Instruction:
How would you implement a function in Python 3 that
imports a genome shipped with pyGeno. Most of the datawraps only contain URLs towards data provided by third parties.

Response:
def importGenome(name, batchSize = 100) :
	"""Import a genome shipped with pyGeno. Most of the datawraps only contain URLs towards data provided by third parties."""
	path = os.path.join(this_dir, "bootstrap_data", "genomes/" + name)
	PG.importGenome(path, batchSize)


Instruction:
Can you implement a function in Python 3 that
imports a SNP set shipped with pyGeno. Most of the datawraps only contain URLs towards data provided by third parties.

Response:
def importSNPs(name) :
	"""Import a SNP set shipped with pyGeno. Most of the datawraps only contain URLs towards data provided by third parties."""
	path = os.path.join(this_dir, "bootstrap_data", "SNPs/" + name)
	PS.importSNPs(path)


Instruction:
Can you generate the documentation for the following Python 3 function
def log(self) :

		self.logs['epochDuration'].append(self.lastEpochDuration)
		self.logs['avg'].append(self.avg)
		self.logs['runtime'].append(self.runtime)
		self.logs['remtime'].append(self.remtime)

Response: logs stats about the progression without printing anything on screen


Instruction:
Given the following Python 3 function, write the documentation
def saveLogs(self, filename) :
		f = open(filename, 'wb')
		cPickle.dump(self.logs, f)
		f.close()

Response: dumps logs into a nice pickle


Instruction:
Explain what the following Python 3 code does
def update(self, label = '', forceRefresh = False, log = False) :
		self.currEpoch += 1
		tim = time.time()
		if (tim - self.lastPrintTime > self.minRefeshTime) or forceRefresh :
			self._update()

			wheelState = self.wheel[self.currEpoch%len(self.wheel)]

			if label == '' :
				slabel = self.label
			else :
				slabel = label

			if self.nbEpochs > 1 :
				ratio = self.currEpoch/self.nbEpochs
				snakeLen = int(self.width*ratio)
				voidLen = int(self.width - (self.width*ratio))

				if snakeLen + voidLen < self.width :
					snakeLen = self.width - voidLen

				self.bar = "%s %s[%s:>%s] %.2f%% (%d/%d) runtime: %s, remaining: %s, avg: %s" %(wheelState, slabel, "~-" * snakeLen, "  " * voidLen, ratio*100, self.currEpoch, self.nbEpochs, self.runtime_hr, self.remtime_hr, self.formatTime(self.avg))
				if self.currEpoch == self.nbEpochs :
					self.close()

			else :
				w = self.width - len(self.miniSnake)
				v = self.currEpoch%(w+1)
				snake = "%s%s%s" %("  " * (v), self.miniSnake, "  " * (w-v))
				self.bar = "%s %s[%s] %s%% (%d/%s) runtime: %s, remaining: %s, avg: %s" %(wheelState, slabel, snake, '?', self.currEpoch, '?', self.runtime_hr, '?', self.formatTime(self.avg))

			sys.stdout.write("\b" * (len(self.bar)+1))
			sys.stdout.write(" " * (len(self.bar)+1))
			sys.stdout.write("\b" * (len(self.bar)+1))
			sys.stdout.write(self.bar)
			sys.stdout.flush()
			self.lastPrintTime = time.time()

			if log :
				self.log()

Response: update the internal state of the object


Instruction:
Explain what the following Python 3 code does
def aux_insertTree(childTree, parentTree):
	if childTree.x1 != None and childTree.x2 != None :
		parentTree.insert(childTree.x1, childTree.x2, childTree.name, childTree.referedObject)

	for c in childTree.children:
		aux_insertTree(c, parentTree)

Response: This a private function that inserts a child tree into a parent tree.


Instruction:
Here you have a function in Python 3, explain what it does
def aux_moveTree(offset, tree):
	if tree.x1 != None and tree.x2 != None :
		tree.x1, tree.x2 = tree.x1+offset, tree.x2+offset

	for c in tree.children:
		aux_moveTree(offset, c)

Response: This function is a private recursive function that translates tree to a given x1 and tree s children to a given x2


Instruction:
Here you have a function in Python 3, explain what it does
def insert(self, x1, x2, name = '', referedObject = []) :

		if x1 > x2 :
			xx1, xx2 = x2, x1
		else :
			xx1, xx2 = x1, x2

		rt = None
		insertId = None
		childrenToRemove = []
		for i in range(len(self.children)) :
			if self.children[i].x1 == xx1 and xx2 == self.children[i].x2 :
				self.children[i].name = self.children[i].name + ' U ' + name
				self.children[i].referedObject.append(referedObject)
				return self.children[i]

			if self.children[i].x1 <= xx1 and xx2 <= self.children[i].x2 :
				return self.children[i].insert(x1, x2, name, referedObject)

			elif xx1 <= self.children[i].x1 and self.children[i].x2 <= xx2 :
				if rt == None :
					if type(referedObject) is types.ListType :
						rt = SegmentTree(xx1, xx2, name, referedObject, self, self.level+1)
					else :
						rt = SegmentTree(xx1, xx2, name, [referedObject], self, self.level+1)

					insertId = i

				rt.__addChild(self.children[i])
				self.children[i].father = rt
				childrenToRemove.append(self.children[i])

			elif xx1 <= self.children[i].x1 and xx2 <= self.children[i].x2 :
				insertId = i
				break

		if rt != None :
			self.__addChild(rt, insertId)
			for c in childrenToRemove :
				self.children.remove(c)
		else :
			if type(referedObject) is types.ListType :
				rt = SegmentTree(xx1, xx2, name, referedObject, self, self.level+1)
			else :
				rt = SegmentTree(xx1, xx2, name, [referedObject], self, self.level+1)

			if insertId != None :
				self.__addChild(rt, insertId)
			else :
				self.__addChild(rt)

		return rt

Response: Insert the segment in it s right place and returns it.


Instruction:
Here you have a function in Python 3, explain what it does
def intersect(self, x1, x2 = None) :

		def condition(x1, x2, tree) :
			#print self.id, tree.x1, tree.x2, x1, x2
			if (tree.x1 != None and tree.x2 != None) and (tree.x1 <= x1 and x1 < tree.x2 or tree.x1 <= x2 and x2 < tree.x2) :
				return True
			return False

		if x2 == None :
			xx1, xx2 = x1, x1
		elif x1 > x2 :
			xx1, xx2 = x2, x1
		else :
			xx1, xx2 = x1, x2

		c1 = self.__dichotomicSearch(xx1)
		c2 = self.__dichotomicSearch(xx2)

		if c1 == -1 or c2 == -1 :
			return []

		if xx1 < self.children[c1].x1 :
			c1 -= 1

		inter = self.__radiateDown(x1, x2, c1, condition)
		if self.children[c1].id == self.children[c2].id :
			inter.extend(self.__radiateUp(x1, x2, c2+1, condition))
		else :
			inter.extend(self.__radiateUp(x1, x2, c2, condition))

		ret = []
		for c in inter :
			ret.extend(c.intersect(x1, x2))

		inter.extend(ret)
		return inter

Response: Returns a list of all segments intersected by x1 and x2.


Instruction:
Can you tell what is the following Python 3 function doing
def __radiateDown(self, x1, x2, childId, condition) :
		"Radiates down: walks self.children downward until condition is no longer verifed or there's no childrens left "
		ret = []
		i = childId
		while 0 <= i :
			if condition(x1, x2, self.children[i]) :
				ret.append(self.children[i])
			else :
				break
			i -= 1
		return ret

Response: Radiates down : walks self. children downward until condition is no longer verifed until there s no childrens left


Instruction:
Make a summary of the following Python 3 code
def __radiateUp(self, x1, x2, childId, condition) :
		"Radiates uo: walks self.children upward until condition is no longer verifed or there's no childrens left "
		ret = []
		i = childId
		while i < len(self.children):
			if condition(x1, x2, self.children[i]) :
				ret.append(self.children[i])
			else :
				break
			i += 1
		return ret

Response: Radiates uo : walks self. children upward until condition is no longer verifed until there s no childrens left


Instruction:
Write a Python 3 function that can
remove all gaps between regions

Response:
def removeGaps(self) :
		"""Remove all gaps between regions"""

		for i in range(1, len(self.children)) :
			if self.children[i].x1 > self.children[i-1].x2:
				aux_moveTree(self.children[i-1].x2-self.children[i].x1, self.children[i])


Instruction:
Can you implement a function in Python 3 that
returns the total length of indexed regions

Response:
def getIndexedLength(self) :
		"""Returns the total length of indexed regions"""
		if self.x1 != None and self.x2 != None:
			return self.x2 - self.x1
		else :
			if len(self.children) == 0 :
				return 0
			else :
				l = self.children[0].x2 - self.children[0].x1
				for i in range(1, len(self.children)) :
					l += self.children[i].x2 - self.children[i].x1 - max(0, self.children[i-1].x2 - self.children[i].x1)
				return l


Instruction:
How would you code a function in Python 3 to
return a list of couples ( x1 x2 ) of all the first level indexed regions

Response:
def getFirstLevel(self) :
		"""returns a list of couples (x1, x2) of all the first level indexed regions"""
		res = []
		if len(self.children) > 0 :
			for c in self.children:
				res.append((c.x1, c.x2))
		else :
			if self.x1 != None :
				res = [(self.x1, self.x2)]
			else :
				res = None
		return res


Instruction:
Create a Python 3 function to
flatten the tree. The tree become a tree of depth 1 where overlapping regions have been merged together.

Response:
def flatten(self) :
		"""Flattens the tree. The tree become a tree of depth 1 where overlapping regions have been merged together"""
		if len(self.children) > 1 :
			children = self.children
			self.emptyChildren()

			children[0].emptyChildren()
			x1 = children[0].x1
			x2 = children[0].x2
			refObjs = [children[0].referedObject]
			name = children[0].name

			for i in range(1, len(children)) :
				children[i].emptyChildren()
				if children[i-1] >= children[i] :
					x2 = children[i].x2
					refObjs.append(children[i].referedObject)
					name += " U " + children[i].name
				else :
					if len(refObjs) == 1 :
						refObjs = refObjs[0]

					self.insert(x1, x2, name, refObjs)
					x1 = children[i].x1
					x2 = children[i].x2
					refObjs = [children[i].referedObject]
					name = children[i].name

			if len(refObjs) == 1 :
				refObjs = refObjs[0]

			self.insert(x1, x2, name, refObjs)


Instruction:
Can you generate the documentation for the following Python 3 function
def move(self, newX1) :
		if self.x1 != None and self.x2 != None :
			offset = newX1-self.x1
			aux_moveTree(offset, self)
		elif len(self.children) > 0 :
			offset = newX1-self.children[0].x1
			aux_moveTree(offset, self)

Response: Moves tree to a new starting position updates x1s of children


Instruction:
Write a Python 3 function for
encoding a binary reprensentation default sequence and polymorphisms list

Response:
def encode(self, sequence):
		"""Returns a tuple (binary reprensentation, default sequence, polymorphisms list)"""

		polymorphisms = []
		defaultSequence = ''
		binSequence = array.array(self.forma.typecode)
		b = 0
		i = 0
		trueI = 0 #not inc in case if poly
		poly = set()
		while i < len(sequence)-1:
			b = b | self.forma[self.charToBin[sequence[i]]]
			if sequence[i+1] == '/' :
				poly.add(sequence[i])
				i += 2
			else :
				binSequence.append(b)
				if len(poly) > 0 :
					poly.add(sequence[i])
					polymorphisms.append((trueI, poly))
					poly = set()

				bb = 0
				while b % 2 != 0 :
					b = b/2

				defaultSequence += sequence[i]
				b = 0
				i += 1
				trueI += 1

		if i < len(sequence) :
			b = b | self.forma[self.charToBin[sequence[i]]]
			binSequence.append(b)
			if len(poly) > 0 :
				if sequence[i] not in poly :
					poly.add(sequence[i])
				polymorphisms.append((trueI, poly))
			defaultSequence += sequence[i]

		return (binSequence, defaultSequence, polymorphisms)


Instruction:
Write a Python 3 function for
getting the list of variants from a polymorphisme

Response:
def __getSequenceVariants(self, x1, polyStart, polyStop, listSequence) :
		"""polyStop, is the polymorphisme at wixh number where the calcul of combinaisons stops"""
		if polyStart < len(self.polymorphisms) and polyStart < polyStop:
			sequence = copy.copy(listSequence)
			ret = []

			pk = self.polymorphisms[polyStart]
			posInSequence = pk[0]-x1
			if posInSequence < len(listSequence) :
				for allele in pk[1] :
					sequence[posInSequence] = allele
					ret.extend(self.__getSequenceVariants(x1, polyStart+1, polyStop, sequence))

			return ret
		else :
			return [''.join(listSequence)]


Instruction:
Here you have a function in Python 3, explain what it does
def getSequenceVariants(self, x1 = 0, x2 = -1, maxVariantNumber = 128) :
		if x2 == -1 :
			xx2 = len(self.defaultSequence)
		else :
			xx2 = x2

		polyStart = None
		nbP = 1
		stopped = False
		j = 0
		for p in self.polymorphisms :
			if p[0] >= xx2 :
				break

			if x1 <= p[0] :
				if polyStart == None :
					polyStart = j

				nbP *= len(p[1])

				if nbP > maxVariantNumber :
					stopped = True
					break
			j += 1

		if polyStart == None :
			return (stopped, [self.defaultSequence[x1:xx2]])

		return (stopped, self.__getSequenceVariants(x1, polyStart, j, list(self.defaultSequence[x1:xx2])))

Response: returns the sequences resulting from all combinaisons between x1 and x2


Instruction:
Here you have a function in Python 3, explain what it does
def getNbVariants(self, x1, x2 = -1) :
		if x2 == -1 :
			xx2 = len(self.defaultSequence)
		else :
			xx2 = x2

		nbP = 1
		for p in self.polymorphisms:
			if x1 <= p[0] and p[0] <= xx2 :
				nbP *= len(p[1])

		return nbP

Response: returns the nb of variants of sequences between x1 and x2


Instruction:
Given the following Python 3 function, write the documentation
def _dichFind(self, needle, currHaystack, offset, lst = None) :

		if len(currHaystack) == 1 :
			if (offset <= (len(self) - len(needle))) and (currHaystack[0] & needle[0]) > 0 and (self[offset+len(needle)-1] & needle[-1]) > 0 :
				found = True
				for i in xrange(1, len(needle)-1) :
					if self[offset + i] & needle[i] == 0 :
						found = False
						break
				if found :
					if lst is not None :
						lst.append(offset)
					else :
						return offset
				else :
					if lst is None :
						return -1
		else :
			if (offset <= (len(self) - len(needle))) :
				if lst is not None :
					self._dichFind(needle, currHaystack[:len(currHaystack)/2], offset, lst)
					self._dichFind(needle, currHaystack[len(currHaystack)/2:], offset + len(currHaystack)/2, lst)
				else :
					v1 = self._dichFind(needle, currHaystack[:len(currHaystack)/2], offset, lst)
					if v1 > -1 :
						return v1

					return self._dichFind(needle, currHaystack[len(currHaystack)/2:], offset + len(currHaystack)/2, lst)
			return -1

Response: dichotomic search, if lst is None, will return the first position found. If it's a list, will return a list of all positions in lst. returns -1 or [] if no match found


Instruction:
Write a Python 3 function for
using KMP algorithm to search the first occurrence in the input_sequence of the pattern. both arguments are integer arrays. return the position of the first occurrence if found ; otherwise - 1.

Response:
def _kmp_search_first(self, pInput_sequence, pPattern):
                """use KMP algorithm to search the first occurrence in the input_sequence of the pattern. both arguments are integer arrays. return the position of the occurence if found; otherwise, -1."""
                input_sequence, pattern = pInput_sequence, [len(bin(e)) for e in pPattern]
                n, m = len(input_sequence), len(pattern)
                d = p = 0
                next = self._kmp_construct_next(pattern)
                while d < n and p < m:
                        p = next[len(bin(input_sequence[d]))][p]
                        d += 1
                if p == m: return d - p
                else: return -1


Instruction:
How would you code a function in Python 3 to
use KMP algorithm to search all occurences in the input_sequence of the pattern. both arguments are integer arrays. return a list of the positions of the occurences if found otherwise [].

Response:
def _kmp_search_all(self, pInput_sequence, pPattern):
                """use KMP algorithm to search all occurrence in the input_sequence of the pattern. both arguments are integer arrays. return a list of the positions of the occurences if found; otherwise, []."""
                r = []
                input_sequence, pattern = [len(bin(e)) for e in pInput_sequence], [len(bin(e)) for e in pPattern]
                n, m = len(input_sequence), len(pattern)
                d = p = 0
                next = self._kmp_construct_next(pattern)
                while d < n:
                        p = next[input_sequence[d]][p]
                        d += 1
                        if p == m:
                                r.append(d - m)
                                p = 0
                return r


Instruction:
Make a summary of the following Python 3 code
def _kmp_find(self, needle, haystack, lst = None):
                if lst != None: return self._kmp_search_all(haystack, needle)
                else: return self._kmp_search_first(haystack, needle)

Response: find with KMP - searching.


Instruction:
Explain what the following Python 3 code does
def findByBiSearch(self, strSeq) :
		arr = self.encode(strSeq)
		return self._dichFind(arr[0], self, 0, lst = None)

Response: returns the first occurence of strSeq in self. Takes polymorphisms into account


Instruction:
Explain what the following Python 3 code does
def findAllByBiSearch(self, strSeq) :
		arr = self.encode(strSeq)
		lst = []
		self._dichFind(arr[0], self, 0, lst)
		return lst

Response: Same as find but returns a list of all occurences


Instruction:
How would you explain what the following Python 3 function does
def find(self, strSeq) :
		arr = self.encode(strSeq)
                return self._kmp_find(arr[0], self)

Response: returns the first occurence of strSeq in self. Takes polymorphisms into account


Instruction:
Can you generate a brief explanation for the following Python 3 code
def findAll(self, strSeq) :
		arr = self.encode(strSeq)
		lst = []
                lst = self._kmp_find(arr[0], self, lst)
		return lst

Response: Same as find but returns a list of all occurences


Instruction:
Here you have a function in Python 3, explain what it does
def decode(self, binSequence):
		try:
			binSeq = iter(binSequence[0])
		except TypeError, te:
			binSeq = binSequence

		ret = ''
		for b in binSeq :
			ch = ''
			for c in self.charToBin :
				if b & self.forma[self.charToBin[c]] > 0 :
					ch += c +'/'
			if ch == '' :
				raise KeyError('Key %d unkowom, bad format' % b)

			ret += ch[:-1]
		return ret

Response: decodes a binary sequence to return a string


Instruction:
Write a Python 3 script to
parse a file and returns a dictionary of the object properties

Response:
def parse(self, filename, gziped = False, stream = False) :
		"""opens a file"""
		self.stream = stream

		if gziped :
			self.f = gzip.open(filename)
		else :
			self.f = open(filename)

		self.filename = filename
		self.gziped = gziped

		lineId = 0
		inLegend = True
		while inLegend :
			ll = self.f.readline()
			l = ll.replace('\r', '\n').replace('\n', '')
			if l[:2] == '##' :
				eqPos = l.find('=')
				key = l[2:eqPos]
				values = l[eqPos+1:].strip()

				if l[eqPos+1] != '<' :
					self.meta[key] = values
				else :
					if key not in self.meta :
						self.meta[key] = {}
					svalues = l[eqPos+2:-1].split(',') #remove the < and > that surounf the string
					idKey = svalues[0].split('=')[1]
					self.meta[key][idKey] = {}
					i = 1
					for v in svalues[1:] :
						sv = v.split("=")
						field = sv[0]
						value = sv[1]
						if field.lower() == 'description' :
							self.meta[key][idKey][field] = ','.join(svalues[i:])[len(field)+2:-1]
							break
						self.meta[key][idKey][field] = value
						i += 1
			elif l[:6] == '#CHROM': #we are in legend
				sl = l.split('\t')
				for i in range(len(sl)) :
					self.legend[sl[i]] = i
					self.dnegel[i] = sl[i]
				break

			lineId += 1

		if not stream :
			self.lines = self.f.readlines()
			self.f.close()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _decode8(self, offset):
        # UTF-8 Strings contain two lengths, as they might differ:
        # 1) the UTF-16 length
        str_len, skip = self._decode_length(offset, 1)
        offset += skip

        # 2) the utf-8 string length
        encoded_bytes, skip = self._decode_length(offset, 1)
        offset += skip

        data = self.m_charbuff[offset: offset + encoded_bytes]

        assert self.m_charbuff[offset + encoded_bytes] == 0, \
            "UTF-8 String is not null terminated! At offset={}".format(offset)

        return self._decode_bytes(data, 'utf-8', str_len)

Response: Decode an UTF - 8 String at the given offset.


Instruction:
Can you generate the documentation for the following Python 3 function
def _decode16(self, offset):
        str_len, skip = self._decode_length(offset, 2)
        offset += skip

        # The len is the string len in utf-16 units
        encoded_bytes = str_len * 2

        data = self.m_charbuff[offset: offset + encoded_bytes]

        assert self.m_charbuff[offset + encoded_bytes:offset + encoded_bytes + 2] == b"\x00\x00", \
            "UTF-16 String is not null terminated! At offset={}".format(offset)

        return self._decode_bytes(data, 'utf-16', str_len)

Response: Decode an UTF - 16 String at the given offset.


Instruction:
Write a Python 3 script to
decode the length of the string at the given offset.

Response:
def _decode_length(self, offset, sizeof_char):
        """
        Generic Length Decoding at offset of string

        The method works for both 8 and 16 bit Strings.
        Length checks are enforced:
        * 8 bit strings: maximum of 0x7FFF bytes (See
        http://androidxref.com/9.0.0_r3/xref/frameworks/base/libs/androidfw/ResourceTypes.cpp#692)
        * 16 bit strings: maximum of 0x7FFFFFF bytes (See
        http://androidxref.com/9.0.0_r3/xref/frameworks/base/libs/androidfw/ResourceTypes.cpp#670)

        :param offset: offset into the string data section of the beginning of
        the string
        :param sizeof_char: number of bytes per char (1 = 8bit, 2 = 16bit)
        :returns: tuple of (length, read bytes)
        """
        sizeof_2chars = sizeof_char << 1
        fmt = "<2{}".format('B' if sizeof_char == 1 else 'H')
        highbit = 0x80 << (8 * (sizeof_char - 1))

        length1, length2 = unpack(fmt, self.m_charbuff[offset:(offset + sizeof_2chars)])

        if (length1 & highbit) != 0:
            length = ((length1 & ~highbit) << (8 * sizeof_char)) | length2
            size = sizeof_2chars
        else:
            length = length1
            size = sizeof_char

        if sizeof_char == 1:
            assert length <= 0x7FFF, "length of UTF-8 string is too large! At offset={}".format(offset)
        else:
            assert length <= 0x7FFFFFFF, "length of UTF-16 string is too large!  At offset={}".format(offset)

        return length, size


Instruction:
Can you write a function in Python 3 where it
applies some fixes to element named and attribute names.

Response:
def _fix_name(self, name):
        """
        Apply some fixes to element named and attribute names.
        Try to get conform to:
        > Like element names, attribute names are case-sensitive and must start with a letter or underscore.
        > The rest of the name can contain letters, digits, hyphens, underscores, and periods.
        See: https://msdn.microsoft.com/en-us/library/ms256152(v=vs.110).aspx

        :param name: Name of the attribute
        :return: a fixed version of the name
        """
        if not name[0].isalpha() and name[0] != "_":
            log.warning("Invalid start for name '{}'".format(name))
            self.packerwarning = True
            name = "_{}".format(name)
        if name.startswith("android:"):
            # Seems be a common thing...
            # Actually this means that the Manifest is likely to be broken, as
            # usually no namespace URI is set in this case.
            log.warning(
                "Name '{}' starts with 'android:' prefix! "
                "The Manifest seems to be broken? Removing prefix.".format(
                    name
                )
            )
            self.packerwarning = True
            name = name[len("android:"):]
        if ":" in name:
            # Print out an extra warning
            log.warning("Name seems to contain a namespace prefix: '{}'".format(name))
        if not re.match(r"^[a-zA-Z0-9._-]*$", name):
            log.warning("Name '{}' contains invalid characters!".format(name))
            self.packerwarning = True
            name = re.sub(r"[^a-zA-Z0-9._-]", "_", name)

        return name


Instruction:
How would you code a function in Python 3 to
fix the value of the attribute to be a string that is not part of the XML.

Response:
def _fix_value(self, value):
        """
        Return a cleaned version of a value
        according to the specification:
        > Char	   ::=   	#x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]

        See https://www.w3.org/TR/xml/#charsets

        :param value: a value to clean
        :return: the cleaned value
        """
        if not self.__charrange or not self.__replacement:
            if sys.maxunicode == 0xFFFF:
                # Fix for python 2.x, surrogate pairs does not match in regex
                self.__charrange = re.compile(
                    u'^([\u0020-\uD7FF\u0009\u000A\u000D\uE000-\uFFFD]|[\uD800-\uDBFF][\uDC00-\uDFFF])*$')
                # TODO: this regex is slightly wrong... surrogates are not matched as pairs.
                self.__replacement = re.compile(u'[^\u0020-\uDBFF\u0009\u000A\u000D\uE000-\uFFFD\uDC00-\uDFFF]')
            else:
                self.__charrange = re.compile(u'^[\u0020-\uD7FF\u0009\u000A\u000D\uE000-\uFFFD\U00010000-\U0010FFFF]*$')
                self.__replacement = re.compile(u'[^\u0020-\uD7FF\u0009\u000A\u000D\uE000-\uFFFD\U00010000-\U0010FFFF]')

        # Reading string until \x00. This is the same as aapt does.
        if "\x00" in value:
            self.packerwarning = True
            log.warning(
                "Null byte found in attribute value at position {}: "
                "Value(hex): '{}'".format(
                    value.find("\x00"),
                    binascii.hexlify(value.encode("utf-8"))
                )
            )
            value = value[:value.find("\x00")]

        if not self.__charrange.match(value):
            log.warning("Invalid character in value found. Replacing with '_'.")
            self.packerwarning = True
            value = self.__replacement.sub('_', value)
        return value


Instruction:
Here you have a function in Python 3, explain what it does
def _apk_analysis(self):
        i = "AndroidManifest.xml"
        try:
            manifest_data = self.zip.read(i)
        except KeyError:
            log.warning("Missing AndroidManifest.xml. Is this an APK file?")
        else:
            ap = AXMLPrinter(manifest_data)

            if not ap.is_valid():
                log.error("Error while parsing AndroidManifest.xml - is the file valid?")
                return

            self.axml[i] = ap
            self.xml[i] = self.axml[i].get_xml_obj()

            if self.axml[i].is_packed():
                log.warning("XML Seems to be packed, operations on the AndroidManifest.xml might fail.")

            if self.xml[i] is not None:
                if self.xml[i].tag != "manifest":
                    log.error("AndroidManifest.xml does not start with a <manifest> tag! Is this a valid APK?")
                    return

                self.package = self.get_attribute_value("manifest", "package")
                self.androidversion["Code"] = self.get_attribute_value("manifest", "versionCode")
                self.androidversion["Name"] = self.get_attribute_value("manifest", "versionName")
                permission = list(self.get_all_attribute_value("uses-permission", "name"))
                self.permissions = list(set(self.permissions + permission))

                for uses_permission in self.find_tags("uses-permission"):
                    self.uses_permissions.append([
                        self.get_value_from_tag(uses_permission, "name"),
                        self._get_permission_maxsdk(uses_permission)
                    ])

                # getting details of the declared permissions
                for d_perm_item in self.find_tags('permission'):
                    d_perm_name = self._get_res_string_value(
                        str(self.get_value_from_tag(d_perm_item, "name")))
                    d_perm_label = self._get_res_string_value(
                        str(self.get_value_from_tag(d_perm_item, "label")))
                    d_perm_description = self._get_res_string_value(
                        str(self.get_value_from_tag(d_perm_item, "description")))
                    d_perm_permissionGroup = self._get_res_string_value(
                        str(self.get_value_from_tag(d_perm_item, "permissionGroup")))
                    d_perm_protectionLevel = self._get_res_string_value(
                        str(self.get_value_from_tag(d_perm_item, "protectionLevel")))

                    d_perm_details = {
                        "label": d_perm_label,
                        "description": d_perm_description,
                        "permissionGroup": d_perm_permissionGroup,
                        "protectionLevel": d_perm_protectionLevel,
                    }
                    self.declared_permissions[d_perm_name] = d_perm_details

                self.valid_apk = True

Response: Run analysis on the APK file and set all the attributes in the class attribute.


Instruction:
Implement a Python 3 function for
getting the filetype guessed for a buffer.

Response:
def _get_file_magic_name(self, buffer):
        """
        Return the filetype guessed for a buffer
        :param buffer: bytes
        :return: str of filetype
        """
        default = "Unknown"
        ftype = None

        try:
            # Magic is optional
            import magic
        except ImportError:
            return default

        try:
            # There are several implementations of magic,
            # unfortunately all called magic
            # We use this one: https://github.com/ahupp/python-magic/
            getattr(magic, "MagicException")
        except AttributeError:
            # Looks like no magic was installed
            return default

        try:
            ftype = magic.from_buffer(buffer[:1024])
        except magic.MagicError as e:
            log.exception("Error getting the magic type!")
            return default

        if not ftype:
            return default
        else:
            return self._patch_magic(buffer, ftype)


Instruction:
Make a summary of the following Python 3 code
def get_files_types(self):
        if self._files == {}:
            # Generate File Types / CRC List
            for i in self.get_files():
                buffer = self.zip.read(i)
                self.files_crc32[i] = crc32(buffer)
                # FIXME why not use the crc from the zipfile?
                # should be validated as well.
                # crc = self.zip.getinfo(i).CRC
                self._files[i] = self._get_file_magic_name(buffer)

        return self._files

Response: Return the files inside the APK with their associated types


Instruction:
How would you implement a function in Python 3 that
patches magic to return correct guess by mime libraries

Response:
def _patch_magic(self, buffer, orig):
        """
        Overwrite some probably wrong detections by mime libraries

        :param buffer: bytes of the file to detect
        :param orig: guess by mime libary
        :return: corrected guess
        """
        if ("Zip" in orig) or ('(JAR)' in orig):
            val = is_android_raw(buffer)
            if val == "APK":
                return "Android application package file"

        return orig


Instruction:
Can you tell what is the following Python 3 function doing
def get_files_crc32(self):
        if self.files_crc32 == {}:
            for i in self.get_files():
                buffer = self.zip.read(i)
                self.files_crc32[i] = crc32(buffer)

        return self.files_crc32

Response: Calculates and returns a dictionary of filenames and CRC32s


Instruction:
Here you have a function in Python 3, explain what it does
def is_multidex(self):
        dexre = re.compile("^classes(\d+)?.dex$")
        return len([instance for instance in self.get_files() if dexre.search(instance)]) > 1

Response: Test if the APK has multiple DEX files


Instruction:
How would you implement a function in Python 3 that
gets the elements in xml files which match with the tag name and the specific attribute name.

Response:
def get_elements(self, tag_name, attribute, with_namespace=True):
        """
        Deprecated: use `get_all_attribute_value()` instead
        Return elements in xml files which match with the tag name and the specific attribute
        :param tag_name: a string which specify the tag name
        :param attribute: a string which specify the attribute
        """
        for i in self.xml:
            if self.xml[i] is None:
                continue
            for item in self.xml[i].findall('.//' + tag_name):
                if with_namespace:
                    value = item.get(self._ns(attribute))
                else:
                    value = item.get(attribute)
                # There might be an attribute without the namespace
                if value:
                    yield self._format_value(value)


Instruction:
Given the following Python 3 function, write the documentation
def _format_value(self, value):
        if len(value) > 0:
            if value[0] == ".":
                value = self.package + value
            else:
                v_dot = value.find(".")
                if v_dot == 0:
                    value = self.package + "." + value
                elif v_dot == -1:
                    value = self.package + "." + value
        return value

Response: Format a value with packagename if not already set


Instruction:
How would you explain what the following Python 3 function does
def get_element(self, tag_name, attribute, **attribute_filter):
        for i in self.xml:
            if self.xml[i] is None:
                continue
            tag = self.xml[i].findall('.//' + tag_name)
            if len(tag) == 0:
                return None
            for item in tag:
                skip_this_item = False
                for attr, val in list(attribute_filter.items()):
                    attr_val = item.get(self._ns(attr))
                    if attr_val != val:
                        skip_this_item = True
                        break

                if skip_this_item:
                    continue

                value = item.get(self._ns(attribute))

                if value is not None:
                    return value
        return None

Response: Return the element in xml files which match with the tag name and the specific attribute name.


Instruction:
Write a Python 3 script to
return true if the attributes of the tag match the ones in attribute filter

Response:
def is_tag_matched(self, tag, **attribute_filter):
        """
        Return true if the attributes matches in attribute filter
        :param tag: specify the tag element
        :type tag: Element
        :param attribute: specify the attribute
        :type attribute: string
        """
        if len(attribute_filter) <= 0:
            return True
        for attr, value in attribute_filter.items():
            # TODO: figure out if both android:name and name tag exist which one to give preference
            _value = tag.get(self._ns(attr))
            if _value is None:
                log.warning("Failed to get the attribute with namespace")
                _value = tag.get(attr)
            if _value != value:
                return False
        return True


Instruction:
Given the following Python 3 function, write the documentation
def name(self):
        if self.m_name == -1 or (self.m_event != const.START_TAG and self.m_event != const.END_TAG):
            return u''

        return self.sb[self.m_name]

Response: Return the name of the tag


Instruction:
Given the following Python 3 function, write the documentation
def namespace(self):
        if self.m_name == -1 or (self.m_event != const.START_TAG and self.m_event != const.END_TAG):
            return u''

        # No Namespace
        if self.m_namespaceUri == 0xFFFFFFFF:
            return u''

        return self.sb[self.m_namespaceUri]

Response: Return the Namespace URI of the current tag.


Instruction:
Write a Python 3 function for
returning the current text assosicated with the current text

Response:
def text(self):
        """
        Return the String assosicated with the current text
        """
        if self.m_name == -1 or self.m_event != const.TEXT:
            return u''

        return self.sb[self.m_name]


Instruction:
Write a Python 3 function for
returning the Namespace URI for the attribute at the given index.

Response:
def getAttributeNamespace(self, index):
        """
        Return the Namespace URI (if any) for the attribute
        """
        uri = self.getAttributeUri(index)

        # No Namespace
        if uri == 0xFFFFFFFF:
            return u''

        return self.sb[uri]


Instruction:
How would you implement a function in Python 3 that
returns the String which represents the attribute name

Response:
def getAttributeName(self, index):
        """
        Returns the String which represents the attribute name
        """
        offset = self._get_attribute_offset(index)
        name = self.m_attributes[offset + const.ATTRIBUTE_IX_NAME]

        res = self.sb[name]
        # If the result is a (null) string, we need to look it up.
        if not res:
            attr = self.m_resourceIDs[name]
            if attr in public.SYSTEM_RESOURCES['attributes']['inverse']:
                res = 'android:' + public.SYSTEM_RESOURCES['attributes']['inverse'][attr]
            else:
                # Attach the HEX Number, so for multiple missing attributes we do not run
                # into problems.
                res = 'android:UNKNOWN_SYSTEM_ATTRIBUTE_{:08x}'.format(attr)

        return res


Instruction:
Create a Python 3 function to
return the type of the attribute at the given index

Response:
def getAttributeValueType(self, index):
        """
        Return the type of the attribute at the given index

        :param index: index of the attribute
        """
        offset = self._get_attribute_offset(index)
        return self.m_attributes[offset + const.ATTRIBUTE_IX_VALUE_TYPE]


Instruction:
Explain what the following Python 3 code does
def getAttributeValueData(self, index):
        offset = self._get_attribute_offset(index)
        return self.m_attributes[offset + const.ATTRIBUTE_IX_VALUE_DATA]

Response: Returns the data of the attribute at the given index


Instruction:
Given the following Python 3 function, write the documentation
def get_qualifier(self):
        res = []

        mcc = self.imsi & 0xFFFF
        mnc = (self.imsi & 0xFFFF0000) >> 16
        if mcc != 0:
            res.append("mcc%d" % mcc)
        if mnc != 0:
            res.append("mnc%d" % mnc)

        if self.locale != 0:
            res.append(self.get_language_and_region())

        screenLayout = self.screenConfig & 0xff
        if (screenLayout & const.MASK_LAYOUTDIR) != 0:
            if screenLayout & const.MASK_LAYOUTDIR == const.LAYOUTDIR_LTR:
                res.append("ldltr")
            elif screenLayout & const.MASK_LAYOUTDIR == const.LAYOUTDIR_RTL:
                res.append("ldrtl")
            else:
                res.append("layoutDir_%d" % (screenLayout & const.MASK_LAYOUTDIR))

        smallestScreenWidthDp = (self.screenConfig & 0xFFFF0000) >> 16
        if smallestScreenWidthDp != 0:
            res.append("sw%ddp" % smallestScreenWidthDp)

        screenWidthDp = self.screenSizeDp & 0xFFFF
        screenHeightDp = (self.screenSizeDp & 0xFFFF0000) >> 16
        if screenWidthDp != 0:
            res.append("w%ddp" % screenWidthDp)
        if screenHeightDp != 0:
            res.append("h%ddp" % screenHeightDp)

        if (screenLayout & const.MASK_SCREENSIZE) != const.SCREENSIZE_ANY:
            if screenLayout & const.MASK_SCREENSIZE == const.SCREENSIZE_SMALL:
                res.append("small")
            elif screenLayout & const.MASK_SCREENSIZE == const.SCREENSIZE_NORMAL:
                res.append("normal")
            elif screenLayout & const.MASK_SCREENSIZE == const.SCREENSIZE_LARGE:
                res.append("large")
            elif screenLayout & const.MASK_SCREENSIZE == const.SCREENSIZE_XLARGE:
                res.append("xlarge")
            else:
                res.append("screenLayoutSize_%d" % (screenLayout & const.MASK_SCREENSIZE))
        if (screenLayout & const.MASK_SCREENLONG) != 0:
            if screenLayout & const.MASK_SCREENLONG == const.SCREENLONG_NO:
                res.append("notlong")
            elif screenLayout & const.MASK_SCREENLONG == const.SCREENLONG_YES:
                res.append("long")
            else:
                res.append("screenLayoutLong_%d" % (screenLayout & const.MASK_SCREENLONG))

        density = (self.screenType & 0xffff0000) >> 16
        if density != const.DENSITY_DEFAULT:
            if density == const.DENSITY_LOW:
                res.append("ldpi")
            elif density == const.DENSITY_MEDIUM:
                res.append("mdpi")
            elif density == const.DENSITY_TV:
                res.append("tvdpi")
            elif density == const.DENSITY_HIGH:
                res.append("hdpi")
            elif density == const.DENSITY_XHIGH:
                res.append("xhdpi")
            elif density == const.DENSITY_XXHIGH:
                res.append("xxhdpi")
            elif density == const.DENSITY_XXXHIGH:
                res.append("xxxhdpi")
            elif density == const.DENSITY_NONE:
                res.append("nodpi")
            elif density == const.DENSITY_ANY:
                res.append("anydpi")
            else:
                res.append("%ddpi" % (density))

        touchscreen = (self.screenType & 0xff00) >> 8
        if touchscreen != const.TOUCHSCREEN_ANY:
            if touchscreen == const.TOUCHSCREEN_NOTOUCH:
                res.append("notouch")
            elif touchscreen == const.TOUCHSCREEN_FINGER:
                res.append("finger")
            elif touchscreen == const.TOUCHSCREEN_STYLUS:
                res.append("stylus")
            else:
                res.append("touchscreen_%d" % touchscreen)

        screenSize = self.screenSize
        if screenSize != 0:
            screenWidth = self.screenSize & 0xffff
            screenHeight = (self.screenSize & 0xffff0000) >> 16
            res.append("%dx%d" % (screenWidth, screenHeight))

        version = self.version
        if version != 0:
            sdkVersion = self.version & 0xffff
            minorVersion = (self.version & 0xffff0000) >> 16
            res.append("v%d" % sdkVersion)
            if minorVersion != 0:
                res.append(".%d" % minorVersion)

        return "-".join(res)

Response: Get the resource name qualifier for the current configuration.


Instruction:
How would you explain what the following Python 3 function does
def make_cache_key(request):
    headers = frozenset(request._p['header'].items())
    path = frozenset(request._p['path'].items())
    query = frozenset(request._p['query'])
    return (request.url, headers, path, query)

Response: Generate a cache key from request object data


Instruction:
Explain what the following Python 3 code does
def check_cache(cache):
    if isinstance(cache, BaseCache):
        return cache
    elif cache is False:
        return DictCache()
    elif cache is None:
        return DummyCache()
    else:
        raise ValueError('Provided cache must implement BaseCache')

Response: check if a cache fits esipy needs or not


Instruction:
Create a Python 3 function for
returning the time left in second for an expires header

Response:
def get_cache_time_left(expires_header):
    """ return the time left in second for an expires header """
    epoch = datetime(1970, 1, 1)
    # this date is ALWAYS in UTC (RFC 7231)
    expire = (
        datetime(
            *parsedate(expires_header)[:6]
        ) - epoch
    ).total_seconds()
    now = (datetime.utcnow() - epoch).total_seconds()
    return int(expire) - int(now)


Instruction:
Write a Python 3 function that can
generate a code_verifier for the current user.

Response:
def generate_code_verifier(n_bytes=64):
    """
    source: https://github.com/openstack/deb-python-oauth2client
    Generates a 'code_verifier' as described in section 4.1 of RFC 7636.
    This is a 'high-entropy cryptographic random string' that will be
    impractical for an attacker to guess.
    Args:
        n_bytes: integer between 31 and 96, inclusive. default: 64
            number of bytes of entropy to include in verifier.
    Returns:
        Bytestring, representing urlsafe base64-encoded random data.
    """
    verifier = base64.urlsafe_b64encode(
        os.urandom(n_bytes)
    ).rstrip(b'=').decode('utf-8')

    # https://tools.ietf.org/html/rfc7636#section-4.1
    # minimum length of 43 characters and a maximum length of 128 characters.
    if len(verifier) < 43:
        raise ValueError("Verifier too short. n_bytes must be > 30.")
    elif len(verifier) > 128:
        raise ValueError("Verifier too long. n_bytes must be < 97.")
    else:
        return verifier


Instruction:
Can you generate a brief explanation for the following Python 3 code
def generate_code_challenge(verifier):
    digest = hashlib.sha256(verifier.encode('utf-8')).digest()
    return base64.urlsafe_b64encode(digest).rstrip(b'=').decode('utf-8')

Response: Generate a code_challenge for a given token.


Instruction:
Make a summary of the following Python 3 code
def add_receiver(self, receiver):
        if not callable(receiver):
            raise TypeError("receiver must be callable")
        self.event_receivers.append(receiver)

Response: Adds a receiver to the list of receivers.


Instruction:
Can you implement a function in Python 3 that
removes a receiver from the list of receivers.

Response:
def remove_receiver(self, receiver):
        """ Remove a receiver to the list of receivers.

        :param receiver: a callable variable
        """
        if receiver in self.event_receivers:
            self.event_receivers.remove(receiver)


Instruction:
Create a Python 3 function for
sending a message to all event receivers that have been triggered and pass them the parameters.

Response:
def send_robust(self, **kwargs):
        """ Trigger all receiver and pass them the parameters
        If an exception is raised it will be catched and displayed as error
        in the logger (if defined).

        :param kwargs: all arguments from the event.
        """
        for receiver in self.event_receivers:
            try:
                receiver(**kwargs)
            except Exception as err:  # pylint: disable=W0703
                if not hasattr(err, '__traceback__'):
                    LOGGER.error(sys.exc_info()[2])
                else:
                    LOGGER.error(getattr(err, '__traceback__'))


Instruction:
How would you implement a function in Python 3 that
gets the app from cache or generate a new one if required

Response:
def __get_or_create_app(self, url, cache_key):
        """ Get the app from cache or generate a new one if required

        Because app object doesn't have etag/expiry, we have to make
        a head() call before, to have these informations first... """
        headers = {"Accept": "application/json"}
        app_url = '%s?datasource=%s' % (url, self.datasource)

        cached = self.cache.get(cache_key, (None, None, 0))
        if cached is None or len(cached) != 3:
            self.cache.invalidate(cache_key)
            cached_app, cached_headers, cached_expiry = (cached, None, 0)
        else:
            cached_app, cached_headers, cached_expiry = cached

        if cached_app is not None and cached_headers is not None:
            # we didn't set custom expire, use header expiry
            expires = cached_headers.get('expires', None)
            cache_timeout = -1
            if self.expire is None and expires is not None:
                cache_timeout = get_cache_time_left(
                    cached_headers['expires']
                )
                if cache_timeout >= 0:
                    return cached_app

            # we set custom expire, check this instead
            else:
                if self.expire == 0 or cached_expiry >= time.time():
                    return cached_app

            # if we have etags, add the header to use them
            etag = cached_headers.get('etag', None)
            if etag is not None:
                headers['If-None-Match'] = etag

            # if nothing makes us use the cache, invalidate it
            if ((expires is None or cache_timeout < 0 or
                 cached_expiry < time.time()) and etag is None):
                self.cache.invalidate(cache_key)

        # set timeout value in case we have to cache it later
        timeout = 0
        if self.expire is not None and self.expire > 0:
            timeout = time.time() + self.expire

        # we are here, we know we have to make a head call...
        res = requests.head(app_url, headers=headers)
        if res.status_code == 304 and cached_app is not None:
            self.cache.set(
                cache_key,
                (cached_app, res.headers, timeout)
            )
            return cached_app

        # ok, cache is not accurate, make the full stuff
        app = App.create(app_url)
        if self.caching:
            self.cache.set(cache_key, (app, res.headers, timeout))

        return app


Instruction:
Write a Python 3 function that can
invalidate all cached endpoints meta included

Response:
def clear_cached_endpoints(self, prefix=None):
        """ Invalidate all cached endpoints, meta included

        Loop over all meta endpoints to generate all cache key the
        invalidate each of them. Doing it this way will prevent the
        app not finding keys as the user may change its prefixes
        Meta endpoint will be updated upon next call.
        :param: prefix the prefix for the cache key (default is cache_prefix)
        """
        prefix = prefix if prefix is not None else self.cache_prefix
        for endpoint in self.app.op.values():
            cache_key = '%s:app:%s' % (prefix, endpoint.url)
            self.cache.invalidate(cache_key)
        self.cache.invalidate('%s:app:meta_swagger_url' % self.cache_prefix)
        self.app = None


Instruction:
Can you tell what is the following Python 3 function doing
def __get_jwks_key(self, jwks_uri, **kwargs):
        jwks_key = kwargs.pop('jwks_key', None)
        if not jwks_key:
            res = self._session.get(jwks_uri)
            res.raise_for_status()
            jwks_key = res.json()

        self.jwks_key_set = None
        self.jwks_key = None
        if 'keys' in jwks_key:
            self.jwks_key_set = {}
            for jwks in jwks_key['keys']:
                self.jwks_key_set[jwks['kid']] = jwks
        else:
            self.jwks_key = jwks_key

Response: Get from jwks_ui all the JWKS keys required to decode JWT Token.


Instruction:
How would you implement a function in Python 3 that
prepares the request parameters to execute the POST call to the OAuth token endpoint.

Response:
def __prepare_token_request(self, params, url=None):
        """Generate the request parameters to execute the POST call to
        get or refresh a token.

        Parameters
        ----------
        params : dictionary
            Contains the parameters that will be given as data to the oauth
            endpoint

        Returns
        -------
        type
            The filled request parameters with all required informations

        """
        if self.secret_key is None:
            params['code_verifier'] = self.code_verifier
            params['client_id'] = self.client_id

        request_params = {
            'headers': self.__get_basic_auth_header(),
            'data': params,
            'url': self.oauth_token if url is None else url,
        }

        return request_params


Instruction:
How would you code a function in Python 3 to
construct the auth uri and returns it

Response:
def get_auth_uri(self, state, scopes=None, implicit=False):
        """Constructs the full auth uri and returns it..

        Parameters
        ----------
        state : string
            The state to pass through the auth process
        scopes : list (optional)
            The list of scope to have
        implicit : Boolean (optional)
            Activate or not the implicit flow

        Returns
        -------
        String
            The generated URL for the user to log into EVE SSO

        """
        if state is None or state == '':
            raise AttributeError('"state" must be non empty, non None string')

        scopes_list = [] if not scopes else scopes
        response_type = 'code' if not implicit else 'token'

        # generate the auth URI
        auth_uri = '%s?response_type=%s&redirect_uri=%s&client_id=%s%s%s' % (
            self.oauth_authorize,
            response_type,
            quote(self.redirect_uri, safe=''),
            self.client_id,
            '&scope=%s' % '+'.join(scopes_list) if scopes else '',
            '&state=%s' % state
        )

        # add code challenge if we have one
        if self.secret_key is None and not implicit:
            auth_uri += '&code_challenge_method=S256&code_challenge=%s' % (
                generate_code_challenge(self.code_verifier)
            )
        return auth_uri


Instruction:
Write a Python 3 script for
returning the params dict for the post request to get the access_token from the refresh_token

Response:
def get_refresh_token_params(self, scope_list=None):
        """ Return the param object for the post() call to get the access_token
        from the refresh_token

        :param code: the refresh token
        :return: a dict with the url, params and header
        """
        if self.refresh_token is None:
            raise AttributeError('No refresh token is defined.')

        params = {
            'grant_type': 'refresh_token',
            'refresh_token': self.refresh_token,
        }

        if scope_list:
            if isinstance(scope_list, list):
                scopes = '+'.join(scope_list)
            else:
                raise AttributeError('scope_list must be a list of scope.')
            params['scope'] = scopes

        return self.__prepare_token_request(params)


Instruction:
Write a Python 3 script to
update the access_token refresh_token and token_expiry attributes from the response body.

Response:
def update_token(self, response_json, **kwargs):
        """ Update access_token, refresh_token and token_expiry from the
        response body.
        The response must be converted to a json object before being passed as
        a parameter

        :param response_json: the response body to use.
        :param token_identifier: the user identifier for the token
        """
        self.token_identifier = kwargs.pop(
            'token_identifier',
            self.token_identifier
        )
        self.access_token = response_json['access_token']
        self.token_expiry = int(time.time()) + response_json['expires_in']

        if 'refresh_token' in response_json:
            self.refresh_token = response_json['refresh_token']


Instruction:
Implement a function in Python 3 to
return true if the token is expired.

Response:
def is_token_expired(self, offset=0):
        """ Return true if the token is expired.

        The offset can be used to change the expiry time:
        - positive value decrease the time (sooner)
        - negative value increase the time (later)
        If the expiry is not set, always return True. This case allow the users
        to define a security object, only knowing the refresh_token and get
        a new access_token / expiry_time without errors.

        :param offset: the expiry offset (in seconds) [default: 0]
        :return: boolean true if expired, else false.
        """
        if self.token_expiry is None:
            return True
        return int(time.time()) >= (self.token_expiry - offset)


Instruction:
Can you write a function in Python 3 where it
refreshes the auth data using the refresh token in auth.

Response:
def refresh(self, scope_list=None):
        """ Update the auth data (tokens) using the refresh token in auth.
        """
        request_data = self.get_refresh_token_params(scope_list)
        res = self._session.post(**request_data)
        if res.status_code != 200:
            raise APIException(
                request_data['url'],
                res.status_code,
                response=res.content,
                request_param=request_data,
                response_header=res.headers
            )
        json_res = res.json()
        self.update_token(json_res)
        return json_res


Instruction:
Write a Python 3 script to
request the token to the auth endpoint.

Response:
def auth(self, code):
        """ Request the token to the /oauth/token endpoint.
        Update the security tokens.

        :param code: the code you get from the auth process
        """
        request_data = self.get_access_token_params(code)

        res = self._session.post(**request_data)
        if res.status_code != 200:
            raise APIException(
                request_data['url'],
                res.status_code,
                response=res.content,
                request_param=request_data,
                response_header=res.headers
            )

        json_res = res.json()
        self.update_token(json_res)
        return json_res


Instruction:
How would you code a function in Python 3 to
revoke the current tokens then empty all stored tokens

Response:
def revoke(self):
        """Revoke the current tokens then empty all stored tokens
        This returns nothing since the endpoint return HTTP/200
        whatever the result is...

        Currently not working with JWT, left here for compatibility.
        """
        if not self.refresh_token and not self.access_token:
            raise AttributeError('No access/refresh token are defined.')

        if self.refresh_token:
            data = {
                'token_type_hint': 'refresh_token',
                'token': self.refresh_token,
            }
        else:
            data = {
                'token_type_hint': 'access_token',
                'token': self.access_token,
            }

        request_data = self.__prepare_token_request(data, self.oauth_revoke)

        self._session.post(**request_data)
        self.access_token = None
        self.refresh_token = None
        self.token_expiry = None


Instruction:
Write a Python 3 script to
decode and verify the token and return the decoded informations.

Response:
def verify(self, kid='JWT-Signature-Key', options=None):
        """Decode and verify the token and return the decoded informations

        Parameters
        ----------
        kid : string
            The JWKS key id to identify the key to decode the token.
            Default is 'JWT-Signature-Key'. Only change if CCP changes it.
        options : Dict
            The dictionary of options for skipping validation steps. See
            https://python-jose.readthedocs.io/en/latest/jwt/api.html#jose.jwt.decode

        Returns
        -------
        Dict
            The JWT informations from the token, such as character name etc.

        Raises
        ------
            jose.exceptions.JWTError: If the signature is invalid in any way.
            jose.exceptions.ExpiredSignatureError: If the signature has expired
            jose.exceptions.JWTClaimsError: If any claim is invalid in any way.
        """
        if self.access_token is None or self.access_token == "":
            raise AttributeError('No access token are available at this time')
        if options is None:
            options = {}

        if self.jwks_key_set is None:
            key = self.jwks_key
        else:
            key = self.jwks_key_set[kid]

        return jwt.decode(
            self.access_token,
            key,
            issuer=self.oauth_issuer,
            options=options
        )


Instruction:
Can you write a function in Python 3 where it
tries to send a request and retries the response.

Response:
def _retry_request(self, req_and_resp, _retry=0, **kwargs):
        """Uses self._request in a sane retry loop (for 5xx level errors).

        Do not use the _retry parameter, use the same params as _request

        Used when ESIClient is initialized with retry_requests=True
        if raise_on_error is True, this will only raise exception after
        all retry have been done

        """
        raise_on_error = kwargs.pop('raise_on_error', False)

        if _retry:
            # backoff delay loop in seconds: 0.01, 0.16, 0.81, 2.56, 6.25
            time.sleep(_retry ** 4 / 100)

        res = self._request(req_and_resp, **kwargs)

        if 500 <= res.status <= 599:
            _retry += 1
            if _retry < 5:
                LOGGER.warning(
                    "[failure #%d] %s %d: %r",
                    _retry,
                    req_and_resp[0].url,
                    res.status,
                    res.data,
                )
                return self._retry_request(
                    req_and_resp,
                    _retry=_retry,
                    raise_on_error=raise_on_error,
                    **kwargs
                )

        if res.status >= 400 and raise_on_error:
            raise APIException(
                req_and_resp[0].url,
                res.status,
                response=res.raw,
                request_param=req_and_resp[0].query,
                response_header=res.header
            )

        return res


Instruction:
How would you explain what the following Python 3 function does
def multi_request(self, reqs_and_resps, threads=20, **kwargs):

        opt = kwargs.pop('opt', {})
        raw_body_only = kwargs.pop('raw_body_only', self.raw_body_only)
        # you shouldnt need more than 100, 20 is probably fine in most cases
        threads = max(min(threads, 100), 1)

        def _multi_shim(req_and_resp):
            """Shim self.request to also return the original request."""

            return req_and_resp[0], self.request(
                req_and_resp,
                raw_body_only=raw_body_only,
                opt=opt,
            )

        results = []

        with ThreadPoolExecutor(max_workers=threads) as pool:
            for result in pool.map(_multi_shim, reqs_and_resps):
                results.append(result)

        return results

Response: Use a threadpool to send multiple requests in parallel.


Instruction:
Write a Python 3 function that can
make a request and fill the response with the fields from the request and response.

Response:
def _request(self, req_and_resp, **kwargs):
        """ Take a request_and_response object from pyswagger.App and
        check auth, token, headers, prepare the actual request and fill the
        response

        Note on performance : if you need more performance (because you are
        using this in a batch) you'd rather set raw_body_only=True, as parsed
        body is really slow. You'll then have to get data from response.raw
        and convert it to json using "json.loads(response.raw)"

        :param req_and_resp: the request and response object from pyswagger.App
        :param raw_body_only: define if we want the body to be parsed as object
                              instead of staying a raw dict. [Default: False]
        :param opt: options, see pyswagger/blob/master/pyswagger/io.py#L144
        :param raise_on_error: boolean to raise an error if HTTP Code >= 400

        :return: the final response.
        """

        opt = kwargs.pop('opt', {})

        # reset the request and response to reuse existing req_and_resp
        req_and_resp[0].reset()
        req_and_resp[1].reset()

        # required because of inheritance
        request, response = super(EsiClient, self).request(req_and_resp, opt)

        # check cache here so we have all headers, formed url and params
        cache_key = make_cache_key(request)
        res = self.__make_request(request, opt, cache_key)

        if res.status_code == 200:
            self.__cache_response(cache_key, res, request.method.upper())

        # generate the Response object from requests response
        response.raw_body_only = kwargs.pop(
            'raw_body_only',
            self.raw_body_only
        )

        try:
            response.apply_with(
                status=res.status_code,
                header=res.headers,
                raw=six.BytesIO(res.content).getvalue()
            )

        except ValueError:
            # catch JSONDecodeError/ValueError when response is not JSON
            raise APIException(
                request.url,
                res.status_code,
                response=res.content,
                request_param=request.query,
                response_header=res.headers
            )

        if 'warning' in res.headers:
            # send in logger and warnings, so the user doesn't have to use
            # logging to see it (at least once)
            LOGGER.warning("[%s] %s", res.url, res.headers['warning'])
            warnings.warn("[%s] %s" % (res.url, res.headers['warning']))

        if res.status_code >= 400 and kwargs.pop('raise_on_error', False):
            raise APIException(
                request.url,
                res.status_code,
                response=response.raw,
                request_param=request.query,
                response_header=response.header
            )

        return response


Instruction:
Can you implement a function in Python 3 that
makes a HEAD request and return a response object.

Response:
def head(self, req_and_resp, **kwargs):
        """ Take a request_and_response object from pyswagger.App, check
        and prepare everything to make a valid HEAD request

        :param req_and_resp: the request and response object from pyswagger.App
        :param opt: options, see pyswagger/blob/master/pyswagger/io.py#L144
        :param raise_on_error: boolean to raise an error if HTTP Code >= 400

        :return: the final response.
        """

        opt = kwargs.pop('opt', {})

        # reset the request and response to reuse existing req_and_resp
        req_and_resp[0].reset()
        req_and_resp[1].reset()

        # required because of inheritance
        request, response = super(EsiClient, self).request(req_and_resp, opt)

        res = self.__make_request(request, opt, method='HEAD')

        response.apply_with(
            status=res.status_code,
            header=res.headers,
            raw=None,
        )

        if 'warning' in res.headers:
            # send in logger and warnings, so the user doesn't have to use
            # logging to see it (at least once)
            LOGGER.warning("[%s] %s", res.url, res.headers['warning'])
            warnings.warn("[%s] %s" % (res.url, res.headers['warning']))

        if res.status_code >= 400 and kwargs.pop('raise_on_error', False):
            raise APIException(
                request.url,
                res.status_code,
                response='',
                request_param=request.query,
                response_header=response.header
            )

        return response


Instruction:
Write a Python 3 script to
cache the response and check if it is expired

Response:
def __cache_response(self, cache_key, res, method):
        """ cache the response

        if method is one of self.__uncached_method__, don't cache anything
        """
        if ('expires' in res.headers
                and method not in self.__uncached_methods__):
            cache_timeout = get_cache_time_left(res.headers.get('expires'))

            # Occasionally CCP swagger will return an outdated expire
            # warn and skip cache if timeout is <0
            if cache_timeout >= 0:
                self.cache.set(
                    cache_key,
                    CachedResponse(
                        status_code=res.status_code,
                        headers=res.headers,
                        content=res.content,
                        url=res.url,
                    ),
                )
            else:
                LOGGER.warning(
                    "[%s] returned expired result: %s", res.url,
                    res.headers)
                warnings.warn("[%s] returned expired result" % res.url)


Instruction:
Write a Python 3 script to
make a new request and return the response or cached response.

Response:
def __make_request(self, request, opt, cache_key=None, method=None):
        """ Check cache, deal with expiration and etag, make the request and
        return the response or cached response.

        :param request: the pyswagger.io.Request object to prepare the request
        :param opt: options, see pyswagger/blob/master/pyswagger/io.py#L144
        :param cache_key: the cache key used for the cache stuff.
        :param method: [default:None] allows to force the method, especially
            useful if you want to make a HEAD request.
            Default value will use endpoint method

        """
        # check expiration and etags
        opt_headers = {}
        cached_response = self.cache.get(cache_key, None)
        if cached_response is not None:
            # if we have expires cached, and still validd
            expires = cached_response.headers.get('expires', None)
            if expires is not None:
                cache_timeout = get_cache_time_left(
                    cached_response.headers['expires']
                )
                if cache_timeout >= 0:
                    return cached_response

            # if we have etags, add the header to use them
            etag = cached_response.headers.get('etag', None)
            if etag is not None:
                opt_headers['If-None-Match'] = etag

            # if nothing makes us use the cache, invalidate everything
            if (expires is None or cache_timeout < 0) and etag is None:
                self.cache.invalidate(cache_key)

        # apply request-related options before preparation.
        request.prepare(
            scheme=self.prepare_schemes(request).pop(),
            handle_files=False
        )
        request._patch(opt)

        # prepare the request and make it.
        method = method or request.method.upper()
        request.header.update(opt_headers)
        prepared_request = self._session.prepare_request(
            Request(
                method=method,
                url=request.url,
                params=request.query,
                data=request.data,
                headers=request.header
            )
        )
        start_api_call = time.time()

        try:
            res = self._session.send(
                prepared_request,
                timeout=self.timeout,
            )

        except (RequestsConnectionError, Timeout) as exc:
            # timeout issue, generate a fake response to finish the process
            # as a normal error 500
            res = CachedResponse(
                status_code=500,
                headers={},
                content=('{"error": "%s"}' % str(exc)).encode('latin-1'),
                url=prepared_request.url
            )

        # event for api call stats
        self.signal_api_call_stats.send(
            url=res.url,
            status_code=res.status_code,
            elapsed_time=time.time() - start_api_call,
            message=res.content if res.status_code != 200 else None
        )

        # if we have HTTP 304 (content didn't change), return the cached
        # response updated with the new headers
        if res.status_code == 304 and cached_response is not None:
            cached_response.headers['Expires'] = res.headers.get('Expires')
            cached_response.headers['Date'] = res.headers.get('Date')
            return cached_response
        return res


Instruction:
Can you tell what is the following Python 3 function doing
def _hash(data):
    hash_algo = hashlib.new('md5')
    hash_algo.update(pickle.dumps(data))
    # prefix allows possibility of multiple applications
    # sharing same keyspace
    return 'esi_' + hash_algo.hexdigest()

Response: generate a hash from data object to be used as cache key


Instruction:
Here you have a function in Python 3, explain what it does
def fit_from_image(self, data, voxelsize, seeds, unique_cls):
        fvs, clsselected = self.features_from_image(data, voxelsize, seeds, unique_cls)
        self.fit(fvs, clsselected)

Response: This Method computes feature vector and train model from the data.


Instruction:
Write a Python 3 function for
loading the current model from a file

Response:
def load(self, mdl_file):
        """
        load model from file. fv_type is not set with this function. It is expected to set it before.
        """
        import dill as pickle

        mdl_file_e = op.expanduser(mdl_file)

        sv = pickle.load(open(mdl_file_e, "rb"))
        self.mdl = sv["mdl"]
        # self.mdl[2] = self.mdl[0]
        # try:
        #     eval(sv['fv_extern_src'])
        #     eval("fv_extern_temp_name  = " + sv['fv_extern_src_name'])
        #     sv['fv_extern'] = fv_extern_temp_name
        # except:
        #     print "pomoc,necoje blbe"
        #     pass

        self.modelparams.update(sv["modelparams"])
        logger.debug("loaded model from path: " + mdl_file_e)


Instruction:
Create a Python 3 function to
function to compute feature vectors from image data.

Response:
def features_from_image(
        self, data, voxelsize, seeds=None, unique_cls=None
    ):  # , voxels=None):
        """
        Input data is 3d image

        :param data: is 3d image
        :param seeds: ndimage with same shape as data, nonzero values means seeds.
        :param unique_cls: can select only fv for seeds from specific class.
        f.e. unique_cls = [1, 2] ignores label 0

        funcion is called twice in graph cut
        first call is with all params, second is only with data.

        based on self.modelparams['fv_type'] the feature vector is computed
        keywords "intensity", "voxels", "fv001", "fv_extern"  can be used.
        modelparams['fv_type'] = 'fv_extern' allows to use external fv function

        Example of exter feature function. For easier implementation of return values use function return_fv_by_seeds().

        def fv_function(data, voxelsize, seeds=None, cl=None):
            data2 = scipy.ndimage.filters.gaussian_filter(data, sigma=5)
            arrs = [data.reshape(-1, 1), data2.reshape(-1, 1)]
            fv = np.concatenate(arrs, axis=1)
            return imcut.features.return_fv_by_seeds(fv, seeds, unique_cls)

        modelparams['fv_extern'] = fv_function
        """

        fv_type = self.modelparams["fv_type"]
        logger.debug("fv_type " + fv_type)
        fv = []
        if fv_type == "intensity":
            fv = data.reshape(-1, 1)

            if seeds is not None:
                logger.debug("seeds: %s", scipy.stats.describe(seeds, axis=None))
                sd = seeds.reshape(-1, 1)
                selection = np.in1d(sd, unique_cls)
                fv = fv[selection]
                sd = sd[selection]
                # sd = sd[]
                return fv, sd
            return fv

        # elif fv_type in ("voxels"):
        #     if seeds is not None:
        #         fv = np.asarray(voxels).reshape(-1, 1)
        #     else:
        #         fv = data
        #         fv = fv.reshape(-1, 1)
        elif fv_type in ("fv001", "FV001", "intensity_and_blur"):

            # intensity in pixel, gaussian blur intensity
            return features.fv_function_intensity_and_smoothing(
                data, voxelsize, seeds, unique_cls
            )

            # from PyQt4.QtCore import pyqtRemoveInputHook
            # pyqtRemoveInputHook()

            # print fv1.shape
            # print fv2.shape
            # print fv.shape
        elif fv_type == "fv_extern":
            fv_function = self.modelparams["fv_extern"]
            return fv_function(data, voxelsize, seeds, unique_cls)

        else:
            logger.error("Unknown feature vector type: " + self.modelparams["fv_type"])
        return fv


Instruction:
Create a Python 3 function for
fitting one class in the database.

Response:
def _fit_one_class(self, clx, cl):
        """ Train clas number cl with data clx.

        Use trainFromImageAndSeeds() function if you want to use 3D image data
        as an input.

        clx: data, 2d matrix
        cl: label, integer

        label: gmmsame, gaussian_kde, dpgmm, stored
        """

        logger.debug("clx " + str(clx[:10, :]))
        logger.debug("clx type" + str(clx.dtype))
        # name = 'clx' + str(cl) + '.npy'
        # print name
        # np.save(name, clx)
        logger.debug("_fit()")
        if self.modelparams["adaptation"] == "original_data":
            if cl in self.mdl.keys():
                return
        # if True:
        #     return

        logger.debug("training continues")

        if self.modelparams["type"] == "gmmsame":
            if len(clx.shape) == 1:
                logger.warning(
                    "reshaping in train will be removed. Use \
                                \ntrainFromImageAndSeeds() function"
                )

                print("Warning deprecated feature in train() function")
                #  je to jen jednorozměrný vektor, tak je potřeba to
                # převést na 2d matici
                clx = clx.reshape(-1, 1)
            gmmparams = self.modelparams["params"]
            self.mdl[cl] = sklearn.mixture.GaussianMixture(**gmmparams)
            self.mdl[cl].fit(clx)

        elif self.modelparams["type"] == "kernel":
            # Not working (probably) in old versions of scikits
            # from sklearn.neighbors.kde import KernelDensity
            from sklearn.neighbors import KernelDensity

            # kernelmodelparams = {'kernel': 'gaussian', 'bandwidth': 0.2}
            kernelmodelparams = self.modelparams["params"]
            self.mdl[cl] = KernelDensity(**kernelmodelparams).fit(clx)
        elif self.modelparams["type"] == "gaussian_kde":
            # print clx
            import scipy.stats

            # from PyQt4.QtCore import pyqtRemoveInputHook
            # pyqtRemoveInputHook()

            # gaussian_kde works only with floating point types
            self.mdl[cl] = scipy.stats.gaussian_kde(clx.astype(np.float))
        elif self.modelparams["type"] == "dpgmm":
            # print 'clx.shape ', clx.shape
            # print 'cl ', cl
            gmmparams = self.modelparams["params"]
            self.mdl[cl] = sklearn.mixture.DPGMM(**gmmparams)
            # todo here is a hack
            # dpgmm z nějakého důvodu nefunguje pro naše data
            # vždy natrénuje jednu složku v blízkosti nuly
            # patrně to bude mít něco společného s parametrem alpha
            # přenásobí-li se to malým číslem, zázračně to chodí
            self.mdl[cl].fit(clx * 0.001)
        elif self.modelparams["type"] == "stored":
            # Classifer is trained before segmentation and stored to pickle
            import pickle

            print("stored")
            logger.warning("deprecated use of stored parameters")

            mdl_file = self.modelparams["params"]["mdl_file"]
            self.mdl = pickle.load(open(mdl_file, "rb"))

        else:
            raise NameError("Unknown model type")


Instruction:
How would you explain what the following Python 3 function does
def likelihood(self, x, cl):

        # sha = x.shape
        # xr = x.reshape(-1, sha[-1])
        # outsha = sha[:-1]
        # from PyQt4.QtCore import pyqtRemoveInputHook
        # pyqtRemoveInputHook()
        logger.debug("likel " + str(x.shape))
        if self.modelparams["type"] == "gmmsame":

            px = self.mdl[cl].score_samples(x)

        # todo ošetřit více dimenzionální fv
        # px = px.reshape(outsha)
        elif self.modelparams["type"] == "kernel":
            px = self.mdl[cl].score_samples(x)
        elif self.modelparams["type"] == "gaussian_kde":
            # print x
            # np.log because it is likelihood
            # @TODO Zde je patrně problém s reshape
            # old
            # px = np.log(self.mdl[cl](x.reshape(-1)))
            # new
            px = np.log(self.mdl[cl](x))
            # px = px.reshape(outsha)
            # from PyQt4.QtCore import pyqtRemoveInputHook
            # pyqtRemoveInputHook()
        elif self.modelparams["type"] == "dpgmm":
            # todo here is a hack
            # dpgmm z nějakého důvodu nefunguje pro naše data
            # vždy natrénuje jednu složku v blízkosti nuly
            # patrně to bude mít něco společného s parametrem alpha
            # přenásobí-li se to malým číslem, zázračně to chodí
            logger.warning(".score() replaced with .score_samples() . Check it.")
            # px = self.mdl[cl].score(x * 0.01)
            px = self.mdl[cl].score_samples(x * 0.01)
        elif self.modelparams["type"] == "stored":
            px = self.mdl[cl].score(x)
        return px

Response: likelihood from data x with fature vector with 4 fatures


Instruction:
Can you implement a function in Python 3 that
gets information about one node in graph

Response:
def inspect_node_neighborhood(nlinks, msinds, node_msindex):
    """
    Get information about one node in graph

    :param nlinks: neighboorhood edges
    :param msinds: indexes in 3d image
    :param node_msindex: int, multiscale index of selected voxel
    :return: node_neighboor_edges_and_weights, node_neighboor_seeds
    """
    # seed_indexes = np.nonzero(node_seed)
    # selected_inds = msinds[seed_indexes]
    # node_msindex = selected_inds[0]

    node_neighbor_edges = get_neighborhood_edes(nlinks, node_msindex)

    node_neighbor_seeds = np.zeros_like(msinds, dtype=np.int8)
    for neighboor_ind in np.unique(node_neighbor_edges[:, :2].ravel()):
        node_neighbor_ind = np.where(msinds == neighboor_ind)
        node_neighbor_seeds[node_neighbor_ind] = 2

    node_neighbor_seeds[np.where(msinds == node_msindex)] = 1

    # node_coordinates = np.unravel_index(selected_voxel_ind, msinds.shape)
    # node_neighbor_coordinates = np.unravel_index(np.unique(node_neighbor_edges[:, :2].ravel()), msinds.shape)
    return node_neighbor_edges, node_neighbor_seeds


Instruction:
How would you explain what the following Python 3 function does
def inspect_node(nlinks, unariesalt, msinds, node_msindex):

    node_unariesalt = unariesalt[node_msindex]
    neigh_edges, neigh_seeds = inspect_node_neighborhood(nlinks, msinds, node_msindex)
    return node_unariesalt, neigh_edges, neigh_seeds

Response: Inspects one node in graph and returns unariesalt edges and weights for each node in graph


Instruction:
Can you tell what is the following Python 3 function doing
def get_node_msindex(msinds, node_seed):
    if type(node_seed) == np.ndarray:
        seed_indexes = np.nonzero(node_seed)
    elif type(node_seed) == list:
        seed_indexes = node_seed
    else:
        seed_indexes = [node_seed]

    selected_nodes_msinds = msinds[seed_indexes]
    node_msindex = selected_nodes_msinds[0]
    return node_msindex

Response: Convert seeds - like selection of voxel to multiscale index.


Instruction:
Can you tell what is the following Python 3 function doing
def relabel_squeeze(data):
    palette, index = np.unique(data, return_inverse=True)
    data = index.reshape(data.shape)
    # realy slow solution
    #        unq = np.unique(data)
    #        actual_label = 0
    #        for lab in unq:
    #            data[data == lab] = actual_label
    #            actual_label += 1

    # one another solution probably slower
    # arr = data
    # data = (np.digitize(arr.reshape(-1,),np.unique(arr))-1).reshape(arr.shape)

    return data

Response: Makes relabeling of data if there are unused values.


Instruction:
Can you tell what is the following Python 3 function doing
def load(self, filename, fv_extern=None):
        self.modelparams["mdl_stored_file"] = filename
        if fv_extern is not None:
            self.modelparams["fv_extern"] = fv_extern

        # segparams['modelparams'] = {
        #     'mdl_stored_file': mdl_stored_file,
        #     # 'fv_extern': fv_function
        # }
        self.mdl = Model(modelparams=self.modelparams)

Response: Load the model stored in the file.


Instruction:
Write a Python 3 function that can
get the segmentation and the :return:

Response:
def __msgc_step12_low_resolution_segmentation(self):
        """
        Get the segmentation and the
        :return:
        """
        import scipy

        start = self._start_time
        # ===== low resolution data processing
        # default parameters
        # TODO segparams_lo and segparams_hi je tam asi zbytecně
        sparams_lo = {
            "boundary_dilatation_distance": 2,
            "block_size": 6,
            "use_boundary_penalties": True,
            "boundary_penalties_weight": 1,
            "tile_zoom_constant": 1,
        }

        sparams_lo.update(self.segparams)
        sparams_hi = copy.copy(sparams_lo)
        # sparams_lo['boundary_penalties_weight'] = (
        #         sparams_lo['boundary_penalties_weight'] *
        #         sparams_lo['block_size'])
        self.segparams = sparams_lo

        self.stats["t1"] = time.time() - start
        # step 1:  low res GC
        hiseeds = self.seeds
        # ms_zoom = 4  # 0.125 #self.segparams['scale']
        # ms_zoom = self.segparams['block_size']
        # loseeds = pyed.getSeeds()
        # logger.debug("msc " + str(np.unique(hiseeds)))
        loseeds = seed_zoom(hiseeds, self.segparams["block_size"])

        hard_constraints = True

        self.seeds = loseeds

        modelparams_hi = self.modelparams.copy()
        # feature vector will be computed from selected voxels
        self.modelparams["use_extra_features_for_training"] = True

        # TODO what with voxels? It is used from here
        # hiseeds and hiimage is used to create intensity model
        self.voxels1 = self.img[hiseeds == 1].reshape(-1, 1)
        self.voxels2 = self.img[hiseeds == 2].reshape(-1, 1)
        # this is how to compute with loseeds resolution but in wrong way
        # self.voxels1 = self.img[self.seeds == 1]
        # self.voxels2 = self.img[self.seeds == 2]

        # self.voxels1 = pyed.getSeedsVal(1)
        # self.voxels2 = pyed.getSeedsVal(2)

        img_orig = self.img

        # TODO this should be done with resize_to_shape_whith_zoom
        zoom = np.asarray(loseeds.shape).astype(np.float) / img_orig.shape
        self.img = scipy.ndimage.interpolation.zoom(img_orig, zoom, order=0)
        voxelsize_orig = self.voxelsize
        logger.debug("zoom %s", zoom)
        logger.debug("vs %s", self.voxelsize)
        self.voxelsize = self.voxelsize * zoom

        # self.img = resize_to_shape_with_zoom(img_orig, loseeds.shape, 1.0 / ms_zoom, order=0)

        # this step set the self.segmentation
        self.__single_scale_gc_run()
        # logger.debug(
        #     'segmentation - max: %d min: %d' % (
        #         np.max(self.segmentation),
        #         np.min(self.segmentation)
        #     )
        # )
        logger.debug(
            "segmentation: %s", scipy.stats.describe(self.segmentation, axis=None)
        )

        self.modelparams = modelparams_hi
        self.voxelsize = voxelsize_orig
        self.img = img_orig
        self.seeds = hiseeds
        self.stats["t2"] = time.time() - start
        return hard_constraints


Instruction:
Write a Python 3 function that can
estimate discontinuity in basis of low resolution image segmentation. :return: discontinuity in low resolution

Response:
def __msgc_step3_discontinuity_localization(self):
        """
        Estimate discontinuity in basis of low resolution image segmentation.
        :return: discontinuity in low resolution
        """
        import scipy

        start = self._start_time
        seg = 1 - self.segmentation.astype(np.int8)
        self.stats["low level object voxels"] = np.sum(seg)
        self.stats["low level image voxels"] = np.prod(seg.shape)
        # in seg is now stored low resolution segmentation
        # back to normal parameters
        # step 2: discontinuity localization
        # self.segparams = sparams_hi
        seg_border = scipy.ndimage.filters.laplace(seg, mode="constant")
        logger.debug("seg_border: %s", scipy.stats.describe(seg_border, axis=None))
        # logger.debug(str(np.max(seg_border)))
        # logger.debug(str(np.min(seg_border)))
        seg_border[seg_border != 0] = 1
        logger.debug("seg_border: %s", scipy.stats.describe(seg_border, axis=None))
        # scipy.ndimage.morphology.distance_transform_edt
        boundary_dilatation_distance = self.segparams["boundary_dilatation_distance"]
        seg = scipy.ndimage.morphology.binary_dilation(
            seg_border,
            # seg,
            np.ones(
                [
                    (boundary_dilatation_distance * 2) + 1,
                    (boundary_dilatation_distance * 2) + 1,
                    (boundary_dilatation_distance * 2) + 1,
                ]
            ),
        )
        if self.keep_temp_properties:
            self.temp_msgc_lowres_discontinuity = seg
        else:
            self.temp_msgc_lowres_discontinuity = None

        if self.debug_images:
            import sed3

            pd = sed3.sed3(seg_border)  # ), contour=seg)
            pd.show()
            pd = sed3.sed3(seg)  # ), contour=seg)
            pd.show()
        # segzoom = scipy.ndimage.interpolation.zoom(seg.astype('float'), zoom,
        #                                                order=0).astype('int8')
        self.stats["t3"] = time.time() - start
        return seg


Instruction:
Can you generate the documentation for the following Python 3 function
def __multiscale_gc_lo2hi_run(self):  # , pyed):
        # from PyQt4.QtCore import pyqtRemoveInputHook
        # pyqtRemoveInputHook()
        self._msgc_lo2hi_resize_init()
        self.__msgc_step0_init()

        hard_constraints = self.__msgc_step12_low_resolution_segmentation()
        # ===== high resolution data processing
        seg = self.__msgc_step3_discontinuity_localization()

        self.stats["t3.1"] = (time.time() - self._start_time)
        graph = Graph(
            seg,
            voxelsize=self.voxelsize,
            nsplit=self.segparams["block_size"],
            edge_weight_table=self._msgc_npenalty_table,
            compute_low_nodes_index=True,
        )

        # graph.run() = graph.generate_base_grid() + graph.split_voxels()
        # graph.run()
        graph.generate_base_grid()
        self.stats["t3.2"] = (time.time() - self._start_time)
        graph.split_voxels()

        self.stats["t3.3"] = (time.time() - self._start_time)

        self.stats.update(graph.stats)
        self.stats["t4"] = (time.time() - self._start_time)
        mul_mask, mul_val = self.__msgc_tlinks_area_weight_from_low_segmentation(seg)
        area_weight = 1
        unariesalt = self.__create_tlinks(
            self.img,
            self.voxelsize,
            self.seeds,
            area_weight=area_weight,
            hard_constraints=hard_constraints,
            mul_mask=None,
            mul_val=None,
        )
        # N-links prepared
        self.stats["t5"] = (time.time() - self._start_time)
        un, ind = np.unique(graph.msinds, return_index=True)
        self.stats["t6"] = (time.time() - self._start_time)

        self.stats["t7"] = (time.time() - self._start_time)
        unariesalt2_lo2hi = np.hstack(
            [unariesalt[ind, 0, 0].reshape(-1, 1), unariesalt[ind, 0, 1].reshape(-1, 1)]
        )
        nlinks_lo2hi = np.hstack([graph.edges, graph.edges_weights.reshape(-1, 1)])
        if self.debug_images:
            import sed3

            ed = sed3.sed3(unariesalt[:, :, 0].reshape(self.img.shape))
            ed.show()
            import sed3

            ed = sed3.sed3(unariesalt[:, :, 1].reshape(self.img.shape))
            ed.show()
            # ed = sed3.sed3(seg)
            # ed.show()
            # import sed3
            # ed = sed3.sed3(graph.data)
            # ed.show()
            # import sed3
            # ed = sed3.sed3(graph.msinds)
            # ed.show()

        # nlinks, unariesalt2, msinds = self.__msgc_step45678_construct_graph(area_weight, hard_constraints, seg)
        # self.__msgc_step9_finish_perform_gc_and_reshape(nlinks, unariesalt2, msinds)
        self.__msgc_step9_finish_perform_gc_and_reshape(
            nlinks_lo2hi, unariesalt2_lo2hi, graph.msinds
        )
        self._msgc_lo2hi_resize_clean_finish()

Response: Run Graph - Cut segmentation with refinement of low resolution multiscale graph.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def __multiscale_gc_hi2lo_run(self):  # , pyed):
        # from PyQt4.QtCore import pyqtRemoveInputHook
        # pyqtRemoveInputHook()

        self.__msgc_step0_init()
        hard_constraints = self.__msgc_step12_low_resolution_segmentation()
        # ===== high resolution data processing
        seg = self.__msgc_step3_discontinuity_localization()
        nlinks, unariesalt2, msinds = self.__msgc_step45678_hi2lo_construct_graph(
            hard_constraints, seg
        )
        self.__msgc_step9_finish_perform_gc_and_reshape(nlinks, unariesalt2, msinds)

Response: Run Graph - Cut segmentation with simplifiyng of high resolution multiscale graph.


Instruction:
Can you generate the documentation for the following Python 3 function
def __ordered_values_by_indexes(self, data, inds):
        # get unique labels and their first indexes
        # lab, linds = np.unique(inds, return_index=True)
        # compute values by indexes
        # values = data.reshape(-1)[linds]

        # alternative slow implementation
        # if there are different data on same index, it will take
        # maximal value
        # lab = np.unique(inds)
        # values = [0]*len(lab)
        # for label in lab:
        #     values[label] = np.max(data[inds == label])
        #
        # values = np.asarray(values)

        # yet another implementation
        values = [None] * (np.max(inds) + 1)

        linear_inds = inds.ravel()
        linear_data = data.ravel()
        for i in range(0, len(linear_inds)):
            # going over all data pixels

            if values[linear_inds[i]] is None:
                # this index is found for first
                values[linear_inds[i]] = linear_data[i]
            elif values[linear_inds[i]] < linear_data[i]:
                # here can be changed maximal or minimal value
                values[linear_inds[i]] = linear_data[i]

        values = np.asarray(values)

        return values

Response: Return values by indexes.


Instruction:
Write a Python 3 script for
functioning computes multiscale indexes of ndarray.

Response:
def __hi2lo_multiscale_indexes(self, mask, orig_shape):  # , zoom):
        """
        Function computes multiscale indexes of ndarray.

        mask: Says where is original resolution (0) and where is small
        resolution (1). Mask is in small resolution.

        orig_shape: Original shape of input data.
        zoom: Usually number greater then 1

        result = [[0 1 2],
                  [3 4 4],
                  [5 4 4]]
        """

        mask_orig = zoom_to_shape(mask, orig_shape, dtype=np.int8)

        inds_small = np.arange(mask.size).reshape(mask.shape)
        inds_small_in_orig = zoom_to_shape(inds_small, orig_shape, dtype=np.int8)
        inds_orig = np.arange(np.prod(orig_shape)).reshape(orig_shape)

        # inds_orig = inds_orig * mask_orig
        inds_orig += np.max(inds_small_in_orig) + 1
        # print 'indexes'
        # import py3DSeedEditor as ped
        # import pdb; pdb.set_trace() # BREAKPOINT

        #  '==' is not the same as 'is' for numpy.array
        inds_small_in_orig[mask_orig == True] = inds_orig[mask_orig == True]  # noqa
        inds = inds_small_in_orig
        # print np.max(inds)
        # print np.min(inds)
        inds = relabel_squeeze(inds)
        logger.debug(
            "Index after relabeling: %s", scipy.stats.describe(inds, axis=None)
        )
        # logger.debug("Minimal index after relabeling: " + str(np.min(inds)))
        # inds_orig[mask_orig==True] = 0
        # inds_small_in_orig[mask_orig==False] = 0
        # inds = (inds_orig + np.max(inds_small_in_orig) + 1) + inds_small_in_orig

        return inds, mask_orig


Instruction:
Here you have a function in Python 3, explain what it does
def interactivity(self, min_val=None, max_val=None, qt_app=None):
        from .seed_editor_qt import QTSeedEditor
        from PyQt4.QtGui import QApplication

        if min_val is None:
            min_val = np.min(self.img)

        if max_val is None:
            max_val = np.max(self.img)

        window_c = (max_val + min_val) / 2  # .astype(np.int16)
        window_w = max_val - min_val  # .astype(np.int16)

        if qt_app is None:
            qt_app = QApplication(sys.argv)

        pyed = QTSeedEditor(
            self.img,
            modeFun=self.interactivity_loop,
            voxelSize=self.voxelsize,
            seeds=self.seeds,
            volume_unit=self.volume_unit,
        )

        pyed.changeC(window_c)
        pyed.changeW(window_w)

        qt_app.exec_()

Response: Interactivity setting with 3d seed editor


Instruction:
Write a Python 3 script for
functioning for manual seed setting. Sets variable seeds and prepares voxels for density model.

Response:
def set_seeds(self, seeds):
        """
        Function for manual seed setting. Sets variable seeds and prepares
        voxels for density model.
        :param seeds: ndarray (0 - nothing, 1 - object, 2 - background,
        3 - object just hard constraints, no model training, 4 - background
        just hard constraints, no model training)
        """
        if self.img.shape != seeds.shape:
            raise Exception("Seeds must be same size as input image")

        self.seeds = seeds.astype("int8")
        self.voxels1 = self.img[self.seeds == 1]
        self.voxels2 = self.img[self.seeds == 2]