Instruction:
How would you code a function in Python 3 to
generate an annotation dictionary for a DataFrame

Response:
def get_annotations(df,annotations,kind='lines',theme=None,**kwargs):
	"""
	Generates an annotations object

	Parameters:
	-----------
		df : DataFrame
			Original DataFrame of values
		annotations : dict or list
			Dictionary of annotations
			{x_point : text}
			or
			List of Plotly annotations
	"""

	for key in list(kwargs.keys()):
		if key not in __ANN_KWARGS:
			raise Exception("Invalid keyword : '{0}'".format(key))

	theme_data = getTheme(theme)

	kwargs['fontcolor']=kwargs.pop('fontcolor',theme_data['annotations']['fontcolor'])
	kwargs['arrowcolor']=kwargs.pop('arrowcolor',theme_data['annotations']['arrowcolor'])
	kwargs['fontsize']=kwargs.pop('fontsize',12)

	def check_ann(annotation):
		local_list=[]

		if 'title' in annotation:
			local_list.append(
					dict(
							text=annotation['title'],
							showarrow=False,
							x=0,
							y=1,
							xref='paper',
							yref='paper',
							font={'size':24 if not 'fontsize' in kwargs else kwargs['fontsize']}
						)
				)

			del annotation['title']
			local_list.append(ann)

		elif 'x' in annotation:
			ann=dict(
								x=annotation['x'],
								y=annotation.get('y',.5),
								xref=annotation.get('xref','x'),
								yref=annotation.get('yref',kwargs.get('yref','y1')),
								text=annotation.get('text'),
								showarrow=annotation.get('showarrow',True),
								arrowhead=annotation.get('arrowhead',7),
								arrowcolor=annotation.get('arrowcolor',kwargs.get('arrowcolor')),
								ax=annotation.get('ax',0),
								ay=annotation.get('ay',-100),
								textangle=annotation.get('textangle',-90),
								font = dict(
									color = annotation.get('fontcolor',annotation.get('color',kwargs.get('fontcolor'))),
									size = annotation.get('fontsize',annotation.get('size',kwargs.get('fontsize')))
								)
								)
			local_list.append(ann)

		else:
			for k,v in list(annotation.items()):
				if kind in ('candlestick','ohlc','candle'):
					d=ta._ohlc_dict(df)
					maxv=df[d['high']].ix[k]
					yref='y2'
				else:
					maxv=df.ix[k].sum() if k in df.index else 0
					yref='y1'
				ann=dict(
								x=k,
								y=maxv,
								xref='x',
								yref=yref,
								text=v,
								showarrow=kwargs.get('showarrow',True),
								arrowhead=kwargs.get('arrowhead',7),
								arrowcolor = kwargs['arrowcolor'],
								ax=kwargs.get('ax',0),
								ay=kwargs.get('ay',-100),
								textangle=kwargs.get('textangle',-90),
								font = dict(
									color = kwargs['fontcolor'],
									size=kwargs['fontsize']
								)
								)
				local_list.append(ann)

		return local_list

	annotations = make_list(annotations)
	_list_ann=[]
	for ann in annotations:
		_list_ann.extend(check_ann(ann))
	return _list_ann


Instruction:
How would you explain what the following Python 3 function does
def get_base_layout(figs):
	layout={}
	for fig in figs:
		if not isinstance(fig,dict):
			fig=fig.to_dict()
		for k,v in list(fig['layout'].items()):
			layout[k]=v
	return layout

Response:
Generates a layout with the union of all properties of multiple
	figures


Instruction:
Implement a Python 3 function for
generating multiple Plotly figures for a given DataFrame.

Response:
def figures(df,specs,asList=False):
	"""
	Generates multiple Plotly figures for a given DataFrame

	Parameters:
	-----------
		df : DataFrame
			Pandas DataFrame
		specs : list(dict)
			List of dictionaries with the properties
			of each figure.
			All properties avaialbe can be seen with
			help(cufflinks.pd.DataFrame.iplot)
		asList : boolean
			If True, then a list of figures is returned.
			Otherwise a single (merged) figure is returned.
			Default : False
	"""
	figs=[]
	for spec in specs:
		figs.append(df.figure(**spec))
	if asList:
		return figs
	else:
		return merge_figures(figs)


Instruction:
Make a summary of the following Python 3 code
def merge_figures(figures):
	figure={}
	data=[]
	for fig in figures:
		for trace in fig['data']:
			data.append(trace)
	layout=get_base_layout(figures)
	figure['data']=data
	figure['layout']=layout
	return figure

Response:
Generates a single Figure from a list of figures
	Parameters :
	-----------
	figures : list of figures
	Parameters


Instruction:
How would you implement a function in Python 3 that
generates a subplot view for a set of figures This is a wrapper for plotly.tools.make_subplots Parameters: ----------- figures : [Figures] List of Plotly Figures shape : (rows,cols) Tuple indicating the size of rows and columns If omitted then the layout is automatically set shared_xaxes : bool Assign shared x axes. If True, subplots in the same grid column have one common shared x-axis at the bottom of the grid. shared_yaxes : bool Assign shared y axes. If True, subplots in the same grid row have one common shared y-axis on the left-hand side of the grid. start_cell : string 'bottom-left' 'top-left' Choose the starting cell in the subplot grid used to set the domains of the subplots. theme : string Layout Theme solar pearl white see cufflinks.getThemes() for all available themes base_layout : layout (dict) Layout to be used as base where the subplots will be added subplot_titles : list(string) List of strings that contains the titles of each plot. horizontal_spacing : float [0,1] Space between subplot columns. vertical_spacing : float Space between subplot rows. specs : list of dicts Subplot specifications. ex1: specs=[[{}, {}], [{'colspan': 2}, None]] ex2: specs=[[{'rowspan': 2}, {}], [None, {}]] - Indices of the outer list correspond to subplot grid rows starting from the bottom. The number of rows in 'specs' must be equal to 'rows'. - Indices of the inner lists correspond to subplot grid columns starting from the left. The number of columns in 'specs' must be equal to 'cols'. - Each item in the 'specs' list corresponds to one subplot in a subplot grid. (N.B. The subplot grid has exactly 'rows' times 'cols' cells.) - Use None for blank a subplot cell (or to move pass a col/row span). - Note that specs[0][0] has the specs of the 'start_cell' subplot. - Each item in 'specs' is a dictionary. The available keys are: * is_3d (boolean, default=False): flag for 3d scenes * colspan (int, default=1): number of subplot columns for this subplot to span. * rowspan (int, default=1): number of subplot rows for this subplot to span. * l (float, default=0.0): padding left of cell * r (float, default=0.0): padding right of cell * t (float, default=0.0): padding right of cell * b (float, default=0.0): padding bottom of cell - Use 'horizontal_spacing' and 'vertical_spacing' to adjust the spacing in between the subplots. insets : list of dicts Inset specifications. - Each item in 'insets' is a dictionary. The available keys are: * cell (tuple, default=(1,1)): (row, col) index of the subplot cell to overlay inset axes onto. * is_3d (boolean, default=False): flag for 3d scenes * l (float, default=0.0): padding left of inset in fraction of cell width * w (float or 'to_end', default='to_end') inset width in fraction of cell width ('to_end': to cell right edge) * b (float, default=0.0): padding bottom of inset in fraction of cell height * h (float or 'to_end', default='to_end') inset height in fraction of cell height ('to_end': to cell top edge)

Response:
def subplots(figures,shape=None,
				  shared_xaxes=False, shared_yaxes=False,
				  start_cell='top-left', theme=None,base_layout=None,
				  **kwargs):
	"""
	Generates a subplot view for a set of figures
	This is a wrapper for plotly.tools.make_subplots

	Parameters:
	-----------
		figures : [Figures]
			List of Plotly Figures
		shape : (rows,cols)
			Tuple indicating the size of rows and columns
			If omitted then the layout is automatically set
		shared_xaxes : bool
			Assign shared x axes.
			If True, subplots in the same grid column have one common
			shared x-axis at the bottom of the grid.
		shared_yaxes : bool
			Assign shared y axes.
			If True, subplots in the same grid row have one common
			shared y-axis on the left-hand side of the grid.
		start_cell : string
				'bottom-left'
				'top-left'
			Choose the starting cell in the subplot grid used to set the
			domains of the subplots.
		theme : string
			Layout Theme
				solar
				pearl
				white
			see cufflinks.getThemes() for all
			available themes
		base_layout : layout (dict)
			Layout to be used as base where the subplots will be
			added
		subplot_titles : list(string)
			List of strings that contains the titles of each
			plot.
		horizontal_spacing : float
				[0,1]
			Space between subplot columns.
		vertical_spacing : float
			Space between subplot rows.
		specs : list of dicts
			Subplot specifications.
				ex1: specs=[[{}, {}], [{'colspan': 2}, None]]
				ex2: specs=[[{'rowspan': 2}, {}], [None, {}]]

			- Indices of the outer list correspond to subplot grid rows
			  starting from the bottom. The number of rows in 'specs'
			  must be equal to 'rows'.

			- Indices of the inner lists correspond to subplot grid columns
			  starting from the left. The number of columns in 'specs'
			  must be equal to 'cols'.

			- Each item in the 'specs' list corresponds to one subplot
			  in a subplot grid. (N.B. The subplot grid has exactly 'rows'
			  times 'cols' cells.)

			- Use None for blank a subplot cell (or to move pass a col/row span).

			- Note that specs[0][0] has the specs of the 'start_cell' subplot.

			- Each item in 'specs' is a dictionary.
				The available keys are:

				* is_3d (boolean, default=False): flag for 3d scenes
				* colspan (int, default=1): number of subplot columns
					for this subplot to span.
				* rowspan (int, default=1): number of subplot rows
					for this subplot to span.
				* l (float, default=0.0): padding left of cell
				* r (float, default=0.0): padding right of cell
				* t (float, default=0.0): padding right of cell
				* b (float, default=0.0): padding bottom of cell

			- Use 'horizontal_spacing' and 'vertical_spacing' to adjust
			  the spacing in between the subplots.

		insets : list of dicts
			Inset specifications.

			- Each item in 'insets' is a dictionary.
				The available keys are:

				* cell (tuple, default=(1,1)): (row, col) index of the
					subplot cell to overlay inset axes onto.
				* is_3d (boolean, default=False): flag for 3d scenes
				* l (float, default=0.0): padding left of inset
					  in fraction of cell width
				* w (float or 'to_end', default='to_end') inset width
					  in fraction of cell width ('to_end': to cell right edge)
				* b (float, default=0.0): padding bottom of inset
					  in fraction of cell height
				* h (float or 'to_end', default='to_end') inset height
					  in fraction of cell height ('to_end': to cell top edge)
	"""
	if not isinstance(figures,list):
		figures=[figures]

	if shape:
		rows,cols=shape
		if len(figures)>rows*cols:
			raise Exception("Invalid shape for the number of figures given")
	else:
		if len(figures)==1:
			cols=1
			rows=1
		elif shared_xaxes:
			cols=1
			rows=len(figures)
		elif shared_yaxes:
			cols=len(figures)
			rows=1
		else:
			cols=2
			rows=len(figures)//2+len(figures)%2
	sp,grid_ref=get_subplots(rows=rows,cols=cols,
				             shared_xaxes=shared_xaxes, shared_yaxes=shared_yaxes,
				             start_cell=start_cell, theme=theme,base_layout=base_layout,
				             **kwargs)
	list_ref=(col for row in grid_ref for col in row)
	for i in range(len(figures)):
		while True:
			lr=next(list_ref)
			if lr is not None:
				break
		for _ in figures[i]['data']:
			for axe in lr:
				_.update({'{0}axis'.format(axe[0]):axe})
			sp['data'].append(_)
	# Remove extra plots
	for k in list(sp['layout'].keys()):
		try:
			if int(k[-1])>len(figures):
				del sp['layout'][k]
		except:
			pass

	# Check for non-cartesian plots
	data=sp['data']
	layout=sp['layout']
	for d in data:
		if d['type']=='pie':
			d['domain']={}
			d['domain']['x']=layout['xaxis{0}'.format(d['xaxis'][1:])]['domain']
			d['domain']['y']=layout['yaxis{0}'.format(d['yaxis'][1:])]['domain']
	return sp


Instruction:
Write a Python 3 function for
generating a subplot view for a set of figures Parameters: ----------- rows : int Number of rows cols : int Number of cols shared_xaxes : bool Assign shared x axes. If True, subplots in the same grid column have one common shared x-axis at the bottom of the gird. shared_yaxes : bool Assign shared y axes. If True, subplots in the same grid row have one common shared y-axis on the left-hand side of the gird. start_cell : string 'bottom-left' 'top-left' Choose the starting cell in the subplot grid used to set the domains of the subplots. theme : string Layout Theme solar pearl white see cufflinks.getThemes() for all available themes horizontal_spacing : float [0,1] Space between subplot columns. vertical_spacing : float Space between subplot rows. specs : list of dicts Subplot specifications. ex1: specs=[[{}, {}], [{'colspan': 2}, None]] ex2: specs=[[{'rowspan': 2}, {}], [None, {}]] - Indices of the outer list correspond to subplot grid rows starting from the bottom. The number of rows in 'specs' must be equal to 'rows'. - Indices of the inner lists correspond to subplot grid columns starting from the left. The number of columns in 'specs' must be equal to 'cols'. - Each item in the 'specs' list corresponds to one subplot in a subplot grid. (N.B. The subplot grid has exactly 'rows' times 'cols' cells.) - Use None for blank a subplot cell (or to move pass a col/row span). - Note that specs[0][0] has the specs of the 'start_cell' subplot. - Each item in 'specs' is a dictionary. The available keys are: * is_3d (boolean, default=False): flag for 3d scenes * colspan (int, default=1): number of subplot columns for this subplot to span. * rowspan (int, default=1): number of subplot rows for this subplot to span. * l (float, default=0.0): padding left of cell * r (float, default=0.0): padding right of cell * t (float, default=0.0): padding right of cell * b (float, default=0.0): padding bottom of cell - Use 'horizontal_spacing' and 'vertical_spacing' to adjust the spacing in between the subplots. insets : list of dicts Inset specifications. - Each item in 'insets' is a dictionary. The available keys are: * cell (tuple, default=(1,1)): (row, col) index of the subplot cell to overlay inset axes onto. * is_3d (boolean, default=False): flag for 3d scenes * l (float, default=0.0): padding left of inset in fraction of cell width * w (float or 'to_end', default='to_end') inset width in fraction of cell width ('to_end': to cell right edge) * b (float, default=0.0): padding bottom of inset in fraction of cell height * h (float or 'to_end', default='to_end') inset height in fraction of cell height ('to_end': to cell top edge)

Response:
def get_subplots(rows=1,cols=1,
				  shared_xaxes=False, shared_yaxes=False,
				  start_cell='top-left', theme=None,base_layout=None,
				  **kwargs):

	"""
	Generates a subplot view for a set of figures

	Parameters:
	-----------
		rows : int
			Number of rows
		cols : int
			Number of cols
		shared_xaxes : bool
			Assign shared x axes.
			If True, subplots in the same grid column have one common
			shared x-axis at the bottom of the gird.
		shared_yaxes : bool
			Assign shared y axes.
			If True, subplots in the same grid row have one common
			shared y-axis on the left-hand side of the gird.
		start_cell : string
				'bottom-left'
				'top-left'
			Choose the starting cell in the subplot grid used to set the
			domains of the subplots.
		theme : string
			Layout Theme
				solar
				pearl
				white
			see cufflinks.getThemes() for all
			available themes
		horizontal_spacing : float
				[0,1]
			Space between subplot columns.
		vertical_spacing : float
			Space between subplot rows.
		specs : list of dicts
			Subplot specifications.
				ex1: specs=[[{}, {}], [{'colspan': 2}, None]]
				ex2: specs=[[{'rowspan': 2}, {}], [None, {}]]

			- Indices of the outer list correspond to subplot grid rows
			  starting from the bottom. The number of rows in 'specs'
			  must be equal to 'rows'.

			- Indices of the inner lists correspond to subplot grid columns
			  starting from the left. The number of columns in 'specs'
			  must be equal to 'cols'.

			- Each item in the 'specs' list corresponds to one subplot
			  in a subplot grid. (N.B. The subplot grid has exactly 'rows'
			  times 'cols' cells.)

			- Use None for blank a subplot cell (or to move pass a col/row span).

			- Note that specs[0][0] has the specs of the 'start_cell' subplot.

			- Each item in 'specs' is a dictionary.
				The available keys are:

				* is_3d (boolean, default=False): flag for 3d scenes
				* colspan (int, default=1): number of subplot columns
					for this subplot to span.
				* rowspan (int, default=1): number of subplot rows
					for this subplot to span.
				* l (float, default=0.0): padding left of cell
				* r (float, default=0.0): padding right of cell
				* t (float, default=0.0): padding right of cell
				* b (float, default=0.0): padding bottom of cell

			- Use 'horizontal_spacing' and 'vertical_spacing' to adjust
			  the spacing in between the subplots.

		insets : list of dicts
			Inset specifications.

			- Each item in 'insets' is a dictionary.
				The available keys are:

				* cell (tuple, default=(1,1)): (row, col) index of the
					subplot cell to overlay inset axes onto.
				* is_3d (boolean, default=False): flag for 3d scenes
				* l (float, default=0.0): padding left of inset
					  in fraction of cell width
				* w (float or 'to_end', default='to_end') inset width
					  in fraction of cell width ('to_end': to cell right edge)
				* b (float, default=0.0): padding bottom of inset
					  in fraction of cell height
				* h (float or 'to_end', default='to_end') inset height
					  in fraction of cell height ('to_end': to cell top edge)
	"""

	if not theme:
		theme = auth.get_config_file()['theme']

	layout= base_layout if base_layout else getLayout(theme,**check_kwargs(kwargs,__LAYOUT_AXIS))
	sp=make_subplots(rows=rows,cols=cols,shared_xaxes=shared_xaxes,
										   shared_yaxes=shared_yaxes,print_grid=False,
											start_cell=start_cell,**kwargs)
	sp, grid_ref = sp.to_dict(), sp._grid_ref

	for k,v in list(layout.items()):
		if 'xaxis' not in k and 'yaxis' not in k:
			sp['layout'].update({k:v})

	def update_axis(fig,layout):
		for axis, n in list(Figure(fig).axis['len'].items()):
			for _ in range(1,n+1):
				for k,v in list(layout['{0}axis'.format(axis)].items()):
					_='' if _==1 else _
					if k not in fig['layout']['{0}axis{1}'.format(axis,_)]:
						fig['layout']['{0}axis{1}'.format(axis,_)][k]=v

	update_axis(sp,layout)
	# 124 - zeroline on the first figure

	# if 'subplot_titles' in kwargs:
	# 	if 'annotations' in layout:
	# 		annotation=sp['layout']['annotations'][0]
	# 	else:
	# 		annotation=getLayout(theme,annotations=[Annotation(text='')])['annotations']
	# 	for ann in sp['layout']['annotations']:
	# 		ann.update(font=dict(color=annotation['font']['color']))

	# def update_items(sp_item,layout,axis):
	# 	for k,v in list(layout[axis].items()):
	# 		sp_item.update({k:v})

	# for k,v in list(sp['layout'].items()):
	# 	if isinstance(v,go.XAxis):
	# 		update_items(v,layout,'xaxis1')
	# 	elif isinstance(v,go.YAxis):
	# 		update_items(v,layout,'xaxis1')

	return sp, grid_ref


Instruction:
Create a Python 3 function for
displaying a matrix with scatter plot for each pair of Series in the DataFrame.

Response:
def scatter_matrix(df,theme=None,bins=10,color='grey',size=2):
	"""
	Displays a matrix with scatter plot for each pair of
	Series in the DataFrame.
	The diagonal shows a histogram for each of the Series

	Parameters:
	-----------
		df : DataFrame
			Pandas DataFrame
		theme : string
			Theme to be used (if not the default)
		bins : int
			Number of bins to use for histogram
		color : string
			Color to be used for each scatter plot
		size : int
			Size for each marker on the scatter plot
	"""
	if not theme:
		theme = auth.get_config_file()['theme']

	figs=[]
	for i in df.columns:
		for j in df.columns:
			if i==j:
				fig=df.iplot(kind='histogram',keys=[i],asFigure=True,bins=bins)
				figs.append(fig)
			else:
				figs.append(df.iplot(kind='scatter',mode='markers',x=j,y=i,asFigure=True,size=size,colors=[color]))
	layout=getLayout(theme)
	layout['xaxis'].update(showgrid=False)
	layout['yaxis'].update(showgrid=False)
	sm=subplots(figs,shape=(len(df.columns),len(df.columns)),shared_xaxes=False,shared_yaxes=False,
					  horizontal_spacing=.05,vertical_spacing=.07,base_layout=layout)
	sm['layout'].update(bargap=.02,showlegend=False)
	return sm


Instruction:
Write a Python 3 function that can
set the axis of the plot

Response:
def _set_axis(self,traces,on=None,side='right',title=''):
	"""
	Sets the axis in which each trace should appear
	If the axis doesn't exist then a new axis is created

	Parameters:
	-----------
		traces : list(str)
			List of trace names
		on : string
			The axis in which the traces should be placed.
			If this is not indicated then a new axis will be
			created
		side : string
			Side where the axis will be placed
				'left'
				'right'
		title : string
			Sets the title of the axis
			Applies only to new axis
	"""
	fig={}
	fig_cpy=fig_to_dict(self).copy()
	fig['data']=fig_cpy['data']
	fig['layout']=fig_cpy['layout']
	fig=Figure(fig)
	traces=make_list(traces)

	def update_data(trace,y):
		anchor=fig.axis['def'][y]['anchor'] if 'anchor' in fig.axis['def'][y] else 'x1'
		idx=fig.trace_dict[trace] if isinstance(trace,str) else trace
		fig['data'][idx]['xaxis']=anchor
		fig['data'][idx]['yaxis']=y

	for trace in traces:
		if on:
			if on not in fig.axis['def']:
				raise Exception('"on" axis does not exists: {0}'.format(on))
			update_data(trace,y=on)
		else:
			curr_x,curr_y=fig.axis['ref'][trace]
			domain='[0.0, 1.0]' if 'domain' not in fig.axis['def'][curr_y] else str(fig.axis['def'][curr_y]['domain'])
			try:
				new_axis=fig.axis['dom']['y'][domain][side]
			except KeyError:
				axis=fig.axis['def'][curr_y].copy()
				### check overlaying values
				axis.update(title=title,overlaying=curr_y,side=side,anchor=curr_x)
				axis_idx=str(fig.axis['len']['y']+1)
				fig['layout']['yaxis{0}'.format(axis_idx)]=axis
				new_axis='y{0}'.format(axis_idx)
			update_data(trace,y=new_axis)

	for k in list(fig.axis['def'].keys()):
		id='{0}axis{1}'.format(k[0],k[-1:])
		if k not in fig.axis['ref_axis']:
			try:
				del fig['layout'][id]
			except KeyError:
				pass

	return fig


Instruction:
Can you tell what is the following Python 3 function doing
def get_shape(kind='line',x=None,y=None,x0=None,y0=None,x1=None,y1=None,span=0,color='red',dash='solid',width=1,
				fillcolor=None,fill=False,opacity=1,xref='x',yref='y'):
	"""
	Returns a plotly shape

	Parameters:
	-----------
		kind : string
			Shape kind
				line
				rect
				circle
		x : float
			x values for the shape.
			This assumes x0=x1
		x0 : float
			x0 value for the shape
		x1 : float
			x1 value for the shape
		y : float
			y values for the shape.
			This assumes y0=y1
		y0 : float
			y0 value for the shape
		y1 : float
			y1 value for the shape
		color : string
			color for shape line
		dash : string
			line style
				solid
				dash
				dashdot
				dot
		width : int
			line width
		fillcolor : string
			shape fill color
		fill : bool
			If True then fill shape
			If not fillcolor then the
			line color will be used
		opacity : float [0,1]
			opacity of the fill
		xref : string
			Sets the x coordinate system
			which this object refers to
				'x'
				'paper'
				'x2' etc
		yref : string
			Sets the y coordinate system
			which this object refers to
				'y'
				'paper'
				'y2' etc
	"""
	if x1 is None:
		if x0 is None:
			if x is None:
				xref='paper'
				x0=0
				x1=1
			else:
				x0=x1=x
		else:
			x1=x0
	else:
		x
	if y1 is None:
		if y0 is None:
			if y is None:
				yref='paper'
				y0=0
				y1=1
			else:
				y0=y1=y
		else:
			y1=y0

	shape = {	'x0':x0,
				'y0':y0,
				'x1':x1,
				'y1':y1,
				'line' : {
					'color':normalize(color),
					'width':width,
					'dash':dash
					},
				'xref':xref,
				'yref':yref
				}

	if kind=='line':
		shape['type']='line'

	elif kind=='circle':
		shape['type']='circle'

	elif kind=='rect':
		shape['type']='rect'
	else:
		raise Exception("Invalid or unkown shape type : {0}".format(kind))

	if (fill or fillcolor) and kind!='line':
		fillcolor = color if not fillcolor else fillcolor
		fillcolor=to_rgba(normalize(fillcolor),opacity)
		shape['fillcolor']=fillcolor

	return shape

Response: Returns a plotly shape for the specified shape.


Instruction:
Given the following Python 3 function, write the documentation
def get_range_selector(steps=['1m','1y'],bgcolor='rgba(150, 200, 250, 0.4)',x=0,y=0.9,
						visible=True,**kwargs):
	"""
	Returns a range selector
	Reference: https://plot.ly/python/reference/#layout-xaxis-rangeselector

	Parameters:
	-----------
		steps : string or list(string)
			Steps for the range
				Examples:
					['1y','2 months','5 weeks','ytd','2mtd']
		bgocolor : string or tuple(color,alpha)
			Background color
				Examples:
					'cyan'
					'rgba(232,214,10,.5)'
					('blue',.3)
		font_size : int
			Font size
		x : float
			Position along the x axis
			Domain (0,1)
		y : float
			Position along the y axis
			Domain (0,1)
	"""
	import string

	def get_step(s):

		term=[]
		stepmode='backward'
		_s=s
		_s=_s.replace(' ','')
		_s=_s.lower()
		if _s in ['reset','all']:
			return {'count':1,'label':s,'step':'all'}
		if _s[-2:]=='td':
			_s=_s[:-2]
			stepmode='todate'
			if _s[0] not in string.digits:
				_s='1'+_s
		if _s[0] not in string.digits:
			raise Exception('Invalid step format: {0}'.format(s))
		while _s[-1] not in string.digits:
			term.append(_s[-1])
			_s=_s[:-1]
		term.reverse()
		term=''.join(term)
		cnt=int(_s)
		term=term[:-1] if (term[-1]=='s' and len(term)>1) else term
		if term in ['y','year','yr']:
					steps='year'
		elif term in ['w','week','wk']:
			steps='week'
		elif term in ['m','month','mth','mnth','mo']:
			steps='month'
		elif term in ['hr','hour']:
			steps='hour'
		elif term in ['min','minute','mn']:
			steps='minute'
		elif term in ['sec','sc','s']:
			steps='second'
		else:
			raise Exception('Invalid step format: {0}'.format(s))
		return {'count':cnt,'label':s,'step':steps,'stepmode':stepmode}

	rangeselector={
		'bgcolor':to_rgba(bgcolor,1),
		'x':x,
		'y':y,
		'visible':visible
	}

	kwargs['fontsize']=kwargs.get('fontsize',13)

	rangeselector=dict_replace_keyword(rangeselector,'font',kwargs,False)

	buttons=[]
	if type(steps) not in (list,tuple):
		steps=[steps]
	for s in steps:
		buttons.append(get_step(s))
	rangeselector['buttons']=buttons
	return rangeselector

Response: Returns a range selector for the current plot.


Instruction:
Can you generate the documentation for the following Python 3 function
def view_label_matrix(L, colorbar=True):
    L = L.todense() if sparse.issparse(L) else L
    plt.imshow(L, aspect="auto")
    plt.title("Label Matrix")
    if colorbar:
        labels = sorted(np.unique(np.asarray(L).reshape(-1, 1).squeeze()))
        boundaries = np.array(labels + [max(labels) + 1]) - 0.5
        plt.colorbar(boundaries=boundaries, ticks=labels)
    plt.show()

Response: Display an n m matrix of labels


Instruction:
Make a summary of the following Python 3 code
def view_overlaps(L, self_overlaps=False, normalize=True, colorbar=True):
    L = L.todense() if sparse.issparse(L) else L
    G = _get_overlaps_matrix(L, normalize=normalize)
    if not self_overlaps:
        np.fill_diagonal(G, 0)  # Zero out self-overlaps
    plt.imshow(G, aspect="auto")
    plt.title("Overlaps")
    if colorbar:
        plt.colorbar()
    plt.show()

Response: Display an matrix of overlaps


Instruction:
Can you tell what is the following Python 3 function doing
def view_conflicts(L, normalize=True, colorbar=True):
    L = L.todense() if sparse.issparse(L) else L
    C = _get_conflicts_matrix(L, normalize=normalize)
    plt.imshow(C, aspect="auto")
    plt.title("Conflicts")
    if colorbar:
        plt.colorbar()
    plt.show()

Response: Display an matrix of conflicts


Instruction:
Write a Python 3 function that can
plot a histogram from a numpy array of probabilities

Response:
def plot_probabilities_histogram(Y_p, title=None):
    """Plot a histogram from a numpy array of probabilities

    Args:
        Y_p: An [n] or [n, 1] np.ndarray of probabilities (floats in [0,1])
    """
    if Y_p.ndim > 1:
        msg = (
            f"Arg Y_p should be a 1-dimensional np.ndarray, not of shape "
            f"{Y_p.shape}."
        )
        raise ValueError(msg)
    plt.hist(Y_p, bins=20)
    plt.xlim((0, 1.025))
    plt.xlabel("Probability")
    plt.ylabel("# Predictions")
    if isinstance(title, str):
        plt.title(title)
    plt.show()


Instruction:
Make a summary of the following Python 3 code
def plot_predictions_histogram(Y_ph, Y, title=None):
    labels = list(set(Y).union(set(Y_ph)))
    edges = [x - 0.5 for x in range(min(labels), max(labels) + 2)]

    plt.hist([Y_ph, Y], bins=edges, label=["Predicted", "Gold"])
    ax = plt.gca()
    ax.set_xticks(labels)
    plt.xlabel("Label")
    plt.ylabel("# Predictions")
    plt.legend(loc="upper right")
    if isinstance(title, str):
        plt.title(title)
    plt.show()

Response: Plot a histogram comparing int predictions vs true labels by class


Instruction:
Write a Python 3 function that can
return True if the logging frequency has been met.

Response:
def check(self, batch_size):
        """Returns True if the logging frequency has been met."""
        self.increment(batch_size)
        return self.unit_count >= self.config["log_train_every"]


Instruction:
Explain what the following Python 3 code does
def increment(self, batch_size):
        self.example_count += batch_size
        self.example_total += batch_size
        if self.log_unit == "seconds":
            self.unit_count = int(self.timer.elapsed())
            self.unit_total = int(self.timer.total_elapsed())
        elif self.log_unit == "examples":
            self.unit_count = self.example_count
            self.unit_total = self.example_total
        elif self.log_unit == "batches":
            self.unit_count += 1
            self.unit_total += 1
        elif self.log_unit == "epochs":
            # Track epoch by example count because otherwise we only know when
            # a new epoch starts, not when an epoch ends
            if self.example_count >= self.epoch_size:
                self.unit_count += 1
                self.unit_total += 1
        else:
            raise Exception(f"Unrecognized log_unit: {self.log_unit}")

Response: Update the total and relative unit counts based on the log_unit.


Instruction:
Here you have a function in Python 3, explain what it does
def calculate_metrics(self, model, train_loader, valid_loader, metrics_dict):
        # Check whether or not it's time for validation as well
        self.log_count += 1
        log_valid = (
            valid_loader is not None
            and self.valid_every_X
            and not (self.log_count % self.valid_every_X)
        )

        metrics_dict = {}

        # Calculate custom metrics
        if self.config["log_train_metrics_func"] is not None:
            func = self.config["log_train_metrics_func"]
            func_list = func if isinstance(func, list) else [func]
            for func in func_list:
                metrics_dict = self._calculate_custom_metrics(
                    model, train_loader, func, metrics_dict, split="train"
                )
        if self.config["log_valid_metrics_func"] is not None and log_valid:
            func = self.config["log_valid_metrics_func"]
            func_list = func if isinstance(func, list) else [func]
            for func in func_list:
                metrics_dict = self._calculate_custom_metrics(
                    model, valid_loader, func, metrics_dict, split="valid"
                )

        # Calculate standard metrics
        metrics_dict = self._calculate_standard_metrics(
            model, train_loader, self.log_train_metrics, metrics_dict, "train"
        )

        if log_valid:
            metrics_dict = self._calculate_standard_metrics(
                model, valid_loader, self.log_valid_metrics, metrics_dict, "valid"
            )

        return metrics_dict

Response: Calculate metrics for the current log count and train_loader.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def log(self, metrics_dict):
        if self.writer:
            self.write_to_file(metrics_dict)

        if self.verbose:
            self.print_to_screen(metrics_dict)
        self.reset()

Response: Print calculated metrics and optionally write to file and print to screen.


Instruction:
Can you generate the documentation for the following Python 3 function
def print_to_screen(self, metrics_dict):
        score_strings = defaultdict(list)
        for split_metric, value in metrics_dict.items():
            split, metric = split_metric.split("/", 1)

            if isinstance(value, float):
                score_strings[split].append(f"{metric}={value:0.3f}")
            else:
                score_strings[split].append(f"{metric}={value}")

        header = f"{self.unit_total} {self.log_unit[:3]}"
        if self.log_unit != "epochs":
            epochs = self.example_total / self.epoch_size
            header += f" ({epochs:0.2f} epo)"
        string = f"[{header}]:"

        if score_strings["train"]:
            train_scores = f"{', '.join(score_strings['train'])}"
            string += f" TRAIN:[{train_scores}]"
        if score_strings["valid"]:
            valid_scores = f"{', '.join(score_strings['valid'])}"
            string += f" VALID:[{valid_scores}]"
        print(string)

Response: Print all metrics in metrics_dict to screen


Instruction:
Can you generate the documentation for the following Python 3 function
def _reduce_output(self, outputs, seq_lengths):
        batch_size = outputs.shape[0]
        reduced = []
        # Necessary to iterate over batch because of different sequence lengths
        for i in range(batch_size):
            if self.lstm_reduction == "mean":
                # Average over all non-padding reduced
                # Use dim=0 because first dimension disappears after indexing
                reduced.append(outputs[i, : seq_lengths[i], :].mean(dim=0))
            elif self.lstm_reduction == "max":
                # Max-pool over all non-padding reduced
                # Use dim=0 because first dimension disappears after indexing
                reduced.append(outputs[i, : seq_lengths[i], :].max(dim=0)[0])
            elif self.lstm_reduction == "last":
                # Take the last output of the sequence (before padding starts)
                # NOTE: maybe better to take first and last?
                reduced.append(outputs[i, seq_lengths[i] - 1, :])
            elif self.lstm_reduction == "attention":
                reduced.append(self._attention(outputs[i, : seq_lengths[i], :]))
            else:
                msg = (
                    f"Did not recognize lstm kwarg 'lstm_reduction' == "
                    f"{self.lstm_reduction}"
                )
                raise ValueError(msg)
        return torch.stack(reduced, dim=0)

Response: Reduces the output of an LSTM step into the hidden state outputs.


Instruction:
Create a Python 3 function for
applying one step of an lstm to the input X which is handled by self. encoder.

Response:
def forward(self, X):
        """Applies one step of an lstm (plus reduction) to the input X, which
        is handled by self.encoder"""
        # Identify the first non-zero integer from the right (i.e., the length
        # of the sequence before padding starts).
        batch_size, max_seq = X.shape[0], X.shape[1]
        seq_lengths = torch.zeros(batch_size, dtype=torch.long)
        for i in range(batch_size):
            for j in range(max_seq - 1, -1, -1):
                if not torch.all(X[i, j] == 0):
                    seq_lengths[i] = j + 1
                    break

        # Sort by length because pack_padded_sequence requires it
        # Save original order to restore before returning
        seq_lengths, perm_idx = seq_lengths.sort(0, descending=True)
        X = X[perm_idx]
        inv_perm_idx = torch.tensor(
            [i for i, _ in sorted(enumerate(perm_idx), key=lambda idx: idx[1])],
            dtype=torch.long,
        )

        # Encode and pack input sequence
        X_packed = rnn_utils.pack_padded_sequence(
            self.encoder.encode(X), seq_lengths, batch_first=True
        )

        # Run LSTM
        outputs, (h_t, c_t) = self.lstm(X_packed)

        # Unpack and reduce outputs
        outputs_unpacked, _ = rnn_utils.pad_packed_sequence(outputs, batch_first=True)
        reduced = self._reduce_output(outputs_unpacked, seq_lengths)
        return reduced[inv_perm_idx, :]


Instruction:
How would you explain what the following Python 3 function does
def choose_other_label(k, y):
    return choice(list(set(range(1, k + 1)) - set([y])))

Response:
Given a cardinality k and true label y return random value in
    { 1... k } \ y.


Instruction:
Can you generate the documentation for the following Python 3 function
def gaussian_bags_of_words(Y, vocab=vocab1k, sigma=1, bag_size=[25, 50], **kwargs):

    def make_distribution(sigma, num_words):
        p = abs(np.random.normal(0, sigma, num_words))
        return p / sum(p)

    num_words = len(vocab)
    word_dists = {y: make_distribution(sigma, num_words) for y in set(Y)}
    bag_sizes = np.random.choice(range(min(bag_size), max(bag_size)), len(Y))

    X = []
    items = []
    for i, (y, length) in enumerate(zip(Y, bag_sizes)):
        x = torch.from_numpy(np.random.choice(num_words, length, p=word_dists[y]))
        X.append(x)
        items.append(" ".join(vocab[j] for j in x))

    return X, items

Response: Generates a list of bags of words based on label assignments Y.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _generate_edges(self, edge_prob):
        self.E, self.parent = [], {}
        for i in range(self.m):
            if random() < edge_prob and i > 0:
                p_i = choice(i)
                self.E.append((p_i, i))
                self.parent[i] = p_i

Response: Generate a random tree - structured dependency graph based on a specific edge probability.


Instruction:
Can you implement a function in Python 3 that
computes the conditional probability that LF i outputs li and that LF j output lj and Y = y.

Response:
def P_conditional(self, i, li, j, lj, y):
        """Compute the conditional probability
            P_\theta(li | lj, y)
            =
            Z^{-1} exp(
                theta_{i|y} \indpm{ \lambda_i = Y }
                + \theta_{i,j} \indpm{ \lambda_i = \lambda_j }
            )
        In other words, compute the conditional probability that LF i outputs
        li given that LF j output lj, and Y = y, parameterized by
            - a class-conditional LF accuracy parameter \theta_{i|y}
            - a symmetric LF correlation paramter \theta_{i,j}
        """
        Z = np.sum([self._P(i, _li, j, lj, y) for _li in range(self.k + 1)])
        return self._P(i, li, j, lj, y) / Z


Instruction:
Write a Python 3 function for
generating an n m label matrix with entries in 0... k

Response:
def _generate_label_matrix(self):
        """Generate an [n,m] label matrix with entries in {0,...,k}"""
        self.L = np.zeros((self.n, self.m))
        self.Y = np.zeros(self.n, dtype=np.int64)
        for i in range(self.n):
            y = choice(self.k, p=self.p) + 1  # Note that y \in {1,...,k}
            self.Y[i] = y
            for j in range(self.m):
                p_j = self.parent.get(j, 0)
                prob_y = self.P_conditional(j, y, p_j, self.L[i, p_j], y)
                prob_0 = self.P_conditional(j, 0, p_j, self.L[i, p_j], y)
                p = np.ones(self.k + 1) * (1 - prob_y - prob_0) / (self.k - 1)
                p[0] = prob_0
                p[y] = prob_y
                self.L[i, j] = choice(self.k + 1, p=p)


Instruction:
How would you implement a function in Python 3 that
computes the true clique conditional probabilities P \ lC | Y by counting given L Y and then computing the ground truth for that cluster.

Response:
def _get_conditional_probs(self):
        """Compute the true clique conditional probabilities P(\lC | Y) by
        counting given L, Y; we'll use this as ground truth to compare to.

        Note that this generates an attribute, self.c_probs, that has the same
        definition as returned by `LabelModel.get_conditional_probs`.

        TODO: Can compute these exactly if we want to implement that.
        """
        # TODO: Extend to higher-order cliques again
        self.c_probs = np.zeros((self.m * (self.k + 1), self.k))
        for y in range(1, self.k + 1):
            Ly = self.L[self.Y == y]
            for ly in range(self.k + 1):
                self.c_probs[ly :: (self.k + 1), y - 1] = (
                    np.where(Ly == ly, 1, 0).sum(axis=0) / Ly.shape[0]
                )


Instruction:
Can you generate a brief explanation for the following Python 3 code
def fit_transform(self, input, **fit_kwargs):
        self.fit(input, **fit_kwargs)
        X = self.transform(input)
        return X

Response: Execute fit and transform in sequence.


Instruction:
Can you tell what is the following Python 3 function doing
def fit(self, sents, **kwargs):
        tokens = list(itertools.chain.from_iterable(sents))
        counter = Counter(tokens)
        self.vocab = self.build_vocab(counter, **kwargs)

Response: Builds a vocabulary object based on the tokens in the input.


Instruction:
Write a Python 3 function for
converting lists of tokens into a Tensor of embedding indices.

Response:
def transform(self, sents):
        """Converts lists of tokens into a Tensor of embedding indices.

        Args:
            sents: A list of lists of tokens (representing sentences)
                NOTE: These sentences should already be marked using the
                mark_entities() helper.
        Returns:
            X: A Tensor of shape (num_items, max_seq_len)
        """

        def convert(tokens):
            return torch.tensor([self.vocab.stoi[t] for t in tokens], dtype=torch.long)

        if self.vocab is None:
            raise Exception(
                "Must run .fit() for .fit_transform() before " "calling .transform()."
            )

        seqs = sorted([convert(s) for s in sents], key=lambda x: -len(x))
        X = torch.LongTensor(pad_sequence(seqs, batch_first=True))
        return X


Instruction:
Can you generate a brief explanation for the following Python 3 code
def pred_to_prob(Y_h, k):
    Y_h = Y_h.clone()
    if Y_h.dim() > 1:
        Y_h = Y_h.squeeze()
    assert Y_h.dim() == 1
    assert (Y_h >= 1).all()
    assert (Y_h <= k).all()
    n = Y_h.shape[0]
    Y_s = torch.zeros((n, k), dtype=Y_h.dtype, device=Y_h.device)
    for i, j in enumerate(Y_h):
        Y_s[i, j - 1] = 1.0
    return Y_s

Response: Converts a 1D tensor of predicted labels into a 2D tensor of probabilistic labels for item i and label k


Instruction:
Explain what the following Python 3 code does
def arraylike_to_numpy(array_like):

    orig_type = type(array_like)

    # Convert to np.ndarray
    if isinstance(array_like, np.ndarray):
        pass
    elif isinstance(array_like, list):
        array_like = np.array(array_like)
    elif issparse(array_like):
        array_like = array_like.toarray()
    elif isinstance(array_like, torch.Tensor):
        array_like = array_like.numpy()
    elif not isinstance(array_like, np.ndarray):
        array_like = np.array(array_like)
    else:
        msg = f"Input of type {orig_type} could not be converted to 1d " "np.ndarray"
        raise ValueError(msg)

    # Correct shape
    if (array_like.ndim > 1) and (1 in array_like.shape):
        array_like = array_like.flatten()
    if array_like.ndim != 1:
        raise ValueError("Input could not be converted to 1d np.array")

    # Convert to ints
    if any(array_like % 1):
        raise ValueError("Input contains at least one non-integer value.")
    array_like = array_like.astype(np.dtype(int))

    return array_like

Response: Convert a 1d array - like ( e. g. list tensor etc. to a np. ndarray


Instruction:
Can you implement a function in Python 3 that
converts a matrix from one label type to another.

Response:
def convert_labels(Y, source, dest):
    """Convert a matrix from one label type to another

    Args:
        Y: A np.ndarray or torch.Tensor of labels (ints)
        source: The convention the labels are currently expressed in
        dest: The convention to convert the labels to

    Conventions:
        'categorical': [0: abstain, 1: positive, 2: negative]
        'plusminus': [0: abstain, 1: positive, -1: negative]
        'onezero': [0: negative, 1: positive]

    Note that converting to 'onezero' will combine abstain and negative labels.
    """
    if Y is None:
        return Y
    if isinstance(Y, np.ndarray):
        Y = Y.copy()
        assert isinstance(Y, int)
    elif isinstance(Y, torch.Tensor):
        Y = Y.clone()
        assert np.sum(Y.numpy() - Y.numpy().astype(int)) == 0.0
    else:
        raise ValueError("Unrecognized label data type.")
    negative_map = {"categorical": 2, "plusminus": -1, "onezero": 0}
    Y[Y == negative_map[source]] = negative_map[dest]
    return Y


Instruction:
Create a Python 3 function to
convert a 2D label matrix L into a 3D tensor where each element in L is a one hot class.

Response:
def label_matrix_to_one_hot(L, k=None):
    """Converts a 2D [n,m] label matrix into an [n,m,k] one hot 3D tensor

    Note that in the returned 3D matrix, abstain votes continue to be
    represented by 0s, not 1s.

    Args:
        L: a [n,m] label matrix with categorical labels (0 = abstain)
        k: the number of classes that could appear in L
            if None, k is inferred as the max element in L
    """
    n, m = L.shape
    if k is None:
        k = L.max()
    L_onehot = torch.zeros(n, m, k + 1)
    for i, row in enumerate(L):
        for j, k in enumerate(row):
            if k > 0:
                L_onehot[i, j, k - 1] = 1
    return L_onehot


Instruction:
How would you implement a function in Python 3 that
merges dictionary y into a copy of x, overwriting elements of x when there is a conflict, except if the element is a dictionary, in which case recurse. misses: what to do if a key in y is not in x 'insert' -> set x[key] = value 'exception' -> raise an exception 'report' -> report the name of the missing key 'ignore' -> do nothing TODO: give example here (pull from tests)

Response:
def recursive_merge_dicts(x, y, misses="report", verbose=None):
    """
    Merge dictionary y into a copy of x, overwriting elements of x when there
    is a conflict, except if the element is a dictionary, in which case recurse.

    misses: what to do if a key in y is not in x
        'insert'    -> set x[key] = value
        'exception' -> raise an exception
        'report'    -> report the name of the missing key
        'ignore'    -> do nothing

    TODO: give example here (pull from tests)
    """

    def recurse(x, y, misses="report", verbose=1):
        found = True
        for k, v in y.items():
            found = False
            if k in x:
                found = True
                if isinstance(x[k], dict):
                    if not isinstance(v, dict):
                        msg = f"Attempted to overwrite dict {k} with " f"non-dict: {v}"
                        raise ValueError(msg)
                    recurse(x[k], v, misses, verbose)
                else:
                    if x[k] == v:
                        msg = f"Reaffirming {k}={x[k]}"
                    else:
                        msg = f"Overwriting {k}={x[k]} to {k}={v}"
                        x[k] = v
                    if verbose > 1 and k != "verbose":
                        print(msg)
            else:
                for kx, vx in x.items():
                    if isinstance(vx, dict):
                        found = recurse(vx, {k: v}, misses="ignore", verbose=verbose)
                    if found:
                        break
            if not found:
                msg = f'Could not find kwarg "{k}" in destination dict.'
                if misses == "insert":
                    x[k] = v
                    if verbose > 1:
                        print(f"Added {k}={v} from second dict to first")
                elif misses == "exception":
                    raise ValueError(msg)
                elif misses == "report":
                    print(msg)
                else:
                    pass
        return found

    # If verbose is not provided, look for an value in y first, then x
    # (Do this because 'verbose' kwarg is often inside one or both of x and y)
    if verbose is None:
        verbose = y.get("verbose", x.get("verbose", 1))

    z = copy.deepcopy(x)
    recurse(z, y, misses, verbose)
    return z


Instruction:
Can you generate a brief explanation for the following Python 3 code
def split_data(
    *inputs,
    splits=[0.5, 0.5],
    shuffle=True,
    stratify_by=None,
    index_only=False,
    seed=None,
):
    """Splits inputs into multiple splits of defined sizes

    Args:
        inputs: correlated tuples/lists/arrays/matrices/tensors to split
        splits: list containing split sizes (fractions or counts);
        shuffle: if True, shuffle the data before splitting
        stratify_by: (None or an input) if not None, use these labels to
            stratify the splits (separating the data into groups by these
            labels and sampling from those, rather than from the population at
            large); overrides shuffle
        index_only: if True, return only the indices of the new splits, not the
            split data itself
        seed: (int) random seed

    Example usage:
        Ls, Xs, Ys = split_data(L, X, Y, splits=[0.8, 0.1, 0.1])
        OR
        assignments = split_data(Y, splits=[0.8, 0.1, 0.1], index_only=True)

    Note: This is very similar to scikit-learn's train_test_split() method,
        but with support for more than two splits.
    """

    def fractions_to_counts(fracs, n):
        """Converts a list of fractions to a list of counts that sum to n"""
        counts = [int(np.round(n * frac)) for frac in fracs]
        # Ensure sum of split counts sums to n
        counts[-1] = n - sum(counts[:-1])
        return counts

    def slice_data(data, indices):
        if isinstance(data, list) or isinstance(data, tuple):
            return [d for i, d in enumerate(data) if i in set(indices)]
        else:
            try:
                # Works for np.ndarray, scipy.sparse, torch.Tensor
                return data[indices]
            except TypeError:
                raise Exception(
                    f"split_data() currently only accepts inputs "
                    f"of type tuple, list, np.ndarray, scipy.sparse, or "
                    f"torch.Tensor; not {type(data)}"
                )

    # Setting random seed
    if seed is not None:
        random.seed(seed)

    try:
        n = len(inputs[0])
    except TypeError:
        n = inputs[0].shape[0]
    num_splits = len(splits)

    # Check splits for validity and convert to fractions
    if all(isinstance(x, int) for x in splits):
        if not sum(splits) == n:
            raise ValueError(
                f"Provided split counts must sum to n ({n}), not {sum(splits)}."
            )
        fracs = [count / n for count in splits]

    elif all(isinstance(x, float) for x in splits):
        if not sum(splits) == 1.0:
            raise ValueError(f"Split fractions must sum to 1.0, not {sum(splits)}.")
        fracs = splits

    else:
        raise ValueError("Splits must contain all ints or all floats.")

    # Make sampling pools
    if stratify_by is None:
        pools = [np.arange(n)]
    else:
        pools = defaultdict(list)
        for i, val in enumerate(stratify_by):
            pools[val].append(i)
        pools = list(pools.values())

    # Make index assignments
    assignments = [[] for _ in range(num_splits)]
    for pool in pools:
        if shuffle or stratify_by is not None:
            random.shuffle(pool)

        counts = fractions_to_counts(fracs, len(pool))
        counts.insert(0, 0)
        cum_counts = np.cumsum(counts)
        for i in range(num_splits):
            assignments[i].extend(pool[cum_counts[i] : cum_counts[i + 1]])

    if index_only:
        return assignments
    else:
        outputs = []
        for data in inputs:
            data_splits = []
            for split in range(num_splits):
                data_splits.append(slice_data(data, assignments[split]))
            outputs.append(data_splits)

        if len(outputs) == 1:
            return outputs[0]
        else:
            return outputs

Response: Splits the data into multiple sets of items for the given size.


Instruction:
Write a Python 3 function for
creating and attaches task_heads to the appropriate network layers.

Response:
def _build_task_heads(self, head_modules):
        """Creates and attaches task_heads to the appropriate network layers"""
        # Make task head layer assignments
        num_layers = len(self.config["layer_out_dims"])
        task_head_layers = self._set_task_head_layers(num_layers)

        # task_head_layers stores the layer whose output is input to task head t
        # task_map stores the task heads that appear at each layer
        self.task_map = defaultdict(list)
        for t, l in enumerate(task_head_layers):
            self.task_map[l].append(t)

        if any(l == 0 for l in task_head_layers) and head_modules is None:
            raise Exception(
                "If any task head is being attached to layer 0 "
                "(the input modules), then you must provide a t-length list of "
                "head_modules, since the output dimension of each input_module "
                "cannot be inferred."
            )

        # Construct heads
        head_dims = [self.K[t] for t in range(self.t)]

        heads = nn.ModuleList()
        for t in range(self.t):
            input_dim = self.config["layer_out_dims"][task_head_layers[t]]
            if self.config["pass_predictions"]:
                for p in self.task_graph.parents[t]:
                    input_dim += head_dims[p]
            output_dim = head_dims[t]

            if head_modules is None:
                head = nn.Linear(input_dim, output_dim)
            elif isinstance(head_modules, list):
                head = head_modules[t]
            else:
                head = copy.deepcopy(head_modules)
            heads.append(head)
        return heads


Instruction:
Can you generate the documentation for the following Python 3 function
def forward(self, x):
        head_outputs = [None] * self.t

        # Execute input layer
        if isinstance(self.input_layer, list):  # One input_module per task
            input_outputs = [mod(x) for mod, x in zip(self.input_layer, x)]
            x = torch.stack(input_outputs, dim=1)

            # Execute level-0 task heads from their respective input modules
            for t in self.task_map[0]:
                head = self.heads[t]
                head_outputs[t] = head(input_outputs[t])
        else:  # One input_module for all tasks
            x = self.input_layer(x)

            # Execute level-0 task heads from the single input module
            for t in self.task_map[0]:
                head = self.heads[t]
                head_outputs[t] = head(x)

        # Execute middle layers
        for i, layer in enumerate(self.middle_layers, start=1):
            x = layer(x)

            # Attach level-i task heads from the ith middle module
            for t in self.task_map[i]:
                head = self.heads[t]
                # Optionally include as input the predictions of parent tasks
                if self.config["pass_predictions"] and bool(self.task_graph.parents[t]):
                    task_input = [x]
                    for p in self.task_graph.parents[t]:
                        task_input.append(head_outputs[p])
                    task_input = torch.stack(task_input, dim=1)
                else:
                    task_input = x
                head_outputs[t] = head(task_input)
        return head_outputs

Response: Returns a list of outputs for tasks 0... t - 1.


Instruction:
Can you write a function in Python 3 where it
converts Y to t - length list of probabilistic labels if necessary

Response:
def _preprocess_Y(self, Y, k=None):
        """Convert Y to t-length list of probabilistic labels if necessary"""
        # If not a list, convert to a singleton list
        if not isinstance(Y, list):
            if self.t != 1:
                msg = "For t > 1, Y must be a list of n-dim or [n, K_t] tensors"
                raise ValueError(msg)
            Y = [Y]

        if not len(Y) == self.t:
            msg = f"Expected Y to be a t-length list (t={self.t}), not {len(Y)}"
            raise ValueError(msg)

        return [EndModel._preprocess_Y(self, Y_t, self.K[t]) for t, Y_t in enumerate(Y)]


Instruction:
Write a Python 3 script for
returning the loss function to use in the train_model routine

Response:
def _get_loss_fn(self):
        """Returns the loss function to use in the train_model routine"""
        criteria = self.criteria.to(self.config["device"])
        loss_fn = lambda X, Y: sum(
            criteria(Y_tp, Y_t) for Y_tp, Y_t in zip(self.forward(X), Y)
        )
        return loss_fn


Instruction:
Make a summary of the following Python 3 code
def _create_L_ind(self, L):
        # TODO: Update LabelModel to keep L, L_ind, L_aug as sparse matrices
        # throughout and remove this line.
        if issparse(L[0]):
            L = [L_t.todense() for L_t in L]

        # Make sure converted to numpy here
        L = self._to_numpy(L)

        L_ind = np.ones((self.n, self.m * self.k))
        for yi, y in enumerate(self.task_graph.feasible_set()):
            for t in range(self.t):
                # A[x::y] slices A starting at x at intervals of y
                # e.g., np.arange(9)[0::3] == np.array([0,3,6])
                L_ind[:, yi :: self.k] *= np.where(
                    np.logical_or(L[t] == y[t], L[t] == 0), 1, 0
                )

            # Set LFs that abstained on all feasible label vectors to all 0s
            L_ind[:, yi :: self.k] *= np.where(sum(L) != 0, 1, 0)

        return L_ind

Response: Convert T label matrices with labels in 0... K_t to a one - hot format.


Instruction:
Write a Python 3 script to
return the probability distribution of the cluster class entries in the task graph.

Response:
def predict_proba(self, L):
        """Returns the task marginals estimated by the model: a t-length list of
        [n,k_t] matrices where the (i,j) entry of the sth matrix represents the
        estimated P((Y_i)_s | \lambda_j(x_i))

        Args:
            L: A t-length list of [n,m] scipy.sparse label matrices with values
                in {0,1,...,k}
        """
        # First, get the estimated probability distribution over the feasible
        # set defined by the TaskGraph
        # This is an [n,k] array, where k = |(feasible set)|
        Y_pf = LabelModel.predict_proba(self, L)
        n, k = Y_pf.shape

        # Now get the per-task marginals
        # TODO: Make this optional, versus just returning the above
        Y_p = [np.zeros((n, k_t)) for k_t in self.task_graph.K]
        for yi, y in enumerate(self.task_graph.feasible_set()):
            for t in range(self.t):
                k_t = int(y[t])
                Y_p[t][:, k_t - 1] += Y_pf[:, yi]
        return Y_p


Instruction:
Create a Python 3 function for
predicting labels for an input X on all tasks.

Response:
def predict(self, X, break_ties="random", return_probs=False, **kwargs):
        """Predicts (int) labels for an input X on all tasks

        Args:
            X: The input for the predict_proba method
            break_ties: A tie-breaking policy (see Classifier._break_ties())
            return_probs: Return the predicted probabilities as well

        Returns:
            Y_p: An n-dim np.ndarray of predictions in {1,...k}
            [Optionally: Y_s: An [n, k] np.ndarray of predicted probabilities]
        """
        Y_s = self._to_numpy(self.predict_proba(X, **kwargs))
        Y_p = self._break_ties(Y_s, break_ties).astype(np.int)
        if return_probs:
            return Y_p, Y_s
        else:
            return Y_p


Instruction:
Create a Python 3 function for
scoring the predictive performance of the Classifier on all tasks in the cluster.

Response:
def score(
        self,
        data,
        metric="accuracy",
        break_ties="random",
        verbose=True,
        print_confusion_matrix=True,
        **kwargs,
    ):
        """Scores the predictive performance of the Classifier on all tasks

        Args:
            data: a Pytorch DataLoader, Dataset, or tuple with Tensors (X,Y):
                X: The input for the predict method
                Y: An [n] or [n, 1] torch.Tensor or np.ndarray of target labels
                    in {1,...,k}
            metric: A metric (string) with which to score performance or a
                list of such metrics
            break_ties: A tie-breaking policy (see Classifier._break_ties())
            verbose: The verbosity for just this score method; it will not
                update the class config.
            print_confusion_matrix: Print confusion matrix (overwritten to False if
                verbose=False)

        Returns:
            scores: A (float) score or a list of such scores if kwarg metric
                is a list
        """
        Y_p, Y, Y_s = self._get_predictions(
            data, break_ties=break_ties, return_probs=True, **kwargs
        )

        # Evaluate on the specified metrics
        return_list = isinstance(metric, list)
        metric_list = metric if isinstance(metric, list) else [metric]
        scores = []
        for metric in metric_list:
            score = metric_score(Y, Y_p, metric, probs=Y_s, ignore_in_gold=[0])
            scores.append(score)
            if verbose:
                print(f"{metric.capitalize()}: {score:.3f}")

        # Optionally print confusion matrix
        if print_confusion_matrix and verbose:
            confusion_matrix(Y, Y_p, pretty_print=True)

        # If a single metric was given as a string (not list), return a float
        if len(scores) == 1 and not return_list:
            return scores[0]
        else:
            return scores


Instruction:
Can you generate the documentation for the following Python 3 function
def _train_model(
        self, train_data, loss_fn, valid_data=None, log_writer=None, restore_state={}
    ):
        """The internal training routine called by train_model() after setup

        Args:
            train_data: a tuple of Tensors (X,Y), a Dataset, or a DataLoader of
                X (data) and Y (labels) for the train split
            loss_fn: the loss function to minimize (maps *data -> loss)
            valid_data: a tuple of Tensors (X,Y), a Dataset, or a DataLoader of
                X (data) and Y (labels) for the dev split
            restore_state: a dictionary containing model weights (optimizer, main network) and training information

        If valid_data is not provided, then no checkpointing or
        evaluation on the dev set will occur.
        """
        # Set model to train mode
        self.train()
        train_config = self.config["train_config"]

        # Convert data to DataLoaders
        train_loader = self._create_data_loader(train_data)
        valid_loader = self._create_data_loader(valid_data)
        epoch_size = len(train_loader.dataset)

        # Move model to GPU
        if self.config["verbose"] and self.config["device"] != "cpu":
            print("Using GPU...")
        self.to(self.config["device"])

        # Set training components
        self._set_writer(train_config)
        self._set_logger(train_config, epoch_size)
        self._set_checkpointer(train_config)
        self._set_optimizer(train_config)
        self._set_scheduler(train_config)

        # Restore model if necessary
        if restore_state:
            start_iteration = self._restore_training_state(restore_state)
        else:
            start_iteration = 0

        # Train the model
        metrics_hist = {}  # The most recently seen value for all metrics
        for epoch in range(start_iteration, train_config["n_epochs"]):
            progress_bar = (
                train_config["progress_bar"]
                and self.config["verbose"]
                and self.logger.log_unit == "epochs"
            )

            t = tqdm(
                enumerate(train_loader),
                total=len(train_loader),
                disable=(not progress_bar),
            )

            self.running_loss = 0.0
            self.running_examples = 0
            for batch_num, data in t:
                # NOTE: actual batch_size may not equal config's target batch_size
                batch_size = len(data[0])

                # Moving data to device
                if self.config["device"] != "cpu":
                    data = place_on_gpu(data)

                # Zero the parameter gradients
                self.optimizer.zero_grad()

                # Forward pass to calculate the average loss per example
                loss = loss_fn(*data)
                if torch.isnan(loss):
                    msg = "Loss is NaN. Consider reducing learning rate."
                    raise Exception(msg)

                # Backward pass to calculate gradients
                # Loss is an average loss per example
                loss.backward()

                # Perform optimizer step
                self.optimizer.step()

                # Calculate metrics, log, and checkpoint as necessary
                metrics_dict = self._execute_logging(
                    train_loader, valid_loader, loss, batch_size
                )
                metrics_hist.update(metrics_dict)

                # tqdm output
                t.set_postfix(loss=metrics_dict["train/loss"])

            # Apply learning rate scheduler
            self._update_scheduler(epoch, metrics_hist)

        self.eval()

        # Restore best model if applicable
        if self.checkpointer:
            self.checkpointer.load_best_model(model=self)

        # Write log if applicable
        if self.writer:
            if self.writer.include_config:
                self.writer.add_config(self.config)
            self.writer.close()

        # Print confusion matrix if applicable
        if self.config["verbose"]:
            print("Finished Training")
            if valid_loader is not None:
                self.score(
                    valid_loader,
                    metric=train_config["validation_metric"],
                    verbose=True,
                    print_confusion_matrix=True,
                )

Response: This function is called by train_model to train the model and store the training information in the internal object self. train_data and self. loss_fn.


Instruction:
Here you have a function in Python 3, explain what it does
def save(self, destination, **kwargs):
        with open(destination, "wb") as f:
            torch.save(self, f, **kwargs)

Response: Serialize and save a model.


Instruction:
Can you tell what is the following Python 3 function doing
def load(source, **kwargs):
        with open(source, "rb") as f:
            return torch.load(f, **kwargs)

Response: Deserialize and load a model.


Instruction:
How would you implement a function in Python 3 that
resumes training of a classifier by reloading the appropriate state_dicts for each model.

Response:
def resume_training(self, train_data, model_path, valid_data=None):
        """This model resume training of a classifier by reloading the appropriate state_dicts for each model

        Args:
           train_data: a tuple of Tensors (X,Y), a Dataset, or a DataLoader of
                X (data) and Y (labels) for the train split
            model_path: the path to the saved checpoint for resuming training
            valid_data: a tuple of Tensors (X,Y), a Dataset, or a DataLoader of
                X (data) and Y (labels) for the dev split
        """
        restore_state = self.checkpointer.restore(model_path)
        loss_fn = self._get_loss_fn()
        self.train()
        self._train_model(
            train_data=train_data,
            loss_fn=loss_fn,
            valid_data=valid_data,
            restore_state=restore_state,
        )


Instruction:
Write a Python 3 script for
restoring the model and optimizer states This helper function restores the model's state to a given iteration so that a user can resume training at any epoch. Args: restore_state: a state_dict dictionary

Response:
def _restore_training_state(self, restore_state):
        """Restores the model and optimizer states

        This helper function restores the model's state to a given iteration so
        that a user can resume training at any epoch.

        Args:
            restore_state: a state_dict dictionary
        """
        self.load_state_dict(restore_state["model"])
        self.optimizer.load_state_dict(restore_state["optimizer"])
        self.lr_scheduler.load_state_dict(restore_state["lr_scheduler"])
        start_iteration = restore_state["iteration"] + 1
        if self.config["verbose"]:
            print(f"Restored checkpoint to iteration {start_iteration}.")

        if restore_state["best_model_found"]:
            # Update checkpointer with appropriate information about best model
            # Note that the best model found so far may not be the model in the
            # checkpoint that is currently being loaded.
            self.checkpointer.best_model_found = True
            self.checkpointer.best_iteration = restore_state["best_iteration"]
            self.checkpointer.best_score = restore_state["best_score"]
            if self.config["verbose"]:
                print(
                    f"Updated checkpointer: "
                    f"best_score={self.checkpointer.best_score:.3f}, "
                    f"best_iteration={self.checkpointer.best_iteration}"
                )
        return start_iteration


Instruction:
How would you explain what the following Python 3 function does
def _create_dataset(self, *data):
        # Make sure data is a tuple of dense tensors
        data = [self._to_torch(x, dtype=torch.FloatTensor) for x in data]
        return TensorDataset(*data)

Response: Converts input data to the appropriate Dataset


Instruction:
Can you create a Python 3 function that
converts input data into a DataLoader

Response:
def _create_data_loader(self, data, **kwargs):
        """Converts input data into a DataLoader"""
        if data is None:
            return None

        # Set DataLoader config
        # NOTE: Not applicable if data is already a DataLoader
        config = {
            **self.config["train_config"]["data_loader_config"],
            **kwargs,
            "pin_memory": self.config["device"] != "cpu",
        }
        # Return data as DataLoader
        if isinstance(data, DataLoader):
            return data
        elif isinstance(data, Dataset):
            return DataLoader(data, **config)
        elif isinstance(data, (tuple, list)):
            return DataLoader(self._create_dataset(*data), **config)
        else:
            raise ValueError("Input data type not recognized.")


Instruction:
Implement a Python 3 function for
computing predictions in batch given a labeled dataset or tuple with Tensors X Y and Y_s.

Response:
def _get_predictions(self, data, break_ties="random", return_probs=False, **kwargs):
        """Computes predictions in batch, given a labeled dataset

        Args:
            data: a Pytorch DataLoader, Dataset, or tuple with Tensors (X,Y):
                X: The input for the predict method
                Y: An [n] or [n, 1] torch.Tensor or np.ndarray of target labels
                    in {1,...,k}
            break_ties: How to break ties when making predictions
            return_probs: Return the predicted probabilities as well

        Returns:
            Y_p: A Tensor of predictions
            Y: A Tensor of labels
            [Optionally: Y_s: An [n, k] np.ndarray of predicted probabilities]
        """
        data_loader = self._create_data_loader(data)
        Y_p = []
        Y = []
        Y_s = []

        # Do batch evaluation by default, getting the predictions and labels
        for batch_num, data in enumerate(data_loader):
            Xb, Yb = data
            Y.append(self._to_numpy(Yb))

            # Optionally move to device
            if self.config["device"] != "cpu":
                Xb = place_on_gpu(Xb)

            # Append predictions and labels from DataLoader
            Y_pb, Y_sb = self.predict(
                Xb, break_ties=break_ties, return_probs=True, **kwargs
            )
            Y_p.append(self._to_numpy(Y_pb))
            Y_s.append(self._to_numpy(Y_sb))
        Y_p, Y, Y_s = map(self._stack_batches, [Y_p, Y, Y_s])
        if return_probs:
            return Y_p, Y, Y_s
        else:
            return Y_p, Y


Instruction:
Explain what the following Python 3 code does
def _break_ties(self, Y_s, break_ties="random"):
        n, k = Y_s.shape
        Y_h = np.zeros(n)
        diffs = np.abs(Y_s - Y_s.max(axis=1).reshape(-1, 1))

        TOL = 1e-5
        for i in range(n):
            max_idxs = np.where(diffs[i, :] < TOL)[0]
            if len(max_idxs) == 1:
                Y_h[i] = max_idxs[0] + 1
            # Deal with "tie votes" according to the specified policy
            elif break_ties == "random":
                Y_h[i] = np.random.choice(max_idxs) + 1
            elif break_ties == "abstain":
                Y_h[i] = 0
            elif isinstance(break_ties, int):
                Y_h[i] = break_ties
            else:
                ValueError(f"break_ties={break_ties} policy not recognized.")
        return Y_h

Response: Break ties in each row of a tensor according to the specified policy


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _to_numpy(Z):
        if Z is None:
            return Z
        elif issparse(Z):
            return Z.toarray()
        elif isinstance(Z, np.ndarray):
            return Z
        elif isinstance(Z, list):
            return np.array(Z)
        elif isinstance(Z, torch.Tensor):
            return Z.cpu().numpy()
        else:
            msg = (
                f"Expected None, list, numpy.ndarray or torch.Tensor, "
                f"got {type(Z)} instead."
            )
            raise Exception(msg)

Response: Converts a None list np. ndarray or torch. Tensor to np. ndarray ; also handles converting sparse input to dense.


Instruction:
Can you write a function in Python 3 where it
converts a None list np. ndarray or torch. Tensor to torch. Tensor ; also handles converting sparse input to dense.

Response:
def _to_torch(Z, dtype=None):
        """Converts a None, list, np.ndarray, or torch.Tensor to torch.Tensor;
        also handles converting sparse input to dense."""
        if Z is None:
            return None
        elif issparse(Z):
            Z = torch.from_numpy(Z.toarray())
        elif isinstance(Z, torch.Tensor):
            pass
        elif isinstance(Z, list):
            Z = torch.from_numpy(np.array(Z))
        elif isinstance(Z, np.ndarray):
            Z = torch.from_numpy(Z)
        else:
            msg = (
                f"Expected list, numpy.ndarray or torch.Tensor, "
                f"got {type(Z)} instead."
            )
            raise Exception(msg)

        return Z.type(dtype) if dtype else Z


Instruction:
Write a Python 3 function for
printing a warning statement just once.

Response:
def warn_once(self, msg, msg_name=None):
        """Prints a warning statement just once

        Args:
            msg: The warning message
            msg_name: [optional] The name of the warning. If None, the msg_name
                will be the msg itself.
        """
        assert isinstance(msg, str)
        msg_name = msg_name if msg_name else msg
        if msg_name not in warnings_given:
            warnings.warn(msg)
        warnings_given.add(msg_name)


Instruction:
Can you generate the documentation for the following Python 3 function
def _stack_batches(X):
        X = [Classifier._to_numpy(Xb) for Xb in X]
        if len(X[0].shape) == 1:
            return np.hstack(X)
        elif len(X[0].shape) == 2:
            return np.vstack(X)
        else:
            raise ValueError(f"Can't stack {len(X[0].shape)}-dim batches.")

Response:
Stack a list of np. ndarrays along the first axis returning an an
        np. ndarray ; note this is only for smooth hanlding of the multi - task
        setting.


Instruction:
Can you tell what is the following Python 3 function doing
def _preprocess_Y(self, Y, k):
        Y = Y.clone()

        # If preds, convert to probs
        if Y.dim() == 1 or Y.shape[1] == 1:
            Y = pred_to_prob(Y.long(), k=k)
        return Y

Response: Convert Y to prob labels if necessary


Instruction:
Can you create a Python 3 function that
returns a [ n k ) tensor of probs ( probabilistic labels ).

Response:
def predict_proba(self, X):
        """Returns a [n, k] tensor of probs (probabilistic labels)."""
        return F.softmax(self.forward(X), dim=1).data.cpu().numpy()


Instruction:
Create a Python 3 function for
executing sparse linear layer

Response:
def forward(self, X):
        """Execute sparse linear layer

        Args:
            X: an [n, h] torch.LongTensor containing up to h indices of features
                whose weights should be looked up and used in a sparse linear
                multiplication.
        """
        return self.W(X).sum(dim=1) + self.b


Instruction:
Here you have a function in Python 3, explain what it does
def splits(
        cls,
        conn_str,
        candidate_def,
        word_dict=None,
        train=0,
        dev=1,
        test=2,
        use_lfs=(0, 0, 0),
        pretrained_word_dict=None,
        max_seq_len=125,
    ):
        """
        Create train/dev/test splits (mapped to split numbers)

        :param conn_str:
        :param candidate_def:
        :param word_dict:
        :param train:
        :param dev:
        :param test:
        :param use_lfs:
        :param pretrained_word_dict:
        :param max_seq_len:
        :return:

        """
        # initialize word_dict if needed
        train_set = cls(
            conn_str,
            candidate_def,
            word_dict=word_dict,
            split=train,
            use_lfs=use_lfs[train],
            pretrained_word_dict=pretrained_word_dict,
            max_seq_len=max_seq_len,
        )
        return (
            train_set,
            cls(
                conn_str,
                candidate_def,
                word_dict=train_set.word_dict,
                split=dev,
                use_lfs=use_lfs[dev],
                max_seq_len=max_seq_len,
            ),
            cls(
                conn_str,
                candidate_def,
                word_dict=train_set.word_dict,
                split=test,
                use_lfs=use_lfs[test],
                max_seq_len=max_seq_len,
            ),
        )

Response: Create train dev and test sets for a single entry in the cache.


Instruction:
Can you tell what is the following Python 3 function doing
def _mark_entities(self, c, markers):
        sent = c.get_parent().words
        positions = [
            [c[i].get_word_start(), c[i].get_word_end()]
            for i in range(self.cardinality)
        ]
        seq = mark_entities(sent, positions, markers=markers, style="insert")
        return [w for w in seq if w.strip()]

Response: Convert Snorkel candidates to marked up sequences


Instruction:
Given the following Python 3 function, write the documentation
def _include_pretrained_vocab(self, pretrained_word_dict, candidates):
        terms = Counter()
        for c in candidates:
            for w in c.get_parent().words:
                if w in pretrained_word_dict:
                    terms[w] += 1
        list(map(self.word_dict.get, terms))

Response:
Include terms available via pretrained embeddings
COOKIES.


Instruction:
Here you have a function in Python 3, explain what it does
def _build_vocab(self, sentences, markers=[]):
        from snorkel.learning.pytorch.rnn.utils import SymbolTable

        vocab = Counter()
        for sent in sentences:
            for w in sent:
                vocab[w] += 1
        word_dict = SymbolTable()
        list(map(word_dict.get, vocab))
        list(map(word_dict.get, markers))
        return word_dict

Response: Build a dictionary of symbol tables based on the sentences and markers.


Instruction:
Make a summary of the following Python 3 code
def _check_L(self, L):
        # TODO: Take this out?
        if issparse(L):
            L = L.todense()

        # Check for correct values, e.g. warning if in {-1,0,1}
        if np.any(L < 0):
            raise ValueError("L must have values in {0,1,...,k}.")

Response: Run some basic checks on the given log matrix L.


Instruction:
Here you have a function in Python 3, explain what it does
def _create_L_ind(self, L):
        # TODO: Update LabelModel to keep L variants as sparse matrices
        # throughout and remove this line.
        if issparse(L):
            L = L.todense()

        L_ind = np.zeros((self.n, self.m * self.k))
        for y in range(1, self.k + 1):
            # A[x::y] slices A starting at x at intervals of y
            # e.g., np.arange(9)[0::3] == np.array([0,3,6])
            L_ind[:, (y - 1) :: self.k] = np.where(L == y, 1, 0)
        return L_ind

Response: Convert a label matrix with labels in 0... k to a one - hot format.


Instruction:
Can you create a Python 3 function that
returns an augmented version of L where each column is an indicator for whether a certain source or clique of sources voted in a certain pattern.

Response:
def _get_augmented_label_matrix(self, L, higher_order=False):
        """Returns an augmented version of L where each column is an indicator
        for whether a certain source or clique of sources voted in a certain
        pattern.

        Args:
            L: An [n,m] scipy.sparse label matrix with values in {0,1,...,k}
        """
        # Create a helper data structure which maps cliques (as tuples of member
        # sources) --> {start_index, end_index, maximal_cliques}, where
        # the last value is a set of indices in this data structure
        self.c_data = {}
        for i in range(self.m):
            self.c_data[i] = {
                "start_index": i * self.k,
                "end_index": (i + 1) * self.k,
                "max_cliques": set(
                    [
                        j
                        for j in self.c_tree.nodes()
                        if i in self.c_tree.node[j]["members"]
                    ]
                ),
            }

        L_ind = self._create_L_ind(L)

        # Get the higher-order clique statistics based on the clique tree
        # First, iterate over the maximal cliques (nodes of c_tree) and
        # separator sets (edges of c_tree)
        if higher_order:
            L_aug = np.copy(L_ind)
            for item in chain(self.c_tree.nodes(), self.c_tree.edges()):
                if isinstance(item, int):
                    C = self.c_tree.node[item]
                    C_type = "node"
                elif isinstance(item, tuple):
                    C = self.c_tree[item[0]][item[1]]
                    C_type = "edge"
                else:
                    raise ValueError(item)
                members = list(C["members"])
                nc = len(members)

                # If a unary maximal clique, just store its existing index
                if nc == 1:
                    C["start_index"] = members[0] * self.k
                    C["end_index"] = (members[0] + 1) * self.k

                # Else add one column for each possible value
                else:
                    L_C = np.ones((self.n, self.k ** nc))
                    for i, vals in enumerate(product(range(self.k), repeat=nc)):
                        for j, v in enumerate(vals):
                            L_C[:, i] *= L_ind[:, members[j] * self.k + v]

                    # Add to L_aug and store the indices
                    if L_aug is not None:
                        C["start_index"] = L_aug.shape[1]
                        C["end_index"] = L_aug.shape[1] + L_C.shape[1]
                        L_aug = np.hstack([L_aug, L_C])
                    else:
                        C["start_index"] = 0
                        C["end_index"] = L_C.shape[1]
                        L_aug = L_C

                    # Add to self.c_data as well
                    id = tuple(members) if len(members) > 1 else members[0]
                    self.c_data[id] = {
                        "start_index": C["start_index"],
                        "end_index": C["end_index"],
                        "max_cliques": set([item]) if C_type == "node" else set(item),
                    }
            return L_aug
        else:
            return L_ind


Instruction:
Given the following Python 3 function, write the documentation
def _build_mask(self):
        self.mask = torch.ones(self.d, self.d).byte()
        for ci in self.c_data.values():
            si, ei = ci["start_index"], ci["end_index"]
            for cj in self.c_data.values():
                sj, ej = cj["start_index"], cj["end_index"]

                # Check if ci and cj are part of the same maximal clique
                # If so, mask out their corresponding blocks in O^{-1}
                if len(ci["max_cliques"].intersection(cj["max_cliques"])) > 0:
                    self.mask[si:ei, sj:ej] = 0
                    self.mask[sj:ej, si:ei] = 0

Response: Build the mask applied to O^{ - 1 } O for the matrix approx constraint


Instruction:
Given the following Python 3 function, write the documentation
def _generate_O(self, L):
        L_aug = self._get_augmented_label_matrix(L)
        self.d = L_aug.shape[1]
        self.O = torch.from_numpy(L_aug.T @ L_aug / self.n).float()

Response:
Form the overlaps matrix which is just all the different observed
        combinations of values of pairs of sources


Instruction:
Can you generate the documentation for the following Python 3 function
def _generate_O_inv(self, L):
        self._generate_O(L)
        self.O_inv = torch.from_numpy(np.linalg.inv(self.O.numpy())).float()

Response: Form the inverse overlaps matrix


Instruction:
Can you create a Python 3 function that
initializes the learned params for the cluster - level parameters.

Response:
def _init_params(self):
        """Initialize the learned params

        - \mu is the primary learned parameter, where each row corresponds to
        the probability of a clique C emitting a specific combination of labels,
        conditioned on different values of Y (for each column); that is:

            self.mu[i*self.k + j, y] = P(\lambda_i = j | Y = y)

        and similarly for higher-order cliques.
        - Z is the inverse form version of \mu.
        """
        train_config = self.config["train_config"]

        # Initialize mu so as to break basic reflective symmetry
        # Note that we are given either a single or per-LF initial precision
        # value, prec_i = P(Y=y|\lf=y), and use:
        #   mu_init = P(\lf=y|Y=y) = P(\lf=y) * prec_i / P(Y=y)

        # Handle single or per-LF values
        if isinstance(train_config["prec_init"], (int, float)):
            prec_init = train_config["prec_init"] * torch.ones(self.m)
        else:
            prec_init = torch.from_numpy(train_config["prec_init"])
            if prec_init.shape[0] != self.m:
                raise ValueError(f"prec_init must have shape {self.m}.")

        # Get the per-value labeling propensities
        # Note that self.O must have been computed already!
        lps = torch.diag(self.O).numpy()

        # TODO: Update for higher-order cliques!
        self.mu_init = torch.zeros(self.d, self.k)
        for i in range(self.m):
            for y in range(self.k):
                idx = i * self.k + y
                mu_init = torch.clamp(lps[idx] * prec_init[i] / self.p[y], 0, 1)
                self.mu_init[idx, y] += mu_init

        # Initialize randomly based on self.mu_init
        self.mu = nn.Parameter(self.mu_init.clone() * np.random.random()).float()

        if self.inv_form:
            self.Z = nn.Parameter(torch.randn(self.d, self.k)).float()

        # Build the mask over O^{-1}
        # TODO: Put this elsewhere?
        self._build_mask()


Instruction:
Can you create a Python 3 function that
returns the full conditional probabilities table as a numpy array.

Response:
def get_conditional_probs(self, source=None):
        """Returns the full conditional probabilities table as a numpy array,
        where row i*(k+1) + ly is the conditional probabilities of source i
        emmiting label ly (including abstains 0), conditioned on different
        values of Y, i.e.:

            c_probs[i*(k+1) + ly, y] = P(\lambda_i = ly | Y = y)

        Note that this simply involves inferring the kth row by law of total
        probability and adding in to mu.

        If `source` is not None, returns only the corresponding block.
        """
        c_probs = np.zeros((self.m * (self.k + 1), self.k))
        mu = self.mu.detach().clone().numpy()

        for i in range(self.m):
            # si = self.c_data[(i,)]['start_index']
            # ei = self.c_data[(i,)]['end_index']
            # mu_i = mu[si:ei, :]
            mu_i = mu[i * self.k : (i + 1) * self.k, :]
            c_probs[i * (self.k + 1) + 1 : (i + 1) * (self.k + 1), :] = mu_i

            # The 0th row (corresponding to abstains) is the difference between
            # the sums of the other rows and one, by law of total prob
            c_probs[i * (self.k + 1), :] = 1 - mu_i.sum(axis=0)
        c_probs = np.clip(c_probs, 0.01, 0.99)

        if source is not None:
            return c_probs[source * (self.k + 1) : (source + 1) * (self.k + 1)]
        else:
            return c_probs


Instruction:
Given the following Python 3 function, write the documentation
def predict_proba(self, L):
        self._set_constants(L)

        L_aug = self._get_augmented_label_matrix(L)
        mu = np.clip(self.mu.detach().clone().numpy(), 0.01, 0.99)

        # Create a "junction tree mask" over the columns of L_aug / mu
        if len(self.deps) > 0:
            jtm = np.zeros(L_aug.shape[1])

            # All maximal cliques are +1
            for i in self.c_tree.nodes():
                node = self.c_tree.node[i]
                jtm[node["start_index"] : node["end_index"]] = 1

            # All separator sets are -1
            for i, j in self.c_tree.edges():
                edge = self.c_tree[i][j]
                jtm[edge["start_index"] : edge["end_index"]] = 1
        else:
            jtm = np.ones(L_aug.shape[1])

        # Note: We omit abstains, effectively assuming uniform distribution here
        X = np.exp(L_aug @ np.diag(jtm) @ np.log(mu) + np.log(self.p))
        Z = np.tile(X.sum(axis=1).reshape(-1, 1), self.k)
        return X / Z

Response: Returns the [ n k ) matrix of label probabilities P Y | \ lambda


Instruction:
Can you generate the documentation for the following Python 3 function
def get_Q(self):
        Z = self.Z.detach().clone().numpy()
        O = self.O.numpy()
        I_k = np.eye(self.k)
        return O @ Z @ np.linalg.inv(I_k + Z.T @ O @ Z) @ Z.T @ O

Response:
Get the model s estimate of Q = \ mu P \ mu^T
        We can then separately extract \ mu subject to additional constraints and return \ mu P \ mu^T


Instruction:
Can you tell what is the following Python 3 function doing
def _set_class_balance(self, class_balance, Y_dev):
        if class_balance is not None:
            self.p = np.array(class_balance)
        elif Y_dev is not None:
            class_counts = Counter(Y_dev)
            sorted_counts = np.array([v for k, v in sorted(class_counts.items())])
            self.p = sorted_counts / sum(sorted_counts)
        else:
            self.p = (1 / self.k) * np.ones(self.k)
        self.P = torch.diag(torch.from_numpy(self.p)).float()

Response: Set a prior for the class balance from the user - provided class_balance Y_dev


Instruction:
Can you generate the documentation for the following Python 3 function
def train_model(
        self,
        L_train,
        Y_dev=None,
        deps=[],
        class_balance=None,
        log_writer=None,
        **kwargs,
    ):
        """Train the model (i.e. estimate mu) in one of two ways, depending on
        whether source dependencies are provided or not:

        Args:
            L_train: An [n,m] scipy.sparse matrix with values in {0,1,...,k}
                corresponding to labels from supervision sources on the
                training set
            Y_dev: Target labels for the dev set, for estimating class_balance
            deps: (list of tuples) known dependencies between supervision
                sources. If not provided, sources are assumed to be independent.
                TODO: add automatic dependency-learning code
            class_balance: (np.array) each class's percentage of the population

        (1) No dependencies (conditionally independent sources): Estimate mu
        subject to constraints:
            (1a) O_{B(i,j)} - (mu P mu.T)_{B(i,j)} = 0, for i != j, where B(i,j)
                is the block of entries corresponding to sources i,j
            (1b) np.sum( mu P, 1 ) = diag(O)

        (2) Source dependencies:
            - First, estimate Z subject to the inverse form
            constraint:
                (2a) O_\Omega + (ZZ.T)_\Omega = 0, \Omega is the deps mask
            - Then, compute Q = mu P mu.T
            - Finally, estimate mu subject to mu P mu.T = Q and (1b)
        """
        self.config = recursive_merge_dicts(self.config, kwargs, misses="ignore")
        train_config = self.config["train_config"]

        # TODO: Implement logging for label model?
        if log_writer is not None:
            raise NotImplementedError("Logging for LabelModel.")

        # Note that the LabelModel class implements its own (centered) L2 reg.
        l2 = train_config.get("l2", 0)

        self._set_class_balance(class_balance, Y_dev)
        self._set_constants(L_train)
        self._set_dependencies(deps)
        self._check_L(L_train)

        # Whether to take the simple conditionally independent approach, or the
        # "inverse form" approach for handling dependencies
        # This flag allows us to eg test the latter even with no deps present
        self.inv_form = len(self.deps) > 0

        # Creating this faux dataset is necessary for now because the LabelModel
        # loss functions do not accept inputs, but Classifer._train_model()
        # expects training data to feed to the loss functions.
        dataset = MetalDataset([0], [0])
        train_loader = DataLoader(dataset)
        if self.inv_form:
            # Compute O, O^{-1}, and initialize params
            if self.config["verbose"]:
                print("Computing O^{-1}...")
            self._generate_O_inv(L_train)
            self._init_params()

            # Estimate Z, compute Q = \mu P \mu^T
            if self.config["verbose"]:
                print("Estimating Z...")
            self._train_model(train_loader, self.loss_inv_Z)
            self.Q = torch.from_numpy(self.get_Q()).float()

            # Estimate \mu
            if self.config["verbose"]:
                print("Estimating \mu...")
            self._train_model(train_loader, partial(self.loss_inv_mu, l2=l2))
        else:
            # Compute O and initialize params
            if self.config["verbose"]:
                print("Computing O...")
            self._generate_O(L_train)
            self._init_params()

            # Estimate \mu
            if self.config["verbose"]:
                print("Estimating \mu...")
            self._train_model(train_loader, partial(self.loss_mu, l2=l2))

Response: Train the label model on the given set of known classes.


Instruction:
Write a Python 3 script for
transforming the input label matrix L to a three - way overlaps tensor.

Response:
def _get_overlaps_tensor(self, L):
        """Transforms the input label matrix to a three-way overlaps tensor.

        Args:
            L: (np.array) An n x m array of LF output labels, in {0,...,k} if
                self.abstains, else in {1,...,k}, generated by m conditionally
                independent LFs on n data points

        Outputs:
            O: (torch.Tensor) A (m, m, m, k, k, k) tensor of the label-specific
            empirical overlap rates; that is,

                O[i,j,k,y1,y2,y3] = P(\lf_i = y1, \lf_j = y2, \lf_k = y3)

            where this quantity is computed empirically by this function, based
            on the label matrix L.
        """
        n, m = L.shape

        # Convert from a (n,m) matrix of ints to a (k_lf, n, m) indicator tensor
        LY = np.array([np.where(L == y, 1, 0) for y in range(self.k_0, self.k + 1)])

        # Form the three-way overlaps matrix
        O = np.einsum("abc,dbe,fbg->cegadf", LY, LY, LY) / n
        return torch.from_numpy(O).float()


Instruction:
Here you have a function in Python 3, explain what it does
def get_mask(self, m):
        mask = torch.ones((m, m, m, self.k_lf, self.k_lf, self.k_lf)).byte()
        for i, j, k in product(range(m), repeat=3):
            if len(set((i, j, k))) < 3:
                mask[i, j, k, :, :, :] = 0
        return mask

Response: Get the three - way overlaps matrix O which is 0 when indices i j k are not unique


Instruction:
Given the following Python 3 function, write the documentation
def _get_char_offsets(self, char_offsets):
        if char_offsets:
            char_offsets = char_offsets
            char_offsets.append(len(self.text))
        else:
            char_offsets = np.zeros(len(self.tokens) + 1)
            for i, tok in enumerate(self.tokens):
                # Add 1 to account for the spaces between tokens
                char_offsets[i + 1] = char_offsets[i] + len(tok) + 1
            char_offsets[-1] = len(self.text)
        return np.array(char_offsets)

Response: Store or calculate char_offsets


Instruction:
Can you write a function in Python 3 where it
predicts the probabilistic labels for a set of class entry.

Response:
def predict_proba(self, L):
        """
        Args:
            L: An [n, m] scipy.sparse matrix of labels
        Returns:
            output: A [n, k] np.ndarray of probabilistic labels
        """
        n = L.shape[0]
        Y_p = np.random.rand(n, self.k)
        Y_p /= Y_p.sum(axis=1).reshape(-1, 1)
        return Y_p