Instruction:
Given the following Python 3 function, write the documentation
def gain_offsets(Idat,Qdat,Udat,Vdat,tsamp,chan_per_coarse,feedtype='l',**kwargs):
    '''
    Determines relative gain error in the X and Y feeds for an
    observation given I and Q (I and V for circular basis) noise diode data.
    '''
    if feedtype=='l':
        #Fold noise diode data and calculate ON OFF differences for I and Q
        I_OFF,I_ON = foldcal(Idat,tsamp,**kwargs)
        Q_OFF,Q_ON = foldcal(Qdat,tsamp,**kwargs)

        #Calculate power in each feed for noise diode ON and OFF
        XX_ON = (I_ON+Q_ON)/2
        XX_OFF = (I_OFF+Q_OFF)/2
        YY_ON = (I_ON-Q_ON)/2
        YY_OFF = (I_OFF-Q_OFF)/2

        #Calculate gain offset (divided by 2) as defined in Heiles (2001)
        G = (XX_OFF-YY_OFF)/(XX_OFF+YY_OFF)

    if feedtype=='c':
        #Fold noise diode data and calculate ON OFF differences for I and Q
        I_OFF,I_ON = foldcal(Idat,tsamp,**kwargs)
        V_OFF,V_ON = foldcal(Vdat,tsamp,**kwargs)

        #Calculate power in each feed for noise diode ON and OFF
        RR_ON = (I_ON+V_ON)/2
        RR_OFF = (I_OFF+V_OFF)/2
        LL_ON = (I_ON-V_ON)/2
        LL_OFF = (I_OFF-V_OFF)/2

        #Calculate gain offset (divided by 2) as defined in Heiles (2001)
        G = (RR_OFF-LL_OFF)/(RR_OFF+LL_OFF)

    return convert_to_coarse(G,chan_per_coarse)

Response: Determines relative gain error in the X and Y feeds for an anon_noise_diode observation given I Q and V for circular basis.


Instruction:
How would you explain what the following Python 3 function does
def apply_Mueller(I,Q,U,V, gain_offsets, phase_offsets, chan_per_coarse, feedtype='l'):
    '''
    Returns calibrated Stokes parameters for an observation given an array
    of differential gains and phase differences.
    '''

    #Find shape of data arrays and calculate number of coarse channels
    shape = I.shape
    ax0 = I.shape[0]
    ax1 = I.shape[1]
    nchans = I.shape[2]
    ncoarse = nchans/chan_per_coarse

    #Reshape data arrays to separate coarse channels
    I = np.reshape(I,(ax0,ax1,ncoarse,chan_per_coarse))
    Q = np.reshape(Q,(ax0,ax1,ncoarse,chan_per_coarse))
    U = np.reshape(U,(ax0,ax1,ncoarse,chan_per_coarse))
    V = np.reshape(V,(ax0,ax1,ncoarse,chan_per_coarse))

    #Swap axes 2 and 3 to in order for broadcasting to work correctly
    I = np.swapaxes(I,2,3)
    Q = np.swapaxes(Q,2,3)
    U = np.swapaxes(U,2,3)
    V = np.swapaxes(V,2,3)

    #Apply top left corner of electronics chain inverse Mueller matrix
    a = 1/(1-gain_offsets**2)
    if feedtype=='l':
        Icorr = a*(I-gain_offsets*Q)
        Qcorr = a*(-1*gain_offsets*I+Q)
        I = None
        Q = None
    if feedtype=='c':
        Icorr = a*(I-gain_offsets*V)
        Vcorr = a*(-1*gain_offsets*I+V)
        I = None
        V = None

    #Apply bottom right corner of electronics chain inverse Mueller matrix
    if feedtype=='l':
        Ucorr = U*np.cos(phase_offsets)-V*np.sin(phase_offsets)
        Vcorr = U*np.sin(phase_offsets)+V*np.cos(phase_offsets)
        U = None
        V = None
    if feedtype=='c':
        Qcorr = Q*np.cos(phase_offsets)+U*np.sin(phase_offsets)
        Ucorr = -1*Q*np.sin(phase_offsets)+U*np.cos(phase_offsets)
        Q = None
        U = None

    #Reshape arrays to original shape
    Icorr = np.reshape(np.swapaxes(Icorr,2,3),shape)
    Qcorr = np.reshape(np.swapaxes(Qcorr,2,3),shape)
    Ucorr = np.reshape(np.swapaxes(Ucorr,2,3),shape)
    Vcorr = np.reshape(np.swapaxes(Vcorr,2,3),shape)

    #Return corrected data arrays
    return Icorr,Qcorr,Ucorr,Vcorr

Response: Applies the Mueller matrix to an array of differential gains and phase differences.


Instruction:
Make a summary of the following Python 3 code
def calibrate_pols(cross_pols,diode_cross,obsI=None,onefile=True,feedtype='l',**kwargs):
    '''
    Write Stokes-calibrated filterbank file for a given observation
    with a calibrator noise diode measurement on the source

    Parameters
    ----------
    cross_pols : string
        Path to cross polarization filterbank file (rawspec output) for observation to be calibrated
    diode_cross : string
        Path to cross polarization filterbank file of noise diode measurement ON the target
    obsI : string
        Path to Stokes I filterbank file of main observation (only needed if onefile=False)
    onefile : boolean
        True writes all calibrated Stokes parameters to a single filterbank file,
        False writes four separate files
    feedtype : 'l' or 'c'
        Basis of antenna dipoles. 'c' for circular, 'l' for linear
    '''
    #Obtain time sample length, frequencies, and noise diode data
    obs = Waterfall(diode_cross,max_load=150)
    cross_dat = obs.data
    tsamp = obs.header['tsamp']

    #Calculate number of coarse channels in the noise diode measurement (usually 8)
    dio_ncoarse = obs.calc_n_coarse_chan()
    dio_nchans = obs.header['nchans']
    dio_chan_per_coarse = dio_nchans/dio_ncoarse
    obs = None
    Idat,Qdat,Udat,Vdat = get_stokes(cross_dat,feedtype)
    cross_dat = None
    #Calculate differential gain and phase from noise diode measurements
    print('Calculating Mueller Matrix variables')
    gams = gain_offsets(Idat,Qdat,Udat,Vdat,tsamp,dio_chan_per_coarse,feedtype,**kwargs)
    psis = phase_offsets(Idat,Qdat,Udat,Vdat,tsamp,dio_chan_per_coarse,feedtype,**kwargs)

    #Clear data arrays to save memory
    Idat = None
    Qdat = None
    Udat = None
    Vdat = None

    #Get corrected Stokes parameters
    print('Opening '+cross_pols)
    cross_obs = Waterfall(cross_pols,max_load=150)
    obs_ncoarse = cross_obs.calc_n_coarse_chan()
    obs_nchans = cross_obs.header['nchans']
    obs_chan_per_coarse = obs_nchans/obs_ncoarse

    print('Grabbing Stokes parameters')
    I,Q,U,V = get_stokes(cross_obs.data,feedtype)

    print('Applying Mueller Matrix')
    I,Q,U,V = apply_Mueller(I,Q,U,V,gams,psis,obs_chan_per_coarse,feedtype)

    #Use onefile (default) to produce one filterbank file containing all Stokes information
    if onefile==True:
        cross_obs.data[:,0,:] = np.squeeze(I)
        cross_obs.data[:,1,:] = np.squeeze(Q)
        cross_obs.data[:,2,:] = np.squeeze(U)
        cross_obs.data[:,3,:] = np.squeeze(V)
        cross_obs.write_to_fil(cross_pols[:-15]+'.SIQUV.polcal.fil')
        print('Calibrated Stokes parameters written to '+cross_pols[:-15]+'.SIQUV.polcal.fil')
        return

    #Write corrected Stokes parameters to four filterbank files if onefile==False
    obs = Waterfall(obs_I,max_load=150)
    obs.data = I
    obs.write_to_fil(cross_pols[:-15]+'.SI.polcal.fil')   #assuming file is named *.cross_pols.fil
    print('Calibrated Stokes I written to '+cross_pols[:-15]+'.SI.polcal.fil')

    obs.data = Q
    obs.write_to_fil(cross_pols[:-15]+'.Q.polcal.fil')   #assuming file is named *.cross_pols.fil
    print('Calibrated Stokes Q written to '+cross_pols[:-15]+'.Q.polcal.fil')

    obs.data = U
    obs.write_to_fil(cross_pols[:-15]+'.U.polcal.fil')   #assuming file is named *.cross_pols.fil
    print('Calibrated Stokes U written to '+cross_pols[:-15]+'.U.polcal.fil')

    obs.data = V
    obs.write_to_fil(cross_pols[:-15]+'.V.polcal.fil')   #assuming file is named *.cross_pols.fil
    print('Calibrated Stokes V written to '+cross_pols[:-15]+'.V.polcal.fil')

Response: Calibrate the given cross polarization filterbanks for a given observation.


Instruction:
Can you generate the documentation for the following Python 3 function
def fracpols(str, **kwargs):
    '''Output fractional linear and circular polarizations for a
    rawspec cross polarization .fil file. NOT STANDARD USE'''

    I,Q,U,V,L=get_stokes(str, **kwargs)
    return L/I,V/I

Response:
Output fractional linear and circular polarizations for a
    rawspec cross polarization. NOT STANDARD USE


Instruction:
Can you generate the documentation for the following Python 3 function
def write_stokefils(str, str_I, Ifil=False, Qfil=False, Ufil=False, Vfil=False, Lfil=False, **kwargs):
    '''Writes up to 5 new filterbank files corresponding to each Stokes
    parameter (and total linear polarization L) for a given cross polarization .fil file'''

    I,Q,U,V,L=get_stokes(str, **kwargs)
    obs = Waterfall(str_I, max_load=150) #Load filterbank file to write stokes data to
    if Ifil:
        obs.data = I
        obs.write_to_fil(str[:-15]+'.I.fil')   #assuming file is named *.cross_pols.fil

    if Qfil:
        obs.data = Q
        obs.write_to_fil(str[:-15]+'.Q.fil')   #assuming file is named *.cross_pols.fil

    if Ufil:
        obs.data = U
        obs.write_to_fil(str[:-15]+'.U.fil')   #assuming file is named *.cross_pols.fil

    if Vfil:
        obs.data = V
        obs.write_to_fil(str[:-15]+'.V.fil')   #assuming file is named *.cross_pols.fil

    if Lfil:
        obs.data = L
        obs.write_to_fil(str[:-15]+'.L.fil')

Response: Writes up to 5 new filterbank files corresponding to each StokesTaxonomy parameter and total linear polarization L for a given cross polarization. fil file


Instruction:
Write a Python 3 script for
writing two new filterbank files containing fractional linear and circular polarization data

Response:
def write_polfils(str, str_I, **kwargs):
    '''Writes two new filterbank files containing fractional linear and
    circular polarization data'''

    lin,circ=fracpols(str, **kwargs)
    obs = Waterfall(str_I, max_load=150)

    obs.data = lin
    obs.write_to_fil(str[:-15]+'.linpol.fil')   #assuming file is named *.cross_pols.fil

    obs.data = circ
    obs.write_to_fil(str[:-15]+'.circpol.fil')


Instruction:
Implement a Python 3 function for
returning the index of the closest in xarr to value val

Response:
def closest(xarr, val):
    """ Return the index of the closest in xarr to value val """
    idx_closest = np.argmin(np.abs(np.array(xarr) - val))
    return idx_closest


Instruction:
Implement a function in Python 3 to
upgrade data from nbits to 8bits

Response:
def unpack(data, nbit):
    """upgrade data from nbits to 8bits

    Notes: Pretty sure this function is a little broken!
    """
    if nbit > 8:
        raise ValueError("unpack: nbit must be <= 8")
    if 8 % nbit != 0:
        raise ValueError("unpack: nbit must divide into 8")
    if data.dtype not in (np.uint8, np.int8):
        raise TypeError("unpack: dtype must be 8-bit")
    if nbit == 8:
        return data
    elif nbit == 4:
        data = unpack_4to8(data)
        return data
    elif nbit == 2:
        data = unpack_2to8(data)
        return data
    elif nbit == 1:
        data = unpack_1to8(data)
        return data


Instruction:
Explain what the following Python 3 code does
def unpack_2to8(data):
    two_eight_lookup = {0: 40,
                        1: 12,
                        2: -12,
                        3: -40}

    tmp = data.astype(np.uint32)
    tmp = (tmp | (tmp << 12)) & 0xF000F
    tmp = (tmp | (tmp << 6)) & 0x3030303
    tmp = tmp.byteswap()
    tmp = tmp.view('uint8')
    mapped = np.array(tmp, dtype=np.int8)
    for k, v in two_eight_lookup.items():
        mapped[tmp == k] = v
    return mapped

Response: Promote 2 - bit unisgned data into 8 - bit unsigned data.


Instruction:
Write a Python 3 script to
promote 2 - bit unisgned data into 8 - bit unsigned data.

Response:
def unpack_4to8(data):
    """ Promote 2-bit unisgned data into 8-bit unsigned data.

    Args:
        data: Numpy array with dtype == uint8

    Notes:
        # The process is this:
        # ABCDEFGH [Bits of one 4+4-bit value]
        # 00000000ABCDEFGH [astype(uint16)]
        # 0000ABCDEFGH0000 [<< 4]
        # 0000ABCDXXXXEFGH [bitwise 'or' of previous two lines]
        # 0000111100001111 [0x0F0F]
        # 0000ABCD0000EFGH [bitwise 'and' of previous two lines]
        # ABCD0000EFGH0000 [<< 4]
        # which effectively pads the two 4-bit values with zeros on the right
        # Note: This technique assumes LSB-first ordering
    """

    tmpdata = data.astype(np.int16)  # np.empty(upshape, dtype=np.int16)
    tmpdata = (tmpdata | (tmpdata << 4)) & 0x0F0F
    # tmpdata = tmpdata << 4 # Shift into high bits to avoid needing to sign extend
    updata = tmpdata.byteswap()
    return updata.view(data.dtype)


Instruction:
Create a Python 3 function for
returning ON - OFF for all Stokes parameters given a cross_pols noise diode measurement

Response:
def get_diff(dio_cross,feedtype,**kwargs):
    '''
    Returns ON-OFF for all Stokes parameters given a cross_pols noise diode measurement
    '''
    #Get Stokes parameters, frequencies, and time sample length
    obs = Waterfall(dio_cross,max_load=150)
    freqs = obs.populate_freqs()
    tsamp = obs.header['tsamp']
    data = obs.data
    obs = None

    I,Q,U,V = get_stokes(data,feedtype)

    #Fold noise diode data
    I_OFF,I_ON = foldcal(I,tsamp,**kwargs)
    Q_OFF,Q_ON = foldcal(Q,tsamp,**kwargs)
    U_OFF,U_ON = foldcal(U,tsamp,**kwargs)
    V_OFF,V_ON = foldcal(V,tsamp,**kwargs)

    #Do ON-OFF subtraction
    Idiff = I_ON-I_OFF
    Qdiff = Q_ON-Q_OFF
    Udiff = U_ON-U_OFF
    Vdiff = V_ON-V_OFF

    return Idiff,Qdiff,Udiff,Vdiff,freqs


Instruction:
Can you generate the documentation for the following Python 3 function
def plot_Stokes_diode(dio_cross,diff=True,feedtype='l',**kwargs):
    '''
    Plots the uncalibrated full stokes spectrum of the noise diode.
    Use diff=False to plot both ON and OFF, or diff=True for ON-OFF
    '''

    #If diff=True, get ON-OFF. If not get ON and OFF separately
    if diff==True:
        Idiff,Qdiff,Udiff,Vdiff,freqs = get_diff(dio_cross,feedtype,**kwargs)
    else:
        obs = Waterfall(dio_cross,max_load=150)
        freqs = obs.populate_freqs()
        tsamp = obs.header['tsamp']
        data = obs.data
        I,Q,U,V = get_stokes(data,feedtype)

        I_OFF,I_ON = foldcal(I,tsamp,**kwargs)
        Q_OFF,Q_ON = foldcal(Q,tsamp,**kwargs)
        U_OFF,U_ON = foldcal(U,tsamp,**kwargs)
        V_OFF,V_ON = foldcal(V,tsamp,**kwargs)

    #Plot spectra
    if diff==True:
        plt.plot(freqs,Idiff,'k-',label='I')
        plt.plot(freqs,Qdiff,'r-',label='Q')
        plt.plot(freqs,Udiff,'g-',label='U')
        plt.plot(freqs,Vdiff,'m-',label='V')
    else:
        plt.plot(freqs,I_ON,'k-',label='I ON')
        plt.plot(freqs,I_OFF,'k--',label='I OFF')
        plt.plot(freqs,Q_ON,'r-',label='Q ON')
        plt.plot(freqs,Q_OFF,'r--',label='Q OFF')
        plt.plot(freqs,U_ON,'g-',label='U ON')
        plt.plot(freqs,U_OFF,'g--',label='U OFF')
        plt.plot(freqs,V_ON,'m-',label='V ON')
        plt.plot(freqs,V_OFF,'m--',label='V OFF')

    plt.legend()
    plt.xlabel('Frequency (MHz)')
    plt.title('Uncalibrated Full Stokes Noise Diode Spectrum')
    plt.ylabel('Power (Counts)')

Response: Plot the uncalibrated full stokes spectrum of the noise diode.


Instruction:
Given the following Python 3 function, write the documentation
def plot_calibrated_diode(dio_cross,chan_per_coarse=8,feedtype='l',**kwargs):
    '''
    Plots the corrected noise diode spectrum for a given noise diode measurement
    after application of the inverse Mueller matrix for the electronics chain.
    '''
    #Get full stokes data for the ND observation
    obs = Waterfall(dio_cross,max_load=150)
    freqs = obs.populate_freqs()
    tsamp = obs.header['tsamp']
    data = obs.data
    obs = None
    I,Q,U,V = get_stokes(data,feedtype)
    data = None

    #Calculate Mueller Matrix variables for each coarse channel
    psis = phase_offsets(I,Q,U,V,tsamp,chan_per_coarse,feedtype,**kwargs)
    G = gain_offsets(I,Q,U,V,tsamp,chan_per_coarse,feedtype,**kwargs)

    #Apply the Mueller matrix to original noise diode data and refold
    I,Q,U,V = apply_Mueller(I,Q,U,V,G,psis,chan_per_coarse,feedtype)
    I_OFF,I_ON = foldcal(I,tsamp,**kwargs)
    Q_OFF,Q_ON = foldcal(Q,tsamp,**kwargs)
    U_OFF,U_ON = foldcal(U,tsamp,**kwargs)
    V_OFF,V_ON = foldcal(V,tsamp,**kwargs)

    #Delete data arrays for space
    I = None
    Q = None
    U = None
    V = None

    #Plot new ON-OFF spectra
    plt.plot(freqs,I_ON-I_OFF,'k-',label='I')
    plt.plot(freqs,Q_ON-Q_OFF,'r-',label='Q')
    plt.plot(freqs,U_ON-U_OFF,'g-',label='U')
    plt.plot(freqs,V_ON-V_OFF,'m-',label='V')

    plt.legend()
    plt.xlabel('Frequency (MHz)')
    plt.title('Calibrated Full Stokes Noise Diode Spectrum')
    plt.ylabel('Power (Counts)')

Response: Plots the corrected noise diode spectrum for a given noise diode measurement.


Instruction:
Here you have a function in Python 3, explain what it does
def plot_phase_offsets(dio_cross,chan_per_coarse=8,feedtype='l',ax1=None,ax2=None,legend=True,**kwargs):
    '''
    Plots the calculated phase offsets of each coarse channel along with
    the UV (or QU) noise diode spectrum for comparison
    '''
    #Get ON-OFF ND spectra
    Idiff,Qdiff,Udiff,Vdiff,freqs = get_diff(dio_cross,feedtype,**kwargs)
    obs = Waterfall(dio_cross,max_load=150)
    tsamp = obs.header['tsamp']
    data = obs.data
    obs = None
    I,Q,U,V = get_stokes(data,feedtype)

    #Get phase offsets and convert to degrees
    coarse_psis = phase_offsets(I,Q,U,V,tsamp,chan_per_coarse,feedtype,**kwargs)
    coarse_freqs = convert_to_coarse(freqs,chan_per_coarse)
    coarse_degs = np.degrees(coarse_psis)

    #Plot phase offsets
    if ax2==None:
        plt.subplot(211)
    else:
        axPsi = plt.axes(ax2)
        plt.setp(axPsi.get_xticklabels(),visible=False)
    plt.plot(coarse_freqs,coarse_degs,'ko',markersize=2,label='Coarse Channel $\psi$')
    plt.ylabel('Degrees')
    plt.grid(True)
    plt.title('Phase Offsets')
    if legend==True:
        plt.legend()

    #Plot U and V spectra
    if ax1==None:
        plt.subplot(212)
    else:
        axUV = plt.axes(ax1)
    plt.plot(freqs,Udiff,'g-',label='U')
    if feedtype=='l':
        plt.plot(freqs,Vdiff,'m-',label='V')
    if feedtype=='c':
        plt.plot(freqs,Qdiff,'r-',label='Q')
    plt.xlabel('Frequency (MHz)')
    plt.ylabel('Power (Counts)')
    plt.grid(True)
    if legend==True:
        plt.legend()

Response: Plots the phase offsets of each coarse channel along with the UV or QU of the noise diode spectrum for comparison.


Instruction:
Given the following Python 3 function, write the documentation
def plot_gain_offsets(dio_cross,dio_chan_per_coarse=8,feedtype='l',ax1=None,ax2=None,legend=True,**kwargs):
    '''
    Plots the calculated gain offsets of each coarse channel along with
    the time averaged power spectra of the X and Y feeds
    '''
    #Get ON-OFF ND spectra
    Idiff,Qdiff,Udiff,Vdiff,freqs = get_diff(dio_cross,feedtype,**kwargs)
    obs = Waterfall(dio_cross,max_load=150)
    tsamp = obs.header['tsamp']
    data = obs.data
    obs = None
    I,Q,U,V = get_stokes(data,feedtype)

    #Get phase offsets and convert to degrees
    coarse_G = gain_offsets(I,Q,U,V,tsamp,dio_chan_per_coarse,feedtype,**kwargs)
    coarse_freqs = convert_to_coarse(freqs,dio_chan_per_coarse)

    #Get X and Y spectra for the noise diode ON and OFF
    #If using circular feeds these correspond to LL and RR
    XX_OFF,XX_ON = foldcal(np.expand_dims(data[:,0,:],axis=1),tsamp,**kwargs)
    YY_OFF,YY_ON = foldcal(np.expand_dims(data[:,1,:],axis=1),tsamp,**kwargs)

    if ax1==None:
        plt.subplot(211)
    else:
        axG = plt.axes(ax1)
        plt.setp(axG.get_xticklabels(),visible=False)
    plt.plot(coarse_freqs,coarse_G,'ko',markersize=2)
    plt.ylabel(r'$\frac{\Delta G}{2}$',rotation=90)
    if feedtype=='l':
        plt.title('XY Gain Difference')
    if feedtype=='c':
        plt.title('LR Gain Difference')
    plt.grid(True)

    if ax2==None:
        plt.subplot(212)
    else:
        axXY = plt.axes(ax2,sharex=axG)
    if feedtype=='l':
        plt.plot(freqs,XX_OFF,'b-',label='XX')
        plt.plot(freqs,YY_OFF,'r-',label='YY')
    if feedtype=='c':
        plt.plot(freqs,XX_OFF,'b-',label='LL')
        plt.plot(freqs,YY_OFF,'r-',label='RR')
    plt.xlabel('Frequency (MHz)')
    plt.ylabel('Power (Counts)')
    if legend==True:
        plt.legend()

Response: Plots the gain offsets of each coarse channel along with the time averaged power spectra of the X and Y feeds.


Instruction:
Write a Python 3 function for
plotting the diode measurement over the observation time series.

Response:
def plot_diode_fold(dio_cross,bothfeeds=True,feedtype='l',min_samp=-500,max_samp=7000,legend=True,**kwargs):
    '''
    Plots the calculated average power and time sampling of ON (red) and
    OFF (blue) for a noise diode measurement over the observation time series
    '''
    #Get full stokes data of ND measurement
    obs = Waterfall(dio_cross,max_load=150)
    tsamp = obs.header['tsamp']
    data = obs.data
    obs = None
    I,Q,U,V = get_stokes(data,feedtype)

    #Calculate time series, OFF and ON averages, and time samples for each
    tseriesI = np.squeeze(np.mean(I,axis=2))
    I_OFF,I_ON,OFFints,ONints = foldcal(I,tsamp,inds=True,**kwargs)
    if bothfeeds==True:
        if feedtype=='l':
            tseriesQ = np.squeeze(np.mean(Q,axis=2))
            tseriesX = (tseriesI+tseriesQ)/2
            tseriesY = (tseriesI-tseriesQ)/2
        if feedtype=='c':
            tseriesV = np.squeeze(np.mean(V,axis=2))
            tseriesR = (tseriesI+tseriesV)/2
            tseriesL = (tseriesI-tseriesV)/2

    stop = ONints[-1,1]

    #Plot time series and calculated average for ON and OFF
    if bothfeeds==False:
        plt.plot(tseriesI[0:stop],'k-',label='Total Power')
        for i in ONints:
            plt.plot(np.arange(i[0],i[1]),np.full((i[1]-i[0]),np.mean(I_ON)),'r-')
        for i in OFFints:
            plt.plot(np.arange(i[0],i[1]),np.full((i[1]-i[0]),np.mean(I_OFF)),'b-')
    else:
        if feedtype=='l':
            diff = np.mean(tseriesX)-np.mean(tseriesY)
            plt.plot(tseriesX[0:stop],'b-',label='XX')
            plt.plot(tseriesY[0:stop]+diff,'r-',label='YY (shifted)')
        if feedtype=='c':
            diff = np.mean(tseriesL)-np.mean(tseriesR)
            plt.plot(tseriesL[0:stop],'b-',label='LL')
            plt.plot(tseriesR[0:stop]+diff,'r-',label='RR (shifted)')

    #Calculate plotting limits
    if bothfeeds==False:
        lowlim = np.mean(I_OFF)-(np.mean(I_ON)-np.mean(I_OFF))/2
        hilim = np.mean(I_ON)+(np.mean(I_ON)-np.mean(I_OFF))/2
        plt.ylim((lowlim,hilim))

    plt.xlim((min_samp,max_samp))
    plt.xlabel('Time Sample Number')
    plt.ylabel('Power (Counts)')
    plt.title('Noise Diode Fold')
    if legend==True:
        plt.legend()


Instruction:
Implement a function in Python 3 to
generate and shows five plots of full calibration on the diode.

Response:
def plot_fullcalib(dio_cross,feedtype='l',**kwargs):
    '''
    Generates and shows five plots: Uncalibrated diode, calibrated diode, fold information,
    phase offsets, and gain offsets for a noise diode measurement. Most useful diagnostic plot to
    make sure calibration proceeds correctly.
    '''

    plt.figure("Multiple Calibration Plots", figsize=(12,9))
    left, width = 0.075,0.435
    bottom, height = 0.45,0.5
    width2 = 0.232
    bottom2, height2 = 0.115,0.0975

    rect_uncal = [left,bottom,width,height]
    rect_cal = [left+width+0.025,bottom,width,height]
    rect_fold = [left,bottom2,width2,0.22]
    rect_gain1 = [left+width2+0.1,bottom2,width2,height2]
    rect_phase1 = [left+width2*2+0.1*2,bottom2,width2,height2]
    rect_gain2 = [left+width2+0.1,bottom2+height2+0.025,width2,height2]
    rect_phase2 = [left+width2*2+0.1*2,bottom2+height2+0.025,width2,height2]

    #--------
    axFold = plt.axes(rect_fold)
    print('Plotting Diode Fold')
    plot_diode_fold(dio_cross,bothfeeds=False,feedtype=feedtype,min_samp=2000,max_samp=5500,legend=False,**kwargs)

    #--------
    print('Plotting Gain Offsets')
    plot_gain_offsets(dio_cross,feedtype=feedtype,ax1=rect_gain2,ax2=rect_gain1,legend=False,**kwargs)

    #--------
    print('Plotting Phase Offsets')
    plot_phase_offsets(dio_cross,feedtype=feedtype,ax1=rect_phase1,ax2=rect_phase2,legend=False,**kwargs)
    plt.ylabel('')

    #--------
    ax_uncal = plt.axes(rect_uncal)
    print('Plotting Uncalibrated Diode')
    plot_Stokes_diode(dio_cross,feedtype=feedtype,**kwargs)

    #--------
    ax_cal = plt.axes(rect_cal,sharey=ax_uncal)
    print('Plotting Calibrated Diode')
    plot_calibrated_diode(dio_cross,feedtype=feedtype,**kwargs)
    plt.ylabel('')
    plt.setp(ax_cal.get_yticklabels(),visible=False)

    plt.savefig(dio_cross[:-4]+'.stokescalib.png',dpi=2000)
    plt.show()


Instruction:
Can you create a Python 3 function that
plots the full - band Stokes I spectrum of the noise diode.

Response:
def plot_diodespec(ON_obs,OFF_obs,calflux,calfreq,spec_in,units='mJy',**kwargs):
    '''
    Plots the full-band Stokes I spectrum of the noise diode (ON-OFF)
    '''

    dspec = diode_spec(ON_obs,OFF_obs,calflux,calfreq,spec_in,**kwargs)
    obs = Waterfall(ON_obs,max_load=150)
    freqs = obs.populate_freqs()
    chan_per_coarse = obs.header['nchans']/obs.calc_n_coarse_chan()
    coarse_freqs = convert_to_coarse(freqs,chan_per_coarse)
    plt.ion()
    plt.figure()
    plt.plot(coarse_freqs,dspec)
    plt.xlabel('Frequency (MHz)')
    plt.ylabel('Flux Density ('+units+')')
    plt.title('Noise Diode Spectrum')


Instruction:
Explain what the following Python 3 code does
def cmd_tool():
    '''Read input and output frequency, and output file name
    '''

    parser = argparse.ArgumentParser(description='Dices hdf5 or fil files and writes to hdf5 or fil.')
    parser.add_argument('-f', '--input_filename', action='store', default=None, dest='in_fname', type=str, help='Name of file to write from (HDF5 or FIL)')
    parser.add_argument('-b', action='store', default=None, dest='f_start', type=float, help='Start frequency in MHz')
    parser.add_argument('-e', action='store', default=None, dest='f_stop', type=float, help='Stop frequency in MHz')
    parser.add_argument('-x', '--output_file', action='store', default=None, dest='out_format', type=str, help='Output file format [.h5 or .fil].')
    parser.add_argument('-o', '--output_filename', action='store', default=None, dest='out_fname', type=str, help='Ouput file name to write (to HDF5 or FIL).')
    parser.add_argument('-l', action='store', default=None, dest='max_load', type=float,help='Maximum data limit to load. Default:1GB')

    args = parser.parse_args()

    if len(sys.argv) == 1:
            logger.error('Indicate file name and start and stop frequencies')
            sys.exit()

    if args.in_fname == None:
            logger.error('Need to indicate input file name')
            sys.exit()

    if args.out_fname == None:
        if (args.out_format == None) or (args.out_format == 'h5'):
            if args.in_fname[len(args.in_fname)-4:] == '.fil':
                args.out_fname = args.in_fname
                args.out_fname = args.out_fname.replace('.fil','_diced.h5')
            elif args.in_fname[len(args.in_fname)-3:] == '.h5':
                args.out_fname = args.in_fname
                args.out_fname = args.out_fname.replace('.h5','_diced.h5')
            else:
                logger.error('Input file not recognized')
                sys.exit()
        elif args.out_format == 'fil':
            if args.in_fname[len(args.in_fname)-4:] == '.fil':
                args.out_fname = args.in_fname
                args.out_fname = args.out_fname.replace('.fil','_diced.fil')
            elif args.in_fname[len(args.in_fname)-3:] == '.h5':
                args.out_fname = args.in_fname
                args.out_fname = args.out_fname.replace('.h5','_diced.fil')
            else:
                logger.error('input file not recognized.')
                sys.exit()

        else:
            logger.error('Must indicate either output file name or valid output file extension.')
            sys.exit()

    elif (args.out_fname[len(args.out_fname)-4:] == '.fil') and (args.out_format == 'h5'):
        logger.error('Output file extension does not match output file name')
        sys.exit()

    elif (args.out_fname[len(args.out_fname)-3:] == '.h5') and (args.out_format == 'fil'):
        logger.error('Output file extension does not match output file name.')
        sys.exit()

    if (args.out_fname[len(args.out_fname)-3:] != '.h5') and (args.out_fname[len(args.out_fname)-4:] != '.fil'):
        logger.error('Indicate output file name with extension, or simply output file extension.')
        sys.exit()

    if args.f_start == None and args.f_stop == None:
        logger.error('Please give either start and/or end frequencies. Otherwise use fil2h5 or h52fil functions.')
        sys.exit()

    if args.f_start == None:
        logger.warning('Lower frequency not given, setting to ' + str(f_min_file) + ' MHz to match file.')

    if args.f_stop == None:
        logger.warning('Higher frequency not given, setting to ' + str(f_max_file) + ' MHz to match file.')

    #Read start frequency and bandwidth from data set
    file_big = Waterfall(args.in_fname, max_load = args.max_load)
    f_min_file = file_big.header['fch1']
    f_max_file = file_big.header['fch1'] + file_big.header['nchans']*file_big.header['foff']

    if f_max_file < f_min_file:
            f_max_file,f_min_file = f_min_file,f_max_file

    FreqBWFile = f_max_file-f_min_file
    stdDF = FreqBWFile / float(file_big.calc_n_coarse_chan())   #stdDF = 2.9296875

    if args.f_stop < args.f_start:
            args.f_stop,args.f_start = args.f_start,args.f_stop

    if args.f_start < f_max_file and args.f_start > f_min_file and args.f_stop > f_max_file:
            args.f_stop = f_max_file
            logger.warning('Higher frequency set to ' + str(f_max_file) + ' MHz to match file.')

    if args.f_stop < f_max_file and args.f_stop > f_min_file and args.f_start < f_min_file:
            args.f_start = f_min_file
            logger.warning('Lower frequency set to ' + str(f_min_file) + ' MHz to match file.')

    if args.f_start < f_min_file and args.f_stop > f_max_file:
            args.f_start = f_min_file
            args.f_stop = f_max_file
            logger.warning('Lower frequency set to ' + str(f_min_file) + ' MHz and higher frequency set to ' + str(f_max_file) + ' MHz to match file.')
            # print '\nindicated frequencies include file frequency span - no need to dice\n'
            # sys.exit()

    if min(args.f_start,args.f_stop) < f_min_file or max(args.f_start,args.f_stop) > f_max_file:
            logger.error('Bandwidth to extract must be within ' + str(f_min_file) + ' MHz and ' + str(f_max_file) + ' MHz.')
            sys.exit()

    # calculate real coarse channel begin and end freqs
    f_start_real = math.floor((min(args.f_start,args.f_stop) - f_min_file)/stdDF)*stdDF + f_min_file
    f_stop_real = f_max_file - math.floor((f_max_file - max(args.f_start,args.f_stop))/stdDF)*stdDF

    # print
    # print "true start frequency is " + str(f_start_real)
    # print "true stop frequency is " + str(f_stop_real)

    logger.info('Writing to ' + args.out_fname)
    logger.info('Extacting from ' + str(f_start_real) + ' MHz to ' + str(f_stop_real) + ' MHz.')

    # create waterfall object
    file_small = Waterfall(args.in_fname, f_start = f_start_real, f_stop = f_stop_real, max_load = args.max_load)

    # write waterfall object
    if args.out_fname[len(args.out_fname)-4:] == '.fil':
        file_small.write_to_fil(args.out_fname)
    elif args.out_fname[len(args.out_fname)-3:] == '.h5':
        file_small.write_to_hdf5(args.out_fname)
    else:
        logger.error('Error in output file creation : verify output file name and extension.')
        sys.exit()

Response:
Command line interface for the
Taxonomy command line interface.


Instruction:
How would you implement a function in Python 3 that
commands line utility for creating HDF5 Filterbank files.

Response:
def cmd_tool(args=None):
    """ Command line utility for creating HDF5 blimpy files. """
    from argparse import ArgumentParser
    parser = ArgumentParser(description="Command line utility for creating HDF5 Filterbank files.")
    parser.add_argument('dirname', type=str, help='Name of directory to read')
    args = parser.parse_args()

    if not HAS_BITSHUFFLE:
        print("Error: the bitshuffle library is required to run this script.")
        exit()

    filelist = glob.glob(os.path.join(args.dirname, '*.fil'))

    for filename in filelist:
        if not os.path.exists(filename + '.h5'):
            t0 = time.time()
            print("\nReading %s header..." % filename)
            fb = Filterbank(filename, load_data=False)

            data_shape = (fb.n_ints_in_file, fb.header['nifs'], fb.header['nchans'])
            data_dtype = fb.data.dtype
            print(data_dtype)

            print("Creating new dataset, %s" % str(data_shape))
            block_size = 0
            h5 = h5py.File(filename + '.h5', 'w')

            h5.attrs['CLASS'] = 'FILTERBANK'

            dset = h5.create_dataset('data',
                              shape=data_shape,
                              compression=bitshuffle.h5.H5FILTER,
                              compression_opts=(block_size, bitshuffle.h5.H5_COMPRESS_LZ4),
                              dtype=data_dtype)

            dset_mask = h5.create_dataset('mask',
                                     shape=data_shape,
                                     compression=bitshuffle.h5.H5FILTER,
                                     compression_opts=(block_size, bitshuffle.h5.H5_COMPRESS_LZ4),
                                     dtype='uint8')

            dset.dims[0].label = "frequency"
            dset.dims[1].label = "feed_id"
            dset.dims[2].label = "time"

            dset_mask.dims[0].label = "frequency"
            dset_mask.dims[1].label = "feed_id"
            dset_mask.dims[2].label = "time"

            # Copy over header information as attributes
            for key, value in fb.header.items():
                dset.attrs[key] = value

            filesize = os.path.getsize(filename)

            if filesize >= MAX_SIZE:
                n_int_per_read = int(filesize / MAX_SIZE / 2)
                print("Filling in with data over %i reads..." % n_int_per_read)
                for ii in range(0, n_int_per_read):
                    print("Reading %i of %i" % (ii + 1, n_int_per_read))
                    #print  ii*n_int_per_read, (ii+1)*n_int_per_read
                    fb = Filterbank(filename, t_start=ii*n_int_per_read, t_stop=(ii+1) * n_int_per_read)
                    dset[ii*n_int_per_read:(ii+1)*n_int_per_read] = fb.data[:]
            else:
                fb = Filterbank(filename)
                print(dset.shape, " -> ", fb.data.shape)
                dset[:] = fb.data[:]

            h5.close()

            t1 = time.time()
            print("Conversion time: %2.2fs" % (t1- t0))


Instruction:
Create a Python 3 function to
open a file in a Waterfall file.

Response:
def open_file(filename, f_start=None, f_stop=None,t_start=None, t_stop=None,load_data=True,max_load=1.):
    """Open a HDF5 or filterbank file

    Returns instance of a Reader to read data from file.

    ================== ==================================================
    Filename extension File type
    ================== ==================================================
    h5, hdf5           HDF5 format
    fil                fil format
    *other*            Will raise NotImplementedError
    ================== ==================================================

    """
    if not os.path.isfile(filename):
        type(filename)
        print(filename)
        raise IOError("No such file or directory: " + filename)

    filename = os.path.expandvars(os.path.expanduser(filename))
    # Get file extension to determine type
    ext = filename.split(".")[-1].strip().lower()

    if six.PY3:
        ext = bytes(ext, 'ascii')

    if h5py.is_hdf5(filename):
        # Open HDF5 file
        return H5Reader(filename, f_start=f_start, f_stop=f_stop, t_start=t_start, t_stop=t_stop,
                        load_data=load_data, max_load=max_load)
    elif sigproc.is_filterbank(filename):
        # Open FIL file
        return FilReader(filename, f_start=f_start, f_stop=f_stop, t_start=t_start, t_stop=t_stop, load_data=load_data, max_load=max_load)
    else:
        raise NotImplementedError('Cannot open this type of file with Waterfall')


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _setup_selection_range(self, f_start=None, f_stop=None, t_start=None, t_stop=None, init=False):

        # This avoids resetting values
        if init is True:
            if t_start is None:
                t_start = self.t_begin
            if t_stop is None:
                t_stop = self.t_end
            if f_start is None:
                f_start = self.f_begin
            if f_stop is None:
                f_stop = self.f_end
        else:
            if f_start is None:
                f_start = self.f_start
            if f_stop is None:
                f_stop = self.f_stop
            if t_start is None:
                t_start = self.t_start
            if t_stop is None:
                t_stop = self.t_stop

        # By now, all values start/stop are populated.

        if t_stop >= 0 and t_start >= 0 and t_stop < t_start:
            t_stop, t_start = t_start,t_stop
            logger.warning('Given t_stop < t_start, assuming reversed values.')
        if f_stop and f_start and f_stop < f_start:
            f_stop, f_start = f_start,f_stop
            logger.warning('Given f_stop < f_start, assuming reversed values.')

        if t_start >= self.t_begin and t_start < self.t_end:
            self.t_start = int(t_start)
        else:
            if init is False or t_start != None:
                logger.warning('Setting t_start = %f, since t_start not given or not valid.'%self.t_begin)
            self.t_start = self.t_begin

        if t_stop <= self.t_end  and t_stop > self.t_begin:
            self.t_stop = int(t_stop)
        else:
            if init is False or t_stop:
                logger.warning('Setting t_stop = %f, since t_stop not given or not valid.'%self.t_end)
            self.t_stop = self.t_end

        if f_start >= self.f_begin and f_start < self.f_end:
            self.f_start = f_start
        else:
            if init is False or f_start:
                logger.warning('Setting f_start = %f, since f_start not given or not valid.'%self.f_begin)
            self.f_start = self.f_begin

        if f_stop <= self.f_end and f_stop > self.f_begin:
            self.f_stop = f_stop
        else:
            if init is False or f_stop:
                logger.warning('Setting f_stop = %f, since f_stop not given or not valid.'%self.f_end)
            self.f_stop = self.f_end

        # Now we have setup bounds, we can calculate shape of selection
        self.selection_shape = self._calc_selection_shape()

Response: Sets up the selection range for the current file.


Instruction:
How would you implement a function in Python 3 that
sets up the data type of the object.

Response:
def _setup_dtype(self):
        """Calculating dtype
        """

        #Set up the data type
        if self._n_bytes  == 4:
            return b'float32'
        elif self._n_bytes  == 2:
            return b'uint16'
        elif self._n_bytes  == 1:
            return b'uint8'
        else:
            logger.warning('Having trouble setting dtype, assuming float32.')
            return b'float32'


Instruction:
Implement a Python 3 function for
calculating size of data of interest.

Response:
def _calc_selection_size(self):
        """Calculate size of data of interest.
        """

        #Check to see how many integrations requested
        n_ints = self.t_stop - self.t_start
        #Check to see how many frequency channels requested
        n_chan = (self.f_stop - self.f_start) / abs(self.header[b'foff'])

        n_bytes  = self._n_bytes
        selection_size = int(n_ints*n_chan*n_bytes)

        return selection_size


Instruction:
Write a Python 3 script to
calculate shape of data of interest.

Response:
def _calc_selection_shape(self):
        """Calculate shape of data of interest.
        """

        #Check how many integrations requested
        n_ints = int(self.t_stop - self.t_start)
        #Check how many frequency channels requested
        n_chan = int(np.round((self.f_stop - self.f_start) / abs(self.header[b'foff'])))

        selection_shape = (n_ints,int(self.header[b'nifs']),n_chan)

        return selection_shape


Instruction:
Make a summary of the following Python 3 code
def _setup_chans(self):

        if self.header[b'foff'] < 0:
            f0 = self.f_end
        else:
            f0 = self.f_begin

        i_start, i_stop = 0, self.n_channels_in_file
        if self.f_start:
            i_start = np.round((self.f_start - f0) / self.header[b'foff'])
        if self.f_stop:
            i_stop  = np.round((self.f_stop - f0)  / self.header[b'foff'])

        #calculate closest true index value
        chan_start_idx = np.int(i_start)
        chan_stop_idx  = np.int(i_stop)

        if chan_stop_idx < chan_start_idx:
            chan_stop_idx, chan_start_idx = chan_start_idx,chan_stop_idx

        self.chan_start_idx =  chan_start_idx
        self.chan_stop_idx = chan_stop_idx

Response: Setup channel borders and indices for the channel tables.


Instruction:
Make a summary of the following Python 3 code
def _setup_freqs(self):

        if self.header[b'foff'] > 0:
            self.f_start = self.f_begin + self.chan_start_idx*abs(self.header[b'foff'])
            self.f_stop = self.f_begin + self.chan_stop_idx*abs(self.header[b'foff'])
        else:
            self.f_start = self.f_end - self.chan_stop_idx*abs(self.header[b'foff'])
            self.f_stop = self.f_end - self.chan_start_idx*abs(self.header[b'foff'])

Response: Sets up the frequency borders based on the channel values.


Instruction:
Here you have a function in Python 3, explain what it does
def populate_timestamps(self,update_header=False):

        #Check to see how many integrations requested
        ii_start, ii_stop = 0, self.n_ints_in_file
        if self.t_start:
            ii_start = self.t_start
        if self.t_stop:
            ii_stop = self.t_stop

        ## Setup time axis
        t0 = self.header[b'tstart']
        t_delt = self.header[b'tsamp']

        if update_header:
            timestamps = ii_start * t_delt / 24./60./60. + t0
        else:
            timestamps = np.arange(ii_start, ii_stop) * t_delt / 24./60./60. + t0

        return timestamps

Response:
Populate time axis.
            timestamps.


Instruction:
Can you generate the documentation for the following Python 3 function
def calc_n_blobs(self, blob_dim):

        n_blobs = int(np.ceil(1.0 * np.prod(self.selection_shape) / np.prod(blob_dim)))

        return n_blobs

Response: Calculates the number of blobs in the data selection.


Instruction:
Write a Python 3 function for
checking if the current selection is too large.

Response:
def isheavy(self):
        """ Check if the current selection is too large.
        """

        selection_size_bytes = self._calc_selection_size()

        if selection_size_bytes > self.MAX_DATA_ARRAY_SIZE:
            return True
        else:
            return False


Instruction:
Write a Python 3 script for
reading header and return a Python dictionary of key = value pairs

Response:
def read_header(self):
        """ Read header and return a Python dictionary of key:value pairs
        """

        self.header = {}

        for key, val in self.h5['data'].attrs.items():
            if six.PY3:
                key = bytes(key, 'ascii')
            if key == b'src_raj':
                self.header[key] = Angle(val, unit='hr')
            elif key == b'src_dej':
                self.header[key] = Angle(val, unit='deg')
            else:
                self.header[key] = val

        return self.header


Instruction:
Can you implement a function in Python 3 that
finds first blob from selection.

Response:
def _find_blob_start(self, blob_dim, n_blob):
        """Find first blob from selection.
        """

        #Convert input frequencies into what their corresponding channel number would be.
        self._setup_chans()

        #Check which is the blob time offset
        blob_time_start = self.t_start + blob_dim[self.time_axis]*n_blob

        #Check which is the blob frequency offset (in channels)
        blob_freq_start = self.chan_start_idx + (blob_dim[self.freq_axis]*n_blob)%self.selection_shape[self.freq_axis]

        blob_start = np.array([blob_time_start, 0, blob_freq_start])

        return blob_start


Instruction:
Here you have a function in Python 3, explain what it does
def read_data(self, f_start=None, f_stop=None,t_start=None, t_stop=None):

        self._setup_selection_range(f_start=f_start, f_stop=f_stop, t_start=t_start, t_stop=t_stop)

        #check if selection is small enough.
        if self.isheavy():
            logger.warning("Selection size of %.2f GB, exceeding our size limit %.2f GB. Instance created, header loaded, but data not loaded, please try another (t,v) selection." % (self._calc_selection_size() / (1024. ** 3), self.MAX_DATA_ARRAY_SIZE / (1024. ** 3)))
            self.data = np.array([0],dtype=self._d_type)
            return None

        #Convert input frequencies into what their corresponding channel number would be.
        self._setup_chans()
        #Update frequencies ranges from channel number.
        self._setup_freqs()

        self.data = self.h5["data"][self.t_start:self.t_stop,:,self.chan_start_idx:self.chan_stop_idx]

Response: Read data from the H5 file.


Instruction:
Write a Python 3 function for
reading a single blob from a selection.

Response:
def read_blob(self,blob_dim,n_blob=0):
        """Read blob from a selection.
        """

        n_blobs = self.calc_n_blobs(blob_dim)
        if n_blob > n_blobs or n_blob < 0:
            raise ValueError('Please provide correct n_blob value. Given %i, but max values is %i'%(n_blob,n_blobs))

        #This prevents issues when the last blob is smaller than the others in time
        if blob_dim[self.time_axis]*(n_blob+1) > self.selection_shape[self.time_axis]:
            updated_blob_dim = (self.selection_shape[self.time_axis] - blob_dim[self.time_axis]*n_blob, 1, blob_dim[self.freq_axis])
        else:
            updated_blob_dim = [int(i) for i in blob_dim]

        blob_start = self._find_blob_start(blob_dim, n_blob)
        blob_end = blob_start + np.array(updated_blob_dim)

        blob = self.h5["data"][int(blob_start[self.time_axis]):int(blob_end[self.time_axis]),
                               :,
                               int(blob_start[self.freq_axis]):int(blob_end[self.freq_axis])
                               ]

#         if self.header[b'foff'] < 0:
#             blob = blob[:,:,::-1]

        return blob


Instruction:
Write a Python 3 function for
reading blimpy header and return a Python dictionary of key value pairs.

Response:
def read_header(self, return_idxs=False):
        """ Read blimpy header and return a Python dictionary of key:value pairs

        Args:
            filename (str): name of file to open

        Optional args:
            return_idxs (bool): Default False. If true, returns the file offset indexes
                                for values

        Returns:
            Python dict of key:value pairs, OR returns file offset indexes for values.

        """
        self.header = sigproc.read_header(self.filename, return_idxs=return_idxs)
        return self.header


Instruction:
Can you write a function in Python 3 where it
reads data from the file and store it in self. data.

Response:
def read_data(self, f_start=None, f_stop=None,t_start=None, t_stop=None):
        """ Read data.
        """

        self._setup_selection_range(f_start=f_start, f_stop=f_stop, t_start=t_start, t_stop=t_stop)

        #check if selection is small enough.
        if self.isheavy():
            logger.warning("Selection size of %.2f GB, exceeding our size limit %.2f GB. Instance created, "
                           "header loaded, but data not loaded, please try another (t,v) selection." % (self._calc_selection_size() / (1024. ** 3), self.MAX_DATA_ARRAY_SIZE / (1024. ** 3)))
            self.data = np.array([0],dtype=self._d_type)
            return None

        #Convert input frequencies into what their corresponding channel number would be.
        self._setup_chans()
        #Update frequencies ranges from channel number.
        self._setup_freqs()

        n_chans = self.header[b'nchans']
        n_chans_selected = self.selection_shape[self.freq_axis]
        n_ifs   = self.header[b'nifs']

        # Load binary data
        f = open(self.filename, 'rb')
        f.seek(int(self.idx_data))

        # now check to see how many integrations requested
        n_ints = self.t_stop - self.t_start

        # Seek to first integration
        f.seek(int(self.t_start * self._n_bytes  * n_ifs * n_chans), 1)

        #Loading  data
        self.data = np.zeros((n_ints, n_ifs, n_chans_selected), dtype=self._d_type)

        for ii in range(n_ints):
            for jj in range(n_ifs):
                f.seek(int(self._n_bytes  * self.chan_start_idx), 1) # 1 = from current location
                dd = np.fromfile(f, count=n_chans_selected, dtype=self._d_type)

                # Reverse array if frequency axis is flipped
#                     if self.header[b'foff'] < 0:
#                         dd = dd[::-1]

                self.data[ii, jj] = dd

                f.seek(int(self._n_bytes  * (n_chans - self.chan_stop_idx)), 1)


Instruction:
How would you explain what the following Python 3 function does
def _find_blob_start(self):

        # Convert input frequencies into what their corresponding channel number would be.
        self._setup_chans()

        # Check which is the blob time offset
        blob_time_start = self.t_start

        # Check which is the blob frequency offset (in channels)
        blob_freq_start = self.chan_start_idx

        blob_start = blob_time_start * self.n_channels_in_file + blob_freq_start

        return blob_start

Response: Find first blob from selection.


Instruction:
Can you tell what is the following Python 3 function doing
def read_blob(self,blob_dim,n_blob=0):

        n_blobs = self.calc_n_blobs(blob_dim)
        if n_blob > n_blobs or n_blob < 0:
            raise ValueError('Please provide correct n_blob value. Given %i, but max values is %i'%(n_blob,n_blobs))

        # This prevents issues when the last blob is smaller than the others in time.
        if blob_dim[self.time_axis]*(n_blob+1) > self.selection_shape[self.time_axis]:
            updated_blob_dim = (int(self.selection_shape[self.time_axis] - blob_dim[self.time_axis]*n_blob), 1, int(blob_dim[self.freq_axis]))
        else:
            updated_blob_dim = [int(i) for i in blob_dim]

        blob_start = self._find_blob_start()
        blob = np.zeros(updated_blob_dim, dtype=self._d_type)

        # EE: For now; also assuming one polarization and one beam.

        # Assuming the blob will loop over the whole frequency range.
        if self.f_start == self.f_begin and self.f_stop == self.f_end:

            blob_flat_size = np.prod(blob_dim)
            updated_blob_flat_size = np.prod(updated_blob_dim)

            # Load binary data
            with open(self.filename, 'rb') as f:
                f.seek(int(self.idx_data + self._n_bytes  * (blob_start + n_blob*blob_flat_size)))
                dd = np.fromfile(f, count=updated_blob_flat_size, dtype=self._d_type)

            if dd.shape[0] == updated_blob_flat_size:
                blob = dd.reshape(updated_blob_dim)
            else:
                logger.info('DD shape != blob shape.')
                blob = dd.reshape((int(dd.shape[0]/blob_dim[self.freq_axis]),blob_dim[self.beam_axis],blob_dim[self.freq_axis]))
        else:

            for blobt in range(updated_blob_dim[self.time_axis]):

                #Load binary data
                with open(self.filename, 'rb') as f:
                    f.seek(int(self.idx_data + self._n_bytes * (blob_start + n_blob*blob_dim[self.time_axis]*self.n_channels_in_file + blobt*self.n_channels_in_file)))
                    dd = np.fromfile(f, count=blob_dim[self.freq_axis], dtype=self._d_type)

                blob[blobt] = dd

#         if self.header[b'foff'] < 0:
#             blob = blob[:,:,::-1]

        return blob

Response: Read a binary blob from a selection.


Instruction:
Here you have a function in Python 3, explain what it does
def read_all(self,reverse=True):
        raise NotImplementedError('To be implemented')

        # go to start of the data
        self.filfile.seek(int(self.datastart))
        # read data into 2-D numpy array
#        data=np.fromfile(self.filfile,dtype=self.dtype).reshape(self.channels,self.blocksize,order='F')
        data=np.fromfile(self.filfile,dtype=self.dtype).reshape(self.blocksize, self.channels)
        if reverse:
            data = data[:,::-1]
        return data

Response: read all the data into a 2 - D numpy array.


Instruction:
Create a Python 3 function to
read a single row of data into a 2 - D numpy array.

Response:
def read_row(self,rownumber,reverse=True):
        """ Read a block of data. The number of samples per row is set in self.channels
            If reverse=True the x axis is flipped.
        """
        raise NotImplementedError('To be implemented')

        # go to start of the row
        self.filfile.seek(int(self.datastart+self.channels*rownumber*(int(self.nbits/8))))
        # read data into 2-D numpy array
        data=np.fromfile(self.filfile,count=self.channels,dtype=self.dtype).reshape(1, self.channels)
        if reverse:
            data = data[:,::-1]
        return data


Instruction:
Given the following Python 3 function, write the documentation
def cmd_tool(args=None):

    from argparse import ArgumentParser

    parser = ArgumentParser(description="Command line utility for reading and plotting blimpy files.")

    parser.add_argument('filename', type=str,
                        help='Name of file to read')
    parser.add_argument('-p', action='store',  default='a', dest='what_to_plot', type=str,
                        help='Show: "w" waterfall (freq vs. time) plot; "s" integrated spectrum plot; \
                        "t" for time series; "mm" for spectrum including min max; "k" for kurtosis; \
                        "a" for all available plots and information; and "ank" for all but kurtosis.')
    parser.add_argument('-b', action='store', default=None, dest='f_start', type=float,
                        help='Start frequency (begin), in MHz')
    parser.add_argument('-e', action='store', default=None, dest='f_stop', type=float,
                        help='Stop frequency (end), in MHz')
    parser.add_argument('-B', action='store', default=None, dest='t_start', type=int,
                        help='Start integration (begin, inclusive) ID ')
    parser.add_argument('-E', action='store', default=None, dest='t_stop', type=int,
                        help='Stop integration (end, exclusive) ID')
    parser.add_argument('-i', action='store_true', default=False, dest='info_only',
                        help='Show info only')
    parser.add_argument('-a', action='store_true', default=False, dest='average',
                       help='average along time axis (plot spectrum only)')
    parser.add_argument('-s', action='store', default='', dest='plt_filename', type=str,
                       help='save plot graphic to file (give filename as argument)')
    parser.add_argument('-S', action='store_true', default=False, dest='save_only',
                       help='Turn off plotting of data and only save to file.')
    parser.add_argument('-D', action='store_false', default=True, dest='blank_dc',
                       help='Use to not blank DC bin.')
    parser.add_argument('-H', action='store_true', default=False, dest='to_hdf5',
                       help='Write file to hdf5 format.')
    parser.add_argument('-F', action='store_true', default=False, dest='to_fil',
                       help='Write file to .fil format.')
    parser.add_argument('-o', action='store', default=None, dest='filename_out', type=str,
                        help='Filename output (if not probided, the name will be the same but with apropiate extension).')
    parser.add_argument('-l', action='store', default=None, dest='max_load', type=float,
                        help='Maximum data limit to load. Default:1GB')

    if args is None:
        args = sys.argv[1:]

    parse_args = parser.parse_args(args)

    # Open blimpy data
    filename = parse_args.filename
    load_data = not parse_args.info_only
    info_only = parse_args.info_only
    filename_out = parse_args.filename_out

    fil = Waterfall(filename, f_start=parse_args.f_start, f_stop=parse_args.f_stop, t_start=parse_args.t_start, t_stop=parse_args.t_stop, load_data=load_data, max_load=parse_args.max_load)
    fil.info()

    #Check the size of selection.
    if fil.container.isheavy() or parse_args.to_hdf5 or parse_args.to_fil:
        info_only = True

    # And if we want to plot data, then plot data.

    if not info_only:
        print('')

        if parse_args.blank_dc:
            logger.info("Blanking DC bin")
            n_coarse_chan = fil.calc_n_coarse_chan()
            fil.blank_dc(n_coarse_chan)

        if parse_args.what_to_plot == "w":
            plt.figure("waterfall", figsize=(8, 6))
            fil.plot_waterfall(f_start=parse_args.f_start, f_stop=parse_args.f_stop)
        elif parse_args.what_to_plot == "s":
            plt.figure("Spectrum", figsize=(8, 6))
            fil.plot_spectrum(logged=True, f_start=parse_args.f_start, f_stop=parse_args.f_stop, t='all')
        elif parse_args.what_to_plot == "mm":
            plt.figure("min max", figsize=(8, 6))
            fil.plot_spectrum_min_max(logged=True, f_start=parse_args.f_start, f_stop=parse_args.f_stop, t='all')
        elif parse_args.what_to_plot == "k":
            plt.figure("kurtosis", figsize=(8, 6))
            fil.plot_kurtosis(f_start=parse_args.f_start, f_stop=parse_args.f_stop)
        elif parse_args.what_to_plot == "t":
            plt.figure("Time Series", figsize=(8, 6))
            fil.plot_time_series(f_start=parse_args.f_start, f_stop=parse_args.f_stop,orientation='h')
        elif parse_args.what_to_plot == "a":
            plt.figure("Multiple diagnostic plots", figsize=(12, 9),facecolor='white')
            fil.plot_all(logged=True, f_start=parse_args.f_start, f_stop=parse_args.f_stop, t='all')
        elif parse_args.what_to_plot == "ank":
            plt.figure("Multiple diagnostic plots", figsize=(12, 9),facecolor='white')
            fil.plot_all(logged=True, f_start=parse_args.f_start, f_stop=parse_args.f_stop, t='all',kutosis=False)

        if parse_args.plt_filename != '':
            plt.savefig(parse_args.plt_filename)

        if not parse_args.save_only:
            if 'DISPLAY' in os.environ.keys():
                plt.show()
            else:
                logger.warning("No $DISPLAY available.")

    else:

        if parse_args.to_hdf5 and parse_args.to_fil:
            raise Warning('Either provide to_hdf5 or to_fil, but not both.')

        if parse_args.to_hdf5:
            if not filename_out:
                filename_out = filename.replace('.fil','.h5')
            elif '.h5' not in filename_out:
                filename_out = filename_out.replace('.fil','')+'.h5'

            logger.info('Writing file : %s'%(filename_out))
            fil.write_to_hdf5(filename_out)
            logger.info('File written.')

        if parse_args.to_fil:
            if not filename_out:
                filename_out = filename.replace('.h5','.fil')
            elif '.fil' not in filename_out:
                filename_out = filename_out.replace('.h5','')+'.fil'

            logger.info('Writing file : %s'%(filename_out))
            fil.write_to_fil(filename_out)
            logger.info('File written.')

Response: Command line tool for reading and plotting blimpy files.


Instruction:
Can you tell what is the following Python 3 function doing
def read_data(self, f_start=None, f_stop=None,t_start=None, t_stop=None):

        self.container.read_data(f_start=f_start, f_stop=f_stop,t_start=t_start, t_stop=t_stop)

        self.__load_data()

Response: Reads data selection if small enough.


Instruction:
Can you tell what is the following Python 3 function doing
def __update_header(self):

        #Updating frequency of first channel from selection
        if self.header[b'foff'] < 0:
            self.header[b'fch1'] = self.container.f_stop
        else:
            self.header[b'fch1'] = self.container.f_start

        #Updating number of coarse channels.
        self.header[b'nchans'] = self.container.selection_shape[self.freq_axis]

        #Updating time stamp for first time bin from selection
        self.header[b'tstart'] = self.container.populate_timestamps(update_header=True)

Response: Updates the header information from the original file to the selection.


Instruction:
Can you create a Python 3 function that
prints header information and other derived information.

Response:
def info(self):
        """ Print header information and other derived information. """

        print("\n--- File Info ---")

        for key, val in self.file_header.items():
            if key == 'src_raj':
                val = val.to_string(unit=u.hour, sep=':')
            if key == 'src_dej':
                val = val.to_string(unit=u.deg, sep=':')
            print("%16s : %32s" % (key, val))

        print("\n%16s : %32s" % ("Num ints in file", self.n_ints_in_file))
        print("%16s : %32s" % ("File shape", self.file_shape))
        print("--- Selection Info ---")
        print("%16s : %32s" % ("Data selection shape", self.selection_shape))
        print("%16s : %32s" % ("Minimum freq (MHz)", self.container.f_start))
        print("%16s : %32s" % ("Maximum freq (MHz)", self.container.f_stop))


Instruction:
Can you write a function in Python 3 where it
writes data to a. fil file.

Response:
def write_to_fil(self, filename_out, *args, **kwargs):
        """ Write data to .fil file.
            It check the file size then decides how to write the file.

        Args:
            filename_out (str): Name of output file
        """

        #For timing how long it takes to write a file.
        t0 = time.time()

        #Update header
        self.__update_header()

        if self.container.isheavy():
            self.__write_to_fil_heavy(filename_out)
        else:
            self.__write_to_fil_light(filename_out)

        t1 = time.time()
        logger.info('Conversion time: %2.2fsec' % (t1- t0))


Instruction:
How would you code a function in Python 3 to
write data to. fil file.

Response:
def __write_to_fil_heavy(self, filename_out, *args, **kwargs):
        """ Write data to .fil file.

        Args:
            filename_out (str): Name of output file
        """

        #Note that a chunk is not a blob!!
        chunk_dim = self.__get_chunk_dimensions()
        blob_dim = self.__get_blob_dimensions(chunk_dim)
        n_blobs = self.container.calc_n_blobs(blob_dim)

        #Write header of .fil file
        n_bytes  = self.header[b'nbits'] / 8
        with open(filename_out, "wb") as fileh:
            fileh.write(generate_sigproc_header(self)) #generate_sigproc_header comes from sigproc.py

        logger.info('Using %i n_blobs to write the data.'% n_blobs)
        for ii in range(0, n_blobs):
            logger.info('Reading %i of %i' % (ii + 1, n_blobs))

            bob = self.container.read_blob(blob_dim,n_blob=ii)

            #Write data of .fil file.
            with open(filename_out, "a") as fileh:
                j = bob
                if n_bytes == 4:
                    np.float32(j.ravel()).tofile(fileh)
                elif n_bytes == 2:
                    np.int16(j.ravel()).tofile(fileh)
                elif n_bytes == 1:
                    np.int8(j.ravel()).tofile(fileh)


Instruction:
Implement a function in Python 3 to
write data to. fil file.

Response:
def __write_to_fil_light(self, filename_out, *args, **kwargs):
        """ Write data to .fil file.

        Args:
            filename_out (str): Name of output file
        """

        n_bytes  = self.header[b'nbits'] / 8
        with open(filename_out, "wb") as fileh:
            fileh.write(generate_sigproc_header(self)) #generate_sigproc_header comes from sigproc.py
            j = self.data
            if n_bytes == 4:
                np.float32(j.ravel()).tofile(fileh)
            elif n_bytes == 2:
                np.int16(j.ravel()).tofile(fileh)
            elif n_bytes == 1:
                np.int8(j.ravel()).tofile(fileh)


Instruction:
Can you write a function in Python 3 where it
writes data to HDF5 file.

Response:
def write_to_hdf5(self, filename_out, *args, **kwargs):
        """ Write data to HDF5 file.
            It check the file size then decides how to write the file.

        Args:
            filename_out (str): Name of output file
        """

        #For timing how long it takes to write a file.
        t0 = time.time()

        #Update header
        self.__update_header()

        if self.container.isheavy():
            self.__write_to_hdf5_heavy(filename_out)
        else:
            self.__write_to_hdf5_light(filename_out)

        t1 = time.time()
        logger.info('Conversion time: %2.2fsec' % (t1- t0))


Instruction:
How would you explain what the following Python 3 function does
def __write_to_hdf5_heavy(self, filename_out, *args, **kwargs):

        block_size = 0

        #Note that a chunk is not a blob!!
        chunk_dim = self.__get_chunk_dimensions()
        blob_dim = self.__get_blob_dimensions(chunk_dim)
        n_blobs = self.container.calc_n_blobs(blob_dim)

        with h5py.File(filename_out, 'w') as h5:

            h5.attrs[b'CLASS'] = b'FILTERBANK'
            h5.attrs[b'VERSION'] = b'1.0'

            if HAS_BITSHUFFLE:
                bs_compression = bitshuffle.h5.H5FILTER
                bs_compression_opts = (block_size, bitshuffle.h5.H5_COMPRESS_LZ4)
            else:
                bs_compression = None
                bs_compression_opts = None
                logger.warning("Warning: bitshuffle not found. No compression applied.")

            dset = h5.create_dataset('data',
                            shape=self.selection_shape,
                            chunks=chunk_dim,
                            compression=bs_compression,
                            compression_opts=bs_compression_opts,
                            dtype=self.data.dtype)

            dset_mask = h5.create_dataset('mask',
                            shape=self.selection_shape,
                            chunks=chunk_dim,
                            compression=bs_compression,
                            compression_opts=bs_compression_opts,
                            dtype='uint8')

            dset.dims[0].label = b"frequency"
            dset.dims[1].label = b"feed_id"
            dset.dims[2].label = b"time"

            dset_mask.dims[0].label = b"frequency"
            dset_mask.dims[1].label = b"feed_id"
            dset_mask.dims[2].label = b"time"

            # Copy over header information as attributes
            for key, value in self.header.items():
                dset.attrs[key] = value

            if blob_dim[self.freq_axis] < self.selection_shape[self.freq_axis]:

                logger.info('Using %i n_blobs to write the data.'% n_blobs)
                for ii in range(0, n_blobs):
                    logger.info('Reading %i of %i' % (ii + 1, n_blobs))

                    bob = self.container.read_blob(blob_dim,n_blob=ii)

                    #-----
                    #Using channels instead of frequency.
                    c_start = self.container.chan_start_idx + ii*blob_dim[self.freq_axis]
                    t_start = self.container.t_start + (c_start/self.selection_shape[self.freq_axis])*blob_dim[self.time_axis]
                    t_stop = t_start + blob_dim[self.time_axis]

                    # Reverse array if frequency axis is flipped
#                     if self.header['foff'] < 0:
#                         c_stop = self.selection_shape[self.freq_axis] - (c_start)%self.selection_shape[self.freq_axis]
#                         c_start = c_stop - blob_dim[self.freq_axis]
#                     else:
                    c_start = (c_start)%self.selection_shape[self.freq_axis]
                    c_stop = c_start + blob_dim[self.freq_axis]
                    #-----

                    logger.debug(t_start,t_stop,c_start,c_stop)

                    dset[t_start:t_stop,0,c_start:c_stop] = bob[:]

            else:

                logger.info('Using %i n_blobs to write the data.'% n_blobs)
                for ii in range(0, n_blobs):
                    logger.info('Reading %i of %i' % (ii + 1, n_blobs))

                    bob = self.container.read_blob(blob_dim,n_blob=ii)
                    t_start = self.container.t_start + ii*blob_dim[self.time_axis]

                    #This prevents issues when the last blob is smaller than the others in time
                    if (ii+1)*blob_dim[self.time_axis] > self.n_ints_in_file:
                        t_stop = self.n_ints_in_file
                    else:
                        t_stop = (ii+1)*blob_dim[self.time_axis]

                    dset[t_start:t_stop] = bob[:]

Response: Write data to HDF5 file.


Instruction:
Create a Python 3 function to
write data to HDF5 file in one go.

Response:
def __write_to_hdf5_light(self, filename_out, *args, **kwargs):
        """ Write data to HDF5 file in one go.

        Args:
            filename_out (str): Name of output file
        """

        block_size = 0

        with h5py.File(filename_out, 'w') as h5:

            h5.attrs[b'CLASS']   = b'FILTERBANK'
            h5.attrs[b'VERSION'] = b'1.0'

            if HAS_BITSHUFFLE:
                bs_compression = bitshuffle.h5.H5FILTER
                bs_compression_opts = (block_size, bitshuffle.h5.H5_COMPRESS_LZ4)
            else:
                bs_compression = None
                bs_compression_opts = None
                logger.warning("Warning: bitshuffle not found. No compression applied.")

            dset = h5.create_dataset('data',
                        data=self.data,
#                          compression='lzf')
                        compression=bs_compression,
                        compression_opts=bs_compression_opts)

            dset_mask = h5.create_dataset('mask',
                        shape=self.file_shape,
#                                 compression='lzf',
                        compression=bs_compression,
                        compression_opts=bs_compression_opts,
                        dtype='uint8')

            dset.dims[0].label = b"frequency"
            dset.dims[1].label = b"feed_id"
            dset.dims[2].label = b"time"

            dset_mask.dims[0].label = b"frequency"
            dset_mask.dims[1].label = b"feed_id"
            dset_mask.dims[2].label = b"time"

            # Copy over header information as attributes
            for key, value in self.header.items():
                dset.attrs[key] = value


Instruction:
Can you create a Python 3 function that
sets the blob dimmentions

Response:
def __get_blob_dimensions(self, chunk_dim):
        """ Sets the blob dimmentions, trying to read around 1024 MiB at a time.
            This is assuming a chunk is about 1 MiB.
        """

        #Taking the size into consideration, but avoiding having multiple blobs within a single time bin.
        if self.selection_shape[self.freq_axis] > chunk_dim[self.freq_axis]*MAX_BLOB_MB:
            freq_axis_size = self.selection_shape[self.freq_axis]
#             while freq_axis_size > chunk_dim[self.freq_axis]*MAX_BLOB_MB:
#                 freq_axis_size /= 2
            time_axis_size = 1
        else:
            freq_axis_size = self.selection_shape[self.freq_axis]
            time_axis_size = np.min([chunk_dim[self.time_axis] * MAX_BLOB_MB * chunk_dim[self.freq_axis] / freq_axis_size, self.selection_shape[self.time_axis]])

        blob_dim = (int(time_axis_size), 1, freq_axis_size)

        return blob_dim


Instruction:
Can you create a Python 3 function that
sets the chunking dimmentions depending on the file format.

Response:
def __get_chunk_dimensions(self):
        """ Sets the chunking dimmentions depending on the file type.
        """

        #Usually '.0000.' is in self.filename
        if np.abs(self.header[b'foff']) < 1e-5:
            logger.info('Detecting high frequency resolution data.')
            chunk_dim = (1,1,1048576) #1048576 is the number of channels in a coarse channel.
            return chunk_dim
        #Usually '.0001.' is in self.filename
        elif np.abs(self.header[b'tsamp']) < 1e-3:
            logger.info('Detecting high time resolution data.')
            chunk_dim = (2048,1,512) #512 is the total number of channels per single band (ie. blc00)
            return chunk_dim
        #Usually '.0002.' is in self.filename
        elif np.abs(self.header[b'foff']) < 1e-2 and np.abs(self.header[b'foff'])  >= 1e-5:
            logger.info('Detecting intermediate frequency and time resolution data.')
            chunk_dim = (10,1,65536)  #65536 is the total number of channels per single band (ie. blc00)
#            chunk_dim = (1,1,65536/4)
            return chunk_dim
        else:
            logger.warning('File format not known. Will use minimum chunking. NOT OPTIMAL.')
            chunk_dim = (1,1,512)
            return chunk_dim


Instruction:
How would you code a function in Python 3 to
grab a portion of data from the file.

Response:
def grab_data(self, f_start=None, f_stop=None,t_start=None, t_stop=None, if_id=0):
        """ Extract a portion of data by frequency range.

        Args:
            f_start (float): start frequency in MHz
            f_stop (float): stop frequency in MHz
            if_id (int): IF input identification (req. when multiple IFs in file)

        Returns:
            (freqs, data) (np.arrays): frequency axis in MHz and data subset
        """

        self.freqs = self.populate_freqs()
        self.timestamps = self.populate_timestamps()

        if f_start is None:
            f_start = self.freqs[0]
        if f_stop is None:
            f_stop = self.freqs[-1]

        i0 = np.argmin(np.abs(self.freqs - f_start))
        i1 = np.argmin(np.abs(self.freqs - f_stop))

        if i0 < i1:
            plot_f    = self.freqs[i0:i1 + 1]
            plot_data = np.squeeze(self.data[t_start:t_stop, ..., i0:i1 + 1])
        else:
            plot_f    = self.freqs[i1:i0 + 1]
            plot_data = np.squeeze(self.data[t_start:t_stop, ..., i1:i0 + 1])

        return plot_f, plot_data


Instruction:
Create a Python 3 function to
command line tool for plotting and viewing info on guppi raw files.

Response:
def cmd_tool(args=None):
    """ Command line tool for plotting and viewing info on guppi raw files """

    from argparse import ArgumentParser

    parser = ArgumentParser(description="Command line utility for creating spectra from GuppiRaw files.")

    parser.add_argument('filename', type=str, help='Name of file to read')
    parser.add_argument('-o', dest='outdir', type=str, default='./', help='output directory for PNG files')
    args = parser.parse_args()

    r = GuppiRaw(args.filename)

    r.print_stats()
    bname = os.path.splitext(os.path.basename(args.filename))[0]
    bname = os.path.join(args.outdir, bname)
    r.plot_histogram(filename="%s_hist.png" % bname)
    r.plot_spectrum(filename="%s_spec.png" % bname)


Instruction:
How would you explain what the following Python 3 function does
def read_header(self):
        start_idx = self.file_obj.tell()
        key, val = '', ''

        header_dict = {}
        keep_reading = True

        first_line = self.file_obj

        try:
            while keep_reading:
                if start_idx + 80 > self.filesize:
                    keep_reading = False
                    raise EndOfFileError("End Of Data File")
                line = self.file_obj.read(80)
                if PYTHON3:
                    line = line.decode("utf-8")
                # print line
                if line.startswith('END'):
                    keep_reading = False
                    break
                else:
                    key, val = line.split('=')
                    key, val = key.strip(), val.strip()

                    if "'" in val:
                        # Items in quotes are strings
                        val = str(val.strip("'").strip())
                    elif "." in val:
                        # Items with periods are floats (if not a string)
                        val = float(val)
                    else:
                        # Otherwise it's an integer
                        val = int(val)

                header_dict[key] = val
        except ValueError:
            print("CURRENT LINE: ", line)
            print("BLOCK START IDX: ", start_idx)
            print("FILE SIZE: ", self.filesize)
            print("NEXT 512 BYTES: \n")
            print(self.file_obj.read(512))
            raise

        data_idx = self.file_obj.tell()

        # Seek past padding if DIRECTIO is being used
        if "DIRECTIO" in header_dict.keys():
            if int(header_dict["DIRECTIO"]) == 1:
                if data_idx % 512:
                    data_idx += (512 - data_idx % 512)

        self.file_obj.seek(start_idx)
        return header_dict, data_idx

Response: Reads next header from the file and returns a dictionary of keyword - value header data and byte index of where the corresponding data block resides.


Instruction:
How would you code a function in Python 3 to
read the first header in file

Response:
def read_first_header(self):
        """ Read first header in file

        Returns:
            header (dict): keyword:value pairs of header metadata
        """
        self.file_obj.seek(0)
        header_dict, pos = self.read_header()
        self.file_obj.seek(0)
        return header_dict


Instruction:
Make a summary of the following Python 3 code
def get_data(self):
        with self as gr:
            while True:
                try:
                    yield gr.read_next_data_block_int8()
                except Exception as e:
                    print("File depleted")
                    yield None, None, None

Response: returns a generator object that reads data at a time


Instruction:
Write a Python 3 function that can
read the next block of data and its header and return it as a tuple of header data and complex64.

Response:
def read_next_data_block_int8(self):
        """ Read the next block of data and its header

        Returns: (header, data)
            header (dict): dictionary of header metadata
            data (np.array): Numpy array of data, converted into to complex64.
        """
        header, data_idx = self.read_header()
        self.file_obj.seek(data_idx)

        # Read data and reshape

        n_chan = int(header['OBSNCHAN'])
        n_pol = int(header['NPOL'])
        n_bit = int(header['NBITS'])
        n_samples = int(int(header['BLOCSIZE']) / (n_chan * n_pol * (n_bit / 8)))

        d = np.fromfile(self.file_obj, count=header['BLOCSIZE'], dtype='int8')

        # Handle 2-bit and 4-bit data
        if n_bit != 8:
            d = unpack(d, n_bit)

        d = d.reshape((n_chan, n_samples, n_pol))  # Real, imag

        if self._d_x.shape != d[..., 0:2].shape:
            self._d_x = np.ascontiguousarray(np.zeros(d[..., 0:2].shape, dtype='int8'))
            self._d_y = np.ascontiguousarray(np.zeros(d[..., 2:4].shape, dtype='int8'))

        self._d_x[:] = d[..., 0:2]
        self._d_y[:] = d[..., 2:4]
        return header, self._d_x, self._d_y


Instruction:
Write a Python 3 script for
reading the next block of data and its header and data and convert it into complex64.

Response:
def read_next_data_block_int8_2x(self):
        """ Read the next block of data and its header

        Returns: (header, data)
            header (dict): dictionary of header metadata
            data (np.array): Numpy array of data, converted into to complex64.
        """
        header, data_idx = self.read_header()
        self.file_obj.seek(data_idx)

        # Read data and reshape

        n_chan = int(header['OBSNCHAN'])
        n_pol = int(header['NPOL'])
        n_bit = int(header['NBITS'])
        n_samples = int(int(header['BLOCSIZE']) / (n_chan * n_pol * (n_bit / 8)))

        d = np.fromfile(self.file_obj, count=header['BLOCSIZE'], dtype='int8')

        header, data_idx = self.read_header()
        self.file_obj.seek(data_idx)
        d2 = np.fromfile(self.file_obj, count=header['BLOCSIZE'], dtype='int8')

        # Handle 2-bit and 4-bit data
        if n_bit != 8:
            d = unpack(d, n_bit)

        d = d.reshape((n_chan, n_samples, n_pol))  # Real, imag
        d2 = d2.reshape((n_chan, n_samples, n_pol))
        d = np.concatenate((d, d2), axis=1)
        print(d.shape)

        if self._d_x.shape != (n_chan, n_samples * 2, n_pol):
            self._d_x = np.ascontiguousarray(np.zeros(d[..., 0:2].shape, dtype='int8'))
            self._d_y = np.ascontiguousarray(np.zeros(d[..., 2:4].shape, dtype='int8'))

        self._d_x[:] = d[..., 0:2]
        self._d_y[:] = d[..., 2:4]
        return header, self._d_x, self._d_y


Instruction:
Implement a Python 3 function for
reading the next block of data and its header and data.

Response:
def read_next_data_block(self):
        """ Read the next block of data and its header

        Returns: (header, data)
            header (dict): dictionary of header metadata
            data (np.array): Numpy array of data, converted into to complex64.
        """
        header, data_idx = self.read_header()
        self.file_obj.seek(data_idx)

        # Read data and reshape

        n_chan = int(header['OBSNCHAN'])
        n_pol = int(header['NPOL'])
        n_bit = int(header['NBITS'])
        n_samples = int(int(header['BLOCSIZE']) / (n_chan * n_pol * (n_bit / 8)))

        d = np.ascontiguousarray(np.fromfile(self.file_obj, count=header['BLOCSIZE'], dtype='int8'))

        # Handle 2-bit and 4-bit data
        if n_bit != 8:
            d = unpack(d, n_bit)

        dshape = self.read_next_data_block_shape()

        d = d.reshape(dshape)  # Real, imag

        if self._d.shape != d.shape:
            self._d = np.zeros(d.shape, dtype='float32')

        self._d[:] = d

        return header, self._d[:].view('complex64')


Instruction:
Explain what the following Python 3 code does
def find_n_data_blocks(self):
        self.file_obj.seek(0)
        header0, data_idx0 = self.read_header()

        self.file_obj.seek(data_idx0)
        block_size = int(header0['BLOCSIZE'])
        n_bits = int(header0['NBITS'])
        self.file_obj.seek(int(header0['BLOCSIZE']), 1)
        n_blocks = 1
        end_found = False
        while not end_found:
            try:
                header, data_idx = self.read_header()
                self.file_obj.seek(data_idx)
                self.file_obj.seek(header['BLOCSIZE'], 1)
                n_blocks += 1
            except EndOfFileError:
                end_found = True
                break

        self.file_obj.seek(0)
        return n_blocks

Response: Seek through the file to find how many data blocks there are in the file


Instruction:
How would you explain what the following Python 3 function does
def print_stats(self):

        header, data = self.read_next_data_block()
        data = data.view('float32')

        print("AVG: %2.3f" % data.mean())
        print("STD: %2.3f" % data.std())
        print("MAX: %2.3f" % data.max())
        print("MIN: %2.3f" % data.min())

        import pylab as plt

Response: Compute some basic stats on the next block of data


Instruction:
Can you generate a brief explanation for the following Python 3 code
def plot_histogram(self, filename=None):
        header, data = self.read_next_data_block()
        data = data.view('float32')

        plt.figure("Histogram")
        plt.hist(data.flatten(), 65, facecolor='#cc0000')
        if filename:
            plt.savefig(filename)
        plt.show()

Response: Plot a histogram of data values


Instruction:
How would you code a function in Python 3 to
plot the spectrum of the current logarithm of the data.

Response:
def plot_spectrum(self, filename=None, plot_db=True):
        """ Do a (slow) numpy FFT and take power of data """
        header, data = self.read_next_data_block()

        print("Computing FFT...")
        d_xx_fft = np.abs(np.fft.fft(data[..., 0]))
        d_xx_fft = d_xx_fft.flatten()

        # Rebin to max number of points
        dec_fac_x = 1
        if d_xx_fft.shape[0] > MAX_PLT_POINTS:
            dec_fac_x = d_xx_fft.shape[0] / MAX_PLT_POINTS

        d_xx_fft = rebin(d_xx_fft, dec_fac_x)

        print("Plotting...")
        if plot_db:
            plt.plot(10 * np.log10(d_xx_fft))
            plt.ylabel("Power [dB]")
        else:
            plt.plot(d_xx_fft)
            plt.ylabel("Power")
            plt.xlabel("Channel")
        plt.title(self.filename)
        if filename:
            plt.savefig(filename)
        plt.show()


Instruction:
Make a summary of the following Python 3 code
def generate_filterbank_header(self, nchans=1, ):
        gp_head = self.read_first_header()
        fb_head = {}

        telescope_str = gp_head.get("TELESCOP", "unknown")
        if telescope_str in ('GBT', 'GREENBANK'):
            fb_head["telescope_id"] = 6
        elif telescope_str in ('PKS', 'PARKES'):
            fb_head["telescop_id"] = 7
        else:
            fb_head["telescop_id"] = 0

        # Using .get() method allows us to fill in default values if not present
        fb_head["source_name"] = gp_head.get("SRC_NAME", "unknown")
        fb_head["az_start"] = gp_head.get("AZ", 0)
        fb_head["za_start"] = gp_head.get("ZA", 0)

        fb_head["src_raj"] = Angle(str(gp_head.get("RA", 0.0)) + "hr")
        fb_head["src_dej"] = Angle(str(gp_head.get("DEC", 0.0)) + "deg")
        fb_head["rawdatafile"] = self.filename

        # hardcoded
        fb_head["machine_id"] = 20
        fb_head["data_type"] = 1  # blio datatype
        fb_head["barycentric"] = 0
        fb_head["pulsarcentric"] = 0
        fb_head["nbits"] = 32

        # TODO - compute these values. Need to figure out the correct calcs
        fb_head["tstart"] = 0.0
        fb_head["tsamp"] = 1.0
        fb_head["fch1"] = 0.0
        fb_head["foff"] = 187.5 / nchans

        # Need to be updated based on output specs
        fb_head["nchans"] = nchans
        fb_head["nifs"] = 1
        fb_head["nbeams"] = 1

        return fb_head

Response: Generate a blimpy header dictionary for the filterbank.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def find_header_size(filename):
    ''' Script to find the header size of a filterbank file'''

    # open datafile
    filfile=open(filename,'rb')
    # go to the start of the file
    filfile.seek(0)
    #read some region larger than the header.
    round1 = filfile.read(1000)
    headersize = round1.find('HEADER_END')+len('HEADER_END')

    return headersize

Response: Script to find the header size of a filterbank file


Instruction:
Write a Python 3 function for
commanding line tool to make a md5sum comparison of two. fil files.

Response:
def cmd_tool(args=None):
    """ Command line tool to make a md5sum comparison of two .fil files. """

    if 'bl' in local_host:
        header_loc = '/usr/local/sigproc/bin/header' #Current location of header command in GBT.
    else:
        raise IOError('Script only able to run in BL systems.')

    p = OptionParser()
    p.set_usage('matchfils <FIL_FILE1> <FIL_FILE2>')
    opts, args = p.parse_args(sys.argv[1:])

    file1 = args[0]
    file2 = args[1]

    #------------------------------------
    #Create batch script

    make_batch_script()

    #------------------------------------
    #First checksum

    headersize1 = find_header_size(file1)
    file_size1 = os.path.getsize(file1)

    #Strip header from file, and calculate the md5sum of the rest.
    #command=['tail','-c',str(file_size1-headersize1),file1,'|','md5sum']
    command=['./tail_sum.sh',file1,str(file_size1-headersize1)]
    print('[matchfils] '+' '.join(command))

    proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    (out, err) = proc.communicate()

    check_sum1 = out.split()[0]
    print('[matchfils] Checksum is:', check_sum1)

    if err:
        raise Error('There is an error.')

    #---
    out,err = reset_outs()

    command=[header_loc,file1]
    print('[matchfils] Header information:')

    proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    (out, err) = proc.communicate()

    header1 = out
    print(header1)

    #------------------------------------
    #Second checksum

    out,err = reset_outs()

    headersize2 = find_header_size(file2)
    file_size2 = os.path.getsize(file2)

    #Strip header from file, and calculate the md5sum of the rest.
    command=['./tail_sum.sh',file2,str(file_size2-headersize2)]
    print('[matchfils] '+' '.join(command))

    proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    (out, err) = proc.communicate()

    check_sum2 = out.split()[0]
    print('[matchfils] Checksum is:', check_sum2)

    if err:
        raise Error('There is an error.')

    #---
    out,err = reset_outs()

    command=[header_loc,file2]
    print('[matchfils] Header information:')

    proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    (out, err) = proc.communicate()

    header2 = out
    print(header2)

    #------------------------------------
    #check the checksums

    if check_sum1 != check_sum2:
        print('[matchfils] Booo! Checksum does not match between files.')
    else:
        print('[matchfils] Hooray! Checksum matches between files.')

    #------------------------------------
    #Remove batch script

    os.remove('tail_sum.sh')


Instruction:
Explain what the following Python 3 code does
def make_h5_file(filename,out_dir='./', new_filename = None, max_load = None):
    ''' Converts file to HDF5 (.h5) format. Default saves output in current dir.
    '''

    fil_file = Waterfall(filename, max_load = max_load)
    if not new_filename:
        new_filename = out_dir+filename.replace('.fil','.h5').split('/')[-1]

    if '.h5' not in new_filename:
        new_filename = new_filename+'.h5'

    fil_file.write_to_hdf5(new_filename)

Response: Converts a file to HDF5 format.


Instruction:
Here you have a function in Python 3, explain what it does
def cmd_tool(args=None):
    from argparse import ArgumentParser

    if not HAS_BITSHUFFLE:
        print("Error: the bitshuffle library is required to run this script.")
        exit()

    parser = ArgumentParser(description="Command line utility for creating HDF5 Raw files.")
    parser.add_argument('filename', type=str, help='Name of filename to read')
    args = parser.parse_args()

    fileroot = args.filename.split('.0000.raw')[0]

    filelist = glob.glob(fileroot + '*.raw')
    filelist = sorted(filelist)

    # Read first file
    r = GuppiRaw(filelist[0])
    header, data = r.read_next_data_block()
    dshape = data.shape #r.read_next_data_block_shape()
    print(dshape)

    n_blocks_total = 0
    for filename in filelist:
        print(filename)
        r = GuppiRaw(filename)
        n_blocks_total += r.n_blocks
    print(n_blocks_total)

    full_dshape = np.concatenate(((n_blocks_total,), dshape))

    # Create h5py file
    h5 = h5py.File(fileroot + '.h5', 'w')
    h5.attrs['CLASS'] = 'GUPPIRAW'
    block_size = 0      # This is chunk block size
    dset = h5.create_dataset('data',
                  shape=full_dshape,
                  #compression=bitshuffle.h5.H5FILTER,
                  #compression_opts=(block_size, bitshuffle.h5.H5_COMPRESS_LZ4),
                  dtype=data.dtype)

    h5_idx = 0
    for filename in filelist:
        print("\nReading %s header..." % filename)
        r = GuppiRaw(filename)
        h5 = h5py.File(filename + '.h5', 'w')

        header, data = r.read_next_data_block()

        for ii in range(0, r.n_blocks):
            t0 = time.time()
            print("Reading block %i of %i" % (h5_idx+1, full_dshape[0]))
            header, data = r.read_next_data_block()
            t1 = time.time()

            t2 = time.time()
            print("Writing block %i of %i" % (h5_idx+1, full_dshape[0]))
            dset[h5_idx, :] = data
            t3 = time.time()
            print("Read: %2.2fs, Write %2.2fs" % ((t1-t0), (t3-t2)))

            h5_idx += 1

            # Copy over header information as attributes
            for key, value in header.items():
                dset.attrs[key] = value

        h5.close()

        t1 = time.time()
        print("Conversion time: %2.2fs" % (t1- t0))

Response: Command line tool for converting guppi raw into HDF5 versions of guppi raw


Instruction:
Can you tell what is the following Python 3 function doing
def foldcal(data,tsamp, diode_p=0.04,numsamps=1000,switch=False,inds=False):
    '''
    Returns time-averaged spectra of the ON and OFF measurements in a
    calibrator measurement with flickering noise diode

    Parameters
    ----------
    data : 2D Array object (float)
        2D dynamic spectrum for data (any Stokes parameter) with flickering noise diode.
    tsamp : float
        Sampling time of data in seconds
    diode_p : float
        Period of the flickering noise diode in seconds
    numsamps : int
        Number of samples over which to average noise diode ON and OFF
    switch : boolean
        Use switch=True if the noise diode "skips" turning from OFF to ON once or vice versa
    inds : boolean
        Use inds=True to also return the indexes of the time series where the ND is ON and OFF
    '''

    halfper = diode_p/2.0

    foldt = halfper/tsamp   #number of time samples per diode switch

    onesec = 1/tsamp    #number of time samples in the first second

    #Find diode switches in units of time samples and round down to the nearest int
    ints = np.arange(0,numsamps)
    t_switch = (onesec+ints*foldt)
    t_switch = t_switch.astype('int')

    ONints = np.array(np.reshape(t_switch[:],(numsamps/2,2)))
    ONints[:,0] = ONints[:,0]+1   #Find index ranges of ON time samples

    OFFints = np.array(np.reshape(t_switch[1:-1],(numsamps/2-1,2)))
    OFFints[:,0] = OFFints[:,0]+1   #Find index ranges of OFF time samples

    av_ON = []
    av_OFF = []

    #Average ON and OFF spectra separately with respect to time
    for i in ONints:
        if i[1]!=i[0]:
            av_ON.append(np.sum(data[i[0]:i[1],:,:],axis=0)/(i[1]-i[0]))

    for i in OFFints:
        if i[1]!=i[0]:
            av_OFF.append(np.sum(data[i[0]:i[1],:,:],axis=0)/(i[1]-i[0]))

    #If switch=True, flip the return statement since ON is actually OFF
    if switch==False:
        if inds==False:
            return np.squeeze(np.mean(av_ON,axis=0)), np.squeeze(np.mean(av_OFF,axis=0))
        else:
            return np.squeeze(np.mean(av_ON,axis=0)), np.squeeze(np.mean(av_OFF,axis=0)),ONints,OFFints
    if switch==True:
        if inds==False:
            return np.squeeze(np.mean(av_OFF,axis=0)), np.squeeze(np.mean(av_ON,axis=0))
        else:
            return np.squeeze(np.mean(av_OFF,axis=0)), np.squeeze(np.mean(av_ON,axis=0)),OFFints,ONints

Response: This function calculates the time - averaged spectra of the ON and OFF measurements in a single calendar file.


Instruction:
How would you implement a function in Python 3 that
integrates over each core channel of a given spectrum and frequencies.

Response:
def integrate_chans(spec,freqs,chan_per_coarse):
    '''
    Integrates over each core channel of a given spectrum.
    Important for calibrating data with frequency/time resolution different from noise diode data

    Parameters
    ----------
    spec : 1D Array (float)
        Spectrum (any Stokes parameter) to be integrated
    freqs : 1D Array (float)
        Frequency values for each bin of the spectrum
    chan_per_coarse: int
        Number of frequency bins per coarse channel
    '''

    num_coarse = spec.size/chan_per_coarse     #Calculate total number of coarse channels

    #Rearrange spectrum by coarse channel
    spec_shaped = np.array(np.reshape(spec,(num_coarse,chan_per_coarse)))
    freqs_shaped = np.array(np.reshape(freqs,(num_coarse,chan_per_coarse)))

    #Average over coarse channels
    return np.mean(spec_shaped[:,1:-1],axis=1)


Instruction:
Implement a Python 3 function for
integrating the frequency bins of a single noise diode into two different components.

Response:
def integrate_calib(name,chan_per_coarse,fullstokes=False,**kwargs):
    '''
    Folds Stokes I noise diode data and integrates along coarse channels

    Parameters
    ----------
    name : str
        Path to noise diode filterbank file
    chan_per_coarse : int
        Number of frequency bins per coarse channel
    fullstokes : boolean
        Use fullstokes=True if data is in IQUV format or just Stokes I, use fullstokes=False if
        it is in cross_pols format
    '''
    #Load data
    obs = Waterfall(name,max_load=150)
    data = obs.data

    #If the data has cross_pols format calculate Stokes I
    if fullstokes==False and data.shape[1]>1:
        data = data[:,0,:]+data[:,1,:]
        data = np.expand_dims(data,axis=1)
    #If the data has IQUV format get Stokes I
    if fullstokes==True:
        data = data[:,0,:]
        data = np.expand_dims(data,axis=1)

    tsamp = obs.header['tsamp']

    #Calculate ON and OFF values
    OFF,ON = foldcal(data,tsamp,**kwargs)

    freqs = obs.populate_freqs()

    #Find ON and OFF spectra by coarse channel
    ON_int = integrate_chans(ON,freqs,chan_per_coarse)
    OFF_int = integrate_chans(OFF,freqs,chan_per_coarse)

    #If "ON" is actually "OFF" switch them
    if np.sum(ON_int)<np.sum(OFF_int):
        temp = ON_int
        ON_int = OFF_int
        OFF_int = temp

    #Return coarse channel spectrum of OFF and ON
    return OFF_int,ON_int