Instruction:
Explain what the following Python 3 code does
def update(self, gist, content):
        # abort if content is False
        if content is False:
            return False

        # request
        url = self._api_url("gists", gist.get("id"))
        data = {"files": {self.filename: {"content": content}}}
        self.output("Sending contents of {} to {}".format(self.file_path, url))
        response = self.requests.patch(url, data=dumps(data))

        # error
        if response.status_code != 200:
            self.oops("Could not update " + gist.get("description"))
            self.oops("PATCH request returned " + str(response.status_code))
            return False

        # success
        self.yeah("Done!")
        self.hey("The URL to this Gist is: {}".format(gist["url"]))
        return True

Response: Updates the contents of a gist at GitHub.


Instruction:
Here you have a function in Python 3, explain what it does
def create(self, content, **kwargs):
        # abort if content is False
        if content is False:
            return False

        # set new gist
        public = bool(kwargs.get("public", True))
        data = {
            "description": self.filename,
            "public": public,
            "files": {self.filename: {"content": content}},
        }

        # send request
        url = self._api_url("gists")
        self.output("Sending contents of {} to {}".format(self.file_path, url))
        response = self.requests.post(url, data=dumps(data))

        # error
        if response.status_code != 201:
            self.oops("Could not create " + self.filename)
            self.oops("POST request returned " + str(response.status_code))
            return False

        # parse created gist
        gist = self._parse_gist(response.json())

        # success
        self.yeah("Done!")
        self.hey("The URL to this Gist is: {}".format(gist["url"]))
        return True

Response: Create a new gist.


Instruction:
How would you implement a function in Python 3 that
asks user which gist to use in case of more than one gist matching the instance filename.

Response:
def _ask_which_gist(self, matches):
        """
        Asks user which gist to use in case of more than one gist matching the
        instance filename.
        :param matches: (list) of dictioaries generated within select_gists()
        :return: (dict) of the selected gist
        """
        # ask user which gist to use
        self.hey("Use {} from which gist?".format(self.filename))
        for count, gist in enumerate(matches, 1):
            self.hey("[{}] {}".format(count, gist.get("description")))

        # get the gist index
        selected = False
        while not selected:
            gist_index = prompt("Type the number: ", type=int) - 1
            try:
                selected = matches[gist_index]
            except IndexError:
                self.oops("Invalid number, please try again.")

        self.output("Using `{}` Gist".format(selected["description"]))
        return selected


Instruction:
Write a Python 3 function that can
receive a gist dict and parse it to GetGist

Response:
def _parse_gist(gist):
        """Receive a gist (dict) and parse it to GetGist"""

        # parse files
        files = list()
        file_names = sorted(filename for filename in gist["files"].keys())
        for name in file_names:
            files.append(
                dict(filename=name, raw_url=gist["files"][name].get("raw_url"))
            )

        # parse description
        description = gist["description"]
        if not description:
            names = sorted(f.get("filename") for f in files)
            description = names.pop(0)

        return dict(
            description=description,
            id=gist.get("id"),
            files=files,
            url=gist.get("html_url"),
        )


Instruction:
Here you have a function in Python 3, explain what it does
def indent(self, message):
        indent = self.indent_char * self.indent_size
        return indent + message

Response: Sets the indent for standardized output


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get(self, url, params=None, **kwargs):
        return requests.get(url, params=params, headers=self.add_headers(**kwargs))

Response: Encapsulte requests. get to use this class instance header


Instruction:
Can you tell what is the following Python 3 function doing
def patch(self, url, data=None, **kwargs):
        return requests.patch(url, data=data, headers=self.add_headers(**kwargs))

Response: Encapsulte requests. patch to use this class instance header


Instruction:
Make a summary of the following Python 3 code
def post(self, url, data=None, **kwargs):
        return requests.post(url, data=data, headers=self.add_headers(**kwargs))

Response: Encapsulte requests. post to use this class instance header


Instruction:
Given the following Python 3 function, write the documentation
def save(self, content):
        # backup existing file if needed
        if os.path.exists(self.file_path) and not self.assume_yes:
            message = "Overwrite existing {}? (y/n) "
            if not confirm(message.format(self.filename)):
                self.backup()

        # write file
        self.output("Saving " + self.filename)
        with open(self.file_path, "wb") as handler:
            if not isinstance(content, bytes):
                content = bytes(content, "utf-8")
            handler.write(content)
        self.yeah("Done!")

Response: Save any given content to the instance file.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def backup(self):
        count = 0
        name = "{}.bkp".format(self.filename)
        backup = os.path.join(self.cwd, name)
        while os.path.exists(backup):
            count += 1
            name = "{}.bkp{}".format(self.filename, count)
            backup = os.path.join(self.cwd, name)
        self.hey("Moving existing {} to {}".format(self.filename, name))
        os.rename(os.path.join(self.cwd, self.filename), backup)

Response: Backups files with the same name of the instance filename


Instruction:
Implement a function in Python 3 to
read the contents of a file in the local file system.

Response:
def read(self, file_path=None):
        """
        Read the contents of a file.
        :param filename: (str) path to a file in the local file system
        :return: (str) contents of the file, or (False) if not found/not file
        """
        if not file_path:
            file_path = self.file_path

        # abort if the file path does not exist
        if not os.path.exists(file_path):
            self.oops("Sorry, but {} does not exist".format(file_path))
            return False

        # abort if the file path is not a file
        if not os.path.isfile(file_path):
            self.oops("Sorry, but {} is not a file".format(file_path))
            return False

        with open(file_path) as handler:
            return handler.read()


Instruction:
Here you have a function in Python 3, explain what it does
def str_matcher(mode):

    def f_raw(inp_str, pos):
        return unique_literal_cache_pool[mode] if inp_str.startswith(mode, pos) else None

    def f_collection(inp_str, pos):
        for each in mode:
            if inp_str.startswith(each, pos):
                return unique_literal_cache_pool[each]
        return None

    if isinstance(mode, str):
        return f_raw

    if len(mode) is 1:
        mode = mode[0]
        return f_raw

    return f_collection

Response: Returns a function that returns a list of tokens that match the input string.


Instruction:
Explain what the following Python 3 code does
def regex_matcher(regex_pat):
    if isinstance(regex_pat, str):
        regex_pat = re.compile(regex_pat)

    def f(inp_str, pos):
        m = regex_pat.match(inp_str, pos)
        return m.group() if m else None

    return f

Response:
generate token names cache
    generate token names


Instruction:
Can you tell what is the following Python 3 function doing
def ast_for_stmts(self, stmts: T) -> None:
        if not stmts:
            raise ValueError('no ast found!')
        head, *equals = stmts

        if head.name is NameEnum.TokenDef:
            self.ast_for_token_def(head)
        elif head.name is NameEnum.TokenIgnore:
            self.ast_for_token_ignore(head)
        else:
            self.ast_for_equals(head)

        for each in equals:
            self.ast_for_equals(each)

        # if every combined parser can reach any other combined,
        # just take any of them and compile it!
        if not self.compile_helper.alone and self._current__combined_parser_name:
            self.compile_helper.alone.add(self._current__combined_parser_name)

Response: Add the ast for the given list of statements.


Instruction:
Make a summary of the following Python 3 code
def _request(self, method, resource_uri, **kwargs):
        data = kwargs.get('data')
        response = method(self.API_BASE_URL + resource_uri,
                          json=data, headers=self.headers)
        response.raise_for_status()
        return response.json()

Response: Perform a method on a resource.


Instruction:
Write a Python 3 script for
getting a resource s nacms.

Response:
def get(self, endpoint, **kwargs):
        """Get a resource.

        Args:
            endpoint: resource endpoint.
        """
        return self._request(requests.get, endpoint, **kwargs)


Instruction:
How would you code a function in Python 3 to
create a resource. Args: endpoint: resource endpoint.

Response:
def post(self, endpoint, **kwargs):
        """Create a resource.

        Args:
            endpoint: resource endpoint.
        """
        return self._request(requests.post, endpoint, **kwargs)


Instruction:
Make a summary of the following Python 3 code
def put(self, endpoint, **kwargs):
        return self._request(requests.put, endpoint, **kwargs)

Response: Update a resource s nacms.


Instruction:
Implement a function in Python 3 to
take Slack formatted text and returns HTML.

Response:
def render(txt):
    """
    Accepts Slack formatted text and returns HTML.
    """

    # Removing links to other channels
    txt = re.sub(r'<#[^\|]*\|(.*)>', r'#\g<1>', txt)

    # Removing links to other users
    txt = re.sub(r'<(@.*)>', r'\g<1>', txt)

    # handle named hyperlinks
    txt = re.sub(r'<([^\|]*)\|([^\|]*)>', r'<a href="\g<1>" target="blank">\g<2>', txt)

    # handle unnamed hyperlinks
    txt = re.sub(r'<([^a|/a].*)>', r'<a href="\g<1>" target="blank">\g<1>', txt)

    # handle ordered and unordered lists
    for delimeter in LIST_DELIMITERS:
        slack_tag = delimeter
        class_name = LIST_DELIMITERS[delimeter]

        # Wrap any lines that start with the slack_tag in

        list_regex = u'(?:^|\n){}\s?(.*)'.format(slack_tag)
        list_repl = r'
\g<1>
'.format(class_name)
        txt = re.sub(list_regex, list_repl, txt)

    # hanlde blockquotes
    txt = re.sub(u'(^|\n)(?:>){3}\s?(.*)$', r'\g<1>
\g<2>
', txt, flags=re.DOTALL)
    txt = re.sub(u'(?:^|\n)>\s?(.*)\n?', r'
\g<1>
', txt)

    # handle code blocks
    txt = re.sub(r'```\n?(.*)```', r'
\g<1>
', txt, flags=re.DOTALL)
    txt = re.sub(r'\n(
)', r'\g<1>', txt)

    # handle bolding, italics, and strikethrough
    for wrapper in FORMATTERS:
        slack_tag = wrapper
        html_tag = FORMATTERS[wrapper]

        # Grab all text in formatted characters on the same line unless escaped
        regex = r'(?<!\\)\{t}([^\{t}|\n]*)\{t}'.format(t=slack_tag)
        repl = r'<{t}>\g<1></{t}>'.format(t=html_tag)
        txt = re.sub(regex, repl, txt)

    # convert line breaks
    txt = txt.replace('\n', '
')

    # clean up bad HTML
    parser = CustomSlackdownHTMLParser(txt)
    txt = parser.clean()

    # convert multiple spaces
    txt = txt.replace(r'  ', ' &nbsp')

    return txt


Instruction:
Make a summary of the following Python 3 code
def _open_list(self, list_type):
        if list_type in LIST_TYPES.keys():
            tag = LIST_TYPES[list_type]
        else:
            raise Exception('CustomSlackdownHTMLParser:_open_list: Not a valid list type.')

        html = '<{t} class="list-container-{c}">'.format(
            t=tag,
            c=list_type
        )
        self.cleaned_html += html
        self.current_parent_element['tag'] = LIST_TYPES[list_type]
        self.current_parent_element['attrs'] = {'class': list_type}

Response: Add an open list tag corresponding to the specification in the LIST_TYPES.


Instruction:
Implement a function in Python 3 to
add an close list tag corresponding to the currently open list found in current_parent_element.

Response:
def _close_list(self):
        """
        Add an close list tag corresponding to the currently open
        list found in current_parent_element.
        """
        list_type = self.current_parent_element['attrs']['class']
        tag = LIST_TYPES[list_type]

        html = '</{t}>'.format(
            t=tag
        )
        self.cleaned_html += html
        self.current_parent_element['tag'] = ''
        self.current_parent_element['attrs'] = {}


Instruction:
Create a Python 3 function for
handling the start tag of the element.

Response:
def handle_starttag(self, tag, attrs):
        """
        Called by HTMLParser.feed when a start tag is found.
        """
        # Parse the tag attributes
        attrs_dict = dict(t for t in attrs)

        # If the tag is a predefined parent element
        if tag in PARENT_ELEMENTS:
            # If parser is parsing another parent element
            if self.current_parent_element['tag'] != '':
                # close the parent element
                self.cleaned_html += '</{}>'.format(self.current_parent_element['tag'])

            self.current_parent_element['tag'] = tag
            self.current_parent_element['attrs'] = {}

            self.cleaned_html += '<{}>'.format(tag)

        # If the tag is a list item
        elif tag == 'li':
            self.parsing_li = True

            # Parse the class name & subsequent type
            class_name = attrs_dict['class']
            list_type = class_name[10:]

            # Check if parsing a list
            if self.current_parent_element['tag'] == 'ul' or self.current_parent_element['tag'] == 'ol':
                cur_list_type = self.current_parent_element['attrs']['class']
                # Parsing a different list
                if cur_list_type != list_type:
                    # Close that list
                    self._close_list()

                    # Open new list
                    self._open_list(list_type)
            # Not parsing a list
            else:
                # if parsing some other parent
                if self.current_parent_element['tag'] != '':
                    self.cleaned_html += '</{}>'.format(self.current_parent_element['tag'])
                # Open new list
                self._open_list(list_type)

            self.cleaned_html += '<{}>'.format(tag)

        # If the tag is a line break
        elif tag == 'br':
            # If parsing a paragraph, close it
            if self.current_parent_element['tag'] == 'p':
                self.cleaned_html += '
'
                self.current_parent_element['tag'] = ''
                self.current_parent_element['attrs'] = {}
            # If parsing a list, close it
            elif self.current_parent_element['tag'] == 'ul' or self.current_parent_element['tag'] == 'ol':
                self._close_list()
            # If parsing any other parent element, keep it
            elif self.current_parent_element['tag'] in PARENT_ELEMENTS:
                self.cleaned_html += '
'
            # If not in any parent element, create an empty paragraph
            else:
                self.cleaned_html += '

'

        # If the tag is something else, like a  or  tag
        else:
            # If not parsing any parent element
            if self.current_parent_element['tag'] == '':
                self.cleaned_html += '
'
                self.current_parent_element['tag'] = 'p'
            self.cleaned_html += '<{}'.format(tag)

            for attr in sorted(attrs_dict.keys()):
                self.cleaned_html += ' {k}="{v}"'.format(
                    k=attr,
                    v=attrs_dict[attr]
                )

            self.cleaned_html += '>'


Instruction:
How would you code a function in Python 3 to
call by HTMLParser. feed when an end tag is found.

Response:
def handle_endtag(self, tag):
        """
        Called by HTMLParser.feed when an end tag is found.
        """
        if tag in PARENT_ELEMENTS:
            self.current_parent_element['tag'] = ''
            self.current_parent_element['attrs'] = ''

        if tag == 'li':
            self.parsing_li = True
        if tag != 'br':
            self.cleaned_html += '</{}>'.format(tag)


Instruction:
Given the following Python 3 function, write the documentation
def handle_data(self, data):
        if self.current_parent_element['tag'] == '':
            self.cleaned_html += '<p>'
            self.current_parent_element['tag'] = 'p'

        self.cleaned_html += data

Response: Called by HTMLParser. feed when text is found.


Instruction:
Can you write a function in Python 3 where it
removes formatting tags added to pre elements.

Response:
def _remove_pre_formatting(self):
        """
        Removes formatting tags added to pre elements.
        """
        preformatted_wrappers = [
            'pre',
            'code'
        ]

        for wrapper in preformatted_wrappers:
            for formatter in FORMATTERS:
                tag = FORMATTERS[formatter]
                character = formatter

                regex = r'(<{w}>.*)<{t}>(.*)</{t}>(.*</{w}>)'.format(
                    t=tag,
                    w=wrapper
                )
                repl = r'\g<1>{c}\g<2>{c}\g<3>'.format(c=character)
                self.cleaned_html = re.sub(regex, repl, self.cleaned_html)


Instruction:
How would you explain what the following Python 3 function does
def clean(self):
        # Calls handle_starttag, handle_endtag, and handle_data
        self.feed()

        # Clean up any parent tags left open
        if self.current_parent_element['tag'] != '':
            self.cleaned_html += '</{}>'.format(self.current_parent_element['tag'])

        # Remove empty
 added after lists
        self.cleaned_html = re.sub(r'(</[u|o]l>)

', r'\g<1>', self.cleaned_html)

        self._remove_pre_formatting()

        return self.cleaned_html

Response: Cleans up any problematic HTML.


Instruction:
Make a summary of the following Python 3 code
def render_author(**kwargs):
    html = '
'

    author_avatar = kwargs.get('author_avatar', None)
    if author_avatar:
        html += ''.format(author_avatar)

    author_name = kwargs.get('author_name', None)
    if author_name:
        html += '
'

        author_link = kwargs.get('author_link', None)
        if author_link:
            html += '{author_name}'.format(
                author_link=author_link,
                author_name=author_name
            )
        else:
            html += author_name

        html += '
'

    author_handle = kwargs.get('author_handle', None)
    if author_handle:
        html += '
{}
'.format(author_handle)

    html += '
'

Response:
Unstrict template block for rendering authors:

            {author_name}

{author_handle}


Instruction:
Here you have a function in Python 3, explain what it does
def render_metadata(**kwargs):
    html = '
'

    service_logo = kwargs.get('service_logo', None)
    if service_logo:
        html += ''.format(service_logo)

    service_name = kwargs.get('service_name', None)
    if service_name:
        html += '
{}
'.format(service_name)

    timestamp = kwargs.get('timestamp', None)
    if timestamp:
        html += '
'

        timestamp_link = kwargs.get('timestamp_link', None)
        if timestamp_link:
            html += '{timestamp}'.format(
                timestamp_link=timestamp_link,
                timestamp=timestamp
            )
        else:
            html += timestamp

        html += '
'

    html += '
'

Response: Render the metadata page.


Instruction:
Explain what the following Python 3 code does
def render_image(**kwargs):
    html = ''

    url = kwargs.get('url', None)
    if url:
        html = '<img'

        alt_text = kwargs.get('alt_text', None)
        if alt_text:
            html += ' alt="{}"'.format(alt_text)

        title = kwargs.get('title', None)
        if title:
            html += ' title="{}"'.format(title)

        html += ' src="{}">'.format(url)

    return html

Response: Render an image in a non - strict template block.


Instruction:
Implement a Python 3 function for
rendering a twitter embed.

Response:
def render_twitter(text, **kwargs):
    """
    Strict template block for rendering twitter embeds.
    """
    author = render_author(**kwargs['author'])
    metadata = render_metadata(**kwargs['metadata'])
    image = render_image(**kwargs['image'])

    html = """

            {author}

{text}

            {metadata}
            {image}

    """.format(
        author=author,
        text=text,
        metadata=metadata,
        image=image
    ).strip()

    return html


Instruction:
How would you code a function in Python 3 to
get the model params from the given LAB directory.

Response:
def get_model(LAB_DIR):
    """ Cannon model params """
    coeffs = np.load("%s/coeffs.npz" %LAB_DIR)['arr_0']
    scatters = np.load("%s/scatters.npz" %LAB_DIR)['arr_0']
    chisqs = np.load("%s/chisqs.npz" %LAB_DIR)['arr_0']
    pivots = np.load("%s/pivots.npz" %LAB_DIR)['arr_0']
    return coeffs, scatters, chisqs, pivots


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_labels(ids_find):
    a = pyfits.open("%s/lamost_catalog_full.fits" %LAB_DIR)
    data = a[1].data
    a.close()
    id_all = data['lamost_id']
    id_all = np.array(id_all)
    id_all = np.array([val.strip() for val in id_all])
    snr_all = data['cannon_snrg']
    chisq_all = data['cannon_chisq']
    teff = data['cannon_teff']
    logg = data['cannon_logg']
    feh = data['cannon_m_h']
    afe = data['cannon_alpha_m']
    ak = data['cannon_a_k']
    labels = np.vstack((teff,logg,feh,afe,ak))
    choose = np.in1d(id_all, ids_find)
    id_choose = id_all[choose]
    label_choose = labels[:,choose]
    snr_choose = snr_all[choose]
    chisq_choose = chisq_all[choose]
    inds = np.array([np.where(id_choose==val)[0][0] for val in ids_find])
    print(id_choose[inds][100])
    print(ids_find[100])
    return label_choose[:,inds], snr_choose[inds], chisq_choose[inds]

Response: Get labels for a list of ids


Instruction:
Create a Python 3 function for
getting the normalized spectra for the lamost model

Response:
def get_normed_spectra():
    """ Spectra to compare with models """
    wl = np.load("%s/wl.npz" %LAB_DIR)['arr_0']
    filenames = np.array(
            [SPEC_DIR + "/Spectra" + "/" + val for val in lamost_id])
    grid, fluxes, ivars, npix, SNRs = lamost.load_spectra(
            lamost_id, input_grid=wl)
    ds = dataset.Dataset(
            wl, lamost_id, fluxes, ivars, [1],
            lamost_id[0:2], fluxes[0:2], ivars[0:2])
    ds.continuum_normalize_gaussian_smoothing(L=50)
    np.savez(SPEC_DIR + "/" + "norm_flux.npz", ds.tr_flux)
    np.savez(SPEC_DIR + "/" + "norm_ivar.npz", ds.tr_ivar)
    return ds.tr_flux, ds.tr_ivar


Instruction:
How would you explain what the following Python 3 function does
def wget_files():
    for f in lamost_id:
        short = (f.split('-')[2]).split('_')[0]
        filename = "%s/%s.gz" %(short,f)
        DIR = "/Users/annaho/Data/Li_Giants/Spectra_APOKASC"
        searchfor = "%s/%s.gz" %(DIR,f)
        if glob.glob(searchfor):
            print("done")
        else:
            #print(searchfor)
            os.system(
                    "wget http://dr2.lamost.org/sas/fits/%s" %(filename))
            new_filename = filename.split("_")[0] + "_" + filename.split("_")[2]
            os.system(
                    "wget http://dr2.lamost.org/sas/fits/%s" %(new_filename))

Response: Pull the files from the LAMOST archive


Instruction:
Explain what the following Python 3 code does
def get_labels():
    cannon_teff = data['cannon_teff_2']
    cannon_logg = data['cannon_logg_2']
    cannon_m_h = data['cannon_m_h']
    cannon_alpha_m = data['cannon_alpha_m']
    cannon_a_k = data['cannon_a_k']
    labels = np.vstack(
            (cannon_teff, cannon_logg, cannon_m_h, cannon_alpha_m, cannon_a_k))
    cannon_chisq = data['cannon_chisq']
    np.savez(DATA_DIR + "chisq.npz", labels)
    np.savez(DATA_DIR + "labels.npz", labels)
    snrg = data['cannon_snrg'] # snrg * 3
    np.savez("snr.npz", snrg)
    return labels.T

Response: Get the labels of the Cannon model spectra


Instruction:
Can you generate a brief explanation for the following Python 3 code
def cannon_normalize(spec_raw):
    spec = np.array([spec_raw])
    wl = np.arange(0, spec.shape[1])
    w = continuum_normalization.gaussian_weight_matrix(wl, L=50)
    ivar = np.ones(spec.shape)*0.5
    cont = continuum_normalization._find_cont_gaussian_smooth(
            wl, spec, ivar, w)
    norm_flux, norm_ivar = continuum_normalization._cont_norm(
            spec, ivar, cont)
    return norm_flux[0]

Response: Normalize according to The Cannon


Instruction:
Can you generate a brief explanation for the following Python 3 code
def resample(grid, wl, flux):
    flux_rs = (interpolate.interp1d(wl, flux))(grid)
    return flux_rs

Response: Resample spectrum onto desired grid


Instruction:
Write a Python 3 script for
generating the gradient spectra for a Cannon object.

Response:
def gen_cannon_grad_spec(choose, coeffs, pivots):
    """ Generate Cannon gradient spectra

    Parameters
    ----------
    labels: default values for [teff, logg, feh, cfe, nfe, afe, ak]
    choose: val of cfe or nfe, whatever you're varying
    low: lowest val of cfe or nfe, whatever you're varying
    high: highest val of cfe or nfe, whatever you're varying
    """
    base_labels = [4800, 2.5, 0.03, 0.10, -0.17, -0.17, 0, -0.16,
            -0.13, -0.15, 0.13, 0.08, 0.17, -0.062]
    label_names = np.array(
            ['TEFF', 'LOGG', 'AK', 'Al', 'Ca', 'C', 'Fe', 'Mg', 'Mn',
            'Ni', 'N', 'O', 'Si', 'Ti'])
    label_atnum = np.array(
            [0, 1, -1, 13, 20, 6, 26, 12, 25, 28, 7, 8, 14, 22])
    # Generate Cannon gradient spectra
    ind = np.where(label_atnum==choose)[0][0]
    low_lab = copy.copy(base_labels)
    high = base_labels[ind]
    if choose > 0:
        low = base_labels[ind] - 0.2
    else: #temperature
        if choose != 0: print("warning...")
        low = base_labels[ind] - 200
    low_lab[ind] = low
    lvec = (train_model._get_lvec(np.array([low_lab]), pivots))[0]
    model_low = np.dot(coeffs, lvec)
    lvec = (train_model._get_lvec(np.array([base_labels]), pivots))[0]
    model_high = np.dot(coeffs, lvec)
    grad_spec = (model_high - model_low) / (high - low)
    return grad_spec


Instruction:
How would you code a function in Python 3 to
get model specting from the given atomic number

Response:
def get_model_spec_ting(atomic_number):
    """
    X_u_template[0:2] are teff, logg, vturb in km/s
    X_u_template[:,3] -> onward, put atomic number
    atomic_number is 6 for C, 7 for N
    """
    DATA_DIR = "/Users/annaho/Data/LAMOST/Mass_And_Age"
    temp = np.load("%s/X_u_template_KGh_res=1800.npz" %DATA_DIR)
    X_u_template = temp["X_u_template"]
    wl = temp["wavelength"]
    grad_spec = X_u_template[:,atomic_number]
    return wl, grad_spec


Instruction:
Create a Python 3 function for
calculating the residuals of the dataset and model object using the dataset and model object ds and model object m.

Response:
def get_residuals(ds, m):
    """ Using the dataset and model object, calculate the residuals and return

    Parameters
    ----------
    ds: dataset object
    m: model object
    Return
    ------
    residuals: array of residuals, spec minus model spec
    """
    model_spectra = get_model_spectra(ds, m)
    resid = ds.test_flux - model_spectra
    return resid


Instruction:
Can you implement a function in Python 3 that
loads the model object for the

Response:
def load_model():
    """ Load the model

    Parameters
    ----------
    direc: directory with all of the model files

    Returns
    -------
    m: model object
    """
    direc = "/home/annaho/TheCannon/code/lamost/mass_age/cn"
    m = model.CannonModel(2)
    m.coeffs = np.load(direc + "/coeffs.npz")['arr_0'][0:3626,:] # no cols
    m.scatters = np.load(direc + "/scatters.npz")['arr_0'][0:3626] # no cols
    m.chisqs = np.load(direc + "/chisqs.npz")['arr_0'][0:3626] # no cols
    m.pivots = np.load(direc + "/pivots.npz")['arr_0']
    return m


Instruction:
How would you implement a function in Python 3 that
loads the dataset for a single date

Response:
def load_dataset(date):
    """ Load the dataset for a single date

    Parameters
    ----------
    date: the date (string) for which to load the data & dataset

    Returns
    -------
    ds: the dataset object
    """
    LAB_DIR = "/home/annaho/TheCannon/data/lamost"
    WL_DIR = "/home/annaho/TheCannon/code/lamost/mass_age/cn"
    SPEC_DIR = "/home/annaho/TheCannon/code/apogee_lamost/xcalib_4labels/output"
    wl = np.load(WL_DIR + "/wl_cols.npz")['arr_0'][0:3626] # no cols
    ds = dataset.Dataset(wl, [], [], [], [], [], [], [])
    test_label = np.load("%s/%s_all_cannon_labels.npz" %(LAB_DIR,date))['arr_0']
    ds.test_label_vals = test_label
    a = np.load("%s/%s_norm.npz" %(SPEC_DIR,date))
    ds.test_flux = a['arr_0']
    ds.test_ivar = a['arr_1']
    ds.test_ID = np.load("%s/%s_ids.npz" %(SPEC_DIR,date))['arr_0']
    return ds


Instruction:
Write a Python 3 function that can
fit a Gaussian to the data

Response:
def fit_gaussian(x, y, yerr, p0):
    """ Fit a Gaussian to the data """
    try:
        popt, pcov = curve_fit(gaussian, x, y, sigma=yerr, p0=p0, absolute_sigma=True)
    except RuntimeError:
        return [0],[0]
    return popt, pcov


Instruction:
Can you generate a brief explanation for the following Python 3 code
def select(yerrs, amps, amp_errs, widths):
    keep_1 = np.logical_and(amps < 0, widths > 1)
    keep_2 = np.logical_and(np.abs(amps) > 3*yerrs, amp_errs < 3*np.abs(amps))
    keep = np.logical_and(keep_1, keep_2)
    return keep

Response: selects an object based on the given error and amplitudes


Instruction:
Can you generate the documentation for the following Python 3 function
def run_all():
    DATA_DIR = "/home/annaho/TheCannon/code/apogee_lamost/xcalib_4labels"
    dates = os.listdir("/home/share/LAMOST/DR2/DR2_release")
    dates = np.array(dates)
    dates = np.delete(dates, np.where(dates=='.directory')[0][0])
    dates = np.delete(dates, np.where(dates=='all_folders.list')[0][0])
    dates = np.delete(dates, np.where(dates=='dr2.lis')[0][0])

    for date in dates:
        if glob.glob("*%s*.txt" %date):
            print("%s done" %date)
        else:
            print("running %s" %date)
            run_one_date(date)

Response: Load the data that we re using to search for Li - rich giants.


Instruction:
Explain what the following Python 3 code does
def gen_cannon_grad_spec(base_labels, choose, low, high, coeffs, pivots):
    # Generate Cannon gradient spectra
    low_lab = copy.copy(base_labels)
    low_lab[choose] = low
    lvec = (train_model._get_lvec(np.array([low_lab]), pivots))[0]
    model_low = np.dot(coeffs, lvec)
    high_lab = copy.copy(base_labels)
    high_lab[choose] = high
    lvec = (train_model._get_lvec(np.array([high_lab]), pivots))[0]
    model_high = np.dot(coeffs, lvec)
    grad_spec = (model_high - model_low) / (high - low)
    return grad_spec

Response: Generate the gradient of a Cannon graph based on the base_labels and coeffs.


Instruction:
Explain what the following Python 3 code does
def get_err(snr):

    quad_terms = np.array(
            [3.11e-3, 1.10e-5, 6.95e-6, 5.05e-6, 4.65e-6, 4.10e-6])
    lin_terms = np.array(
            [-0.869, -2.07e-3, -1.40e-3, -1.03e-3, -1.13e-3, -7.29e-4])
    consts = np.array([104, 0.200, 0.117, 0.114, 0.156, 0.0624])
    err = quad_terms[:,None] * snr**2 + lin_terms[:,None] * snr + consts[:,None]

    # find the minimum of the quadratic function
    min_snr = -lin_terms / (2*quad_terms)
    min_err = quad_terms * min_snr**2 + lin_terms * min_snr + consts
    mask = (snr[:,None] > min_snr).T
    for i in range(0,len(min_err)):
        err[i][mask[i]] = min_err[i]

    return err

Response: Get approximate scatters from SNR.


Instruction:
Create a Python 3 function to
get colors from a catalog

Response:
def get_colors(catalog):
    """
    Pull colors from catalog

    Parameters
    ----------
    catalog: filename
    """
    print("Get Colors")
    a = pyfits.open(catalog)
    data = a[1].data
    a.close()
    all_ids = data['LAMOST_ID_1']
    all_ids = np.array([val.strip() for val in all_ids])
    # G magnitude
    gmag = data['gpmag']
    gmag_err = data['e_gpmag']
    # R magnitude
    rmag = data['rpmag']
    rmag_err = data['e_rpmag']
    # I magnitude
    imag = data['ipmag']
    imag_err = data['e_ipmag']
    # W1
    W1 = data['W1mag']
    W1_err = data['e_W1mag']
    # W1
    W2 = data['W2mag']
    W2_err = data['e_W2mag']
    # J magnitude
    Jmag = data['Jmag']
    Jmag_err = data['e_Jmag']
    # H magnitude
    Hmag = data['Hmag']
    Hmag_err = data['e_Hmag']
    # K magnitude
    Kmag = data['Kmag']
    Kmag_err = data['e_Kmag']
    # Stack
    mag = np.vstack((
        gmag, rmag, imag, Jmag, Hmag, Kmag, W2, W1)) # 8, nobj
    mag_err = np.vstack((
        gmag_err, rmag_err, imag_err, Jmag_err,
        Hmag_err, Kmag_err, W2_err, W1_err))
    # Make g-r, r-i, i-J, etc
    col = mag[:-1] - mag[1:]
    col_ivar = 1/(mag_err[:-1]**2 + mag_err[1:]**2)

    # There's something wrong with the i-band, I think..so the second color r-i
    #bad = col[:,1] < 0.0
    #col_ivar[bad] = 0.0

    return all_ids, col, col_ivar


Instruction:
How would you explain what the following Python 3 function does
def draw_spectra(md, ds):
    coeffs_all, covs, scatters, red_chisqs, pivots, label_vector = model.model
    nstars = len(dataset.test_SNR)
    cannon_flux = np.zeros(dataset.test_flux.shape)
    cannon_ivar = np.zeros(dataset.test_ivar.shape)
    for i in range(nstars):
        x = label_vector[:,i,:]
        spec_fit = np.einsum('ij, ij->i', x, coeffs_all)
        cannon_flux[i,:] = spec_fit
        bad = dataset.test_ivar[i,:] == SMALL**2
        cannon_ivar[i,:][~bad] = 1. / scatters[~bad] ** 2
    return cannon_flux, cannon_ivar

Response: Generates the best fit spectra for all the test objects


Instruction:
Can you tell what is the following Python 3 function doing
def overlay_spectra(model, dataset):
    best_flux, best_ivar = draw_spectra(model, dataset)
    coeffs_all, covs, scatters, all_chisqs, pivots, label_vector = model.model

    # Overplot original spectra with best-fit spectra
    print("Overplotting spectra for ten random stars")
    res = dataset.test_flux-best_flux
    lambdas = dataset.wl
    npix = len(lambdas)
    nstars = best_flux.shape[0]
    pickstars = []
    for i in range(10):
        pickstars.append(random.randrange(0, nstars-1))
    for i in pickstars:
        print("Star %s" % i)
        ID = dataset.test_ID[i]
        spec_orig = dataset.test_flux[i,:]
        bad = dataset.test_flux[i,:] == 0
        lambdas = np.ma.array(lambdas, mask=bad, dtype=float)
        npix = len(lambdas.compressed())
        spec_orig = np.ma.array(dataset.test_flux[i,:], mask=bad)
        spec_fit = np.ma.array(best_flux[i,:], mask=bad)
        ivars_orig = np.ma.array(dataset.test_ivar[i,:], mask=bad)
        ivars_fit = np.ma.array(best_ivar[i,:], mask=bad)
        red_chisq = np.sum(all_chisqs[:,i], axis=0) / (npix - coeffs_all.shape[1])
        red_chisq = np.round(red_chisq, 2)
        fig,axarr = plt.subplots(2)
        ax1 = axarr[0]
        im = ax1.scatter(lambdas, spec_orig, label="Orig Spec",
                         c=1 / np.sqrt(ivars_orig), s=10)
        ax1.scatter(lambdas, spec_fit, label="Cannon Spec", c='r', s=10)
        ax1.errorbar(lambdas, spec_fit,
                     yerr=1/np.sqrt(ivars_fit), fmt='ro', ms=1, alpha=0.7)
        ax1.set_xlabel(r"Wavelength $\lambda (\AA)$")
        ax1.set_ylabel("Normalized flux")
        ax1.set_title("Spectrum Fit: %s" % ID)
        ax1.set_title("Spectrum Fit")
        ax1.set_xlim(min(lambdas.compressed())-10, max(lambdas.compressed())+10)
        ax1.legend(loc='lower center', fancybox=True, shadow=True)
        ax2 = axarr[1]
        ax2.scatter(spec_orig, spec_fit, c=1/np.sqrt(ivars_orig), alpha=0.7)
        ax2.errorbar(spec_orig, spec_fit, yerr=1 / np.sqrt(ivars_fit),
                     ecolor='k', fmt="none", ms=1, alpha=0.7)
        #fig.subplots_adjust(right=0.8)
        #cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.7])
        fig.colorbar()
        #fig.colorbar(
        #        im, cax=cbar_ax,
        #        label="Uncertainties on the Fluxes from the Original Spectrum")
        xlims = ax2.get_xlim()
        ylims = ax2.get_ylim()
        lims = [np.min([xlims, ylims]), np.max([xlims, ylims])]
        ax2.plot(lims, lims, 'k-', alpha=0.75)
        textstr = "Red Chi Sq: %s" % red_chisq
        props = dict(boxstyle='round', facecolor='palevioletred', alpha=0.5)
        ax2.text(0.05, 0.95, textstr, transform=ax2.transAxes, fontsize=14,
                 verticalalignment='top', bbox=props)
        ax2.set_xlim(xlims)
        ax2.set_ylim(ylims)
        ax2.set_xlabel("Orig Fluxes")
        ax2.set_ylabel("Fitted Fluxes")
        plt.tight_layout()
        filename = "best_fit_spec_Star%s.png" % i
        print("Saved as %s" % filename)
        fig.savefig(filename)
        plt.close(fig)

Response: Overlays the original spectra with the best fit spectra.


Instruction:
Make a summary of the following Python 3 code
def residuals(cannon_set, dataset):
    print("Stacking spectrum fit residuals")
    res = dataset.test_fluxes - cannon_set.test_fluxes
    bad = dataset.test_ivars == SMALL**2
    err = np.zeros(len(dataset.test_ivars))
    err = np.sqrt(1. / dataset.test_ivars + 1. / cannon_set.test_ivars)
    res_norm = res / err
    res_norm = np.ma.array(res_norm,
                           mask=(np.ones_like(res_norm) *
                                 (np.std(res_norm,axis=0) == 0)))
    res_norm = np.ma.compress_cols(res_norm)

    for i in range(len(cannon_set.get_plotting_labels())):
        label_name = cannon_set.get_plotting_labels()[i]
        print("Plotting residuals sorted by %s" % label_name)
        label_vals = cannon_set.tr_label_vals[:,i]
        sorted_res = res_norm[np.argsort(label_vals)]
        mu = np.mean(sorted_res.flatten())
        sigma = np.std(sorted_res.flatten())
        left, width = 0.1, 0.65
        bottom, height = 0.1, 0.65
        bottom_h = left_h = left+width+0.1
        rect_scatter = [left, bottom, width, height]
        rect_histx = [left, bottom_h, width, 0.1]
        rect_histy = [left_h, bottom, 0.1, height]
        plt.figure()
        axScatter = plt.axes(rect_scatter)
        axHistx = plt.axes(rect_histx)
        axHisty = plt.axes(rect_histy)
        im = axScatter.imshow(sorted_res, cmap=plt.cm.bwr_r,
                              interpolation="nearest", vmin=mu - 3. * sigma,
                              vmax=mu + 3. * sigma, aspect='auto',
                              origin='lower', extent=[0, len(dataset.wl),
                                                      min(label_vals),
                                                      max(label_vals)])
        cax, kw = colorbar.make_axes(axScatter.axes, location='bottom')
        plt.colorbar(im, cax=cax, orientation='horizontal')
        axScatter.set_title(
                r"Spectral Residuals Sorted by ${0:s}$".format(label_name))
        axScatter.set_xlabel("Pixels")
        axScatter.set_ylabel(r"$%s$" % label_name)
        axHisty.hist(np.std(res_norm,axis=1)[~np.isnan(np.std(res_norm, axis=1))], orientation='horizontal', range=[0,2])
        axHisty.axhline(y=1, c='k', linewidth=3, label="y=1")
        axHisty.legend(bbox_to_anchor=(0., 0.8, 1., .102),
                       prop={'family':'serif', 'size':'small'})
        axHisty.text(1.0, 0.5, "Distribution of Stdev of Star Residuals",
                     verticalalignment='center', transform=axHisty.transAxes,
                     rotation=270)
        axHisty.set_ylabel("Standard Deviation")
        start, end = axHisty.get_xlim()
        axHisty.xaxis.set_ticks(np.linspace(start, end, 3))
        axHisty.set_xlabel("Number of Stars")
        axHisty.xaxis.set_label_position("top")
        axHistx.hist(np.std(res_norm, axis=0)[~np.isnan(np.std(res_norm, axis=0))], range=[0.8,1.1])
        axHistx.axvline(x=1, c='k', linewidth=3, label="x=1")
        axHistx.set_title("Distribution of Stdev of Pixel Residuals")
        axHistx.set_xlabel("Standard Deviation")
        axHistx.set_ylabel("Number of Pixels")
        start, end = axHistx.get_ylim()
        axHistx.yaxis.set_ticks(np.linspace(start, end, 3))
        axHistx.legend()
        filename = "residuals_sorted_by_label_%s.png" % i
        plt.savefig(filename)
        print("File saved as %s" % filename)
        plt.close()

    # Auto-correlation of mean residuals
    print("Plotting Auto-Correlation of Mean Residuals")
    mean_res = res_norm.mean(axis=0)
    autocorr = np.correlate(mean_res, mean_res, mode="full")
    pkwidth = int(len(autocorr)/2-np.argmin(autocorr))
    xmin = int(len(autocorr)/2)-pkwidth
    xmax = int(len(autocorr)/2)+pkwidth
    zoom_x = np.linspace(xmin, xmax, len(autocorr[xmin:xmax]))
    fig, axarr = plt.subplots(2)
    axarr[0].plot(autocorr)
    axarr[0].set_title("Autocorrelation of Mean Spectral Residual")
    axarr[0].set_xlabel("Lag (# Pixels)")
    axarr[0].set_ylabel("Autocorrelation")
    axarr[1].plot(zoom_x, autocorr[xmin:xmax])
    axarr[1].set_title("Central Peak, Zoomed")
    axarr[1].set_xlabel("Lag (# Pixels)")
    axarr[1].set_ylabel("Autocorrelation")
    filename = "residuals_autocorr.png"
    plt.savefig(filename)
    print("saved %s" % filename)
    plt.close()

Response: Stack spectrum fit residuals


Instruction:
How would you explain what the following Python 3 function does
def _find_contpix_given_cuts(f_cut, sig_cut, wl, fluxes, ivars):
    f_bar = np.median(fluxes, axis=0)
    sigma_f = np.var(fluxes, axis=0)
    bad = np.logical_and(f_bar==0, sigma_f==0)
    cont1 = np.abs(f_bar-1) <= f_cut
    cont2 = sigma_f <= sig_cut
    contmask = np.logical_and(cont1, cont2)
    contmask[bad] = False
    return contmask

Response: Find and return continuum pixels given the flux and sigma cuts.


Instruction:
Create a Python 3 function to
find continuum pix in the spec meeting a set target fraction.

Response:
def _find_contpix(wl, fluxes, ivars, target_frac):
    """ Find continuum pix in spec, meeting a set target fraction

    Parameters
    ----------
    wl: numpy ndarray
        rest-frame wavelength vector

    fluxes: numpy ndarray
        pixel intensities

    ivars: numpy ndarray
        inverse variances, parallel to fluxes

    target_frac: float
        the fraction of pixels in spectrum desired to be continuum

    Returns
    -------
    contmask: boolean numpy ndarray
        True corresponds to continuum pixels
    """
    print("Target frac: %s" %(target_frac))
    bad1 = np.median(ivars, axis=0) == SMALL
    bad2 = np.var(ivars, axis=0) == 0
    bad = np.logical_and(bad1, bad2)
    npixels = len(wl)-sum(bad)
    f_cut = 0.0001
    stepsize = 0.0001
    sig_cut = 0.0001
    contmask = _find_contpix_given_cuts(f_cut, sig_cut, wl, fluxes, ivars)
    if npixels > 0:
        frac = sum(contmask)/float(npixels)
    else:
        frac = 0
    while (frac < target_frac):
        f_cut += stepsize
        sig_cut += stepsize
        contmask = _find_contpix_given_cuts(f_cut, sig_cut, wl, fluxes, ivars)
        if npixels > 0:
            frac = sum(contmask)/float(npixels)
        else:
            frac = 0
    if frac > 0.10*npixels:
        print("Warning: Over 10% of pixels identified as continuum.")
    print("%s out of %s pixels identified as continuum" %(sum(contmask),
                                                          npixels))
    print("Cuts: f_cut %s, sig_cut %s" %(f_cut, sig_cut))
    return contmask


Instruction:
Make a summary of the following Python 3 code
def _find_contpix_regions(wl, fluxes, ivars, frac, ranges):
    contmask = np.zeros(len(wl), dtype=bool)
    for chunk in ranges:
        start = chunk[0]
        stop = chunk[1]
        contmask[start:stop] = _find_contpix(
                wl[start:stop], fluxes[:,start:stop], ivars[:,start:stop], frac)
    return contmask

Response: Find continuum pix in a spectrum split into chunks


Instruction:
Create a Python 3 function to
load the reference data and assign each object a random integer from 0 to 7. Save the IDs.

Response:
def group_data():
    """ Load the reference data, and assign each object
    a random integer from 0 to 7. Save the IDs. """

    tr_obj = np.load("%s/ref_id.npz" %direc_ref)['arr_0']
    groups = np.random.randint(0, 8, size=len(tr_obj))
    np.savez("ref_groups.npz", groups)


Instruction:
Explain what the following Python 3 code does
def train(ds, ii):
    print("Loading model")
    m = model.CannonModel(2)
    print("Training...")
    m.fit(ds)
    np.savez("./ex%s_coeffs.npz" %ii, m.coeffs)
    np.savez("./ex%s_scatters.npz" %ii, m.scatters)
    np.savez("./ex%s_chisqs.npz" %ii, m.chisqs)
    np.savez("./ex%s_pivots.npz" %ii, m.pivots)
    fig = m.diagnostics_leading_coeffs(ds)
    plt.savefig("ex%s_leading_coeffs.png" %ii)
    # m.diagnostics_leading_coeffs_triangle(ds)
    # m.diagnostics_plot_chisq(ds)
    return m

Response: Train the Cannon model on a dataset object.


Instruction:
Here you have a function in Python 3, explain what it does
def xvalidate():

    print("Loading data")
    groups = np.load("ref_groups.npz")['arr_0']
    ref_label = np.load("%s/ref_label.npz" %direc_ref)['arr_0']
    ref_id = np.load("%s/ref_id.npz" %direc_ref)['arr_0']
    ref_flux = np.load("%s/ref_flux.npz" %direc_ref)['arr_0']
    ref_ivar = np.load("%s/ref_ivar.npz" %direc_ref)['arr_0']
    wl = np.load("%s/wl.npz" %direc_ref)['arr_0']

    num_models = 8

    for ii in np.arange(num_models):
        print("Leaving out group %s" %ii)
        train_on = groups != ii
        test_on = groups == ii

        tr_label = ref_label[train_on]
        tr_id = ref_id[train_on]
        tr_flux = ref_flux[train_on]
        tr_ivar = ref_ivar[train_on]
        print("Training on %s objects" %len(tr_id))
        test_label = ref_label[test_on]
        test_id = ref_id[test_on]
        test_flux = ref_flux[test_on]
        test_ivar = ref_ivar[test_on]
        print("Testing on %s objects" %len(test_id))

        print("Loading dataset...")
        ds = dataset.Dataset(
                wl, tr_id, tr_flux, tr_ivar, tr_label,
                test_id, test_flux, test_ivar)
        ds.set_label_names(
                ['T_{eff}', '\log g', '[M/H]', '[\\alpha/Fe]', 'AKWISE'])
        fig = ds.diagnostics_SNR()
        plt.savefig("ex%s_SNR.png" %ii)
        fig = ds.diagnostics_ref_labels()
        plt.savefig("ex%s_ref_label_triangle.png" %ii)
        np.savez("ex%s_tr_snr.npz" %ii, ds.tr_SNR)

        # train a model
        m = train(ds, ii)

        # test step
        ds.tr_label = test_label # to compare the results
        test(ds, m, ii)

Response: Train a model on the remaining 1 / 8 of the sample.


Instruction:
Implement a Python 3 function for
calculating the weighted standard deviation of a sequence of values.

Response:
def weighted_std(values, weights):
    """ Calculate standard deviation weighted by errors """
    average = np.average(values, weights=weights)
    variance = np.average((values-average)**2, weights=weights)
    return np.sqrt(variance)


Instruction:
Create a Python 3 function to
estimate the scatter in a region of the spectrum taken to be continuum

Response:
def estimate_noise(fluxes, contmask):
    """ Estimate the scatter in a region of the spectrum
    taken to be continuum """
    nstars = fluxes.shape[0]
    scatter = np.zeros(nstars)
    for i,spec in enumerate(fluxes):
        cont = spec[contmask]
        scatter[i] = stats.funcs.mad_std(cont)
    return scatter


Instruction:
Make a summary of the following Python 3 code
def load_ref_spectra():
    data_dir = "/Users/annaho/Data/AAOmega/ref_spectra"
    # Load the files & count the number of training objects
    ff = glob.glob("%s/*.txt" %data_dir)
    nstars = len(ff)
    print("We have %s training objects" %nstars)

    # Read the first file to get the wavelength array
    f = ff[0]
    data = Table.read(f, format="ascii.fast_no_header")
    wl = data['col1']
    npix = len(wl)
    print("We have %s pixels" %npix)

    tr_flux = np.zeros((nstars,npix))
    tr_ivar = np.zeros(tr_flux.shape)

    for i,f in enumerate(ff):
        data = Table.read(f, format="ascii.fast_no_header")
        flux = data['col2']
        tr_flux[i,:] = flux
        sigma = data['col3']
        tr_ivar[i,:] = 1.0 / sigma**2

    return np.array(ff), wl, tr_flux, tr_ivar

Response: Load the training spectra from the first file in the directory and return the wavelength array


Instruction:
Can you write a function in Python 3 where it
loads the data for the AAOmega object.

Response:
def load_data():
    data_dir = "/Users/annaho/Data/AAOmega"
    out_dir = "%s/%s" %(data_dir, "Run_13_July")

    """ Use all the above functions to set data up for The Cannon """
    ff, wl, tr_flux, tr_ivar = load_ref_spectra()

    """ pick one that doesn't have extra dead pixels """
    skylines = tr_ivar[4,:] # should be the same across all obj
    np.savez("%s/skylines.npz" %out_dir, skylines)

    contmask = np.load("%s/contmask_regions.npz" %data_dir)['arr_0']
    scatter = estimate_noise(tr_flux, contmask)
    ids, labels = load_labels()

    # Select the objects in the catalog corresponding to the files
    inds = []
    ff_short = []
    for fname in ff:
        val = fname.split("/")[-1]
        short = (val.split('.')[0] + '.' + val.split('.')[1])
        ff_short.append(short)
        if short in ids:
            ind = np.where(ids==short)[0][0]
            inds.append(ind)

    # choose the labels
    tr_id = ids[inds]
    tr_label = labels[inds]

    # find the corresponding spectra
    ff_short = np.array(ff_short)
    inds = np.array([np.where(ff_short==val)[0][0] for val in tr_id])
    tr_flux_choose = tr_flux[inds]
    tr_ivar_choose = tr_ivar[inds]
    scatter_choose = scatter[inds]
    np.savez("%s/wl.npz" %out_dir, wl)
    np.savez("%s/ref_id_all.npz" %out_dir, tr_id)
    np.savez("%s/ref_flux_all.npz" %out_dir, tr_flux_choose)
    np.savez("%s/ref_ivar_all.npz" %out_dir, tr_ivar_choose)
    np.savez("%s/ref_label_all.npz" %out_dir, tr_label)
    np.savez("%s/ref_spec_scat_all.npz" %out_dir, scatter_choose)

    # now, the test spectra
    test_id, test_flux = load_test_spectra()
    scatter = estimate_noise(test_flux, contmask)
    np.savez("%s/test_id.npz" %out_dir, test_id)
    np.savez("%s/test_flux.npz" %out_dir, test_flux)
    np.savez("%s/test_spec_scat.npz" %out_dir, scatter)


Instruction:
Create a Python 3 function for
taking the scatters and skylines and make final ivars

Response:
def make_full_ivar():
    """ take the scatters and skylines and make final ivars """

    # skylines come as an ivar
    # don't use them for now, because I don't really trust them...
    # skylines = np.load("%s/skylines.npz" %DATA_DIR)['arr_0']

    ref_flux = np.load("%s/ref_flux_all.npz" %DATA_DIR)['arr_0']
    ref_scat = np.load("%s/ref_spec_scat_all.npz" %DATA_DIR)['arr_0']
    test_flux = np.load("%s/test_flux.npz" %DATA_DIR)['arr_0']
    test_scat = np.load("%s/test_spec_scat.npz" %DATA_DIR)['arr_0']
    ref_ivar = np.ones(ref_flux.shape) / ref_scat[:,None]**2
    test_ivar = np.ones(test_flux.shape) / test_scat[:,None]**2

    # ref_ivar = (ref_ivar_temp * skylines[None,:]) / (ref_ivar_temp + skylines)
    # test_ivar = (test_ivar_temp * skylines[None,:]) / (test_ivar_temp + skylines)

    ref_bad = np.logical_or(ref_flux <= 0, ref_flux > 1.1)
    test_bad = np.logical_or(test_flux <= 0, test_flux > 1.1)
    SMALL = 1.0 / 1000000000.0
    ref_ivar[ref_bad] = SMALL
    test_ivar[test_bad] = SMALL
    np.savez("%s/ref_ivar_corr.npz" %DATA_DIR, ref_ivar)
    np.savez("%s/test_ivar_corr.npz" %DATA_DIR, test_ivar)


Instruction:
Given the following Python 3 function, write the documentation
def _sinusoid(x, p, L, y):
    N = int(len(p)/2)
    n = np.linspace(0, N, N+1)
    k = n*np.pi/L
    func = 0
    for n in range(0, N):
        func += p[2*n]*np.sin(k[n]*x)+p[2*n+1]*np.cos(k[n]*x)
    return func

Response: Returns the sinusoid contfunc evaluated at input x for the continuum.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _weighted_median(values, weights, quantile):
    sindx = np.argsort(values)
    cvalues = 1. * np.cumsum(weights[sindx])
    if cvalues[-1] == 0: # means all the values are 0
        return values[0]
    cvalues = cvalues / cvalues[-1] # div by largest value
    foo = sindx[cvalues > quantile]
    if len(foo) == 0:
        return values[0]
    indx = foo[0]
    return values[indx]

Response: Calculate a weighted median of a set of values


Instruction:
How would you explain what the following Python 3 function does
def _find_cont_gaussian_smooth(wl, fluxes, ivars, w):
    print("Finding the continuum")
    bot = np.dot(ivars, w.T)
    top = np.dot(fluxes*ivars, w.T)
    bad = bot == 0
    cont = np.zeros(top.shape)
    cont[~bad] = top[~bad] / bot[~bad]
    return cont

Response: Finds the weighted mean block of spectra with Gaussian smoothed fluxes.


Instruction:
Can you tell what is the following Python 3 function doing
def _cont_norm_gaussian_smooth(dataset, L):
    print("Gaussian smoothing the entire dataset...")
    w = gaussian_weight_matrix(dataset.wl, L)

    print("Gaussian smoothing the training set")
    cont = _find_cont_gaussian_smooth(
            dataset.wl, dataset.tr_flux, dataset.tr_ivar, w)
    norm_tr_flux, norm_tr_ivar = _cont_norm(
            dataset.tr_flux, dataset.tr_ivar, cont)
    print("Gaussian smoothing the test set")
    cont = _find_cont_gaussian_smooth(
            dataset.wl, dataset.test_flux, dataset.test_ivar, w)
    norm_test_flux, norm_test_ivar = _cont_norm(
            dataset.test_flux, dataset.test_ivar, cont)
    return norm_tr_flux, norm_tr_ivar, norm_test_flux, norm_test_ivar

Response: Continuum normalize by dividing by a Gaussian - weighted smoothed spectrum


Instruction:
Can you create a Python 3 function that
fits a continuum to a single segment of spectra.

Response:
def _find_cont_fitfunc(fluxes, ivars, contmask, deg, ffunc, n_proc=1):
    """ Fit a continuum to a continuum pixels in a segment of spectra

    Functional form can be either sinusoid or chebyshev, with specified degree

    Parameters
    ----------
    fluxes: numpy ndarray of shape (nstars, npixels)
        training set or test set pixel intensities

    ivars: numpy ndarray of shape (nstars, npixels)
        inverse variances, parallel to fluxes

    contmask: numpy ndarray of length (npixels)
        boolean pixel mask, True indicates that pixel is continuum

    deg: int
        degree of fitting function

    ffunc: str
        type of fitting function, chebyshev or sinusoid

    Returns
    -------
    cont: numpy ndarray of shape (nstars, npixels)
        the continuum, parallel to fluxes
    """
    nstars = fluxes.shape[0]
    npixels = fluxes.shape[1]
    cont = np.zeros(fluxes.shape)

    if n_proc == 1:
        for jj in range(nstars):
            flux = fluxes[jj,:]
            ivar = ivars[jj,:]
            pix = np.arange(0, npixels)
            y = flux[contmask]
            x = pix[contmask]
            yivar = ivar[contmask]
            yivar[yivar == 0] = SMALL**2
            if ffunc=="sinusoid":
                p0 = np.ones(deg*2) # one for cos, one for sin
                L = max(x)-min(x)
                pcont_func = _partial_func(_sinusoid, L=L, y=flux)
                popt, pcov = opt.curve_fit(pcont_func, x, y, p0=p0,
                                           sigma=1./np.sqrt(yivar))
            elif ffunc=="chebyshev":
                fit = np.polynomial.chebyshev.Chebyshev.fit(x=x,y=y,w=yivar,deg=deg)
            for element in pix:
                if ffunc=="sinusoid":
                    cont[jj,element] = _sinusoid(element, popt, L=L, y=flux)
                elif ffunc=="chebyshev":
                    cont[jj,element] = fit(element)
    else:
        # start mp.Pool
        pool = mp.Pool(processes=n_proc)
        mp_results = []
        for i in xrange(nstars):
            mp_results.append(pool.apply_async(\
                _find_cont_fitfunc,
                (fluxes[i, :].reshape((1, -1)),
                 ivars[i, :].reshape((1, -1)),
                 contmask[:]),
                {'deg':deg, 'ffunc':ffunc}))
        # close mp.Pool
        pool.close()
        pool.join()

        cont = np.array([mp_results[i].get().flatten() for i in xrange(nstars)])

    return cont


Instruction:
How would you code a function in Python 3 to
find the continuum in the given set of fluxes and the given mask.

Response:
def _find_cont_fitfunc_regions(fluxes, ivars, contmask, deg, ranges, ffunc,
                               n_proc=1):
    """ Run fit_cont, dealing with spectrum in regions or chunks

    This is useful if a spectrum has gaps.

    Parameters
    ----------
    fluxes: ndarray of shape (nstars, npixels)
        training set or test set pixel intensities

    ivars: numpy ndarray of shape (nstars, npixels)
        inverse variances, parallel to fluxes

    contmask: numpy ndarray of length (npixels)
        boolean pixel mask, True indicates that pixel is continuum

    deg: int
        degree of fitting function

    ffunc: str
        type of fitting function, chebyshev or sinusoid

    Returns
    -------
    cont: numpy ndarray of shape (nstars, npixels)
        the continuum, parallel to fluxes
    """
    nstars = fluxes.shape[0]
    npixels = fluxes.shape[1]
    cont = np.zeros(fluxes.shape)
    for chunk in ranges:
        start = chunk[0]
        stop = chunk[1]
        if ffunc=="chebyshev":
            output = _find_cont_fitfunc(fluxes[:,start:stop],
                                        ivars[:,start:stop],
                                        contmask[start:stop],
                                        deg=deg, ffunc="chebyshev",
                                        n_proc=n_proc)
        elif ffunc=="sinusoid":
            output = _find_cont_fitfunc(fluxes[:,start:stop],
                                        ivars[:,start:stop],
                                        contmask[start:stop],
                                        deg=deg, ffunc="sinusoid",
                                        n_proc=n_proc)
        cont[:, start:stop] = output

    return cont


Instruction:
Explain what the following Python 3 code does
def _find_cont_running_quantile(wl, fluxes, ivars, q, delta_lambda,
                                verbose=False):
    """ Perform continuum normalization using a running quantile

    Parameters
    ----------
    wl: numpy ndarray
        wavelength vector
    fluxes: numpy ndarray of shape (nstars, npixels)
        pixel intensities
    ivars: numpy ndarray of shape (nstars, npixels)
        inverse variances, parallel to fluxes
    q: float
        the desired quantile cut
    delta_lambda: int
        the number of pixels over which the median is calculated

    Output
    ------
    norm_fluxes: numpy ndarray of shape (nstars, npixels)
        normalized pixel intensities
    norm_ivars: numpy ndarray of shape (nstars, npixels)
        rescaled pixel invariances
    """
    cont = np.zeros(fluxes.shape)
    nstars = fluxes.shape[0]
    for jj in range(nstars):
        if verbose:
            print("cont_norm_q(): working on star [%s/%s]..." % (jj+1, nstars))
        flux = fluxes[jj,:]
        ivar = ivars[jj,:]
        for ll, lam in enumerate(wl):
            indx = (np.where(abs(wl-lam) < delta_lambda))[0]
            flux_cut = flux[indx]
            ivar_cut = ivar[indx]
            cont[jj, ll] = _weighted_median(flux_cut, ivar_cut, q)
    return cont

Response: Perform continuum normalization using a running quantile


Instruction:
Can you tell what is the following Python 3 function doing
def _cont_norm_running_quantile_mp(wl, fluxes, ivars, q, delta_lambda,
                                   n_proc=2, verbose=False):
    """
    The same as _cont_norm_running_quantile() above,
    but using multi-processing.

    Bo Zhang (NAOC)
    """
    nStar = fluxes.shape[0]

    # start mp.Pool
    mp_results = []
    pool = mp.Pool(processes=n_proc)
    for i in xrange(nStar):
        mp_results.append(pool.apply_async(\
            _find_cont_running_quantile,
            (wl, fluxes[i, :].reshape((1, -1)), ivars[i, :].reshape((1, -1)),
             q, delta_lambda), {'verbose': False}))
        if verbose:
            print('@Bo Zhang: continuum normalizing star [%d/%d] ...'\
                  % (i + 1, nStar))
    # close mp.Pool
    pool.close()
    pool.join()

    # reshape results --> cont
    cont = np.zeros_like(fluxes)
    for i in xrange(nStar):
        cont[i, :] = mp_results[i].get() #.flatten()
    norm_fluxes = np.ones(fluxes.shape)
    norm_fluxes[cont!=0] = fluxes[cont!=0] / cont[cont!=0]
    norm_ivars = cont**2 * ivars

    print('@Bo Zhang: continuum normalization finished!')
    return norm_fluxes, norm_ivars

Response: This function normalizes the running quantiles in a running process.


Instruction:
Write a Python 3 function that can
perform continuum normalization using running quantile

Response:
def _cont_norm_running_quantile_regions(wl, fluxes, ivars, q, delta_lambda,
                                        ranges, verbose=True):
    """ Perform continuum normalization using running quantile, for spectrum
    that comes in chunks
    """
    print("contnorm.py: continuum norm using running quantile")
    print("Taking spectra in %s chunks" % len(ranges))
    nstars = fluxes.shape[0]
    norm_fluxes = np.zeros(fluxes.shape)
    norm_ivars = np.zeros(ivars.shape)
    for chunk in ranges:
        start = chunk[0]
        stop = chunk[1]
        output = _cont_norm_running_quantile(
                wl[start:stop], fluxes[:,start:stop],
                ivars[:,start:stop], q, delta_lambda)
        norm_fluxes[:,start:stop] = output[0]
        norm_ivars[:,start:stop] = output[1]
    return norm_fluxes, norm_ivars


Instruction:
How would you code a function in Python 3 to
perform continuum normalization using running quantile for spectrum

Response:
def _cont_norm_running_quantile_regions_mp(wl, fluxes, ivars, q, delta_lambda,
                                           ranges, n_proc=2, verbose=False):
    """
    Perform continuum normalization using running quantile, for spectrum
    that comes in chunks.

    The same as _cont_norm_running_quantile_regions(),
    but using multi-processing.

    Bo Zhang (NAOC)
    """
    print("contnorm.py: continuum norm using running quantile")
    print("Taking spectra in %s chunks" % len(ranges))
    # nstars = fluxes.shape[0]
    nchunks = len(ranges)
    norm_fluxes = np.zeros(fluxes.shape)
    norm_ivars = np.zeros(ivars.shape)
    for i in xrange(nchunks):
        chunk = ranges[i, :]
        start = chunk[0]
        stop = chunk[1]
        if verbose:
            print('@Bo Zhang: Going to normalize Chunk [%d/%d], pixel:[%d, %d] ...'
                  % (i+1, nchunks, start, stop))
        output = _cont_norm_running_quantile_mp(
            wl[start:stop], fluxes[:, start:stop],
            ivars[:, start:stop], q, delta_lambda,
            n_proc=n_proc, verbose=verbose)
        norm_fluxes[:, start:stop] = output[0]
        norm_ivars[:, start:stop] = output[1]
    return norm_fluxes, norm_ivars


Instruction:
How would you implement a function in Python 3 that
performs continuum normalization for spectra in chunks

Response:
def _cont_norm_regions(fluxes, ivars, cont, ranges):
    """ Perform continuum normalization for spectra in chunks

    Useful for spectra that have gaps

    Parameters
    ---------
    fluxes: numpy ndarray
        pixel intensities
    ivars: numpy ndarray
        inverse variances, parallel to fluxes
    cont: numpy ndarray
        the continuum
    ranges: list or np ndarray
        the chunks that the spectrum should be split into

    Returns
    -------
    norm_fluxes: numpy ndarray
        normalized pixel intensities
    norm_ivars: numpy ndarray
        rescaled inverse variances
    """
    nstars = fluxes.shape[0]
    norm_fluxes = np.zeros(fluxes.shape)
    norm_ivars = np.zeros(ivars.shape)
    for chunk in ranges:
        start = chunk[0]
        stop = chunk[1]
        output = _cont_norm(fluxes[:,start:stop],
                           ivars[:,start:stop],
                           cont[:,start:stop])
        norm_fluxes[:,start:stop] = output[0]
        norm_ivars[:,start:stop] = output[1]
    for jj in range(nstars):
        bad = (norm_ivars[jj,:] == 0.)
        norm_fluxes[jj,:][bad] = 1.
    return norm_fluxes, norm_ivars


Instruction:
Implement a Python 3 function for
training the spectral model on the data set.

Response:
def train(self, ds):
        """ Run training step: solve for best-fit spectral model """
        if self.useErrors:
            self.coeffs, self.scatters, self.new_tr_labels, self.chisqs, self.pivots, self.scales = _train_model_new(ds)
        else:
            self.coeffs, self.scatters, self.chisqs, self.pivots, self.scales = _train_model(ds)


Instruction:
Make a summary of the following Python 3 code
def infer_spectra(self, ds):
        lvec_all = _get_lvec(ds.test_label_vals, self.pivots, self.scales, derivs=False)
        self.model_spectra = np.dot(lvec_all, self.coeffs.T)

Response: Infer the model spectra for the object holding the test set.


Instruction:
How would you explain what the following Python 3 function does
def plot_contpix(self, x, y, contpix_x, contpix_y, figname):
        fig, axarr = plt.subplots(2, sharex=True)
        plt.xlabel(r"Wavelength $\lambda (\AA)$")
        plt.xlim(min(x), max(x))
        ax = axarr[0]
        ax.step(x, y, where='mid', c='k', linewidth=0.3,
                label=r'$\theta_0$' + "= the leading fit coefficient")
        ax.scatter(contpix_x, contpix_y, s=1, color='r',
                label="continuum pixels")
        ax.legend(loc='lower right',
                prop={'family':'serif', 'size':'small'})
        ax.set_title("Baseline Spectrum with Continuum Pixels")
        ax.set_ylabel(r'$\theta_0$')
        ax = axarr[1]
        ax.step(x, y, where='mid', c='k', linewidth=0.3,
             label=r'$\theta_0$' + "= the leading fit coefficient")
        ax.scatter(contpix_x, contpix_y, s=1, color='r',
                label="continuum pixels")
        ax.set_title("Baseline Spectrum with Continuum Pixels, Zoomed")
        ax.legend(loc='upper right', prop={'family':'serif',
            'size':'small'})
        ax.set_ylabel(r'$\theta_0$')
        ax.set_ylim(0.95, 1.05)
        print("Diagnostic plot: fitted 0th order spec w/ cont pix")
        print("Saved as %s.png" % (figname))
        plt.savefig(figname)
        plt.close()

Response: Plot baseline spec with continuum pix overlaid


Instruction:
How would you code a function in Python 3 to
call plot_contpix once for each nth of the spectrum

Response:
def diagnostics_contpix(self, data, nchunks=10, fig = "baseline_spec_with_cont_pix"):
        """ Call plot_contpix once for each nth of the spectrum """
        if data.contmask is None:
            print("No contmask set")
        else:
            coeffs_all = self.coeffs
            wl = data.wl
            baseline_spec = coeffs_all[:,0]
            contmask = data.contmask
            contpix_x = wl[contmask]
            contpix_y = baseline_spec[contmask]
            rem = len(wl)%nchunks
            wl_split = np.array(np.split(wl[0:len(wl)-rem],nchunks))
            baseline_spec_split = np.array(
                    np.split(baseline_spec[0:len(wl)-rem],nchunks))
            nchunks = wl_split.shape[0]
            for i in range(nchunks):
                fig_chunk = fig + "_%s" %str(i)
                wl_chunk = wl_split[i,:]
                baseline_spec_chunk = baseline_spec_split[i,:]
                take = np.logical_and(
                        contpix_x>wl_chunk[0], contpix_x<wl_chunk[-1])
                self.plot_contpix(
                        wl_chunk, baseline_spec_chunk,
                        contpix_x[take], contpix_y[take], fig_chunk)


Instruction:
Can you create a Python 3 function that
produces a set of diagnostic plots for the model

Response:
def diagnostics_plot_chisq(self, ds, figname = "modelfit_chisqs.png"):
        """ Produce a set of diagnostic plots for the model

        Parameters
        ----------
        (optional) chisq_dist_plot_name: str
            Filename of output saved plot
        """
        label_names = ds.get_plotting_labels()
        lams = ds.wl
        pivots = self.pivots
        npixels = len(lams)
        nlabels = len(pivots)
        chisqs = self.chisqs
        coeffs = self.coeffs
        scatters = self.scatters

        # Histogram of the chi squareds of ind. stars
        plt.hist(np.sum(chisqs, axis=0), color='lightblue', alpha=0.7,
                bins=int(np.sqrt(len(chisqs))))
        dof = len(lams) - coeffs.shape[1]   # for one star
        plt.axvline(x=dof, c='k', linewidth=2, label="DOF")
        plt.legend()
        plt.title("Distribution of " + r"$\chi^2$" + " of the Model Fit")
        plt.ylabel("Count")
        plt.xlabel(r"$\chi^2$" + " of Individual Star")
        print("Diagnostic plot: histogram of the red chi squareds of the fit")
        print("Saved as %s" %figname)
        plt.savefig(figname)
        plt.close()


Instruction:
How would you explain what the following Python 3 function does
def calc_mass(nu_max, delta_nu, teff):
    NU_MAX = 3140.0 # microHz
    DELTA_NU = 135.03 # microHz
    TEFF = 5777.0
    return (nu_max/NU_MAX)**3 * (delta_nu/DELTA_NU)**(-4) * (teff/TEFF)**1.5

Response: Calculate the mass of a node based on the given nu_max delta_nu and teff.


Instruction:
Here you have a function in Python 3, explain what it does
def calc_mass_2(mh,cm,nm,teff,logg):
    CplusN = calc_sum(mh,cm,nm)
    t = teff/4000.
    return (95.8689 - 10.4042*mh - 0.7266*mh**2
            + 41.3642*cm - 5.3242*cm*mh - 46.7792*cm**2
            + 15.0508*nm - 0.9342*nm*mh - 30.5159*nm*cm - 1.6083*nm**2
            - 67.6093*CplusN + 7.0486*CplusN*mh + 133.5775*CplusN*cm + 38.9439*CplusN*nm - 88.9948*CplusN**2
            - 144.1765*t + 5.1180*t*mh - 73.7690*t*cm - 15.2927*t*nm + 101.7482*t*CplusN + 27.7690*t**2
            - 9.4246*logg + 1.5159*logg*mh + 16.0412*logg*cm + 1.3549*logg*nm - 18.6527*logg*CplusN + 28.8015*logg*t - 4.0982*logg**2)

Response: Calculates mass of mass 2 from mass 1 and mass 2 and mass 3.


Instruction:
Explain what the following Python 3 code does
def corner(xs, bins=20, range=None, weights=None, color="k",
           smooth=None, smooth1d=None,
           labels=None, label_kwargs=None,
           show_titles=False, title_fmt=".2f", title_kwargs=None,
           truths=None, truth_color="#4682b4",
           scale_hist=False, quantiles=None, verbose=False, fig=None,
           max_n_ticks=5, top_ticks=False, use_math_text=False,
           hist_kwargs=None, **hist2d_kwargs):
    """
    Make a *sick* corner plot showing the projections of a data set in a
    multi-dimensional space. kwargs are passed to hist2d() or used for
    `matplotlib` styling.

    Parameters
    ----------
    xs : array_like (nsamples, ndim)
        The samples. This should be a 1- or 2-dimensional array. For a 1-D
        array this results in a simple histogram. For a 2-D array, the zeroth
        axis is the list of samples and the next axis are the dimensions of
        the space.

    bins : int or array_like (ndim,) (optional)
        The number of bins to use in histograms, either as a fixed value for
        all dimensions, or as a list of integers for each dimension.

    weights : array_like (nsamples,)
        The weight of each sample. If `None` (default), samples are given
        equal weight.

    color : str (optional)
        A ``matplotlib`` style color for all histograms.

    smooth, smooth1d : float (optional)
       The standard deviation for Gaussian kernel passed to
       `scipy.ndimage.gaussian_filter` to smooth the 2-D and 1-D histograms
       respectively. If `None` (default), no smoothing is applied.

    labels : iterable (ndim,) (optional)
        A list of names for the dimensions. If a ``xs`` is a
        ``pandas.DataFrame``, labels will default to column names.

    label_kwargs : dict (optional)
        Any extra keyword arguments to send to the `set_xlabel` and
        `set_ylabel` methods.

    show_titles : bool (optional)
        Displays a title above each 1-D histogram showing the 0.5 quantile
        with the upper and lower errors supplied by the quantiles argument.

    title_fmt : string (optional)
        The format string for the quantiles given in titles. If you explicitly
        set ``show_titles=True`` and ``title_fmt=None``, the labels will be
        shown as the titles. (default: ``.2f``)

    title_kwargs : dict (optional)
        Any extra keyword arguments to send to the `set_title` command.

    range : iterable (ndim,) (optional)
        A list where each element is either a length 2 tuple containing
        lower and upper bounds or a float in range (0., 1.)
        giving the fraction of samples to include in bounds, e.g.,
        [(0.,10.), (1.,5), 0.999, etc.].
        If a fraction, the bounds are chosen to be equal-tailed.

    truths : iterable (ndim,) (optional)
        A list of reference values to indicate on the plots.  Individual
        values can be omitted by using ``None``.

    truth_color : str (optional)
        A ``matplotlib`` style color for the ``truths`` makers.

    scale_hist : bool (optional)
        Should the 1-D histograms be scaled in such a way that the zero line
        is visible?

    quantiles : iterable (optional)
        A list of fractional quantiles to show on the 1-D histograms as
        vertical dashed lines.

    verbose : bool (optional)
        If true, print the values of the computed quantiles.

    plot_contours : bool (optional)
        Draw contours for dense regions of the plot.

    use_math_text : bool (optional)
        If true, then axis tick labels for very large or small exponents will
        be displayed as powers of 10 rather than using `e`.

    max_n_ticks: int (optional)
        Maximum number of ticks to try to use

    top_ticks : bool (optional)
        If true, label the top ticks of each axis

    fig : matplotlib.Figure (optional)
        Overplot onto the provided figure object.

    hist_kwargs : dict (optional)
        Any extra keyword arguments to send to the 1-D histogram plots.

    **hist2d_kwargs : (optional)
        Any remaining keyword arguments are sent to `corner.hist2d` to generate
        the 2-D histogram plots.
    """
    if quantiles is None:
        quantiles = []
    if title_kwargs is None:
        title_kwargs = dict()
    if label_kwargs is None:
        label_kwargs = dict()

    # Try filling in labels from pandas.DataFrame columns.
    if labels is None:
        try:
            labels = xs.columns
        except AttributeError:
            pass

    # Deal with 1D sample lists.
    xs = np.atleast_1d(xs)
    if len(xs.shape) == 1:
        xs = np.atleast_2d(xs)
    else:
        assert len(xs.shape) == 2, "The input sample array must be 1- or 2-D."
        xs = xs.T
    assert xs.shape[0] <= xs.shape[1], "I don't believe that you want more " \
                                       "dimensions than samples!"

    # Parse the weight array.
    if weights is not None:
        weights = np.asarray(weights)
        if weights.ndim != 1:
            raise ValueError("Weights must be 1-D")
        if xs.shape[1] != weights.shape[0]:
            raise ValueError("Lengths of weights must match number of samples")

    # Parse the parameter ranges.
    if range is None:
        if "extents" in hist2d_kwargs:
            logging.warn("Deprecated keyword argument 'extents'. "
                         "Use 'range' instead.")
            range = hist2d_kwargs.pop("extents")
        else:
            range = [[x.min(), x.max()] for x in xs]
            # Check for parameters that never change.
            m = np.array([e[0] == e[1] for e in range], dtype=bool)
            if np.any(m):
                raise ValueError(("It looks like the parameter(s) in "
                                  "column(s) {0} have no dynamic range. "
                                  "Please provide a `range` argument.")
                                 .format(", ".join(map(
                                     "{0}".format, np.arange(len(m))[m]))))

    else:
        # If any of the extents are percentiles, convert them to ranges.
        # Also make sure it's a normal list.
        range = list(range)
        for i, _ in enumerate(range):
            try:
                emin, emax = range[i]
            except TypeError:
                q = [0.5 - 0.5*range[i], 0.5 + 0.5*range[i]]
                range[i] = quantile(xs[i], q, weights=weights)

    if len(range) != xs.shape[0]:
        raise ValueError("Dimension mismatch between samples and range")

    # Parse the bin specifications.
    try:
        bins = [float(bins) for _ in range]
    except TypeError:
        if len(bins) != len(range):
            raise ValueError("Dimension mismatch between bins and range")

    # Some magic numbers for pretty axis layout.
    K = len(xs)
    factor = 2.0           # size of one side of one panel
    lbdim = 0.5 * factor   # size of left/bottom margin
    trdim = 0.2 * factor   # size of top/right margin
    whspace = 0.05         # w/hspace size
    plotdim = factor * K + factor * (K - 1.) * whspace
    dim = lbdim + plotdim + trdim

    # Create a new figure if one wasn't provided.
    if fig is None:
        fig, axes = pl.subplots(K, K, figsize=(dim, dim))
    else:
        try:
            axes = np.array(fig.axes).reshape((K, K))
        except:
            raise ValueError("Provided figure has {0} axes, but data has "
                             "dimensions K={1}".format(len(fig.axes), K))

    # Format the figure.
    lb = lbdim / dim
    tr = (lbdim + plotdim) / dim
    fig.subplots_adjust(left=lb, bottom=lb, right=tr, top=tr,
                        wspace=whspace, hspace=whspace)

    # Set up the default histogram keywords.
    if hist_kwargs is None:
        hist_kwargs = dict()
    hist_kwargs["color"] = hist_kwargs.get("color", color)
    if smooth1d is None:
        hist_kwargs["histtype"] = hist_kwargs.get("histtype", "step")

    for i, x in enumerate(xs):
        # Deal with masked arrays.
        if hasattr(x, "compressed"):
            x = x.compressed()

        if np.shape(xs)[0] == 1:
            ax = axes
        else:
            ax = axes[i, i]
        # Plot the histograms.
        if smooth1d is None:
            n, _, _ = ax.hist(x, bins=bins[i], weights=weights,
                              range=range[i], **hist_kwargs)
        else:
            if gaussian_filter is None:
                raise ImportError("Please install scipy for smoothing")
            n, b = np.histogram(x, bins=bins[i], weights=weights,
                                range=range[i])
            n = gaussian_filter(n, smooth1d)
            x0 = np.array(list(zip(b[:-1], b[1:]))).flatten()
            y0 = np.array(list(zip(n, n))).flatten()
            ax.plot(x0, y0, **hist_kwargs)

        if truths is not None and truths[i] is not None:
            ax.axvline(truths[i], color=truth_color)

        # Plot quantiles if wanted.
        if len(quantiles) > 0:
            qvalues = quantile(x, quantiles, weights=weights)
            for q in qvalues:
                ax.axvline(q, ls="dashed", color=color)

            if verbose:
                print("Quantiles:")
                print([item for item in zip(quantiles, qvalues)])

        if show_titles:
            title = None
            if title_fmt is not None:
                # Compute the quantiles for the title. This might redo
                # unneeded computation but who cares.
                q_16, q_50, q_84 = quantile(x, [0.16, 0.5, 0.84],
                                            weights=weights)
                q_m, q_p = q_50-q_16, q_84-q_50

                # Format the quantile display.
                fmt = "{{0:{0}}}".format(title_fmt).format
                title = r"${{{0}}}_{{-{1}}}^{{+{2}}}$"
                title = title.format(fmt(q_50), fmt(q_m), fmt(q_p))

                # Add in the column name if it's given.
                if labels is not None:
                    title = "{0} = {1}".format(labels[i], title)

            elif labels is not None:
                title = "{0}".format(labels[i])

            if title is not None:
                ax.set_title(title, **title_kwargs)

        # Set up the axes.
        ax.set_xlim(range[i])
        if scale_hist:
            maxn = np.max(n)
            ax.set_ylim(-0.1 * maxn, 1.1 * maxn)
        else:
            ax.set_ylim(0, 1.1 * np.max(n))
        ax.set_yticklabels([])
        ax.xaxis.set_major_locator(MaxNLocator(max_n_ticks, prune="lower"))

        if i < K - 1:
            if top_ticks:
                ax.xaxis.set_ticks_position("top")
                [l.set_rotation(45) for l in ax.get_xticklabels()]
            else:
                ax.set_xticklabels([])
        else:
            [l.set_rotation(45) for l in ax.get_xticklabels()]
            if labels is not None:
                ax.set_xlabel(labels[i], **label_kwargs)
                ax.xaxis.set_label_coords(0.5, -0.3)

            # use MathText for axes ticks
            ax.xaxis.set_major_formatter(
                ScalarFormatter(useMathText=use_math_text))

        for j, y in enumerate(xs):
            if np.shape(xs)[0] == 1:
                ax = axes
            else:
                ax = axes[i, j]
            if j > i:
                ax.set_frame_on(False)
                ax.set_xticks([])
                ax.set_yticks([])
                continue
            elif j == i:
                continue

            # Deal with masked arrays.
            if hasattr(y, "compressed"):
                y = y.compressed()

            hist2d(y, x, ax=ax, range=[range[j], range[i]], weights=weights,
                   color=color, smooth=smooth, bins=[bins[j], bins[i]],
                   **hist2d_kwargs)

            if truths is not None:
                if truths[i] is not None and truths[j] is not None:
                    ax.plot(truths[j], truths[i], "s", color=truth_color)
                if truths[j] is not None:
                    ax.axvline(truths[j], color=truth_color)
                if truths[i] is not None:
                    ax.axhline(truths[i], color=truth_color)

            ax.xaxis.set_major_locator(MaxNLocator(max_n_ticks, prune="lower"))
            ax.yaxis.set_major_locator(MaxNLocator(max_n_ticks, prune="lower"))

            if i < K - 1:
                ax.set_xticklabels([])
            else:
                [l.set_rotation(45) for l in ax.get_xticklabels()]
                if labels is not None:
                    ax.set_xlabel(labels[j], **label_kwargs)
                    ax.xaxis.set_label_coords(0.5, -0.3)

                # use MathText for axes ticks
                ax.xaxis.set_major_formatter(
                    ScalarFormatter(useMathText=use_math_text))

            if j > 0:
                ax.set_yticklabels([])
            else:
                [l.set_rotation(45) for l in ax.get_yticklabels()]
                if labels is not None:
                    ax.set_ylabel(labels[i], **label_kwargs)
                    ax.yaxis.set_label_coords(-0.3, 0.5)

                # use MathText for axes ticks
                ax.yaxis.set_major_formatter(
                    ScalarFormatter(useMathText=use_math_text))

    return fig

Response: A simple corner plot showing the projections of a data set in a multi - dimensional space.


Instruction:
Implement a function in Python 3 to
like np. percentile but returns a list of the n - th quantile of the elements in x.

Response:
def quantile(x, q, weights=None):
    """
    Like numpy.percentile, but:

    * Values of q are quantiles [0., 1.] rather than percentiles [0., 100.]
    * scalar q not supported (q must be iterable)
    * optional weights on x

    """
    if weights is None:
        return np.percentile(x, [100. * qi for qi in q])
    else:
        idx = np.argsort(x)
        xsorted = x[idx]
        cdf = np.add.accumulate(weights[idx])
        cdf /= cdf[-1]
        return np.interp(q, cdf, xsorted).tolist()


Instruction:
Implement a Python 3 function for
plotting a 2 - D histogram of samples.

Response:
def hist2d(x, y, bins=20, range=None, weights=None, levels=None, smooth=None,
           ax=None, color=None, plot_datapoints=True, plot_density=True,
           plot_contours=True, no_fill_contours=False, fill_contours=False,
           contour_kwargs=None, contourf_kwargs=None, data_kwargs=None,
           **kwargs):
    """
    Plot a 2-D histogram of samples.

    Parameters
    ----------
    x, y : array_like (nsamples,)
       The samples.

    levels : array_like
        The contour levels to draw.

    ax : matplotlib.Axes (optional)
        A axes instance on which to add the 2-D histogram.

    plot_datapoints : bool (optional)
        Draw the individual data points.

    plot_density : bool (optional)
        Draw the density colormap.

    plot_contours : bool (optional)
        Draw the contours.

    no_fill_contours : bool (optional)
        Add no filling at all to the contours (unlike setting
        ``fill_contours=False``, which still adds a white fill at the densest
        points).

    fill_contours : bool (optional)
        Fill the contours.

    contour_kwargs : dict (optional)
        Any additional keyword arguments to pass to the `contour` method.

    contourf_kwargs : dict (optional)
        Any additional keyword arguments to pass to the `contourf` method.

    data_kwargs : dict (optional)
        Any additional keyword arguments to pass to the `plot` method when
        adding the individual data points.
    """
    if ax is None:
        ax = pl.gca()

    # Set the default range based on the data range if not provided.
    if range is None:
        if "extent" in kwargs:
            logging.warn("Deprecated keyword argument 'extent'. "
                         "Use 'range' instead.")
            range = kwargs["extent"]
        else:
            range = [[x.min(), x.max()], [y.min(), y.max()]]

    # Set up the default plotting arguments.
    if color is None:
        color = "k"

    # Choose the default "sigma" contour levels.
    if levels is None:
        levels = 1.0 - np.exp(-0.5 * np.arange(0.5, 2.1, 0.5) ** 2)

    # This is the color map for the density plot, over-plotted to indicate the
    # density of the points near the center.
    density_cmap = LinearSegmentedColormap.from_list(
        "density_cmap", [color, (1, 1, 1, 0)])

    # This color map is used to hide the points at the high density areas.
    white_cmap = LinearSegmentedColormap.from_list(
        "white_cmap", [(1, 1, 1), (1, 1, 1)], N=2)

    # This "color map" is the list of colors for the contour levels if the
    # contours are filled.
    rgba_color = colorConverter.to_rgba(color)
    contour_cmap = [list(rgba_color) for l in levels] + [rgba_color]
    for i, l in enumerate(levels):
        contour_cmap[i][-1] *= float(i) / (len(levels)+1)

    # We'll make the 2D histogram to directly estimate the density.
    try:
        H, X, Y = np.histogram2d(x.flatten(), y.flatten(), bins=bins,
                                 range=range, weights=weights)
    except ValueError:
        raise ValueError("It looks like at least one of your sample columns "
                         "have no dynamic range. You could try using the "
                         "'range' argument.")

    if smooth is not None:
        if gaussian_filter is None:
            raise ImportError("Please install scipy for smoothing")
        H = gaussian_filter(H, smooth)

    # Compute the density levels.
    Hflat = H.flatten()
    inds = np.argsort(Hflat)[::-1]
    Hflat = Hflat[inds]
    sm = np.cumsum(Hflat)
    sm /= sm[-1]
    V = np.empty(len(levels))
    for i, v0 in enumerate(levels):
        try:
            V[i] = Hflat[sm <= v0][-1]
        except:
            V[i] = Hflat[0]
    V.sort()
    m = np.diff(V) == 0
    if np.any(m):
        logging.warning("Too few points to create valid contours")
    while np.any(m):
        V[np.where(m)[0][0]] *= 1.0 - 1e-4
        m = np.diff(V) == 0
    V.sort()

    # Compute the bin centers.
    X1, Y1 = 0.5 * (X[1:] + X[:-1]), 0.5 * (Y[1:] + Y[:-1])

    # Extend the array for the sake of the contours at the plot edges.
    H2 = H.min() + np.zeros((H.shape[0] + 4, H.shape[1] + 4))
    H2[2:-2, 2:-2] = H
    H2[2:-2, 1] = H[:, 0]
    H2[2:-2, -2] = H[:, -1]
    H2[1, 2:-2] = H[0]
    H2[-2, 2:-2] = H[-1]
    H2[1, 1] = H[0, 0]
    H2[1, -2] = H[0, -1]
    H2[-2, 1] = H[-1, 0]
    H2[-2, -2] = H[-1, -1]
    X2 = np.concatenate([
        X1[0] + np.array([-2, -1]) * np.diff(X1[:2]),
        X1,
        X1[-1] + np.array([1, 2]) * np.diff(X1[-2:]),
    ])
    Y2 = np.concatenate([
        Y1[0] + np.array([-2, -1]) * np.diff(Y1[:2]),
        Y1,
        Y1[-1] + np.array([1, 2]) * np.diff(Y1[-2:]),
    ])

    if plot_datapoints:
        if data_kwargs is None:
            data_kwargs = dict()
        data_kwargs["color"] = data_kwargs.get("color", color)
        data_kwargs["ms"] = data_kwargs.get("ms", 2.0)
        data_kwargs["mec"] = data_kwargs.get("mec", "none")
        data_kwargs["alpha"] = data_kwargs.get("alpha", 0.1)
        ax.plot(x, y, "o", zorder=-1, rasterized=True, **data_kwargs)

    # Plot the base fill to hide the densest data points.
    if (plot_contours or plot_density) and not no_fill_contours:
        ax.contourf(X2, Y2, H2.T, [V.min(), H.max()],
                    cmap=white_cmap, antialiased=False)

    if plot_contours and fill_contours:
        if contourf_kwargs is None:
            contourf_kwargs = dict()
        contourf_kwargs["colors"] = contourf_kwargs.get("colors", contour_cmap)
        contourf_kwargs["antialiased"] = contourf_kwargs.get("antialiased",
                                                             False)
        ax.contourf(X2, Y2, H2.T, np.concatenate([[0], V, [H.max()*(1+1e-4)]]),
                    **contourf_kwargs)

    # Plot the density map. This can't be plotted at the same time as the
    # contour fills.
    elif plot_density:
        ax.pcolor(X, Y, H.max() - H.T, cmap=density_cmap)

    # Plot the contour edge colors.
    if plot_contours:
        if contour_kwargs is None:
            contour_kwargs = dict()
        contour_kwargs["colors"] = contour_kwargs.get("colors", color)
        ax.contour(X2, Y2, H2.T, V, **contour_kwargs)

    ax.set_xlim(range[0])
    ax.set_ylim(range[1])


Instruction:
How would you explain what the following Python 3 function does
def make_classifier(self, name, ids, labels):
        if not all(np.in1d(ids, self.ids)):
            raise ValueError("Hay ids de textos que no se encuentran \
                              almacenados.")
        setattr(self, name, SGDClassifier())
        classifier = getattr(self, name)
        indices = np.searchsorted(self.ids, ids)
        classifier.fit(self.tfidf_mat[indices, :], labels)

Response: Enregistro un clasificador SVM sobre los textos cargados.


Instruction:
Can you tell what is the following Python 3 function doing
def classify(self, classifier_name, examples, max_labels=None,
                 goodness_of_fit=False):
        """Usar un clasificador SVM para etiquetar textos nuevos.

        Args:
            classifier_name (str): Nombre del clasidicador a usar.
            examples (list or str): Se espera un ejemplo o una lista de
                ejemplos a clasificar en texto plano o en ids.
            max_labels (int, optional): Cantidad de etiquetas a devolver para
                cada ejemplo. Si se devuelve mas de una el orden corresponde a
                la plausibilidad de cada etiqueta. Si es None devuelve todas
                las etiquetas posibles.
            goodness_of_fit (bool, optional): Indica si devuelve o no una
                medida de cuan buenas son las etiquetas.
        Nota:
            Usa el clasificador de `Scikit-learn <http://scikit-learn.org/>`_

        Returns:
            tuple (array, array): (labels_considerados, puntajes)
                labels_considerados: Las etiquetas que se consideraron para
                    clasificar.
                puntajes: Cuanto más alto el puntaje, más probable es que la
                    etiqueta considerada sea la adecuada.
        """
        classifier = getattr(self, classifier_name)
        texts_vectors = self._make_text_vectors(examples)
        return classifier.classes_, classifier.decision_function(texts_vectors)

Response: Usar un clasificador SVM para etiquetar textos nuevos.