"""The main warehose pipeline definition.
"""

import hashlib
import json
import logging
import os
import re
import struct
from io import BytesIO
from pathlib import Path, posixpath

import bonobo
import boto3
import mondrian
import pydicom
from bonobo.config import use
from botocore.exceptions import ClientError

from warehouse.components import constants, services

# set up logging
mondrian.setup(excepthook=True)
logger = logging.getLogger()

s3_resource = boto3.resource("s3")
s3_client = boto3.client("s3")

BUCKET_NAME = os.getenv("WAREHOUSE_BUCKET", default="chest-data-warehouse")
bucket = s3_resource.Bucket(BUCKET_NAME)

DRY_RUN = bool(os.getenv("DRY_RUN", default=False))

KB = 1024

###
# Helpers
###
def object_exists(key):
    """ Checking whether a given object exists in our work bucket

    :param key: the object key in question
    :type key: string
    :raises botocore.exceptions.ClientError: if there's any transfer error
    :return: True if object exists in the work bucket
    :rtype: boolean
    """
    try:
        bucket.Object(key).load()
    except ClientError as e:
        if e.response["Error"]["Code"] == "404":
            return False
        else:
            raise ClientError
    else:
        return True

def get_date_from_key(key):
    """ Extract date from an object key from the bucket's directory pattern,
    for a given prefix

    :param key: the object key in question
    :type key: string
    :return: the extracted date if found
    :rtype: string or None
    """
    date_match = re.match(r"^.+/(?P<date>\d{4}-\d{2}-\d{2})/.+", key)
    if date_match:
        return date_match.group("date")

def get_submitting_centre_from_object(obj):
    """Extract the SubmittingCentre value from an S3 object that is
    a JSON file in the expected format.

    :param obj: the S3 object of the JSON file to process
    :type obj: boto3.resource('s3').ObjectSummary
    :return: the value defined for SubmittingCentre in the file
    :rtype: string or None
    """
    file_content = obj.Object().get()["Body"].read().decode("utf-8")
    try:
        json_content = json.loads(file_content)
    except json.decoder.JSONDecodeError:
        logger.error("Couldn't decode contents of {obj.key} as JSON.")
        raise
    return json_content.get("SubmittingCentre")

def patient_in_training_set(
    patient_id, training_percent=constants.TRAINING_PERCENTAGE
):
    """ Separating patient ID's into training and validation sets, check
    which one this ID should fall into.

    It uses a hashing (sha512) to get pseudo-randomisation based on ID,
    and do the cut-off with a set percentage.

    :param patient_id: the candidate patient ID
    :type patient_id: string
    :param training_percent: the percentage of patience to assign to the training set (defaults to the global TRAINING_PERCENTAGE)
    :type training_percent: int
    :return: True if the patient ID should fall into the training set
    :rtype: boolean
    """
    return (
        int(
            hashlib.sha512(
                patient_id.strip().upper().encode("utf-8")
            ).hexdigest(),
            16,
        )
        % 100
        < training_percent
    )

def inplace_nullify(d, key):
    """
    Recurse through a dictionary and set the value `key` to `None`

    Extracted from https://bitbucket.org/scicomcore/dcm2slimjson/src/master/dcm2slimjson/main.py

    :param d: dict to modify
    :type d: dict
    :param key: specific key to modify
    :type key: anything that can be a dict key
    """
    if isinstance(d, list):
        [inplace_nullify(_, key) for _ in d]

    if isinstance(d, dict):
        for k, v in d.items():

            if k == key:
                d[k] = None

            if isinstance(v, (dict, list)):
                inplace_nullify(v, key)

def scrub_dicom(fd):
    """Remove binary data and other unusuaed sections from a DICOM image.

    Extracted from https://bitbucket.org/scicomcore/dcm2slimjson/src/master/dcm2slimjson/main.py

    :param fd: image data to scrub
    :type fd: pydicom.FileDataset
    :return: the scrubbed image data
    :rtype: dict
    """

    # Use a large value to bypass binary data handler
    out = fd.to_json_dict(bulk_data_threshold=1e20)

    # Drop binary data
    inplace_nullify(out, "InlineBinary")

    # Remove Value of Interest (VOI) transform data
    inplace_nullify(out, "00283010")

    return out

class PartialDicom:
    """Download partial DICOM files iteratively, to save
    on traffic.
    """

    def __init__(self, obj, initial_range_kb=20):
        # Default value of 20Kb initial range is based on
        # tests run on representative data
        self._found_image_tag = False
        self.obj = obj
        self.range_kb = initial_range_kb

    def _stop_when(self, tag, VR, length):
        """Custom stopper for the DICOM reader, to stop
        at the pixel data, but also note whether that
        tag was actually reached.
        """
        self._found_image_tag = tag == (0x7FE0, 0x0010)
        return self._found_image_tag

    def download(self):
        """Download file iteratively, and return the image data
        """
        with BytesIO() as tmp:
            while True:
                tmp.seek(0)
                toprange = (self.range_kb * KB) - 1
                stream = self.obj.get(Range=f"bytes=0-{toprange}")["Body"]
                tmp.write(stream.read())
                tmp.seek(0)
                try:
                    image_data = pydicom.filereader.read_partial(
                        tmp, stop_when=self._stop_when
                    )
                    if self._found_image_tag or tmp.tell() < toprange:
                        # We've found the image tag, or there was not image tag
                        # to be found in this image
                        break
                except (OSError, struct.error):
                    # Can happen when file got truncated in the middle of a data field
                    pass
                except Exception:
                    raise
                self.range_kb *= 2
        return image_data

###
# Transformation steps
###
@use("config")
def load_config(config):
    """Load configuration from the bucket
    """
    try:
        obj = bucket.Object(constants.CONFIG_KEY).get()
        contents = json.loads(obj["Body"].read().decode("utf-8"))
        config.set_config(contents)
        yield
    except ClientError as ex:
        if ex.response["Error"]["Code"] == "NoSuchKey":
            logger.warning(
                "No configuration found in the bucket! (not going to do any loading)"
            )
        else:
            raise

@use("keycache")
@use("patientcache")
@use("inventory")
def load_existing_files(keycache, patientcache, inventory):
    """ Loading existing files from the training and
    validation sets into the keycache.

    :param keycache: the key cache service (provided by bonobo)
    :type keycache: Keycache
    """
    # Set up our listing function.
    if inventory.enabled:
        listing = inventory.filter_keys
    else:
        # When using the original listing without inventory, we need to
        # transform the objects returned by the filter
        def listing(Prefix):
            return map(
                lambda obj: obj.key, bucket.objects.filter(Prefix=Prefix)
            )

    patient_file_name = re.compile(
        r"^.+/data/(?P<patient_id>.*)/(?:data|status)_\d{4}-\d{2}-\d{2}.json$"
    )
    for group, prefix in [
        ("validation", constants.VALIDATION_PREFIX),
        ("training", constants.TRAINING_PREFIX),
    ]:
        for key in listing(Prefix=prefix):
            m = patient_file_name.match(key)
            if m:
                # It is a patient file
                patient_id = m.group("patient_id")
                patientcache.add(patient_id, group)
            else:
                # It is an image file
                try:
                    keycache.add(key)
                except services.DuplicateKeyError:
                    logger.exception(f"{key} is duplicate in cache.")
                    continue
    return bonobo.constants.NOT_MODIFIED

@use("config")
@use("inventory")
@use("rawsubfolderlist")
def extract_raw_folders(config, inventory, rawsubfolderlist):
    """ Extractor: get all date folders within the `raw/` data drop

    :return: subfolders within the `raw/` prefix (yield)
    :rtype: string
    """
    for site_raw_prefix in config.get_raw_prefixes():
        if not site_raw_prefix.endswith("/"):
            site_raw_prefix += "/"

        if inventory.enabled:
            prefixes = inventory.list_folders(site_raw_prefix)
        else:
            result = s3_client.list_objects(
                Bucket=BUCKET_NAME, Prefix=site_raw_prefix, Delimiter="/"
            )
            prefixes = [p.get("Prefix") for p in result.get("CommonPrefixes")]
        # list folders in date order
        for folder in sorted(prefixes, reverse=False):
            for subfolder in rawsubfolderlist.get():
                yield folder + subfolder

@use("inventory")
def extract_raw_files_from_folder(folder, inventory):
    """ Extract files from a given date folder in the data dump

    :param folder: the folder to process
    :type key: string
    :return: each object (yield)
    :rtype: boto3.resource('s3').ObjectSummary
    """
    listing = inventory.filter if inventory.enabled else bucket.objects.filter
    for obj in listing(Prefix=folder):
        yield "process", obj, None

@use("keycache")
@use("config")
@use("patientcache")
def process_image(*args, keycache, config, patientcache):
    """ Processing images from the raw dump

    Takes a single image, downloads it into temporary storage
    and extracts its metadata.

    The metadata is then uploaded here, except if the file already exists.

    If the image file already exists at the correct location, it's not passed
    on to the next step.

    :param obj: the object in question
    :type obj: boto3.resource('s3').ObjectSummary
    :param keycache: the key cache service (provided by bonobo)
    :type keycache: Keycache
    :return: a task name, the original object, and a new key where it should be copied within the bucket
    :rtype: (string, boto3.resource('s3').ObjectSummary, string)
    """
    # check file type
    task, obj, _ = args
    if task != "process" or Path(obj.key).suffix.lower() != ".dcm":
        # not an image, don't do anything with it
        return bonobo.constants.NOT_MODIFIED

    # check if work is already done
    image_in_cache = keycache.exists(obj.key)
    image_uuid = Path(obj.key).stem
    metadata_in_cache = keycache.exists(f"{image_uuid}.json")
    if metadata_in_cache and image_in_cache:
        # files exist, nothing to do here
        return

    # download the image
    image_data = PartialDicom(obj.Object()).download()
    if image_data is None:
        # we couldn't read the image data correctly
        logger.warning(
            f"Object '{obj.key}' couldn't be loaded as a DICOM file, skipping!"
        )
        return

    # extract the required data from the image
    patient_id = image_data.PatientID
    study_id = image_data.StudyInstanceUID
    series_id = image_data.SeriesInstanceUID
    group = patientcache.get_group(patient_id)
    if group is not None:
        training_set = group == "training"
    else:
        logger.error(
            f"Image without patient data: {obj.key}; "
            + f"included patient ID: {patient_id}; "
            + "skipping!"
        )
        return
    prefix = (
        constants.TRAINING_PREFIX
        if training_set
        else constants.VALIDATION_PREFIX
    )
    image_type = constants.MODALITY.get(
        image_data["Modality"].value, "unknown"
    )

    date = get_date_from_key(obj.key)
    if date:
        # the location of the new files
        new_key = posixpath.join(
            prefix,
            image_type,
            patient_id,
            study_id,
            series_id,
            Path(obj.key).name,
        )
        metadata_key = posixpath.join(
            prefix,
            f"{image_type}-metadata",
            patient_id,
            study_id,
            series_id,
            f"{image_uuid}.json",
        )
        # send off to copy or upload steps
        if not object_exists(new_key):
            yield "copy", obj, new_key
        if not object_exists(metadata_key):
            yield "metadata", metadata_key, image_data

def process_dicom_data(*args):
    """Process DICOM images, by scrubbing the image data

    :param task: task informatiomn, needs to be equal to "metadata" to be processed here
    :type task: string
    :param metadata_key: location to upload the extracted metadata later
    :type metadata_key: string
    :param image_data: DICOM image data
    :type image_data: pydicom.FileDataset
    :return: metadata key and scrubbed image data, if processed
    :rtype: tuple
    """
    task, metadata_key, image_data, = args
    if task == "metadata":
        scrubbed_image_data = scrub_dicom(image_data)
        yield "upload", metadata_key, json.dumps(scrubbed_image_data)

def upload_text_data(*args):
    """Upload the text data to the correct bucket location.

    :param task: selector to run this task or not, needs to be "upload" to process a file
    :type task: string
    :param outgoing_key: location to upload the data
    :type outgoing_key: string
    :param outgoing_data: text to file content to upload
    :type outgoing_data: string
    """
    task, outgoing_key, outgoing_data, = args
    if (
        task == "upload"
        and outgoing_key is not None
        and outgoing_data is not None
    ):
        if DRY_RUN:
            logger.info(f"Would upload to key: {outgoing_key}")
        else:
            bucket.put_object(Body=outgoing_data, Key=outgoing_key)

        return bonobo.constants.NOT_MODIFIED

@use("config")
@use("patientcache")
def process_patient_data(*args, config, patientcache):
    """Processing patient data from the raw dump

    Get the patient ID from the filename, do a training/validation
    test split, and create the key for the new location for the
    next processing step to copy things to.

    :param obj: the object in question
    :type obj: boto3.resource('s3').ObjectSummary
    :return: a task name, the original object, and a new key where it should be copied within the bucket
    :rtype: (string, boto3.resource('s3').ObjectSummary, string)
    """
    task, obj, _ = args
    if task != "process" or Path(obj.key).suffix.lower() != ".json":
        # Not a data file, don't do anything with it
        yield bonobo.constants.NOT_MODIFIED

    m = re.match(
        r"^(?P<patient_id>.*)_(?P<outcome>data|status)$", Path(obj.key).stem
    )
    if m is None:
        # Can't interpret this file based on name, skip
        return

    patient_id = m.group("patient_id")
    outcome = m.group("outcome")

    group = patientcache.get_group(patient_id)
    if group is not None:
        training_set = group == "training"
    else:
        # patient group is not cached
        submitting_centre = get_submitting_centre_from_object(obj)
        if submitting_centre is None:
            logger.error(
                f"{obj.key} does not have 'SubmittingCentre' entry, skipping!"
            )
            return

        config_group = config.get_site_group(submitting_centre)
        if config_group is None:
            logger.warning(
                f"Site '{submitting_centre}' is not in configuration, skipping!"
            )
            return
        if config_group == "split":
            training_set = patient_in_training_set(
                patient_id, config.get_training_percentage()
            )
        else:
            # deciding between "training" and "validation" groups.
            training_set = config_group == "training"
        patientcache.add(
            patient_id, "training" if training_set else "validation"
        )

    prefix = (
        constants.TRAINING_PREFIX
        if training_set
        else constants.VALIDATION_PREFIX
    )
    date = get_date_from_key(obj.key)
    if date is not None:
        new_key = f"{prefix}data/{patient_id}/{outcome}_{date}.json"
        if not object_exists(new_key):
            yield "copy", obj, new_key

def data_copy(*args):
    """Copy objects within the bucket

    Only if both original object and new key is provided.

    :param task: selector to run this task or not, needs to be "copy" to process a file
    :type task: string
    :param obj: the object key in question
    :type obj: boto3.resource('s3').ObjectSummary
    :param obj: the new key to copy data to
    :type obj: string
    :return: standard constant for bonobo "load" steps, so they can be chained
    :rtype: bonobo.constants.NOT_MODIFIED
    """
    task, obj, new_key, = args
    if task == "copy" and obj is not None and new_key is not None:
        if DRY_RUN:
            logger.info(f"Would copy: {obj.key} -> {new_key}")
        else:
            bucket.copy({"Bucket": obj.bucket_name, "Key": obj.key}, new_key)

        return bonobo.constants.NOT_MODIFIED

###
# Graph setup
###
def get_graph(**options):
    """
    This function builds the graph that needs to be executed.

    :return: bonobo.Graph
    """
    graph = bonobo.Graph()

    graph.add_chain(
        load_config,
        load_existing_files,
        extract_raw_folders,
        extract_raw_files_from_folder,
    )

    graph.add_chain(data_copy, _input=None, _name="copy")

    graph.add_chain(
        # bonobo.Limit(30),
        process_patient_data,
        _input=extract_raw_files_from_folder,
        _output="copy",
    )

    graph.add_chain(
        # bonobo.Limit(30),
        process_image,
        _input=process_patient_data,
        _output="copy",
    )

    graph.add_chain(process_dicom_data, upload_text_data, _input=process_image)

    return graph

def get_services(**options):
    """
    This function builds the services dictionary, which is a simple dict of names-to-implementation used by bonobo
    for runtime injection.

    It will be used on top of the defaults provided by bonobo (fs, http, ...). You can override those defaults, or just
    let the framework define them. You can also define your own services and naming is up to you.

    :return: dict
    """
    config = services.PipelineConfig()
    keycache = services.KeyCache()
    patientcache = services.PatientCache()
    rawsubfolderlist = services.SubFolderList()

    if bool(os.getenv("SKIP_INVENTORY", default=False)):
        inventory = services.Inventory()
    else:
        inventory = services.Inventory(main_bucket=BUCKET_NAME)

    return {
        "config": config,
        "keycache": keycache,
        "patientcache": patientcache,
        "rawsubfolderlist": rawsubfolderlist,
        "inventory": inventory,
    }

def main():
    """Execute the pipeline graph
    """
    parser = bonobo.get_argument_parser()
    with bonobo.parse_args(parser) as options:
        bonobo.run(get_graph(**options), services=get_services(**options))

if __name__ == "__main__":
    main()


# Copyright (c) 2019 The Boule Developers.
# Distributed under the terms of the BSD 3-Clause License.
# SPDX-License-Identifier: BSD-3-Clause
#
# This code is part of the Fatiando a Terra project (https://www.fatiando.org)
#
"""
Module for defining and setting the reference ellipsoid.
"""
from warnings import warn
import attr
import numpy as np

# Don't let ellipsoid parameters be changed to avoid messing up calculations
# accidentally.
@attr.s(frozen=True)
class Ellipsoid:
    """
    Reference oblate ellipsoid.

    The ellipsoid is oblate and spins around it's minor axis. It is defined by
    four parameters (semi-major axis, flattening, geocentric gravitational
    constant, and angular velocity) and offers other derived quantities.

    **All attributes of this class are read-only and cannot be changed after
    instantiation.**

    All parameters are in SI units.

    .. note::

        Use :class:`boule.Sphere` if you desire zero flattening because there
        are singularities for this particular case in the normal gravity
        calculations.

    Parameters
    ----------
    name : str
        A short name for the ellipsoid, for example ``'WGS84'``.
    semimajor_axis : float
        The semi-major axis of the ellipsoid (equatorial radius), usually
        represented by "a" [meters].
    flattening : float
        The flattening of the ellipsoid (f) [adimensional].
    geocentric_grav_const : float
        The geocentric gravitational constant (GM) [m^3 s^-2].
    angular_velocity : float
        The angular velocity of the rotating ellipsoid (omega) [rad s^-1].
    long_name : str or None
        A long name for the ellipsoid, for example ``"World Geodetic System
        1984"`` (optional).
    reference : str or None
        Citation for the ellipsoid parameter values (optional).

    Examples
    --------

    We can define an ellipsoid by setting the 4 key numerical parameters:

    >>> ellipsoid = Ellipsoid(
    ...     name="oblate-ellipsoid",
    ...     long_name="Oblate Ellipsoid",
    ...     semimajor_axis=1,
    ...     flattening=0.5,
    ...     geocentric_grav_const=1,
    ...     angular_velocity=0,
    ... )
    >>> print(ellipsoid) # doctest: +ELLIPSIS
    Ellipsoid(name='oblate-ellipsoid', ...)
    >>> print(ellipsoid.long_name)
    Oblate Ellipsoid

    The class defines several derived attributes based on the input parameters:

    >>> print("{:.2f}".format(ellipsoid.semiminor_axis))
    0.50
    >>> print("{:.2f}".format(ellipsoid.mean_radius))
    0.83
    >>> print("{:.2f}".format(ellipsoid.linear_eccentricity))
    0.87
    >>> print("{:.2f}".format(ellipsoid.first_eccentricity))
    0.87
    >>> print("{:.2f}".format(ellipsoid.second_eccentricity))
    1.73

    """

    name = attr.ib()
    semimajor_axis = attr.ib()
    flattening = attr.ib()
    geocentric_grav_const = attr.ib()
    angular_velocity = attr.ib()
    long_name = attr.ib(default=None)
    reference = attr.ib(default=None)

    @flattening.validator
    def _check_flattening(
        self, flattening, value
    ):  # pylint: disable=no-self-use,unused-argument
        """
        Check if flattening is valid
        """
        if value < 0 or value >= 1:
            raise ValueError(
                f"Invalid flattening '{value}'. "
                "Should be greater than zero and lower than 1."
            )
        if value == 0:
            raise ValueError(
                "Flattening equal to zero will lead to errors in normal gravity. "
                "Use boule.Sphere for representing ellipsoids with zero flattening."
            )
        if value < 1e-7:
            warn(
                f"Flattening is too close to zero ('{value}'). "
                "This may lead to inaccurate results and division by zero errors. "
                "Use boule.Sphere for representing ellipsoids with zero flattening."
            )

    @semimajor_axis.validator
    def _check_semimajor_axis(
        self, semimajor_axis, value
    ):  # pylint: disable=no-self-use,unused-argument
        """
        Check if semimajor_axis is positive
        """
        if not value > 0:
            raise ValueError(
                f"Invalid semi-major axis '{value}'. Should be greater than zero."
            )

    @geocentric_grav_const.validator
    def _check_geocentric_grav_const(
        self, geocentric_grav_const, value
    ):  # pylint: disable=no-self-use,unused-argument
        """
        Warn if geocentric_grav_const is negative
        """
        if value < 0:
            warn(f"The geocentric gravitational constant is negative: '{value}'")

    @property
    def semiminor_axis(self):
        "The small (polar) axis of the ellipsoid [meters]"
        return self.semimajor_axis * (1 - self.flattening)

    @property
    def linear_eccentricity(self):
        "The linear eccentricity [meters]"
        return np.sqrt(self.semimajor_axis ** 2 - self.semiminor_axis ** 2)

    @property
    def first_eccentricity(self):
        "The first eccentricity [adimensional]"
        return self.linear_eccentricity / self.semimajor_axis

    @property
    def second_eccentricity(self):
        "The second eccentricity [adimensional]"
        return self.linear_eccentricity / self.semiminor_axis

    @property
    def mean_radius(self):
        """
        The arithmetic mean radius :math:`R_1=(2a+b)/3` [Moritz1988]_ [meters]
        """
        return 1 / 3 * (2 * self.semimajor_axis + self.semiminor_axis)

    @property
    def emm(self):
        r"Auxiliary quantity :math:`m = \omega^2 a^2 b / (GM)`"
        return (
            self.angular_velocity ** 2
            * self.semimajor_axis ** 2
            * self.semiminor_axis
            / self.geocentric_grav_const
        )

    @property
    def gravity_equator(self):
        """
        The norm of the gravity vector on the ellipsoid at the equator [m/s²]
        """
        ratio = self.semiminor_axis / self.linear_eccentricity
        arctan = np.arctan2(self.linear_eccentricity, self.semiminor_axis)
        aux = (
            self.second_eccentricity
            * (3 * (1 + ratio ** 2) * (1 - ratio * arctan) - 1)
            / (3 * ((1 + 3 * ratio ** 2) * arctan - 3 * ratio))
        )
        axis_mul = self.semimajor_axis * self.semiminor_axis
        result = self.geocentric_grav_const * (1 - self.emm - self.emm * aux) / axis_mul
        return result

    @property
    def gravity_pole(self):
        "The norm of the gravity vector on the ellipsoid at the poles [m/s²]"
        ratio = self.semiminor_axis / self.linear_eccentricity
        arctan = np.arctan2(self.linear_eccentricity, self.semiminor_axis)
        aux = (
            self.second_eccentricity
            * (3 * (1 + ratio ** 2) * (1 - ratio * arctan) - 1)
            / (1.5 * ((1 + 3 * ratio ** 2) * arctan - 3 * ratio))
        )
        result = (
            self.geocentric_grav_const * (1 + self.emm * aux) / self.semimajor_axis ** 2
        )
        return result

    def geocentric_radius(self, latitude, geodetic=True):
        r"""
        Distance from the center of the ellipsoid to its surface.

        The geocentric radius and is a function of the geodetic latitude
        :math:`\phi` and the semi-major and semi-minor axis, a and b:

        .. math::

            R(\phi) = \sqrt{\dfrac{
                (a^2\cos\phi)^2 + (b^2\sin\phi)^2}{
                (a\cos\phi)^2 + (b\sin\phi)^2 }
            }

        See https://en.wikipedia.org/wiki/Earth_radius#Geocentric_radius

        The same could be achieved with
        :meth:`boule.Ellipsoid.geodetic_to_spherical` by passing any value for
        the longitudes and heights equal to zero. This method provides a
        simpler and possibly faster alternative.

        Alternatively, the geocentric radius can also be expressed in terms of
        the geocentric (spherical) latitude :math:`\theta`:

        .. math::

            R(\theta) = \sqrt{\dfrac{1}{
                (\frac{\cos\theta}{a})^2 + (\frac{\sin\theta}{b})^2 }
            }

        This can be useful if you already have the geocentric latitudes and
        need the geocentric radius of the ellipsoid (for example, in spherical
        harmonic analysis). In these cases, the coordinate conversion route is
        not possible since we need the radial coordinates to do that in the
        first place.

        .. note::

            No elevation is taken into account (the height is zero). If you
            need the geocentric radius at a height other than zero, use
            :meth:`boule.Ellipsoid.geodetic_to_spherical` instead.

        Parameters
        ----------
        latitude : float or array
            Latitude coordinates on geodetic coordinate system in degrees.
        geodetic : bool
            If True (default), will assume that latitudes are geodetic
            latitudes. Otherwise, will that they are geocentric spherical
            latitudes.

        Returns
        -------
        geocentric_radius : float or array
            The geocentric radius for the given latitude(s) in the same units
            as the ellipsoid axis.

        """
        latitude_rad = np.radians(latitude)
        coslat, sinlat = np.cos(latitude_rad), np.sin(latitude_rad)
        # Avoid doing this in favour of having the user do the conversions when
        # possible. It's not the case here, so we made an exception.
        if geodetic:
            radius = np.sqrt(
                (
                    (self.semimajor_axis ** 2 * coslat) ** 2
                    + (self.semiminor_axis ** 2 * sinlat) ** 2
                )
                / (
                    (self.semimajor_axis * coslat) ** 2
                    + (self.semiminor_axis * sinlat) ** 2
                )
            )
        else:
            radius = np.sqrt(
                1
                / (
                    (coslat / self.semimajor_axis) ** 2
                    + (sinlat / self.semiminor_axis) ** 2
                )
            )
        return radius

    def prime_vertical_radius(self, sinlat):
        r"""
        Calculate the prime vertical radius for a given geodetic latitude

        The prime vertical radius is defined as:

        .. math::

            N(\phi) = \frac{a}{\sqrt{1 - e^2 \sin^2(\phi)}}

        Where :math:`a` is the semi-major axis and :math:`e` is the first
        eccentricity.

        This function receives the sine of the latitude as input to avoid
        repeated computations of trigonometric functions.

        Parameters
        ----------
        sinlat : float or array-like
            Sine of the latitude angle.

        Returns
        -------
        prime_vertical_radius : float or array-like
            Prime vertical radius given in the same units as the semi-major
            axis

        """
        return self.semimajor_axis / np.sqrt(
            1 - self.first_eccentricity ** 2 * sinlat ** 2
        )

    def geodetic_to_spherical(self, longitude, latitude, height):
        """
        Convert from geodetic to geocentric spherical coordinates.

        The geodetic datum is defined by this ellipsoid. The coordinates are
        converted following [Vermeille2002]_.

        Parameters
        ----------
        longitude : array
            Longitude coordinates on geodetic coordinate system in degrees.
        latitude : array
            Latitude coordinates on geodetic coordinate system in degrees.
        height : array
            Ellipsoidal heights in meters.

        Returns
        -------
        longitude : array
            Longitude coordinates on geocentric spherical coordinate system in
            degrees.
            The longitude coordinates are not modified during this conversion.
        spherical_latitude : array
            Converted latitude coordinates on geocentric spherical coordinate
            system in degrees.
        radius : array
            Converted spherical radius coordinates in meters.

        """
        latitude_rad = np.radians(latitude)
        coslat, sinlat = np.cos(latitude_rad), np.sin(latitude_rad)
        prime_vertical_radius = self.prime_vertical_radius(sinlat)
        # Instead of computing X and Y, we only compute the projection on the
        # XY plane: xy_projection = sqrt( X**2 + Y**2 )
        xy_projection = (height + prime_vertical_radius) * coslat
        z_cartesian = (
            height + (1 - self.first_eccentricity ** 2) * prime_vertical_radius
        ) * sinlat
        radius = np.sqrt(xy_projection ** 2 + z_cartesian ** 2)
        spherical_latitude = np.degrees(np.arcsin(z_cartesian / radius))
        return longitude, spherical_latitude, radius

    def spherical_to_geodetic(self, longitude, spherical_latitude, radius):
        """
        Convert from geocentric spherical to geodetic coordinates.

        The geodetic datum is defined by this ellipsoid. The coordinates are
        converted following [Vermeille2002]_.

        Parameters
        ----------
        longitude : array
            Longitude coordinates on geocentric spherical coordinate system in
            degrees.
        spherical_latitude : array
            Latitude coordinates on geocentric spherical coordinate system in
            degrees.
        radius : array
            Spherical radius coordinates in meters.

        Returns
        -------
        longitude : array
            Longitude coordinates on geodetic coordinate system in degrees.
            The longitude coordinates are not modified during this conversion.
        latitude : array
            Converted latitude coordinates on geodetic coordinate system in
            degrees.
        height : array
            Converted ellipsoidal height coordinates in meters.

        """
        spherical_latitude = np.radians(spherical_latitude)
        k, big_z, big_d = self._spherical_to_geodetic_terms(spherical_latitude, radius)
        latitude = np.degrees(
            2 * np.arctan(big_z / (big_d + np.sqrt(big_d ** 2 + big_z ** 2)))
        )
        height = (
            (k + self.first_eccentricity ** 2 - 1)
            / k
            * np.sqrt(big_d ** 2 + big_z ** 2)
        )
        return longitude, latitude, height

    def _spherical_to_geodetic_terms(self, spherical_latitude, radius):
        "Calculate intermediate terms needed for the conversion."
        # Offload computation of these intermediate variables here to clean up
        # the main function body
        cos_latitude = np.cos(spherical_latitude)
        big_z = radius * np.sin(spherical_latitude)
        p_0 = radius ** 2 * cos_latitude ** 2 / self.semimajor_axis ** 2
        q_0 = (1 - self.first_eccentricity ** 2) / self.semimajor_axis ** 2 * big_z ** 2
        r_0 = (p_0 + q_0 - self.first_eccentricity ** 4) / 6
        s_0 = self.first_eccentricity ** 4 * p_0 * q_0 / 4 / r_0 ** 3
        t_0 = np.cbrt(1 + s_0 + np.sqrt(2 * s_0 + s_0 ** 2))
        u_0 = r_0 * (1 + t_0 + 1 / t_0)
        v_0 = np.sqrt(u_0 ** 2 + q_0 * self.first_eccentricity ** 4)
        w_0 = self.first_eccentricity ** 2 * (u_0 + v_0 - q_0) / 2 / v_0
        k = np.sqrt(u_0 + v_0 + w_0 ** 2) - w_0
        big_d = k * radius * cos_latitude / (k + self.first_eccentricity ** 2)
        return k, big_z, big_d

    def normal_gravity(
        self, latitude, height, si_units=False
    ):  # pylint: disable=too-many-locals
        """
        Calculate normal gravity at any latitude and height.

        Computes the magnitude of the gradient of the gravity potential
        (gravitational + centrifugal) generated by the ellipsoid at the given
        latitude and (geometric) height. Uses of a closed form expression of
        [LiGotze2001]_.

        Parameters
        ----------
        latitude : float or array
            The (geodetic) latitude where the normal gravity will be computed
            (in degrees).
        height : float or array
            The ellipsoidal (geometric) height of computation the point (in
            meters).
        si_units : bool
            Return the value in mGal (False, default) or SI units (True)

        Returns
        -------
        gamma : float or array
            The normal gravity in mGal.

        """
        # Warn if height is negative
        if np.any(height < 0):
            warn(
                "Formulas used are valid for points outside the ellipsoid."
                "Height must be greater than or equal to zero."
            )

        sinlat = np.sin(np.deg2rad(latitude))
        coslat = np.sqrt(1 - sinlat ** 2)
        # The terms below follow the variable names from Li and Goetze (2001)
        cosbeta_l2, sinbeta_l2, b_l, q_0, q_l, big_w = self._normal_gravity_terms(
            sinlat, coslat, height
        )
        # Put together gamma using 3 terms
        term1 = self.geocentric_grav_const / (b_l ** 2 + self.linear_eccentricity ** 2)
        term2 = (0.5 * sinbeta_l2 - 1 / 6) * (
            self.semimajor_axis ** 2
            * self.linear_eccentricity
            * q_l
            * self.angular_velocity ** 2
            / ((b_l ** 2 + self.linear_eccentricity ** 2) * q_0)
        )
        term3 = -cosbeta_l2 * b_l * self.angular_velocity ** 2
        gamma = (term1 + term2 + term3) / big_w
        if si_units:
            return gamma
        # Convert gamma from SI to mGal
        return gamma * 1e5

    def _normal_gravity_terms(self, sinlat, coslat, height):
        "Calculate intermediate terms needed for the calculations."
        # Offload computation of these intermediate variables here to clean up
        # the main function body
        beta = np.arctan2(self.semiminor_axis * sinlat, self.semimajor_axis * coslat)
        zl2 = (self.semiminor_axis * np.sin(beta) + height * sinlat) ** 2
        rl2 = (self.semimajor_axis * np.cos(beta) + height * coslat) ** 2
        big_d = (rl2 - zl2) / self.linear_eccentricity ** 2
        big_r = (rl2 + zl2) / self.linear_eccentricity ** 2
        cosbeta_l2 = 0.5 * (1 + big_r) - np.sqrt(0.25 * (1 + big_r ** 2) - 0.5 * big_d)
        sinbeta_l2 = 1 - cosbeta_l2
        b_l = np.sqrt(rl2 + zl2 - self.linear_eccentricity ** 2 * cosbeta_l2)
        q_0 = 0.5 * (
            (1 + 3 * (self.semiminor_axis / self.linear_eccentricity) ** 2)
            * np.arctan2(self.linear_eccentricity, self.semiminor_axis)
            - 3 * self.semiminor_axis / self.linear_eccentricity
        )
        q_l = (
            3
            * (1 + (b_l / self.linear_eccentricity) ** 2)
            * (
                1
                - b_l
                / self.linear_eccentricity
                * np.arctan2(self.linear_eccentricity, b_l)
            )
            - 1
        )
        big_w = np.sqrt(
            (b_l ** 2 + self.linear_eccentricity ** 2 * sinbeta_l2)
            / (b_l ** 2 + self.linear_eccentricity ** 2)
        )
        return cosbeta_l2, sinbeta_l2, b_l, q_0, q_l, big_w


# Default configuration for Cherry Py REST server

import cherrypy

config = {
    'global': {
        'server.thread_pool' : 30,  # Thread pool size (sys def = 10)
        'server.socket_host' : '0.0.0.0', # Current host
        'server.socket_port' : 8000,
        'server.max_request_body_size' : 400 * (1024 ** 2),  # N megabytes
        'engine.autoreload_on' : False,  # Restart when source files changed?
        'tools.gzip.on' : True,     # Support gzip compression
        'log.screen' : True,
        },
    '/': {
        'request.dispatch' : cherrypy.dispatch.MethodDispatcher(),
        # Let user see tracebacks.  Very useful for debugging, but
        # not typically considered good form as it exposes information
        # to users that could be exploited
        'request.show_tracebacks' : True,
        #change timeout duration
        'response.timeout' : 1080 #18 minutes
        }
    }


import gym
import numpy as np
import matplotlib.pyplot as plt

num_images = 100
env = gym.make('Pong-v0')
env.reset()
done = False
path = 'env_images_test/'
for i in range(1000):
    # env.render()
    state, reward, done, _ = env.step(env.action_space.sample())
    state = state[35:195:2, ::2, 0]
    state[np.logical_or(state == 144, state == 109)] = 0
    state[state != 0] = 1
    state = state.astype(np.float)
    # state = np.expand_dims(state, axis=-1)
    if i>=500 and i<=500+num_images:
        plt.imsave(path+'{}.png'.format(i), state, cmap='gray')
    # print(state.shape)


# -*- coding: utf-8 -*-
'''Run a prediction for a comment through the reddit May 2015 hate speech model'''

from sklearn.feature_extraction.text import TfidfVectorizer
from string import punctuation
from nltk import word_tokenize
from nltk.stem import snowball
import xgboost as xgb
import cPickle as pickle
import numpy as np
import pandas as pd

stemmer = snowball.SnowballStemmer("english")

def stem_tokens(tokens, stemmer):
    '''Stem the tokens.'''
    stemmed = []
    for item in tokens:
        stemmed.append(stemmer.stem(item))
    return stemmed

def tokenize(text):
    '''Tokenize & stem. Stems automatically for now.
    Leaving "stemmer" out of function call, so it works with TfidfVectorizer'''
    tokens = word_tokenize(text)
    stems = stem_tokens(tokens, stemmer)
    return stems

def predict_comment(comment, classes, bst, vect):
    '''
    Where "comment" is the comment by the user, to be passed in.
    classes =
    '''
    comment_tfidf = vect.transform([comment])
    comment_xgb = xgb.DMatrix(comment_tfidf)
    yprob = bst.predict(comment_xgb).reshape(1, 5)  # hard coding -- only one comment at a time in this case.
    ylabel = classes[np.argmax(yprob, axis=1)]

    # print('The class is: {0} with probability {1}%'.format(ylabel, round(100 * np.max(yprob), 1)))

    return ylabel, round(100*np.max(yprob), 1), comment

def main():
    classes = ['Not Hate', 'Size Hate', 'Gender Hate', 'Race Hate', 'Religion Hate']

    # load saved xgboost model
    bst = xgb.Booster()
    bst.load_model('../FinalModel/modelv1/BuildModel/hatespeech.model')
    # load tf-idf matrix
    # tfidf_X = pickle.load(open('../FinalModel/BuildModel/tfidf_X.p', 'rb'))
    vect = pickle.load(open('../FinalModel/modelv1/BuildModel/vect.p', 'rb'))

    # get comment from user
    comment = raw_input('Enter comment: ')
    # predict class of comment
    predict_comment(comment, classes, bst, vect)

    predict = raw_input("Enter 'y' to get another prediction.")

    while predict == 'y':
        # get comment from user
        comment = raw_input('Enter comment: ')
        # predict class of comment
        predict_comment(comment, classes, bst, vect)
        predict = raw_input("Enter 'y' to get another prediction.")

if __name__ == "__main__":
    main()


'''
Created on Sep 18, 2016

@author: Julia Yu
'''

import os
import json

class compareAPIResult():
    def compareAPIResult(self, dellID, apiName, actualResult):
        baselineFilePath = os.path.join(os.path.dirname(os.path.realpath('__file__')),'testdata/ptagentTestResult_%s_baseline.json'%dellID)
        with open(baselineFilePath) as f:
            baseResults=json.load(f)
        for baseline in baseResults:
            if baseline['api'] == apiName:
                expectedResult = baseline['result']
                break
        if cmp(actualResult, expectedResult) == 0:
            return True
        else:
            return False


# -*- coding: utf-8 -*-
from gtts import gTTS
from playsound import playsound
from threading import Thread
from datetime import date
from fuzzywuzzy import process
import sys
import os
import time
import sched
import requests
import random
import json
import threading

# this is a full text version of omega, for debugging
val = 1

def omegaSpeak(text):
    print(text)

def omegaTaskCompleted(user):
    # possible responses
    val = ''
    # priant 1 random number in the range of 1 and 2
    for x in range(1):
        val = random.randint(1, 2)
    if val == 1:
        omegaSpeak('ok, done')
    if val == 2:
        omegaSpeak('That has finished %s' % user)

def takingCommands(request):
    if request == False:
        try:
            # command = r.recognize_google(audio)
            print(' 🧑 You said: {}'.format(command))
            omegaCommands(command.lower())
        except SystemExit:
            sys.exit()
        except:
            print('')
    else:
        omegaCommands(request)

def omegaCommands(command):
    # get the current user before every command
    name = readFromData()
    if command == 'omega':
        omegaSpeak('Yes %s' % name)
    if 'shutdown' in command:
        omegaSpeak("ok, closing")

    if 'what is the date' in command:
        omegaSpeak('One second %s ' % name)
        today = str(date.today())
        omegaSpeak(today)
    if 'new user' in command:
        print('new user')
    if 'hello' in command:
        omegaSpeak('Hello %s' % name)
        omegaTaskCompleted(name)
    if 'what is my name' in command:
        if name == '':
            omegaSpeak('I cannot see you')
        elif name == 'no user':
            omegaSpeak('I cannot see you')
        else:
            omegaSpeak('You are %s' % name)
    if 'turn off' in command:
        lights(command, 'off', user)
    if 'turn on' in command:
        lights(command, 'on', user)

# need an alwasys listening function and then it calls the commands function

# hue light control
# command is the command given
# state is off or on
# user is omega tries to get the user that it sees

def lights(command, state, user):
    hueLights = []
    # get lights
    resp = requests.get('https://discovery.meethue.com/')
    ipAddress = ''
    for ip in resp.json():
        ipAddress = ip['internalipaddress']
    lights = requests.get(
        'http://%s/api/pf3enyrZJz4uvwgYf90t9E2FzLM4tW2GeGSnO-ut/lights' % ipAddress)
    # split the command at the '
    if 'turn off the' in command:
        command = command.split('turn off the')
    else:
        command = command.split('turn on the')
    lightName = command[1]
    # put all the light name from hue in array
    for i in lights.json():
        hueLights.append(
            {'index': i, 'name': '{}' .format(lights.json()[i]['name'])})
    # fuzzy search for best matching light name
    finalLight = process.extractOne(lightName, hueLights)
    print(finalLight[0]['index'])
    # set state of light
    if state == 'on':
        payload = " {\"on\":true}"
    else:
        payload = " {\"on\":false}"
    requests.put(
        'http://%s/api/pf3enyrZJz4uvwgYf90t9E2FzLM4tW2GeGSnO-ut/lights/%s/state' % (ipAddress, finalLight[0]['index']), data=payload)
    #     else:
    omegaTaskCompleted(user)

def typeToAssist():
    command = input('What is the command?')
    omegaCommands(command)

def readFromData():
    with open('../person.txt') as json_file:
        data = json.load(json_file)
        return data["person"]

while val == 1:
    typeToAssist()


from graphviz import Digraph

#We arrange the graph using Networkx and then paint it using Cytoscape.
G = Digraph(comment="Lung Cancer Experiment")
G.node('1', "Tobacco Use")
G.node('2', "Lung Cancer")
G.node('3', "Secondhand Smoke")
G.node('4', "Heavy Smoking")
G.node('5', "Complications")
G.node('6', "Death")
G.node('7', "Symptoms")
G.node('8', "Bone Pain")
G.node('9', "Treatable")
G.node('10',"Radiation Therapy")
G.node('11',"Bleeding")
G.node('12',"Chemical Imbalances")
G.node('13',"Pain")
G.node('14',"Speech Difficulties")
G.node('15',"Breathing Difficulties")
G.node('16',"Metastasize")
G.node('17',"Collect Fluid")
G.node('19',"Fatigue")
G.node('20',"Loss of appetite")
G.node('21',"Workplace")
G.node('22',"Radon Gas")
G.edge('1','2')
G.edge('3','2')
G.edge('4','2')
G.edge('2','5')
G.edge('2','5')
G.edge('2','6')
G.edge('2','7')
G.edge('2','8')
G.edge('2','9')
G.edge('2','10')
G.edge('2','11')
G.edge('2','12')
G.edge('2','13')
G.edge('2','14')
G.edge('2','15')
G.edge('2','16')
G.edge('2','17')
G.edge('9','19')
G.edge('9','20')
G.edge('21','2')
G.edge('22','2')
G.render("causal_graph")


from sklearn.neighbors import KNeighborsClassifier, NearestNeighbors
import numpy as np
from lmnn import LargeMarginNearestNeighbor as LMNN

class imls():
    def __init__(self, k=3, mu=0.5, coef=5, randomState=np.random):
        self.coef = coef
        self.k = k
        self.mu = mu
        self.randomState = randomState

    def fitPredict(self, Xtrain, ytrain, Xtest):
        while self.coef*self.k > Xtrain.shape[0]:
            self.coef -= 1
        nn = NearestNeighbors(n_neighbors=self.coef*self.k)
        nn.fit(Xtrain)
        nearestNeighbors = nn.kneighbors(Xtest, return_distance=False)
        lmnn = LMNN(k=self.k,randomState=self.randomState, mu=self.mu)
        lmnn.fit(Xtrain, ytrain)
        Xtrain = lmnn.transform(Xtrain)
        Xtest = lmnn.transform(Xtest)
        nn = NearestNeighbors(n_neighbors=self.coef*self.k)
        nn.fit(Xtrain)
        newNearestNeighbors = nn.kneighbors(Xtest, return_distance=False)
        matching = np.array([len(np.intersect1d(
               nearestNeighbors[i],newNearestNeighbors[i]))>=int(self.coef*0.8)
                             for i in range(len(nearestNeighbors))])
        while matching.all() == False:
            nearestNeighbors = newNearestNeighbors.copy()
            lmnn = LMNN(k=self.k,randomState=self.randomState, mu=self.mu)
            lmnn.fit(Xtrain, ytrain)
            Xtrain = lmnn.transform(Xtrain)
            Xtest = lmnn.transform(Xtest)
            nn = NearestNeighbors(n_neighbors=self.coef*self.k)
            nn.fit(Xtrain)
            newNearestNeighbors = nn.kneighbors(Xtest, return_distance=False)
            matching = np.array([len(np.intersect1d(
              nearestNeighbors[i], newNearestNeighbors[i]))>=int(self.coef*0.8)
                                 for i in range(len(nearestNeighbors))])
        knc = KNeighborsClassifier(n_neighbors=self.k)
        knc.fit(Xtrain, ytrain)
        return knc.predict(Xtest)


"""
Give the Host and Gene name to print the express gene names
"""
import argparse
import re
import sys
from assignment5 import my_io
from assignment5 import config

def main():
    """
    Main function
    :return: Prints out Expressed genes in Mammal in STD.OUT format
    """
    args = get_cli_args()
    # function to check for passed arguments
    temp_host, gene = check_passed_args(args)
    # getting scientific name
    host = modify_host_name(temp_host)
    # file name and absolute path
    file = "/".join((config.get_unigene_directory(),
                     host, gene + "." + config.get_unigene_extension()))
    # checking if file exist in directory or not
    _check_file_exist(file, host, gene)
    # List of expressed genes sorted alphabetically
    tissue_strig = get_gene_data(file)
    # Print output on STD.OUT
    print_ouput(temp_host, gene, tissue_strig)

def print_ouput(host, gene, tissue_string):
    """
    Print Expressed gene name on STD OUT
    :param host: Name of the host
    :param gene: Name of the gene passed
    :param tissue_string: sorted list of expressed genes
    :return: None
    """
    print(f"In {host}, There are {len(tissue_string)} "
          f"tissues that {gene} is expressed in:\n")

    for i, value in enumerate(tissue_string, start=1):
        print(f"{i}. {value.title()}")

def get_gene_data(gene_file):
    """
    Get Epressed gene names from host file.
    :param gene_file: Absolute path to gene file of host
    :return: Sorted list of expresssed genes in host
    """
    fh_in = my_io.get_fh(gene_file, "r")

    tissue_strig = []

    for line in fh_in:
        if re.search("EXPRESS", line):
            line = line.replace("\n", "")
            line = re.sub('[A-Z]', "", line)
            tissue_strig = line.split("|")
            tissue_strig = [x.strip(' ') for x in tissue_strig]

    my_io.get_fh(fh_in, "close")

    return sorted(tissue_strig)

def _check_file_exist(file, temp_host, gene):
    # check for the existence of file
    if my_io.is_valid_gene_file_name(file):
        # using f-strings
        print(f"\nFound Gene {gene} for {temp_host}")
    else:
        print("Not found")
        print(f"Gene {gene} does not exist for {temp_host}. "
              f"exiting now...", file=sys.stderr)
        sys.exit()

def modify_host_name(host_name):
    """
    Get Scientific name from dictionary exist in config file
    if name with "_" is passed it can be treated as scientifc name
    :param host_name: Argument passed in CLI otions
    :return: Scientific name for Host
    """
    scientific_name = ""

    if "_" in host_name:
        scientific_name = host_name
    else:
        if host_name.lower() in list(config.get_host_keywords().keys()):
            scientific_name = config.get_host_keywords()[host_name.lower()]
        else:
            _print_host_directories()
            scientific_name = host_name

    return scientific_name

def _print_host_directories():
    """
    Internal function to print the name of valid Hosts data available
    in directory scientific and non-scientific both (case-insensitive)
    :return: NONE exits the program
    """

    print("\nEither the Host Name you are searching for is not in the database"
          "\nor If you are trying to use the scientific name please "
          "put the name in double quotes:\n"
          "\n\"Scientific name\"\n"
          "\nHere is a (non-case sensitive) list of available Hosts by scientific name\n")

    for i, value in enumerate(set(list(config.get_host_keywords().values())), start=1):
        print(f"{i}. {value}")

    print("\nHere is a (non-case sensitive) list of available Hosts by common name\n")

    for i, key in enumerate(list(config.get_host_keywords().keys()), start=1):
        print(f"{i}. {key.title()}")

def check_passed_args(args):
    """
    Check how many arguments passed, if NONE: return
    default file options
    :param args: Argparse file arguments, passed in CLI
    :return: Names of the files to open
    """

    host = "Homo_sapiens"
    gene = "TGM1"
    args_to_return1 = ""
    args_to_return2 = ""

    if len(sys.argv) > 2:
        args_to_return1 = args.HOST
        args_to_return2 = args.GENE
    else:
        args_to_return1 = host
        args_to_return2 = gene

    return args_to_return1, args_to_return2

def get_cli_args():
    """
    Get Command Line Argument function to read arguments from command
    line using argparse
    :return: Argument Parser object with all the required options
    """
    parser = argparse.ArgumentParser(description="Give the Host and Gene name")

    parser.add_argument("-host",
                        dest="HOST",
                        type=str,
                        help="Name of Host",
                        required=False)

    parser.add_argument("-gene",
                        dest="GENE",
                        type=str,
                        help="Name of Gene",
                        required=False)

    return parser.parse_args()

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-

"""Utilities for building paths."""

import logging
import os
import tarfile
from typing import Optional
from urllib.parse import urlparse
from urllib.request import urlretrieve

import pandas as pd

from .constants import PYOBO_HOME

__all__ = [
    'get_prefix_directory',
    'prefix_directory_join',
    'get_prefix_obo_path',
    'get_url_filename',
    'ensure_path',
    'ensure_df',
    'ensure_excel',
    'ensure_tar_df',
]

logger = logging.getLogger(__name__)

def get_prefix_directory(prefix: str, *, version: Optional[str] = None) -> str:
    """Get the directory."""
    if version:
        directory = os.path.abspath(os.path.join(PYOBO_HOME, prefix, version))
    else:
        directory = os.path.abspath(os.path.join(PYOBO_HOME, prefix))
    os.makedirs(directory, exist_ok=True)
    return directory

def prefix_directory_join(prefix: str, *parts: str, version: Optional[str] = None) -> str:
    """Join the parts onto the prefix directory."""
    rv = os.path.join(get_prefix_directory(prefix, version=version), *parts)
    os.makedirs(os.path.dirname(rv), exist_ok=True)
    return rv

def get_prefix_obo_path(prefix: str) -> str:
    """Get the canonical path to the OBO file."""
    return prefix_directory_join(prefix, f"{prefix}.obo")

def get_url_filename(url: str) -> str:
    """Get the filename from the end of the URL."""
    parse_result = urlparse(url)
    return os.path.basename(parse_result.path)

def ensure_path(
    prefix: str,
    url: str,
    *,
    version: Optional[str] = None,
    path: Optional[str] = None,
) -> str:
    """Download a file if it doesn't exist."""
    if path is None:
        path = get_url_filename(url)

    if version:
        path = prefix_directory_join(prefix, path, version=version)
    else:
        path = prefix_directory_join(prefix, path)

    if not os.path.exists(path):
        logger.info('[%s] downloading OBO from %s', prefix, url)
        urlretrieve(url, path)

    return path

def ensure_df(
    prefix: str,
    url: str,
    *,
    version: Optional[str] = None,
    path: Optional[str] = None,
    sep: str = '\t',
    **kwargs,
) -> pd.DataFrame:
    """Download a file and open as a dataframe."""
    path = ensure_path(prefix, url, version=version, path=path)
    return pd.read_csv(path, sep=sep, **kwargs)

def ensure_excel(
    prefix: str,
    url: str,
    *,
    version: Optional[str] = None,
    path: Optional[str] = None,
    **kwargs,
) -> pd.DataFrame:
    """Download an excel file and open as a dataframe."""
    path = ensure_path(prefix, url, version=version, path=path)
    return pd.read_excel(path, **kwargs)

def ensure_tar_df(
    prefix: str,
    url: str,
    inner_path: str,
    *,
    version: Optional[str] = None,
    path: Optional[str] = None,
    **kwargs,
) -> pd.DataFrame:
    """Download a tar file and open as a dataframe."""
    path = ensure_path(prefix, url, version=version, path=path)
    with tarfile.open(path) as tar_file:
        with tar_file.extractfile(inner_path) as file:
            return pd.read_csv(file, **kwargs)


#!/usr/bin/python3
""" states.py objects that handle all default API actions
    using GET, POST, DELETE"""
from models import storage
from models.state import State
from flask import jsonify, abort, request
from api.v1.views import app_views

@app_views.route('/states', methods=['GET'], strict_slashes=False)
@app_views.route('/states/<state_id>', methods=['GET'], strict_slashes=False)
def get_state_id(state_id=None):
    """ returns the state dictionary """
    if state_id is None:
        obj_list = []
        objs = storage.all("State")
        for obj in objs.values():
            obj_list.append(obj.to_dict())
        return jsonify(obj_list)
    obj = storage.get("State", state_id)
    if obj is None:
        abort(404)
    return jsonify(obj.to_dict())

@app_views.route('/states/<state_id>', methods=['DELETE'],
                 strict_slashes=False)
def delete_state(state_id=None):
    """ returns the state id """
    obj = storage.get("State", state_id)
    if obj is None:
        abort(404)
    storage.delete(obj)
    storage.save()
    return jsonify({}), 200

@app_views.route('/states', methods=['POST'], strict_slashes=False)
def new_state():
    """ creates a new state """
    if request.get_json() is None:
        abort(400, "Not a JSON")
    dic = request.get_json()
    if 'name' not in dic:
        abort(400, "Missing name")
    obj = State(**dic)
    storage.new(obj)
    storage.save()
    return jsonify(obj.to_dict()), 201

@app_views.route('/states/<state_id>', methods=['PUT'], strict_slashes=False)
def update_state(state_id=None):
    """ Updates the states """
    if request.get_json() is None:
        abort(400, "Not a JSON")
    obj = storage.get("State", state_id)
    if obj is None:
        abort(404)
    dic = request.get_json()
    dic.pop('created_at', None)
    dic.pop('updated_at', None)
    dic.pop('id', None)
    for key, value in dic.items():
        setattr(obj, key, value)
    obj.save()
    return jsonify(obj.to_dict()), 200


#!/usr/bin/python3

from random import random, choice, randint
from locale import getlocale
from os.path import exists, join
from datetime import datetime, timedelta
from math import floor

import sqlite3

MALE_FIRST_NAMES = 'maleFirstNames'

class DataSource:

    MALE_NAMES = 'maleNames'
    FEMALE_NAMES = 'femaleNames'
    LAST_NAMES = 'lastNames'

    def __init__(self, dataSource):
        self.dataSource = dataSource

    def loadDataItems(self, localesDir, dataItems=None, locales=None):
        pass
        # if localesDir == None or len(localesDir.strip()) == 0:
            # raise Exception('localesDir not specified')
            #
        # if not exists(localesDir):
            # raise Exception("%s not found" % localesDir)
            #
        # self.localesDir = localesDir
        #
        # if dataItems == None or len(dataItems) == 0:
            # dataItems = DataSource.ALL_DATA_ITEMS
            #
        # if locales == None or len(locales) == 0:
            # locales = [getlocale()[0]]
        #
        # for dataItem in dataItems:
            # if dataItem not in DataSource.ALL_DATA_ITEMS:
                # raise Exception('unrecognized data item %s' % dataItem)
            # for locale in locales:
                # self.loadDataItem(dataItem, locale)

    def loadDataItem(self, dataItem, locale, *posArgs, **keywords):
        raise Exception('implement this method in subclass')

    def randomMaleName(self):
        return self.randomChoice(DataSource.MALE_NAMES)

    def randomFemaleName(self):
        return self.randomChoice(DataSource.FEMALE_NAMES)

    def randomLastName(self):
        return self.randomChoice(DataSource.LAST_NAMES)

class InMemoryDataSource(DataSource):

    def __init__(self):
        self.dataItems = {}

    def loadDataItem(self, dataItem, locale, *posArgs, **keywords):
        if 'values' not in keywords.keys():
            raise Exception('values not specified')

        self.dataItems[dataItem] = keywords['values']

    def randomChoice(self, dataItemName):
        if dataItemName not in self.dataItems.keys():
            raise Exception(dataItemName + " not present in data items")
        return choice(self.dataItems[dataItemName])

class SqliteDataSource(DataSource):

    def __init__(self):
        self.conn = None

    def open(self, dbFile):
        '''Opens an existing sqllite file if file exists.  Will create one if
        it does not exist'''

        self.conn = sqlite3.connect(dbFile)

    def loadDataItem(self, dataItem, locale):
        # Call base class method to validate that  files exist
        DataSource.loadDataItem(self, dataItem, locale)
        print('loadDataItem')

        cursor = self.conn.cursor()

        if not self.hasTable('nameControlTable'):
            cursor.execute(
                """create table if not exists maleFirstNames (
                    tableName text)""")

        if self.hasTable(dataItem):
            if self.hasRecords(dataItem):
                cursor.execute('delete maleFirstNames')
        else:
            cursor.execute(
                """create table if not exists maleFirstNames (
                    name text,
                    randSort integer)""")

        sourceFile = open(self.sourceFilePath, 'r')

        for line in sourceFile:
            line = line.strip()
            print(line)
            cursor.execute(
                "insert into maleFirstNames (name) values (?)",
                (line,))

        sourceFile.close()

    def hasTable(self, tableName):
        cursor = self.conn.cursor()
        cursor.execute(
            "select * from sqlite_master where tbl_name = ?",
            (tableName,))
        for row in cursor:
            return True
        return False

    def hasRecords(self, tableName):
        cursor = self.conn.cursor()
        cursor.execute(
            "select count(*) from %s" % tableName)
        for row in cursor:
            if row[0] == 0:
                return False
        return True

    def randomMaleName(self):
        pass

    def randomFemaleName(self):
        pass

    def close(self):
        if self.conn != None:
            self.conn.close()

    # def __init__(self):
        # self.conn = None
        #
    # def reLoad(self):
        # """opens an existing sqlite database"""
        # pass
        #
    # def load(self, localesDir, databaseFile, locales=None, dataItems=None):
        # """clears database and loads data into sqlite database from file."""
        # # check if localesDirectory exists
        #
        #
        #
        # if locales == None or len(locales) == 0:
            # locales = [getlocale()[0]]
        #
        # for locale in locales:
            # # do this in a cross platform way
            # localeDir = localesDir + '/' + locale
            # print(localeDir)
            #
    # def close(self):
        # self.conn.close()

class DataGenerationRule:
    def __init__(self):
        pass

class PersonAgeRule(DataGenerationRule):
    def getAge(self):
        return 90 * random()

class PersonSexRule(DataGenerationRule):
    def getSex(self):
        x = random()
        if x <= 0.495:
            return 'male'
        if x > 0.495 and x < 0.99:
            return 'female'
        return 'unknown'

class PersonContext:

    def __init__(self):
        self.currentDateTime = datetime.now()

        self.ageRule = PersonAgeRule()
        self.sexRule = PersonSexRule()

class Person:

    SEXES = ['male', 'female', 'unknown']

    def __init__(self, dataSource, context=None, **keywords):

        if context == None:
            context = PersonContext()

        if 'sex' in keywords.keys():
            self.sex = keywords['sex']
            if self.sex not in Person.SEXES:
                raise(ValueError())
        else:
            self.sex = context.sexRule.getSex()

        self.age = context.ageRule.getAge()

        self.dateOfBirth = datetime.today() - timedelta(self.age * 365.25)
        self.age = floor(self.age)

        self.firstName, tempSex = self.firstOrMiddleName(
            dataSource,
            self.sex,
            'firstName',
            **keywords)

        self.middleName, tempSex = self.firstOrMiddleName(
            dataSource,
            tempSex,
            'middleName',
            **keywords)

        self.lastName = dataSource.randomLastName()

    def firstOrMiddleName(self, dataSource, sex, nameType, **keywords):
        if nameType in keywords.keys():
            return keywords[nameType]
        if sex == 'male' or (sex == 'unknown' and random() < 0.5):
            return dataSource.randomMaleName(), 'male'
        return dataSource.randomFemaleName(), 'female'

    def __str__(self):
        array = {
            'sex': self.sex,
            'firstName': self.firstName,
            'middleName': self.middleName,
            'lastName': self.lastName,
            'dateOfBirth':self.dateOfBirth,
            'age':self.age}
        return str(array)

class Household:

    def __init__(self, dataSource, context=None, **keywords):
        numberOfAdults = randint(1,5)
        print("there are %d adults in the household")

        self.members = []

        if numberOfAdults > 1:
            if self.hasMarriedCouple():
                self.generateMarriedCouple(dataSource)

    def __str__(self):
        return str([str(x) for x in self.members])

    def hasMarriedCouple(self):
        return choice([True, False])

    def generateMarriedCouple(self, dataSource):
        spouse1 = Person(dataSource)
        self.members.append(spouse1)
        if self.isHomosexualCouple():
            pass

    def isHomosexualCouple(self):
        return randint(1,50) == 1
        # if 'middleName' in keywords.keys():
            # self.middleName = keywords['middleName']
        # else:
            # self.middleName = randomMiddleName(self.sex)
            #
        # if 'lastName' in keywords.keys():
            # self.lastName = keywords['lastName']
        # self.lastName = randomLastName(self.sex)
        #
        # if 'minAge' in keywords.keys():
            # minAge = keywords['minAge']
            # # add check to ensure minAge is numeric
        # else:
            # minAge = 0
            #
        # if 'maxAge' in keywords.keys():
            # maxAge = keywords['maxAge']
            # # add check to ensure maxAge is numeric
        # else:
            # maxAge = 110
            #
        # self.age = randint(minAge*100, maxAge*100)/100.0

# def randomCouple(**keywords):
    # """Will generate two random people who may or may not have the last name.
    # This function will override values for the sex and lastName keywords"""
    # if random() < 0.05:
        # # same - sex couple
        # keywords['sex'] = randomSex()
        # if random() < 0.33:
            # # share last_name
            # keywords['lastName'] = randomLastName()
            # return RandomPerson(**keywords), RandomPerson(**keywords)
        # return RandomPerson(**keywords), RandomPerson(**keywords)
#
    # if random() < 0.70 and 'lastName' not in keywords.keys():
        # keywords['lastName'] = randomLastName()
        #
    # keywords['sex'] = 'M'
    # person1 = RandomPerson(**keywords)
    #
    # keywords['sex'] = 'F'
    # person2 = RandomPerson(**keywords)
    # return person1, person2

if __name__ == '__main__':
    pass


import sys, os

sys.path.insert(0, os.path.abspath('./'))

from tools.deep_learning.cnn_utils import evaluate_prediction

possible_selection_metrics = ["best_loss", "best_balanced_accuracy", "last_checkpoint"]

def get_test_prediction(test_diagnosis_df, merged_df):
    import pandas as pd

    test_diagnosis_df=merged_df.merge(test_diagnosis_df, on=["participant_id"], how="right")
    test_diagnosis_df=dict(list(test_diagnosis_df.groupby("T1w_mri_field")))
    res_dict={}
    for key in [1.5, 3]:
        res_dict["test_%sT"%str(key)]=test_diagnosis_df[key]
    return res_dict

def load_test_and_separate(results_dict, args, cnn_classification_dir, selection_metric, mode,  fold, merged_df=None, stat_dict=None):
    import pandas as pd

    if args.bayesian:
        for test_mode, values_df in stat_dict[fold][selection_metric].groupby("mode"):
            if "test" in test_mode:
                prediction_column = "predicted_label_from_%s" % args.ba_inference_mode
                test_diagnosis_eval = evaluate_prediction(values_df[["true_label"]].values.astype(int),
                                                          values_df[[prediction_column]].values.astype(int))
                test_diagnosis_df = pd.DataFrame(test_diagnosis_eval, index=[0])
                test_diagnosis_df = test_diagnosis_df.assign(fold=fold,
                                                             mode=test_mode)

                results_dict[selection_metric] = pd.concat(
                    [results_dict[selection_metric], test_diagnosis_df],
                    axis=0)
    else:
        test_diagnosis_path = os.path.join(cnn_classification_dir, selection_metric,
                                           '%s_image_level_prediction.tsv' % (mode))
        test_diagnosis_df = pd.read_csv(test_diagnosis_path, sep='\t')
        prediction_column = "predicted_label"

        test_diagnosis_dict = get_test_prediction(test_diagnosis_df, merged_df)
        for test_mode in test_diagnosis_dict:
            test_diagnosis_eval = evaluate_prediction(test_diagnosis_dict[test_mode][["true_label"]].values.astype(int),
                                                      test_diagnosis_dict[test_mode][[prediction_column]].values.astype(int))
            test_diagnosis_df = pd.DataFrame(test_diagnosis_eval, index=[0])
            test_diagnosis_df = test_diagnosis_df.assign(fold=fold,
                                                         mode=test_mode)
            results_dict[selection_metric] = pd.concat(
                [results_dict[selection_metric], test_diagnosis_df],
                axis=0)
    return results_dict

def load_mode_results(results_dict, args, cnn_classification_dir, selection_metric, mode,  fold, load_from_ba_predictions=False,stat_dict=None):
    import pandas as pd
    if args.bayesian and load_from_ba_predictions:
        values_df = stat_dict[fold][selection_metric].groupby("mode").get_group(mode)
        prediction_column = "predicted_label_from_%s" % args.ba_inference_mode
        test_diagnosis_eval = evaluate_prediction(values_df[["true_label"]].values.astype(int),
                                                  values_df[[prediction_column]].values.astype(int))
        test_diagnosis_df = pd.DataFrame(test_diagnosis_eval, index=[0])
        test_diagnosis_df = test_diagnosis_df.assign(fold=fold,
                                                     mode=mode)

        results_dict[selection_metric] = pd.concat(
            [results_dict[selection_metric], test_diagnosis_df],
            axis=0)
    else:
        test_diagnosis_path = os.path.join(cnn_classification_dir, selection_metric,
                                           '%s_image_level_metrics.tsv' % (mode))
        if os.path.exists(test_diagnosis_path):
            test_diagnosis_df = pd.read_csv(test_diagnosis_path, sep='\t')
            test_diagnosis_df = test_diagnosis_df.assign(fold=fold,
                                                         mode=mode)
            test_diagnosis_df = test_diagnosis_df.drop(["total_loss", "image_id", ], axis=1)

            results_dict[selection_metric] = pd.concat([results_dict[selection_metric], test_diagnosis_df],
                                                       axis=0)
    return results_dict

def get_results(args, aggregation_type="average"):
    # aggregation_type=[average, separate, together]
    import pandas as pd
    import os
    import pathlib
    import numpy as np

    merged_df = None
    stat_dict = None
    if args.bayesian:
        stat_dict = get_uncertainty_distribution(args, aggregation_type="separate")
    else:
        if args.separate_by_MS:
            merged_df = pd.read_csv(args.merged_file, sep="\t")
            merged_df=merged_df[["participant_id", "T1w_mri_field"]]

    results_dict = {}
    currentDirectory = pathlib.Path(args.model_path)
    currentPattern = "fold-*"

    for fold_dir in currentDirectory.glob(currentPattern):
        fold = int(str(fold_dir).split("-")[-1])
        cnn_classification_dir = os.path.join(args.model_path, 'fold-%i' % fold, 'cnn_classification')
        if args.selection_metrics is None:
            selection_metrics = []
            for f in os.scandir(cnn_classification_dir):
                metric=os.path.basename(os.path.normpath(f.path))
                if metric in possible_selection_metrics:
                    selection_metrics.append(metric)
        else:
            selection_metrics=args.selection_metrics

        for selection_metric in selection_metrics:
            if not selection_metric in results_dict.keys():
                results_dict[selection_metric] = pd.DataFrame()
            modes = ['train', 'validation']
            for ms_el in args.MS_list:
                modes.append('test_' + ms_el)

            for mode in modes:
                if "test" in mode:
                    if args.separate_by_MS:
                        results_dict=load_test_and_separate(results_dict, args, cnn_classification_dir, selection_metric, mode,
                                               fold, merged_df, stat_dict=stat_dict)
                    else:
                        results_dict=load_mode_results(results_dict, args, cnn_classification_dir, selection_metric, mode,  fold, load_from_ba_predictions=True, stat_dict=stat_dict)

                else:
                    results_dict=load_mode_results(results_dict, args, cnn_classification_dir, selection_metric, mode, fold,
                                      load_from_ba_predictions=False, stat_dict=None)

    resulting_metrics_dict = {}
    if aggregation_type=="average":
        for key in results_dict.keys():
            res_df = results_dict[key].drop(["fold"], axis=1)
            resulting_metrics_dict[key] = res_df.groupby(["mode"], as_index=False, sort=False).agg(np.mean)
        resulting_metrics_dict = {aggregation_type: resulting_metrics_dict}

    elif aggregation_type=="separate":
        for key in results_dict.keys():
            metric_dict = dict(list(results_dict[key].groupby("fold")))
            for fold in metric_dict.keys():
                if fold not in resulting_metrics_dict.keys():
                    resulting_metrics_dict[fold] = {}
                resulting_metrics_dict[fold][key] = metric_dict[fold]
    else:
        resulting_metrics_dict={"all":results_dict}
    return resulting_metrics_dict

def get_uncertainty_distribution(args, aggregation_type="average"):
    import pandas as pd
    import os
    import pathlib
    import numpy as np

    if args.separate_by_MS:
        merged_df = pd.read_csv(args.merged_file, sep="\t")
        merged_df=merged_df[["participant_id", "T1w_mri_field"]]
    currentDirectory = pathlib.Path(args.model_path)
    currentPattern = "fold-*"
    stat_dict = {}
    for fold_dir in currentDirectory.glob(currentPattern):
        fold = int(str(fold_dir).split("-")[-1])
        cnn_classification_dir = os.path.join(args.model_path, 'fold-%i' % fold, 'cnn_classification')

        if args.selection_metrics is None:
            selection_metrics = []
            for f in os.scandir(cnn_classification_dir):
                metric=os.path.basename(os.path.normpath(f.path))
                if metric in possible_selection_metrics:
                    selection_metrics.append(metric)
        else:
            selection_metrics = args.selection_metrics

        for selection_metric in selection_metrics:
            if not selection_metric in stat_dict.keys():
                stat_dict[selection_metric] = pd.DataFrame()
            modes = ['test_' + ms_el for ms_el in args.MS_list]

            for mode in modes:
                test_diagnosis_path = os.path.join(cnn_classification_dir, selection_metric,
                                                   "bayesian_statistics", '%s_image_level_stats.tsv' % (mode))
                test_diagnosis_df = pd.read_csv(test_diagnosis_path, sep='\t')
                test_diagnosis_df["class_variance"] = test_diagnosis_df["class_variance"].apply(
                    lambda x: x[1:-1].split()).apply(lambda x: [float(i) for i in x])

                if "test" in mode and args.separate_by_MS:
                    test_diagnosis_dict = get_test_prediction(test_diagnosis_df, merged_df)
                    for key in test_diagnosis_dict:
                        test_diagnosis_dict[key] = test_diagnosis_dict[key].assign(fold=fold, mode=key)
                        stat_dict[selection_metric] = pd.concat([stat_dict[selection_metric], test_diagnosis_dict[key]],
                                                                axis=0)
                else:
                    test_diagnosis_df = test_diagnosis_df.assign(fold=fold, mode=mode)
                    stat_dict[selection_metric] = pd.concat([stat_dict[selection_metric], test_diagnosis_df], axis=0)
                # stat_dict[selection_metric].reset_index(inplace=True, drop=True)

    resulting_stat_dict = {}
    if aggregation_type=="average" or aggregation_type=="all":
        for key in stat_dict.keys():
            stat_df = stat_dict[key]
            additional_colums_df = stat_df[
                ["true_label", "predicted_label_from_mean", "predicted_label_from_mode", "mode", "participant_id"]]
            additional_colums_df = additional_colums_df.groupby(["mode", "participant_id"], as_index=False,
                                                                sort=False).agg(pd.Series.mode)
            stat_df = stat_df.drop(
                ["true_label", "predicted_label_from_mean", "predicted_label_from_mode", "fold"], axis=1)
            resulting_stat_dict[key] = stat_df.groupby(["mode", "participant_id"], as_index=False, sort=False).agg(np.mean)
            resulting_stat_dict[key]=resulting_stat_dict[key].merge(additional_colums_df, on=["mode", "participant_id"], how="right")
        resulting_stat_dict = {aggregation_type: resulting_stat_dict}

    elif aggregation_type == "separate":
        for key in stat_dict.keys():
            metric_dict = dict(list(stat_dict[key].groupby("fold")))
            for fold in metric_dict.keys():
                if fold not in resulting_stat_dict.keys():
                    resulting_stat_dict[fold] = {}
                resulting_stat_dict[fold][key] = metric_dict[fold]

    return resulting_stat_dict

def get_history(args, aggregation_type="average"):
    import pandas as pd
    import os
    import pathlib
    import numpy as np

    currentDirectory = pathlib.Path(args.model_path)
    currentPattern = "fold-*"
    history_df = pd.DataFrame()
    for fold_dir in currentDirectory.glob(currentPattern):
        fold = int(str(fold_dir).split("-")[-1])
        history = pd.read_csv(os.path.join(args.model_path, 'fold-%i' % fold, 'training.tsv'), sep='\t')
        history = history.assign(fold=fold)
        history_df = pd.concat([history_df, history], axis=0)
    if aggregation_type == "average":
        history_df = history_df[
            ["epoch", "balanced_accuracy_train", "loss_train", "balanced_accuracy_valid", "loss_valid"]]
        history_df = {aggregation_type: history_df.groupby("epoch", as_index=False).agg(np.mean)}

    elif aggregation_type == "all":
        history_df={aggregation_type:history_df}
    else:
        history_df = dict(list(history_df.groupby("fold")))
    return history_df

def reshape_dictionary(dict_sample):
    res = dict()
    for key, val in dict_sample.items():
        for key_in, val_in in val.items():
            if key_in not in res:
                temp = dict()
            else:
                temp = res[key_in]
            temp[key] = val_in
            res[key_in] = temp
    return res

def get_data_generic(args, reshape_dict=True):
    data = {}
    for data_type in args.data_types:
        data[data_type] = eval("get_%s" % data_type)(args, args.aggregation_type)
    #data is now in format {data_type: {fold_0:, ...fold_n etc}}

    #toDo: turn off this function?
    if reshape_dict:
        # reshape data to format  {fold_0: {data_type_1:, ...data_type_i etc}}
        data = reshape_dictionary(data)
    return data


import sys

import cv2 as cv

def inside(r, q):
    rx, ry, rw, rh = r
    qx, qy, qw, qh = q
    return rx > qx and ry > qy and rx + rw < qx + qw and ry + rh < qy + qh

def draw_detections(img, rects, thickness=1):
    for x, y, w, h in rects:
        # the HOG detector returns slightly larger rectangles than the real objects.
        # so we slightly shrink the rectangles to get a nicer output.
        pad_w, pad_h = int(0.15*w), int(0.05*h)
        cv.rectangle(img, (x+pad_w, y+pad_h), (x+w-pad_w, y+h-pad_h), (0, 255, 0), thickness)

def find_faces_haar(frame, face_cascade):
    gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)

    faces = face_cascade.detectMultiScale(
        gray,
        scaleFactor=1.1,
        minNeighbors=5,
        minSize=(30, 30),
        flags=cv.CASCADE_SCALE_IMAGE
    )

    # Draw a rectangle around the faces
    for (x, y, w, h) in faces:
        cv.rectangle(frame, (x, y), (x+w, y+h), (0, 255, 0), 2)

    return frame

def find_faces_hog(img, hog):
    found, w = hog.detectMultiScale(img, winStride=(8,8), padding=(32,32), scale=1.05)
    found_filtered = []
    for ri, r in enumerate(found):
        for qi, q in enumerate(found):
            if ri != qi and inside(r, q):
                break
        else:
            found_filtered.append(r)
    draw_detections(img, found)
    draw_detections(img, found_filtered, 3)
    return img

if __name__ == '__main__':
    hog = cv.HOGDescriptor()
    hog.setSVMDetector(cv.HOGDescriptor_getDefaultPeopleDetector())

    face_cascade = cv.CascadeClassifier(sys.argv[1])

    cap = cv.VideoCapture(1)
    while True:
        ret, img = cap.read()

        # face detection using HOG
        # img = find_faces_hog(img, hog=hog)

        # HAAR
        img = find_faces_haar(img, face_cascade)

        # show the result.
        cv.imshow('capture', img)
        ch = cv.waitKey(1)
        if ch == 27:
            break
    cv.destroyAllWindows()


import logging
from unittest.mock import patch

from django.test import override_settings

import pytest

from thenewboston_node.business_logic.blockchain.base import BlockchainBase
from thenewboston_node.business_logic.blockchain.memory_blockchain import MemoryBlockchain
from thenewboston_node.business_logic.blockchain.mock_blockchain import MockBlockchain
from thenewboston_node.business_logic.tests.factories import add_blocks_to_blockchain
from thenewboston_node.business_logic.tests.mocks.storage_mock import StorageMock
from thenewboston_node.business_logic.utils.iter import get_generator

logger = logging.getLogger(__name__)

LARGE_MEMORY_BLOCKCHAIN_SIZE = 100

MEMORY_BLOCKCHAIN_CLASS = 'thenewboston_node.business_logic.blockchain.memory_blockchain.MemoryBlockchain'
FILE_BLOCKCHAIN_CLASS = 'thenewboston_node.business_logic.blockchain.file_blockchain.FileBlockchain'
MOCK_BLOCKCHAIN_CLASS = 'thenewboston_node.business_logic.blockchain.mock_blockchain.MockBlockchain'

@pytest.fixture
def get_head_block_mock():
    with patch.object(MockBlockchain, 'get_head_block', return_value=None) as mock:
        yield mock

@pytest.fixture
def get_next_block_number_mock():
    with patch.object(MockBlockchain, 'get_next_block_number', return_value=0) as mock:
        yield mock

@pytest.fixture
def get_next_block_identifier_mock():
    with patch.object(MockBlockchain, 'get_next_block_identifier', return_value='next-block-identifier') as mock:
        yield mock

@pytest.fixture
def get_account_state_mock():
    with patch.object(MockBlockchain, 'get_account_balance', return_value=430) as mock:
        yield mock

@pytest.fixture
def get_account_lock_mock():
    with patch.object(MockBlockchain, 'get_account_balance_lock', return_value='fake-balance-lock') as mock:
        yield mock

def yield_forced_blockchain(class_, class_kwargs=None):
    blockchain_settings = {'class': class_, 'kwargs': class_kwargs or {}}

    BlockchainBase.clear_instance_cache()
    with override_settings(BLOCKCHAIN=blockchain_settings):
        blockchain = BlockchainBase.get_instance()
        yield blockchain
    BlockchainBase.clear_instance_cache()

def yield_and_init_forced_blockchain(class_, blockchain_genesis_state, class_kwargs=None):
    blockchain = next(yield_forced_blockchain(class_, class_kwargs))
    blockchain.add_blockchain_state(blockchain_genesis_state)
    blockchain.validate()
    yield blockchain

@pytest.fixture
def memory_blockchain(blockchain_genesis_state):
    blockchain = MemoryBlockchain()
    blockchain.add_blockchain_state(blockchain_genesis_state)
    blockchain.validate()
    yield blockchain

@pytest.fixture
def forced_memory_blockchain(blockchain_genesis_state):
    yield from yield_and_init_forced_blockchain(MEMORY_BLOCKCHAIN_CLASS, blockchain_genesis_state)

@pytest.fixture
def forced_file_blockchain(blockchain_genesis_state, blockchain_directory):
    yield from yield_and_init_forced_blockchain(
        FILE_BLOCKCHAIN_CLASS, blockchain_genesis_state, class_kwargs={'base_directory': blockchain_directory}
    )

@pytest.fixture(autouse=True)  # Autouse for safety reasons
def forced_mock_blockchain(blockchain_genesis_state):
    yield from yield_and_init_forced_blockchain(MOCK_BLOCKCHAIN_CLASS, blockchain_genesis_state)

@pytest.fixture
def large_blockchain(treasury_account_key_pair):
    blockchain = BlockchainBase.get_instance()

    accounts = blockchain.get_first_blockchain_state().account_states
    assert len(accounts) == 1
    treasury_account, account_state = list(accounts.items())[0]
    assert treasury_account_key_pair.public == treasury_account
    assert account_state.balance > 10000000000  # tons of money present

    add_blocks_to_blockchain(blockchain, 100, treasury_account_key_pair.private)
    yield blockchain

@pytest.fixture
def file_blockchain_w_memory_storage(
    forced_file_blockchain, blockchain_genesis_state, forced_mock_network, get_primary_validator_mock,
    get_preferred_node_mock
):
    block_storage_mock = patch.object(forced_file_blockchain, 'block_storage', StorageMock())
    arf_storage_mock = patch.object(forced_file_blockchain, 'account_root_files_storage', StorageMock())

    with block_storage_mock, arf_storage_mock:
        forced_file_blockchain.add_blockchain_state(blockchain_genesis_state)
        forced_file_blockchain.validate()
        yield forced_file_blockchain

@pytest.fixture
def blockchain_base(blockchain_genesis_state):
    blockchain = BlockchainBase()
    with patch.object(blockchain, 'yield_blockchain_states', get_generator([blockchain_genesis_state])):
        yield blockchain


import asyncio
import random

import discord

#Token that connects with the bot
TOKEN = 'SECRET KEY'

client = discord.Client()

alive = True
channels = []

reply = ['Mijn gemeente heeft rond de 41.000 inwoners',
         'Je kan campussen van de UA vinden in de gemeente waar ik woon',
         'Ik woon niet ver van een Fort',
         'Als ge zat zijt, kunt ge op minder dan een half uur van de vizit tot bij mij thuis wandelen']

@client.event
async def on_message(message):
    if message.author == client.user:
        return
    print(message)
    command = message.content.split(' ')
    if command[0] == 'bot':
        channel = client.get_channel(int(command[1]))
        if command[2] == 'conv_biokot':
            await asyncio.sleep(12)
            await channel.send(
                'Awel das goe, dan babbel ik rechts van u wel met toffere mensen.')
        if command[2] == 'conv_hagar':
            await asyncio.sleep(16)
            await channel.send(
                'Ik hoor het al Thomas, gij vindt mij genen toffe. Ik zal me wel aan de rechterkant van de tafel '
                'zetten, zet gij u maar bij die middelste drie aan tafel. ‘T is al goe.')
        if command[2] == 'conv_all':
            await asyncio.sleep(2)
            await channel.send(
                'Awel das goe, dan babbel ik rechts van u wel met toffere mensen.')
            await channel.send(
                'Ik hoor het al Thomas, gij vindt mij genen toffe. Ik zal me wel aan de rechterkant van de tafel '
                'zetten, zet gij u maar bij die middelste drie aan tafel. ‘T is al goe.')
    print(message.channel.type)
    if not message.guild:
        if 'waar' in message.content or 'Waar' in message.content:
            await message.channel.send(reply[random.randint(0, len(reply)-1)])

@client.event
async def on_ready():
    print('Logged in as')
    print(client.user.name)
    print(client.user.id)
    print('------')

if __name__ == '__main__':
    client.run(TOKEN)


import requests
import re

tolurl = 'https://www.ximalaya.com/revision/album/v1/getTracksList?albumId=18943952&pageNum=3'
myHeader = {
    'User-Agent':
	'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:81.0) Gecko/20100101 Firefox/81.0'
}
content = requests.get(tolurl,headers = myHeader)
jsonContent = content.json()
myDatab = jsonContent['data']['tracks']
#print(myDatab)
urlList = []
for i in myDatab:
    urlList.append('https://www.ximalaya.com'+i['url'])

print(urlList)


#encoding:utf-8
#By:Eastmount CSDN 2021-08-20
import cv2
import numpy as np
import matplotlib.pyplot as plt

#读取图片
img = cv2.imread('lena-hd.png')

#灰度转换
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

#直方图均衡化处理
result = cv2.equalizeHist(gray)

#显示图像
plt.subplot(221)
plt.imshow(gray, cmap=plt.cm.gray), plt.axis("off"), plt.title('(a)')
plt.subplot(222)
plt.imshow(result, cmap=plt.cm.gray), plt.axis("off"), plt.title('(b)')
plt.subplot(223)
plt.hist(img.ravel(), 256), plt.title('(c)')
plt.subplot(224)
plt.hist(result.ravel(), 256), plt.title('(d)')
plt.show()


import json
import logging

from django.contrib.auth.decorators import login_required
from django.http import (
    Http404,
    HttpResponse,
    HttpResponseForbidden
)
from papermerge.core.models import (
    BaseTreeNode,
    Page,
    Access
)
from papermerge.core.models.kvstore import (get_currency_formats,
                                            get_date_formats, get_kv_types,
                                            get_numeric_formats)

logger = logging.getLogger(__name__)

@login_required
def metadata(request, model, id):
    """
    model can be either node or page. Respectively
    id will be the 'id' of either node or page.
    E.g.
    POST /metadata/page/55 # will update metadata for page id=55
    POST /metadata/node/40 # will update metadata for node id=40
    """
    if model == 'node':
        _Klass = BaseTreeNode
    else:
        _Klass = Page
    try:
        item = _Klass.objects.get(id=id)
    except _Klass.DoesNotExist:
        raise Http404("Node does not exists")

    kvstore = []

    if request.method == 'GET':
        for kv in item.kv.all():
            kvstore.append(kv.to_dict())
    else:
        if isinstance(item, BaseTreeNode):
            node = item
        else:
            node = item.document

        if request.user.has_perm(Access.PERM_WRITE, node):
            kv_data = json.loads(request.body)
            if 'kvstore' in kv_data:
                if isinstance(kv_data['kvstore'], list):
                    item.kv.update(
                        _sanitize_kvstore_list(kv_data['kvstore'])
                    )
        else:
            return HttpResponseForbidden()

    return HttpResponse(
        json.dumps(
            {
                'kvstore': kvstore,
                'currency_formats': get_currency_formats(),
                'date_formats': get_date_formats(),
                'numeric_formats': get_numeric_formats(),
                'kv_types': get_kv_types()

            }
        ),
        content_type="application/json"
    )

def _sanitize_kvstore_list(kvstore_list):
    """
    Creates a new dictionay only with allowed keys.
    """
    new_kvstore_list = []
    allowed_keys = [
        'id',
        'key',
        'value',
        'kv_type',
        'kv_format',
        'kv_inherited',
    ]

    for item in kvstore_list:
        sanitized_kvstore_item = {
            allowed_key: item.get(allowed_key, None)
            for allowed_key in allowed_keys
        }
        new_kvstore_list.append(sanitized_kvstore_item)

    return new_kvstore_list


import numpy, scipy, sklearn.feature_selection, sys, pandas, scipy.stats
sys.path.append("/Users/pwangel/Gene_Analysis")
from ga_utils import *
from scipy import optimize

def transform_distribution(dataframe, scale_parameter):

    for i_col in dataframe.columns:

        col_range  = dataframe[i_col].max()-dataframe[i_col].min()
        n_bins     = int(col_range/dataframe[i_col].std()/scale_parameter)+2
        starting_point = dataframe[i_col].mean() + 0.5*dataframe[i_col].std()*scale_parameter
        minimum    = dataframe[i_col].min()
        maximum    = dataframe[i_col].max()

        upper_bins = numpy.array([dataframe[i_col].std()*scale_parameter*pos+starting_point for pos in range(n_bins//2+1)])
        lower_bins = numpy.array([-dataframe[i_col].std()*scale_parameter*pos+starting_point for pos in range(n_bins//2+1)])
        all_bins   = numpy.concatenate((lower_bins[::-1], upper_bins[1:]))
        sel        = (all_bins >= minimum) & (all_bins <= maximum)
        all_bins   = all_bins[sel]
        labels     = numpy.concatenate((numpy.arange(start =-len(lower_bins[lower_bins>=minimum])+2, stop=0), numpy.arange(start=0, stop=len(upper_bins[upper_bins <= maximum]))))
        dataframe[i_col] = pandas.cut(dataframe[i_col], bins=all_bins, labels=labels)

    return dataframe

'''Finds common genes between a group of RNASeq and Microarray datasets.'''
'''Then plots 2d Histogram comparing measure correlation coefficients between Microarray and RNASeq'''

sample_threshold  = 20

from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
from plotly.graph_objs import *
import plotly.figure_factory as ff

# Read in data

RNASeq_data, labels = read_file_lists.smash_into_matrix('/Users/pwangel/Data/ensembl_hg19_v69_mm9_v67/', 'RNASeq', sample_threshold, 'TMM RPKM (Log2)', 'ENSG', False)
Microarray_data, labels = read_file_lists.smash_into_matrix('/Users/pwangel/Data/ensembl_hg19_v69_mm9_v67/', 'Microarray', sample_threshold, None, 'ENSG', False)

print("Comparing RNASeq and Microarray gene content")

RNASeq_data        = RNASeq_data.loc[(RNASeq_data.count(axis=1).values > len(RNASeq_data.columns)/2)]
Microarray_data    = Microarray_data.loc[(Microarray_data.count(axis=1).values > len(Microarray_data.columns)/2)]

common_genes       = numpy.intersect1d(RNASeq_data.index.values, Microarray_data.index.values)

print("Kept %d common genes" %len(common_genes))
iu = numpy.triu_indices(len(common_genes),1)

df_gene_list       = pandas.DataFrame(index=common_genes)

RNASeq_data        = df_gene_list.merge(RNASeq_data, how='left', left_index=True, right_index=True, validate='1:1')
Microarray_data    = df_gene_list.merge(Microarray_data, how='left', left_index=True, right_index=True, validate='1:1').sample(frac=0.5, replace=False, axis=1)

print("Have dataframes of size:")

print(RNASeq_data.shape)
print(Microarray_data.shape)

# Let's do an experimental transform
#bins = 100
#labels = numpy.linspace(start=0.0, stop=1.0, num=bins+1)
#labels = labels[:-1]+0.5*(labels[1]-labels[0])
#labels = scipy.stats.lognorm.ppf(labels, s=0.75)

#for i_col in RNASeq_data.columns:
#	RNASeq_data[i_col] = pandas.qcut(RNASeq_data[i_col], q=bins, labels =labels , retbins=False).astype(float)

#for i_col in Microarray_data.columns:
#	Microarray_data[i_col] = pandas.qcut(Microarray_data[i_col], q=bins, labels =labels , retbins=False).astype(float)

#RNASeq_correlation_array      = run_parallel.parallelized_correlation(RNASeq_data.sort_index()).as_matrix()[iu].astype(float)
#non_parRNASeq_correlation_array      = RNASeq_data.sort_index().transpose().corr(method='pearson', min_periods=10).as_matrix()[iu]
#print(len(RNASeq_correlation_array[RNASeq_correlation_array!= non_parRNASeq_correlation_array]))

print("Completed RNASeq correlations")

#Microarray_correlation_array  = Microarray_data.sample(n=1000, axis=1, replace =False).sort_index().transpose().corr(method='pearson', min_periods=10).as_matrix()[iu]
#Microarray_correlation_array   = run_parallel.parallelized_correlation(Microarray_data.sort_index()).as_matrix()[iu].astype(float)
print("Completed Microarray correlations")

sel = (~numpy.isnan(Microarray_correlation_array)) & (~numpy.isnan(RNASeq_correlation_array))
heat, x_bins, y_bins = numpy.histogram2d(x=Microarray_correlation_array[sel], y=RNASeq_correlation_array[sel], bins=200)
heat                 = numpy.power(heat, 1.0) #shows the trend more clearly, i hope

fitfunc = lambda p, x: p[0]+ p[1]*x # target function
errfunc = lambda p, x, y, weight: (fitfunc(p, x) - y)/weight # distance to the target function
p0 = [0.1, 0.5] # initial guess for the parameters

weights   = []
y_plot    = []
for i_col in range(len(x_bins)-1):
    weights.append(1.0/scipy.stats.norm.fit(heat[:,i_col])[1])
    y_plot.append(y_bins[numpy.argmax(heat[:,i_col])])

p1, success = optimize.leastsq(errfunc, p0[:], args=(x_bins[:-1], y_plot, weights))

data_to_plot = [Heatmap(z=heat, x=x_bins, y=y_bins)]
data_to_plot.append(Scatter(x=[-0.9, 0.9], y=[-0.9, 0.9], mode='lines', name='1:1', \
                            line=dict(color = 'black', width=2, dash='dash')))
data_to_plot.append(Scatter(x=x_bins, y=fitfunc(p1, x_bins[:-1]), mode='lines', name='y=%.2f x +%.2f' %(p1[1], p1[0]), \
                            line=dict(color = 'k', width=2, dash='dash')))
data_to_plot.append(Scatter(x=x_bins, y=y_plot, mode='lines', name='Max density', \
                            line=dict(color = 'white', width=2, dash='dash')))

fig = Figure(data=data_to_plot,
        layout=Layout(title='p(pearson)', xaxis=dict(title='micorarray correlation coefficient'), yaxis=dict(title='rnaseq correlation coefficient')) )
plot(fig, auto_open=False, filename='/users/pwangel/plotlyworkspace/transformed_Microarray_vs_RNASeq_pearson.html')