# -*- coding: utf-8 -*-
#
# Copyright (C) 2020-2021 CERN.
# Copyright (C) 2021 Graz University of Technology.
#
# Invenio-Vocabularies is free software; you can redistribute it and/or
# modify it under the terms of the MIT License; see LICENSE file for more
# details.

"""Commands to create and manage vocabulary."""

import csv
from os.path import dirname, join

import click
from flask.cli import with_appcontext
from flask_principal import Identity
from invenio_access import any_user
from invenio_db import db

from invenio_vocabularies.contrib.subjects.subjects import subject_record_type
from invenio_vocabularies.records.models import VocabularyType
from invenio_vocabularies.services.service import VocabulariesService

data_directory = join(dirname(__file__), "data")

def get_available_vocabularies():
    """Specify the available vocabularies."""
    return {
        "licenses": {
            "path": join(data_directory, "licenses.csv"),
        },
        "subjects": {
            "path": join(data_directory, "subjects.csv"),
            "specific": _create_subjects_vocabulary,
        },
    }

def _load_csv_data(path):
    with open(path) as f:
        reader = csv.DictReader(f, skipinitialspace=True)
        dicts = [row for row in reader]
        return dicts

def _create_subjects_vocabulary(vocabulary_type_name, source_path):
    identity = Identity(1)
    identity.provides.add(any_user)
    service = subject_record_type.service_cls()

    rows = _load_csv_data(source_path)

    records = []
    for row in rows:
        metadata = {
            "title": row["title"],
            "term": row["id"],
            "identifier": row["id"],
            "scheme": row["scheme"],
        }

        record = service.create(
            identity=identity,
            data={
                "metadata": metadata,
            },
        )

        records.append(record)

    return records

def _create_vocabulary(vocabulary_type_name, source_path):
    identity = Identity(1)
    identity.provides.add(any_user)
    service = VocabulariesService()

    # Load data
    rows = _load_csv_data(source_path)

    # Create vocabulary type
    vocabulary_type = VocabularyType(name=vocabulary_type_name)
    db.session.add(vocabulary_type)
    db.session.commit()

    i18n = ["title", "description"]  # Attributes with i18n support
    other = ["icon"]  # Other top-level attributes

    default_language = "en"  # Static (dependent on the files)

    metadata = {"title": {}, "description": {}, "props": {}}

    records = []
    for row in rows:
        for attribute in row:
            value = row[attribute]
            if attribute in i18n:
                metadata[attribute][default_language] = value
            elif any(map(lambda s: value.startswith(s + "_"), i18n)):
                [prefix_attr, language] = attribute.split("_", 1)
                metadata[prefix_attr][language] = value
            elif attribute in other:
                metadata[attribute] = value
            else:
                metadata["props"][attribute] = value

        # Create record
        record = service.create(
            identity=identity,
            data={
                "metadata": metadata,
                "vocabulary_type_id": vocabulary_type.id,
            },
        )

        records.append(record)

    return records

@click.group()
def vocabularies():
    """Vocabularies command."""
    pass

@vocabularies.command(name="import")
@click.argument(
    "vocabulary_types",
    nargs=-1,
    type=click.Choice([v for v in get_available_vocabularies()]),
)
@with_appcontext
def load(vocabulary_types):
    """Index CSV-based vocabularies in Elasticsearch."""
    click.echo("creating vocabularies...", color="blue")

    for vocabulary_type in vocabulary_types:
        vocabulary = get_available_vocabularies()[vocabulary_type]
        if VocabularyType.query.filter_by(name=vocabulary_type).count() > 0:
            click.echo(
                "vocabulary type {} already exists, skipping".format(
                    vocabulary_type
                ),
                color="red",
            )
            continue

        click.echo(
            "creating vocabulary type {}...".format(vocabulary_type),
            color="blue",
        )

        fun_create_vocabulary = vocabulary.get("specific", _create_vocabulary)
        items = fun_create_vocabulary(vocabulary_type, vocabulary["path"])

        click.echo(
            "created {} vocabulary items successfully".format(len(items)),
            color="green",
        )
    click.echo("vocabularies created", color="green")


"""
t3 common module
"""

import datetime
import os
import string
from typing import Optional, Tuple

from rmgpy.species import Species

from arc.species.converter import molecules_from_xyz

VERSION = '0.1.0'

t3_path = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))  # absolute path to the T3 folder
DATA_BASE_PATH = os.path.join(t3_path, 'tests', 'data')
SIMULATE_DATA_BASE_PATH = os.path.join(t3_path, 'tests', 'test_simulate_adapters', 'data')
EXAMPLES_BASE_PATH = os.path.join(t3_path, 'examples')
IPYTHON_SIMULATOR_EXAMPLES_PATH = os.path.join(t3_path, 'ipython', 'simulator_adapter_examples')
PROJECTS_BASE_PATH = os.path.join(t3_path, 'Projects')
VALID_CHARS = "-_=.,%s%s" % (string.ascii_letters, string.digits)

def get_species_by_label(label: str,
                         species_list: list,
                         ) -> Optional[Species]:
    """
    Get a species from a list of species by its label.

    Args:
        label (str): A species label.
        species_list (list): Entries are RMG Species objects.

    Returns:
        Optional[Species]: The corresponding species from the species_list.
                           Returns ``None`` if no species was found.
    """
    for species in species_list:
        if species.label == label or species.to_chemkin() == label:
            return species
    if '(' in label and ')' in label:
        # try by the RMG species index
        for species in species_list:
            if species.index == int(label.split('(')[-1].split(')')[0]):
                return species
    return None

def dict_to_str(dictionary: dict,
                level: int = 0,
                ) -> str:
    """
    A helper function to log dictionaries in a pretty way.

    Args:
        dictionary (dict): A general python dictionary.
        level (int): A recursion level counter, sets the visual indentation.

    Returns:
        str: A text representation for the dictionary.
    """
    message = ''
    for key, value in dictionary.items():
        if isinstance(value, dict):
            message += ' ' * level * 2 + str(key) + ':\n' + dict_to_str(value, level + 1)
        else:
            message += ' ' * level * 2 + str(key) + ': ' + str(value) + '\n'
    return message

def delete_root_rmg_log(project_directory: str) -> None:
    """
    Delete the 'RMG.log' file left in the root output directory, it's a left-over.

    Args:
        project_directory (str): The path to the root output folder.
    """
    rmg_log_path = os.path.join(project_directory, 'RMG.log')
    if os.path.isfile(rmg_log_path):
        os.remove(rmg_log_path)

def get_rmg_species_from_a_species_dict(species_dict: dict,
                                        raise_error: bool = False,
                                        ) -> Optional[Species]:
    """
    Get an RMG Species instance that corresponds to a species specified under the rmg.species
    section of the T3 input file (a species dictionary).

    Args:
        species_dict (dict): The species dictionary to process.
        raise_error (bool, optional): Whether to raise an error if a Species instance cannot be generated.
                                      Default: ``False``.

    Raises:
        ValueError: If the species dictionary does not have a specified structure (if ``raise_error`` is ``True``).

    Returns:
        Species: The corresponding RMG species instance.
    """
    species = None
    errored = False
    if species_dict['adjlist'] is not None:
        species = Species(label=species_dict['label']).from_adjacency_list(species_dict['adjlist'])
    elif species_dict['smiles'] is not None:
        species = Species(label=species_dict['label'], smiles=species_dict['smiles'])
    elif species_dict['inchi'] is not None:
        species = Species(label=species_dict['label'], inchi=species_dict['inchi'])
    elif species_dict['xyz'] is not None:
        for xyz in species_dict['xyz']:
            mol_bo = molecules_from_xyz(xyz=xyz)[1]
            if mol_bo is not None:
                species = Species(label=species_dict['label']).from_adjacency_list(mol_bo.to_adjacency_list())
                break
        else:
            errored = True
    else:
        errored = True
    if errored and raise_error:
        raise ValueError(f"The species corresponding to {species_dict['label']} does not have a specified structure.")
    return species

def time_lapse(t0: datetime.datetime) -> datetime.timedelta:
    """
    A helper function returning the elapsed time since t0.

    Args:
        t0 (datetime.datetime): The initial time the count starts from.

    Returns: datetime.timedelta
        The time difference between now and t0.
    """
    return datetime.datetime.now() - t0

def convert_termination_time_to_seconds(termination_time: Tuple[float, str]):
    """
    Converts the termination_time tuple from the RMG reactor to seconds.
    This is necessary for the RMS adapters since the Julia solver expects
    the integration bounds to be in units of seconds.

    Args:
        termination_time (Tuple[float, str]):  Termination time for simulating in the RMG reactor. Example: [5, 'hours']

    Returns:
        t_final (float): The termination time in seconds.
    """
    unit_conversion = {'micro-s': 1e-6,
                       'ms': 1e-3,
                       's': 1,
                       'hrs': 3600,
                       'hours': 3600,
                       'days': 3600*24,
                       }
    t_final, units = termination_time
    t_final = t_final * unit_conversion[units]
    return t_final


import time
from random import randint, uniform
from selenium import webdriver
from itertools import repeat

# Add odd shit here
site_list = ["yahoo.com"]

def site_select():
    i = randint(0, len(site_list) - 1)
    return (site_list[i])

firefox_profile = webdriver.FirefoxProfile()
firefox_profile.set_preference("browser.privatebrowsing.autostart", True)
driver = webdriver.Firefox(firefox_profile=firefox_profile)

# Visits a site, clicks a random number links, sleeps for random spans between
def visit_site():
    new_site = site_select()
    driver.get(new_site)
    print("Visiting: " + new_site)
    time.sleep(uniform(1, 15))

    for i in repeat(None, randint(1, 3)) :
        try:
            links = driver.find_elements_by_css_selector('a')
            l = links[randint(0, len(links)-1)]
            time.sleep(1)
            print("clicking link")
            l.click()
            time.sleep(uniform(0, 120))
        except Exception as e:
            print("Something went wrong with the link click.")
            print(type(e))

while(True):
    visit_site()
    time.sleep(uniform(4, 80))


from django.urls import path
from client import views
from django.conf import settings
from django.conf.urls.static import static

app_name="client"
urlpatterns = [
    path('', views.home, name='home'),
    path('menu/', views.menu, name='menu'),
    path('profile/', views.profile, name='profile'),
    path('preview/', views.preview, name='preview'),
    path('submit/', views.submit, name='submit'),
    path('update/', views.update, name='update'),
    path('register/', views.SignUpPage, name='register'),
    path('login/', views.LoginPage, name='login'),
    path('logout/', views.logout_view, name='logout'),
]
urlpatterns += static(settings.MEDIA_URL,document_root=settings.MEDIA_ROOT)


import requests

from backend.utils.requestutil import atbRequests

class TransactionRequest():

    def __init__(self, bank_id, acc_id):
        self.bank_id = bank_id
        self.acc_id = acc_id
        self.request_url = ''
        self.data = None
        atbR = atbRequests()
        self.request_token = atbR.generate_login_token()

    def transaction_request(self):
        atb_headers = {
            'Authorization': 'DirectLogin token={}'.format(self.request_token)
        }
        self.request_url = 'https://api.leapos.ca/obp/v4.0.0/my/banks/{}/accounts/{}/transactions'.format(self.bank_id, self.acc_id)
        response = requests.get(self.request_url, headers=atb_headers)
        data = response.json()
        self.data = data
        return self.data

    def format_data(self):
        out_data = []
        for row in self.data.get('transactions'):
            f_data = {
                'type': row.get('details').get('type'),
                'timestamp': row.get('details').get('completed'),
                'description': row.get('details').get('description'),
                'balance': row.get('details').get('new_balance').get('amount'),
                'change': row.get('details').get('value').get('amount')
            }
            out_data.append(f_data)
        return out_data


from PIL import ImageFont, ImageDraw, Image
import os, shutil
from time import time, gmtime, strftime

class Data:

    def ready_data(self, w, h, path, testing):

        # #check if folder exists if its doesn't create it and if it does
        # #delete and make it a new

        if not os.path.exists('fonts'):
            raise ValueError("please provide input directory of fonts")

        if w<15 or h<15:
            raise ValueError("please provide height and weight >15")

        if not os.path.exists(path):
            os.makedirs(path)
            print("path created")

        # #populate array with .ttf files
        ttf = []
        print("populating data")
        for root, dirs, files in os.walk(r'fonts'):
            if(testing==False):
                for file in files:
                    if file.endswith('.ttf'):
                        ttf.append(file)
            else:
                ttf.append(files[0])

        # #declare Width and Height Parameters

        self.build_data(ttf, w, h, path)

    def build_data(self, ttf, W, H, path):

        for i in range(0, ttf.__len__()):
            letter = 65

            font_name = str(ttf[int(i)])
            font_name = font_name[0:font_name.__len__() - 4]

            print("Drawing images for "+font_name)
            for iter in range(0, 2):
                append = "upper"
                if(letter==97):
                    append="lower"

                for alpha in range(0, 26):

                    im = Image.new("RGBA", (W, H), (255, 255, 255))

                    draw = ImageDraw.Draw(im)

                    try:

                        if not os.path.exists(os.path.join(path, font_name)):
                            os.makedirs(os.path.join(path,font_name))

                        font = ImageFont.truetype(os.path.join('fonts', ttf[int(i)]), H-10)

                        w, h = draw.textsize(chr(alpha+letter), font=font)

                        draw.text(((W-w)/2,((H-h)/2)-2), chr(letter+alpha), (0, 0, 0), font=font)

                        im.save(os.path.join(path,font_name, font_name +"_"+append+ "_"+chr(letter+alpha)+".png"), 'png')

                    except OSError:
                        pass
                        shutil.rmtree(os.path.join(path,font_name))
                letter = 97

        #uncomment below to have the manifold run
        #FontManifold

if __name__ == '__main__':
    print("program starting at: " + strftime(" %H:%M:%S", gmtime()))
    t0 = time()
    Data().ready_data(60, 60, "test_data", False)
    print("finished after " + str((time() - t0) / 60.0) + " minutes")
    print("program program finished at: " + strftime(" %H:%M:%S", gmtime()))


#   Licensed under the Apache License, Version 2.0 (the "License"); you may
#   not use this file except in compliance with the License. You may obtain
#   a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#   License for the specific language governing permissions and limitations
#   under the License.
#
'''CSS ELK cluster v1 action implementations'''
import logging

from osc_lib import utils
from osc_lib.command import command
from otcextensions.common import sdk_utils

from otcextensions.i18n import _

LOG = logging.getLogger(__name__)

def _flatten_output(obj):
    data = {
        'id': obj.id,
        'clusterId': obj.clusterId,
        'name': obj.name,
        'clustername': obj.clusterName,
        'started': obj.started,
        'size': obj.size,
        'description': obj.description,
        'status': obj.status,
        'updated': obj.updated,
    }

"""
def _flatten_output(obj):
    data = {
        'id': obj.id,
        'clusterid': obj.clusterId,
        'name': obj.name,
        'clustername': obj.clusterName,
        'status': obj.status,
        'created': obj.created,
    }
"""

def _get_columns(item):
    column_map = {
    }
    return sdk_utils.get_osc_show_columns_for_sdk_resource(item, column_map)

class DeleteCluster(command.Command):
    _description = _('Delete CSS Cluster')

    def get_parser(self, prog_name):
        parser = super(DeleteCluster, self).get_parser(prog_name)
        parser.add_argument(
            'cluster',
            metavar='<cluster>',
            nargs='+',
            help=_('Name or ID of the tracker to delete.')
        )
        return parser

    def take_action(self, parsed_args):

        if parsed_args.cluster:
            client = self.app.client_manager.css
            for cluster in parsed_args.cluster:
                client.delete_cluster(cluster=cluster, ignore_missing=False)

class CreateSnapshot(command.ShowOne):
    _description = _('Create a single CSS snapshot')

    columns = (
        'clusterid',
        'name',
        'clustername',
        'status',
        'created'
        # 'dimensions',
        # 'metric_name',
        # 'unit',
    )

    table_columns = (
        'name',
        # 'dimensions.name',
        # 'dimensions.value',
        # 'metric_name',
        # 'unit',
    )

    def get_parser(self, prog_name):
        parser = super(CreateSnapshot, self).get_parser(prog_name)

        parser.add_argument(
            '--clusterid',
            metavar='<clusterid>',
            required=True,
            help=_('Specifies the CSS Cluster name OBS bucket name. The value is a string of '
                   '0 to 64 characters and can contain uppercase and '
                   'lowercase letters (a to z and A to Z), digits (0 to '
                   '9), hyphens (-), underscores (_), and periods (.). '
                   'In addition, it must start and end with a letter.')
        )

        parser.add_argument(
            '--name',
            metavar='<name>',
            required=True,
            help=_('Specifies the CSS Cluster name OBS bucket name. The value is a string of '
                   '0 to 64 characters and can contain uppercase and '
                   'lowercase letters (a to z and A to Z), digits (0 to '
                   '9), hyphens (-), underscores (_), and periods (.). '
                   'In addition, it must start and end with a letter.')
        )
        parser.add_argument(
            '--description',
            metavar='<description>',
            help=_('Specifies the prefix of a log that needs to be stored '
                   'in an OBS  bucket. The value is a string of 0 to 64 '
                   'characters and can contain uppercase and lowercase '
                   'letters (a to z and A to Z), digits (0 to 9), '
                   'hyphens (-), underscores (_), and periods (.)')
        )
        parser.add_argument(
            '--indices',
            metavar='<indices>',
            help=_('Specifies whether SMN is supported. When the value is '
                   '`false`, `topic_id` and `operations` can be left empty.')
        )
        return parser

    def take_action(self, parsed_args):

        client = self.app.client_manager.css

        attrs = {}

        if parsed_args.name:
            attrs['name'] = parsed_args.name
        if parsed_args.clusterid:
            attrs['clusterid'] = parsed_args.clusterid
        if parsed_args.description:
            attrs['description'] = parsed_args.description
        if parsed_args.indices:
            attrs['indices'] = parsed_args.indices

        obj = client.create_snapshot(**attrs)

        """
        obj = list(obj)

        obj = obj[0]["backup"]

        table = (self.columns,
                 (utils.get_dict_properties(
                     _flatten_output(s), self.columns
                 ) for s in obj))
        return table

        """

        display_columns, columns = _get_columns(obj)
        data = utils.get_item_properties(obj, columns)

        return (display_columns, data)

class ShowCluster(command.ShowOne):
    _description = _('Show details of a CSS cluster')

    def get_parser(self, prog_name):
        parser = super(ShowCluster, self).get_parser(prog_name)
        parser.add_argument(
            'cluster',
            metavar='<cluster>',
            # default='system',
            help=_('Cluster name (currently only `system`)')
        )
        return parser

    def take_action(self, parsed_args):
        client = self.app.client_manager.css

        data = client.get_cluster(
            cluster=parsed_args.cluster,
        )

        display_columns, columns = _get_columns(data)
        data = utils.get_item_properties(data, columns)

        return (display_columns, data)

class RestartCluster(command.ShowOne):
    _description = _('Show details of a CSS cluster')

    def get_parser(self, prog_name):
        parser = super(RestartCluster, self).get_parser(prog_name)
        parser.add_argument(
            'cluster',
            metavar='<cluster>',
            # default='system',
            help=_('Cluster name (currently only `system`)')
        )
        return parser

    def take_action(self, parsed_args):
        client = self.app.client_manager.css

        data = client.restart_cluster(
            cluster=parsed_args.cluster,
        )

        display_columns, columns = _get_columns(data)
        data = utils.get_item_properties(data, columns)

        return (display_columns, data)

class Snapshots(command.Lister):
    _description = _('List of CSS Backups')
    columns = (
        'id',
        'clusterid',
        'clustername',
        'name',
        'clustername',
        'status',
        'created'
        # 'dimensions',
        # 'metric_name',
        # 'unit',
    )

    def get_parser(self, prog_name):
        parser = super(Snapshots, self).get_parser(prog_name)
        parser.add_argument(
            'cluster_id',
            metavar='<cluster_id>',
            help=_('Cluster id')
        )
        return parser

    def take_action(self, parsed_args):
        client = self.app.client_manager.css

        query = {}
        if parsed_args.cluster_id:
            query['cluster_id'] = parsed_args.cluster_id

        """
        if parsed_args.namespace:
            query['namespace'] = parsed_args.namespace
        if parsed_args.metric_name:
            query['metric_name'] = parsed_args.metric_name
        if parsed_args.unit:
            query['unit'] = parsed_args.unit
        """

        data = client.snapshots(**query)
        # data = client.clusters()

        data = list(data)
        # data = data["backups"]
        # for s in range(data):
        #    if data[s] is None:
        #        data.remove(s)

        table = (self.columns,
                 (utils.get_dict_properties(
                     _flatten_output(s), self.columns
                 ) for s in data))
        return table


import unittest2 as unittest
from genweb.core.testing import GENWEB_INTEGRATION_TESTING
from genweb.core.testing import GENWEB_FUNCTIONAL_TESTING
from AccessControl import Unauthorized
from zope.component import getMultiAdapter, queryUtility
from zope.interface import alsoProvides
from Products.CMFCore.utils import getToolByName

from plone.testing.z2 import Browser
from plone.app.testing import TEST_USER_ID, TEST_USER_NAME
from plone.app.testing import login, logout
from plone.app.testing import setRoles
from plone.app.testing import applyProfile

from plone.portlets.interfaces import IPortletManager
from plone.portlets.interfaces import IPortletAssignmentMapping

from genweb.core.interfaces import IHomePage
from genweb.theme.portlets import homepage

import transaction

class IntegrationTest(unittest.TestCase):

    layer = GENWEB_INTEGRATION_TESTING

    def setUp(self):
        self.portal = self.layer['portal']
        self.request = self.layer['request']

    def testPortalConstrains(self):
        portal_allowed_types = ['Folder', 'File', 'Image', 'Document']
        setRoles(self.portal, TEST_USER_ID, ['Manager'])
        login(self.portal, TEST_USER_NAME)
        self.assertEqual(sorted([ct.id for ct in self.portal.allowedContentTypes()]), sorted(portal_allowed_types))

    def testLinkBehavior(self):
        """Test for Link behavior and related index and metadata"""
        portal = self.layer['portal']
        setRoles(portal, TEST_USER_ID, ['Manager'])
        login(portal, TEST_USER_NAME)
        portal.invokeFactory('Folder', 'f2', title=u"Soc una carpeta")
        f2 = portal['f2']
        f2.invokeFactory('Link', 'enllac', title=u"Soc un link")
        link = f2['enllac']
        link.open_link_in_new_window = False
        link.reindexObject()

        self.assertEqual(link.open_link_in_new_window, False)

        results = portal.portal_catalog.searchResults(portal_type='Link')
        self.assertEqual(results[0].open_link_in_new_window, False)

        link.open_link_in_new_window = True
        link.reindexObject()

        results = portal.portal_catalog.searchResults(portal_type='Link')
        self.assertEqual(results[0].open_link_in_new_window, True)

    def testHomePageMarkerInterface(self):
        self.assertTrue(IHomePage.providedBy(self.portal['front-page']))

    def testAdapters(self):
        setRoles(self.portal, TEST_USER_ID, ['Manager'])
        login(self.portal, TEST_USER_NAME)
        self.portal.invokeFactory('Document', 'test_adapter', title=u"Soc una pagina")
        from genweb.core.adapters import IImportant
        obj = IImportant(self.portal.test_adapter)
        self.assertEqual(obj.is_important, False)
        obj.is_important = True
        obj2 = IImportant(self.portal.test_adapter)
        self.assertEqual(obj2.is_important, True)

    def test_favorites(self):
        setRoles(self.portal, TEST_USER_ID, ['Manager'])
        login(self.portal, TEST_USER_NAME)
        self.portal.invokeFactory('Folder', 'prova', title=u"Soc una carpeta")
        prova = self.portal['prova']
        prova.invokeFactory('Folder', 'prova', title=u"Soc una carpeta")
        prova2 = prova['prova']

        from genweb.core.adapters.favorites import IFavorite
        IFavorite(prova2).add(TEST_USER_NAME)
        self.assertTrue(TEST_USER_NAME in IFavorite(prova2).get())
        self.assertTrue(TEST_USER_NAME not in IFavorite(prova).get())

    def test_protected_content(self):
        setRoles(self.portal, TEST_USER_ID, ['Manager'])
        login(self.portal, TEST_USER_NAME)
        self.portal.invokeFactory('Folder', 'test_folder', title=u"Soc una carpeta")
        self.portal.test_folder.invokeFactory('Document', 'test_document', title=u"Soc un document")
        from genweb.core.interfaces import IProtectedContent
        alsoProvides(self.portal.test_folder, IProtectedContent)
        setRoles(self.portal, TEST_USER_ID, ['Reader', 'Editor'])

        self.portal.test_folder.manage_delObjects('test_document')

        self.assertRaises(Unauthorized, self.portal.manage_delObjects, 'test_folder')

class FunctionalTest(unittest.TestCase):

    layer = GENWEB_FUNCTIONAL_TESTING

    def setUp(self):
        self.portal = self.layer['portal']
        self.request = self.layer['request']
        self.app = self.layer['app']
        self.browser = Browser(self.app)
        setRoles(self.portal, TEST_USER_ID, ['Manager'])

        # Create a portlet in a slot
        benvingut = self.portal['front-page']
        manager = queryUtility(IPortletManager, name='genweb.portlets.HomePortletManager2', context=benvingut)
        assignments = getMultiAdapter((benvingut, manager), IPortletAssignmentMapping)
        homepage_assignment = homepage.Assignment()
        assignments['homepage'] = homepage_assignment
        transaction.commit()
        setRoles(self.portal, TEST_USER_ID, ['Member'])

    def testHomePagePortlet(self):
        portalURL = self.portal.absolute_url()

        self.browser.open(portalURL)

        self.assertTrue('Congratulations! You have successfully installed Plone.' in self.browser.contents)


import numpy as np
from scipy import linalg

def dipole_field(xv, b0=-31100):
    """
    """
    bx = lambda x,y,z: 3*b0*x*z/np.sqrt(x*x+y*y+z*z)**5
    by = lambda x,y,z: 3*b0*y*z/np.sqrt(x*x+y*y+z*z)**5
    bz = lambda x,y,z: b0*(2*z*z-x*x-y*y)/np.sqrt(x*x+y*y+z*z)**5

    bvec = np.r_[bx(*xv), by(*xv), bz(*xv)]

    return bvec

def dipole_gradient(xv, b0=-31100, r_e=6371, dx=0.01):
    """Estimate gradient of dipole field

    Uses finite differences becauseit saves dev time over
    calculating derivatives analytically.
    """

    bx = lambda x, y, z: 3*b0*x*z/np.sqrt(x*x+y*y+z*z)**5
    by = lambda x, y, z: 3*b0*y*z/np.sqrt(x*x+y*y+z*z)**5
    bz = lambda x, y, z: b0*(2*z*z-x*x-y*y)/np.sqrt(x*x+y*y+z*z)**5
    df = lambda f, x, d: (f(x+d)-f(x-d))/(2*d)

    dbxdx = df(lambda x: bx(x, xv[1], xv[2]), xv[0], dx)
    dbxdy = df(lambda x: bx(xv[0], x, xv[2]), xv[1], dx)
    dbxdz = df(lambda x: bx(xv[0], xv[1], x), xv[2], dx)
    gradbx = np.r_[dbxdx, dbxdy, dbxdz]/r_e

    dbydx = df(lambda x: by(x, xv[1], xv[2]), xv[0], dx)
    dbydy = df(lambda x: by(xv[0], x, xv[2]), xv[1], dx)
    dbydz = df(lambda x: by(xv[0], xv[1], x), xv[2], dx)
    gradby = np.r_[dbydx, dbydy, dbydz]/r_e

    dbzdx = df(lambda x: bz(x, xv[1], xv[2]), xv[0], dx)
    dbzdy = df(lambda x: bz(xv[0], x, xv[2]), xv[1], dx)
    dbzdz = df(lambda x: bz(xv[0], xv[1], x), xv[2], dx)
    gradbz = np.r_[dbzdx, dbzdy, dbzdz]/r_e

    gradb = np.c_[gradbx, gradby, gradbz]

    return gradb

def grad_bmag(xv, r_e=6371, dx=0.01):
    """Calculate the gradient of the magnetic field magnitude"""
    btot = lambda x, y, z: linalg.norm(dipole_field([x, y, z]))
    df = lambda f, x: (f(x + dx) - f(x - dx))/(2*dx)
    bgradx = df(lambda x: btot(x, xv[1], xv[2]), xv[0])
    bgrady = df(lambda x: btot(xv[0], x, xv[2]), xv[1])
    bgradz = df(lambda x: btot(xv[0], xv[1], x), xv[2])
    return np.r_[bgradx, bgrady, bgradz]/r_e


#!/usr/bin/env python3

# Author: Jeffrey Grover
# Purpose: Create a new bed file where entries made of bins from the original
# This is most useful for creating metaplots or looking at expression over a
# fraction of the original features
# Created: 2018-01-15
# Refactored: 2019-06-22

import csv
from argparse import ArgumentParser

# Subroutine functions

def parse_bed_to_dict(input_bed):
    bed_dict = {}
    with open(input_bed, 'r') as input_handle:
        for line in input_handle:
            entry = line.strip().split('\t')
            feature_id = entry[3]
            try:
                score = entry[4]
            except IndexError:
                score = '.'
            try:
                strand = entry[5]
            except IndexError:
                strand = '.'
            bed_dict[feature_id] = {
                'chromosome': entry[0],
                'start': int(entry[1]),
                'stop': int(entry[2]),
                'score': score,
                'strand': strand
            }
    return bed_dict

def create_upstream_bins(bed_dict, window_size, bin_width, output_prefix):
    num_bins = int(window_size / bin_width)
    for i in range(num_bins):
        bin_number = i + 1  # Humans count starting at 1
        bin_start = window_size - (bin_width * i)
        bin_stop = bin_start - (bin_width - 1)
        upstream_file = '%su_%s_%s-%s.bed' % (output_prefix, str(bin_number).zfill(2), bin_start, bin_stop)
        with open(upstream_file, 'w') as upstream_handle:
            upstream_writer = csv.writer(upstream_handle, delimiter='\t')
            for feature_id in bed_dict:
                chromosome = bed_dict[feature_id]['chromosome']
                score = bed_dict[feature_id]['score']
                strand = bed_dict[feature_id]['strand']
                start = bed_dict[feature_id]['start']
                stop = bed_dict[feature_id]['stop']

                # Bins for + and unstranded start at the feature - window size

                if strand in ('+', '.'):
                    start = start - (window_size - (bin_width * i))
                    stop = start + (bin_width - 1)

                # Bins for - strand start at 1 beyond the stop

                elif strand == '-':
                    stop = stop + (window_size - (bin_width * i))
                    start = stop - (bin_width - 1)

                # Output bins, but only if the bins are valid coordinates

                if start >= 0 and stop >= 0:
                    output_row = [chromosome, start, stop, feature_id, score, strand]
                    upstream_writer.writerow(output_row)

def create_downstream_bins(bed_dict, window_size, bin_width, output_prefix):
    num_bins = int(window_size / bin_width)
    for i in range(num_bins):
        bin_number = i + 1
        bin_start = (bin_width * i) + 1
        bin_stop = (bin_start - 1) + bin_width
        downstream_file = '%sd_%s_%s-%s.bed' % (output_prefix, str(bin_number).zfill(2), bin_start, bin_stop)
        with open(downstream_file, 'w') as downstream_handle:
            downstream_writer = csv.writer(downstream_handle, delimiter='\t')
            for feature_id in bed_dict:
                chromosome = bed_dict[feature_id]['chromosome']
                score = bed_dict[feature_id]['score']
                strand = bed_dict[feature_id]['strand']
                start = bed_dict[feature_id]['start']
                stop = bed_dict[feature_id]['stop']

                # Bins for + and unstranded start at 1 beyond the feature stop

                if strand in ('+', '.'):
                    start = stop + ((bin_width * i) + 1)
                    stop = start + (bin_width - 1)

                # Bins for - strand start 1 before the start

                elif strand == '-':
                    stop = start - ((bin_width * i) + 1)
                    start = stop - (bin_width - 1)

                # Output bins, but only if the bins are valid coordinates

                if start >= 0 and stop >= 0:
                    output_row = [chromosome, start, stop, feature_id, score, strand]
                    downstream_writer.writerow(output_row)

def create_body_bins(bed_dict, window_size, bin_width, output_prefix):
    num_bins = int(window_size / bin_width)
    for i in range(num_bins):
        bin_number = i + 1
        bin_perc_length = bin_number * (100 / num_bins)
        body_file = '%sb_%s_%s.bed' % (output_prefix, str(bin_number).zfill(2), bin_perc_length)
        with open(body_file, 'w') as body_handle:
            body_writer = csv.writer(body_handle, delimiter='\t')
            for feature_id in bed_dict:
                chromosome = bed_dict[feature_id]['chromosome']
                score = bed_dict[feature_id]['score']
                strand = bed_dict[feature_id]['strand']
                start = bed_dict[feature_id]['start']
                stop = bed_dict[feature_id]['stop']
                feature_length = stop - start
                body_bin_width = (feature_length / num_bins)

                # For + strand or unstranded bin coordinates start at start + 1
                # First and last will be 1nt shorter to so exact start can be its own bin

                if strand in ('+', '.'):
                    if i == 0:
                        start = start + 1
                    else:
                        start = int(round((start + (body_bin_width * i)), 0))
                    stop = int(round((start + body_bin_width), 0) - 1)

                # For - strand bins are numbered from the bed file's "stop"
                # First will be 1nt shorter to so exact stop can be its own bin

                elif strand == '-':
                    if i == 0:
                        stop = int(round((stop - (body_bin_width * i)), 0) - 2)
                    else:
                        stop = int(round((stop - (body_bin_width * i)), 0) - 1)
                    start = int(round((stop - body_bin_width), 0))

                # Output bins, but only if the bins are valid coordinates

                if start >= 0 and stop >= 0:
                    output_row = [chromosome, start, stop, feature_id, score, strand]
                    body_writer.writerow(output_row)

def get_start_coords(bed_dict, output_prefix):
    start_file = '%sstart.bed' % output_prefix
    with open(start_file, 'w') as start_handle:
        start_writer = csv.writer(start_handle, delimiter='\t')
        for feature_id in bed_dict:
            chromosome = bed_dict[feature_id]['chromosome']
            score = bed_dict[feature_id]['score']
            strand = bed_dict[feature_id]['strand']
            start = bed_dict[feature_id]['start']
            stop = bed_dict[feature_id]['stop']

            # Start coordinates are zero-based already for + or unstranded

            if strand in ('+', '.'):
                stop = start + 1

            # For - strand some extra work is required

            elif strand == '-':
                start = stop - 1

            # Output bins

            output_row = [chromosome, start, stop, feature_id, score, strand]
            start_writer.writerow(output_row)

def get_stop_coords(bed_dict, output_prefix):
    stop_file = '%sstop.bed' % output_prefix
    with open(stop_file, 'w') as stop_handle:
        stop_writer = csv.writer(stop_handle, delimiter='\t')
        for feature_id in bed_dict:
            chromosome = bed_dict[feature_id]['chromosome']
            score = bed_dict[feature_id]['score']
            strand = bed_dict[feature_id]['strand']
            start = bed_dict[feature_id]['start']
            stop = bed_dict[feature_id]['stop']

            # Stop coordinate begins at stop -1 for + or unstranded

            if strand in ('+', '.'):
                start = stop - 1

            # Already zero baed for - strand

            elif strand == '-':
                stop = start + 1

            # Output bins

            output_row = [chromosome, start, stop, feature_id, score, strand]
            stop_writer.writerow(output_row)

# Parse command line options

def get_args():
    parser = ArgumentParser(
        description='Create separate bed files for desired bin sizes up and '
        'downstream of an input bed file. Divides feature bodies into an equal '
        'number of bins as well. Be sure to use bin widths that can be divided '
        'evenly into your window size or it will likely fail. Start and stop of'
        ' each feature is also output as a bin of length 1.')
    parser.add_argument('input_bed',
                        help='bed file to process',
                        metavar='FILE.bed')
    parser.add_argument('-w', '--window',
                        help='Size of up and downstream windows (nt)',
                        type=int,
                        metavar='INT')
    parser.add_argument('-b', '--bin_width',
                        help='Bin size (nt)',
                        type=int,
                        metavar='INT')
    parser.add_argument('-p', '--prefix',
                        help='Prefix for output files. Files will be output as: Prefix[body|up|down]_bin#_length[start-stop|percent].bed',
                        metavar='PREFIX')
    return parser.parse_args()

# Run the functions, create the bins

def main(args):
    bed_dict = parse_bed_to_dict(args.input_bed)
    create_body_bins(bed_dict, args.window, args.bin_width, args.output_prefix)
    create_upstream_bins(bed_dict, args.window, args.bin_width, args.output_prefix)
    create_downstream_bins(bed_dict, args.window, args.bin_width, args.output_prefix)
    get_start_coords(bed_dict, args.output_prefix)
    get_stop_coords(bed_dict, args.output_prefix)

if __name__ == '__main__':
    args = get_args()
    main(args)


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#     file: api.py
#     date: 2018-03-20
#   author: paul.dautry
#  purpose:
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# =============================================================================
#  IMPORTS
# =============================================================================
from pathlib import Path
from slugify import slugify
from argparse import ArgumentParser, Namespace
from traceback import print_exc
from mkctf.helper.logger import Logger
from mkctf.helper.config import load_config
from mkctf.command.init import init
from mkctf.command.show import show
from mkctf.command.build import build
from mkctf.command.deploy import deploy
from mkctf.command.status import status
from mkctf.command.create import create
from mkctf.command.delete import delete
from mkctf.command.enable import enable
from mkctf.command.export import export
from mkctf.command.disable import disable
from mkctf.command.configure import configure
from mkctf.command.renew_flag import renew_flag
from mkctf.object.repository import Repository
# =============================================================================
#  CONFIGURATION
# =============================================================================
DEFAULT_SIZE = 32
DEFAULT_TIMEOUT = 4 # seconds
# =============================================================================
#  CLASSES
# =============================================================================
class MKCTFAPI:
    """Provides access to all functionalities programmatically

    """
    @staticmethod
    def parse_args():
        """Parse command line arguments

        Returns:
            Namespace -- [description]
        """
        p = ArgumentParser(add_help=True,
                           description="Manage CTF challenges repository.")
        p.add_argument('-q', '--quiet', action='store_true',
                       help="decrease program verbosity")
        p.add_argument('-d', '--debug', action='store_true',
                       help="output debug messages")
        p.add_argument('--no-color', action='store_true',
                       help="disable colored output")
        p.add_argument('-r', '--repo-root', type=Path, default=Path.cwd(),
                       help="repository's root folder absolute path.")
        p.add_argument('-j', '--json', action='store_true',
                       help="json formatted output.")
        p.add_argument('-f', '--force', action='store_true',
                       help="do not ask for confirmation.")
        # -- add subparsers
        sps = p.add_subparsers(dest='command', metavar='COMMAND')
        sps.required = True
        # ---- init
        init_p = sps.add_parser('init', help="initializes mkctf repository.")
        init_p.set_defaults(func=init)
        # ---- show
        show_p = sps.add_parser('show', help="shows challenges.")
        show_p.add_argument('-c', '--category', help="challenge's category.")
        show_p.add_argument('-s', '--slug', help="challenge's slug.")
        show_p.set_defaults(func=show)
        # ---- create
        create_p = sps.add_parser('create', help="creates a challenge.")
        create_p.set_defaults(func=create)
        # ---- delete
        delete_p = sps.add_parser('delete', help="deletes a challenge.")
        delete_p.add_argument('category', help="challenge's category.")
        delete_p.add_argument('slug', help="challenge's slug.")
        delete_p.set_defaults(func=delete)
        # ---- configure
        configure_p = sps.add_parser('configure', help="edits repository's config "
                                                       "or challenge's config.")
        configure_p.add_argument('-c', '--category', help="challenge's category.")
        configure_p.add_argument('-s', '--slug', help="challenge's slug.")
        configure_p.set_defaults(func=configure)
        # ---- enable
        enable_p = sps.add_parser('enable', help="enables a challenge.")
        enable_p.add_argument('category', help="challenge's category.")
        enable_p.add_argument('slug', help="challenge's slug.")
        enable_p.set_defaults(func=enable)
        # ---- disable
        disable_p = sps.add_parser('disable', help="disables a challenge.")
        disable_p.add_argument('category', help="challenge's category.")
        disable_p.add_argument('slug', help="challenge's slug.")
        disable_p.set_defaults(func=disable)
        # ---- export
        export_p = sps.add_parser('export', help="exports enabled static "
                                                 "challenges.")
        export_p.add_argument('export_dir', type=Path,
                              help="folder where archives must be written. If "
                                   "the folder does not exist it will be "
                                   "created.")
        export_p.add_argument('-c', '--category', help="challenge's category.")
        export_p.add_argument('-s', '--slug', help="challenge's slug.")
        export_p.add_argument('--include-disabled', action='store_true',
                              help="export disabled challenges too.")
        export_p.set_defaults(func=export)
        # ---- renew-flag
        renew_flag_p = sps.add_parser('renew-flag',
                                       help="renews flags. You might want to "
                                            "build and deploy/export after that.")
        renew_flag_p.add_argument('-c', '--category', help="challenge's category.")
        renew_flag_p.add_argument('-s', '--slug', help="challenge's slug.")
        renew_flag_p.add_argument('--size', type=int, default=DEFAULT_SIZE,
                                  help="flag's random string size (in bytes).")
        renew_flag_p.set_defaults(func=renew_flag)
        # ---- build
        build_p = sps.add_parser('build',
                                 help="builds challenges. After building "
                                      "challenges you might want to deploy/export.")
        build_p.add_argument('-c', '--category', help="challenge's category.")
        build_p.add_argument('-s', '--slug', help="challenge's slug.")
        build_p.add_argument('-t', '--timeout', type=int, default=DEFAULT_TIMEOUT,
                             help="override default timeout for subprocesses.")
        build_p.set_defaults(func=build)
        # ---- deploy
        deploy_p = sps.add_parser('deploy', help="deploy challenges.")
        deploy_p.add_argument('-c', '--category', help="challenge's category.")
        deploy_p.add_argument('-s', '--slug', help="challenge's slug.")
        deploy_p.add_argument('-t', '--timeout', type=int, default=DEFAULT_TIMEOUT,
                              help="override default timeout for subprocesses.")
        deploy_p.set_defaults(func=deploy)
        # ---- status
        status_p = sps.add_parser('status', help="check deployed challenge's "
                                                 "status using exploit/exploit.")
        status_p.add_argument('-c', '--category', help="challenge's category.")
        status_p.add_argument('-s', '--slug', help="challenge's slug.")
        status_p.add_argument('-t', '--timeout', type=int, default=DEFAULT_TIMEOUT,
                              help="override default timeout for subprocesses.")
        status_p.set_defaults(func=status)

        args = p.parse_args()

        args.configuration = None

        return args

    def __init__(self, repo_root,
                 debug, quiet, no_color,
                 out=None):
        """Constructs a new instance

        Arguments:
            repo_root {Path} -- [description]
            debug {bool} -- [description]
            quiet {bool} -- [description]
            no_color {bool} -- [description]

        Keyword Arguments:
            out {IOBase} -- [description] (default: {None})
        """

        super(MKCTFAPI, self).__init__()
        self.logger = Logger(debug, quiet, no_color, out)

        self.repo_root = Path(repo_root)
        self.logger.debug('repo_root: {}'.format(self.repo_root))

        self.glob_conf_path = Path.home() / '.config/mkctf.yml'
        self.logger.debug('glob_conf_path: {}'.format(self.glob_conf_path))

        self.glob_conf = load_config(self.glob_conf_path)
        self.logger.debug('glob_conf: {}'.format(self.glob_conf))

        self.repo_conf_path = self.repo_root / self.glob_conf['files']['config']['repository']
        self.logger.debug('repo_conf_path: {}'.format(self.repo_conf_path))

        self.repo = Repository(self.logger, self.repo_conf_path, self.glob_conf)

    async def perform(self, ns):
        """Performs a comand using given Namespace ns

        Arguments:
            ns {[type]} -- [description]

        Returns:
            [type] -- [description]
        """
        self.logger.info("mkctf starts...")
        self.logger.debug("ns: {}".format(ns))

        if ns.command != 'init' and self.repo.get_conf() is None:
            self.logger.fatal("mkctf repository must be initialized first. "
                              "Run `mkctf init` first.")

        try:
            # -----------------------------------------------------------------
            # 'result' content depending on json argument value:
            # if ns.json:
            #   result = dict or None
            # else:
            #   result = True or False
            # -----------------------------------------------------------------
            result = await ns.func(ns, self.repo, self.logger)
        except Exception as e:
            print_exc()
            self.logger.fatal("Ouuuuupss.....:(")

        if result:
            self.logger.info("mkctf ended successfully." )
        else:
            self.logger.error("mkctf ended with errors.")

        return result

    def __ns(self, func):
        """Creates a standard Namespace used by all functions of the API

        Arguments:
            func {function} -- [description]

        Returns:
            Namespace -- [description]
        """
        ns = Namespace()
        ns.json = True
        ns.force = True
        ns.no_color = True
        ns.command = func.__name__
        ns.func = func
        return ns

    def init(self):
        """API wrapper for 'init' command

        Returns:
            [type] -- [description]
        """
        ns = self.__ns(init)
        # perform
        return self.perform(ns)

    def show(self):
        """API wrapper for 'show' command

        Returns:
            [type] -- [description]
        """
        ns = self.__ns(show)
        # perform
        return self.perform(ns)

    def create(self,
               category,
               name,
               flag,
               points,
               parameters={},
               enabled=False,
               standalone=True):
        """API wrapper for 'create' command

        Arguments:
            category {str} -- [description]
            name {str} -- [description]
            flag {str} -- [description]
            points {int} -- [description]
            parameters {dict} -- [description]
            enabled {bool} -- [description]
            standalone {bool} -- [description]

        Returns:
            [type] -- [description]
        """
        ns = self.__ns(create)
        # parameters
        ns.configuration = {
                'name': name,
                'slug': slugify(name),
                'flag': flag,
                'points': points,
                'enabled': enabled,
                'category': category,
                'parameters': parameters,
                'standalone': standalone
        }
        # perform
        return self.perform(ns)

    def delete(self, category=None, slug=None):
        """API wrapper for 'delete' command

        Keyword Arguments:
            category {str} -- [description] (default: {None})
            slug {str} -- [description] (default: {None})

        Returns:
            [type] -- [description]
        """
        ns = self.__ns(delete)
        # parameters
        ns.category = category
        ns.slug = slug
        # perform
        return self.perform(ns)

    def configure(self, configuration, category=None, slug=None):
        """API wrapper for 'configure' command

        Arguments:
            configuration {dict} -- [description]

        Keyword Arguments:
            category {str} -- [description] (default: {None})
            slug {str} -- [description] (default: {None})

        Returns:
            [type] -- [description]
        """
        ns = self.__ns(enable)
        # parameters
        ns.configuration = configuration
        ns.category = category
        ns.slug = slug
        # perform
        return self.perform(ns)

    def enable(self, category, slug):
        """API wrapper for 'enable' command

        Arguments:
            category {str} -- [description]
            slug {str} -- [description]

        Returns:
            [type] -- [description]
        """
        ns = self.__ns(enable)
        # parameters
        ns.category = category
        ns.slug = slug
        # perform
        return self.perform(ns)

    def disable(self, category, slug):
        """API wrapper for 'disable' command

        Arguments:
            category {str} -- [description]
            slug {str} -- [description]

        Returns:
            [type] -- [description]
        """
        ns = self.__ns(disable)
        # parameters
        ns.category = category
        ns.slug = slug
        # perform
        return self.perform(ns)

    def export(self, export_dir,
               category=None, slug=None,
               include_disabled=False):
        """API wrapper for 'export' command

        Arguments:
            export_dir {Path} -- [description]

        Keyword Arguments:
            category {str} -- [description] (default: {None})
            slug {str} -- [description] (default: {None})
            include_disabled {bool} -- [description] (default: {False})

        Returns:
            [type] -- [description]
        """
        ns = self.__ns(export)
        # parameters
        ns.export_dir = export_dir
        ns.category = category
        ns.slug = slug
        ns.include_disabled = include_disabled
        # perform
        return self.perform(ns)

    def renew_flag(self, category=None, slug=None, size=DEFAULT_SIZE):
        """API wrapper for 'renew_flag' command

        Keyword Arguments:
            category {str} -- [description] (default: {None})
            slug {str} -- [description] (default: {None})
            size {int} -- [description] (default: {DEFAULT_SIZE})

        Returns:
            [type] -- [description]
        """
        ns = self.__ns(renew_flag)
        # parameters
        ns.category = category
        ns.slug = slug
        ns.size = size
        # perform
        return self.perform(ns)

    def build(self, category=None, slug=None, timeout=DEFAULT_TIMEOUT):
        """API wrapper for 'build' command

        Keyword Arguments:
            category {str} -- [description] (default: {None})
            slug {str} -- [description] (default: {None})
            timeout {int} -- [description] (default: {DEFAULT_TIMEOUT})

        Returns:
            [type] -- [description]
        """
        ns = self.__ns(build)
        # parameters
        ns.category = category
        ns.slug = slug
        ns.timeout = timeout
        # perform
        return self.perform(ns)

    def deploy(self, category=None, slug=None, timeout=DEFAULT_TIMEOUT):
        """API wrapper for 'deploy' command

        Keyword Arguments:
            category {str} -- [description] (default: {None})
            slug {str} -- [description] (default: {None})
            timeout {int} -- [description] (default: {DEFAULT_TIMEOUT})

        Returns:
            [type] -- [description]
        """
        ns = self.__ns(deploy)
        # parameters
        ns.category = category
        ns.slug = slug
        ns.timeout = timeout
        # perform
        return self.perform(ns)

    def status(self, category=None, slug=None, timeout=DEFAULT_TIMEOUT):
        """API wrapper for 'status' command

        Keyword Arguments:
            category {str} -- [description] (default: {None})
            slug {str} -- [description] (default: {None})
            timeout {int} -- [description] (default: {DEFAULT_TIMEOUT})

        Returns:
            [type] -- [description]
        """
        ns = self.__ns(status)
        # parameters
        ns.category = category
        ns.slug = slug
        ns.timeout = timeout
        # perform
        return self.perform(ns)


# -*- coding: utf-8 -*-
import pymongo
from bson.objectid import ObjectId
from db import DB
import datetime

class PollIpRepository(DB):

    def __init__(self,pollId):
        DB.__init__(self)
        self.table = self.database['ip_%s'%pollId]

    def CreateIndex(self):
        self.table.ensure_index('ip',unique=False)

    def GetIpCount(self,dataId=None,ip=None,ipLimitTime=None):
        '''返回指定时间内的ip数'''
        condition ={}
        if dataId:
            condition['data_id'] = dataId
        if ip:
            condition['ip'] = ip
        if ipLimitTime:
            startTime = datetime.datetime.now() - datetime.timedelta(seconds=ipLimitTime)
            condition['isrtdate'] = {'$gte':startTime}
        cur = self.table.find(condition)
        return cur.count()

    def Search(self,data_id=None,ip=None,sort='isrtdate',desc=True,start=0,count=10):
        condition = {}

        #

        #如果用户按照uid或者id搜索。则将条件设置为唯一搜索
        if data_id:
            condition['data_id'] = data_id

        if ip:
            condition['ip'] = ip

        cur = self.table.find(condition)
        #满足条件的总记录数
        totalCount = cur.count()
        #设定排序
        sortType = pymongo.ASCENDING
        if desc:
            sortType = pymongo.DESCENDING
        cur = cur.skip(start).limit(count).sort(sort, sortType)
        recordList =list()
        for item in cur:
            recordList.append(item)
        return {'list':recordList,'total_count':totalCount}

    def Save(self,data):
        self.table.save(data)
        return True

    def Remove(self,id):
        self.table.remove({'id':id})
        return True

    def RemoveAll(self):
        self.table.remove()
        return True

if __name__=='__main__':
    pass


# -*- coding: utf-8 -*-

"""Processing a list of power plants in Germany.

SPDX-FileCopyrightText: 2016-2021 Uwe Krien <krien@uni-bremen.de>

SPDX-License-Identifier: MIT
"""
__copyright__ = "Uwe Krien <krien@uni-bremen.de>"
__license__ = "MIT"

# Python libraries
import os
import logging

# External libraries
import pandas as pd
from workalendar.europe import Germany

# oemof libraries
import demandlib.bdew as bdew

# internal modules
from reegis import config as cfg
from reegis import bmwi
from reegis import geometries
from reegis import energy_balance
from reegis import coastdat
from reegis import inhabitants

def heat_demand(year):
    """
    Fetch heat demand per sector from the federal states energy balances.

    If the share between domestic and retail does not exist the share from
    the german balance is used. If this value also not exists a default
    share of 0.5 is used.

    Parameters
    ----------
    year

    Returns
    -------
    pandas.DataFrame

    Examples
    --------
    >>> hd=heat_demand(2014)
    >>> hd.loc[('MV', 'domestic'), 'district heating']
    5151.5
    """
    eb = energy_balance.get_usage_balance(year)
    eb.sort_index(inplace=True)

    # get fraction of domestic and retail from the german energy balance
    share = energy_balance.get_domestic_retail_share(year)

    # Use 0.5 for both sectors if no value is given
    share.fillna(0.5, inplace=True)

    # Divide domestic and retail by the value of the german energy balance if
    # the sum of domestic and retail does not equal the value given in the
    # local energy balance.
    check_value = True
    for state in eb.index.get_level_values(0).unique():
        for col in eb.columns:
            check = (
                eb.loc[(state, "domestic"), col]
                + eb.loc[(state, "retail"), col]
                - eb.loc[(state, "domestic and retail"), col]
            ).round()
            if check < 0:
                for sector in ["domestic", "retail"]:
                    try:
                        eb.loc[(state, sector), col] = (
                            eb.loc[(state, "domestic and retail"), col]
                            * share.loc[col, sector]
                        )
                    except KeyError:
                        eb.loc[(state, sector), col] = (
                            eb.loc[(state, "domestic and retail"), col] * 0.5
                        )

                check = (
                    eb.loc[(state, "domestic"), col]
                    + eb.loc[(state, "retail"), col]
                    - eb.loc[(state, "domestic and retail"), col]
                ).round()

                if check < 0:
                    logging.error(
                        "In {0} the {1} sector results {2}".format(
                            state, col, check
                        )
                    )
                    check_value = False
    if check_value:
        logging.debug("Divides 'domestic and retail' without errors.")

    # Reduce energy balance to the needed columns and group by fuel groups.
    eb = eb.loc[
        (slice(None), ["industrial", "domestic", "retail"]),
    ]

    eb = eb.groupby(by=cfg.get_dict("FUEL_GROUPS_HEAT_DEMAND"), axis=1).sum()

    # Remove empty columns
    for col in eb.columns:
        if not (
            eb.loc[(slice(None), "domestic"), col].sum() > 0
            or eb.loc[(slice(None), "retail"), col].sum() > 0
            or eb.loc[(slice(None), "industrial"), col].sum() > 0
        ):
            del eb[col]

    # The use of electricity belongs to the electricity sector. It is possible
    # to connect it to the heating sector for future scenarios.
    del eb["electricity"]
    del eb["total"]  # if electricity is removed total is not correct anymore.

    # get fraction of mechanical energy use and subtract it from the balance to
    # get the use of heat only.
    share_mech = share_of_mechanical_energy_bmwi(year)
    for c in share_mech.columns:
        for i in share_mech.index:
            eb.loc[(slice(None), c), i] -= (
                eb.loc[(slice(None), c), i] * share_mech.loc[i, c]
            )
    eb.sort_index(inplace=True)

    return eb

def share_of_mechanical_energy_bmwi(year):
    """
    Get share of mechanical energy from the overall energy use per sector.

    Parameters
    ----------
    year : int

    Returns
    -------
    pandas.DataFrame

    Examples
    --------
    >>> share_of_mechanical_energy_bmwi(2014).loc['oil', 'retail']
    0.078

    """
    mech = pd.DataFrame()
    fs = bmwi.read_bmwi_sheet_7("a")
    fs.sort_index(inplace=True)
    sector = "Industrie"

    total = float(fs.loc[(sector, "gesamt"), year])
    mech[sector] = (
        fs.loc[(sector, "mechanische Energie"), year].div(total).round(3)
    )

    fs = bmwi.read_bmwi_sheet_7("b")
    fs.sort_index(inplace=True)
    for sector in fs.index.get_level_values(0).unique():
        total = float(fs.loc[(sector, "gesamt"), year])
        mech[sector] = (
            fs.loc[(sector, "mechanische Energie"), year]
            .div(total)
            .astype(float)
            .round(3)
        )
    mech.drop(" - davon Strom", inplace=True)
    mech.drop("mechanische Energie", inplace=True)
    ren_col = {
        "Industrie": "industrial",
        "Gewerbe, Handel, Dienstleistungen ": "retail",
        "private Haushalte": "domestic",
    }
    ren_index = {
        " - davon Öl": "oil",
        " - davon Gas": "natural gas",
    }
    mech.index.name = ""
    mech.rename(columns=ren_col, inplace=True)
    mech.rename(index=ren_index, inplace=True)
    mech.fillna(0, inplace=True)
    return mech

def get_heat_profile_from_demandlib(
    temperature, annual_demand, sector, year, build_class=1
):
    """
    Create an hourly load profile from the annual demand using the demandlib.

    Parameters
    ----------
    temperature : pandas.Series
    annual_demand : float
    sector : str
    year : int
    build_class : int

    Returns
    -------
    pandas.DataFrame

    Examples
    --------
    >>> temperature=pd.Series(list(range(50)), index=pd.date_range(
    ...     '2014-05-03 12:00', periods=50, freq='h'))
    >>> temperature = 10 + temperature * 0.1
    >>> hp=get_heat_profile_from_demandlib(
    ...     temperature, 5345, 'retail', 2014)
    >>> int(round(hp.sum()))
    5302
    """
    cal = Germany()
    holidays = dict(cal.holidays(year))

    if "efh" in sector:
        shlp_type = "EFH"
    elif "mfh" in sector:
        shlp_type = "MFH"
    elif "domestic" in sector:
        shlp_type = "MFH"
    elif "retail" in sector:
        shlp_type = "ghd"
        build_class = 0
    elif "industrial" in sector:
        shlp_type = "ghd"
        build_class = 0
    else:
        raise AttributeError('"{0}" is an unknown sector.'.format(sector))
    return bdew.HeatBuilding(
        temperature.index,
        holidays=holidays,
        temperature=temperature,
        shlp_type=shlp_type,
        wind_class=0,
        building_class=build_class,
        annual_heat_demand=annual_demand,
        name=sector,
        ww_incl=True,
    ).get_bdew_profile()

def get_heat_profiles_by_federal_state(
    year, to_csv=None, state=None, weather_year=None
):
    """
    Get heat profiles by state, sector and fuel. Use the pandas `groupby`
    method to group the results.

    The unit of the resulting data is TJ.

    Parameters
    ----------
    year : int
        Year of the demand data set.
    to_csv : str
        Path to the csv file.
    state : list or None
        List of abbreviations of federal states. If None a table with all
        federal states will be returned. Valid values are: BB, BE, BW, BY, HB,
        HE, HH, MV, NI, NW, RP, SH, SL, SN, ST, TH
    weather_year : int or None
        Can be used if the year of the weather data differs from the year of
        the demand data. If None the year parameter will be used. Use with
        care, because the demand data may include implicit weather effects.

    Returns
    -------
    pd.DataFrame

    Examples
    --------
    >>> fn=os.path.join(os.path.expanduser('~'),
    ...     'heat_profile.reegis_doctest.csv')
    >>> hp=get_heat_profiles_by_federal_state(2014, state=['BE', 'BB'],
    ...                                         to_csv=fn)
    >>> hp.groupby(level=[0, 1], axis=1).sum().sum().round(1)
    BB  domestic      66822.4
        industrial    69668.0
        retail        23299.5
    BE  domestic      67382.1
        industrial     6162.8
        retail        39364.9
    dtype: float64
    >>> round(hp.groupby(level=[0, 2], axis=1).sum().sum().loc['BB'], 1)
    district heating    17646.9
    gas                  3916.3
    hard coal           21378.4
    lignite              5630.5
    natural gas         63840.8
    oil                 16257.4
    other                1112.1
    re                  30007.4
    dtype: float64
    >>> hp_MWh=hp.div(0.0036)
    >>> round(hp_MWh.groupby(level=[2], axis=1).sum().sum().loc['lignite'], 1)
    1671427.4
    >>> round(hp.sum().sum(), 1)
    272699.7
    """

    if weather_year is None:
        weather_year = year

    building_class = {}
    for (k, v) in cfg.get_dict("building_class").items():
        for s in v.split(", "):
            building_class[s] = int(k)

    demand_state = heat_demand(year).sort_index()

    temperatures = coastdat.federal_state_average_weather(
        weather_year, "temp_air"
    )

    temperatures = temperatures.tz_convert("Europe/Berlin")

    my_columns = pd.MultiIndex(levels=[[], [], []], codes=[[], [], []])
    heat_profiles = pd.DataFrame(columns=my_columns)

    if state is None:
        states = demand_state.index.get_level_values(0).unique()
    else:
        states = state

    # for region in demand_state.index.get_level_values(0).unique():
    for region in states:
        logging.info("Creating heat profile for {}".format(region))
        tmp = demand_state.loc[region].groupby(level=0).sum()
        temperature = temperatures[region] - 273
        for fuel in tmp.columns:
            logging.debug(
                "{0} - {1} ({2})".format(region, fuel, building_class[region])
            )
            for sector in tmp.index:
                heat_profiles[
                    (region, sector, fuel)
                ] = get_heat_profile_from_demandlib(
                    temperature,
                    tmp.loc[sector, fuel],
                    sector,
                    year,
                    building_class[region],
                )
    heat_profiles.sort_index(1, inplace=True)

    if to_csv is not None:
        heat_profiles.to_csv(to_csv)
    return heat_profiles

def get_heat_profiles_by_region(
    regions, year, name="region", from_csv=None, to_csv=None, weather_year=None
):
    """
    Get heat profiles for any region divided by sector and fuel. Use the
    pandas `groupby` method to group the results.

    The unit of the resulting data is TJ.

    Parameters
    ----------
    year : int
        Year of the demand data set.
    regions : geopandas.geoDataFrame
        A table with region geometries and there id as index.
    name : str
        Name of the regions set.
    from_csv : str
        Path to the file of the demand state profiles.
    to_csv : str
        Path with filename of the output file.
    weather_year : int or None
        Can be used if the year of the weather data differs from the year of
        the demand data. If None the year parameter will be used. Use with
        care, because the demand data may include implicit weather effects.

    Returns
    -------
    pd.DataFrame

    Examples
    --------
    >>> from reegis import geometries
    >>> fn=os.path.join(os.path.expanduser('~'),
    ...                   'heat_profile.reegis_doctest.csv')
    >>> regions=geometries.load(
    ...     cfg.get('paths', 'geometry'),
    ...     cfg.get('geometry', 'de21_polygons'))
    >>> hpl=get_heat_profiles_by_region(regions, 2014, from_csv=fn)
    >>> round(hpl.sum().sum(), 1)
    272699.7
    >>> os.remove(fn)

    """
    if weather_year is None:
        weather_year = year

    # Get demand by federal state
    if from_csv is None:
        from_csv = os.path.join(
            cfg.get("paths", "demand"),
            cfg.get("demand", "heat_profile_state_var").format(
                year=year, weather_year=weather_year
            ),
        )
    if not os.path.isfile(from_csv):
        get_heat_profiles_by_federal_state(
            year, to_csv=from_csv, weather_year=weather_year
        )
    demand_state = pd.read_csv(from_csv, index_col=[0], header=[0, 1, 2])

    # Create empty MulitIndex DataFrame to take the results
    four_level_columns = pd.MultiIndex(
        levels=[[], [], [], []], codes=[[], [], [], []]
    )
    demand_region = pd.DataFrame(
        index=demand_state.index, columns=four_level_columns
    )

    # Get inhabitants for federal states and the given regions
    ew = inhabitants.get_share_of_federal_states_by_region(year, regions, name)

    # Use the inhabitants to recalculate the demand from federal states to
    # the given regions.
    for i in ew.items():
        state = i[0][1]
        region = i[0][0]
        share = i[1]
        if state in demand_state.columns.get_level_values(0).unique():
            for sector in (
                demand_state[state].columns.get_level_values(0).unique()
            ):
                for fuel in demand_state[state, sector].columns:
                    demand_region[region, fuel, sector, state] = (
                        demand_state[state, sector, fuel] * share
                    )
    demand_region.sort_index(1, inplace=True)
    demand_region = demand_region.groupby(level=[0, 1, 2], axis=1).sum()

    if to_csv is not None:
        demand_region.to_csv(to_csv)

    return demand_region

if __name__ == "__main__":
    pass


"""Class for training / evaluating pytorch networks

Reference Implementations:
- https://github.com/keras-team/keras/blob/master/keras/engine/training.py
"""

from tensorflow.python.keras.callbacks import (
    BaseLogger, CallbackList, History, ProgbarLogger
)
import torch

class Model(object):
    """Model for training / evaluating pytorch networks

    Reference Implementation:
    - https://github.com/keras-team/keras/blob/master/keras/engine/training.py
    """

    def __init__(self, network, device=None):
        """Init

        :param network: pytorch network to train or evaluate
        :type network: torch.nn.Module
        :param device: device to train the network on, e.g. 'cuda:0'
        :type device: str
        """

        self.network = network
        self.device = device

        self._compiled = False
        # these are set in the `compile` method
        self.optimizer = None
        self.loss = None

        self.history = History()
        self.stop_training = False

    def _assert_compiled(self):
        """Raise a value error if the model is not compiled

        This is a convenience wrapper to avoid duplicating these lines in
        multiple methods.

        :raises: RuntimeError if `self._compiled` is not True
        """

        if not self._compiled:
            msg = ('Model must be compiled before training; please call '
                   'the `compile` method before training.')
            raise RuntimeError(msg)

    def _default_callbacks(self):
        """Return default callbacks automatically applied during training

        By default, the following callbacks are automatically applied during
        training:
        - tensorflow.keras.callbacks.BaseLogger
        - tensorflow.keras.callbacks.ProgbarLogger
        - tensorflow.keras.callbacks.History (which is the `Model.history`
          attribute set in `Model.__init__`)

        :return: callbacks automatically applied to every Model
        :rtype: list
        """

        default_callbacks = [
            BaseLogger(), ProgbarLogger(count_mode='steps'), self.history
        ]
        return default_callbacks

    def compile(self, optimizer, loss):
        """Setup the model for training

        This sets `self.optimizer` and `self.loss` in place.

        :param optimizer: class name of the optimizer to use when training, one
         of those from `torch.optim` (e.g. `Adam`)
        :type optimizer: str
        :param loss: class name of the loss to use when training, one of those
         from `torch.nn` (e.g. `CrossEntropyLoss`)
        :type loss: str
        :raises AttributeError: if an invalid optimizer or loss function is
         specified
        """

        try:
            Optimizer = getattr(torch.optim, optimizer)
        except AttributeError:
            msg = (
                '`optimizer` must be a `str` representing an optimizer from '
                'the `torch.optim` package, and {} is not a valid one.'
            )
            raise AttributeError(msg.format(optimizer))
        self.optimizer = Optimizer(self.network.parameters())

        try:
            Loss = getattr(torch.nn, loss)
        except AttributeError:
            msg = (
                '`loss` must be a `str` representing a loss from '
                'the `torch.nn` package, and {} is not a valid one.'
            )
            raise AttributeError(msg.format(loss))
        self.loss = Loss()

        self._compiled = True

    def evaluate_generator(self, generator, n_steps):
        """Evaluate the network on batches of data generated from `generator`

        :param generator: a generator yielding batches indefinitely, where each
         batch is a tuple of (inputs, targets)
        :type generator: generator
        :param n_steps: number of batches to evaluate on
        :type n_steps: int
        :return: scalar test loss
        :rtype: float
        """

        self._assert_compiled()

        if self.device:
            self.network.to(self.device)

        total_loss = 0
        n_obs = 0
        for _ in range(n_steps):
            inputs, targets = next(generator)
            n_obs += inputs.shape[0]

            loss = self.test_on_batch(inputs, targets)
            total_loss += loss

        return total_loss / n_obs

    def fit_generator(self, generator, n_steps_per_epoch, n_epochs=1,
                      validation_data=None, n_validation_steps=None):
        """Train the network on batches of data generated from `generator`

        :param generator: a generator yielding batches indefinitely, where each
         batch is a tuple of (inputs, targets)
        :type generator: generator
        :param n_steps_per_epoch: number of batches to train on in one epoch
        :type n_steps_per_epoch: int
        :param n_epochs: number of epochs to train the model
        :type n_epochs: int
        :param validation_data: generator yielding batches to evaluate the loss
         on at the end of each epoch, where each batch is a tuple of (inputs,
         targets)
        :type validation_data: generator
        :param n_validation_steps: number of batches to evaluate on from
         `validation_data`
        :raises RuntimeError: if only one of `validation_data` and
         `n_validation_steps` are passed in
        """

        default_callbacks = self._default_callbacks()
        callbacks = CallbackList(default_callbacks)

        self._assert_compiled()

        invalid_inputs = (
            (validation_data is not None and n_validation_steps is None) or
            (n_validation_steps is not None and validation_data is None)
        )
        if invalid_inputs:
            msg = (
                '`validation_data` and `n_validation_steps` must both be '
                'passed, or neither.'
            )
            raise RuntimeError(msg)

        if self.device:
            self.network.to(self.device)

        callbacks.set_params({
            'epochs': n_epochs,
            'metrics': ['loss', 'val_loss'],
            'steps': n_steps_per_epoch,
            'verbose': True
        })
        callbacks.set_model(self)

        callbacks.on_train_begin()
        for idx_epoch in range(n_epochs):
            if self.stop_training:
                break

            epoch_logs = {}
            callbacks.on_epoch_begin(idx_epoch)

            for idx_batch in range(n_steps_per_epoch):
                batch_logs = {'batch': idx_batch, 'size': 1}
                callbacks.on_batch_begin(idx_batch, batch_logs)

                inputs, targets = next(generator)
                loss = self.train_on_batch(inputs, targets)

                batch_logs['loss'] = loss
                callbacks.on_batch_end(idx_batch, batch_logs)

                if self.stop_training:
                    break

            if validation_data:
                val_loss = self.evaluate_generator(
                    validation_data, n_validation_steps
                )
                epoch_logs['val_loss'] = val_loss
            callbacks.on_epoch_end(idx_epoch, epoch_logs)
        callbacks.on_train_end()

    def load_weights(self, fpath_weights):
        """Loads all layer weights from the provided `fpath_weights`

        :param fpath_weights: fpath_weights to load the model from
        :type fpath_weights: str
        """

        self.network.load_state_dict(torch.load(fpath_weights))

    def save_weights(self, fpath_weights, overwrite=True):
        """Dumps all layers and weights to the provided `fpath_weights`

        The weights can be loaded into a `Model` with the same topology using
        the `Model.load_weights` method.

        :param fpath_weights: fpath_weights to save the model to
        :type fpath_weights: str
        :param overwrite: overwrite an existing file at `fpath_weights`
         (if present); only True is currently supported
        :type overwrite: bool
        """

        assert overwrite, '`overwrite=False` is not supported!'
        torch.save(self.network.state_dict(), fpath_weights)

    def test_on_batch(self, inputs, targets):
        """Evaluate the model on a single batch of samples

        :param inputs: inputs to predict on
        :type inputs: torch.Tensor
        :param targets: targets to compare model predictions to
        :type targets: torch.Tensor
        :return: scalar test loss
        :rtype: float
        """

        self._assert_compiled()

        self.network.train(mode=False)
        if self.device:
            inputs = inputs.to(self.device)
            targets = targets.to(self.device)

        outputs = self.network(inputs)
        loss = self.loss(outputs, targets)

        return loss.tolist()

    def train_on_batch(self, inputs, targets):
        """Run a single forward / backward pass on a single batch of data

        :param inputs: inputs to use in the forward / backward pass
        :type inputs: torch.Tensor
        :param targets: targets to use in the forward / backward pass
        :type targets: torch.Tensor
        :return: scalar training loss
        :rtype: float
        """

        self._assert_compiled()

        self.network.train(mode=True)
        if self.device:
            inputs = inputs.to(self.device)
            targets = targets.to(self.device)

        outputs = self.network(inputs)
        loss = self.loss(outputs, targets)

        self.optimizer.zero_grad()
        loss.backward()
        self.optimizer.step()

        return loss.tolist()


"""
Migration script to create the genome_index_tool_data table.
"""

import datetime
import logging

from sqlalchemy import (
    Column,
    DateTime,
    ForeignKey,
    Integer,
    MetaData,
    String,
    Table
)

from galaxy.model.migrate.versions.util import (
    create_table,
    drop_table
)

log = logging.getLogger(__name__)
now = datetime.datetime.utcnow
metadata = MetaData()

GenomeIndexToolData_table = Table("genome_index_tool_data", metadata,
                                  Column("id", Integer, primary_key=True),
                                  Column("job_id", Integer, ForeignKey("job.id"), index=True),
                                  Column("dataset_id", Integer, ForeignKey("dataset.id"), index=True),
                                  Column("deferred_job_id", Integer, ForeignKey("deferred_job.id"), index=True),
                                  Column("transfer_job_id", Integer, ForeignKey("transfer_job.id"), index=True),
                                  Column("fasta_path", String(255)),
                                  Column("created_time", DateTime, default=now),
                                  Column("modified_time", DateTime, default=now, onupdate=now),
                                  Column("indexer", String(64)),
                                  Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True))

def upgrade(migrate_engine):
    print(__doc__)
    metadata.bind = migrate_engine
    metadata.reflect()

    create_table(GenomeIndexToolData_table)

def downgrade(migrate_engine):
    metadata.bind = migrate_engine
    metadata.reflect()

    drop_table(GenomeIndexToolData_table)


import numpy as np
import matplotlib.pyplot as plt

fig = plt.figure()
# syntax for 3-D projection
ax = plt.axes(projection='3d')
# defining all 3 axes
z = np.linspace(0, 1, 100)
x = z**2
y = z**3
# plotting
ax.plot3D(x, y, z, 'r')
ax.set_title('3D line graph plotting')
plt.show()


import re
import os
import datetime

class appCache(object):
	__modelString = "CACHE MANIFEST\n#%s:v3\n\n%s\n\n# All other resources (e.g. sites) require the user to be online.\nNETWORK:\n*"
	def __init__(self, path, name="appCacheFile"):
		if os.path.isfile(path):
			self.path = path
			self.name = name
			self.dirname = os.path.dirname(path)
			self.fileAppCache = os.path.join(self.dirname, self.name + '.appcache')
		else:
			raise Exception('Path doesn\'t contain a valid file!')

	def _readFile(self):
		if os.path.exists(self.path):
			with open(self.path) as file:
				readed = file.read()
			return re.findall('href=[\"\']([^\'\"].*?\.css)[\'\"]', readed) + re.findall('src=[\"\']([^\'\"].*?\.js)[\'\"]', readed)
		return False

	def createAppCache(self):
		pathsString = self._readFile();

		with open(self.fileAppCache, 'w+') as file:
			file.write(__modelString % (datetime.date.today(), pathsString))

	def printAppCache(self):
		pathsString = "";

		for item in self._readFile():
			pathsString += item + '\n'

		file.write(__modelString % (datetime.date.today(), pathsString))

appcache = appCache('D:\wwwroot\QA\MarketPlace.Web\index.html', 'appMarketplace')
appcache.createAppCache()


import json

class Settings:
    def __init__(self, defpath):
        with open(defpath,'r') as deffile:
            deff = json.load(deffile)
            self.dateformat  = deff["meta"]["dateformat"]
            self.delimiter   = deff["meta"]["delimiter"]
            self.encoding    = deff["meta"]["encoding"]
            self.locale      = deff["meta"]["locale"]
            self.accountName = deff["meta"]["accountName"]
            self.header      = int(deff["meta"]["header"])
            self.date        = int(deff["columns"]["date"])
            self.amount      = int(deff["columns"]["amount"])
            self.memo        = int(deff["columns"]["memo"])
            self.payee       = int(deff["columns"]["payee"])


"""
  The test file need to be run inside the gui_testing directory,
  with application centered on the (1920,1080) resolution main screen.
  The test cases are for the system testing, with the tests visulized.
  The test cases are testing the user story 1, mainly with the functionalities
  regarding:
    1. add, delete, update client information; choose client to add equipments
    2. add, delete equipments; choose equipment to add new run with a pair of CSVs
    3. add, delete new run

"""

import unittest
import os
import shutil
from time import sleep
import pyautogui as pg

# W0311: bad indentation; E1101:no-member; C0325: Unnecessary parens
# pylint: disable=W0311, E1101, C0325

# use the current directory, where the script exists
CWD = os.path.dirname(os.path.realpath(__file__))

class TestUserStory1(unittest.TestCase):
  """The user story 1 related system testing

  Args:
      unittest ([type]): [The default testing module from Python]
  """
  @classmethod
  def setUpClass(self):
    """ Create or replace previous folder """
    try:
      folder = os.getcwd() + "\\US01 Fail"
      shutil.rmtree(folder)
    except FileNotFoundError:
      print("[There's no such directory]")

    try:
      folder = os.getcwd() + "\\US01 Success"
      shutil.rmtree(folder)
    except FileNotFoundError:
      print("[There's no such directory]")

    try:
      folder = os.getcwd() + "\\US01"
      shutil.rmtree(folder)
    except FileNotFoundError:
      print("[There's no such directory]")

    try:
      self.cwd = os.getcwd() # for printing the current directory
      # print(self.cwd)
      os.mkdir("US01")
      os.chdir("US01") # cd to the directory for saving screenshots
      self.storage = '%s/' % os.getcwd() # escape the cwd string
    except:
      print("[Something is wrong when creating the directory]")

  @classmethod
  def tearDownClass(self):
    """ End the test, determine if tests succeed or fail
    Then, rename the folder.
    """
    file_list = os.listdir()
    # print(file_list)
    # iterate through the filename string, if there's a keyword "fail", rename the folder
    for filename in file_list:
      if "fail" in filename:
        # print("Some tests failed")
        os.chdir("..")
        os.rename("US01", "US01 Fail")
        return
    os.chdir("..")
    os.rename("US01", "US01 Success")
    # print("All tests passed")

  def test_a_add_new_client(self):
    ''' add a new client '''
    try:
      # The pyautogui doesn't recognize the pop up window -> use the coordination
      btn_add_new_client = pg.locateOnScreen(
        os.path.join(CWD, 'screenshot' ,'main_win' ,'add_new_client.png'),
        grayscale=False, confidence=.9)

      pg.moveTo(btn_add_new_client)
      pg.click()
    except:
      print("[There's no such button matching the picture]")
      raise
    # add a new client
    try:
      pg.moveTo(942, 464, 0.6) # cal number
      pg.click()
      pg.write("CAL00004")

      pg.moveTo(942, 502, 0.6) # client name
      pg.click()
      pg.write("Test client name")

      pg.moveTo(942, 540, 0.6) # address 1
      pg.click()
      pg.write("Test client address1")

      pg.moveTo(942, 577, 0.6) # address 2
      pg.click()
      pg.write("Test client address2")

      pg.moveTo(942, 634, 0.6)
      pg.click()
    except:
      print("[There's somthing wrong]")
    # locate the picture to see whether it exists
    try:
      pg.locateOnScreen(os.path.join(CWD,'screenshot', 'main_win', 'cal_CAL0004.png'))
      # succeeds, screenshot the application
      try:
        pg.screenshot(
          os.path.join(CWD,'US01', 'test add_new_client success .png'),
          region=(520, 127, (1405-520), (870-127))
        )
      except:
        print("[Fail to locate the folder]")
    except:
      print("[Fail to find the picture or add new client function fails]")
      pg.screenshot(
        os.path.join(CWD,'US01', 'test add_new_client success .png'),
        region=(520, 127, (1405-520), (870-127))
      )
      raise

  def test_b_add_equipment(self):
    ''' add an equipment, the information show up on GUI screen correctly'''
    # choose a client to add an equipment
    try:
      pg.moveTo(632, 487, 0.7)
      pg.click()

      btn_choose_client = pg.locateOnScreen(
        os.path.join(CWD, 'screenshot' ,'main_win' ,'choose_client.png'),
        grayscale=False, confidence=.9)
      pg.moveTo(btn_choose_client)
      pg.click()
    except:
      print("[Fail to find the picture or choose function fails]")
      raise

    # move to the button
    try:
      btn_add_new_equipment = pg.locateOnScreen(
        os.path.join(CWD, 'screenshot' ,'main_win_info' ,'info_add_new_equipment.png'),
        grayscale=False, confidence=.9)
      pg.moveTo(btn_add_new_equipment)
      pg.click()
    except:
      print("[Fail to find the picture or add new equipment function fails]")
      raise
    # finishing adding the equipment
    try:
      pg.moveTo(947, 473, 0.7)
      pg.click()
      pg.write("1234")

      pg.moveTo(999, 513, 0.7)
      pg.click()
      pg.write("test mode")

      pg.moveTo(962, 540, 0.7) # submit
      pg.click()
    except:
      print("[Fail to find the picture or add new equipment function fails]")
      raise

    try:
      pg.locateOnScreen(os.path.join(CWD, 'screenshot', 'main_win_info', 'info_id.png'))
      # print(result) # if successful, would print out the box area
      try:
        pg.screenshot(
          os.path.join(CWD,'US01', 'test add_new_equipemnt success .png'),
          region=(520, 127, (1405-520), (870-127))
        )
      except:
        print("[Fail to locate the folder]")
    except:
      print("[Fail to find the picture or add new equipment function fails]")
      pg.screenshot(
        os.path.join(CWD,'US01', 'test add_new_equipemnt fail .png'),
        region=(520, 127, (1405-520), (870-127))
      )
      raise

  def test_c_add_new_run(self):
    ''' add a pair of files to a run, timestamp marked, information on screen '''

    # check the equipment info correct or not
    try:
      pg.locateOnScreen(
        os.path.join(CWD, 'screenshot', 'main_win', 'cal_CAL0004.png'),
        grayscale=False,
        confidence = .7
      )
    except:
        raise("[CAL Number incorrect, or cannot locate the picture]")
    try:
      pg.locateOnScreen(
        os.path.join(CWD, 'screenshot', 'main_win_info', 'info_client_name.png'),
        grayscale=False,
        confidence = .7
      )
    except:
        raise("[client name incorrect, or cannot locate the picture]")
    try:
      pg.locateOnScreen(
        os.path.join(CWD, 'screenshot', 'main_win_info', 'info_serial.png'),
        grayscale=False,
        confidence = .7
      )
    except:
      raise("[serial number incorrect, or cannot locate the picture]")
    try:
      pg.locateOnScreen(
        os.path.join(CWD, 'screenshot', 'main_win_info', 'info_id.png'),
        grayscale=False,
        confidence = .7
      )
    except:
      raise("[id incorrect, or cannot locate the picture]")

    # if all data correctly shown on the form, choose an equipment to add new run
    try:
      pg.moveTo(604, 606, 0.7)
      pg.click()

      btn_choose_equipment = pg.locateOnScreen(
        os.path.join(CWD, 'screenshot' ,'main_win_info' ,'info_choose_equipment.png'),
        grayscale=False, confidence=.9)

      pg.moveTo(btn_choose_equipment)
      pg.click()
    except:
      print("[Fail to find the picture or choose equipment function fails]")
      raise

    # go on testing the add new run functions
    try:
      btn_add_new_run = pg.locateOnScreen(
        os.path.join(CWD, 'screenshot', 'main_win_equip', 'equip_add_new_run.png'),
        grayscale=False,
        confidence = .9
      )

      pg.moveTo(btn_add_new_run)
      pg.click()
    except:
      raise("cannot locate the button add new run")

    # check the submit is successful
    try:
      pg.moveTo(674, 470, 0.7) # client file
      pg.click()

      pg.moveTo(977, 560, 0.7) # file
      pg.click()

      pg.moveTo(1281, 912, 0.7) # open
      pg.click()

      pg.moveTo(669, 539, 0.7) # lab file
      pg.click()

      pg.moveTo(907, 589, 0.7) # file
      pg.click()

      pg.moveTo(1281, 912, 0.7)
      pg.click()

      pg.moveTo(967, 601, 0.7) #submit
      pg.click()

      pg.locateOnScreen(
        os.path.join(CWD, 'screenshot', 'main_win', 'cal_CAL0004.png'),
        grayscale=False,
        confidence = .7
      )
      try:
        pg.screenshot(
          os.path.join(CWD,'US01', 'test add_new_run success .png'),
          region=(520, 127, (1405-520), (870-127))
        )
      except:
        print("[Fail to locate the folder]")
    except:
      pg.screenshot(
        os.path.join(CWD,'US01', 'test add_new_run fail .png'),
        region=(520, 127, (1405-520), (870-127))
      )
      raise("[CAL Number incorrect, or cannot locate the picture, or submit fails]")

  def test_d_delete_run(self):
    ''' delete a run in the equipment page '''
    try:
      pg.moveTo(685, 568, 0.7) # row data
      pg.click()

      btn_del = pg.locateOnScreen(
        os.path.join(CWD, 'screenshot', 'main_win_equip', 'equip_del_run.png'),
        grayscale=False,
        confidence = .9
      )
      pg.moveTo(btn_del)
      pg.click()

      pg.moveTo(933, 534, 0.7) # confirm delete the run
      pg.click()

      # if cannot find the data id, meaning the data has been deleted successfully
      # directly screenshot the data id for scanning
      try:
        # data
        data_x = 543
        data_y = 555
        pg.screenshot(
          os.path.join(CWD, 'screenshot', 'main_win_equip', 'equip_data.png'),
          region=(
            data_x,
            data_y,
            (817 - data_x),
            (587 - data_y)
          )
        )
        # print(data)
      except:
        print("[Saving file failed]")

      sleep(1) # prepare for picture locating
      try:
        data = pg.locateOnScreen(
          os.path.join(CWD, 'screenshot', 'main_win_equip', 'equip_data.png'),
          grayscale=False,
        )
        if data is None:
          print("[Data successfully deleted]")
          sleep(1) # pop window image get screenshot
          pg.screenshot(
          os.path.join(CWD,'US01', 'test_delete_run success .png'),
          region=(520, 127, (1405-520), (870-127))
          )
        else:
          print("[Data deletion failed]")
          pg.screenshot(
          os.path.join(CWD,'US01', 'test_delete_run fail .png'),
          region=(520, 127, (1405-520), (870-127))
          )
      except:
        print("[Fail to locate the filepath]")

    except:
      pg.screenshot(
      os.path.join(CWD,'US01', 'test_delete_run fail .png'),
      region=(520, 127, (1405-520), (870-127))
      )
      raise("[Data deletion failed]")

  def test_e_del_client(self):
    """Return to homepage to delete a client
    """
    try:
      btn_home = pg.locateOnScreen(
        os.path.join(CWD, 'screenshot', 'main_win_info', 'info_home.png'),
        grayscale=False,
        confidence = .8
      )
      pg.moveTo(btn_home)
      pg.click()
      sleep(1)
    except:
      raise("[Fail to find the picture or home function fails]")

    try:
      pg.moveTo(669, 484, 0.7) # selecte one client
      pg.click()

      pg.moveTo(1235, 387, 0.7) # delete client button, fail to capture the picture
      pg.click()

      pg.moveTo(941, 533, 0.7)
      pg.click()
    except:
      raise("[Fail to find the picture delete function fails]")

    sleep(1) # for the picture locating precision
    try:
      image = pg.locateOnScreen(
        os.path.join(CWD, 'screenshot', 'main_win', 'cal_CAL0004.png'),
        grayscale=False,
        confidence = .9
      )
      # if image cannot be found, it would be None; otherwise screenshot the failure
      if image is None:
        print("[The client has been deleted successfully]")
        try:
          pg.screenshot(
            os.path.join(CWD,'US01', 'test_del_client success .png'),
            region=(520, 127, (1405-520), (870-127))
            )
        except:
          print("[Fail to locate the folder]")
      else:
        print("[Fail to delete the client, or the picture detection fail]")
        pg.screenshot(
          os.path.join(CWD,'US01', 'test_del_client fail .png'),
          region=(520, 127, (1405-520), (870-127))
          )
    except:
      print("[Fail to locate the filepath]")

# test the client update button -> manully, successful
if __name__ == '__main__':
    suite = unittest.TestLoader().loadTestsFromTestCase(TestUserStory1)
    unittest.TextTestRunner(verbosity=2).run(suite)