import requests
from bs4 import BeautifulSoup
import re
import urllib.request
import time
import csv

def get_request_years():
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36'}
    page = requests.get("http://www.stats.gov.cn/tjsj/ndsj/", headers = headers)

    html = BeautifulSoup(page.content, 'html.parser')
    table = html.find('table', 'ztzw_tab')
    print(table)
    links = table.findAll('a')
    years_hrefs = []
    for link in links:
        years_hrefs.append(link['href'])

    print(years_hrefs)
    return years_hrefs

def get_data_for_year(year):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36'}
    left_link = re.sub('indexch', 'left', year)
    base_link = re.sub('indexch.htm', '', year)
    base_year = re.findall('\d+', base_link)[0]
    year_page = requests.get(left_link, headers=headers)
    year_html = BeautifulSoup(year_page.content)
    if len(year_html.findAll('ul', {'id': 'foldinglist'})) != 0:
        folding_lists = year_html.findAll('ul', {'id': 'foldinglist'})
    else:
        folding_lists = year_html.findAll('ul', {'id': re.compile('divOne_*')})

    for folding_list in folding_lists:
        li_lists = folding_list.findAll('li')

        for li_list in li_lists:
            file = li_list.find("a").get('href')
            name = li_list.find("a").text.strip()
            name = re.sub('\W+', '', name)
            print(file)

            if '.jpg' in file:
                retries = 3
                success = False
                while not success and retries >= 0:
                    if retries == 0:
                        raise Exception("cause of the problem, time out")

                    try:
                        urllib.request.urlretrieve(base_link + file,
                                                   'C:\\Users\\jocel\\OneDrive\\Desktop\\test\\' + base_year + name + '.jpg')
                        success = True
                    except Exception as e:
                        wait = retries * 30
                        time.sleep(wait)
                        retries -= 1
                        print(e)
            elif '简要说明' in name:
                pass
            elif '主要统计指标解释' in name:
                pass
            elif '.htm' in file:
                retries = 3
                success = False
                while not success and retries >= 0:
                    if retries == 0:
                        raise Exception("cause of the problem, time out")

                    try:
                        print(file)
                        print(base_link)
                        add = re.sub(r'\b.htm\b', '.xls', base_link + file)
                        print(add)
                        urllib.request.urlretrieve(add,
                                                   'C:\\Users\\jocel\\OneDrive\\Desktop\\test\\' + base_year + name + '.xls')

                        success = True
                    except Exception as e:
                        wait = retries * 30
                        time.sleep(wait)
                        retries -= 1
                        print(e)

            else:
                raise Exception("cause of the problem")

def flow():
    years_hrefs = get_request_years()
    for year_href in years_hrefs:
        print(year_href)
        get_data_for_year(year_href)

if __name__== "__main__":
  flow()


import fiona
import numpy as np
import pandas as pd
import geopandas as gpd
import shapely
from shapely.geometry import Point, Polygon
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
import descartes
from sklearn.linear_model import LogisticRegression
import geoplot as gplt
import geoplot.crs as gcrs
import joblib
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.model_selection import cross_val_score
from imblearn.over_sampling import SMOTE
from sklearn.feature_selection import RFE
import statsmodels.api as sm
from sklearn.ensemble import RandomForestClassifier
from sklearn.impute import SimpleImputer

RES_CONST = 0.25
pd.set_option('display.max_columns', None)

infile1  ="https://pasta.lternet.edu/package/data/eml/edi/267/2/1c716f66bf3572a37a9f67035f9e02ac".strip()
infile1  = infile1.replace("https://","http://")

dt1 =pd.read_csv(infile1
          ,skiprows=1
            ,sep=","
                ,quotechar='"'
           , names=[
                    "lakecode",
                    "lakename",
                    "continent",
                    "country",
                    "state",
                    "IntermittentIceCover",
                    "Latitude_dd",
                    "Longitude_dd",
                    "Elevation_m",
                    "MeanAnnualAirTemp_c",
                    "SurfaceArea_km2",
                    "MeanDepth_m",
                    "MaximumDepth_m",
                    "Volume_mcm",
                    "WatershedArea_km2",
                    "ShorelineLength_km",
                    "ResidenceTime_days",
                    "MeanDischarge_m3_sec",
                    "Slope_degrees",
                    "ShorelineDevelopment",
                    "JFMCloudCover_perc",
                    "JFMPrecipitation_mm",
                    "DistanceToCoast_km",
                    "MaximumDistanceToLand_km"    ]
    )
# Coerce the data into the types specified in the metadata
dt1.lakecode=dt1.lakecode.astype('category')
dt1.lakename=dt1.lakename.astype('category')
dt1.continent=dt1.continent.astype('category')
dt1.country=dt1.country.astype('category')
dt1.state=dt1.state.astype('category')
dt1.IntermittentIceCover=dt1.IntermittentIceCover.astype('category')
dt1.Latitude_dd=pd.to_numeric(dt1.Latitude_dd,errors='coerce')
dt1.Longitude_dd=pd.to_numeric(dt1.Longitude_dd,errors='coerce')
dt1.Elevation_m=pd.to_numeric(dt1.Elevation_m,errors='coerce')
dt1.MeanAnnualAirTemp_c=pd.to_numeric(dt1.MeanAnnualAirTemp_c,errors='coerce')
dt1.SurfaceArea_km2=pd.to_numeric(dt1.SurfaceArea_km2,errors='coerce')
dt1.MeanDepth_m=pd.to_numeric(dt1.MeanDepth_m,errors='coerce')
dt1.MaximumDepth_m=pd.to_numeric(dt1.MaximumDepth_m,errors='coerce')
dt1.Volume_mcm=pd.to_numeric(dt1.Volume_mcm,errors='coerce')
dt1.WatershedArea_km2=pd.to_numeric(dt1.WatershedArea_km2,errors='coerce')
dt1.ShorelineLength_km=pd.to_numeric(dt1.ShorelineLength_km,errors='coerce')
dt1.ResidenceTime_days=pd.to_numeric(dt1.ResidenceTime_days,errors='coerce')
dt1.MeanDischarge_m3_sec=pd.to_numeric(dt1.MeanDischarge_m3_sec,errors='coerce')
dt1.Slope_degrees=pd.to_numeric(dt1.Slope_degrees,errors='coerce')
dt1.ShorelineDevelopment=pd.to_numeric(dt1.ShorelineDevelopment,errors='coerce')
dt1.JFMCloudCover_perc=pd.to_numeric(dt1.JFMCloudCover_perc,errors='coerce')
dt1.JFMPrecipitation_mm=pd.to_numeric(dt1.JFMPrecipitation_mm,errors='coerce')
dt1.DistanceToCoast_km=pd.to_numeric(dt1.DistanceToCoast_km,errors='coerce')
dt1.MaximumDistanceToLand_km=pd.to_numeric(dt1.MaximumDistanceToLand_km,errors='coerce')

dt = dt1.filter(['Latitude_dd', 'Longitude_dd', 'IntermittentIceCover'])
dt['IntermittentIceCover'] = dt['IntermittentIceCover'].map({'Y': 1, 'N': 0})

dt = gpd.GeoDataFrame(dt, geometry = gpd.points_from_xy(dt.Longitude_dd, dt.Latitude_dd))

annualLakes= dt[dt['IntermittentIceCover']==0]
print(annualLakes.shape)
intermittentLakes= dt[dt['IntermittentIceCover']==1]
print(intermittentLakes.shape)

world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))

ax = gplt.polyplot(world, projection=gplt.crs.NorthPolarStereo(), facecolor='whitesmoke', figsize = (15, 15))

gplt.pointplot(annualLakes, color = 'black', ax = ax, s = 10, label = 'Annual winter ice')
gplt.pointplot(intermittentLakes, color = 'tab:orange', ax = ax, s = 10, label = 'Intermittent winter ice')
lgnd = plt.legend(loc="lower left", scatterpoints=1, fontsize=18)
lgnd.legendHandles[0]._sizes = [100]
lgnd.legendHandles[1]._sizes = [100]
plt.savefig('trainingLakeMap.png', bbox_inches='tight')
plt.clf()


import os
from configparser import ConfigParser

import pandas as pd
from ecgpandas.loader import Loader

def statistics(statements, original_statements, lookup):
    print("With multi and modifiers")
    added_and_removed(original_statements, statements)

    print("With multi and no modifiers")
    statement_types = lookup.rename(columns={'TopLvlFolder': 'Group'}).Type.to_frame()
    statements = pd.merge(statements, statement_types, left_on='Statement', right_index=True)
    original_statements = pd.merge(original_statements, statement_types, left_on='Statement', right_index=True)

    statements = statements.loc[statements.Type == 2]
    original_statements = original_statements.loc[original_statements.Type == 2]

    statements.drop('Type', axis=1, inplace=True)
    original_statements.drop('Type', axis=1, inplace=True)

    added_and_removed(original_statements, statements)

    print("No multi and no modifiers")
    statements = statements.groupby(['PatientID', 'Date'], group_keys=False).filter(lambda g: len(g) == 1)
    original_statements = original_statements.groupby(['PatientID', 'Date'], group_keys=False).filter(lambda g: len(g) == 1)

    added_and_removed(original_statements, statements)

    print("Most important, our data set")
    iterative_count_differences(original_statements, statements)

def iterative_count_differences(original, statements):
    original.set_index(['PatientID', 'Date'], inplace=True)
    statements.set_index(['PatientID', 'Date'], inplace=True)

    original.drop('index', axis=1, inplace=True)
    statements.drop('index', axis=1, inplace=True)

    original_dict = original.squeeze().to_dict()
    statement_dict = statements.squeeze().to_dict()

    total = statements.shape[0]
    changed = 0

    for idx in statement_dict.keys():
        if idx in original_dict:
            if statement_dict[idx] != original_dict[idx]:
                changed += 1

    print("Ratio changed records by doctors {}".format(changed/total))

def added_and_removed(original_statements, statements):
    statements.reset_index(inplace=True)
    original_statements.reset_index(inplace=True)
    records_with_added = statements.merge(original_statements.drop_duplicates(), on=['PatientID', 'Date', 'Statement'],
                                          how='left', indicator=True)
    records_with_added = records_with_added.loc[records_with_added._merge == 'left_only']
    records_with_added.set_index(['PatientID', 'Date'], inplace=True)
    number_with_added = records_with_added.index.unique().shape[0]
    ratio_records_with_added_statements = number_with_added / statements.index.unique().shape[0]
    print("Ratio changed records with added statments by doctors {}".format(ratio_records_with_added_statements))
    records_with_removed = statements.merge(original_statements.drop_duplicates(),
                                            on=['PatientID', 'Date', 'Statement'], how='right', indicator=True)
    records_with_removed = records_with_removed.loc[records_with_removed._merge == 'right_only']
    records_with_removed.set_index(['PatientID', 'Date'], inplace=True)
    number_with_removed = records_with_removed.index.unique().shape[0]
    ratio_records_with_removed_statements = number_with_removed / statements.index.unique().shape[0]
    print("Ratio changed records with removed statments by doctors {}".format(ratio_records_with_removed_statements))

def smart_stats(original, statements):
    pass

def main():
    config_parser = ConfigParser(allow_no_value=True)
    config_parser.read("../local.conf")

    path = config_parser.get('Default', 'Path')

    statements_path = os.path.join(os.path.sep, path, "SW10", "Parsed", "with_modifiers", "statement.csv")
    original_statements_path = os.path.join(os.path.sep, path, "SW10", "Parsed", "with_modifiers",
                                            "original_statement.csv")

    print("Loading statements and machine generated statements")
    statements = pd.read_csv(statements_path)
    original = pd.read_csv(original_statements_path)

    statements.set_index(["PatientID", "Date"], inplace=True)
    original.set_index(["PatientID", "Date"], inplace=True)

    lookup = Loader(path).load_statement_lookup()

    statement_types = lookup.rename(columns={'TopLvlFolder': 'Group'}).Type.to_frame()
    statements = pd.merge(statements, statement_types, left_on='Statement', right_index=True)
    original = pd.merge(original, statement_types, left_on='Statement', right_index=True)

    smart_stats(original, statements)

    # statistics(statements, original_statements, lookup)

if __name__ == '__main__':
    main()


import os
import logging
import traceback
from threading import RLock
from flask import Flask, request, send_file
from tempfile import mkstemp
from werkzeug.wsgi import ClosingIterator
from werkzeug.exceptions import HTTPException
from pantomime import FileName, normalize_mimetype, mimetype_extension

from convert.converter import Converter, ConversionFailure
from convert.formats import load_mime_extensions
from .document_types import *

logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger('convert')
lock = RLock()
extensions = load_mime_extensions()
converter = Converter()

class ShutdownMiddleware:
    def __init__(self, application):
        self.application = application

    def post_request(self):
        if app.is_dead:
            os._exit(127)

    def __call__(self, environ, after_response):
        iterator = self.application(environ, after_response)
        try:
            return ClosingIterator(iterator, [self.post_request])
        except Exception:
            traceback.print_exc()
            return iterator

app = Flask("convert")
app.is_dead = False
app.wsgi_app = ShutdownMiddleware(app.wsgi_app)

@app.route("/")
def info():
    if app.is_dead:
        return ("BUSY", 503)
    return ("OK", 200)

@app.route("/convert", methods=['POST'])
def convert():
    acquired = lock.acquire(timeout=1)
    if app.is_dead or not acquired:
        return ("BUSY", 503)
    timeout = int(request.args.get('timeout', 1000))
    upload_file = None
    output_format = request.form.get('format')
    if not output_format in LIBREOFFICE_EXPORT_TYPES:
        return ("%s format is not supported" % (output_format), 400)
    try:
        for upload in request.files.values():
            file_name = FileName(upload.filename)
            mime_type = normalize_mimetype(upload.mimetype)
            if not file_name.has_extension:
                file_name.extension = extensions.get(mime_type)
            if not file_name.has_extension:
                file_name.extension = mimetype_extension(mime_type)
            fd, upload_file = mkstemp(suffix=file_name.safe())
            os.close(fd)
            log.info('Convert to %s: %s [%s]',
                     output_format, upload_file, mime_type)
            upload.save(upload_file)
            converter.convert_file(upload_file, output_format, timeout)
            output_filename = "%s.%s" % (converter.OUT, output_format)
            log.info("Send file %s [Mime-type: %s]" %
                     (output_filename, OUTPUT_MIME_TYPES[output_format]))
            return send_file(output_filename,
                             mimetype=OUTPUT_MIME_TYPES[output_format],
                             attachment_filename=output_filename)
        return ('No file uploaded', 400)
    except HTTPException:
        raise
    except ConversionFailure as ex:
        app.is_dead = True
        return (str(ex), 400)
    except Exception as ex:
        app.is_dead = True
        log.error('Error: %s', ex)
        return ('FAIL', 503)
    finally:
        if upload_file is not None and os.path.exists(upload_file):
            os.unlink(upload_file)
        if os.path.exists(converter.OUT):
            os.unlink(converter.OUT)
        lock.release()


import json

from django.test import RequestFactory

from tally_ho.libs.permissions import groups
from tally_ho.apps.tally.models.center import Center
from tally_ho.apps.tally.models.candidate import Candidate
from tally_ho.libs.models.enums.form_state import FormState
from tally_ho.libs.models.enums.entry_version import EntryVersion
from tally_ho.libs.models.enums.center_type import CenterType
from tally_ho.apps.tally.views.reports import (
    administrative_areas_reports as admin_reports,
)
from tally_ho.libs.tests.test_base import (
    create_electrol_race, create_result_form, create_station,\
    create_reconciliation_form, create_sub_constituency, create_tally,\
    create_region, create_constituency, create_office, create_result,\
    create_candidates, TestBase, create_ballot
)
from tally_ho.libs.tests.fixtures.electrol_race_data import (
    electrol_races
)

class TestAdministrativeAreasReports(TestBase):
    def setUp(self):
        self.factory = RequestFactory()
        self._create_permission_groups()
        self._create_and_login_user()
        self._add_user_to_group(self.user, groups.TALLY_MANAGER)
        self.tally = create_tally()
        self.tally.users.add(self.user)
        self.electrol_race = create_electrol_race(
            self.tally,
            **electrol_races[0]
        )
        ballot = create_ballot(self.tally, electrol_race=self.electrol_race)
        self.region = create_region(tally=self.tally)
        office = create_office(tally=self.tally, region=self.region)
        self.constituency = create_constituency(tally=self.tally)
        self.sc =\
            create_sub_constituency(code=1, field_office='1', ballots=[ballot])
        center, _ = Center.objects.get_or_create(
            code='1',
            mahalla='1',
            name='1',
            office=office,
            region='1',
            village='1',
            active=True,
            tally=self.tally,
            sub_constituency=self.sc,
            center_type=CenterType.GENERAL,
            constituency=self.constituency
            )
        self.station = create_station(
            center=center, registrants=20, tally=self.tally
            )
        self.result_form = create_result_form(
            tally=self.tally,
            form_state=FormState.ARCHIVED,
            office=office,
            center=center,
            station_number=self.station.station_number,
            ballot=ballot)
        self.recon_form = create_reconciliation_form(
            result_form=self.result_form,
            user=self.user,
            number_ballots_inside_box=20,
            number_cancelled_ballots=0,
            number_spoiled_ballots=0,
            number_unstamped_ballots=0,
            number_unused_ballots=0,
            number_valid_votes=20,
            number_invalid_votes=0,
            number_ballots_received=20,
            )
        votes = 20
        create_candidates(
            self.result_form, votes=votes, user=self.user,
            num_results=1, tally=self.tally
            )
        for result in self.result_form.results.all():
            result.entry_version = EntryVersion.FINAL
            result.save()
            # create duplicate final results
            create_result(self.result_form, result.candidate, self.user, votes)

    def test_sub_constituency_turn_out_and_votes_summary_reports(self):
        """
        Test that the sub constituency turn out and votes summary reports are
        rendered as expected.
        """
        # add
        view = admin_reports.SummaryReportDataView.as_view()
        request = self.factory.post('/sub-constituency-summary-report')
        request.user = self.user
        response = view(
            request,
            tally_id=self.tally.pk,
            region_id=self.region.pk,
            constituency_id=self.constituency.pk
            )

        # Sub Constituency votes summary report tests
        code, valid_votes, invalid_votes, cancelled_votes, _, _, _ =\
            json.loads(
                response.content.decode())['data'][0]

        self.assertEquals(
            code, '{}	'.format(self.sc.code))
        self.assertEquals(
            valid_votes,
            '{}	'.format(
                self.recon_form.number_valid_votes))
        self.assertEquals(
            invalid_votes,
            '{}	'.format(
                self.recon_form.number_invalid_votes))
        self.assertEquals(
            cancelled_votes,
            '{}	'.format(
                self.recon_form.number_cancelled_ballots))

        view = admin_reports.ProgressiveReportDataView.as_view()
        request = self.factory.get('/sub-cons-progressive-report-list')
        request.user = self.user
        response = view(
            request,
            tally_id=self.tally.pk,
            region_id=self.region.pk,
            constituency_id=self.constituency.pk)
        candidates_count = Candidate.objects.filter(
            tally__id=self.tally.pk).count()

        # Sub Constituency progressive report tests
        code, num_candidates, num_votes, _, _, _ =\
            json.loads(
                response.content.decode())['data'][0]

        self.assertEquals(
            code, '{}	'.format(self.sc.code))
        self.assertEquals(
            num_votes,
            '{}	'.format(
                self.result_form.num_votes))
        self.assertEquals(
            num_candidates,
            '{}	'.format(
                candidates_count))

    def apply_filter(self, data):
        view = admin_reports.ResultFormResultsListDataView.as_view()
        request = self.factory.post('/form-results', data=data)
        request.user = self.user
        response = view(
            request,
            tally_id=self.tally.pk,
        )
        return response

    def test_result_form_result_list_data_view_filters(self):
        """
        Test ResultFormResultsListDataView filters
        """
        # test race type filter
        data = {
            "data": str(
                {
                        "election_level_names":
                        ["Presidential"],
                        "sub_race_type_names":
                        ["ballot_number_presidential_runoff"]
                    }
                )
            }
        response = self.apply_filter(data)
        self.assertEquals(
            len(json.loads(response.content.decode())['data']), 0)
        data = {
            "data": str(
                {
                        "election_level_names": ["Presidential"],
                        "sub_race_type_names": ["ballot_number_presidential"]
                    }
                )
            }
        response = self.apply_filter(data)
        self.assertEquals(
            len(json.loads(response.content.decode())['data']), 2)

        # test center filter
        data = {'data': '{"select_1_ids": ["-1"]}'}  # non existent id
        response = self.apply_filter(data)
        self.assertEquals(
            len(json.loads(response.content.decode())['data']), 0)
        center_id = self.station.center.id
        data = {'data': '{"select_1_ids": ' + f'["{center_id}"]' + '}'}
        response = self.apply_filter(data)
        self.assertEquals(
            len(json.loads(response.content.decode())['data']), 2)

        # test stations filter
        data = {'data': '{"select_2_ids": ["-1"]}'}  # non existent id
        response = self.apply_filter(data)
        self.assertEquals(
            len(json.loads(response.content.decode())['data']), 0)
        station_id = self.station.id
        data = {'data': '{"select_2_ids": ' + f'["{station_id}"]' + '}'}
        response = self.apply_filter(data)
        self.assertEquals(
            len(json.loads(response.content.decode())['data']), 2)

        # test ballot status filter
        data = {'data': '{"ballot_status": ["not_available_for_release"]}'}
        response = self.apply_filter(data)
        self.assertEquals(
            len(json.loads(response.content.decode())['data']), 2)
        data = {'data': '{"ballot_status": ["available_for_release"]}'}
        response = self.apply_filter(data)
        self.assertEquals(
            len(json.loads(response.content.decode())['data']), 0)

        # test station filter
        data = {'data': '{"station_status": ["active"]}'}
        response = self.apply_filter(data)
        self.assertEquals(
            len(json.loads(response.content.decode())['data']), 2)
        data = {'data': '{"station_status": ["inactive"]}'}
        response = self.apply_filter(data)
        self.assertEquals(
            len(json.loads(response.content.decode())['data']), 0)

        # test candidate status
        data = {'data': '{"candidate_status": ["active"]}'}
        response = self.apply_filter(data)
        self.assertEquals(
            len(json.loads(response.content.decode())['data']), 2)
        data = {'data': '{"candidate_status": ["inactive"]}'}
        response = self.apply_filter(data)
        self.assertEquals(
            len(json.loads(response.content.decode())['data']), 0)

        # test station percentage processed
        data = {'data': '{"percentage_processed": "10"}'}
        response = self.apply_filter(data)
        self.assertEquals(
            len(json.loads(response.content.decode())['data']), 2)


"""
Dylan Copley
MAE 5020
Homework 1
Problem 2

Same polygon gen thing except it makes stars... Was showing off to a friend.
"""

import numpy as np
import matplotlib.pyplot as plt
import imageio
import os

#function that plots the polygon, then returns the decoded rgb and image property info
def polygon_anim(s):
    # defining angles
    angles = [(360 / (2 * s)) * n for n in range(0, 2 * s)]

    # defining points from angles, list comprehension except each value is a list itself. index 0 and 1 are x and y.
    points = [[np.cos(n * (np.pi / 180)), np.sin(n * (np.pi / 180))] for n in angles]

    # simply appending the last point as the initial point. This way when lines are needed to be drawn the last points
    # and the initial point will have its line drawn.
    points.append(points[0])

    #generated twice the number of angles/vertices. Every other vertice has half the radial length, creating star shape
    for n in range(0, len(points) - 1):
        #modulus operator to test if divisible by 2, same as saying "every other", or "every even"
        if n % 2 == 1:
            points[n][0] = points[n][0] / 2
            points[n][1] = points[n][1] / 2

    #defining a subplot to play around with draw modes
    fig, ax = plt.subplots()

    #for each point,
    for n in range(0,len(points)-1):
        #bit tricky to read,
        #plt.plot([x1,x2],[y1,y2],'red line solid dot points')
        #points[n][0] are x value, points[n][1] are y value.
        #have to do a ax subplot plot to send current frame
        ax.plot([points[n][0],points[n+1][0]],[points[n][1],points[n+1][1]],'ro-')

    #pretifying
    ax.set_title("Number of sides: " + str(s))
    ax.grid(True)
    ax.set_xlim([-2,2])
    #took an image of plot window, discovered that the scaling ratio of x to y was about 1.343
    #this makes the x and y scales the same, and a prettier image.
    ax.set_ylim([-2/1.343,2/1.343])
    #showing origin
    ax.plot([0],[0],'bo')

    #maximize and grab current frame. I have no clue how I got this code, I borrowed it from an old program.
    fig.canvas.draw()
    image = np.frombuffer(fig.canvas.tostring_rgb(), dtype='uint8')
    image = image.reshape(fig.canvas.get_width_height()[::-1] + (3,))

    #return frame generated
    return image

#getting root directory, need to save the gif file somewhere
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))

#very scary function
#imageio.mimsave((PATH),[list comprehension with each value being a current frame image],fps=n)
#create a gif file stored in the root directory of the program.
#list comprehension generating 27 polygons from 3 to 14. defined fps as 4.
#image is saved as animated_polygon.gif. file will overwrite if program ran again

imageio.mimsave((ROOT_DIR + "\\animated_star.gif"),[polygon_anim(s) for s in range(4,12)], fps=4)


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import re
import pandas as pd
import numpy as np

# In[2]:

fh=open('isear.txt')
label=[]
sen=[]
for line in fh:
    lis=re.findall('[a-zA-Z]+',line)
    label.append(lis[0])
    sen.append(' '.join(lis[1:]))

# In[3]:

print(label)

# In[4]:

print(sen)

# In[5]:

import csv

# In[6]:

with open ('data2.csv','w') as f:
    writer=csv.writer(f)
    writer.writerows(zip(label,sen))

# In[7]:

df=pd.read_csv('data2.csv')

# In[8]:

df.head()

# In[9]:

df.rename(columns={'ID':'label','CITY COUN SUBJ SEX AGE RELI PRAC FOCC MOCC FIEL EMOT WHEN LONG INTS ERGO TROPHO TEMPER EXPRES MOVE EXP EXP EXP PARAL CON EXPC PLEA PLAN FAIR CAUS COPING MORL SELF RELA VERBAL NEUTRO Field Field Field MYKEY SIT STATE':'sentence'},inplace=True)

# In[ ]:

# In[10]:

df.head()

# In[11]:

df.isnull().sum()

# In[ ]:

# In[12]:

'''
Not using NLTK as Spacy is more faster and accurate in Lemmatization and removing stop words.
corpus=[]
for i in range(7666):
    sentence=re.sub('[^a-zA-Z]', ' ',df['sentence'][i])
    sentence=sentence.lower()
    setence=sentence.split()
    ws=WordLemmatizer()
    sentence=[ws.lemmatize(s) for s in sentence if not s in stopwords.words('english')]
    sentence=' '.join(sentence)
    corpus.append(sentence)'''

# In[13]:

import spacy
nlp=spacy.load('en_core_web_sm')

# In[14]:

print(nlp.Defaults.stop_words)

# In[ ]:

# In[15]:

# Removing stop words
corpus=[]
for i in range(7666):
    sentence=re.sub('[^a-zA-Z]', ' ',df['sentence'][i])
    sentence=sentence.lower()
    sentence=sentence.split()

    sentence=[s for s in sentence if not nlp.vocab[s].is_stop]
    sentence=' '.join(sentence)
    corpus.append(sentence)

# In[16]:

corpus

# In[17]:

#Lemmatization
corpus2=[]
for i in range(7666):
    sent=nlp(corpus[i])

    sent2=[s.lemma_ for s in sent ]
    sentence2=' '.join(sent2)
    corpus2.append(sentence2)

# In[18]:

corpus2

# In[19]:

df.head()

# In[20]:

df['cleaned_sentence']=corpus2

# In[21]:

df.head()

# In[22]:

df.label.value_counts()

# In[23]:

#WordCloud Analysis

# In[24]:

get_ipython().system('pip install wordcloud')
from wordcloud import WordCloud
import matplotlib.cm
import matplotlib.pyplot as plt

# In[25]:

depressive_words = ' '.join(list(df[df['label'] == 'sadness']['cleaned_sentence']))
depressive_wc = WordCloud(width = 512,height = 512, collocations=False, colormap=matplotlib.cm.inferno).generate(depressive_words)
plt.figure(figsize = (8, 6), facecolor = 'k')
plt.imshow(depressive_wc)
plt.axis('off')
plt.tight_layout(pad = 0)
plt.show()

# In[26]:

depressive_words = ' '.join(list(df[df['label'] == 'joy']['cleaned_sentence']))
depressive_wc = WordCloud(width = 512,height = 512, collocations=False, colormap=matplotlib.cm.inferno).generate(depressive_words)
plt.figure(figsize = (8, 6), facecolor = 'k')
plt.imshow(depressive_wc)
plt.axis('off')
plt.tight_layout(pad = 0)
plt.show()

# In[27]:

df['emotion'] = df['label'].apply(lambda c: 'Positive' if c =='sadness' else 'Negative')

# In[28]:

df['emotion'].value_counts()

# In[ ]:

# In[ ]:

# In[ ]:

# In[29]:

df5=pd.read_csv('sentiment_tweets3.csv')

# In[30]:

df5.head()

# In[31]:

df5 = df5.drop(['Unnamed: 0'],axis=1)

# In[32]:

df5.label.value_counts()

# In[33]:

df5

# In[34]:

df5=df5.iloc[6000:]

# In[35]:

df5.info()

# In[36]:

corpus=[]
for i in range(6000,10314):
    sentence=re.sub('[^a-zA-Z]', ' ',df5['message'][i])
    sentence=sentence.lower()
    sentence=sentence.split()

    sentence=[s for s in sentence if not nlp.vocab[s].is_stop]
    sentence=' '.join(sentence)
    corpus.append(sentence)

# In[37]:

corpus2=[]
for i in corpus:
    sent=nlp(i)
    sent2=[s.lemma_ for s in sent ]
    sentence2=' '.join(sent2)
    corpus2.append(sentence2)

# In[38]:

len(corpus2)

# In[39]:

df5['cleaned_sentence']=corpus2

# In[40]:

df5=df5[['label','message','cleaned_sentence']]

# In[41]:

df5.head()

# In[42]:

df.head()

# In[43]:

df5.rename(columns={'message':'sentence'},inplace=True)

# In[44]:

df3=df5[df5['label']>=0]

# In[45]:

df3.info()

# In[46]:

df.info()

# In[47]:

df4=df.append(df3)

# In[48]:

df4.head()

# In[49]:

df4['emotion'] = df4['label'].apply(lambda c: 'Positive' if c !=0 and c!='joy' else 'Negative')

# In[50]:

df4.info()

# In[51]:

df4['emotion'].value_counts()

# In[52]:

df4.info()

# In[53]:

df4.to_csv('cleaned_data.csv')

# In[54]:

from sklearn.model_selection import train_test_split

X = df4['cleaned_sentence']
y = df4['emotion']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,random_state=42)

# In[55]:

from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.svm import LinearSVC

text_clf = Pipeline([('tfidf', TfidfVectorizer()),
                     ('clf', LinearSVC()),
])

# Feed the training data through the pipeline
text_clf.fit(X_train, y_train)

# In[56]:

def process(str):
    corpus=[]

    sentence=re.sub('[^a-zA-Z]', ' ',str)
    sentence=sentence.lower()
    sentence=sentence.split()

    sentence=[s for s in sentence if not nlp.vocab[s].is_stop]
    sentence=' '.join(sentence)

    sent=nlp(sentence)
    sent2=[s.lemma_ for s in sent ]
    sentence2=' '.join(sent2)
    return(sentence2)

# In[57]:

string=str(input("Enter Message :"))
string2=process(string)
z=pd.Series(string2)
predictions = text_clf.predict(z)
predictions

# In[58]:

predictions2=text_clf.predict(X_test)
from sklearn import metrics
print(metrics.confusion_matrix(y_test,predictions2))

# In[59]:

print(metrics.classification_report(y_test,predictions2))

#

# In[60]:

print(metrics.accuracy_score(y_test,predictions2))

# In[62]:

depressive_words = ' '.join(list(df4[df4['emotion'] == 'Negative']['cleaned_sentence']))
depressive_wc = WordCloud(width = 512,height = 512, collocations=False, colormap="Set1").generate(depressive_words)
plt.figure(figsize = (10,8), facecolor = 'k')
plt.imshow(depressive_wc)
plt.axis('off')
plt.tight_layout(pad = 0)
plt.show()


import asyncio
import aiohttp

async def download(visits, site):
    count = 0
    for _ in range(visits):
        print(f"downloading {site}")
        async with aiohttp.ClientSession() as session:
            async with session.get(site) as response:
                html = await response.text()
                count += len(html)
    message = f"{site} returned {count//1000}K characters"
    return message

async def main():
    # note each download yields immediately so that other downloads
    # can run in parallel.  main waits until all results are available
    response = await asyncio.gather(
        download(15, "http://ibm.com"),
        download(20, "http://bbc.co.uk"),
        download(25, "http://abc.com")
    )
    print(response)

asyncio.run(main())

src/17 Threading and Concurrency/AsyncIO/07.futures.py


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Libraries
import os
import argparse
import sys
import time
import json
import re
import numpy as np
import pandas as pd
import statistics as stat
import lstm_binary
import lstm_multiclass
from pprint import pprint
from datetime import datetime
from shutil import copy2
from sklearn.model_selection import train_test_split
import pickle

# constants
models_path = '../data/local/models/'
data_path = '../data/local/staging/'

REFPATH = "./"
PROJECT_ROOT = "/Users/nscsekhar/Desktop/nscsekhar/Desktop/Surya/Personal/MIDS/W210/Project/team_cyber/"
MULTI_TOKENIZER_FILE = PROJECT_ROOT + "saved_models/multiclass_tokenizer.pkl"
MULTI_CATEGORIES_FILE = PROJECT_ROOT + "saved_models/multiclass_categories.pkl"
MULTI_MODEL_JSON = PROJECT_ROOT + "saved_models/multiclass_LSTM.json"
MULTI_MODEL_H5 = PROJECT_ROOT + "saved_models/multiclass_LSTM.h5"

def valid_filename(filename, path=''):
    '''valid_filename: determines if the given filename is a real file. Assumes that the file is in the current working directory for the program.

    returns: given file name
    '''
    if path == '':
        path = os.getcwd()+'/'

    if not os.path.isfile(path+filename):
        msg = "The given file '{}' does not exist at '{}'.".format(
            filename,
            path
            )
        raise argparse.ArgumentTypeError(msg)

    return filename

def parse_args():
    '''parse_args: parse command line arguments

    return: dictionary of arguments
    '''
    parser = argparse.ArgumentParser(
        description='Runs models either in train or inference mode.',
        prog='models',
        epilog='Models requires at least one of -t/--train or -i/--inference to operate correctly. Both may be provided for sequential analysis.')
    parser.add_argument('config_file',
                        type=valid_filename,
                        metavar='CONFIG_FILE',
                        help="File path to the requex configuration file. File must be in JSON format.")
    parser.add_argument('-t', '--train',
                        metavar='TRAINING_FILE',
                        nargs=1,
                        help="Runs models in training mode. Training will be run on the given file. The training file must be a prepared '.csv' file. The program will search for the file in the models, then staging, and finally downloads directories.")
    parser.add_argument('-i', '--inference',
                        metavar='INFERENCE_FILE',
                        nargs=1,
                        help="Runs models in inference mode. Inference will be run on the given file. The inference file must be a list of domains separated by carriage returns with no header. The program will search for the file in the models, then staging, and finally downloads directories.")
    parser.add_argument('-m', '--model', choices=['binary', 'multiclass'],
                        metavar='model_type',
                        required=True,
                        help="This required option indicates which type of model is being built or used. Using 'binary' selects a benign/malicious model. Using 'multiclass' will classify the malware family for each malicious classified entry.")

    return vars(parser.parse_args())

def get_config_filename(filename=None):
    '''get_config_filename: returns a verified Requex configuration file name. This function handles the ambiguity around whether the module was called from a shell with command line arguments or if called from another program using the run() function. If filename is none, the function assumes that there are

    return: string; valid filename.
    '''
    if filename is None:
        # get command line arguments
        args = parse_args()
        filename = args['config_file']
    else:
        # filename provided, verify the file exists
        if not os.path.isfile(filename):
            print("The given file '{}' does not exist at '{}'.".format(
                filename,
                os.getcwd()
                ))
            exit(1)
    return filename

def get_config(filename):
    '''get_config: reads the configuration JSON file and stores values in a dictionary for processing.

    PRE: assumes the file already exists

    return: dict of configuration settings
    '''

    with open(filename, "r") as f:
        config = json.load(f)

    return config

def get_file_date(filename):
    '''get_file_date: extracts file date from file name. File date must be in YYYY-MM-DD format.

    returns: datetime object of file date.
    '''
    date = re.search(r'\d\d\d\d-\d\d-\d\d|$', filename).group()
    year, month, day = date.split('-')
    return datetime(int(year), int(month), int(day))

def write_to_train_logfile(metrics, logpath, stdout=True):
    '''write_to_train_logfile: writes metadata in the metrics dict to a logfile
    '''
    # constants
    logfile = 'requex_training_log.csv'

    # write to logfile
    stamp = datetime.utcnow().strftime('%Y-%m-%d-%H:%M')

    # extract the filename
    # filename = os.path.basename(datafile)

    if stdout:
        # print("info:{:>10} rows: {:>10} malicious, {:>10} benign, a {:>3.3f} ratio".format(total_rows, malicious_rows, benign_rows, ratio))
        print('info: {}, {}, {}, {:>3.3f}s, {:>3.2f} MB, {} rows: {} malicious, {} benign, {:>3.3f} ratio, {}, {} categories, train rows: {}, test rows: {}, train time: {:>3.3f}s, inference time: {:>3.3f}s'.format(
             stamp, metrics['filename'], metrics['filedate'], metrics['time'], metrics['memory'], metrics['total_rows'], metrics['malicious_rows'], metrics['benign_rows'], metrics['ratio'], metrics['model'], metrics['categories'], metrics['train_rows'], metrics['test_rows'], metrics['train_time'], metrics['inference_time']))

    with open(logpath+logfile, 'at') as log:
        log.write('{}, {}, {}, {:>3.3f}, {:>3.2f}, {}, {}, {}, {:>3.3f}, {}, {}, {}, {}, {:>3.3f}, {:>3.3f}\n'.format(
             stamp, metrics['filename'], metrics['filedate'], metrics['time'], metrics['memory'], metrics['total_rows'], metrics['malicious_rows'], metrics['benign_rows'], metrics['ratio'], metrics['model'], metrics['categories'], metrics['train_rows'], metrics['test_rows'], metrics['train_time'], metrics['inference_time']))

def copy_models(src, dst):
    '''copy_models: copies the source file (src) to the dst directory. src must be a file and dst must be a directory. Exclusions is an optional parameter that allows for files with certain file names to be excluded from being moved.
    '''
    # check to see if a directory for the dst directory exists
    if not os.path.isdir(dst):
        # directory does not exist, create it
        os.mkdir(dst)

    # verify whether the source and destination are the same
    src_path, filename = os.path.split(src)
    if os.path.isfile(dst+filename):
        print("A file by the name '{}' already exists. File not copied. Processing will continue using the file already in the '{}' directory.".format(filename, dst))
    elif os.path.isfile(src):
        copy2(src, dst)
    else:
        print("The given file '{}' does not exist.".format(src))
        exit(1)

def get_training_data(filename, metrics, logpath):
    '''get_training_data: reads the csv file into a pandas dataframe

    return: pandas dataframe
    '''
    # constants
    MB = 1024*1024

    start_time = time.time()
    df = pd.read_csv(filename,
                     sep=',',
                     parse_dates=[0],
                     dtype={1: int, 2: str, 3: str},
                     engine='c')
    end_time = time.time()
    read_time = end_time - start_time

    # calculate the memory footprint of the dataframe
    memory = sys.getsizeof(df)/MB

    filedate = get_file_date(filename)
    total = df.shape[0]
    benign = df.loc[df['dga'] == 0].shape[0]
    malicious = df.loc[df['dga'] == 1].shape[0]
    ratio = malicious / benign

    # write to logfile
    # write_to_train_logfile(logpath, filename, filedate.strftime('%Y-%m-%d'), read_time, memory, total, malicious, '2',benign, ratio)
    metrics = {
        'filename': filename,
        'filedate': filedate.strftime('%Y-%m-%d'),
        'time': read_time,
        'memory': memory,
        'total_rows': total,
        'malicious_rows': malicious,
        'benign_rows': benign,
        'ratio': ratio,
        'categories': 0,
        'model': 'unknown',
        'train_rows': 0,
        'test_rows': 0,
        'train_time': 0,
        'inference_rows': 0,
        'inference_time': 0,
        'inference_time_mean': 0.0
    }

    return df, metrics

def prep_training_dataset_binary(df):
    '''prep_training_dataset_binary: creates X, Y datasets for training and testing.

    returns: pandas dataframe x4: X_train, X_test, Y_train, Y_test
    '''
    # create X, Y dataframes. X = 'domain' and the model will try to
    # predict Y the catengory index.
    X = df['domain']
    Y = df['dga']

    X_train, X_test, Y_train, Y_test = train_test_split(
        X, Y, test_size=0.2, random_state=23)

    return X_train, X_test, Y_train, Y_test

def prep_training_dataset_multiclass(df, categories_file):
    '''prep_training_dataset_multiclass: creates X, Y datasets for training and testing.

    returns: pandas dataframe x4: X_train, X_test, Y_train, Y_test and the number of uniques
    '''

    # factorize the malware column
    df['catIndex'], uniques = pd.factorize(df['malware'], sort=True)

    # display factorized values
    # print('malware uniques: total - {}\n{}'.format(len(uniques), uniques))
    # print('catIndex uniques: {}'.format(
    #       pd.unique(df['catIndex'].sort_values())))

    # record the categories to disk
    with open(categories_file, 'wb') as f:
        pickle.dump(uniques, f, protocol=pickle.HIGHEST_PROTOCOL)

    # create X, Y dataframes. X = 'domain' and the model will try to
    # predict Y the catengory index.
    X = df['domain']
    Y = df['catIndex']

    X_train, X_test, Y_train, Y_test = train_test_split(
        X, Y, test_size=0.2, random_state=23)

    return X_train, X_test, Y_train, Y_test, len(uniques)

def get_model_info(model_type, config):
    '''get_model_info: returns a dictionary with key value pairs of model file keys and model file names. The model file names are full path names anchored to the root_dir and placed in the models directory.

    type: a string indicating the type of model ['binary', ['multiclass']
    config: a dict filled with configuration parameters

    return: dict of model:filename pairs
    '''

    if model_type == 'binary':
        model = config['binary_model']
    elif model_type == 'multiclass':
        model = config['multiclass_model']
    else:
        # this branch shouldn't happen with the way parse_args() written
        msg = "error: unsupported model type '{}'.".format(model_type)
        raise argparse.ArgumentTypeError(msg)
        exit(1)

    root_dir = config['root_dir']
    models_dir = config['models_dir']

    model = {
        'model_json': root_dir+models_dir+model['model_json'],
        'model_H5': root_dir+models_dir+model['model_H5'],
        'model_tokenizer': root_dir+models_dir+model['model_tokenizer'],
        'model_categories': root_dir+models_dir+model['model_categories'],
        'model_algorithm': model['model_algorithm']
    }

    return model

def find_file(filename, config):
    '''find_file: looks for the  file in a few directories and moves it into the models_dir. Returns the full path to the training file.

    return: string of full file path in the models_dir or an empty string
    '''
    root_dir = config['root_dir']
    downloads_dir = config['downloads_dir']
    staging_dir = config['staging_dir']
    models_dir = config['models_dir']

    # look for file in models_dir
    # look for file in staging_dir
    # look for file in downloads_dir
    if os.path.isfile(root_dir+models_dir+filename):
        return root_dir+models_dir
    elif os.path.isfile(root_dir+staging_dir+filename):
        return root_dir+staging_dir
    elif os.path.isfile(root_dir+downloads_dir+filename):
        return root_dir+downloads_dir
    else:
        return ''
        # msg = "The given file '{}' does not exist at any of these locations '{}', '{}', '{}'.".format(
        #     filename,
        #     models_dir,
        #     staging_dir,
        #     downloads_dir
        #     )
        # print(msg)
        # exit(1)

def get_model_type(model_type):
    '''get_model_type: evaluates model_type to see if it is a valid option. If model_type is empty, function will attempt to pull the parameters from the command line. This function should mirror the choices in parse_args() for -m/--models.

    return: a string with the model_type; empty string if not correct.
    '''
    if model_type is '':
        args = parse_args()
        return args['model']
    elif model_type.lower() == 'binary':
        return 'binary'
    elif model_type.lower() == 'multiclass':
        return 'multiclass'
    else:
        return ''

def get_train_file(filename, config):
    '''get_train_file: evaluates the filename as well as command line arguments to get the training file name. Verifies that the training file exists.

    returns: string of a filename or empty string if not valid.
    '''
    root_dir = config['root_dir']
    models_dir = config['models_dir']

    if filename == '':
        # no filename provided, attempt to get it from the command line
        # parameters
        args = parse_args()
        train_file = args['train']
        if train_file is not None:
            # extract the filename from the parameter list
            train_file = train_file[0]
            location = find_file(train_file, config)
            if location == '':
                # file was not found
                return ''
            else:
                copy_models(location+train_file, root_dir+models_dir)
                return root_dir+models_dir+train_file
        else:
            # the command line parameter for train_file was also None
            return ''
    else:
        # filename was provided
        location = find_file(train_file, config)
        if location == '':
            # file was not found
            return ''
        else:
            copy_models(location+train_file, root_dir+models_dir)
            return root_dir+models_dir+train_file

def get_inference_file(filename, config):
    '''get_inference_file: evaluates the filename as well as command line arguments to get the inference file name. Verifies that the inference file exists.

    returns: string of a filename or empty string if not valid.
    '''
    root_dir = config['root_dir']
    models_dir = config['models_dir']

    if filename == '':
        # no filename provided, attempt to get it from the command line
        # parameters
        args = parse_args()
        inference_file = args['inference']
        if inference_file is not None:
            # extract the filename from the parameter list
            inference_file = inference_file[0]
            location = find_file(inference_file, config)
            if location == '':
                # file was not found
                return ''
            else:
                copy_models(location+inference_file, root_dir+models_dir)
                return root_dir+models_dir+inference_file
        else:
            # the command line parameter for inference_file was also None
            return ''
    else:
        # filename was provided
        location = find_file(inference_file, config)
        if location == '':
            # file was not found
            return ''
        else:
            copy_models(location+inference_file, root_dir+models_dir)
            return root_dir+models_dir+inference_file

def load_inference_data(filename):
    '''load_inference_data: reads data from the given filename. The file must be a text file with '\n' at the end of each entry, one entry per line.

    returns: list of data to be analyzed
    '''
    domains = []
    with open(filename, 'rt', newline='\n') as f:
        lines = f.readlines()

    for line in lines:
        domains.append(line.strip())

    return domains

def write_predictions(domains, predictions, model_type, model_algo, version, config):
    '''write_predictions: takes a 1-D list of domains and predictions and writes them to the inference file output. File name will be 'predictions_YYYY-MM-DD.txt'.
    '''

    # create filename
    root_dir = config['root_dir']
    models_dir = config['models_dir']

    # get the current date and time:
    datestamp = time.strftime('%Y-%m-%d', time.gmtime())
    timestamp = time.strftime('%H:%M.%S', time.gmtime())

    output_file = root_dir+models_dir+model_type+model_algo+'_predictions_'+datestamp+'_v'+version+'.csv'

    # write the predictions to disk
    with open(output_file, 'wt') as f:
        f.write('creation_date: {} {}\n'.format(datestamp, timestamp))
        for i, p in enumerate(predictions):
            # print('i: {}, p: {}, domains: {}'.format(i, p, domains[i]))
            f.write('{}, {}\n'.format(domains[i], p))

def get_version_number(filename):
    '''get_version_number: extracts the version number from the filename.

    returns: string with a version number in it.
    '''
    basename = os.path.basename(filename)
    reg = re.compile(r'(?:_v\d+)|$', flags=re.IGNORECASE)
    return re.search(reg, basename).group()[2:]

def run(config_file=None, model_type='', train_file='', inference_file=''):
    # get configuration parameters
    config_file = get_config_filename(config_file)
    config = get_config(config_file)
    # print('configuration settings:')
    # pprint(config)

    # parse function/command line parameters
    model_type = get_model_type(model_type)
    train_file = get_train_file(train_file, config)
    inference_file = get_inference_file(inference_file, config)

    # assemble the path to the log directory
    root_dir = config['root_dir']
    models_dir = config['models_dir']
    logpath = root_dir+models_dir

    if model_type == '':
        print("error: an invalid model type was given. See the -h/--help command line options for valid model choices.")
        exit(1)

    if train_file == '' and inference_file == '':
        print("error: neither train or inference were given as arguments. Please run again, but with either -t/--train or -i/--inference options (or both) enabled.")
        exit(1)

    # get the model information from the configuration file
    model_info = get_model_info(model_type, config)
    metrics = {
        'filename': '',
        'filedate': '',
        'time': 0.0,
        'memory': 0.0,
        'total_rows': 0,
        'malicious_rows': 0,
        'benign_rows': 0,
        'ratio': 0.0,
        'categories': 0,
        'model': 'unknown',
        'train_rows': 0,
        'test_rows': 0,
        'train_time': 0,
        'inference_rows': 0,
        'inference_time': 0,
        'inference_time_mean': 0.0
    }

    if train_file != '':
        # a training file was provided
        model_version = get_version_number(train_file)

        # get training data from disk
        df, metrics = get_training_data(train_file, metrics, logpath)

        if model_type == 'binary':
            X_train, X_test, Y_train, Y_test = prep_training_dataset_binary(df)
            metrics['model'] = model_type
            metrics['categories'] = 2
            metrics['train_rows'] = X_train.shape[0]
            metrics['test_rows'] = X_test.shape[0]
            # pprint(metrics)
            print('info: {} – training started.'.format(time.strftime('%Y-%m-%d %H:%M.%S', time.gmtime())))
            train_model = lstm_binary.LSTMBinary()
            start_time = time.time()
            train_model.train(X_train, Y_train)
            end_time = time.time()
            train_time = end_time - start_time
            metrics['train_time'] = train_time
            print('info: {} – training ended. Train time {:>3.3f}s.'.format(time.strftime('%Y-%m-%d %H:%M.%S', time.gmtime()), train_time))
            write_to_train_logfile(metrics, logpath, True)

            train_model.save(model_info['model_tokenizer'],
                             model_info['model_json'],
                             model_info['model_H5'])

        elif model_type == 'multiclass':
            # create X and Y and split into train and test
            X_train, X_test, Y_train, Y_test, categories = prep_training_dataset_multiclass(
                df, model_info['model_categories'])
            metrics['model'] = model_type
            metrics['categories'] = categories
            metrics['train_rows'] = X_train.shape[0]
            metrics['test_rows'] = X_test.shape[0]

            print('info: {} – training started.'.format(time.strftime('%Y-%m-%d %H:%M.%S', time.gmtime())))
            start_time = time.time()
            train_model = lstm_multiclass.LSTMMulti()
            train_model.train(X_train, Y_train)
            end_time = time.time()
            train_time = end_time - start_time
            metrics['train_time'] = train_time
            print('info: {} – training ended. Train time {:>3.3f}s.'.format(time.strftime('%Y-%m-%d %H:%M.%S', time.gmtime()), train_time))
            write_to_train_logfile(metrics, logpath, True)

            train_model.save(model_info['model_tokenizer'],
                             model_info['model_categories'],
                             model_info['model_json'],
                             model_info['model_H5'])
        else:
            print("error: unrecognized model type.")
            exit(1)
        # train the model (which model is set by models input)
        # train_model.train(X_train, Y_train)
        # train_model.save(TOKENIZER_FILE, MODEL_JSON, MODEL_H5)
        # save the model to disk

    if inference_file != '':
        # an inference file was provided
        model_version = get_version_number(inference_file)
        print('inference file: {}'.format(inference_file))
        if model_type == 'binary':
            metrics['filename'] = inference_file
            metrics['filedate'] = time.strftime('%Y-%m-%d', time.gmtime())

            predict_model = lstm_binary.LSTMBinary()
            predict_model.load(model_info['model_tokenizer'],
                               model_info['model_json'],
                               model_info['model_H5'])
            domains = load_inference_data(inference_file)
            # print("Number of domains: ", len(domains))
            # print("Top 10:\n", domains[:10])

            # run predictions, record timings
            timestamp = time.strftime('%Y-%m-%d %H:%M.%S', time.gmtime())
            print('info: {} – inference started.'.format(timestamp))
            start_time = time.time()
            predictions = predict_model.predict(domains)
            end_time = time.time()
            prediction_time = end_time - start_time
            domain_count = len(domains)
            metrics['inference_rows'] = domain_count
            metrics['inference_time'] = prediction_time
            metrics['inference_time_mean'] = prediction_time / domain_count
            timestamp = time.strftime('%Y-%m-%d %H:%M.%S', time.gmtime())
            print('info: {} – inference ended. Inference time {:>3.3f}s.'.format(timestamp, prediction_time))

            # reshape the predictions
            predictions = np.reshape(predictions, [predictions.shape[0], ]).tolist()
            # print(predictions[:10])
            # print("domains: {}, predictions: {}".format(len(domains), len(predictions)))

            # write the predictions to file
            write_predictions(domains, predictions, model_type, model_info['model_algorithm'], model_version, config)

            # write_to_train_logfile(metrics, logpath, True)
        elif model_type == 'multiclass':
            metrics['filename'] = inference_file
            metrics['filedate'] = time.strftime('%Y-%m-%d', time.gmtime())

            predict_model = lstm_multiclass.LSTMMulti()
            predict_model.load(model_info['model_tokenizer'],
                               model_info['model_categories'],
                               model_info['model_json'],
                               model_info['model_H5'])
            domains = load_inference_data(inference_file)
            # print("Number of domains: ", len(domains))
            # print("Top 10:\n", domains[:10])

            # run predictions, record timings
            timestamp = time.strftime('%Y-%m-%d %H:%M.%S', time.gmtime())
            print('info: {} – inference started.'.format(timestamp))
            start_time = time.time()
            predictions, pred_prob = predict_model.predict(domains)
            end_time = time.time()
            prediction_time = end_time - start_time
            domain_count = len(domains)
            metrics['inference_rows'] = domain_count
            metrics['inference_time'] = prediction_time
            metrics['inference_time_mean'] = prediction_time / domain_count
            timestamp = time.strftime('%Y-%m-%d %H:%M.%S', time.gmtime())
            print('info: {} – inference ended. Inference time {:>3.3f}s.'.format(timestamp, prediction_time))

            # reshape the predictions
            # predictions = np.reshape(predictions, [predictions.shape[0], ]).tolist()
            # print(predictions[:10])
            # print("domains: {}, predictions: {}".format(len(domains), len(predictions)))

            # write the predictions to file
            write_predictions(domains, predictions, model_type, model_info['model_algorithm'], model_version, config)
        else:
            print("error: unrecognized model type.")
            exit(1)
        # get test data
        # load the model (based on models input)
        # testmodel = lstm_binary.LSTMBinary()
        # testmodel.load(BINARY_TOKENIZER_FILE, BINARY_MODEL_JSON, BINARY_MODEL_H5)
        # make predictions
        # urllist = ["www.google.com", "www.netflix.com", "plvklpgwivery.com"]
        # urltypes = testmodel.predict(urllist)
        # print("URL type:", urltypes)

if __name__ == '__main__':
    run()


import torch
import torch.autograd as autograd
import torch.nn as nn
import torch.optim as optim
import numpy as np
torch.manual_seed(1)
from sklearn.metrics import roc_auc_score
from sklearn.metrics import f1_score
import copy

##########################################################

label_to_ix=np.load('label_to_ix.npy').item()
ix_to_label=np.load('ix_to_label.npy')
training_data=np.load('training_data.npy')
test_data=np.load('test_data.npy')
val_data=np.load('val_data.npy')
word_to_ix=np.load('word_to_ix.npy').item()
ix_to_word=np.load('ix_to_word.npy')
newwikivec=np.load('newwikivec.npy')
wikivoc=np.load('wikivoc.npy').item()

wikisize=newwikivec.shape[0]
rvocsize=newwikivec.shape[1]
wikivec=autograd.Variable(torch.FloatTensor(newwikivec))

batchsize=32

def preprocessing(data):

    new_data=[]
    for i, note, j in data:
        templabel=[0.0]*len(label_to_ix)
        for jj in j:
            if jj in wikivoc:
                templabel[label_to_ix[jj]]=1.0
        templabel=np.array(templabel,dtype=float)
        new_data.append((i, note, templabel))
    new_data=np.array(new_data)

    lenlist=[]
    for i in new_data:
        lenlist.append(len(i[0]))
    sortlen=sorted(range(len(lenlist)), key=lambda k: lenlist[k])
    new_data=new_data[sortlen]

    batch_data=[]

    for start_ix in range(0, len(new_data)-batchsize+1, batchsize):
        thisblock=new_data[start_ix:start_ix+batchsize]
        mybsize= len(thisblock)
        numword=np.max([len(ii[0]) for ii in thisblock])
        main_matrix = np.zeros((mybsize, numword), dtype= np.int)
        for i in range(main_matrix.shape[0]):
            for j in range(main_matrix.shape[1]):
                try:
                    if thisblock[i][0][j] in word_to_ix:
                        main_matrix[i,j] = word_to_ix[thisblock[i][0][j]]

                except IndexError:
                    pass       # because initialze with 0, so you pad with 0

        xxx2=[]
        yyy=[]
        for ii in thisblock:
            xxx2.append(ii[1])
            yyy.append(ii[2])

        xxx2=np.array(xxx2)
        yyy=np.array(yyy)
        batch_data.append((autograd.Variable(torch.from_numpy(main_matrix)),autograd.Variable(torch.FloatTensor(xxx2)),autograd.Variable(torch.FloatTensor(yyy))))
    return batch_data
batchtraining_data=preprocessing(training_data)
batchtest_data=preprocessing(test_data)
batchval_data=preprocessing(val_data)

######################################################################
# Create the model:

Embeddingsize=100
hidden_dim=200
class CNN(nn.Module):

    def __init__(self, batch_size, vocab_size, tagset_size):
        super(CNN, self).__init__()
        self.hidden_dim = hidden_dim
        self.word_embeddings = nn.Embedding(vocab_size+1, Embeddingsize, padding_idx=0)
        self.embed_drop = nn.Dropout(p=0.2)

        self.hidden2tag = nn.Linear(300, tagset_size)

        self.convs1 = nn.Conv1d(Embeddingsize,100,3)
        self.convs2 = nn.Conv1d(Embeddingsize,100,4)
        self.convs3 = nn.Conv1d(Embeddingsize,100,5)

        self.layer2 = nn.Linear(Embeddingsize, 1,bias=False)
        self.embedding=nn.Linear(rvocsize,Embeddingsize)
        self.vattention=nn.Linear(Embeddingsize,Embeddingsize)

        self.sigmoid = nn.Sigmoid()
        self.tanh = nn.Tanh()
        self.dropout = nn.Dropout(0.2)

    def forward(self, vec1, nvec, wiki, simlearning):

        thisembeddings=self.word_embeddings(vec1)
        thisembeddings = self.embed_drop(thisembeddings)
        thisembeddings=thisembeddings.transpose(1,2)

        output1=self.tanh(self.convs1(thisembeddings))
        output1=nn.MaxPool1d(output1.size()[2])(output1)

        output2=self.tanh(self.convs2(thisembeddings))
        output2=nn.MaxPool1d(output2.size()[2])(output2)

        output3=self.tanh(self.convs3(thisembeddings))
        output3=nn.MaxPool1d(output3.size()[2])(output3)

        output4 = torch.cat([output1,output2,output3], 1).squeeze(2)

        if simlearning==1:
            nvec=nvec.view(batchsize,1,-1)
            nvec=nvec.expand(batchsize,wiki.size()[0],-1)
            wiki=wiki.view(1,wiki.size()[0],-1)
            wiki=wiki.expand(nvec.size()[0],wiki.size()[1],-1)
            new=wiki*nvec
            new=self.embedding(new)
            vattention=self.sigmoid(self.vattention(new))
            new=new*vattention
            vec3=self.layer2(new)
            vec3=vec3.view(batchsize,-1)

        vec2 = self.hidden2tag(output4)
        if simlearning==1:
            tag_scores = self.sigmoid(vec2.detach()+vec3)
        else:
            tag_scores = self.sigmoid(vec2)

        return tag_scores

######################################################################
# Train the model:

topk=10

def trainmodel(model, sim):
    print ('start_training')
    modelsaved=[]
    modelperform=[]
    topk=10

    bestresults=-1
    bestiter=-1
    for epoch in range(5000):
        model.train()

        lossestrain = []
        recall=[]
        for mysentence in batchtraining_data:
            model.zero_grad()

            targets = mysentence[2].cuda()
            tag_scores = model(mysentence[0].cuda(),mysentence[1].cuda(),wikivec.cuda(),sim)
            loss = loss_function(tag_scores, targets)
            loss.backward()
            optimizer.step()
            lossestrain.append(loss.data.mean())
        print (epoch)
        modelsaved.append(copy.deepcopy(model.state_dict()))
        print ("XXXXXXXXXXXXXXXXXXXXXXXXXXXX")
        model.eval()

        recall=[]
        for inputs in batchval_data:

            targets = inputs[2].cuda()
            tag_scores = model(inputs[0].cuda(),inputs[1].cuda() ,wikivec.cuda(),sim)

            loss = loss_function(tag_scores, targets)

            targets=targets.data.cpu().numpy()
            tag_scores= tag_scores.data.cpu().numpy()

            for iii in range(0,len(tag_scores)):
                temp={}
                for iiii in range(0,len(tag_scores[iii])):
                    temp[iiii]=tag_scores[iii][iiii]
                temp1=[(k, temp[k]) for k in sorted(temp, key=temp.get, reverse=True)]
                thistop=int(np.sum(targets[iii]))
                hit=0.0
                for ii in temp1[0:max(thistop,topk)]:
                    if targets[iii][ii[0]]==1.0:
                        hit=hit+1
                if thistop!=0:
                    recall.append(hit/thistop)

        print ('validation top-',topk, np.mean(recall))

        modelperform.append(np.mean(recall))
        if modelperform[-1]>bestresults:
            bestresults=modelperform[-1]
            bestiter=len(modelperform)-1

        if (len(modelperform)-bestiter)>5:
            print (modelperform,bestiter)
            return modelsaved[bestiter]

model = CNN(batchsize, len(word_to_ix), len(label_to_ix))
model.cuda()

loss_function = nn.BCELoss()
optimizer = optim.Adam(model.parameters())

basemodel= trainmodel(model, 0)
torch.save(basemodel, 'CNN_model')

model = CNN(batchsize, len(word_to_ix), len(label_to_ix))
model.cuda()
model.load_state_dict(basemodel)
loss_function = nn.BCELoss()
optimizer = optim.Adam(model.parameters())
KSImodel= trainmodel(model, 1)
torch.save(KSImodel, 'KSI_CNN_model')

def testmodel(modelstate, sim):
    model = CNN(batchsize, len(word_to_ix), len(label_to_ix))
    model.cuda()
    model.load_state_dict(modelstate)
    loss_function = nn.BCELoss()
    model.eval()
    recall=[]
    lossestest = []

    y_true=[]
    y_scores=[]

    for inputs in batchtest_data:

        targets = inputs[2].cuda()

        tag_scores = model(inputs[0].cuda(),inputs[1].cuda() ,wikivec.cuda(),sim)

        loss = loss_function(tag_scores, targets)

        targets=targets.data.cpu().numpy()
        tag_scores= tag_scores.data.cpu().numpy()

        lossestest.append(loss.data.mean())
        y_true.append(targets)
        y_scores.append(tag_scores)

        for iii in range(0,len(tag_scores)):
            temp={}
            for iiii in range(0,len(tag_scores[iii])):
                temp[iiii]=tag_scores[iii][iiii]
            temp1=[(k, temp[k]) for k in sorted(temp, key=temp.get, reverse=True)]
            thistop=int(np.sum(targets[iii]))
            hit=0.0

            for ii in temp1[0:max(thistop,topk)]:
                if targets[iii][ii[0]]==1.0:
                    hit=hit+1
            if thistop!=0:
                recall.append(hit/thistop)
    y_true=np.concatenate(y_true,axis=0)
    y_scores=np.concatenate(y_scores,axis=0)
    y_true=y_true.T
    y_scores=y_scores.T
    temptrue=[]
    tempscores=[]
    for  col in range(0,len(y_true)):
        if np.sum(y_true[col])!=0:
            temptrue.append(y_true[col])
            tempscores.append(y_scores[col])
    temptrue=np.array(temptrue)
    tempscores=np.array(tempscores)
    y_true=temptrue.T
    y_scores=tempscores.T
    y_pred=(y_scores>0.5).astype(np.int)
    print ('test loss', np.mean(lossestest))
    print ('top-',topk, np.mean(recall))
    print ('macro AUC', roc_auc_score(y_true, y_scores,average='macro'))
    print ('micro AUC', roc_auc_score(y_true, y_scores,average='micro'))
    print ('macro F1', f1_score(y_true, y_pred, average='macro')  )
    print ('micro F1', f1_score(y_true, y_pred, average='micro')  )

print ('CNN alone:           ')
testmodel(basemodel, 0)
print ('XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX')
print ('KSI+CNN:           ')
testmodel(KSImodel, 1)


from codeModule import *
import colorama, time, os, sys
from colorama import Fore, Back, Style

class Stack:
    def __init__(self):
        self.stack = []

    def __str__(self):
        t = self.stack[::-1]
        out = "___"
        for i in t:
            out = "{}\n {} ".format(out, i)
        out = out + "\n" + "___"
        return out

    def pop(self):
        if self.stack:
            return self.stack.pop()
        else:
            return 0

    def push(self, val):
        self.stack.append(val)

def cont(v, a, b):
    if v < a:
        return b
    if v > b:
        return a
    return v

class Pointer:
    def __init__(self, diagram):
        self.UP = 0
        self.RIGHT = 1
        self.DOWN = 2
        self.LEFT = 3

        self.diagram = diagram
        self.maxX = self.diagram.calcMaxX()
        self.maxY = len(self.diagram.code)
        self.x = 0
        self.y = 0
        self.dir = self.RIGHT

    def forward(self):
        if self.dir == self.UP:
            self.y = cont(self.y - 1, 0, self.maxY-1)
        elif self.dir == self.DOWN:
            self.y = cont(self.y + 1, 0, self.maxY-1)
        elif self.dir == self.RIGHT:
            self.x = cont(self.x + 1, 0, self.maxX-1)
        elif self.dir == self.LEFT:
            self.x = cont(self.x - 1, 0, self.maxX-1)

    def turnRight(self):
        self.dir = cont(self.dir + 1, self.UP, self.LEFT)

    def turnLeft(self):
        self.dir = cont(self.dir - 1, self.UP, self.LEFT)

class Diagram:
    def __init__(self, code="", verbose=False):
        if code:
            self.code = code
        else:
            self.getCode()
        self.stack = Stack()
        self.pointer = Pointer(self)
        self.v = verbose
        if self.v:
            colorama.init()

    def getCode(self):
        if len(sys.argv) == 2:
            fName = sys.argv[1]
        else:
            fName = input("File Name: ")
        print()
        with open(fName, "r") as file:
            data = file.read().split("\n")
        del data[-1]
        #print(data)
        for i in range(len(data)):
            data[i] = list(data[i])
        #print(data)
        self.code = data
        mx = self.calcMaxX()
        for i in self.code:
            for t in range(mx - len(i)):
                i.append(' ')

    def calcMaxX(self):
        lst = list(map(lambda x: len(x), self.code))
        return max(lst)

    def run(self):
        while True:
            if self.v:
                if os.name == "nt":
                    os.system('cls')
                elif os.name == "posix":
                    os.system('clear')
                for y in range(len(self.code)):
                    for x in range(len(self.code[y])):
                        #print("{}".format(Style.RESET_ALL), end="")
                        style = ""
                        if x == self.pointer.x and y == self.pointer.y:
                            style = Back.RED
                        print("{}{}".format(style, self.code[y][x]), end="")
                        print("{}".format(Style.RESET_ALL), end="")
                    print()
                time.sleep(0.1)
                #print()
            exec(codes[self.code[self.pointer.y][self.pointer.x]])
            self.pointer.forward()

if __name__ == "__main__":
    d = Diagram(verbose=False)
    d.run()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Mar  3 13:59:12 2020

@author: astah
"""

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats import norm, weibull_min, chi2, lognorm, kstest
from scipy.optimize import curve_fit
from read_write import determine_file_name_e1, write_contour, read_contour
from contour_statistics import points_outside
from plot import PlottedSample, plot_contour
from read_write import read_dataset
plt.close('all')

plt.close('all')
#%% Functions to fit
# Power function
def power3(x, a, b, c):
    return a + b * x ** c

# Exponential function
def exp3(x, a, b, c):
    return a + b * np.exp(c * x)

#%% Read dataset A, B  or C.
DATASET_CHAR = 'A'
file_path = '../datasets/' + DATASET_CHAR + '.txt'
sample_hs, sample_tz, label_hs, label_tz= read_dataset(file_path)

df = pd.read_csv(file_path, sep='; ')
#%% Inspect the marginal distributions

weib_par1 = weibull_min.fit(df[df.columns[1]], loc=0)
logn_par1 = lognorm.fit(df[df.columns[1]], loc=0)

weib_par2 = weibull_min.fit(df[df.columns[2]], loc=0)
logn_par2 = lognorm.fit(df[df.columns[2]], loc=0)

#%% Goodness of fit

print(kstest(df[df.columns[1]].values, 'weibull_min', args=weib_par1))
print(kstest(df[df.columns[1]].values, 'lognorm', args=logn_par1))

print(kstest(df[df.columns[2]].values, 'weibull_min', args=weib_par2))
print(kstest(df[df.columns[2]].values, 'lognorm', args=logn_par2))

#%% Plot the distributions
#n_bins = 100

#plt.figure()
#plt.subplot(211)
#n1, bins1, _ = plt.hist(df[df.columns[1]], n_bins, density=True, label = df.columns[1])
#plt.plot(bins1, weibull_min.pdf(bins1,*weib_par1), label='Weibull')
#plt.plot(bins1, lognorm.pdf(bins1,*logn_par1), label='Lognorm')
#plt.legend(loc='best')
#plt.subplot(212)
#n, bins, _ = plt.hist(df[df.columns[2]], n_bins, density=True, label = df.columns[2])
#plt.plot(bins, weibull_min.pdf(bins,*weib_par2), label='Weibull')
#plt.plot(bins, lognorm.pdf(bins,*logn_par2), label='Lognorm')
#plt.legend(loc='best')

#%% Bin the data to find the conditoinal marginal distribution
s_min = df[df.columns[1]].min()
s_max = df[df.columns[1]].max()

bin_size = 0.5
s_bins = np.arange(np.floor(s_min), np.ceil(s_max), bin_size) + bin_size/2
s_binedges = s_bins + bin_size/2

s_ind_bin = np.digitize(df[df.columns[1]], bins=s_binedges)

unique, counts = np.unique(s_ind_bin, return_counts=True)

ind_min_bin = unique[counts>10][0]
ind_max_bin = unique[counts>10][-1]
x_bins = s_bins[ind_min_bin:ind_max_bin+1]
real_bins = np.zeros(len(x_bins))

logn_par_cond = np.zeros((len(x_bins),3))
mu_cond = np.zeros(len(x_bins))
sig_cond = np.zeros(len(x_bins))

plot_bins = np.arange(0,14,0.2)

for i in range(len(x_bins)):
    mask1 = s_ind_bin == i + ind_min_bin
    real_bins[i] = df[df.columns[1]][mask1].mean()
    logn_par_cond[i,:] = lognorm.fit(df[df.columns[2]][mask1], floc=0)
    mu_cond[i] = np.mean(np.log(df[df.columns[2]][mask1]))
    sig_cond[i] = np.std(np.log(df[df.columns[2]][mask1]))
#    plt.figure()
#    b = plt.hist(df[df.columns[2]][mask1], bins= plot_bins, density=True)
#    plt.plot(b[1], lognorm.pdf(b[1],*logn_par_cond[i,:]), color='g')

#bounds = ([0, 0, -np.inf], [np.inf, np.inf, np.inf])
bounds = ([-1, 0, -np.inf], [np.inf, np.inf, np.inf])
p0_mu = [0, 2, 0.1]
p0_sig = [0.1, 0.1, -0.3]

mu_vars = curve_fit(power3, real_bins, mu_cond, p0=p0_mu, bounds=bounds)[0]
sig_vars = curve_fit(exp3, real_bins, sig_cond, p0=p0_sig, bounds=bounds)[0]

sig_func = curve_fit(exp3, real_bins, logn_par_cond[:,0], p0=p0_sig, bounds=bounds)[0]
mu_func = curve_fit(power3, real_bins, np.log(logn_par_cond[:,2]), p0=p0_mu, bounds=bounds)[0]

plt.figure()
plt.subplot(211)
plt.plot(real_bins, np.log(logn_par_cond[:,2]), 'o')
plt.plot(real_bins, mu_cond, 'o')
plt.plot(x_bins, power3(x_bins, *mu_func))
plt.ylabel(r'$\mu$: scale parameter')
plt.subplot(212)
plt.plot(real_bins, logn_par_cond[:,0], 'o')
plt.plot(real_bins, sig_cond, 'o')
plt.plot(x_bins, exp3(x_bins, *sig_func))
plt.plot(x_bins, exp3(x_bins, *sig_vars))
plt.ylabel(r'$\sigma$: shape parameter')

#%% Perform the IDS

T1 = 1
T20 = 20

#beta1 = norm.ppf(1- 10/(T1*len(df)))
#beta20 = norm.ppf(1- 10/(T20*len(df)))
beta1 = np.sqrt(chi2.ppf(1- 10/(T1*len(df)), df=2))
beta20 = np.sqrt(chi2.ppf(1- 10/(T20*len(df)), df=2))

phi = np.linspace(0, 2 * np.pi, 360, endpoint=False)

u0_1 = beta1*np.cos(phi)
u1_1 = beta1*np.sin(phi)

u0_20 = beta20*np.cos(phi)
u1_20 = beta20*np.sin(phi)

x1_1 = lognorm.ppf( norm.cdf(u1_1), *logn_par1)
x1_20 = lognorm.ppf( norm.cdf(u1_20), *logn_par1)

# The weibull conditional distribution
sig_x1_1 =  exp3(x1_1, *sig_func)
mu_x1_1 =  power3(x1_1, *mu_func)

sig_x1_20 =  exp3(x1_20, *sig_func)
mu_x1_20 =  power3(x1_20, *mu_func)

x0_1 = lognorm.ppf( norm.cdf(u0_1), sig_x1_1, loc=0, scale=np.exp(mu_x1_1))
x0_20 = lognorm.ppf( norm.cdf(u0_20), sig_x1_20, loc=0, scale=np.exp(mu_x1_20))
#%%
h = sns.jointplot(x= df.columns[2] , y=df.columns[1] , data=df, s=5)
h.x, h.y = x0_1, x1_1
h.plot_joint(plt.plot, color='C1')
h.x, h.y = x0_20, x1_20
h.plot_joint(plt.plot, color='C2')

#%% E1 requirements:
# Save the contours as csv files in the required format.
folder_name = 'contour_coordinates/'
file_name_1 = determine_file_name_e1('Asta', 'Hannesdottir', DATASET_CHAR, T1)
write_contour(x1_1, #y-axis
              x0_1,
              folder_name + file_name_1,
              label_x=df.columns[1],
              label_y=df.columns[2])
file_name_20 = determine_file_name_e1('Asta', 'Hannesdottir', DATASET_CHAR, T20)
write_contour(x1_20,
              x0_20,
              folder_name + file_name_20,
              label_x=df.columns[1],
              label_y=df.columns[2])

# Read the contours from the csv files.
(contour_hs_1, contour_tz_1) = read_contour(folder_name + file_name_1)
(contour_hs_20, contour_tz_20) = read_contour(folder_name + file_name_20)

# Find datapoints that exceed the 20-yr contour.
hs_outside, tz_outside, hs_inside, tz_inside = \
    points_outside(contour_hs_20,
                   contour_tz_20,
                   np.asarray(df[df.columns[1]].values),
                   np.asarray(df[df.columns[2]].values))
print('Number of points outside the contour: ' +  str(len(hs_outside)))
#%%
nan_mask = np.isnan(contour_tz_20)

fig = plt.figure(figsize=(5, 5), dpi=150)
ax = fig.add_subplot(111)

plotted_sample = PlottedSample(x=np.asarray(sample_tz),
                               y=np.asarray(sample_hs),
                               ax=ax,
                               x_inside=tz_inside,
                               y_inside=hs_inside,
                               x_outside=tz_outside,
                               y_outside=hs_outside,
                               return_period=T20)
# Plot the 1-year contour.
plot_contour(x=contour_tz_1,
             y=contour_hs_1,
             ax=ax,
             contour_label=str(T1) + '-yr contour',
             x_label=label_tz,
             y_label=label_hs,
             line_style='b--',
             plotted_sample=plotted_sample)

# Plot the 20-year contour and the sample.
plot_contour(x=contour_tz_20[~nan_mask],
             y=contour_hs_20[~nan_mask],
             ax=ax,
             contour_label=str(T20) + '-yr contour',
             x_label=label_tz,
             y_label=label_hs,
             line_style='b-')#,
#             plotted_sample=plotted_sample)
plt.title('Dataset ' + DATASET_CHAR)
plt.show()
plt.savefig('../results/figures/hannesdottir_asta_dataset_'+DATASET_CHAR+'_1_20.png', dpi=300)


from datetime import datetime, timedelta
from typing import Any, MutableMapping, cast

import pytest

from snuba import state
from snuba.clickhouse.columns import ColumnSet
from snuba.datasets.entities import EntityKey
from snuba.query.conditions import (
    BooleanFunctions,
    ConditionFunctions,
    binary_condition,
)
from snuba.query.data_source.simple import Entity as QueryEntity
from snuba.query.expressions import Column, Expression, Literal
from snuba.query.logical import Query
from snuba.query.timeseries_extension import TimeSeriesExtension
from snuba.request.request_settings import HTTPRequestSettings
from snuba.schemas import validate_jsonschema

def build_time_condition(
    time_columns: str, from_date: datetime, to_date: datetime
) -> Expression:
    return binary_condition(
        BooleanFunctions.AND,
        binary_condition(
            ConditionFunctions.GTE,
            Column(f"_snuba_{time_columns}", None, time_columns),
            Literal(None, from_date),
        ),
        binary_condition(
            ConditionFunctions.LT,
            Column(f"_snuba_{time_columns}", None, time_columns),
            Literal(None, to_date),
        ),
    )

test_data = [
    (
        {
            "from_date": "2019-09-19T10:00:00",
            "to_date": "2019-09-19T12:00:00",
            "granularity": 3600,
        },
        build_time_condition(
            "timestamp", datetime(2019, 9, 19, 10), datetime(2019, 9, 19, 12)
        ),
        3600,
    ),
    (
        {
            "from_date": "1970-01-01T10:00:00",
            "to_date": "2019-09-19T12:00:00",
            "granularity": 3600,
        },
        build_time_condition(
            "timestamp", datetime(2019, 9, 18, 12), datetime(2019, 9, 19, 12)
        ),
        3600,
    ),
    (
        {
            "from_date": "2019-09-19T10:05:30,1234",
            "to_date": "2019-09-19T12:00:34,4567",
        },
        build_time_condition(
            "timestamp",
            datetime(2019, 9, 19, 10, 5, 30),
            datetime(2019, 9, 19, 12, 0, 34),
        ),
        60,
    ),
]

@pytest.mark.parametrize(
    "raw_data, expected_ast_condition, expected_granularity", test_data,
)
def test_query_extension_processing(
    raw_data: MutableMapping[str, Any],
    expected_ast_condition: Expression,
    expected_granularity: int,
) -> None:
    state.set_config("max_days", 1)
    extension = TimeSeriesExtension(
        default_granularity=60,
        default_window=timedelta(days=5),
        timestamp_column="timestamp",
    )

    valid_data = validate_jsonschema(
        raw_data, cast(MutableMapping[str, Any], extension.get_schema())
    )
    query = Query(QueryEntity(EntityKey.EVENTS, ColumnSet([])))

    request_settings = HTTPRequestSettings()

    extension.get_processor().process_query(query, valid_data, request_settings)
    assert query.get_condition() == expected_ast_condition
    assert query.get_granularity() == expected_granularity


#Modulo servidor para el Proyecto 1 de Redes 1 Enero Marzo 2018
#Integrantes: Salvador Gonzalez   - 10-10296
#			  Valentina Hernandez - 10-10352
from __future__ import print_function
from time import sleep
import datetime as d
import os,signal
import sys
import socket as s

#Funcion para listar las descargas completadas de los libros:
def listCompleted():
	print()
	try:
		file = open("./Downladed_Books.txt","r")
		for line in file:
			print(line,end="")
		print()
	except:
		print("No se han completado descargas todavia...")
#Funcion encargada de obtener la lista de todos los clientes que han consultado al servidor
def getClients():
	print()
	try:
		file = open("./sessions.txt","r")
		for line in file:
			print(line,end="")
		file.close()
	except:
		print("El servidor todavia no ha sido consultado...")
#Funcion encargada de Mandarle la lista de los pdfs presentes en el servidor al cliente
#@Param socket: cliente al cual se le enviara la informacion
def getBookList(socket):
	booklist = os.listdir("./Pdfs")
	for book in booklist:
		socket.send(book)
		socket.send(" ")
	socket.send("#")
#Funcion para listar el numero de descargas x cliente y por libro
def listBookClient():
	ocurrencias_c = {}#Diccionario para las ocurrencias de los clientes
	ocurrencias_b = {}#Diccionario para las ocurrencias de los libros
	print()
	try:
		file = open("./Client_Books.txt","r")
		for line in file:
			client,libro,nulo = line.split("-")
			if client in ocurrencias_c:
				ocurrencias_c[client] += 1
			else:
				ocurrencias_c[client] = 1
			if libro in ocurrencias_b:
				ocurrencias_b[libro] += 1
			else:
				ocurrencias_b[libro] = 1
		print("Numero de descargas por libro")
		for key in ocurrencias_b:
			print("Libro: %s - Descargas: %s"%(key,ocurrencias_b[key]))
		print()
		print("Numero de descargas por cliente")
		for key in ocurrencias_c:
			print("Cliente: %s - Descargas: %s"%(key,ocurrencias_c[key]))
	except:
		print("No se han realizado descargas en este servidor....")
#Funcion para listar las descargas en curso
def listDownloads():
	downList = os.listdir("./Server_info")
	if downList:
		for download in downList:
			file = open("./Server_info/%s"%(download),"r")
			print()
			print(file.readline(),end="")
			file.close()
		print()
	else:
		print("\nNo existen descargas en curso actualmente....")
#Funcion para el envio del alchivo al cliente
#@Param socket: cliente al cual se le enviara el archivo
#@Param info: informacion del cliente al cual se le enviara el pdf tupla (IP,PUERTO)
#@Param pid: PID del proceso que envia el archivo para llevar el estatus de descarga
def sendBook(socket,info,pid):
	book = socket.recv(200)#Variable para contener la informacion del libro (paquete de tamano fijo)
	book = filter(lambda x: x!=" ",book)#obtenemos el nombre
	filesize = str(os.path.getsize("./Pdfs/%s"%(book)))#Tamagno en bytes del archivo a enviar
	while(len(filesize) < 10):
		filesize+=" "
	socket.send(filesize)
	filesize = filter(lambda x: x != " ",filesize)
	#Archivo que deseamos enviar
	file = open("./Pdfs/%s"%(book),"r")
	read = 0 #Variable para los bytes leidos
	data = " "#Variable para guardar la data
	while(data != ""):
		#Archivo para guardar el status de la descarga
		status = open("./Server_info/%s.txt"%(pid),"w")
		data = file.read(1000)
		read += len(data)
		socket.send(data)
		status.write("CLiente: %s - Libro: %s - %s de %s bytes..."%(info[0],book,read,filesize))
		status.close()
		sleep(3)
	file.close()
	os.remove("./Server_info/%s.txt"%(pid))
	#Si se completa la descarga/ anadimos el libro a la lista de descargas
	date = d.datetime.now()
	file = open("./Downladed_Books.txt","a")
	file.write("Libro: %s - Fecha: %s-%s-%s - Hora: %s:%s:%s\n"%(book,date.day,date.month,date.year,date.hour,date.minute,date.second))
	file.close()
	#Registro para el num de descargas x libro x cliente
	file = open("./Client_Books.txt","a")
	file.write("%s-%s-\n"%(info[0],book))
	file.close()

def main():
	if (len(sys.argv) != 2):
		print("La invocacion para el servidor debe ser de la forma: ",end="")
		print("python server.py <numero de puerto puerto>")
		return
	port = sys.argv[1]#Numero de puerto para el socket
	server_info = ("",int(port))#Informacion de nuestro servidor
	#la tupla contiene informacion referente a (Direccion Ip,Puerto de escucha)
	#Definimos el socket de para nuestro servidor
	#AF_INET y SOCK_STREAM son parametros para utilizar Protocolo TCP
	#AF_INET es para direcciones IPv4
	print("Generando socket...")
	sleep(1)
	try:
		server_socket = s.socket(s.AF_INET,s.SOCK_STREAM)
		print("socket creado...")
		sleep(1)
	except:
		print("Error al generar el Socket para el servidor...")
		return
	print("Asociando puerto %s al socket..." % port)
	sleep(1)
	try:
		server_socket.bind(server_info)
		print("Asociancion establecida...")
		sleep(1)
	except:
		print("Error al asociar el puerto al socket...")
		return
	#Creamos el directorio para la informacion del server:
	try:
		os.mkdir("./Server_info")
	except:
		files = os.listdir("./Server_info")
		for file in files:
			os.remove("./Server_info/%s"%(file))
	#Proceso hijo para la escuha de las peticiones
	connection_handler = os.fork()
	#Codigo para el proceso hijo
	if connection_handler == 0:
		#Guardamos el PID del hijo para cerrarlo al finalizar el programa
		p_id = str(os.getpid())
		file = open("./listen_process.txt","w")
		file.write(p_id)
		file.close()
		while True:
			#Listen se encarga de escuchar las peticiones, el valor 100 esel tama;o de la cola
			#de peticiones que puede recibir el servidor
			server_socket.listen(100)
			#Esperamos aceptar las peticiones de los clientes
			#client_info es una tupla de la forma (IP,PUERTO)
			client_socket,client_info = server_socket.accept()
			#Anadimos la sesion
			file = open("./sessions.txt","a")
			date = d.datetime.now()
			file.write("Cliente: %s Fecha: %s-%s-%s Hora: %s:%s:%s\n" %(client_info[0],date.day,date.month,date.year,date.hour,date.minute,date.second))
			file.close()
			#Se vuelve a crear un fork para atender las peticiones una ve escuchadas
			request_processor = os.fork()
			#Codigo para el proceso hijo encargado de atender/ejecutar las peticiones
			if (request_processor == 0):
				p_id = str(os.getpid())
				option = client_socket.recv(1)
				if option == "2":
					getBookList(client_socket)
				elif option == "3":
					sendBook(client_socket,client_info,p_id)
				else:
					pass
				quit()
	#codigo para el proceso padre - Menu principal
	else:
		option = 0
		while(option < 5):
			print("\nBienvenido al sistema de gestion del servidor... Que opcion desea ejecutar")
			print("1: LIBROS_DESCARGADOS")
			print("2: CLIENTES_QUE_CONSULTARON")
			print("3: NUM_DESCARGASxLIBROxCLIENTE")
			print("4: DESCARGAS_EN_CURSO")
			print("5: Salir...")
			option = raw_input("Opcion: ")
			if option.isdigit():
				option = int(option)
				#Opcion para el listado de descargas completas
				if option ==  1:
					listCompleted()
				#Opcion para los clientes que me han consultado
				elif option == 2:
					getClients()
				elif option == 3:
					listBookClient()
				#Opcion para listar las descargas en curso
				elif option == 4:
					listDownloads()
				#Opcion para finalizar la corrida
				elif option == 5:
					#Cerramos el socket del server
					server_socket.close()
					#Matamos el proceso listener para dejar libre el puerto
					file = open("./listen_process.txt","r")
					p_id = int(file.read())
					os.kill(p_id,signal.SIGKILL)
					os.remove("./listen_process.txt")
					#Borramos el contenido de la carpera Server_info
					info = os.listdir("./Server_info")
					for file in info:
						os.remove("./Server_info/%s"%(file))
					os.rmdir("./Server_info")
					return
				else:
					print("Ha ingresado una opcion invalida...")
					option = 0
			else:
				print("Ha ingresado una opcion invalida...")
				option = 0

if __name__ == '__main__':
	main()


import torch
import torch.backends.cudnn as cudnn

from models.FSRCNN.model import Net
from trainer import Trainer

class FSRCNNTrainer(Trainer):
    def __init__(self, config, training_loader, valid_loader):
        super(FSRCNNTrainer, self).__init__(config, training_loader, valid_loader, "fsrcnn")

    def build_model(self):
        self.model = Net(num_channels=3, upscale_factor=self.upscale_factor).to(self.device)
        self.model.weight_init(mean=0.0, std=0.2)
        self.criterion = torch.nn.MSELoss()
        torch.manual_seed(self.seed)

        if self.GPU_IN_USE:
            torch.cuda.manual_seed(self.seed)
            cudnn.benchmark = True
            self.criterion.cuda()

        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.lr)
        self.scheduler = torch.optim.lr_scheduler.MultiStepLR(self.optimizer, milestones=[50, 75, 100], gamma=0.5)  # lr decay


import json
import random
from argparse import ArgumentParser
from collections import defaultdict
from os import makedirs, listdir
from os.path import exists, join, isfile, basename

def ensure_dir_exists(dir_path):
    if not exists(dir_path):
        makedirs(dir_path)

def get_valid_sources(all_sources):
    return [s for s in all_sources if exists(s)]

def print_data_sources_stat(data_sources):
    print('Specified {} valid data sources:'.format(len(data_sources)))
    for data_source in data_sources:
        print('   - {}'.format(data_source))

def parse_records(data_sources):
    num_records = defaultdict(int)
    out_records = dict()
    for data_source in data_sources:
        data_type = basename(data_source).split('.')[0]

        with open(data_source) as input_stream:
            for line_id, line in enumerate(input_stream):
                if line_id == 0:
                    continue

                line_elements = line.strip().split(',')
                if len(line_elements) != 4:
                    continue

                label, video_name, start, end = line_elements

                segment_id = num_records[video_name]
                segment_name = f'{video_name}_segment{segment_id}'

                num_records[video_name] += 1
                out_records[segment_name] = dict(
                    label=int(label),
                    data_type=data_type
                )

    return out_records

def validate_videos(records, videos_dir, extension):
    downloaded_videos = set(
        f.replace(f'.{extension}', '')
        for f in listdir(videos_dir)
        if isfile(join(videos_dir, f)) and f.endswith(extension)
    )
    all_videos = set(video_name for video_name in records.keys())

    valid_videos = downloaded_videos & all_videos
    out_records = {video_name: records[video_name] for video_name in valid_videos}

    return out_records

def split_train_val_subsets(records, test_ratio=0.1):
    assert 0.0 < test_ratio < 1.0

    by_labels = defaultdict(list)
    for video_name, content in records.items():
        by_labels[content['label']].append(video_name)

    clustered_segments = dict()
    for label, segments in by_labels.items():
        videos = defaultdict(list)
        for segment in segments:
            video, _ = segment.split('_segment')
            videos[video].append(segment)

        clustered_segments[label] = videos

    out_records = dict()
    for label, videos in clustered_segments.items():
        num_records = len(by_labels[label])
        assert num_records > 1

        video_names = list(videos.keys())
        num_videos = len(video_names)
        assert num_videos > 1

        num_test_samples = min(num_records - 1, max(1, int(num_records * test_ratio)))
        num_test_videos = min(num_videos - 1, max(1, int(num_videos * test_ratio)))

        num_selected_test_samples = 0
        test_videos = []
        for test_video_name in random.sample(video_names, num_test_videos):
            test_videos.append(test_video_name)
            segments = videos[test_video_name]

            for segment in segments:
                out_records[segment] = dict(label=label, data_type='val')

            num_selected_test_samples += len(segments)
            if num_selected_test_samples >= num_test_samples:
                break

        train_videos = list(set(video_names) - set(test_videos))
        for train_video_name in train_videos:
            segments = videos[train_video_name]

            for segment in segments:
                out_records[segment] = dict(label=label, data_type='train')

    return out_records

def build_classmap(records):
    labels = set(record['label'] for record in records.values())
    return {class_name: i for i, class_name in enumerate(sorted(labels))}

def convert_annot(records, classmap, extension):
    out_records = dict()
    for video_name, content in records.items():
        label_id = classmap[content['label']]
        out_records[f'{video_name}.{extension}'] = label_id, content['data_type']

    return out_records

def group_by_type(annotation):
    out_data = defaultdict(list)
    for video_name, (label_id, data_type) in annotation.items():
        out_data[data_type].append((video_name, label_id))

    return out_data

def write_classmap(classmap, out_path):
    with open(out_path, 'w') as output_stream:
        json.dump(classmap, output_stream)

def write_annot(records, out_path):
    with open(out_path, 'w') as output_stream:
        for video_name, label_id in records:
            output_stream.write(f'{video_name} {label_id}\n')

def main():
    parser = ArgumentParser()
    parser.add_argument('--sources', '-s', nargs='+', type=str, required=True)
    parser.add_argument('--videos_dir', '-v', type=str, required=True)
    parser.add_argument('--output_dir', '-o', type=str, required=True)
    parser.add_argument('--extension', '-e', type=str, required=False, default='avi')
    parser.add_argument('--test_ratio', '-r', type=float, required=False, default=0.1)
    args = parser.parse_args()

    ensure_dir_exists(args.output_dir)

    data_sources = get_valid_sources(args.sources)
    print_data_sources_stat(data_sources)
    assert len(data_sources) > 0

    records = parse_records(data_sources)
    print(f'Found {len(records)} records.')

    classmap = build_classmap(records)
    print(f'Found {len(classmap)} unique classes.')

    out_classmap_path = join(args.output_dir, 'classmap.json')
    write_classmap(classmap, out_classmap_path)
    print(f'Dumped classmap to: {out_classmap_path}')

    records = validate_videos(records, args.videos_dir, args.extension)
    print(f'Validated {len(records)} videos.')

    records = split_train_val_subsets(records, args.test_ratio)

    annot = convert_annot(records, classmap, args.extension)
    split_annot = group_by_type(annot)

    for data_type, records in split_annot.items():
        out_annot_path = join(args.output_dir, f'{data_type}.txt')
        write_annot(records, out_annot_path)
        print(f'Dumped annot to: {out_annot_path}')

if __name__ == '__main__':
    main()