# -*- coding: utf-8 -*-
"""
@author: Afnan
Note: some of this is carried over from lab 1
"""
from bs4 import BeautifulSoup
from typing import Tuple
import csv
import re
# BEGIN PART FROM https://towardsdatascience.com/implementing-a-trie-data-structure-in-python-in-less-than-100-lines-of-code-a877ea23c1a1
class TrieNode(object):
    """
    Trie node implementation.
    """

    def __init__(self, char: str):
        self.char = char
        self.children = []
        # Is it the last character of the word.
        self.word_finished = False
        # How many times this character appeared in the addition process
        self.counter = 1

def add(root, word: str):
    """
    Adding a word in the trie structure
    """
    node = root
    for char in word:
        found_in_child = False
        # Search for the character in the children of the present `node`
        for child in node.children:
            if child.char == char:
                # We found it, increase the counter by 1 to keep track that another
                # word has it as well
                child.counter += 1
                # And point the node to the child that contains this char
                node = child
                found_in_child = True
                break
        # We did not find it so add a new chlid
        if not found_in_child:
            new_node = TrieNode(char)
            node.children.append(new_node)
            # And then point node to the new child
            node = new_node
    # Everything finished. Mark it as the end of a word.
    node.word_finished = True

def find_prefix(root, prefix: str) -> Tuple[bool, int]:
    """
    Check and return
      1. If the prefix exsists in any of the words we added so far
      2. If yes then how may words actually have the prefix
    """
    node = root
    # If the root node has no children, then return False.
    # Because it means we are trying to search in an empty trie
    if not root.children:
        return False, 0
    for char in prefix:
        char_not_found = True
        # Search through all the children of the present `node`
        for child in node.children:
            if child.char == char:
                # We found the char existing in the child.
                char_not_found = False
                # Assign node as the child containing the char and break
                node = child
                break
        # Return False anyway when we did not find a char.
        if char_not_found:
            return False, 0
    # Well, we are here means we have found the prefix. Return true to indicate that
    # And also the counter of the last node. This indicates how many words have this
    # prefix
    return True, node.counter

# END PART FROM https://towardsdatascience.com/implementing-a-trie-data-structure-in-python-in-less-than-100-lines-of-code-a877ea23c1a1

# This method works like str.split, but splits for as many times as a delimiter shows up in the doc
# It is also original work based on prior knowledge of how string splits work in Python.
def multi_splitter(input_string, delimiter):
    out_strings = []
    new_sub = str(input_string).split(delimiter)
    for str_element in new_sub:
        sub = str_element.split("</D>")
        out_strings.append(sub[0])
    return out_strings

def get_text(place, sources, places_bag_vector, t_type):
    # This portion involving reading the body text in from the file mostly done by Kumar
    print (place)
    total_text = ""
    for source in sources:
        with open(source) as f:
            data = f.read()
            soup = BeautifulSoup(data, 'html.parser') # parse using HTML parser, close to structure of these files
            reuters_tags = soup.find_all('reuters')
            for reuter_tag in reuters_tags: # get information stored within each reuters tag
                if t_type == 'topics':
                    p_tag = reuter_tag.topics
                else:
                    p_tag = reuter_tag.places
                d_tags = p_tag.find_all('d') # find all places/topics mentioned
                for d_tag in d_tags:
                    for child in d_tag.children: # find relevant tags to current call and add text to a master string
                        if(place == child):
                            try:
                                total_text += reuter_tag.body.get_text()
                            except:
                                total_text += ""

    # This subsequent section is devoted to removing a few bits of rather unwieldy extra characters in our
    # output string. We wanted to retain as many words as possible, so more tedious methods of extraction,
    # such as removing '\n' from the MIDDLE of the word was required. This part written by Afnan.
    array = total_text.split()
    new_array = []
    for word in array: # each word gets examined and picked apart if it contains the offending characters
        new_word = ""
        if '\n' in word: # removing line breaks, wherever they may occur
            subword = word.split('\n')
            for part in subword:
                if '\n' not in part:
                    new_word += part
                    word = new_word
        new_word = ""
        if '.' in word: # removing punctuation
            subword = word.split('.')
            for part in subword:
                if '.' not in part:
                    new_word += part
                    word = new_word
        new_word = ""
        if ',' in word: # removing punctuation
            subword = word.split(',')
            for part in subword:
                if ',' not in part:
                    new_word += part
                    word = new_word
        new_word = ""
        if '"' in word: # removing punctuation
            subword = word.split('"')
            for part in subword:
                if '"' not in part:
                    new_word += part
                    word = new_word
        word += " "
        new_array.append(word)

    cleaned_text = ""
    for newword in new_array:# now removing some final pesky words as well as any numbers we don't want in our analysis
        if "reuter" not in newword.lower() and "\x03" not in newword and '"' not in newword and newword.isdigit() == False:
            cleaned_text += newword

    # Create vector and return to calling function
    places_bag_vector[place] = cleaned_text
    # output looks like: {'afghanistan' : 'Pakistan complained to the United Nations today that...', 'algeria' : 'Liquefied natural gas imports from Algeria...', ....}
    return places_bag_vector

def weighted_get_text(header_importance, place, sources, weighted_bag_vector, t_type):

    # This portion involving reading the body text in from the file mostly done by Kumar
    print (place)
    total_text = ""
    for source in sources:
        with open(source) as f:
            data = f.read()
            soup = BeautifulSoup(data, 'html.parser') # parse using HTML parser, close to structure of these files
            reuters_tags = soup.find_all('reuters')
            for reuter_tag in reuters_tags: # get information stored within each reuters tag
                if t_type == 'topics':
                    p_tag = reuter_tag.topics
                else:
                    p_tag = reuter_tag.places

                d_tags = p_tag.find_all('d') # find all places/topics mentioned
                for d_tag in d_tags:
                    for child in d_tag.children: # find relevant tags to current call and add text to a master string
                        if(place == child):

                            try:
                                total_text += reuter_tag.body.get_text()
                            except:
                                total_text += ""

                            title_tags = reuter_tag.find_all('title')
                            if (len(title_tags) > 0):
                                for title_tag in title_tags:
                                    for i in range(header_importance):
                                        total_text += title_tag.get_text()
                                        total_text += ' '

    # This subsequent section is devoted to removing a few bits of rather unwieldy extra characters in our
    # output string. We wanted to retain as many words as possible, so more tedious methods of extraction,
    # such as removing '\n' from the MIDDLE of the word was required. This part written by Afnan.
    array = total_text.split()
    new_array = []
    for word in array: # each word gets examined and picked apart if it contains the offending characters
        new_word = ""
        if '\n' in word: # removing line breaks, wherever they may occur
            subword = word.split('\n')
            for part in subword:
                if '\n' not in part:
                    new_word += part
                    word = new_word
        new_word = ""
        if '.' in word: # removing punctuation
            subword = word.split('.')
            for part in subword:
                if '.' not in part:
                    new_word += part
                    word = new_word
        new_word = ""
        if ',' in word: # removing punctuation
            subword = word.split(',')
            for part in subword:
                if ',' not in part:
                    new_word += part
                    word = new_word
        new_word = ""
        if '"' in word: # removing punctuation
            subword = word.split('"')
            for part in subword:
                if '"' not in part:
                    new_word += part
                    word = new_word
                    new_word = ""
        if '<' in word: # removing punctuation
            subword = word.split('<')
            for part in subword:
                if '<' not in part:
                    new_word += part
                    word = new_word
                    new_word = ""
        if '>' in word: # removing punctuation
            subword = word.split('>')
            for part in subword:
                if '>' not in part:
                    new_word += part
                    word = new_word
        word += " "
        new_array.append(word)

    cleaned_text = ""
    for newword in new_array:# now removing some final pesky words as well as any numbers we don't want in our analysis
        if "reuter" not in newword.lower() and "\x03" not in newword and '"' not in newword and newword.isdigit() == False:
            if (re.search(place, newword, re.IGNORECASE)):
                for i in range(header_importance * 2):
                    cleaned_text += newword
            else:
                cleaned_text += newword

    cleaned_text.rstrip()

    wordList = []
    words = re.findall(r'\b[a-zA-Z]+\b', cleaned_text)
    for word in words:
        wordList.append(word.lower())

    wordPairs = []
    for word in set(wordList):
        wordPairs.append([word, wordList.count(word)])

    cleaned_text = ""
    for wordPair in wordPairs:
        if wordPair[1] > 1:
            for i in range(wordPair[1]):
                cleaned_text += wordPair[0] + ' '

    # Create vector and return to calling function
    weighted_bag_vector[place] = cleaned_text
    # output looks like: {'afghanistan' : 'Pakistan complained to the United Nations today that...', 'algeria' : 'Liquefied natural gas imports from Algeria...', ....}
    return weighted_bag_vector

if __name__ == "__main__":
    sources = ["files/reut2-000.sgm", "files/reut2-001.sgm", "files/reut2-002.sgm", \
               "files/reut2-003.sgm", "files/reut2-004.sgm", "files/reut2-005.sgm", \
               "files/reut2-006.sgm", "files/reut2-007.sgm", "files/reut2-008.sgm", \
               "files/reut2-009.sgm", "files/reut2-010.sgm", "files/reut2-011.sgm", \
               "files/reut2-012.sgm", "files/reut2-013.sgm", "files/reut2-014.sgm", \
               "files/reut2-015.sgm", "files/reut2-016.sgm", "files/reut2-017.sgm", \
               "files/reut2-018.sgm", "files/reut2-019.sgm", "files/reut2-020.sgm", \
               "files/reut2-021.sgm"]

    total_blank_places = 0
    total_blank_topics = 0
    total_countries = []
    total_topics = []
    root = TrieNode('*')

    # Here, my algorithm for splitting the elements of the TOPICS and PLACES fields is my original work
    for source in sources:
        with open(source) as f: # Open the file and read line by line to a list array
            array = []
            for line in f:
                array.append(line)
        # Since PLACES were contained within one line of code according to the data I saw, I assumed
        # that any line with the PLACES tag would contain all of the location info for that article
        places = []
        for index in array: # Look at lines containing the "PLACES" tag and read those into a separate list
            if "<PLACES>" in index:
                places.append(index)
        # Once I got the line, I split the string on the multiple "<D>" tags to extract the location
        # information within
        new_places = []
        for place in places:
            new_places.extend(multi_splitter(place, "<D>")) # Using the helpful method above, I split on one or more <D> tags
        new_places = [x for x in new_places if x not in ('', '/', '\n', 'PLACES', '/PLACES')]# I then removed instances of tag information or blank information from the overall list

        # One trick I learned in coding Python for work is that by casting a list as a set,
        # you can remove duplicates in one line of code since sets do not contain duplicates
        distinct_countries = set(new_places)
        total_countries.extend(distinct_countries)

        # Next I moved onto TOPICS, using many of the same methods
        # that I used for PLACES to count and extract the information
        topics = []
        for index in array:
            if "<TOPICS>" in index:
                topics.append(index)

        # Once again I used the same string split method to extract the contents of each field
        tops = []
        for topic in topics:
            tops.extend(multi_splitter(topic, "<D>"))
        tops = [x for x in tops if x not in ('', '/', '\n', 'TOPICS', '/TOPICS')]

        # Counted distinct topics using the same cast to set
        distinct_topics = set(tops)
        # You may notice the issue with simply extending the list of total topics
        # There may end up being duplicates between documents that are not addressed
        # I address this issue in the final step: printing the statistics after all loops are finished
        total_topics.extend(distinct_topics)

    # Here, we create all output vectors already sorted into training and test groups based on cross-validation where k = 21
    # These files are then fed into the classifier program
    for i in range(3):
        training_sources = sources[:i] + sources[i+1:]
        test_sources = []
        test_sources.append(sources[i])
        # Here we begin to make our bag of words vectors
        # First we make the training groups

        # TEST SET FOR SPEED
        total_countries = ['afghanistan', 'uk', 'france', 'canada','turkey','usa','japan','pakistan']
        total_topics = ['acq', 'alum', 'lumber', 'jobs', 'interest', 'income','trade', 'wheat']
        # TEST

        #
        #   Vector 1: Generic Bag of Words method. Uses get_text to create bag of words.
        #
        bag_vector = {}
        for country in sorted(set(total_countries)):
            if "<PLACES>" not in country:
                get_text(country, training_sources, bag_vector, 'places')
        with open('place_bag_train' + str(i) + '.csv', 'w') as csv_file:
            writer = csv.writer(csv_file)
            writer.writerow(["country", "text"])
            for key, value in bag_vector.items():
                writer.writerow([key, value])

        bag_vector = {}
        for topic in sorted(set(total_topics)):
            if "<TOPICS>" not in topic:
                get_text(topic, training_sources, bag_vector, 'topics')
        with open('topic_bag_train' + str(i) + '.csv', 'w') as csv_file:
            writer = csv.writer(csv_file)
            writer.writerow(["topic", "text"])
            for key, value in bag_vector.items():
                writer.writerow([key, value])

        # Testing bag of words data.
        bag_vector = {}
        for country in sorted(set(total_countries)):
            if "<PLACES>" not in country:
                get_text(country, test_sources, bag_vector, 'places')
        with open('place_bag_test' + str(i) + '.csv', 'w') as csv_file:
            writer = csv.writer(csv_file)
            writer.writerow(["id", "text"])
            for key, value in bag_vector.items():
                writer.writerow([key, value])

        bag_vector = {}
        for topic in sorted(set(total_topics)):
            if "<TOPICS>" not in topic:
                get_text(topic, test_sources, bag_vector, 'topics')
        with open('topic_bag_test' + str(i) + '.csv', 'w') as csv_file:
            writer = csv.writer(csv_file)
            writer.writerow(["id", "text"])
            for key, value in bag_vector.items():
                writer.writerow([key, value])

        #
        #   Vector 2: Weighted Bag of Words method. Uses weighted_get_text to create bag of words. Title words are five times as important,
        #   the name of the place or topic is ten times as important, and words which only appear once are removed from the bag of words.
        #   Words are added multiple times based on their importance.
        #
        weighted_bag_vector = {}
        for country in sorted(set(total_countries)):
            if "<PLACES>" not in country:
                weighted_get_text(5, country, training_sources, weighted_bag_vector, 'places')
        with open('place_weighted_train' + str(i) + '.csv', 'w') as csv_file:
            writer = csv.writer(csv_file)
            writer.writerow(["country", "text"])
            for key, value in weighted_bag_vector.items():
                writer.writerow([key, value])

        weighted_bag_vector = {}
        for topic in sorted(set(total_topics)):
            if "<TOPICS>" not in topic:
                weighted_get_text(5, topic, training_sources, weighted_bag_vector, 'topics')
        with open('topic_weighted_train' + str(i) + '.csv', 'w') as csv_file:
            writer = csv.writer(csv_file)
            writer.writerow(["topic", "text"])
            for key, value in weighted_bag_vector.items():
                writer.writerow([key, value])

        # Testing modified bag of words data.
        weighted_bag_vector = {}
        for country in sorted(set(total_countries)):
            if "<PLACES>" not in country:
                weighted_get_text(5, country, test_sources, weighted_bag_vector, 'places')
        with open('place_weighted_test' + str(i) + '.csv', 'w') as csv_file:
            writer = csv.writer(csv_file)
            writer.writerow(["id", "text"])
            for key, value in weighted_bag_vector.items():
                writer.writerow([key, value])

        weighted_bag_vector = {}
        for topic in sorted(set(total_topics)):
            if "<TOPICS>" not in topic:
                weighted_get_text(5, topic, test_sources, weighted_bag_vector, 'topics')
        with open('topic_weighted_test' + str(i) + '.csv', 'w') as csv_file:
            writer = csv.writer(csv_file)
            writer.writerow(["id", "text"])
            for key, value in weighted_bag_vector.items():
                writer.writerow([key, value])


import requests
import json
import yaml
from pprint import pprint
import os
from jinja2 import Environment, FileSystemLoader

PATH = os.path.dirname(os.path.abspath(__file__))
TEMPLATE_ENVIRONMENT = Environment(
    autoescape=False,
    loader=FileSystemLoader(os.path.join(PATH, 'templates')),
    trim_blocks=False)

def load_settings():
    with open("settings.yml", 'r') as stream:
        try:
           settings = yaml.load(stream)
        except yaml.YAMLError as exc:
            print(exc)
    return settings

def get_token(settings):
    payload= {"apikey":settings['api_key'],
              "username":settings['user_name'],
              "userkey":settings['user_key']}
    r = requests.post(settings['url'] + "/login", json=payload)
    response = r.json()
    token = response["token"]
    return token

def get_series(settings, token):
    headers = {'Authorization': 'Bearer ' + token}
    payload = {'name': 'Star Trek Discovery'}
    r = requests.get(settings['url'] + "/search/series",
                     params=payload, headers=headers)
    response = r.json()
    return response

def get_series_details(settings, token, sid):
    headers = {'Authorization': 'Bearer ' + token}
    r = requests.get(settings['url'] + "/series/" +
                     str(sid) + "/episodes", headers=headers)
    response = r.json()
    return response

def render_template(template_filename, context):
    return TEMPLATE_ENVIRONMENT.get_template(template_filename).render(context)

def create_md(series, series_details):
   fname = "output.md"
   context = { 'series_details': series_details,
               'series': series['data'][0]
              }
   with open(fname, 'w') as f:
       content = render_template('output.j2', context)
       f.write(content)

def main():
    settings = load_settings()
    token = get_token(settings)
    series = get_series(settings, token)
    series_id = series['data'][0]['id']
    series_details = get_series_details(settings, token, series_id)
    create_md(series, series_details)

if __name__ == "__main__":
    main()


import logging
import requests

from ask_sdk_core.skill_builder import SkillBuilder
from ask_sdk_core.dispatch_components import AbstractRequestHandler
from ask_sdk_core.dispatch_components import AbstractExceptionHandler
import ask_sdk_core.utils as ask_utils
from ask_sdk_core.handler_input import HandlerInput

from ask_sdk_model import Response

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

class LaunchRequestHandler(AbstractRequestHandler):
    """Handler for Skill Launch."""
    def can_handle(self, handler_input):
        # type: (HandlerInput) -> bool

        return ask_utils.is_request_type("LaunchRequest")(handler_input)

    def handle(self, handler_input):
        # type: (HandlerInput) -> Response
        speak_output = "Ciao, puoi chiedermi di darti l'ultimo articolo"

        return (
            handler_input.response_builder
                .speak(speak_output)
                .ask(speak_output)
                .response
        )

class BlogIntentHandler(AbstractRequestHandler):
    """Handler for Hello World Intent."""
    def can_handle(self, handler_input):
        # type: (HandlerInput) -> bool
        return ask_utils.is_intent_name("BlogIntent")(handler_input)

    def handle(self, handler_input):
        # type: (HandlerInput) -> Response

        data = requests.get("https://www.20tab.com/api/articles.json?page_size=1").json()

        try:
            title = data["results"][0]['title_it']
            speak_output = f"L'ultimo articolo ha come titolo: {title}"
        except KeyError:
            speak_output = "Mi dispiace ma non ho trovato alcun articolo"

        return (
            handler_input.response_builder
                .speak(speak_output)
                # .ask("add a reprompt if you want to keep the session open for the user to respond")
                .response
        )

class HelpIntentHandler(AbstractRequestHandler):
    """Handler for Help Intent."""
    def can_handle(self, handler_input):
        # type: (HandlerInput) -> bool
        return ask_utils.is_intent_name("AMAZON.HelpIntent")(handler_input)

    def handle(self, handler_input):
        # type: (HandlerInput) -> Response
        speak_output = "Puoi chiedermi di dirti qual'è l'ultimo articolo"

        return (
            handler_input.response_builder
                .speak(speak_output)
                .ask(speak_output)
                .response
        )

class CancelOrStopIntentHandler(AbstractRequestHandler):
    """Single handler for Cancel and Stop Intent."""
    def can_handle(self, handler_input):
        # type: (HandlerInput) -> bool
        return (ask_utils.is_intent_name("AMAZON.CancelIntent")(handler_input) or
                ask_utils.is_intent_name("AMAZON.StopIntent")(handler_input))

    def handle(self, handler_input):
        # type: (HandlerInput) -> Response
        speak_output = "Goodbye!"

        return (
            handler_input.response_builder
                .speak(speak_output)
                .response
        )

class SessionEndedRequestHandler(AbstractRequestHandler):
    """Handler for Session End."""
    def can_handle(self, handler_input):
        # type: (HandlerInput) -> bool
        return ask_utils.is_request_type("SessionEndedRequest")(handler_input)

    def handle(self, handler_input):
        # type: (HandlerInput) -> Response

        # Any cleanup logic goes here.

        return handler_input.response_builder.response

class CatchAllExceptionHandler(AbstractExceptionHandler):
    """Generic error handling to capture any syntax or routing errors. If you receive an error
    stating the request handler chain is not found, you have not implemented a handler for
    the intent being invoked or included it in the skill builder below.
    """
    def can_handle(self, handler_input, exception):
        # type: (HandlerInput, Exception) -> bool
        return True

    def handle(self, handler_input, exception):
        # type: (HandlerInput, Exception) -> Response
        logger.error(exception, exc_info=True)

        speak_output = "Si è verificato un errore. Riprova"

        return (
            handler_input.response_builder
                .speak(speak_output)
                .ask(speak_output)
                .response
        )

# The SkillBuilder object acts as the entry point for your skill, routing all request and response
# payloads to the handlers above. Make sure any new handlers or interceptors you've
# defined are included below. The order matters - they're processed top to bottom.

sb = SkillBuilder()

sb.add_request_handler(LaunchRequestHandler())
sb.add_request_handler(BlogIntentHandler())
sb.add_request_handler(HelpIntentHandler())
sb.add_request_handler(CancelOrStopIntentHandler())
sb.add_request_handler(SessionEndedRequestHandler())

sb.add_exception_handler(CatchAllExceptionHandler())

skill = sb.create()


from django.contrib.staticfiles.testing import StaticLiveServerCase
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import unittest
import sys

class NewVisitorTest(StaticLiveServerCase):

	@classmethod
	def setUpClass(cls):
		for arg in sys.argv:
			if 'liveserver' in arg:
				cls.server_url = 'http://' + arg.split('=')[1]
				return
		super().setUpClass()
		cls.server_url = cls.live_server_url

	@classmethod
	def tearDownClass(cls):
		if cls.server_url == cls.live_server_url:
			super().tearDownClass()

	def setUp(self):
		self.browser = webdriver.Firefox()
		self.browser.implicitly_wait(3)

	def tearDown(self):
		self.browser.quit()

	def check_for_row_in_list_table(self, row_text):
		table = self.browser.find_element_by_id('id_list_table')
		rows = table.find_elements_by_tag_name('tr')
		self.assertIn(row_text, [row.text for row in rows])

	def test_can_start_a_list_and_finish_it_later(self):
		# Anita heard about a new online to-do list
		# and decided to check out its home page
		self.browser.get(self.server_url)
		self.assertIn('To-Do', self.browser.title)
		header_text = self.browser.find_element_by_tag_name('h1').text
		self.assertIn('To-Do', header_text)

		# She is invited to enter an item right away
		inputbox = self.browser.find_element_by_id('id_new_item')
		self.assertEqual(
			inputbox.get_attribute('placeholder'),
			'Enter a to-do item'
		)

		# She enters "Do shopping" into a textbox
		inputbox.send_keys('Do shopping')

		# When she hits enter the page updates to show a table with
		# "1: Do shopping" as an item
		inputbox.send_keys(Keys.ENTER)
		anita_list_url = self.browser.current_url
		self.assertRegex(anita_list_url, '/lists/.+/$')
		self.check_for_row_in_list_table('1: Do shopping')

		# There is still a textbox inviting her to enter another item
		# she enters "Have fun"
		inputbox = self.browser.find_element_by_id('id_new_item')
		inputbox.send_keys('Have fun')
		inputbox.send_keys(Keys.ENTER)

		# The page updates again to show both items in the list
		self.check_for_row_in_list_table('1: Do shopping')
		self.check_for_row_in_list_table('2: Have fun')

		# A new user, Darwin, comes along

		## We use a new browser session to make sure that no information
		## of Anita's is coming through from cookies etc #
		self.browser.quit()
		self.browser = webdriver.Firefox()

		# Darwin visits the home page. There is no sign of Anita's items
		self.browser.get(self.server_url)
		page_text = self.browser.find_element_by_tag_name('body').text
		self.assertNotIn('Have fun', page_text)
		self.assertNotIn('Do shopping', page_text)

		# Darwin starts a new list by entering an item. He's just a baby so...
		inputbox = self.browser.find_element_by_id('id_new_item')
		inputbox.send_keys('Cause trouble')
		inputbox.send_keys(Keys.ENTER)

		# Darwin gets his own url
		darwin_list_url = self.browser.current_url
		self.assertRegex(darwin_list_url, '/lists/.+/$')
		self.assertNotEqual(darwin_list_url, anita_list_url)

		# There is no trace of Anita's list
		page_text = self.browser.find_element_by_tag_name('body').text
		self.assertNotIn('Have fun', page_text)
		self.assertIn('Cause trouble', page_text)

		# Both are now happy campers

	def test_layout_and_styling(self):
		# Anita goes to the site
		self.browser.get(self.server_url)
		self.browser.set_window_size(1024, 768)

		# She notices the input box is centred
		inputbox = self.browser.find_element_by_id('id_new_item')
		self.assertAlmostEqual(
			inputbox.location['x'] + inputbox.size['width'] / 2,
			512, delta=7)

		# She starts a new list and sees the input centred there too
		inputbox.send_keys('testing\n')
		inputbox = self.browser.find_element_by_id('id_new_item')
		self.assertAlmostEqual(
			inputbox.location['x'] + inputbox.size['width'] / 2,
			512, delta=7)


import numpy as np
import matplotlib.pyplot as plt

from sklearn import datasets
from sklearn.model_selection import train_test_split

from keras.models import Sequential, Model
from keras.layers import Input, concatenate, multiply, Dense, Permute, Reshape, LSTM, Activation
from keras import optimizers
from keras.utils.np_utils import to_categorical
import keras.backend as K

data = datasets.load_digits()

X_data = data.images
y_data = data.target

y_data = to_categorical(y_data)

X_train, X_test, y_train, y_test = train_test_split(X_data, y_data, test_size = 0.2, random_state = 777)

def attention_3d_block(inputs):
    # inputs.shape = (batch_size, time_steps, input_dim)
    input_dim = int(inputs.shape[2])
    a = Permute((2, 1))(inputs)
    a = Reshape((input_dim, TIME_STEPS))(a) # this line is not useful. It's just to know which dimension is what.
    a = Dense(TIME_STEPS, activation='softmax')(a)
    if SINGLE_ATTENTION_VECTOR:
        a = Lambda(lambda x: K.mean(x, axis=1), name='dim_reduction')(a)
        a = RepeatVector(input_dim)(a)
    a_probs = Permute((2, 1), name='attention_vec')(a)
    output_attention_mul = multiply([inputs, a_probs], name='attention_mul')
    return output_attention_mul

def attention_lstm():
    inputs = Input(shape=(TIME_STEPS, INPUT_DIM,))
    attention_mul = attention_3d_block(inputs)
    lstm_units = 32
    attention_mul = LSTM(lstm_units, return_sequences=False)(attention_mul)
    output = Dense(10, activation='softmax')(attention_mul)
    model = Model(inputs=[inputs], outputs=output)

    model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

    return model

model = attention_lstm()
model.fit([X_train], y_train, epochs = 100, batch_size = 64, validation_split = 0.3, verbose = 0)

attention_vector = get_activations(model, [X_train[0]], print_shape_only=True, layer_name = 'attention_vec')[0].reshape(8,8)

plt.imshow(attention_vector)
plt.show()


# Python imports
import logging

# ONSA imports
from charles.utils.utils import *
from charles.utils.inventory_utils import *
from charles.utils.ipam_utils import *
from charles.constants import *
from charles.models import *

def deleted_automated_request(service):
  logging.debug("deleted_automated_request")
  client = get_client(service['client_id'])
  service_data = {}

  try:
    parameters = an_parameters(client, service)
    config = {
      "client": client['name'],
      "service_type":  service['service_type'],
      "service_id": service['service_id'],
      "op_type": "DELETE"}

    config['parameters'] = {
                "service_vlan": service['vlan_id'],
                "an_client_port": parameters['an_client_port'],
                "an_uplink_ports": parameters['an_uplink_ports'],
                "access_port_services":parameters['access_port_services']
                  }
    config['devices'] = [{"vendor": parameters['vendor'],
        "model": parameters['model'], "mgmt_ip": parameters['mgmt_ip']}]

    service_state = DELETEINPROGRESS_SERVICE_STATE

    logging.debug(f'releasing service resources for service {service}')
    if service['public_network']:
      destroy_subnet(client['name'], service['service_id'])
    if service['wan_network']:
      destroy_subnet(client['name'], service['service_id'])
    release_access_port(service['access_port_id'])
    release_vlan(service['access_node_id'], service['vlan_id'])
    logging.debug("deleting service")

    #Send message(job) to Queue so workers can take it
    configure_service(config)
  except BaseException as e:
    logging.error(e)
    service_state = DELETEERROR_SERVICE_STATE

  service_data['service_state'] = service_state
  update_jeangrey_service(service['service_id'], service_data)
  service = update_charles_service(service, service_state)
  return service

def an_activated_automated_request(service):
  logging.debug("an_activated_automated_request")
  client = get_client(service['client_id'])
  service_data = {}

  try:
    parameters = an_parameters(client, service)
    config = {
      "client": client['name'],
      "service_type":  service['service_type'],
      "service_id": service['service_id'],
      "op_type": "CREATE"}

    backbone_parameters = bb_parameters(client, service)

    config['parameters'] = {
                "service_vlan": service['vlan_id'],
                "an_client_port": parameters['an_client_port'],
                "an_uplink_ports": parameters['an_uplink_ports'],
                "access_port_services":parameters['access_port_services']
                  }
    config['devices'] = [{"vendor": parameters['vendor'],
        "model": parameters['model'], "mgmt_ip": parameters['mgmt_ip']}]

    service_state = "an_activation_in_progress"
    logging.debug("configuring service")

    #Send message(job) to Queue so workers can take it
    configure_service(config)
  except BaseException:
    service_state = "ERROR"
    destroy_subnet(client['name'], service['service_id'])

  service_data['service_state'] = service_state
  service_data['public_network'] = backbone_parameters['public_network']
  service_data['wan_network'] = backbone_parameters['wan_network']
  update_jeangrey_service(service['service_id'], service_data)
  service = update_charles_service(service, service_state)
  return service

def an_parameters(client, service):
  try:
    access_port = get_access_port(service['access_port_id'])
    access_node = get_access_node(service['access_node_id'])
    an_device_model = get_device_model(access_node['device_model_id'])
    access_port_services = get_access_port_services(service['access_port_id'])

    services = ""
    for my_service in access_port_services:
      if my_service['id'] is not service['service_id']:
        services+= str(my_service['id']) + "-"
    services += str(service['service_id'])

    parameters = { 'provider_vlan': access_node['provider_vlan'],
                  'an_client_port': access_port['port'],
                  'an_uplink_ports': access_node['uplink_ports'],
                  'mgmt_ip': access_node['mgmt_ip'],
                  'model': an_device_model['model'],
                  'vendor': an_device_model['brand'],
                  'access_port_services': services }

    logging.debug(f'parameters: {parameters}')

    return parameters
  except BaseException:
    logging.error("Unable to fetch parameters")
    raise InvalidParametersException("Unable to fetch parameters")

  # service_data = {}

def bb_parameters(client, service):
  try:
    location = get_location(service['location_id'])
    logging.debug(f'location: {location}')

    router_node = get_router_node(service['router_node_id'])
    logging.debug(f'router_node: {router_node}')

    rn_device_model = get_device_model(router_node['device_model_id'])

    logging.debug(f'looking network with prefix {service["prefix"]} for service ID {service["service_id"]} client {client["name"]}')
    public_network = assign_network(client['name'], service['service_id'],IPAM_PUBLIC_NETWORK,service['prefix'])
    logging.debug(f'network: {public_network}')
    wan_network = assign_network(client['name'], service['service_id'],IPAM_MGMT_WAN,IPAM_MGMT_WANPREFIX)
    logging.debug(f'network: {wan_network}')

    parameters = {
                'pop_size': location['pop_size'],
                'public_network' : public_network,
                'wan_network' : wan_network
                }

    parameters['router_node'] = { 'vendor': rn_device_model['brand'],
                                  'model': rn_device_model['model'],
                                  'mgmt_ip': router_node['mgmt_ip']
                                }

    logging.debug(f'parameters: {parameters}')
    return parameters
  except BaseException:
    logging.error("Unable to fetch parameters")
    raise InvalidParametersException("Unable to fetch parameters")

    # """
    # Fetch  logical units
    # """
    # if service['logical_unit_id'] is None:
    #   free_logical_units = get_free_logical_units(router_node['id'])
    #   if free_logical_units is None:
    #       logging.warning('No available logical units')
    #       parameters['status'] = ERR_NO_LOGICALUNITS
    #       return parameters
    #   else:
    #       logical_unit_id = free_logical_units[0]['id']
    #       client_network = get_client_network(client['name'], service['id'], service['prefix'])
    #       if client_network:
    #           add_logical_unit_to_router_node(router_node['id'], logical_unit_id, service['id'])
    #       else:
    #           logging.warning('No public networks available')
    #           parameters['status'] = ERR_NO_PUBLICNETWORKS
    #           return parameters

    #   wan_network = get_wan_mpls_network(location['name'], client['name'], service['id'])

    # else:
    #   logical_unit_id = service['logical_unit_id']
    #   client_network = service['public_network']
    #   wan_network = service['wan_network']

def bb_data_ack_automated_request(service):
  # if DEBUG: print("bb_data_ack_automated_request")
  # client = get_client(service['client_id'])
  # parameters = bb_parameters(client, service)

  # if DEBUG: print(parameters)

  # #Handle parameters ERROR
  # if parameters is not None:
  #   service_data = {  'logical_unit_id': parameters['logical_unit_id'],
  #                     'wan_network':  parameters['wan_network']
  #                   }
  #   service_data['public_network'] = parameters['client_network']
  #   service_state = "bb_data_ack"

  # else:
  #   service_data = {}
  #   service_state = "ERROR"

  # service_data['service_state']=service_state
  # update_jeangrey_service(service['service_id'], service_data)
  # service = update_charles_service(service, service_state)
  # service.update(service_data)
  # my_service = service
  # return my_service
  pass

def bb_activated_automated_request(service):
  # if DEBUG: print("bb_activated_automated_request")
  # client = get_client(service['client_id'])
  # parameters = bb_parameters(client, service)

  # config = {
  #      "client" : client['name'],
  #      "service_type" :  service['service_type'],
  #      "service_id" : service['service_id'],
  #      "op_type" : "CREATE" }

  # config['parameters'] =  {
  #                         "pop_size" : parameters['pop_size'],
  #                         "an_uplink_interface" : parameters['an_uplink_interface'],
  #                         "an_uplink_ports" :   parameters['an_uplink_ports'],
  #                         "logical_unit" : parameters['logical_unit_id'],
  #                         "provider_vlan" : parameters['provider_vlan'],
  #                         "service_vlan" : service['vlan_id'],
  #                         "an_client_port" : parameters['an_client_port'],
  #                         "wan_cidr": service['wan_network'],
  #                         "client_cidr": service['public_network']
  #                         }

  # config['devices'] =  [{"vendor": parameters['router_node']['vendor'],"model": parameters['router_node']['model'],"mgmt_ip": parameters['router_node']['mgmt_ip']}]

  # configure_service(config)
  # # service_data = {}
  # # service_data['service_state'] = "bb_activation_in_progress"
  # service_state = "bb_activation_in_progress"
  # service_data = {'service_state' : service_state}
  # update_jeangrey_service(service['service_id'], service_data)
  # service = update_charles_service(service, service_state)

  # return service
  pass

def an_data_ack_automated_request(service):
  # if DEBUG: print("an_data_ack_automated_request")
  # # service_data = {}
  # # service_data['service_state'] = "an_data_ack"
  # service_state = "an_data_ack"
  # service_data = {'service_state' : service_state}
  # update_jeangrey_service(service['service_id'], service_data)
  # service = update_charles_service(service, service_state)
  # return service
  pass

def cpe_data_ack_automated_request(service):
  # if DEBUG: print("cpe_data_ack_automated_request")
  # client = get_client(service['client_id'])
  # parameters = cpe_parameters(client, service)
  # customer_location = get_customer_location(service['client_id'], service['customer_location_id'])

  # if parameters['client_port_id']:
  #   service_data = { "client_port_id": parameters['client_port_id']}

  # service_state = "cpe_data_ack"

  # service_data['service_state']=service_state
  # update_jeangrey_service(service['service_id'], service_data)
  # service = update_charles_service(service, service_state)
  # service.update(service_data)
  # return service
  pass

def service_activated_automated_request(service):
  # if DEBUG: print("service_activated_automated_request")
  # client = get_client(service['client_id'])
  # parameters = cpe_parameters(client, service)
  # config = {
  #    "client" : client['name'],
  #    "service_type" :  service['service_type'],
  #    "service_id" : service['service_id'],
  #    "op_type" : "CREATE" }

  # service_data = {}

  # if parameters is not None:
  #   config['parameters'] =  {
  #               "service_vlan" : service['vlan_id'],
  #               "bandwidth" : service['bandwidth'],
  #               "on_client_port" : parameters['client_node']['interface_name'],
  #               "on_uplink_port" : parameters['client_node']['uplink_port'],
  #               "wan_cidr": service['wan_network'],
  #               "client_cidr": service['public_network']
  #              }

  #   config['devices'] = [{"vendor": parameters['client_node']['vendor'], "model": parameters['client_node']['model'], "mgmt_ip": parameters['client_node']['mgmt_ip']}]

  #   service_state = "cpe_activation_in_progress"
  #   configure_service(config)
  # else:
  #   service_state = "ERROR"

  # service_data['service_state'] = service_state
  # update_jeangrey_service(service['service_id'], service_data)
  # service = update_charles_service(service, service_state)
  # return service
  pass

def cpe_parameters(client, service):
  pass
  # location = get_location(service['location_id'])
  # client_node = get_client_node(service['client_node_sn'])
  # parameters = {}

  # if service['client_port_id'] is None:
  #   customer_location = get_customer_location(client['id'],service['customer_location_id'])
  #   client_port_id = fetch_cpe_port_id(service['client_node_sn'], client['name'], customer_location)
  #   client_port = get_client_port(service['client_node_sn'], client_port_id)
  #   parameters['client_port_id'] = client_port_id

  # else:
  #   client_port = get_client_port(service['client_node_sn'], service['client_port_id'])

  # parameters['client_node'] = { 'vendor': client_node['vendor'],
  #                               'model': client_node['model'],
  #                               'mgmt_ip': client_node['mgmt_ip'],
  #                               'interface_name': client_port['interface_name'],
  #                               'wan_network': service['wan_network'],
  #                               'uplink_port' : client_node['uplink_port']
  #                             }

  # return parameters


# -*- coding: utf-8 -*-
import json
import sys
from ConfigParser import NoOptionError

import click
from syncano.exceptions import SyncanoDoesNotExist
from syncano_cli.base.connection import create_connection, get_instance_name
from syncano_cli.config import ACCOUNT_CONFIG_PATH

from .utils import print_response

@click.group()
def top_execute():
    pass

@top_execute.command()
@click.option('--config', help=u'Account configuration file.')
@click.option('--instance-name', help=u'Instance name.')
@click.argument('script_endpoint_name')
@click.option('--payload', help=u'Script payload in JSON format.')
def execute(config, instance_name, script_endpoint_name, payload):
    """
    Execute script endpoint in given instance
    """
    config = config or ACCOUNT_CONFIG_PATH
    instance_name = get_instance_name(config, instance_name)
    try:
        connection = create_connection(config)
        instance = connection.Instance.please.get(instance_name)
        se = instance.script_endpoints.get(instance_name, script_endpoint_name)
        data = json.loads((payload or '').strip() or '{}')
        response = se.run(**data)
        print_response(response)
    except NoOptionError:
        click.echo(u'ERROR: Do a login first: syncano login.')
        sys.exit(1)
    except SyncanoDoesNotExist as e:
        click.echo(u'ERROR: {}'.format(e))
        sys.exit(1)
    except ValueError as e:
        click.echo(u'ERROR: Invalid payload format: {error}'.format(error=e))
        sys.exit(1)


import pygame
from pygame.locals import *

#Pygame draw class. Draws the different sections to the pygame screen. DrawStringData draws readable data.
#DrawGraph draws the graph. DrawPosition draws the position.
class DrawPlayer():
	def __init__(self, surface, start_xPos, start_yPos):
		self.surface = surface
		self.start_xPos = start_xPos
		self.start_yPos = start_yPos

		self.number = 0
		self.hearth = 0
		self.breath = 0
		self.position = 0
		self.hearthGraph = []
		self.breathGraph = []
		self.positionCircle = 0

	def drawBox(self):
		pygame.draw.line(self.surface, pygame.Color(155,155,155), (self.start_xPos, self.start_yPos), (self.start_xPos+500, self.start_yPos))
		pygame.draw.line(self.surface, pygame.Color(155,155,155), (self.start_xPos, self.start_yPos+100), (self.start_xPos+500, self.start_yPos+100))
		pygame.draw.line(self.surface, pygame.Color(155,155,155), (self.start_xPos, self.start_yPos), (self.start_xPos+100, self.start_yPos))
		pygame.draw.line(self.surface, pygame.Color(155,155,155), (self.start_xPos+500, self.start_yPos), (self.start_xPos+500, self.start_yPos+100))
		pygame.draw.line(self.surface, pygame.Color(155,155,155), (self.start_xPos+250, self.start_yPos), (self.start_xPos+250, self.start_yPos+100))

	def drawStringData(self, name, hearth, breath, xPos, yPos):
		pygame.font.init()
		font = pygame.font.SysFont('Calibri', 15)
		self.number = font.render('#'+str(name), False, (0, 0, 0))
		self.surface.blit(self.number, (0, self.start_yPos))

		self.position = font.render('xPos:'+str(xPos)+", yPos:"+str(yPos), False, (0, 0, 0))
		self.surface.blit(self.position, (0, self.start_yPos+25))

		self.hearth = font.render('Current Heart:'+str(hearth), False, (255, 0, 0))
		self.surface.blit(self.hearth, (0, self.start_yPos+50))

		self.breath = font.render('Current Breath:'+str(breath), False, (0, 0, 255))
		self.surface.blit(self.breath, (0, self.start_yPos+75))

	def drawGraph(self, hearthList, breathList):
		pygame.draw.line(self.surface, pygame.Color(0, 0,0), (self.start_xPos+250+250, self.start_yPos), (self.start_xPos+250+250, self.start_yPos+100))

		i = 0
		prev_value = 0
		for value in hearthList:
			if prev_value:
				pygame.draw.line(self.surface, pygame.Color(255, 0,0), (self.start_xPos+250+250-i, self.start_yPos+100-prev_value/2), (self.start_xPos+250+250-i-1, self.start_yPos+100-value/2))
			prev_value = value
			i = i + 1

		i = 0
		prev_value = 0
		for value in breathList:
			if prev_value:
				pygame.draw.line(self.surface, pygame.Color(0, 0,255), (self.start_xPos+250+250-i, self.start_yPos+100-prev_value/2), (self.start_xPos+250+250-i-1, self.start_yPos+100-value/2))
			prev_value = value
			i = i + 1

	def drawPosition(self, xPos, yPos, name):
		#Limits the positions inside bounds.
		xPos = xPos + 1 #These two maps (0,0) to "1,1" on screen in a 5x5m square
		yPos = yPos + 1
		if xPos <= 0:
			xPos = 0
		elif xPos >= 5:
			xPos = 5
		if yPos <= 0:
			yPos = 0
		elif yPos >= 5:
			yPos = 5
		#Maps real life meters into pixels on the screen.
		xPos = int((xPos)*250/5) + 500 #Change these if neccessary.
		yPos = int((yPos)*250/5)
		center = [xPos, yPos]
		self.positionCircle = pygame.draw.circle(self.surface, (255, 255, 0), center, 10)

		pygame.font.init()
		font = pygame.font.SysFont('Calibri', 15)
		self.number = font.render(str(name), False, (0, 0, 0))
		self.surface.blit(self.number, (xPos-4, yPos-7))


import sqlite3
import pandas as pd
filename = "FIORIBOM"

# con = sqlite3.connect(filename+".db")
con = sqlite3.connect("fbomdb.sqlite")
#
# CREATE/APPEND TABLE FIORIBOM - NAME SAME AS EXCEL AND SHEET, ACCEPTS XLSX
wb = pd.read_excel(filename+'.xlsx', sheet_name=None)
for sheet in wb:
    wb[sheet].to_sql(sheet, con, index=False, if_exists='append')
 #
# GET LENGTH OF TABLE
cur = con.cursor()
cur.execute("SELECT * FROM FIORIBOM")
rows = cur.fetchall()
print(len(rows))
#
# # GET LENGTH OF TABLE
def get_release():
    con = sqlite3.connect("fbomdb.sqlite")
    cur = con.cursor()
    cur.execute("SELECT DISTINCT ReleaseName FROM FIORIBOM ORDER BY ReleaseName DESC")
    rows = cur.fetchall()
    # print(len(rows))
    releases = list()
    for row in rows:
        if(('Wave' not in str(row[0])) and ('SFIN' not in str(row[0]))):
            # print(row[0])
            releases.append(row[0])
    con.commit()
    con.close()
    return releases

def get_app_details(release, app_list):
    con = sqlite3.connect("fbomdb.sqlite")
    cur = con.cursor()
    app_details = list()
    for app in app_list:
        # print(app)
        if (app[2] == 'NGUI'):
            # app_query = """SELECT P3Area, FioriID, AppName, AppType, ApplicationComponent, FormFactors, BusinessRoleName, PrimaryODataServiceName FROM FIORIBOM WHERE ReleaseName = '""" + release + """', SemanticObject = '""" + app[0] + """', SemanticAction = '""" + app[1] + """'"""
            app_query = "SELECT P3Area,\
                                PO,\
                                PortfolioCategory,\
                                FioriID,\
                                AppName,\
                                AppType,\
                                UITechnology,\
                                DeliveryModel,\
                                ApplicationComponent,\
                                AppDocumentationLink,\
                                FormFactors,\
                                BusinessRoleName,\
                                BusinessCatalogName,\
                                PrimaryODataServiceName,\
                                AdditionalODataServices,\
                                'Development:_Product_Owner',\
                                'Development:_Vice_President',\
                                'Development:_Component_Owner',\
                                'Development:_Delivery_Manager',\
                                'IMS:_Main_Manager_Responsible',\
                                'IMS:_VP_Director' FROM FIORIBOM WHERE ReleaseName = '%s' AND SemanticObject = '%s' AND SemanticAction = '%s'" % (release, app[0], app[1])
            # print(app_query)
            cur.execute(app_query)
            fetched_data = cur.fetchone()
            if fetched_data is not None:
                app_details.append(fetched_data)
            # print(app_details)
    con.commit()
    con.close()
    # print(app_details)
    return app_details
# channel =   SelectField('Channel',choices=a,)

# cur = con.cursor()
# cur.execute("DROP TABLE FIORIBOM")

# DELETE ALL ENTRIES FROM TABLE
# cur = con.cursor()
# cur.execute("DELETE FROM FIORIBOM")
# rows = cur.fetchall()
# print(len(rows))

# cur = con.cursor()
# cur.execute("SELECT * FROM FIORIBOM WHERE ReleaseName = 'S/4HANA 1511'")
# rows = cur.fetchall()
# print(len(rows))

# cur = con.cursor()
# cur.execute("SELECT AppName, AppType FROM FIORIBOM WHERE ReleaseName = 'S/4HANA 1511' AND SemanticObject = 'Project' AND SemanticAction = 'analyzePlanActual'")
# rows = cur.fetchall()
# for row in rows:
#     print(row[0])
#     print(row[1])

# AppName = cur.fetchone()[0]
# print(AppName)
# AppType = cur.fetchone()[1]
# print(AppType)

# if cur.getCount() > 0:
#     cur.moveToFirst()
#     AppName = cur.getString(cur.getColumnIndex("AppName"));
# print(AppName)

con.commit()
con.close()


# -*- coding: utf-8 -*-
"""
Created on Sat Jun 25 18:00:23 2016

@author: limeng

"""

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
from matplotlib.colors import LinearSegmentedColormap

DMR_STATE = 'output/dmr.state'
DMR_PARAMETER = 'output/dmr.parameters'
DMR_FEATURES = 'output/features.txt'

TOPIC_K = 8

# 读出每个词对应的主题内容
# {doc: {word : topic}}
word_topic_mapping = dict()
with open(DMR_STATE, 'r') as f:
    next(f)
    for line in f:
        record = line.split()
        idx = record[0]
        if idx not in word_topic_mapping:
            word_topic_mapping[idx] = dict()
        word_topic_mapping[idx][record[-2]] = int(record[-1])

print('Extract topic for each word ...')

# 统计不同位置主题出现的频率
# {position : {topic : cnt}}
position_topic_mapping = dict()
with open(DMR_FEATURES, 'r') as f:
    for idx, line in enumerate(f):
        position = line.split()[0]
        if position not in position_topic_mapping:
            topic_list = np.zeros(TOPIC_K)

        # 对于每个文档，计算主题分布，并累加到对应的位置上
        parr = np.zeros(TOPIC_K)
        if str(idx) not in word_topic_mapping:
            continue
        for word, word_topic in word_topic_mapping[str(idx)].items():
            parr[word_topic] += 1
        parr = parr / np.sum(parr)
        topic_list = topic_list + parr
        position_topic_mapping[position] = topic_list

print('Topic for each position ...')

# 归一化
tmp_map = dict()
for loc, topic in position_topic_mapping.items():
    tmp_map[loc] = topic / np.sum(topic)

position_topic_mapping = tmp_map
print('Normalization ...')

# 对于所有的地理位置，以中心地点为标记绘制指定主题的热图
for topic_num in range(TOPIC_K):
    # m  = Basemap(projection='ortho',lon_0=0,lat_0=0,resolution='l',\
    #             llcrnrx=-1000*1000,llcrnry=-1000*1000,
    #             urcrnrx=+1150*1000,urcrnry=+1700*1000)

    print("Plotting for topic: ", topic_num)
    m = Basemap(llcrnrlon=-180,llcrnrlat=-80,
                urcrnrlon=180,urcrnrlat=80,projection='mill')
    m.drawcoastlines()
    m.drawcountries()
    # m.drawstates()

    lon_bins = np.linspace(-180, 180, 72) # 36 bins
    lat_bins = np.linspace(-80, 80, 36) # 72 bins
    density = np.zeros([36, 72])

    for x, y in np.ndindex(density.shape):
        loc_str = 'lbloc_' + str(x) + '_' + str(y)
        if loc_str not in position_topic_mapping:
            continue
        density[x][y] = position_topic_mapping[loc_str][topic_num]

    lon_bins_2d, lat_bins_2d = np.meshgrid(lon_bins, lat_bins)
    xs, ys = m(lon_bins_2d, lat_bins_2d) # will be plotted using pcolormesh

    cdict = {'red':  ( (0.0,  1.0,  1.0),
                       (1.0,  0.9,  1.0) ),
             'green':( (0.0,  1.0,  1.0),
                       (1.0,  0.03, 0.0) ),
             'blue': ( (0.0,  1.0,  1.0),
                       (1.0,  0.16, 0.0) ) }
    custom_map = LinearSegmentedColormap('custom_map', cdict)
    plt.register_cmap(cmap=custom_map)

    plt.pcolormesh(xs, ys, density, cmap="custom_map")
    plt.show()
    # xs.shape = (72, 36)
    # ys.shape = (72, 36)


#!/usr/bin/python

import requests
from pandas import DataFrame

def scrape_batteries(working_ion, filter_property="average_voltage"):
    print("Scraping batteries from battery-explorer")
    print("Working ion: {}".format(working_ion))
    print("Filtering property: {}".format(filter_property))

    batteries = {}

    # Scrape over the full range of average voltage in the materials database
    for i in range(0, 150):
        url = 'https://www.materialsproject.org/batteries/search?query={{"working_ion":{{"$in":["{ion}"]}},"average_voltage":{{"$gte":{0},"$lte":{1}}}}}' \
            .format(-2.1 + i / 10.0, -1.9 + i / 10.0, ion=working_ion, property=filter_property)

        response = requests.get(url)
        response_batteries_list = response.json()

        assert response.status_code == 200
        assert isinstance(response_batteries_list, list)
        assert not isinstance(response_batteries_list, dict)

        for battery in response_batteries_list:
            batteries[battery["battid"]] = battery

        print('Found {} batteries and {} unique batteries so far. Searches run: {}' \
              .format(len(batteries), len(set(batteries)), i))

    return batteries

def fetch_battery_from_api(battery, apikey):
    print("Scraping battery {}".format(battery["battid"]))

    api_key_header = {"X-API-KEY": apikey}
    url = "https://www.materialsproject.org/rest/v2/battery/{}".format(battery["battid"])
    battery_get = requests.get(url, headers=api_key_header)
    response_obj = battery_get.json()

    assert battery_get.status_code == 200
    assert response_obj["valid_response"] == True

    battery_data = response_obj["response"][0]

    if len(battery_data["adj_pairs"]) > 2:
        print("Found a battery with too many materials, write code to handle it. {} material pairs found."
              .format(len(battery_data["adj_pairs"])))
        return None

    return (battery_data["battid"], battery_data)

def compose_battery_data(battery):
    print(battery)
    return [battery["battid"],
            battery["adj_pairs"][0]['id_discharge'],
            battery["adj_pairs"][0]['id_charge'],
            battery['reduced_cell_formula'],
            battery['type'],
            #battery['spacegroup']['symbol'],
            battery['average_voltage'],
            battery['capacity_grav'],
            battery['capacity_vol'],
            battery['energy_grav'],
            battery['energy_vol'],
            battery["adj_pairs"][0]['stability_charge'],
            battery["adj_pairs"][0]['stability_discharge']]

def scrape_battery_data_to_csv(working_ion, output_filename, apikey):
    print("Fetching batteries from Materials Project")

    batteries_list = scrape_batteries(working_ion=working_ion)

    print("Found {} batteries with working ion {}".format(len(batteries_list.values()), working_ion))

    battery_materials = dict(filter(
        lambda element: element != None,
        [fetch_battery_from_api(battery, apikey) for battery in batteries_list.values()]))

    print("Fetched {} battery data objects from rest api".format(len(battery_materials.values())))

    data_columns = ['Battid',
                    'Discharged_ID',
                    'Charged_ID',
                    'Reduced_Cell_Formula',
                    'Type',
                    #'Spacegroup',
                    'Average_Voltage',
                    'Capacity_Grav',
                    'Capacity_Vol',
                    'Specific_E_Wh/kg',
                    'E Density Wh/l',
                    'Stability Charge',
                    'Stability Discharge']

    battery_data = [compose_battery_data(battery) for battery in list(battery_materials.values())]

    df = DataFrame.from_records(battery_data, columns=data_columns)

    print("Exporting selected battery data to file: {}".format(output_filename))
    df.to_csv(output_filename, index=False)

# Example for scraping all Mg-batteries and exporting them to mg_batteries.csv
#scrape_battery_data_to_csv("Mg", "mg_batteries.csv", "GKDHNwKre8uiowqhPh")
# atomtypes = ['Y','Na','Al','Zn']

# for i in atomtypes:
#     filename = i + '_batteries.csv'
#     print('i: ', i )
#     print('Filename: ', filename)
#     scrape_battery_data_to_csv(i, filename, "GKDHNwKre8uiowqhPh")

scrape_battery_data_to_csv("Mg", "Mg_batteries.csv", "GKDHNwKre8uiowqhPh")


from custom.model_development import display_results, getstats_fromstream
from custom.model_development import load_datasets, make_datasets
import matplotlib.pyplot as plt
import numpy as np
from pathlib import Path
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score, precision_score
from sklearn.metrics import recall_score, make_scorer
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from sklearn.model_selection import RepeatedStratifiedKFold, train_test_split
import xgboost as xgb
from xgboost.sklearn import XGBClassifier

#set types and number of examples from each class in each set
set_types = ['training', 'test']
num_ex = [260, 64]
datasets_dict = dict(zip(set_types, num_ex))

class_names = ['empty', 'cloud']
seed = 52

params_file = Path('/home/bob/development/experiment_data/2018/'
                   'cloudparams_s20.csv')

dir_training_images = ('/home/bob/development/atomic_cloud_training_data/'
                       'training_data')

path_images_left = Path(dir_training_images, 'left_clouds/seed_13')
path_images_right = Path(dir_training_images, 'right_clouds/seed_31')

dir_model_main = ('/home/bob/development/ml_algorithms/atomic_cloud/'
                  'gbc_binary_classify/models')

path_model_id = Path(dir_model_main, f'num_ex_{num_ex[0]}_seed_{seed}')

#Tune using training set of left-side clouds.
make_datasets(class_names, datasets_dict, path_images_left, path_model_id, seed)

X_L, X_test_L, y_L, y_test_L = load_datasets(path_model_id,
                                             path_images_left)

_, scale_L, _ = getstats_fromstream(path_model_id, path_images_left)
scale_rnd_L = np.around(scale_L)
X_scl_L = X_L / scale_rnd_L
X_test_scl_L = X_test_L / scale_rnd_L

X_train_scl_L, X_eval_scl_L, y_train_L, y_eval_L = train_test_split(
    X_scl_L, y_L, train_size=0.8, random_state=123)

#Tune using training set of right-side clouds.
#make_datasets(class_names, datasets_dict, path_images_right, path_model_id, seed)
#
#X_R, X_test_R, y_train_R, y_R = load_datasets(path_model_id,
#                                              path_images_right)
#
#_, scale_R, _ = getstats_fromstream(path_model_id, path_images_right)
#scale_rnd_R = np.around(scale_R)
#X_scl_R = X_R / scale_rnd_R
#X_test_scl_R = X_test_R / scale_rnd_R

#Tuning stages.

#Stage 1 - Search for learning_rate
#var_params = {'colsample_bytree': 0.15,
#              'gamma': 0,
#              'max_depth': 10,
#              'min_child_weight': 3,
#              'reg_alpha': 0,
#              'reg_lambda': 0,
#              'subsample': 1}
#learning_rates = [0.0001, 0.001, 0.01, 0.1, 1]
#params_grid = [{'learning_rate': learning_rates}]

#Stage 2 - Search for max_depth and min_child_weight.
#var_params = {'colsample_bytree': 0.15,
#              'gamma': 0,
#              'learning_rate': 0.001,
#              'reg_alpha': 0,
#              'reg_lambda': 0,
#              'subsample': 1}
##max_depth = [3, 7, 11, 15, 19, 24]
##min_child_weight = [1, 3, 5, 7, 9]
#max_depth = [5, 20, 50, 75, 100]
#min_child_weight = [3, 10, 20, 30]
#params_grid = [{'max_depth': max_depth,
#                'min_child_weight': min_child_weight}]
#param_dists = params_grid
#n_iter = 18

#Stage 3 - Search for colsample_bytree and subsample.
#var_params = {'gamma': 0,
#              'learning_rate': 0.001,
#              'max_depth': 9,
#              'min_child_weight': 5,
#              'reg_alpha': 0,
#              'reg_lambda': 0}
#colsample_bytree = [0.02, 0.05, 0.1, 0.2, 0.5]
#colsample_bytree = [i/10 + 0.05 for i in range(1, 5)]
#subsample = [i / 10 for i in range(2, 11, 2)]
#params_grid = [{'colsample_bytree': colsample_bytree,
#                'subsample': subsample}]
#param_dists = params_grid
#n_iter = 25

#Stage 4 - Search for gamma, reg_alpha and reg_lambda.
#var_params = {'colsample_bytree': 0.35,
#              'learning_rate': 0.001,
#              'max_depth': 9,
#              'min_child_weight': 5,
#              'subsample': 0.4}
#gamma = [0.001, 0.01, 0.1, 1, 10]
#reg_alpha = [0.001, 0.01, 0.1, 1, 10]
#reg_lambda = [0.001, 0.01, 0.1, 1, 10]
#gamma = [0.01, 0.1, 1]
#reg_alpha = [10, 100, 1000]
#reg_lambda = [10, 100, 1000]
#params_grid = [{'gamma': gamma, 'reg_alpha': reg_alpha, 'reg_lambda': [0]},
#               {'gamma': gamma, 'reg_alpha': [0], 'reg_lambda': reg_lambda}]
#gamma = [0.1]
#reg_lambda = [i * 10 for i in range(1, 11)]
#params_grid = [{'gamma': gamma, 'reg_alpha': [0], 'reg_lambda': reg_lambda}]
#param_dists = params_grid
#n_iter = 25

#Stage 5 - Check learning_rate again.
var_params = {'colsample_bytree': 0.35,
              'gamma': 0.1,
              'max_depth': 9,
              'min_child_weight': 5,
              'reg_alpha': 0,
              'reg_lambda': 100,
              'subsample': 0.4}
learning_rates = [i / 10000 for i in range(5, 16)]
params_grid = [{'learning_rate': learning_rates}]

early_stopping_rounds = 20
eval_set = [(X_eval_scl_L, y_eval_L)]
fixed_params = {'n_estimators': 500,
                'random_state': 10,
                'verbosity': 0}

estimator = XGBClassifier(**fixed_params, **var_params)

crossval = RepeatedStratifiedKFold(n_splits=6, n_repeats=3, random_state=3)

my_prec_scorer = make_scorer(precision_score, pos_label=class_names[0])
my_recall_scorer = make_scorer(recall_score, pos_label=class_names[0])

metrics = {'accuracy': make_scorer(accuracy_score),
           'precision': my_prec_scorer,
           'recall': my_recall_scorer}

print(f'# Tuning hyper-parameters')
print()

search_params = {'estimator': estimator,
                 'cv': crossval,
                 'scoring': metrics,
                 'refit': False}

model_search = GridSearchCV(**search_params,
                            param_grid=params_grid)

#model_search = RandomizedSearchCV(**search_params,
#                                  param_distributions=param_dists,
#                                  n_iter=n_iter)

#Fit model with left-side clouds.
model_search.fit(X_train_scl_L, y_train_L, eval_set=eval_set,
                 eval_metric='auc', early_stopping_rounds=20,
                 verbose=False)

#Fit model with right-side clouds.
#model_search.fit(X_train_scl_R, y_train_R, eval_set=eval_set,
#                 eval_metric='auc', early_stopping_rounds=20,
#                 verbose=False)

#print("Grid scores by metric on development set:")
#print()

results = model_search.cv_results_
display_results(results, metrics)

# Run this to check performance on the test set.
#model_L_params = {'colsample_bytree': 0.35,
#                  'gamma': 0.1,
#                  'learning_rate': 0.001,
#                  'max_depth': 9,
#                  'min_child_weight': 5,
#                  'n_estimators': 500,
#                  'random_state': 10,
#                  'reg_alpha': 0,
#                  'reg_lambda': 100,
#                  'subsample': 0.4,
#                  'random_state': 10,
#                  'verbosity': 0}
#model_L = XGBClassifier(**model_L_params)
#model_L.fit(X_train_scl_L, y_train_L, eval_set=eval_set,
#            eval_metric='auc', early_stopping_rounds=20,
#            verbose=False)
#print("Detailed classification report:")
#print()
#y_true, y_pred = y_test_L, model_search.predict(X_test_scl_L)
#print(classification_report(y_true, y_pred))
#print()


"""
#1. whitespace before '('
def f ():
    print("Hello")
f()

#2. missing whitespace arount operator
print(5+ 4)

#3. missing whitespace after ','
print([2,3,4])

#4. unexpected spaces around keyword / parameter equals
def f(arg=0):
    return 2**arg
f(arg = 2)

#5. expected 2 blank lines, found 1
def f1():
    return "Hello"

def f2():
    return "World"

#6. multiple statements on one line (color)
if True: print("hello")

#7. multiple statements on one line (semicolon)
print("hello"); print("world")

#8. comparison to None should be 'if cond is None:'
def f(x):
    if x % 2 == 0:
        return True

r = f(x)

if r == None:
    print("odd")

#9. comparison to True should be 'if cond is True:' or 'if cond:'
def f(x):
    if x % 2 == 0:
        return True

r = f(x)

if r == True:
    print("even")

"""
from printermodule import *

try:
	hello() # hello
	world() # world
except:
	pass

import helloworldpkg.hello
import helloworldpkg.world

helloworldpkg.hello.printh()
helloworldpkg.world.printw()

import logging
import traceback

logging.basicConfig(filename='logfile.log', filemode='w', level=logging.INFO)
logging.raiseExceptions = True

def div(a, b):
	try:
		return a/b, None
	except Exception as e:
		return e, traceback.format_exc()

def run_with_log(func, a, b):
	out, trace = div(a,b)
	if trace is not None:
		return logging.exception(f'{out}\n{trace}')

run_with_log(div, 6, 3)
run_with_log(div, 6, 0)
run_with_log(div, "5", 2)


# -*- coding: utf-8 -*-
"""
Compute least-cost distance to port
"""
from concurrent.futures import as_completed
import geopandas as gpd
import logging
import numpy as np
import os
import pandas as pd
from scipy.spatial import cKDTree
from skimage.graph import MCP_Geometric
import time
from warnings import warn

from reV.handlers.exclusions import ExclusionLayers
from reVX.utilities import ExclusionsConverter
from reVX.utilities.utilities import log_versions, coordinate_distance
from rex.utilities.execution import SpawnProcessPool
from rex.utilities.loggers import log_mem
from rex.utilities.utilities import (check_res_file, get_lat_lon_cols,
                                     parse_table, row_col_indices)

logger = logging.getLogger(__name__)

class DistanceToPorts:
    """
    Class to compute the least cost distance from offshore pixels to port
    locations. The distance to coast exclusion layer will be used to calculate
    least cost paths around land masses and other such obstructions. Produces
    the least cost distance from each offshore pixel to the nearest port. If
    a distance to port layer already exists it can be updated with the least
    cost distance to new ports.

    NOTE: Computing the least cost distance is both memory and computationally
    intensive! One EAGLE a bigmem node was needed to run in parallel and a
    medium (178GB) memory node is needed to run in serial.
    """
    def __init__(self, ports, excl_fpath, input_dist_layer='dist_to_coast'):
        """
        Parameters
        ----------
        ports : str
            Path to shape, csv, or json file containing ports to compute
            least cost distance to
        excl_fpath: str
            Path to exclusions .h5 file with distance to coast layer
        input_dist_layer : str, optional
            Exclusions layer name with distance to coast,
            by default 'dist_to_coast'
        """
        log_versions(logger)
        self._input_dist_layer = input_dist_layer
        self._ports_fpath = ports
        self._ports, self._cost_arr, self._profile = self._parse_ports(
            ports, excl_fpath, input_dist_layer=input_dist_layer)
        log_mem(logger)

    def __repr__(self):
        msg = "{} from {}".format(self.__class__.__name__, self.ports)

        return msg

    @property
    def ports(self):
        """
        DataFrame of port locations

        Returns
        -------
        DataFrame
        """
        return self._ports

    @property
    def cost_arr(self):
        """
        Cost array, used to compute least cost distance to ports
        Offshore pixels have a value of 90 (pixel size), onshore pixels have a
        value of -1

        Returns
        -------
        ndarray
        """
        return self._cost_arr

    @classmethod
    def _parse_lat_lons(cls, excl_fpath, input_dist_layer='dist_to_coast'):
        """
        Parse cost array, profile, and latitude and longitude coordinates

        Parameters
        ----------
        excl_fpath: str
            Path to exclusions .h5 file with distance to coast layer
        input_dist_layer : str, optional
            Exclusions layer name with distance to coast values,
            by default 'dist_to_coast'

        Returns
        -------
        lat_lon : pandas.DataFrame
            Table mapping the offshore pixel coordiantes (lat, lon) to thier
            position (row, col) within the distance to shore array/layer/raster
        cost_arr : ndarray
            Cost array with offshore pixels set to 90 (pixel width) and
            onshore pixels set to -1.
        profile : dict
            Profile (transform, crs, etc.) of arr raster
        """
        hsds = check_res_file(excl_fpath)[1]
        with ExclusionLayers(excl_fpath, hsds=hsds) as tif:
            profile = tif.profile
            cost_arr = tif[input_dist_layer]
            lat = tif['latitude'].astype(np.float32)
            lon = tif['longitude'].astype(np.float32)

        mask = cost_arr > 0
        cost_arr = np.where(mask, 90, -1).astype(np.int8)

        mask = mask.ravel()
        ids = np.arange(lat.size, dtype=np.uint32)[mask]
        row_len = lat.shape[1]
        lat = lat.ravel()[mask]
        lon = lon.ravel()[mask]
        del mask

        rows, cols = row_col_indices(ids, row_len)
        del ids
        del row_len

        lat_lon = pd.DataFrame({'latitude': lat,
                                'longitude': lon,
                                'row': rows,
                                'col': cols})

        return lat_lon, cost_arr, profile

    @staticmethod
    def _check_ports_coords(port_coords, lat_lon):
        """
        Check port coordinates to make sure they are within the resource domain

        Parameters
        ----------
        port_coords : ndarray
            nx2 array of (lat, lon) port coordinates
        lat_lon : ndarray
            nx2 array of (lat, lon) offshore coordinates

        Returns
        -------
        check : ndarray
            Boolean array indicating which ports are outside (False) the
            resource domain.
        """
        lat_min, lat_max = np.sort(lat_lon[:, 0])[[0, -1]]
        lon_min, lon_max = np.sort(lat_lon[:, 1])[[0, -1]]

        lat = port_coords[:, 0]
        check = lat < lat_min
        check |= lat > lat_max

        lon = port_coords[:, 1]
        check |= lon < lon_min
        check |= lon > lon_max

        if any(check):
            bad_coords = port_coords[check]
            msg = ("Ports with coordinates ({}) are outsides of the "
                   "resource domain: (({}, {}), ({}, {})) and will not be used"
                   " to to compute the least cost distance!"
                   .format(bad_coords, lat_min, lon_min, lat_max, lon_max))
            logger.warning(msg)
            warn(msg)

        return ~check

    @staticmethod
    def _create_port_names(ports):
        """
        Create port names from "PORT_NAME" and "STATE", confirm all names are
        unique

        Parameters
        ----------
        ports : geopandas.GeoDataFrame | pandas.DataFrame
            DataFrame of port locations and their mapping to the offshore
            pixels for least cost distance computation

        Returns
        -------
        ports : geopandas.GeoDataFrame | pandas.DataFrame
            DataFrame of port locations and their mapping to the offshore
            pixels for least cost distance computation which a unique port
            name added
        """
        name = None
        state = None
        for c in ports.columns:
            if c.lower() == 'port_name':
                if name is not None:
                    msg = ('Multiple potential "port names" were found: '
                           '({}, {})!'.format(name, c))
                    logger.error(msg)
                    raise RuntimeError(msg)
                else:
                    name = c.lower()

            if 'state' in c.lower():
                if state is not None:
                    msg = ('Multiple potential "states" were found: '
                           '({}, {})!'.format(state, c))
                    logger.error(msg)
                    raise RuntimeError(msg)

            if state is not None and name is not None:
                break

        ports['name'] = (ports['PORT_NAME'].astype(str) + '_'
                         + ports['ps_STATE'].astype(str))
        counts = ports['name'].value_counts()
        if np.any(counts > 1):
            msg = ('Ports must have unique names! The following duplicate '
                   'names were provided: {}'.format(counts[counts > 1]))
            logger.error(msg)
            raise RuntimeError(msg)

        return ports

    @classmethod
    def _parse_ports(cls, ports, excl_fpath, input_dist_layer='dist_to_coast'):
        """
        Load ports for disc. Can be provided as a shape, csv, or json file.
        In all cases the ports latitude and longitude coordinates must be
        provided. Map the ports locations to the nearest offshore pixel in the
        distance to coast layer/array/raster. Compute the distance from the
        ports actual position to the nearest offshore pixel.

        Parameters
        ----------
        ports : str
            Path to shape, csv, or json file containing ports to compute
            least cost distance to
        excl_fpath: str
            Path to exclusions .h5 file with distance to coast layer
        input_dist_layer : str, optional
            Exclusions layer name with distance to coast values,
            by default 'dist_to_coast'

        Returns
        -------
        ports : geopandas.GeoDataFrame | pandas.DataFrame
            DataFrame of port locations and their mapping to the offshore
            pixels for least cost distance computation
        cost_arr : ndarray
            Cost array with offshore pixels set to 90 (pixel width) and
            onshore pixels set to -1.
        profile : dict
            Profile (transform, crs, etc.) of arr raster
        """
        if ports.endswith('.shp'):
            ports = gpd.read_file(ports, ignore_geometry=True)
        else:
            ports = parse_table(ports)

        pixels, cost_arr, profile = \
            cls._parse_lat_lons(excl_fpath, input_dist_layer=input_dist_layer)
        lat_lon_cols = get_lat_lon_cols(pixels)
        pixel_coords = pixels[lat_lon_cols].values

        tree = cKDTree(pixel_coords)  # pylint: disable=not-callable

        lat_lon_cols = get_lat_lon_cols(ports)
        port_coords = ports[lat_lon_cols].values.astype('float32')
        # remove ports that are outside pixel bounds
        mask = cls._check_ports_coords(port_coords, pixel_coords)
        port_coords = port_coords[mask]
        ports = ports.loc[mask]
        _, idx = tree.query(port_coords)

        pixels = pixels.iloc[idx]
        pixel_coords = pixel_coords[idx]

        ports['row'] = pixels['row'].values
        ports['col'] = pixels['col'].values
        ports['dist_to_pixel'] = coordinate_distance(port_coords, pixel_coords)

        ports = cls._create_port_names(ports)

        return ports, cost_arr, profile

    @classmethod
    def lc_dist_to_port(cls, cost_arr, port_idx, port_dist,
                        geotiff=None, profile=None):
        """
        Compute the least cost dist from the port coordinates to all
        offshore coordinates in km

        Parameters
        ----------
        cost_arr : ndarray
            Cost array with offshore pixels set to 90 (pixel width) and
            onshore pixels set to -1.
        port_idx : list | tuple | ndarray
            Port (row, col) index, used as starting point for least cost
            distance
        port_dist : float
            Distance from port to pixel that corresponds to port_idx in meters
        geotiff : str, optional
            Output geotiff file path to save least cost distance to ports,
            by default None
        profile : dict, optional
            Profile (transform, crs, etc.) of cost array raster, needed to
            write distance to ports array to geotiff, by default None

        Returns
        -------
        lc_dist : ndarray, optional
            Least cost distance from port to all offshore pixels in km
        """
        try:
            ts = time.time()
            logger.debug('Port that is {:.4f} km from nearest offshore pixel '
                         '{}.'.format(port_dist, port_idx))
            if not isinstance(port_idx, np.ndarray):
                port_idx = np.array(port_idx)

            if len(port_idx) == 2:
                port_idx = np.expand_dims(port_idx, 0)

            mcp = MCP_Geometric(cost_arr)
            lc_dist = mcp.find_costs(starts=port_idx)[0].astype('float32')
            lc_dist /= 1000
            lc_dist += port_dist

            lc_dist[cost_arr == -1] = -1

            tt = (time.time() - ts) / 60
            logger.debug('- Least cost distance computed in {:.4f} minutes'
                         .format(tt))
            if geotiff is not None:
                logger.debug('Saving least cost distance to port to '
                             f'{geotiff}')
                msg = ('Profile is needed to write least cost distance to '
                       'ports to {}!'.format(geotiff))
                assert profile is not None, msg
                ExclusionsConverter.write_geotiff(geotiff, profile, lc_dist)
            else:
                return lc_dist
        except Exception:
            logger.exception('- Error computing least cost distance to port!')

    def distance_to_ports(self, out_dir, max_workers=1, replace=False):
        """
        Compute the least cost distance from each offshore pixel to the nearest
        port in km

        Parameters
        ----------
        out_dir : str
            Directory to save distance to port geotiffs too.
        max_workers : int, optional
            Number of workers to use for setback computation, if 1 run in
            serial, if > 1 run in parallel with that many workers, if None
            run in parallel on all available cores, by default 1
        replace : bool, optional
            Flag to replace existing ports geotiffs, by default False
        """
        if max_workers is None:
            max_workers = os.cpu_count()

        f_name = name = os.path.basename(self._ports_fpath).split('.')[0]
        if max_workers > 1:
            logger.info('Computing least cost distance to ports in parallel '
                        'using {} workers'.format(max_workers))
            loggers = [__name__, 'reVX']
            with SpawnProcessPool(max_workers=max_workers,
                                  loggers=loggers) as exe:
                futures = []
                for _, port in self.ports.iterrows():
                    name = port['name'].replace(' ', '_')
                    geotiff = '{}-{}.tif'.format(f_name, name)
                    geotiff = os.path.join(out_dir, geotiff)
                    if os.path.exists(geotiff) and not replace:
                        msg = ('{} already exists and will be skipped! To '
                               'replace it set "replace=True"'
                               .format(geotiff))
                        logger.warning(msg)
                        warn(msg)
                    else:
                        logger.debug('Computing least cost distance to {}'
                                     .format(name))
                        port_idx = port[['row', 'col']].values
                        port_dist = port['dist_to_pixel']
                        future = exe.submit(
                            self.lc_dist_to_port, self.cost_arr,
                            port_idx, port_dist, geotiff=geotiff,
                            profile=self._profile)
                        futures.append(future)

                for i, future in enumerate(as_completed(futures)):
                    logger.debug('Computed least cost distance for {} of {} '
                                 'ports'.format((i + 1), len(futures)))
                    log_mem(logger)
        else:
            logger.info('Computing least cost distance to ports in serial')
            for i, port in self.ports.iterrows():
                name = port['name'].replace(' ', '_')
                geotiff = '{}-{}.tif'.format(f_name, name)
                geotiff = os.path.join(out_dir, geotiff)
                if os.path.exists(geotiff) and not replace:
                    msg = ('{} already exists and will be skipped! To '
                           'replace it set "replace=True"'
                           .format(geotiff))
                    logger.warning(msg)
                    warn(msg)
                else:
                    logger.debug('Computing least cost distance to {}'
                                 .format(name))
                    port_idx = port[['row', 'col']].values
                    port_dist = port['dist_to_pixel']
                    self.lc_dist_to_port(
                        self.cost_arr, port_idx, port_dist, geotiff=geotiff,
                        profile=self._profile)
                    logger.debug('Computed least cost distance for {} of {} '
                                 'ports'.format((i + 1), len(self.ports)))
                    log_mem(logger)

    @classmethod
    def run(cls, ports, excl_fpath, out_dir, input_dist_layer='dist_to_coast',
            max_workers=1, replace=False):
        """
        Compute the least cost distance from offshore pixels to port
        locations in km. The distance to coast exclusion layer will be used to
        calculate least cost paths around land masses and other such
        obstructions. Produces the least cost distance from each offshore
        pixel to the nearest port. If a distance to port layer already exists
        it can be updated with the least cost distance to new ports.

        Parameters
        ----------
        ports : str
            Path to shape, csv, or json file containing ports to compute
            least cost distance to
        excl_fpath: str
            Path to exclusions .h5 file with distance to coast layer. Will also
            be the file into which the least cost distance to port is saved.
        out_dir : str
            Directory to save distance to port geotiffs too.
        input_dist_layer : str, optional
            Exclusions layer with distance to coast values,
            by default 'dist_to_coast'
        max_workers : int, optional
            Number of workers to use for setback computation, if 1 run in
            serial, if > 1 run in parallel with that many workers, if None
            run in parallel on all available cores, by default 1
        replace : bool, optional
            Flag to replace existing ports geotiffs, by default False
        """
        logger.info('Computing least cost distance to ports in {}'
                    .format(ports))
        dtp = cls(ports, excl_fpath, input_dist_layer=input_dist_layer)

        dtp.distance_to_ports(out_dir, max_workers=max_workers,
                              replace=replace)


#!/usr/bin/env python3
# coding=utf-8
# 抓取天气预报，实现用时访问的策略，即用迭代器，而不是简单的for,避免了占用大量内存，显示慢
# for循环的本质就是先生成一个迭代对象，然后不断地调用next()
# 下面的例子网站，点击后会自动下载一个文件，Json格式，用r.json()转换为python的字典格式
# 要求：从网站抓取需要的城市气温信息，并依次显示；
# 问题：如果一次抓取所有城市天气再显示，显示第一个城市气温时，有很高的延时，并且浪费存储空间；
# 期望从‘用时访问’的策略，并且能把所有城市气温装到一个对象里，可用for语句进行迭代。如何实现？
# 方案：
# 把一个类作为一个迭代器使用  需要在类中实现两个方法 __iter__() 与 __next__() 。
#  __iter__() 方法返回一个特殊的迭代器对象， 这个迭代器对象实现了 __next__() 方法并通过 StopIteration 异常标识迭代的完成。
#  __next__() 方法（Python 2 里是 next()）会返回下一个迭代器对象。

# 这个例子可以取代34那个例子，我参考网上例子修改的
import requests

class WeatherIterator(object):
    def __init__(self, cities):
        self.cities = cities
        self.index = 0

    def __iter__(self):
        # 报错：Can't instantiate abstract class WeatherIterator with abstract methods __next__，
        # 即：不能用抽象的方法实例化抽象类WeatherIterator ——— 将next方法改为__next__()方法；
        return self

    def get_weather(self, city):
        r = requests.get(u'http://wthrcdn.etouch.cn/weather_mini?city=' + city)
        data = r.json()['data']['forecast'][0]
        return '%s: %s, %s' % (city, data['low'], data['high'])

    def __next__(self):
        if self.index == len(self.cities):   # 迭代的结束条件必须有，否则就是死循环
            raise StopIteration
        # 正常迭代情况，即每次迭代出一个城市的气温信息；self.cities[self.index]，得到需要的城市名字
        city = self.cities[self.index]
        self.index += 1
        return self.get_weather(city)

if __name__ == '__main__':
    for x in WeatherIterator([u'北京', u'上海', u'广州', u'长春']):
        print(x)


'''
Created on 2018年6月19日

@author: admin
'''
import unittest
from PO import MenuPage
from selenium import webdriver
from appium import webdriver
from Public import driver
class Test(unittest.TestCase):

    def setUp(self):
        self.driver=webdriver.Remote('http://127.0.0.1:4723/wd/hub',driver.MyDriver.desired_caps)

    def tearDown(self):
        self.driver.quit()

    def test_replay(self):
        menu=MenuPage.menu_page(self.driver)
        menu.click_menbtn()
        menu.click_replay()
        if menu.get_replay_title()=='回放 ':
            print("成功跳转回放页 :"+menu.get_replay_title())

if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main()


import random
from typing import List

class Solution:

    def __init__(self, w: List[int]):
        self.arr = [0]
        for item in w:
            self.arr.append(self.arr[-1] + item)

    def pickIndex(self) -> int:
        x = random.random() * self.arr[-1]
        p1, p2 = 0, len(self.arr) - 1

        while p1 <= p2:
            mid = (p2 - p1) // 2 + p1
            if self.arr[mid] <= x < self.arr[mid + 1]:
                return mid
            elif self.arr[mid] > x:
                p2 = mid - 1
            else:
                p1 = mid + 1

        return -1

if __name__ == "__main__":
    operations = ["Solution", "pickIndex", "pickIndex", "pickIndex", "pickIndex", "pickIndex"]
    values = [[[1, 3]], [1], [1], [1], [0], [1]]
    random.seed(10)  # I've picked this randomly
    sol = None
    for op, v in zip(operations, values):
        if op == "Solution":
            sol = Solution(v[0])
        elif op == "pickIndex":
            print(sol.pickIndex(), '==', v[0])


import pandas as pd
import simplejson as json

def calculate(url):
    """
        This function outlined below  downloads and computes the aggregation of functional and non-functional water sources
        per community. The result is merged with the total number of functional water sources in all communities.
        Additionally, a ranking is based on the percentage of faulty water sources per community.
    """
    df = pd.read_json(url)
    df = df[['water_functioning', 'communities_villages']]
    df = df.groupby(['communities_villages', 'water_functioning']).size().unstack()
    df = df.fillna(0)

    df['total'] = df['yes'] + df['no']
    df['fault_percentage'] = df['no'] / df['total']
    df['ranking'] = df['fault_percentage'].rank(ascending=False)

    total_functional = df['yes'].sum()
    df = df.rename(columns={'yes': 'functional', 'no':'non_functional'})
    df = df[['functional', 'non_functional', 'total', 'ranking']]
    records = df.to_json(orient='index')
    result = '{"number_functional": %s, "communities": %s}' % (total_functional, records)
    return json.loads(result)

data = calculate('https://raw.githubusercontent.com/onaio/ona-tech/master/data/water_points.json')
print(data['number_functional'])
print(data['communities'])