"""
Functions to unpack Simrad EK60 .raw files
Modification from original source (mentioned below) included:
- python 3.6 compatibility
- strip off dependency on other mi-instrument functions
- unpack split-beam angle data
- unpack various additional variables
- support saving to netCDF file

Original source for unpacking power data part:
oceanobservatories/mi-instrument @https://github.com/oceanobservatories/mi-instrument
Authors: Ronald Ronquillo & Richard Han

"""

from collections import defaultdict
from struct import unpack_from, unpack
import numpy as np
import re
import os
from datetime import datetime as dt
from matplotlib.dates import date2num
import pytz
import echopype as ep

# Set constants for unpacking .raw files
BLOCK_SIZE = 1024*4             # Block size read in from binary file to search for token
LENGTH_SIZE = 4
DATAGRAM_HEADER_SIZE = 12
CONFIG_HEADER_SIZE = 516
CONFIG_TRANSDUCER_SIZE = 320

# set global regex expressions to find all sample, annotation and NMEA sentences
SAMPLE_REGEX = b'RAW\d{1}'
SAMPLE_MATCHER = re.compile(SAMPLE_REGEX, re.DOTALL)
FILENAME_REGEX = r'(?P<prefix>\S*)-D(?P<date>\d{1,})-T(?P<time>\d{1,})'
FILENAME_MATCHER = re.compile(FILENAME_REGEX, re.DOTALL)

# Reference time "seconds since 1900-01-01 00:00:00"
REF_TIME = date2num(dt(1900, 1, 1, 0, 0, 0))

WINDOWS_EPOCH = dt(1601, 1, 1)
NTP_EPOCH = dt(1900, 1, 1)
NTP_WINDOWS_DELTA = (NTP_EPOCH - WINDOWS_EPOCH).total_seconds()

# Numpy data type object for unpacking the Sample datagram including the header from binary *.raw
sample_dtype = np.dtype([('length1', 'i4'),  # 4 byte int (long)
                         # Datagram header
                         ('datagram_type', 'a4'),  # 4 byte string
                         ('low_date_time', 'u4'),  # 4 byte int (long)
                         ('high_date_time', 'u4'),  # 4 byte int (long)
                         # Sample datagram
                         ('channel_number', 'i2'),  # 2 byte int (short)
                         ('mode', 'i2'),  # 2 byte int (short): whether split-beam or single-beam
                         ('transducer_depth', 'f4'),  # 4 byte float
                         ('frequency', 'f4'),  # 4 byte float
                         ('transmit_power', 'f4'),  # 4 byte float
                         ('pulse_length', 'f4'),  # 4 byte float
                         ('bandwidth', 'f4'),  # 4 byte float
                         ('sample_interval', 'f4'),  # 4 byte float
                         ('sound_velocity', 'f4'),  # 4 byte float
                         ('absorption_coefficient', 'f4'),  # 4 byte float
                         ('heave', 'f4'),  # 4 byte float
                         ('roll', 'f4'),  # 4 byte float
                         ('pitch', 'f4'),  # 4 byte float
                         ('temperature', 'f4'),  # 4 byte float
                         ('trawl_upper_depth_valid', 'i2'),  # 2 byte int (short)
                         ('trawl_opening_valid', 'i2'),  # 2 byte int (short)
                         ('trawl_upper_depth', 'f4'),  # 4 byte float
                         ('trawl_opening', 'f4'),  # 4 byte float
                         ('offset', 'i4'),  # 4 byte int (long)
                         ('count', 'i4')])  # 4 byte int (long): number of items to unpack for power_data
sample_dtype = sample_dtype.newbyteorder('<')

power_dtype = np.dtype([('power_data', '<i2')])     # 2 byte int (short)

angle_dtype = np.dtype([('athwartship', '<i1'), ('alongship', '<i1')])     # 1 byte ints

def read_config_header(chunk):
    """
    Reads the EK60 raw data file configuration header information
    from the byte string passed in as a chunk
    @param chunk data chunk to read the config header from
    @return: configuration header
    """
    # setup unpack structure and field names
    field_names = ('survey_name', 'transect_name', 'sounder_name',
                   'version', 'transducer_count')
    fmt = '<128s128s128s30s98sl'

    # read in the values from the byte string chunk
    values = list(unpack(fmt, chunk))
    values.pop(4)  # drop the spare field

    # strip the trailing zero byte padding from the strings
    # for i in [0, 1, 2, 3]:
    for i in range(4):
        values[i] = values[i].strip(b'\x00')

    # create the configuration header dictionary
    config_header = dict(zip(field_names, values))
    return config_header

def read_config_transducer(chunk):
    """
    Reads the EK60 raw data file configuration transducer information
    from the byte string passed in as a chunk
    @param chunk data chunk to read the configuration transducer information from
    @return: configuration transducer information
    """

    # setup unpack structure and field names
    field_names = ('channel_id', 'beam_type', 'frequency', 'gain',
                   'equiv_beam_angle', 'beam_width_alongship', 'beam_width_athwartship',
                   'angle_sensitivity_alongship', 'angle_sensitivity_athwartship',
                   'angle_offset_alongship', 'angle_offset_athwartship', 'pos_x', 'pos_y',
                   'pos_z', 'dir_x', 'dir_y', 'dir_z', 'pulse_length_table', 'gain_table',
                   'sa_correction_table', 'gpt_software_version')
    fmt = '<128sl15f5f8s5f8s5f8s16s28s'

    # read in the values from the byte string chunk
    values = list(unpack(fmt, chunk))

    # convert some of the values to arrays
    pulse_length_table = np.array(values[17:22])
    gain_table = np.array(values[23:28])
    sa_correction_table = np.array(values[29:34])

    # strip the trailing zero byte padding from the strings
    for i in [0, 35]:
        values[i] = values[i].strip(b'\x00')

    # put it back together, dropping the spare strings
    config_transducer = dict(zip(field_names[0:17], values[0:17]))
    config_transducer[field_names[17]] = pulse_length_table
    config_transducer[field_names[18]] = gain_table
    config_transducer[field_names[19]] = sa_correction_table
    config_transducer[field_names[20]] = values[35]
    return config_transducer

def read_header(filehandle):
    # Read binary file a block at a time
    raw = filehandle.read(BLOCK_SIZE)

    # Read the configuration datagram, output at the beginning of the file
    length1, = unpack_from('<l', raw)
    byte_cnt = LENGTH_SIZE

    # Configuration datagram header
    byte_cnt += DATAGRAM_HEADER_SIZE

    # Configuration: header
    config_header = read_config_header(raw[byte_cnt:byte_cnt+CONFIG_HEADER_SIZE])
    byte_cnt += CONFIG_HEADER_SIZE
    config_transducer = []
    for num_transducer in range(config_header['transducer_count']):
        config_transducer.append(read_config_transducer(raw[byte_cnt:byte_cnt+CONFIG_TRANSDUCER_SIZE]))
        byte_cnt += CONFIG_TRANSDUCER_SIZE

    # Compare length1 (from beginning of datagram) to length2 (from the end of datagram) to
    # the actual number of bytes read. A mismatch can indicate an invalid, corrupt, misaligned,
    # or missing configuration datagram or a reverse byte order binary data file.
    # A bad/missing configuration datagram header is a significant error.
    length2, = unpack_from('<l', raw, byte_cnt)
    if not (length1 == length2 == byte_cnt-LENGTH_SIZE):
        print('Possible file corruption or format incompatibility.')
#         raise InstrumentDataException(
#             "Length of configuration datagram and number of bytes read do not match: length1: %s"
#             ", length2: %s, byte_cnt: %s. Possible file corruption or format incompatibility." %
#             (length1, length2, byte_cnt+LENGTH_SIZE))
    byte_cnt += LENGTH_SIZE
    filehandle.seek(byte_cnt)
    return config_header, config_transducer

def windows_to_ntp(windows_time):
    """
    Convert a windows file timestamp into Network Time Protocol
    :param windows_time:  100ns since Windows time epoch
    :return:
    """
    return windows_time / 1e7 - NTP_WINDOWS_DELTA

def build_windows_time(high_word, low_word):
    """
    Generate Windows time value from high and low date times.

    :param high_word:  high word portion of the Windows datetime
    :param low_word:   low word portion of the Windows datetime
    :return:  time in 100ns since 1601/01/01 00:00:00 UTC
    """
    return (high_word << 32) + low_word

def process_sample(input_file, transducer_count):
    # log.trace('Processing one sample from input_file: %r', input_file)
    # print('Processing one sample from input_file')

    # Read and unpack the Sample Datagram into numpy array
    sample_data = np.fromfile(input_file, dtype=sample_dtype, count=1)
    channel = sample_data['channel_number'][0]

    # Check for a valid channel number that is within the number of transducers config
    # to prevent incorrectly indexing into the dictionaries.
    # An out of bounds channel number can indicate invalid, corrupt,
    # or misaligned datagram or a reverse byte order binary data file.
    # Log warning and continue to try and process the rest of the file.
    if channel < 0 or channel > transducer_count:
        print('Invalid channel: %s for transducer count: %s. \n\
        Possible file corruption or format incompatibility.' % (channel, transducer_count))

    # Convert high and low bytes to internal time
    windows_time = build_windows_time(sample_data['high_date_time'][0], sample_data['low_date_time'][0])
    ntp_time = windows_to_ntp(windows_time)

    count = sample_data['count'][0]

    # Extract array of power data
    power_data = np.fromfile(input_file, dtype=power_dtype, count=count).astype('f8')

    # Read the athwartship and alongship angle measurements
    if sample_data['mode'][0] > 1:
        angle_data = np.fromfile(input_file, dtype=angle_dtype, count=count)
    else:
        angle_data = []

    # Read and compare length1 (from beginning of datagram) to length2
    # (from the end of datagram). A mismatch can indicate an invalid, corrupt,
    # or misaligned datagram or a reverse byte order binary data file.
    # Log warning and continue to try and process the rest of the file.
    len_dtype = np.dtype([('length2', '<i4')])  # 4 byte int (long)
    length2_data = np.fromfile(input_file, dtype=len_dtype, count=1)
    if not (sample_data['length1'][0] == length2_data['length2'][0]):
        print('Mismatching beginning and end length values in sample datagram: \n\
        length1: %d, length2: %d.\n\
        Possible file corruption or format incompatibility.' % (sample_data['length1'][0], length2_data['length2'][0]))

    return channel, ntp_time, sample_data, power_data, angle_data

def append_metadata(metadata, channel, sample_data):
    # Fixed across ping
    metadata['channel'].append(channel)
    metadata['transducer_depth'].append(sample_data['transducer_depth'][0])          # [meters]
    metadata['frequency'].append(sample_data['frequency'][0])                        # [Hz]
    metadata['sound_velocity'].append(sample_data['sound_velocity'][0])              # [m/s]
    metadata['absorption_coeff'].append(sample_data['absorption_coefficient'][0])    # [dB/m]
    metadata['temperature'].append(sample_data['temperature'][0])                    # [degC]
    metadata['mode'].append(sample_data['mode'][0])             # >1: split-beam, 0: single-beam

    return metadata

def load_ek60_raw(raw_filename):
    """
    Parse the *.raw file.
    @param raw_filename absolute path/name to file to be parsed
    """
    print('%s  unpacking file: %s' % (dt.now().strftime('%H:%M:%S'), raw_filename))

    with open(raw_filename, 'rb') as input_file:  # read ('r') input file using binary mode ('b')

        config_header, config_transducer = read_header(input_file)
        transducer_count = config_header['transducer_count']

        position = input_file.tell()

        # *_data_temp_dict are for storing different channels within each ping
        # content of *_temp_dict are saved to *_data_dict whenever all channels of the same ping are unpacked
        # see below comment "Check if we have enough records to produce a new row of data"

        # Initialize output structure
        first_ping_metadata = defaultdict(list)  # metadata for each channel
        power_data_dict = defaultdict(list)      # echo power
        angle_data_dict = defaultdict(list)      # alongship and athwartship electronic angle
        tr_data_dict = defaultdict(list)         # transmit signal metadata
        data_times = []                          # ping time
        motion = []                              # pitch, roll, heave

        # Read binary file a block at a time
        raw = input_file.read(BLOCK_SIZE)

        # Flag used to check if data are from the same ping
        last_time = None

        while len(raw) > 4:
            # We only care for the Sample datagrams, skip over all the other datagrams
            match = SAMPLE_MATCHER.search(raw)

            if match:
                # Offset by size of length value
                match_start = match.start() - LENGTH_SIZE

                # Seek to the position of the length data before the token to read into numpy array
                input_file.seek(position + match_start)

                # try:
                next_channel, next_time, next_sample, next_power, next_angle = \
                    process_sample(input_file, transducer_count)  # read each sample

                # Check if it's from different channels within the same ping
                # next_time=last_time when it's the same ping but different channel
                if next_time != last_time:   # if data is from a new ping
                    # Clear out our temporary dictionaries and set the last time to this time
                    sample_temp_dict = defaultdict(list)
                    power_temp_dict = defaultdict(list)
                    angle_temp_dict = defaultdict(list)    # include both alongship and athwartship angle
                    last_time = next_time    # update ping time

                # Store this data
                sample_temp_dict[next_channel] = next_sample
                power_temp_dict[next_channel] = next_power
                angle_temp_dict[next_channel] = next_angle

                # Check if we have enough records to produce a new row of data
                # if yes this means that data from all transducer channels have been read for a particular ping
                # a new row of data means all channels of data from one ping
                # if only 2 channels of data were received but there are a total of 3 transducers,
                # the data are not stored in the final power_data_dict
                if len(sample_temp_dict) == len(power_temp_dict) == \
                        len(angle_temp_dict) == transducer_count:

                    # if this is the first ping from all channels,
                    # create metadata particle and store the frequency / bin_size
                    if not power_data_dict:

                        # Initialize each channel to defaultdict
                        for channel in power_temp_dict:
                            first_ping_metadata[channel] = defaultdict(list)
                            angle_data_dict[channel] = []

                        # Fill in metadata for each channel
                        for channel, sample_data in sample_temp_dict.items():
                            append_metadata(first_ping_metadata[channel], channel, sample_data)

                    # Save data and metadata from each ping to *_data_dict
                    data_times.append(next_time)
                    motion.append(np.array([(sample_temp_dict[1]['heave'],  # all channels have the same motion
                                             sample_temp_dict[1]['pitch'],
                                             sample_temp_dict[1]['roll'])],
                                           dtype=[('heave', 'f4'), ('pitch', 'f4'), ('roll', 'f4')]))
                    for channel in power_temp_dict:
                        power_data_dict[channel].append(power_temp_dict[channel])
                        if any(angle_temp_dict[channel]):   # if split-beam data
                            angle_data_dict[channel].append(angle_temp_dict[channel])
                        tr = np.array([(sample_temp_dict[channel]['frequency'],
                                        sample_temp_dict[channel]['transmit_power'],
                                        sample_temp_dict[channel]['pulse_length'],
                                        sample_temp_dict[channel]['bandwidth'],
                                        sample_temp_dict[channel]['sample_interval'])],
                                      dtype=[('frequency', 'f4'), ('transmit_power', 'f4'),
                                             ('pulse_length', 'f4'), ('bandwidth', 'f4'),
                                             ('sample_interval', 'f4')])
                        tr_data_dict[channel].append(tr)

                # except InvalidTransducer:
                #   pass

            else:
                input_file.seek(position + BLOCK_SIZE - 4)

            # Need current position in file to increment for next regex search offset
            position = input_file.tell()
            # Read the next block for regex search
            raw = input_file.read(BLOCK_SIZE)

        data_times = np.array(data_times)
        # Convert to numpy array and decompress power data to dB
        for channel in power_data_dict:
            power_data_dict[channel] = np.array(power_data_dict[channel]) * 10. * np.log10(2) / 256.
            if angle_data_dict[channel]:  # if split-beam data
                angle_data_dict[channel] = np.array(angle_data_dict[channel])
            else:  # if single-beam data
                angle_data_dict[channel] = []
            tr_data_dict[channel] = np.array(tr_data_dict[channel])

        return first_ping_metadata, data_times, motion, \
               power_data_dict, angle_data_dict, tr_data_dict, \
               config_header, config_transducer

def save_raw_to_nc(raw_filename):
    """
    Save data from RAW to netCDF format
    :param raw_filename:
    :return:
    """
    # Load data from RAW file
    first_ping_metadata, data_times, motion, \
    power_data_dict, angle_data_dict, tr_data_dict, \
    config_header, config_transducer = load_ek60_raw(raw_filename)

    # Get nc filename
    filename = os.path.splitext(os.path.basename(raw_filename))[0]
    nc_path = os.path.join(os.path.split(raw_filename)[0], filename + '.nc')
    fm = FILENAME_MATCHER.match(filename)

    # Create nc file by creating top-level group
    tl_attrs = ('Conventions', 'keywords',
                'sonar_convention_authority', 'sonar_convention_name', 'sonar_convention_version',
                'summary', 'title')
    tl_vals = ('CF-1.7, SONAR-netCDF4, ACDD-1.3', 'EK60',
               'ICES', 'SONAR-netCDF4', '1.7',
               '', '')
    tl_dict = dict(zip(tl_attrs, tl_vals))
    tl_dict['date_created'] = dt.strptime(fm.group('date') + '-' + fm.group('time'),
                                          '%Y%m%d-%H%M%S').isoformat() +'Z'
    ep.set_attrs_toplevel(nc_path, tl_dict)

    # Environment group
    freq = np.array([x['frequency'][0] for x in first_ping_metadata.values()], dtype='float32')
    abs_val = np.array([x['absorption_coeff'][0] for x in first_ping_metadata.values()], dtype='float32')
    ss_val = np.array([x['sound_velocity'][0] for x in first_ping_metadata.values()], dtype='float32')
    env_attrs = ('frequency', 'absorption_coeff', 'sound_speed')
    env_vals = (freq, abs_val, ss_val)
    env_dict = dict(zip(env_attrs, env_vals))
    ep.set_group_environment(nc_path, env_dict)

    # Provenance group
    prov_attrs = ('conversion_software_name', 'conversion_software_version', 'conversion_time')
    prov_vals = ('echopype', 'v0.1', dt.now(tz=pytz.utc).isoformat(timespec='seconds'))  # use UTC time
    prov_dict = dict(zip(prov_attrs, prov_vals))
    ep.set_group_provenance(nc_path, os.path.basename(raw_filename), prov_dict)

    # Sonar group
    sonar_attrs = ('sonar_manufacturer', 'sonar_model', 'sonar_serial_number',
                   'sonar_software_name', 'sonar_software_version', 'sonar_type')
    sonar_vals = ('Simrad', config_header['sounder_name'].decode('utf-8'), '',
                  '', config_header['version'].decode('utf-8'), 'echosounder')
    sonar_dict = dict(zip(sonar_attrs, sonar_vals))
    ep.set_group_sonar(nc_path, sonar_dict)

    # Beam group
    beam_dict = dict()
    beam_dict['beam_mode'] = 'vertical'
    beam_dict['conversion_equation_t'] = 'type_3'  # type_3 is EK60 conversion
    beam_dict['ping_time'] = data_times   # here in seconds since 1900-01-01
                                          # this makes it easier for xarray.to_netcdf conversion
    beam_dict['backscatter_r'] = np.array([power_data_dict[x] for x in power_data_dict])
    beam_dict['beamwidth_receive_major'] = np.array([x['beam_width_alongship']
                                                     for x in config_transducer.__iter__()], dtype='float32')
    beam_dict['beamwidth_receive_minor'] = np.array([x['beam_width_athwartship']
                                                     for x in config_transducer.__iter__()], dtype='float32')
    beam_dict['beamwidth_transmit_major'] = np.array([x['beam_width_alongship']
                                                      for x in config_transducer.__iter__()], dtype='float32')
    beam_dict['beamwidth_transmit_minor'] = np.array([x['beam_width_athwartship']
                                                      for x in config_transducer.__iter__()], dtype='float32')
    beam_dict['beam_direction_x'] = np.array([x['dir_x'] for x in config_transducer.__iter__()], dtype='float32')
    beam_dict['beam_direction_y'] = np.array([x['dir_y'] for x in config_transducer.__iter__()], dtype='float32')
    beam_dict['beam_direction_z'] = np.array([x['dir_z'] for x in config_transducer.__iter__()], dtype='float32')
    beam_dict['equivalent_beam_angle'] = np.array([x['equiv_beam_angle']
                                                   for x in config_transducer.__iter__()], dtype='float32')
    beam_dict['gain_correction'] = np.array([x['gain'] for x in config_transducer.__iter__()], dtype='float32')
    beam_dict['non_quantitative_processing'] = np.array([0, ] * freq.size, dtype='int32')
    beam_dict['sample_interval'] = np.array([x['sample_interval'] for x in tr_data_dict.values()],
                                            dtype='float32').squeeze()  # dimension frequency
    beam_dict['sample_time_offset'] = np.array([2, ] * freq.size,
                                               dtype='int32')  # set to 2 for EK60 data, NOT from sample_data['offset']
    beam_dict['transmit_duration_nominal'] = np.array([x['pulse_length']
                                                       for x in tr_data_dict.values()], dtype='float32').squeeze()
    beam_dict['transmit_power'] = np.array([x['transmit_power']
                                            for x in tr_data_dict.values()], dtype='float32').squeeze()
    beam_dict['transmit_bandwidth'] = np.array([x['bandwidth']
                                                for x in tr_data_dict.values()], dtype='float32').squeeze()
    # Below not in convention
    beam_dict['frequency'] = freq
    beam_dict['range_bin'] = np.arange(power_data_dict[1].shape[1])
    beam_dict['channel_id'] = [x['channel_id'].decode('utf-8') for x in config_transducer.__iter__()]
    beam_dict['gpt_software_version'] = [x['gpt_software_version'].decode('utf-8')
                                         for x in config_transducer.__iter__()]
    idx = [np.argwhere(tr_data_dict[x + 1]['pulse_length'][0] == config_transducer[x]['pulse_length_table']).squeeze()
           for x in range(len(config_transducer))]
    beam_dict['sa_correction'] = np.array([x['sa_correction_table'][y]
                                           for x, y in zip(config_transducer.__iter__(), np.array(idx))])

    # Below is in Platform group in convention
    beam_dict['transducer_offset_x'] = np.array([x['pos_x'] for x in config_transducer.__iter__()], dtype='float32')
    beam_dict['transducer_offset_y'] = np.array([x['pos_y'] for x in config_transducer.__iter__()], dtype='float32')
    beam_dict['transducer_offset_z'] = np.array([x['pos_z'] for x in config_transducer.__iter__()],
                                                dtype='float32') + \
                                       np.array([x['transducer_depth'][0] for x in first_ping_metadata.values()],
                                                dtype='float32')
    ep.set_group_beam(nc_path, beam_dict)

    # Platform group
    platform_dict = dict()
    platform_dict['platform_name'] = config_header['survey_name'].decode('utf-8')
    if re.search('OOI', platform_dict['platform_name']):
        platform_dict['platform_type'] = 'subsurface mooring'  # if OOI
    else:
        platform_dict['platform_type'] = 'ship'  # default to ship
    platform_dict['time'] = data_times    # here in seconds since 1900-01-01
                                          # this makes it easier for xarray.to_netcdf conversion
    platform_dict['pitch'] = np.array([x['pitch'] for x in motion.__iter__()], dtype='float32').squeeze()
    platform_dict['roll'] = np.array([x['roll'] for x in motion.__iter__()], dtype='float32').squeeze()
    platform_dict['heave'] = np.array([x['heave'] for x in motion.__iter__()], dtype='float32').squeeze()
    platform_dict['water_level'] = np.int32(0)  # set to 0 for EK60 since this is not separately recorded
                                                # and is part of transducer_depth
    ep.set_group_platform(nc_path, platform_dict)


# uncompyle6 version 3.7.4
# Python bytecode 3.7 (3394)
# Decompiled from: Python 3.7.9 (tags/v3.7.9:13c94747c7, Aug 17 2020, 18:58:18) [MSC v.1900 64 bit (AMD64)]
# Embedded file name: T:\InGame\Gameplay\Scripts\Server\sims\fixup\sim_info_favorites_fixup_action.py
# Compiled at: 2020-07-28 18:38:17
# Size of source mod 2**32: 2528 bytes
from sims.fixup.sim_info_fixup_action import _SimInfoFixupAction
from sims4.tuning.tunable import TunableList, TunableCasPart, TunableReference, TunableTuple, TunableRange
from tag import TunableTag
import services, sims4.random

class _SimInfoFavoritesFixupAction(_SimInfoFixupAction):
    FACTORY_TUNABLES = {'favorite_info_list': TunableList(description='\n            A List of favorite categories to randomly set on new Sims.\n            ',
                             tunable=TunableTuple(description='\n                Tuple containing a tag to set the favorite of, and a list of object definitions\n                and weights to choose for the favorite.\n                ',
                             favorite_tag=TunableTag(description='\n                    The tag to set the favorite of.\n                    ',
                             filter_prefixes=('Func', )),
                             favorite_definition_ids=TunableList(description='\n                    A list of tuples of object definitions, and corresponding weights \n                    One definition is chosen by a weighted random and set as the favorite \n                    for the tag above. \n                    ',
                             tunable=TunableTuple(weight=TunableRange(description='\n                            The weight to use for the choosing of this object. Any positive value works.\n                            ',
                             tunable_type=float,
                             default=1.0,
                             minimum=0),
                             object=TunableReference(description='\n                            The definition of the object to choose as a favorite.\n                            ',
                             manager=(services.definition_manager()))))))}

    def __call__(self, sim_info):
        favorites_tracker = sim_info.favorites_tracker
        if favorites_tracker is None:
            return
        for favorite_info in self.favorite_info_list:
            favorite = sims4.random.weighted_random_item((favorite_info.favorite_definition_ids), flipped=True)
            favorites_tracker.set_favorite(favorite_info.favorite_tag, None, favorite.id)


import ROOT
from itertools import product, combinations
import math
import numpy as np
from collections import OrderedDict, Counter
import sys

from PhysicsTools.Heppy.analyzers.core.Analyzer   import Analyzer
from PhysicsTools.Heppy.analyzers.core.AutoHandle import AutoHandle
from PhysicsTools.Heppy.physicsobjects.Muon       import Muon
from PhysicsTools.Heppy.physicsobjects.Electron   import Electron
from PhysicsTools.Heppy.physicsobjects.Tau        import Tau
from PhysicsTools.HeppyCore.utils.deltar          import deltaR, deltaR2, bestMatch

from CMGTools.WTau3Mu.physicsobjects.Tau3MuMET    import Tau3MuMET
from CMGTools.WTau3Mu.analyzers.resonances        import resonances, sigmas_to_exclude

class HFBDTComputer(Analyzer):
    def declareHandles(self):
        super(HFBDTComputer, self).declareHandles()
        self.handles['vertices'] = AutoHandle("offlineSlimmedPrimaryVertices", 'std::vector<reco::Vertex>', fallbackLabel="offlinePrimaryVertices")
        self.handles['PFCands']  = AutoHandle('packedPFCandidates', 'vector<pat::PackedCandidate>')

    def beginLoop(self, setup):
        super(HFBDTComputer, self).beginLoop(setup)

    @staticmethod
    def trkrel(pfs, mu, sv):
        '''compute the ratio between the sum of pT of PFCands close to a muon, and the muon pT.
        '''
        return 1.*sum([pf.pt() for pf in pfs if deltaR(mu, pf) < 0.3 and HFBDTComputer.dca_fv(tk=pf, sv=sv) < 0.1]) / mu.pt()

    @staticmethod
    def dca_fv(tk, sv):
        '''compute the 3D impact parameter between a track and the secondary vertex
        '''
        return math.sqrt(tk.dxy(sv.position())**2+tk.dz(sv.position())**2)

    @staticmethod
    def fv_d3Dsig(pv, sv):
        '''compute the significance of the 3D distance between the primary and secondary vertex
        '''
        return ROOT.VertexDistance3D().distance(pv, sv).significance()

    @staticmethod
    def d0sig(mu, bs):
        '''compute the displacement significance of a muon track from the beamspot
        '''
        return mu.innerTrack().dxy(bs)/mu.innerTrack().dxyError(bs)

    @staticmethod
    def best_pv(pvs, sv, tau):
        ''' find the PV which minimises the angular distance (i.e. maximises the cosine) between
        the PV-SV segment and the tau momentum vector in the 3D space
        '''
        pv, max_cos = None, -2
        for vtx in pvs:
            seg = ROOT.math.XYZVector(
                sv.x()-vtx.x(),
                sv.y()-vtx.y(),
                sv.z()-vtx.z()
            )
            cos = tau.Dot(seg) / (tau.R()*seg.R())
            if cos > max_cos:
                max_cos = cos
                pv      = vtx
        if pv is None:
            raise ValueError("Could not find a PV. This should not be happening.")
        return pv, max_cos

    @staticmethod
    def trigger_match(objects, muons, filt):
        matches = []
        for i, mu in enumerate(muons):
            objs = sorted([ob for ob in objects if ob.filter(filt) and deltaR(mu, ob) < 0.1], key = lambda ob: deltaR(mu, ob))
            while len(objs) and objs[0] in matches:
                objs = objs[1:]
            matches.append(objs[0] if len(objs) else None)
        return matches

    def process(self, event):
        self.readCollections(event.input)
        event.pvs = self.handles['vertices'].product()
        event.pfs = [pf for pf in self.handles['PFCands' ].product() if
            pf.pt() > 1                            and
            abs(pf.eta()) < 2.4                    and
            pf.trackerLayersWithMeasurement() > 5  and
            pf.pixelLayersWithMeasurement() > 1    and
            pf.trackHighPurity()                   and
            all(deltaR(pf, mu)>0.01 for mu in event.muons)]

        tau = ROOT.math.XYZVector(event.tau3mu.refitTau.px(), event.tau3mu.refitTau.py(), event.tau3mu.refitTau.pz())
        sv  = event.tau3muRefit.refittedVertex
        pv, max_cos = self.best_pv(pvs=event.pvs, sv=sv, tau=tau)
        # fill the BDT variables
        event.cLP           = max(mu.physObj.combinedQuality().chi2LocalPosition     for mu in event.muons)
        event.tKink         = max(mu.physObj.combinedQuality().trkKink               for mu in event.muons)
        event.segmComp      = min(mu.physObj.segmentCompatibility()                  for mu in event.muons)
        event.fv_nC         = event.tau3muRefit.refittedVertex.chi2()
        event.fv_dphi3D     = math.acos(max_cos)
        event.fv_d3Dsig     = self.fv_d3Dsig(pv=pv, sv=sv)
        event.mindca_iso    = min(self.dca_fv(tk=pf, sv=sv)                          for pf in event.pfs  )
        event.trkrel        = min(self.trkrel(pfs=event.pfs, mu=mu.physObj, sv=sv)   for mu in event.muons)
        event.d0sig_min     = min(self.d0sig(mu=mu.physObj, bs=event.bs)             for mu in event.muons)

        event.evt           = event.eventId
        event.tripletMass   = event.tau3muRefit.massMuons()
        event.puFactor      = event.puWeight

        event.d0sig_max     = max(self.d0sig(mu=mu.physObj, bs=event.bs)             for mu in event.muons)
        event.Pt_tripl      = event.tau3muRefit.sumPtMuons()

        for info in event.trigger_infos:
            name = '_'.join(info.name.split('_')[:-1])
            if name in self.cfg_ana.match_triggers.keys():
                filters = self.cfg_ana.match_triggers[name]
                matched = self.trigger_match(objects=info.objects, muons=event.muons, filt=filters[0])
                if name==getattr(self.cfg_ana, 'reference_trigger', 'NONE') and len(matched)>2:
                    event.tkmu_from_ref = 1 if matched[0] is None else 2 if matched[1] is None else 3
                getattr(event.tau3mu, 'matched_triggers')[name] = len([mu for mu in matched if mu is not None])>1
                getattr(event       , 'prescales'       )[name] = info.prescale

        return True


#from doo.doo import Thread2
from mp.utils import normr,almostEq
from mp.geom import angBetween
from numpy import eye,dot,diag,cross,zeros
from numpy.random import randn
import logging; L = logging.Logger(__file__)
L.setLevel(logging.DEBUG)
from nibabel.quaternions import angle_axis2mat,mat2quat,quat2angle_axis

def getState(thread):
    "get position and orientation of ends"
    xyz = thread.getXYZ()
    s0,s1,e1,e0 = xyz.T[[0,1,-2,-1]]
    return s0,normr(s1-s0),e0,normr(e0-e1)

def setOrs(thread,pos1,or1,pos2,or2):
    "set orientation of ends"
    old_pos1,old_or1,old_pos2,old_or2 = getState(thread)
    thread.applyMotion(pos1-old_pos1,minRot(or1,old_or1),
                       pos2-old_pos2,minRot(or2,old_or2))

def applyTwist(thread,twist1,twist2):
    "twist the ends"
    _,or1,_,or2 = getState(thread)
    thread.applyMotion(zeros(3),angle_axis2mat(twist1,or1),zeros(3),angle_axis2mat(twist2,or2))

def minRot(ax1,ax2):
    "find the rotation matrix that takes ax1 to ax2"
    if almostEq(ax1,ax2):
        L.debug("minRot: same vector")
        return eye(3)
    elif almostEq(ax1,-ax2):
        L.debug("minRot: opp vector")
        return -diag([-1,-1,0])
    else:
        ax_rot = cross(ax2,ax1)
        return angle_axis2mat(angBetween(ax1,ax2),ax_rot)

def infTwist(xyz):
    pts = xyz.T
    ors = normr(pts[1:] - pts[:-1])

    start2end_rope = eye(3)
    for (or1,or2) in zip(ors[:-1],ors[1:]):
        start2end_rope = dot(minRot(or2,or1),start2end_rope)

    assert almostEq(dot(start2end_rope,ors[0]),ors[-1])
    end2start_min = minRot(ors[0],ors[-1])

    twist_mat = dot(end2start_min,start2end_rope)
    ang,_ = quat2angle_axis(mat2quat(twist_mat))
    return ang

def test_infTwist():
    xyz = randn(3,100)
    infTwist(xyz)

def test_minRot():
    for _ in xrange(100):
        x1 = normr(randn(3))
        x2 = normr(randn(3))
        assert almostEq(x1,dot(minRot(x1,x2),x2))

if __name__ == "__main__":
    test_minRot()
    test_infTwist()


def ChessboardTraveling(pos_str):
    pos_tuple = (int(pos_str[1]), int(pos_str[3]), int(pos_str[6]), int(pos_str[8]))
    (pos_x_one, pos_y_one, pos_x_two, pos_y_two) = pos_tuple
    combination_counter = 0
    x_counter = 0
    y_counter = 0
    x_limit = pos_x_two - pos_x_one
    y_limit = pos_y_two - pos_y_one
    for num in range(1, x_limit + 1):
        x_counter = num
        while x_counter <= x_limit:
            x_counter += 1
            if y_counter <= y_limit:
                y_counter += 1
            print("(" + str(x_counter) + "," , str(y_counter) + ")")
        combination_counter += 1

    for num in range(1, y_limit + 1):
        y_counter = num
        while y_counter <= y_limit:
            y_counter += 1
            if x_counter <= x_limit:
                x_counter += 1
            print("(" + str(x_counter) + "," , str(y_counter) + ")")
        combination_counter += 1

    return combination_counter

print(ChessboardTraveling("(1 1)(3 3)"))
input("press ENTER to exit the program")


"""
README
===============================================================================
This is an adapter to represent axographio objects as neo objects.

axographio is a file i/o Python module that can read in axograph ".axgx" files.
It is available under a BSD-3-Clause license and can be installed from pip.
The following file types are supported:

 - AXGX/AXGD (Axograph X file format)

Based on stimfitio.pyfrom neo.io

11 JUL 2018, W. Hart, Swinburne University, Australia
"""

# needed for python 3 compatibility
from __future__ import absolute_import

from datetime import datetime
import os
import sys

import numpy as np
import quantities as pq

from neo.io.baseio import BaseIO
from neo.core import Block, Segment, AnalogSignal

try:
    import axographio
except ImportError as err:
    HAS_AXOGRAPHIO = False
    AXOGRAPHIO_ERR = err
else:
    HAS_AXOGRAPHIO = True
    AXOGRAPHIO_ERR = None

class AxographIO(BaseIO):
    """
    Class for converting an Axographio object to a Neo object.
    Provides a standardized representation of the data as defined by the neo
    project; this is useful to explore the data with an increasing number of
    electrophysiology software tools that rely on the Neo standard.

    axographio is a file i/o Python module that can read in axograph ".axgx" files.
    It is available under a BSD-3-Clause license and can be installed from pip.
    The following file types are supported:

    - AXGX/AXGD (Axograph X file format)

    Example usage:
        >>> import neo
        >>> neo_obj = neo.io.AxographIO("file.axgx")
        or
        >>> import axographio
        >>> axo_obj = axographio.read("file.axgx")
        >>> neo_obj = neo.io.AxographIO(axo_obj)
    """

    is_readable = True
    is_writable = False

    supported_objects = [Block, Segment, AnalogSignal]
    readable_objects = [Block]
    writeable_objects = []

    has_header = False
    is_streameable = False

    read_params = {Block: []}
    write_params = None

    name = 'AXOGRAPH'
    extensions = ['axgx', 'axgd']

    mode = 'file'

    def __init__(self, filename=None):
        """
        Arguments:
            filename : Either a filename or an axographio object
        """
        if not HAS_AXOGRAPHIO:
            raise AXOGRAPHIO_ERR

        BaseIO.__init__(self)

        if hasattr(filename, 'lower'):
            self.filename = filename
            self.axo_obj = None
        else:
            self.axo_obj = filename
            self.filename = None

    def read_block(self, **kargs):
        if self.filename is not None:
            self.axo_obj = axographio.read(self.filename)

        # Build up the block
        blk = Block()

        blk.rec_datetime = None
        if self.filename is not None:
            # modified time is not ideal but less prone to
            # cross-platform issues than created time (ctime)
            blk.file_datetime = datetime.fromtimestamp(os.path.getmtime(self.filename))

            # store the filename if it is available
            blk.file_origin = self.filename

        # determine the channel names and counts
        _, channel_ordering = np.unique(self.axo_obj.names[1:], return_index=True)
        channel_names = np.array(self.axo_obj.names[1:])[np.sort(channel_ordering)]
        channel_count = len(channel_names)

        # determine the time signal and sample period
        sample_period = self.axo_obj.data[0].step * pq.s
        start_time = self.axo_obj.data[0].start * pq.s

        # Attempt to read units from the channel names
        channel_unit_names = [x.split()[-1].strip('()') for x in channel_names]
        channel_units = []

        for unit in channel_unit_names:
            try:
                channel_units.append(pq.Quantity(1, unit))
            except LookupError:
                channel_units.append(None)

        # Strip units from channel names
        channel_names = [' '.join(x.split()[:-1]) for x in channel_names]

        # build up segments by grouping axograph columns
        for seg_idx in range(1, len(self.axo_obj.data), channel_count):
            seg = Segment(index=seg_idx)

            # add in the channels
            for chan_idx in range(0, channel_count):
                signal = pq.Quantity(
                    self.axo_obj.data[seg_idx + chan_idx], channel_units[chan_idx])
                analog = AnalogSignal(signal,
                                      sampling_period=sample_period, t_start=start_time,
                                      name=channel_names[chan_idx], channel_index=chan_idx)
                seg.analogsignals.append(analog)

            blk.segments.append(seg)

        blk.create_many_to_one_relationship()

        return blk


import numpy as np
import math
import matplotlib.pyplot as plt

def target(w, x, y): #target function of log likehood method
    ret = 0
    n = len(x)
    for i in range(0,n):
        #print(x[i], w)
        rate = w.dot(x[i])
        #print(w, x[i], rate)
        ret += -rate*y[i] + math.log(1+math.exp(rate))
    return ret
pass
def test(w, testx, testy): #test and return accuracy
    n = len(testx)
    sum = 0
    for i in range(0,n):
        rate = w.dot(testx[i])
        #print("here",w, testx[i], rate)
        activation = 1./(math.exp(-rate)+1.)
        #print (rate)
        #print ("act", activation)
        if activation < 0.5 and testy[i] == 0 or (activation > 0.5 and testy[i] == 1):
           sum += 1
        elif activation == 0.5:
            sum += 0.5
    return sum/n
pass
def newton(weight, trainx, trainy):#Newton optimization of log likehood
    cur = 0
    n = len(trainx[0])
    while 1:
        old = cur
        #print(weight, trainx, trainy)
        cur = target(weight, trainx, trainy)
        print(cur, old)
        if abs(old-cur) <= 0.000001:
            break
        dl = np.matrix(np.zeros(n)).astype(float)
        ddl = np.zeros((n,n)).astype(float)
        #print (n,dl, ddl)
        for i in range(0,len(trainx)):
            rate = weight.dot(trainx[i])
            #print(weight, trainx[i], weight.dot(trainx[i]))
            p = 1. - 1./(1.+math.exp(rate))
            #print("p=", p)
            #print(dl)
            dl += (-trainy[i]+p)*trainx[i]
            #print (i, np.matrix(trainx[i]).T, trainx[i], np.matrix(trainx[i]).T*trainx[i])
            ddl += np.matrix(trainx[i]).T*trainx[i]*p*(1-p)
        #print ("dl,ddl=",dl, np.linalg.pinv(ddl),dl*np.linalg.pinv(ddl))
        weight -= dl*np.linalg.pinv(ddl)
        #print(cur, weight)
    #print(test(weight, testx, testy))
    return weight
pass
def kFoldeCrossValidation(k, data, y):
    n = len(data[0])
    weight_final = np.zeros(n).astype(float)
    weight_final = np.matrix(weight_final)
    for it in range(0,k):
        weight = np.zeros(n).astype(float)#initialize train data and test data
        weight = np.matrix(weight)
        trainx = data[:math.floor(n*it/k)] + data[math.floor(n*(it+1)/k):]
        trainy = y[:math.floor(n*it/k)] + y[math.floor(n*(it+1)/k):]
        testx = data[math.floor(n*it/k):math.floor(n*(it+1)/k)]
        testy = y[math.floor(n*it/k):math.floor(n*(it+1)/k)]

        testx = np.array(testx).astype(float)#transfer data into array structure
        testy = np.array(testy).astype(int)
        trainx = np.array(trainx).astype(float)
        trainy = np.array(trainy).astype(int)
        #print(trainx, trainy)
        weight = newton(weight, trainx, trainy)

        weight_final += weight
    weight_final /= k
    data = np.array(data).astype(float)
    y = np.array(y).astype(int)
    print(test(weight_final, data, y))
    print(weight_final)
    return weight_final
pass
def Virtualization(data, y, weight_final): #Virtualization the scatter plot and draw the classification line,ONLY WORKS IN 2D
    x0,y0,x1,y1 = [],[],[],[]
    data = np.array(data).astype(float)
    y = np.array(y).astype(int)#transfer data into numerical foramt
    for i in range(0,n):#classify two kinds of data
        if y[i] == 1:
            x0.append(data[i][0])
            y0.append(data[i][1])
        else:
            x1.append(data[i][0])
            y1.append(data[i][1])
    fig = plt.figure()
    ax = fig.add_subplot(1,1, 1)
    #print(x0, y0)
    ax.set_title("Logic Regression of Melon Example")
    ax.set_xlabel("Density of melons")
    ax.set_ylabel("Sugar content")
    ax.scatter(x0,y0, label = 'Bad Melons', marker = 'o')#Plot scatter
    ax.scatter(x1,y1,  label = 'Good Melons', marker = 's')
    weight_final = np.array(weight_final)
    X = []
    Y = []
    for i in [1,9]:
        X.append(i*0.1)
        Y.append(-(i*0.1*weight_final[0][0] + weight_final[0][2])/weight_final[0][1])
    print (X,Y)
    ax.plot(X,Y, label = 'classification line')#Plot classification line
    ax.legend()
    plt.show()

file = open("data.txt")
data = []
y = []
n = 0
while 1:#read data
    line = file.readline().split()
    if not line:
        break
    pass
    #line = line.split(' ')
    k = len(line)
    #print (k, data, y)
    lines = []
    for i in range(1,k-1):
        lines.append(line[i])
    lines.append(1)
    data.append(lines)
    if line[-1] == '0':
        y.append(0)
    else:
        y.append(1)
    n += 1

print(data, y)
weight_final = kFoldeCrossValidation(10, data, y)

Virtualization(data, y, weight_final)


import requests
import json
from queue import Queue
import os
import time
from lxml import etree
import threading
import pymongo
import urllib3
from retrying import retry
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
import re

class SinaNews:
    def __init__(self):
        self.headers = {
            "Accept": "*/*",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Host": "feed.mix.sina.com.cn",
            "Proxy-Connection": "keep-alive",
            "Referer": "http://tech.sina.com.cn/chuangshiji/",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
        }
        self.details_headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "max-age=0",
            "Connection": "keep-alive",
            "Host": "tech.sina.com.cn",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"
        }
        self.proxies = {
            "https": "120.24.68.197:16818",
            "http": "120.24.68.197:16818"
        }
        self.MONGO_URL = "192.168.0.8"
        self.MONGO_PORT = 27017
        self.MONGO_DB = "sina_news"
        self.MONGO_COLLECTION = "sina_news_list"
        self.client = pymongo.MongoClient(self.MONGO_URL)
        self.db = self.client[self.MONGO_DB]
        self.coll = self.db[self.MONGO_COLLECTION]
        self.url_queue = Queue()
        self.cy_url_queue = Queue()
        self.content = Queue()
        self.cy_content = Queue()
        self.data = Queue()
        self.image_news_queue = Queue()

    @retry(stop_max_attempt_number=4)
    def _parse_url(self, url, headers, return_str=False):
        r = requests.get(url, headers=headers, proxies=self.proxies, timeout=5)
        # assert r.status_code == 200
        return r.content.decode("utf-8")

    # @retry(stop_max_attempt_number=5)
    def get_news_list(self):
        for i in range(1):
            start_url = "http://feed.mix.sina.com.cn/api/roll/get?pageid=223&lid=23&num=50&versionNumber=1.2.8&page{}"
            response = self._parse_url(start_url.format(i), headers=self.headers)
            item_url_dict = json.loads(response)["result"]["data"]

            for k in item_url_dict:
                # print(k)
                details_url = (k["url"]).replace('["', "").replace('"]', "").replace('\\', "")
                if "internet" not in details_url or "http://slide.tech.sina.com.cn/" not in details_url:
                    self.url_queue.put(details_url)

    # @retry(stop_max_attempt_number=5)
    def get_details_page(self):
        """chuang shi ji """
        while True:
            page_url = self.url_queue.get()
            print(page_url)
            if "csj" in page_url:
                # print(page_url)
                html = self._parse_url(url=page_url, headers=self.details_headers)
                print(page_url)
                time.sleep(1)
                self.content.put(html)
            else:
                # print(page_url)
                html = self._parse_url(url=page_url, headers=self.details_headers)
                print(page_url)
                time.sleep(1)
                self.cy_content.put(html)

            self.url_queue.task_done()

    def parse_cy_item(self):
        while True:
            doc = self.cy_content.get()
            text = etree.HTML(doc)
            news_title = text.xpath('//div[@class="second-title"]/text()') if text.xpath(
                '//div[@class="second-title"]/text()') else None
            if news_title is not None:
                news_title = news_title[0]

            news_date = text.xpath('//div[@class="date-source"]/span[1]/text()') if text.xpath(
                '//div[@class="date-source"]/span[1]/text()') else None
            if news_date is not None:
                news_date = news_date[0]

            news_author = text.xpath('//div[@class="date-source"]/span[2]/text()') if text.xpath(
                '//div[@class="date-source"]/span[2]/text()') else None
            if news_author is not None:
                news_author = news_author[0]
            else:
                news_author = text.xpath('//div[@class="date-source"]/span[2]/a/text()') if text.xpath(
                    '//div[@class="date-source"]/span[2]/a/text()') else None
                if news_author is not None:
                    news_author = news_author[0]

            news_tag = text.xpath('//div[@id="keywords"]/a/text()') if text.xpath(
                '//div[@id="keywords"]/a/text()') else []

            news_url = text.xpath('//meta[@property="toutiao"]/@content') if text.xpath(
                '//meta[@property="toutiao"]/@content') else None
            if news_url is not None:
                news_url = news_url[0]

            news_cate = text.xpath('//div[@class="path"]//span[@class="spliter"]/a/text()') if text.xpath(
                '//div[@class="path"]//span[@class="spliter"]/a/text()') else None
            if news_cate is not None:
                news_cate = news_cate[0]

            news_content = text.xpath('//div[@id="artibody"]') if text.xpath('//div[@id="artibody"]') else None
            if news_content is not None:
                news_content = news_content[0]
            content = etree.tostring(news_content, encoding="utf-8").decode("utf-8")

            data = {
                "title": news_title,
                "time": news_date,
                "categories": news_cate,
                "url": news_url,
                "author": news_author,
                "tag": news_tag,
                "content": content,
            }
            self.data.put(data)
            self.cy_content.task_done()

    def parse_item(self):
        """
        1. 创事记页面解析
        :return:
        """
        while True:
            doc = self.content.get()
            text = etree.HTML(doc)
            news_title = text.xpath('//h1[@id="artibodyTitle"]/text()') if text.xpath(
                '//h1[@id="artibodyTitle"]/text()') else None
            if news_title is not None:
                news_title = news_title[0]

            news_date = text.xpath('//span[@id="pub_date"]/text()') if text.xpath(
                '//span[@id="pub_date"]/text()') else None
            if news_date is not None:
                news_date = news_date[0].strip()

            news_cate = text.xpath('//span[@id="media_name"]/a[1]/text()') if text.xpath(
                '//span[@id="media_name"]/a[1]/text()') else None
            if news_cate is not None:
                news_cate = news_cate[0]

            news_author = text.xpath('//span[@id="author_ename"]/a/text()') if text.xpath(
                '//span[@id="author_ename"]/a/text()') else None
            if news_author is not None:
                news_author = news_author[0]

            news_tag = text.xpath('//p[@class="art_keywords"]/a/text()') if text.xpath(
                '//p[@class="art_keywords"]/a/text()') else None
            if news_tag is not None:
                news_tag = news_tag

            news_url = text.xpath('//div[@class="blk_tw"]/h3/a/@href') if text.xpath(
                '//div[@class="blk_tw"]/h3/a/@href') else None
            if news_url is not None:
                news_url = news_url[0]
            else:
                news_url = text.xpath('//meta[@property="toutiao"]/@content') if text.xpath(
                    '//meta[@property="toutiao"]/@content') else None
                if news_url is not None:
                    news_url = news_url[0]

            news_content = text.xpath('//div[@id="artibody"]') if text.xpath('//div[@id="artibody"]') else None
            if news_content is not None:
                news_content = news_content[0]
            content = etree.tostring(news_content, encoding="utf-8").decode("utf-8")

            data = {
                "title": news_title,
                "time": news_date,
                "categories": news_cate,
                "url": news_url,
                "author": news_author,
                "tag": news_tag,
                "content": content,
            }
            self.data.put(data)
            self.content.task_done()

    def save_data(self):
        while True:
            data = self.data.get()
            try:
                if self.db[self.MONGO_COLLECTION].insert(data):
                    print('存储到MongoDB成功')
            except Exception as e:
                print(e)
                print('存储到MongoDB失败')
            self.data.task_done()

    def run(self):
        self.get_news_list()

        thread_list = []
        for i in range(1):
            t2 = threading.Thread(target=self.get_details_page)
            thread_list.append(t2)

        # for i in range(1):
        #     t3 = threading.Thread(target=self.parse_item)
        #     thread_list.append(t3)
        #
        # for i in range(1):
        #     t4 = threading.Thread(target=self.parse_cy_item)
        #     thread_list.append(t4)
        #
        # for i in range(1):
        #     t5 = threading.Thread(target=self.save_data)
        #     thread_list.append(t5)
        #
        # for t in thread_list:
        #     t.setDaemon(True)  # 设置守护线程，说明该线程不重要，主线程结束，子线程结束
        #     t.start()  # 线程启动
        #
        # for q in [self.url_queue, self.content, self.data]:
        #     q.join()  # 等待，让主线程等待，队列计数为0之后才会结束，否则会一直等待

if __name__ == '__main__':
    sn = SinaNews()
    sn.run()


from django.contrib import admin

# Register your models here.
from .models import Task

def change_status(modeladmin, request, queryset):
    queryset.update(task_status='Complete')

change_status.short_description = "Mark selected tasks as complete"

class TaskAdmin(admin.ModelAdmin):
    list_display = ('task_name', 'task_status', 'task_priority', 'target_date')
    list_filter = ('task_status', 'task_priority', 'target_date')
    search_fields = ['task_name', 'task_status', 'task_priority']
    # list_editable = ['task_status']
    actions = [change_status]

admin.site.register(Task, TaskAdmin)


# -*- coding: utf-8 -*-
"""
Created on Fri Jun 12 14:07:57 2020

@author: USER
"""

import requests
from bs4 import BeautifulSoup as bs
import pandas as pd
import re

link = 'https://hh.ru'
text = 'инженер электрик'
area_num = 2

def page_data(soup):
    #dataframe
    vacancies = pd.DataFrame()
    #блок со всеми вакансиями листа
    vac_block = soup.find('div', {'class':'vacancy-serp'})
    #список вакансий
    vac_list = vac_block.find_all('div', {'class':'vacancy-serp-item'}, recurcive=False)
    #данные по каждой вакансии
    for vac in vac_list:
        vac_dict = {}
        vac_dict['name'] = vac.find('a', {'data-qa':'vacancy-serp__vacancy-title'}).text
        vac_dict['salary'] = vac.find('div', {'class':'vacancy-serp-item__sidebar'}).text
        if '-' in vac_dict['salary']:
            res = re.search(r'([\d ]+\D*[\d ]*)-([\d ]+\D*[\d ]*)',vac_dict['salary'])
            vac_dict['min_salary'] = res.group(1)
            vac_dict['max_salary'] = res.group(2)
        elif 'от' in vac_dict['salary']:
            res = re.search(r'([\d]+\D*[\d]*) ',vac_dict['salary'])
            vac_dict['min_salary'] = res.group(1)

        elif 'до' in vac_dict['salary']:
            res = re.search(r'([\d]+\D*[\d]*) ',vac_dict['salary'])
            vac_dict['max_salary'] = res.group(1)

        else:
            None
        if vac_dict['salary']:
            vac_dict['currency'] = re.search(r' ([\w\.]+$)',vac_dict['salary']).group(1)
        else:
            None
        vac_dict['link'] = vac.find('a', {'data-qa':'vacancy-serp__vacancy-title'})['href']
        if vac.find('a', {'data-qa':'vacancy-serp__vacancy-employer'}):
            vac_dict['company'] = vac.find('a', {'data-qa':'vacancy-serp__vacancy-employer'}).text
        else:
            vac_dict['company']='not_found'
        vac_dict['city'] = vac.find('span', {'data-qa':'vacancy-serp__vacancy-address'}).text
        vac_dict['source'] = 'hh.ru'
        vacancies = vacancies.append(vac_dict, ignore_index = True)
    return(vacancies)

def scrap_hh(link, text, area_num):
    area = area_num
    vacancy_name = text
    url = link
    params = {'area':area,
              'st':'searchVacancy',
              'text': vacancy_name,
              'fromSearch':'true'}
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'}
    rq = requests.get(url+'/search/vacancy', params=params, headers=headers)
    soup = bs(rq.text, 'lxml')
    result = pd.DataFrame()
    result = page_data(soup)
    print(f'страница {1} добавлена')
    pager_block = soup.find('div', {'data-qa':'pager-block'})
    next_page_tail = pager_block.find('a', {'data-qa':'pager-next'})
    i  = 2
    while next_page_tail:
        soup = bs(requests.get(url+next_page_tail['href'], headers = headers).text, 'lxml')
        result = result.append(page_data(soup), ignore_index = True)
        pager_block = soup.find('div', {'data-qa':'pager-block'})
        next_page_tail = pager_block.find('a', {'data-qa':'pager-next'})
        print(f'страница {i} добавлена')
        i = i + 1
    return(result)

df = scrap_hh(link, text, area_num).loc[:, ['name','city', 'min_salary', 'max_salary', 'currency', 'link', 'source']]

print(df)


from web import form
import json, hashlib, zipfile, os, glob, time
import web, utils, decompiler

#web.config.debug = False
render = web.template.render('templates/')
render_base = web.template.render('templates/', base='base')

urls = ( '/', 'Index',
	'/about', 'Index',
	'/copyright', 'Index',
	'/login', 'Login',
	'/upload', 'Uploader',
	'/decompile/([a-fA-F0-9]{32})/?', 'Decompiler',

	#'/busca/()(q=([a-zA-Z0-9]*))?', 'Buscador', #EL BUSCADOR SE HACE EN JS
	#'/report/([a-fA-F0-9]{32})/busca/(q=([a-zA-Z0-9]*))?', 'Buscador',
	'/report/([a-fA-F0-9]{32})/info/classes.json', 'JsonAllClasses',
	'/report/([a-fA-F0-9]{32})', 'RedirReport',
	'/report/([a-fA-F0-9]{32})/', 'Report',
	'/report/([a-fA-F0-9]{32})/(0x.[0-9a-fA-F]+|@.*/.*)', 'JsonRecursos',
	'/report/([a-fA-F0-9]{32})/res/(@.*/.*)', 'RecursosView',#TODO
	'/report/([a-fA-F0-9]{32})/(([a-zA-Z_][a-zA-Z_0-9$]*\.?)*)', 'View',

	)
app = web.application(urls, globals())

sesion = web.session.Session(app, web.session.DiskStore('sessions'))

########## LOGIN ZONE ##########
def check_sesion():
	if not web.config.debug and not sesion.get('logged_in', False):
		raise web.seeother('/login')

class Login:
	login_form = form.Form(
		form.Textbox('username'),
		form.Password('password'),
		form.Button('Login'),
		)

	def valid_user(self):
		i = web.input()
		return i.get('username') == i.get('password')

	def GET(self, out=None):
		if web.input().get('out') is not None:
			sesion.logged_in = False
			raise web.seeother('/')
		return render_base.login(self.login_form())

	def POST(self):
		log = self.login_form()
		if log.validates() and self.valid_user():
			sesion.logged_in = True
			raise web.seeother('/')
		log.valid = None
		return render_base.login(log)

########## FUNCTION ZONE ##########
def checkCache(md5):
	if web.config.debug: return
	last_mod = utils.getLastModifed(utils.REPORTS_DIR + md5)
	if 'HTTP_IF_MODIFIED_SINCE' in web.ctx.env and web.ctx.env['HTTP_IF_MODIFIED_SINCE'] == last_mod:
		raise web.notmodified()
	etag = utils.getEtag(last_mod)
	if 'HTTP_IF_NONE_MATCH' in web.ctx.env and web.ctx.env['HTTP_IF_NONE_MATCH'] == etag:
		raise web.notmodified()
	web.header('Last-Modified', last_mod)
	web.header('Etag', etag)

class Index:
	def GET(self):
		check_sesion()
		return render_base.index()

class Decompiler:
	def GET(self, md5):
		check_sesion()
		files = glob.glob(utils.FILE_UPLOAD_DIR + md5 + '.*')
		if files:
			dec = decompiler.Decompiler()
			dec.decompile(files[0])
			raise web.seeother('/report/'+md5+'/')
		else:
			return render_base.upload("El fichero no se encuentra en el sistema")

class RedirReport:
	def GET(self, md5):
		raise web.seeother('/report/'+md5+'/')
class Report:
	def GET(self, md5):
		check_sesion()
		checkCache(md5)
		if os.path.isdir(utils.REPORTS_DIR + md5):
			#TODO ver que pasa cuando hay un dex y un apk con el mismo md5
			sample = []
			if os.path.isfile(utils.REPORTS_DIR + md5 + "/sample.apk"):
				sample += ['Descargar muestra']
			if os.path.isfile(utils.REPORTS_DIR + md5 + "/classes.dex"):
				sample += ['Descargar dex']
			if os.path.isfile(utils.REPORTS_DIR + md5 + "/classes.dex.dex2jar.jar"):
				sample += ['Descargar jar']
			#TODO insertar enlace si es un dex y tengo el apk

			manifest = ""
			if os.path.isfile(utils.REPORTS_DIR + md5 +'/smali/AndroidManifest.xml'):
				with open(utils.REPORTS_DIR + md5 +'/smali/AndroidManifest.xml', 'r') as fp:
					manifest = fp.read()
					fp.close()
			recursos = utils.findRecursos(utils.REPORTS_DIR + md5 +'/smali/res/')
			return render_base.report(md5, sample, manifest, recursos)
		elif glob.glob(utils.FILE_UPLOAD_DIR + md5 + '.*'):
			raise web.seeother('/decompile/'+md5)
		else:
			return render_base.upload("El fichero no se encuentra en el sistema")

class JsonRecursos():
	def GET(self, md5, key):
		check_sesion()
		checkCache(md5)
		web.header('Content-Type', 'application/json')
		return utils.getRecurso(md5, key)

class JsonAllClasses():
	def GET(self, md5):
		check_sesion()
		checkCache(md5)
		ret = []
		clases = utils.findPackages(utils.REPORTS_DIR + md5 +'/ddx/')
		for pack in clases:
			ret += [{"pack":pack,"clases":clases[pack]}]
		web.header('Content-Type', 'application/json')
		return json.dumps(ret)

class RecursosView():
	def GET(self, md5, key):
		check_sesion()
		checkCache(md5)
		absolute_path = utils.getPathFromRecurso(md5, key);
		if absolute_path:
			if absolute_path.endswith(".xml"):
				file = ""
				with open(absolute_path, 'r') as fp:
					file = fp.read()
					fp.close()
				return render_base.view(md5, file, "xml:nocontrols",True)
			raise web.seeother(absolute_path[1:])
		return web.notfound()

class View():
	def GET(self, md5, clase, class_name):
		check_sesion()
		checkCache(md5)
		type = None
		file = clase.replace(".","/")

		if os.path.isfile(utils.REPORTS_DIR+md5+"/smali/smali/"+file+".smali"):
			type = "smali:nocontrols"
			file = utils.REPORTS_DIR+md5+"/smali/smali/"+file+".smali"
		elif os.path.isfile(utils.REPORTS_DIR+md5+"/ddx/"+file+".ddx"):
			type = "ddx:nocontrols"
			file = utils.REPORTS_DIR+md5+"/ddx/"+file+".ddx"
		else:
			return web.notfound()

		with open(file, 'r') as fp:
			file = None
			file = fp.read()
			fp.close()
		return render_base.view(md5, file, type, False)

class Uploader:
	def GET(self):
		check_sesion()
		return render_base.upload(None)

	def POST(self):
		check_sesion()
		ficheros = web.input(android_file={})
		error = "Solo se soportan ficheros APK y DEX"

		resumen = hashlib.md5()
		resumen.update(ficheros['android_file'].value)
		md5_file = resumen.hexdigest()
		#TODO ya tengo el fichero?

		magic = ficheros['android_file'].file.read(3)
		file = None
		if magic[:2] == "PK":
			file = utils.FILE_UPLOAD_DIR + md5_file + ".apk"
		elif magic == "dex":
			file = utils.FILE_UPLOAD_DIR + md5_file + ".dex"

		#TODO ya tengo el fichero?
		if file:
			with open(file, 'wb') as fp:
				fp.write(magic)
				fp.write(ficheros['android_file'].file.read())
				fp.close()
				if file.endswith(".apk"):
					error = "No es un fichero APK valido"
					if zipfile.is_zipfile(file):
						zip = zipfile.ZipFile(file, 'r')
						error_file = zip.testzip()
						if error_file is not None:
							error = "Existen fichero erroneos en el APK ("+error_file+")"
						else:
							elementos = zip.namelist()
							if "classes.dex" in elementos and "AndroidManifest.xml" in elementos:
								error = None
						zip.close()
				else:
					error = None
				if error:
					os.remove(file)
		if error:
			return render_base.upload(error)
		raise web.seeother('/report/'+md5_file)

def test():
	app.run()

if __name__ == "__main__":
	app.run()


#!/usr/local/bin/python3
# -*- coding:utf-8 -*-
# CIS: Commodity Inventory System (商品盘点系统)

from openpyxl import load_workbook
import sqlite3
import os
import time
import re

# ------------------------- 全局变量 start -------------------------
DATABASE_PATH = '/Users/Penfree/Workspace/Programs/PythonProjects/Codes/commoditiesImport/commodities.db'
RT = '' # RECORDS_TABLE_NAME
MT = 'main' # MAIN_TABLE_NAME
CONNECT = sqlite3.connect(DATABASE_PATH)
CURSOR = CONNECT.cursor()
# ------------------------- 全局变量 end -------------------------

# ------------------------- 工具集 start -------------------------
def createTable(t_name):
    ''' 创建表

        t_name: table name, 表名, 不能为空, 为空时报错返回
    '''
    log('新建作业表')
    if isinstance(t_name, str) and len(t_name) > 0:
        log('检查作业表名')
    else:
        log('作业表名无效')
        return
    #
    sql = "CREATE TABLE IF NOT EXISTS {} \
    (idx INTEGER PRIMARY KEY AUTOINCREMENT,\
    barcode TEXT NOT NULL UNIQUE,\
    name TEXT NOT NULL,\
    brief TEXT DEFAULT '',\
    brand TEXT DEFAULT 'NA',\
    placeOfOrigin TEXT DEFAULT '见包装',\
    unit TEXT DEFAULT 'PCS',\
    spec TEXT DEFAULT 'NA',\
    cate TEXT DEFAULT 'NA',\
    retailPrice TEXT DEFAULT '0',\
    shelfLife TEXT DEFAULT '见包装',\
    commission TEXT DEFAULT '0',\
    points TEXT DEFAULT '1',\
    discount TEXT DEFAULT '1',\
    weighed TEXT DEFAULT '0',\
    settlementType TEXT DEFAULT '2',\
    viprice TEXT DEFAULT '0',\
    tradePrice TEXT DEFAULT '0',\
    storeNum TEXT DEFAULT '0',\
    cost TEXT DEFAULT '0',\
    saleOnWechat TEXT DEFAULT '1',\
    existing TEXT DEFAULT '1');".format(t_name)
    #
    CURSOR.execute(sql)
    CONNECT.commit()

def isStr(content):
    ''' 判断内容是否为有效的字符串
        0, 否: 则返回None
        1, 是: 则返回被判断对象本身
    '''
    if isinstance(content, str):
        return content
    else:
        return

def log(content):
    ''' 按开发文档格式打印
    '''
    if isStr(content):
        print('[CIS] {}'.format(content))
    else:
        print('[CIS] ')

def wLog(content):
    ''' warnings log 警示日志
    '''
    if isStr(content):
        log('**{}**'.format(content))

def uin(placeholder):
    ''' user input 提示用户输入
    '''
    if isStr(placeholder):
        content = input('[CIS] {} >> '.format(placeholder))
        return content
    else:
        log('请提示用户输入的内容')
        return

def isArray(obj):
    ''' 判断对象是否为数组, 不区分是list还是tuple
        0, 返回None
        1, 返回被判断对象本身
    '''
    if isinstance(obj, list) or isinstance(obj, tuple):
        return obj
    else:
        return

def searchSql(k, t):
    ''' 生成搜索商品SQL语句

        k: keyword 搜索关键字

        t: table 表名
    '''
    if isStr(k) and isStr(t):
        return 'SELECT * FROM {0} WHERE barcode LIKE "%{1}%" OR name LIKE "%{1}%"'.format(t, k)
    else:
        log('参数错误')
        return

def receipt(prompt, filter = r'', required = False):
    ''' 接收用户的输入

        prompt: 输入内容提示

        filter: 过滤规则(正则表达式)

        required: 是否必填(可不传值, 默认为0)
    '''
    if isStr(prompt):
        pass
    else:
        log('请增加用户输入的内容')

    while True:
        content = uin(prompt)
        if required:
            if 0 == len(content):
                wLog('本项必填')
                continue
        if isStr(filter) and filter:
            # 为插入数据库定制, 限制用户输入内容
            r = re.search(filter, content)
            if r:
                # 有匹配结果时, 返回匹配结果
                r = r.group()
                return '{}'.format(r)
            elif required:
                # 没有匹配到结果时, 查看本项是否为必填项, 若是则让用户重新输入, 否则进入else:->返回一个空字符串
                log('输入有误, 请检查!')
                continue
            else:
                return ''
        else:
            # 过滤规则错误
            return content

def convertArrayToStr(commodityInfo):
    ''' 将数组的所有元素按顺序拼接为字符串

        commodityInfo: 商品信息, 列表类型
    '''
    result = ''
    if isValidCommodity(commodityInfo):
        split = ''
        for v in commodityInfo:
            # 当result为空时, 不需要间隔符
            if result and (not split):
                split = ','
            #
            v_upper = str(v).upper()
            if (not str(v)) or 'NULL' == v_upper or 'NONE' == v_upper:
                tempV = 'Null'
            else:
                tempV = '\"{}\"'.format(str(v))
            result = '{}{}{}'.format(result, split, tempV)
        return result
    else:
        log('商品信息无效')
        return

def isValidCommodity(commodityInfo):
    ''' 按格式判断商品信息是否有效

        commodityInfo: 长度为22的数组
    '''
    if isArray(commodityInfo) and 22 == len(commodityInfo):
        return commodityInfo
    else:
        return

# ------------------------- 工具集 end -------------------------

# ------------------------- 功能模块 start -------------------------
def updateMainTable():
    ''' 更新主数据库, 耗时操作, 请谨慎调用.
        为提升效率, 本操作取消逐行比对商品信息,
        直接清空数据库主表后再重新灌入数据
    '''
    createTable('main')
    wLog('本操作会覆盖同步数据库, 可能会造成数据丢失, 若无必要请勿更新!!!')
    update = uin('是否更新主数据库(yes?):')
    if 'YES' != update.upper():
        log('主数据库未更新')
        return
    #
    path = '/Volumes/storage/DataBank/商品数据/03商品总目录/重要_在售商品总汇.xlsx'
    try:
        # 清空数据库
        CURSOR.execute('DELETE FROM main')
        CONNECT.commit()
        # 准备数据
        wb = load_workbook(path)
        ws = wb[wb.sheetnames[0]]
        # 进度
        progress = 1
        for row in ws.iter_rows(min_row = 2, max_col = 20):
            try:
                cvs = '' # cell value series
                for col in row:
                    v = col.value
                    cvs = "{}, \'{}\'".format(cvs, v)
                # 主数据库与excel文档少首尾两个字段("索引(idx)" 和 "在档(existing)")
                insertSql = "INSERT INTO main VALUES ({}{}, '1')".format(progress, cvs)
                #=======================================^这里不需要"," cvs首位存在一个","
                progress += 1
                #
                CURSOR.execute(insertSql)
                CONNECT.commit()
            except:
                continue
    except FileNotFoundError:
        log('"重要_在售商品总汇.xlsx"文件不存在')
    except:
        log('错误, 无法更新主数据库')

def creatRecordsTable():
    ''' 每一次运行本程序, 都创建一个作业表用于记录商品信息操作
    '''
    global RT
    log('作业表不存在, 准备创建')
    # 以时间组成作业表名, 格式:"月_日_时_分_秒"
    t = time.localtime()
    RT = 't_{:02}{:02}{:02}{:02}{:02}'.\
    format(t.tm_mon, t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec)
    createTable(RT)
    log('创建完成')
    # 用log文件保存创建过的作业表名
    with open('records.log', 'a') as f:
        f.write(RT + '\n')

def query():
    ''' 商品信息查询入口
    '''
    while True:
        k = uin('请输入商品名或条形码(<q> to exit)')
        if 'Q' == k.upper():
            return
        if not k:
            continue
        else:
            queryInRecords(k)

def queryInRecords(k):
    ''' 在作业表中搜寻商品信息, 当查无结果时自动搜寻主表
        1, 如果存在于主表, 则将该条信息直接复制到作业表
        2, 不存在于主表, 则创建新商品信息.
        k: keyword 关键字
    '''
    log('搜寻作业表')
    if not RT:
        # 作业表不存在, 创建作业表, 在主表搜寻
        creatRecordsTable()
        queryInMain(k)
    else:
        # 在作业表中搜寻
        results = resultsFromDB(k, RT)
        if results:
            r = resultSelector(results)
            showCommodityInfo(r)
            modify = uin('<m>修改, <Any Key>跳过')
            if 'M' == modify.upper():
                newInfo = updateCommodity(r)
                if isValidCommodity(newInfo):
                    CURSOR.execute('DELETE FROM {} WHERE barcode = "{}"'.format(RT, r[1]))
                    CURSOR.execute('INSERT INTO {} VALUES ({})'.format(RT, convertArrayToStr(newInfo)))
                    CONNECT.commit()
        else:
            log('未发现商品')
            queryInMain(k)

def queryInMain(k):
    ''' 在主表中搜寻, 若商品信息存在, 则将商品信息复制到作业表
        若主表中不存在该商品信息, 则创建!
    '''
    log('搜寻主表')
    results = resultsFromDB(k, MT)
    if results:
        r = resultSelector(results)
        if isValidCommodity(r):
            showCommodityInfo(r)
            copyInfoToRecords(r)
    else:
        log('未发现商品')
        typedIn = uin('是否添加(ABort 取消)?')
        if 'AB' == typedIn.upper():
            return
        else:
            newCommodity = updateCommodity(None, 1)
            if isValidCommodity(newCommodity):
                CURSOR.execute('INSERT INTO {} VALUES ({})'.format(RT, convertArrayToStr(newCommodity)))
                CONNECT.commit()
            else:
                log('新建的商品信息有误, 请检查后重试!')

def resultsFromDB(k, tableName = MT):
    ''' 在数据库中查找数据

        k: keyword, 关键字

        tableName: 表名, 默认是MT(Main Table)
    '''
    if isStr(k) and k and isStr(tableName) and tableName:
        pass
    else:
        log('查询条件有误')
        return
    #
    sql = searchSql(k, tableName)
    if sql:
        try:
            CURSOR.execute(sql)
            results = CURSOR.fetchall()
            return results
        except:
            log('查询商品信息出错')
            return
    else:
        log('查询条件不正确, 无法查询')
        return

def resultSelector(resultsList):
    ''' 多个结果时, 让用户选择一个结果返回
        单个结果时, 直接返回
    '''
    if not isArray(resultsList):
        return
    if 1 == len(resultsList):
        return resultsList[0]
    #
    for r in resultsList:
        showCommodityInfo(r, resultsList.index(r))
    while True:
        idx = uin('请选择商品(<q> to exit)')
        if 'q' == idx:
            return
        num = re.search(r'([0-9]+|0+)', idx)
        if num:
            num = int(num.group())
            if num >= len(resultsList):
                log('超出范围, 请重新选择')
                continue
            else:
                return resultsList[num]
        else:
            log('输入有误, 请重新输入')
            continue

def showCommodityInfo(commodityInfo, idx = 0):
    ''' 展示结果
    '''
    if isValidCommodity(commodityInfo):
        log('{0:02} {1[1]} - {1[2]} - {1[18]} - {1[7]} - {1[10]} - {1[19]} - {1[9]}'.format(idx, commodityInfo))

def copyInfoToRecords(commodityInfo):
    ''' 从主表复制商品信息到作业表
        复制之前询问是否修改信息
    '''
    modify = uin('<m>修改, <Any Key>跳过')
    if 'M' == modify.upper():
        tempInfo = updateCommodity(commodityInfo)
    else:
        tempInfo = list(commodityInfo)
        tempInfo[0] = 'Null' # 为避免主键冲突造成崩溃, 让其自匹配
    #
    if isValidCommodity(tempInfo):
        sql = 'INSERT INTO {} VALUES ({})'.format(RT, convertArrayToStr(tempInfo))
        CURSOR.execute(sql)
        CONNECT.commit()

def updateCommodity(commodityInfo, method = 0):
    ''' 更新商品信息(修改 or 新建)

        mothod: 类型
        0, 修改商品(默认值)
        1, 新建商品
    '''
    newCommodity = []
    if method:
        # 新建商品信息
        method = True # 修正method参数值类型
        newCommodity = ['NULL' for i in range(22)]
    else:
        method = False
        if isValidCommodity(commodityInfo):
            newCommodity = list(commodityInfo)
        else:
            log('商品信息错误, 无法修改')
            return
    #
    name = receipt('商品名', required = method)
    barcode = receipt('条形码', r'\d{6,16}', required = method)
    spec = receipt('规格')
    shelfLife = receipt('保质期', r'\d{0,5}')
    cost = receipt('成本价', r'([1-9]\d*|0)\.{0,1}\d{0,2}')
    retail = receipt('零售价', r'([1-9]\d*|0)\.{0,1}\d{0,2}')
    stored = receipt('*** 库存 ***', r'[1-9]\d{0,5}')

    #
    if name:
        newCommodity[2] = name
    if barcode:
        newCommodity[1] = barcode
    if spec:
        newCommodity[7] = spec
    if shelfLife:
        newCommodity[10] = shelfLife
    if cost:
        newCommodity[19] = cost
    if retail:
        newCommodity[9] = retail
    if stored:
        newCommodity[18] = stored

    # 手动修改以下两个值(是否微店销售, 是否存在main table)
    # 当商品名/条形码/成本价/零售价/规格任意一个参数更改都视作新产品, 都将'是否存在主表'值设为0.
    if (name or barcode or cost or retail) and ('0' != newCommodity[-1]):
        newCommodity[-1] = '0'

    if '1' != newCommodity[-2]:
        newCommodity[-2] = '1'
    return newCommodity
# ------------------------- 功能模块 end -------------------------

# ------------------------- 程序入口 start -------------------------
def main():
    ''' 主函数, 程序入口
    '''
    log('您好, 欢迎使用本系统.')
    updateMainTable()
    query()
    # 任务完成后, 断开数据库连接
    CONNECT.close()
#
main()
# ------------------------- 程序入口 end -------------------------


#!/usr/bin/python3
from PyQt5.QtCore import Qt,QTimer,QDateTime,QDate,QTime
from PyQt5.QtWidgets import (QApplication, QGridLayout, QLabel, QLineEdit, QMessageBox, QPushButton, QTextEdit, QWidget)
from PyQt5.QtGui import QPixmap
import math,sys,datetime

class Main_window(QWidget):
	def set_time(self):
		self.czas = datetime.datetime(2015,2,3,14,2)
		self.teraz = datetime.datetime.now()
		self.pozostaly=self.czas - self.teraz
		self.time_label.setText("Pozostało: "+str(self.pozostaly))

	def __init__(self):
		super().__init__()
		self.setWindowTitle("Zegar Śmierci")
		self.layout=QGridLayout()
		self.time_label=QLabel()
		self.layout.addWidget(self.time_label)
		self.setLayout(self.layout)
		self.timer=QTimer()
		self.timer.timeout.connect(self.set_time)
		self.timer.start(0)
		self.obrazek = QPixmap("Mentzen.png")
		self.mietek = QLabel()
		self.mietek.setPixmap(self.obrazek)
		self.layout.addWidget(self.mietek)

app = QApplication(sys.argv)
window = Main_window()
window.show()
sys.exit(app.exec())


#!/usr/bin/python
# -*- coding: utf-8 -*-
""" A Tank Game to teach coding those unfamiliar with it"""

import pygame
import sys
from pygame.locals import *
from PIL import Image
import random
import time

pygame.init()

# Set the width and height of the screen

infoObject = pygame.display.Info()
size = (infoObject.current_w, infoObject.current_h - 30)

# Used to manage how fast the screen updates

clock = pygame.time.Clock()
class tank:

    def __init__(self):
        self.origional_image = pygame.image.load('KV-2_Tank.png')
        self.image = self.origional_image
        self.rotation = 0
        self.coords = (0, 0)
        self.move_count = 0  # # Counts the number of tims a function is iterated

    def rotate(self, degrees):
        """ This program will rotate 1 degree everytime called, untill it has
            reached the desired rotation, then it will advance the program counter
        """

        global program_counter

        if degrees > 0 and self.move_count < degrees:
            self.rotation += 1
            self.move_count += 1
        elif degrees < 0 and self.move_count > degrees:
            self.rotation -= 1
            self.move_count -= 1
        else:
            self.move_count = 0
            program_counter += 1

            # # Converting large negetive rotations to the corresponding small positive ones

            if abs(self.rotation) > 180:
                if self.rotation < 0:
                    self.rotation = 360 + self.rotation
                elif self.rotation > 0:
                    self.rotation = -360 + self.rotation

    def move(self, distance):
        """ This program will move 10 pixels everytime called, untill it has
            reached the desired position, then it will advance the program counter
        """

        global program_counter

        self.move_count += 1
        (x, y) = self.coords

        if distance < 0:
            factor = -1
        elif distance == 0:
            factor = 0
        else:
            factor = 1

        if self.move_count < abs(distance):
            y += factor * -(5 - abs(self.rotation) / 18)
            if abs(self.rotation) <= 90:
                x -= factor * self.rotation / 18
            elif self.rotation > 0:
                x -= factor * (180 - self.rotation) / 18
            else:
                x -= factor * -(180 - abs(self.rotation)) / 18
        else:

            program_counter += 1
            self.move_count = 0

        self.coords = (x, y)

    def fire(self):
        """ Fires a bullet from the tank"""

        global program_counter
        global bullet_list
        x,y = self.coords
        bullet_list.append(bullet((x, y- 1/2 * self.image.get_height()), self.rotation))
        program_counter += 1

    def delay(self, milliseconds):
        """Delays the tank's movement"""

        global program_counter
        time.sleep(milliseconds * 0.001)
        program_counter += 1

    def draw(self):
        """ Draws the image of the tank on the playing surface"""

        self.image = _rot_center(self.origional_image, self.rotation)
        screen.blit(self.image, self.coords)

class bullet:

    def __init__(self, coordinates, rotation):
        self.origional_image = \
            pygame.transform.scale(pygame.image.load('bullet.png'),
                                   (20, 10))
        self.image = self.origional_image
        self.rotation = rotation
        self.coords = coordinates
        self.move_count = 0  ## Counts the number of tims a function is iterated

    def move(self, distance):
        """ This program will move 10 pixels everytime called, untill it has
        reached the desired position, then it will advance the program counter
        """

        self.move_count += 1
        (x, y) = self.coords

        if distance < 0:
            factor = -1
        elif distance == 0:
            factor = 0
        else:
            factor = 1

        if self.move_count < abs(distance):
            y += factor * -(5 - abs(self.rotation) / 18)
            if abs(self.rotation) <= 90:
                x -= factor * self.rotation / 18
            elif self.rotation > 0:
                x -= factor * (180 - self.rotation) / 18
            else:
                x -= factor * -(180 - abs(self.rotation)) / 18
        else:

            self.move_count = 0

        self.coords = (x, y)

    def draw(self):
        """ Draws the image of the tank on the playing surface"""

        self.image = _rot_center(self.origional_image, self.rotation
                                 + 90)
        screen.blit(self.image, self.coords)

def _rot_center(pygame_image, degrees):
    """ Rotate an image while keeping its center and size, by converting to pil, rotating and back"""

    image_string = pygame.image.tostring(pygame_image, 'RGBA', False)
    PIL_image = Image.frombytes('RGBA', pygame_image.get_size(),
                                image_string)
    PIL_image = PIL_image.rotate(degrees)
    image_string = PIL_image.tobytes()
    pygame_image = pygame.image.fromstring(image_string,
            PIL_image.size, 'RGBA')
    return pygame_image

def get_program(nest):
    """ This function gets the tank program from the user"""

    user_in = True
    program = []

    # # Gets program from user

    while user_in:
        user_in = input('>>> ' + nest * '  ')
        if 'tank.' in user_in:
            user_in = 'tank_program.append("' + user_in + '")'

        # # If a conditional, it runs 'get_program' to get the conditional
        # # the code under the conditional,and appends it as one string
        # # separated by '\n' to the program list. It counts the number of
        # # 'nests' to print the correct number of tabs.

        if ':' in user_in:
            nested_program = get_program(nest + 1)
            for command in nested_program:
                user_in += '\n' + (nest + 1) * '  ' + command
        program.append(user_in)
    return program

def start(program):
    """ This function is called, given the input data, collected from the user
        and processed by 'get_program'. Given this list of commands, the computer
        moves the tank on the screen.
    """

    # # Calculating the moves of the Tank. Compiling

    global program_counter
    while True:
        if program_counter < len(program) - 1:
            exec(program[program_counter])
            program_counter += 1
        else:
            break

    # # Executing the moves of the tank. Interpreting

    program_counter = 0
    while True:
        if program_counter < len(tank_program):
            print(tank_program[program_counter])
            exec(tank_program[program_counter])
        screen.fill((255, 255, 255))
        for bullet in bullet_list:
            bullet.move(10)
            bullet.draw()

        tank.draw()
        pygame.display.flip()
        time.sleep(0.001)
        clock.tick(60)

program = get_program(0)
tank_program = []
program_counter = 0
tank = tank()
bullet_list = []

screen = pygame.display.set_mode(size)  # , FULLSCREEN)
start(program)
# Close the window and quit.
pygame.quit()

Tank Game/Outdated/Tank Game compiled.py


#===== type hints ===========
from typing import *

x1 = 10  # type: int
# x2 = 10  # type: str

x3: int = 15

def my_plus(x: int, y: Optional[int]=None) -> int:
    if y is None:
        y = 88
    return x + y

print(my_plus(1, 2))
print(my_plus(2))
# print(my_plus("hello", "world"))

def g(x: Sequence[int]) -> None:
    print(len(x))
    print(x[1])
    for i in x:
        print(i)
    print()

g([1,2,3,4])
g((1,2,3,4))
# g('ssjlsjisjsij')

#===== defaultdict =====
from collections import defaultdict
from pprint import pprint
names = '''bob alice anna abba bot cat caro candy dom doc dog dot ab'''.split()
d = defaultdict(list)
for name in names:
    feature = name[0]
    d[feature].append(name)

pprint(d, depth=40)

pprint(sorted(names, key=len))


import logging
import traceback

from aiohttp import web
from aiohttp.web_exceptions import HTTPError

import settings
from core.utils import json_to_str

log = logging.getLogger(__name__)

def _form_error_response(
    user_message: str, status_code: int, internal_message: str = None
):
    """Функция для формирования ответа с ошибками

    :param user_message:
    :param status_code:
    :param internal_message:
    :return:
    """
    body = {
        "errors": [
            {
                "message": user_message,
                "internal_message": internal_message,
            }
        ]
    }
    return web.Response(
        body=json_to_str(body).encode("utf-8"),
        content_type="application/json",
        status=status_code
    )

async def request_middleware(app, handler):
    """Прослойка для отлова базовых ошибок

    :param app:
    :param handler:
    :return:
    """
    async def middleware(request):
        try:
            response = await handler(request)
        except HTTPError as e:
            response = _form_error_response(
                user_message=e.reason,
                status_code=e.status_code
            )
        except:
            log.exception("Unexpected exception")
            response = _form_error_response(
                user_message="Sorry, the internal server error",
                internal_message=traceback.format_exc(),
                status_code=500
            )

        log.info(f"{request.method} {request.path}{request.query_string}")
        response.headers.add("Access-Control-Allow-Origin", settings.ACCESS_CONTROL_ALLOW_ORIGIN)
        response.headers.add("Access-Control-Allow-Headers", settings.ACCESS_CONTROL_ALLOW_HEADERS)
        return response

    return middleware


"""Implemente uma função que,dada uma lista de strings,
devolve uma lista contendo,para cada string, o seu número
de caracteres. Sua função  deve fazer isso recursivamente"""
quantidade = []
def Qnt(lista):
    global quantidade
    string = lista[0]
    aux = string
    aux = list(aux)
    quantidade.append(len(aux))
    lista.remove(string)
    if len(lista) == 0:
        return quantidade
    return Qnt(lista)

print(Qnt(['abc','defg','asodiasdo','fsdjfoskdfjso','abcde']))


# coding: utf8
# try something like

#display all the available programs and allow them to be edited
@auth.requires_login()
def index():
    rows = db(db.program.id > 0).select()
    test = SQLFORM.grid(db.program.id > 0)
    return dict(rows=rows,test=test)

#create a new flag path, starting with the name of the program.
#year will automatically be inserting by web2py
@auth.requires_login()
def new():
    form = SQLFORM(db.program)
    if form.validate():
        form.vars.id = db.program.insert(**dict(form.vars))
        redirect(URL(c='programs',f='dates',args=[form.vars.id]))
    return dict(form=form)

#receive the id of the program just created by new and use it to build a
#list of dates that the program will exist
@auth.requires_login()
def dates():
    prog_id = request.args[0]
    program = db.program(db.program.id == prog_id)
    dates_query = db.prog_date.program_ref == prog_id
    rows = db(dates_query).select()
    form = SQLFORM(db.prog_date)
    form.add_button('Next Step', URL('programs','authorized_users',args=[prog_id]))
    if form.validate():
        form.vars.id = db.prog_date.insert(program_ref = prog_id,
        program_date = form.vars.program_date)
        redirect(URL(c='programs',f='dates',args=[prog_id]))
    return dict(rows = rows, form = form, program = program)
#receive the id of the prgoram created by new and use it to build a database of
#who can access the program just created.
@auth.requires_login()
def authorized_users():
    prog_id = request.args[0]
    program = db.program(db.program.id == prog_id)
    rows = db(db.prog_allowable.program_ref == prog_id).select()
    form = SQLFORM(db.prog_allowable)
    form.add_button('Next Step',URL('programs','program_requires',args=[prog_id]))
    if form.validate():
        form.vars.id = db.prog_allowable.insert(program_ref = prog_id, user_type = form.vars.user_type)
        redirect(URL('programs','authorized_users',args=[prog_id]))
    return dict(program=program,rows=rows,form=form)
#receive the id of the program created by new and use it to build a database set of
#what features are required to be displayed in this program
@auth.requires_login()
def program_requires():
    prog_id = request.args[0]
    program = db.program(db.program.id == prog_id)
    rows = db(db.prog_requires.program_ref == prog_id).select()
    form = SQLFORM(db.prog_requires)
    if form.validate():
        form.vars.id = db.prog_requires.insert(program_ref = prog_id, features = form.vars.features)
        redirect(URL('programs','program',args=[prog_id]))
    return dict(program = program, rows = rows, form = form)

@auth.requires_login()
def academic_advising():
    prog_id = request.args[0]
    program = db.program(db.program.id == prog_id)
    rows = db(db.academic_advising.program_ref == prog_id).select()
    form = SQLFORM(db.academic_advising)
    def validateaa(form):
        form.vars.id = prog_id
    if form.process(onvalidation = validateaa).accepted:
        response.flash = T('success')
        redirect(URL('programs','academic_advising',args=[prog_id]))
    return dict(program = program, rows = rows, form = form)

@auth.requires_login()
def afternoon_advising():
    prog_id = request.args[0]
    program = db.program(db.program.id == prog_id)
    rows = db(db.afternoon_advising.program_ref == prog_id).select()
    form = SQLFORM(db.afternoon_advising)
    def validateaaf(form):
        form.vars.id = prog_id
    if form.process(onvalidation = validateaaf).accepted:
        response.flash = T('success')
        redirect(URL('programs','afternoon_advising',args=[prog_id]))
    return dict(program = program, rows = rows, form = form)

@auth.requires_login()
def breakout_sessions():
    prog_id = request.args[0]
    program = db.program(db.program.id == prog_id)
    rows = db(db.breakout_sessions.program_ref == prog_id).select()
    form = SQLFORM(db.breakout_sessions)
    def validatebs(form):
        form.vars.id = prog_id
    if form.process(onvalidation = validatebs).accepted:
        response.flash = T('success')
        redirect(URL('programs','breakout_sessions',args=[prog_id]))
    return dict(program = program, rows = rows, form = form)

@auth.requires_login()
def department_advising():
    prog_id = request.args[0]
    program = db.program(db.program.id == prog_id)
    rows = db(db.department_advising.program_ref == prog_id).select()
    form = SQLFORM(db.department_advising)
    def validateda(form):
        form.vars.id = prog_id
    if form.process(onvalidation = validateda).accepted:
        response.flash = T('success')
        redirect(URL('programs','department_advising',args=[prog_id]))
    return dict(program = program, rows = rows, form = form)

@auth.requires_login()
def online_enrollment():
    prog_id = request.args[0]
    program = db.program(db.program.id == prog_id)
    rows = db(db.online_enrollment.program_ref == prog_id).select()
    form = SQLFORM(db.online_enrollment)
    def validateda(form):
        form.vars.id = prog_id
    if form.process(onvalidation = validateda).accepted:
        response.flash = T('success')
        redirect(URL('programs','online_enrollment',args=[prog_id]))
    return dict(program = program, rows = rows, form = form)

@auth.requires_login()
def academic_cluster():
    prog_id = request.args[0]
    program = db.program(db.program.id == prog_id)
    rows = db(db.academic_cluster.program_ref == prog_id).select()
    form = SQLFORM(db.academic_cluster)
    def validateac(form):
        form.vars.id = prog_id
    if form.process(onvalidation = validateac).accepted:
        response.flash = T('success')
        redirect(URL('programs','academic_cluster',args=[prog_id]))
    return dict(program = program, rows = rows, form = form)

#**********
#Meat of this controller, the user should be able to populate the program using the features requested
#should only option to CRUD on a table if the feature was requested, otherwise the user should not be
#allowed to input items for that specific program if the feature was not requested.
@auth.requires_login()
def program():
    prog_id = request.args[0]
    program = db.program(db.program.id == prog_id)
    dates = db(db.prog_date.program_ref == prog_id).select()
    ath_users = db(db.prog_allowable.program_ref == prog_id).select()
    features = db.prog_requires(db.prog_requires.program_ref == prog_id)
    if dates is None:
        response.flash = T('The program is missing dates!')
        redirect(URL('programs','dates',args=[program.id]))
    if ath_users is None:
        response.flash = T('The program is missing user authorizations!')
        redirect(URL('programs','authorized_users',args[program.id]))
    if features is None:
        response.flash = T('The program does not have any features')
        redirect(URL('prgorams','program_requires',args=[program.id]))
    return dict(program = program, dates = dates, ath_users = ath_users, features = features)

@auth.requires_login()
def schedule():
    prog_id = request.args[0]
    program = db.program(db.program.id == prog_id)
    form = SQLFORM(db.schedule)
    form.add_button(program.name + ' Main',URL('programs','program',args=[prog_id]))
    rows = db(db.schedule.program_ref == prog_id).select(orderby=db.schedule.start)
    if form.validate():
        form.vars.id = db.schedule.insert(program_ref = prog_id, item = form.vars.item,
            start = form.vars.start, end = form.vars.end, loc = form.vars.loc,
            summary = form.vars.summary, note= form.vars.note)
        redirect(URL('programs','schedule',args=[prog_id]))
    return dict(form = form, program = program, rows = rows)
@auth.requires_login()
def schedule_augment():
    prog_id = request.args[0]
    schedule_id = request.args[1]
    schedule_item = db.schedule(db.schedule.id == schedule_id)
    program = db.program(db.program.id == prog_id)
    form = SQLFORM(db.schedule_augment)
    if form.validate():
        form.vars.id = db.schedule_augment.insert(program_ref = prog_id, schedule_ref = schedule_id,
            augment = form.vars.augment)
        redirect(URL('programs','schedule'))
    return dict(form=form, schedule_item = schedule_item,program = program)

from xml.etree import ElementTree as ET
from io import BytesIO
import ast
@auth.requires_login()
def maps():
    prog_id = request.args[0]
    program = db.program(db.program.id == prog_id)
    map_form = SQLFORM(db.flag_path)
    paths = db(db.flag_path.program_ref == prog_id).select()
    def validatePath(form):
        form.vars.id = prog_id
    if map_form.validate():
        map_form.vars.id = db.flag_path.insert(program_ref = prog_id, color = map_form.vars.color,
            color_hex = map_form.vars.color_hex, start_loc = map_form.vars.start_loc, end_loc = map_form.vars.end_loc,
            coor_list = map_form.vars.coor_list,
            major = map_form.vars.major)
        if(map_form.vars.kml == ''):
            response.flash = T('Flag path processed successfully')
            return dict(map_form = map_form, paths = paths)
        record = db.flag_path(db.flag_path.id == map_form.vars.id)
        kml_file_like = BytesIO(map_form.vars.value)
        r_str = ''
        touples = []
        namespace = '{http://www.opengis.net/kml/2.2}'
        root = ET.parse(kml_file_like).getroot()
        document = root.find(namespace+'Document').find(namespace+'Folder')
        placemarks = document.findall(namespace+'Placemark')
        i = 0
        for mark in placemarks:
            point = mark.find(namespace+'Point').find(namespace+'coordinates')
            touples.append(ast.literal_eval(point.text))
        record.coor_list = touples
        record.update_record()
        response.flash = T('Flag path processed successfully')
    return dict(map_form = map_form, paths = paths,program = program)