#!/usr/bin/env python3

__version__ = "0.4.5"
__author__ = "Jens Luebeck"

import argparse
import copy
from math import log
import operator
import sys

from ac_util import *
import get_genes

tot_min_del = 5000  # minimum size of deletion before non-trivial
minCycleSize = 10000
compCycContCut = 50000
cycCut = 0.12
compCut = 0.3
min_upper_cn = 4.5
decomposition_strictness = 0.1

# bfb thresholds
min_score_for_bfb = 0.25
# min_fb_reads_for_bfb = 10
fb_dist_cut = 25000

#graph properties
graph_cns = defaultdict(IntervalTree)

# ------------------------------------------------------------
# Methods to compute values used in classification
def get_size(cycle, segSeqD):
    return sum(segSeqD[abs(x)][2] - segSeqD[abs(x)][1] for x in cycle)

def weightedCycleAmount(cycle, cn, segSeqD):
    # get length of cycle
    sc_length = get_size(cycle, segSeqD) / 1000.
    return sc_length * cn

def get_diff(e1, e2, segSeqD):
    p1_abs = segSeqD[abs(e1)]
    p2_abs = segSeqD[abs(e2)]
    if e1 == 0 or e2 == 0:
        return 1

    p1_end = p1_abs[2] if e1 > 0 else p1_abs[1]
    p2_start = p2_abs[1] if e2 > 0 else p2_abs[2]
    return abs(p2_start - p1_end)

def isCircular(cycle):
    circular = False if cycle[0] == 0 and cycle[-1] == 0 else True
    return circular

def isRearranged(cycle, segSeqD):
    # check if it contains regions from multiple chroms
    chromList = [segSeqD[abs(ind)][0] for ind in cycle if ind != 0]
    if len(set(chromList)) > 1:
        return True

    max_del_size = 0
    for i in range(0, len(cycle) - 1):
        if cycle[i] == 0 or cycle[i + 1] == 0:
            continue
        if cycle[i] < 0 and cycle[i + 1] > 0 or cycle[i] > 0 and cycle[i + 1] < 0:
            return True

        dist_diff = get_diff(cycle[i], cycle[i + 1], segSeqD)
        max_del_size = max(dist_diff, max_del_size)
        # tot_del_size += dist_diff
        # if tot_del_size > tot_min_del:
        #     print("Delsize")
        #     return True
    if max_del_size > tot_min_del:
        return True

    return False

def tot_rearr_edges(graphf, add_chr_tag):
    rearr_e = 0
    with open(graphf) as infile:
        for line in infile:
            if line.startswith("discordant"):
                fields = line.rstrip().rsplit()
                lbp, rbp = fields[1].split("->")
                lchrom, lpd = lbp.rsplit(":")
                rchrom, rpd = rbp.rsplit(":")
                if add_chr_tag and not lchrom.startswith("chr"):
                    lchrom = "chr" + lchrom
                    rchrom = "chr" + rchrom

                lpos, ldir = int(lpd[:-1]), lpd[-1]
                rpos, rdir = int(rpd[:-1]), rpd[-1]
                if ldir == rdir:
                    # if lchrom == rchrom and abs(rpos - lpos) < fb_dist_cut:
                    rearr_e += 1

                elif abs(rpos - lpos) > fb_dist_cut:
                    rearr_e += 1

    return rearr_e

def decompositionComplexity(graphf, cycleList, cycleCNs, segSeqD, feature_inds, exclude_inds, add_chr_tag):
    #construct intervaltree of valid regions
    hit_region_it = defaultdict(IntervalTree)
    for i in feature_inds:
        cycle = cycleList[i]
        for cv in cycle:
            if cv != 0:
                c, s, e = segSeqD[abs(cv)]
                hit_region_it[c].addi(s, e+1)

    hf_cut = 0.8
    totalGraphWeight = 0
    segs = 0
    with open(graphf) as infile:
        for line in infile:
            if line.startswith("sequence"):
                fields = line.rsplit()
                c, s, e = fields[1].rsplit(":")[0], int(fields[1].rsplit(":")[1][:-1]), int(fields[2].rsplit(":")[1][:-1])+1
                if add_chr_tag and not c.startswith('chr'):
                    c = "chr" + c

                if not hit_region_it[c][s:e]:
                    continue

                cn = float(fields[3])
                size = float(fields[5]) / 1000.

                # if cn > 1:
                segs += 1
                totalGraphWeight += (size * cn)

            elif line.startswith("BreakpointEdge"):
                break

    # cycleWeights = [0] * len(feature_inds)
    # for ind, cycle in enumerate(cycleList):
    cycleWeights = []
    new_feat_inds = set()
    for ind, cycle in enumerate(cycleList):
        if ind not in exclude_inds:
            hits = False
            for cv in cycle:
                if cv != 0:
                    c, s, e = segSeqD[abs(cv)]
                    if hit_region_it[c][s:e]:
                        hits = True
                        break
            if hits:
                wca = weightedCycleAmount(cycle, cycleCNs[ind], segSeqD)
                if ind in feature_inds:
                    new_feat_inds.add(len(cycleWeights))

                cycleWeights.append(wca)

    # scW = sorted(cycleWeights, reverse=True)
    # cf = cycleWeights[0]/totalGraphWeight

    cf = 0
    fe_ent = 0
    added_cf = 0
    cInd = 0
    if totalGraphWeight > 0:
        while cf + added_cf < hf_cut and cInd < len(cycleWeights):
            if cInd in new_feat_inds:
                cf += added_cf
                if added_cf > 0:
                    fe_ent += (added_cf * log(added_cf))

                added_cf = cycleWeights[cInd] / float(totalGraphWeight)

            cInd += 1

        cf+=added_cf
        cf = round(cf, 5)
        rf = (1 - cf)
        # print(rf, cf, totalGraphWeight)
        if rf > 0:
            fu_ent = -1 * rf * log(rf)
        else:
            fu_ent = 0

    else:
        print("Warning: total graph weight <= 0")
        fu_ent = 0

    seg_ent = log(1.0 / segs) if segs > 0 else 0
    return fu_ent - fe_ent - seg_ent, fu_ent - fe_ent, -1 * seg_ent

# Compute f (foldback fraction) from the edges in the AA graph alone
def compute_f_from_AA_graph(graphf, add_chr_tag):
    with open(graphf) as infile:
        fbCount, nonFbCount, fbEdges, maxCN = 0, 0, 0, 0
        for line in infile:
            fields = line.rstrip().rsplit()
            if line.startswith("discordant"):
                lbp, rbp = fields[1].split("->")
                lchrom, lpd = lbp.rsplit(":")
                rchrom, rpd = rbp.rsplit(":")
                if add_chr_tag and not lchrom.startswith('chr'):
                    lchrom = "chr" + lchrom
                    rchrom = "chr" + rchrom

                lpos, ldir = int(lpd[:-1]), lpd[-1]
                rpos, rdir = int(rpd[:-1]), rpd[-1]

                if lcD[lchrom][lpos] or lcD[rchrom][rpos]:
                    continue

                elif fields[0] == "discordant" and rchrom == lchrom and abs(rpos - lpos) <= 2000 and rdir == '-' and ldir == '+':
                    continue

                rSupp = int(fields[3])
                if ldir == rdir:
                    if lchrom == rchrom and abs(rpos - lpos) < fb_dist_cut:
                        fbCount += rSupp
                        fbEdges += 1

                    else:
                        nonFbCount += rSupp

                else:
                    nonFbCount += rSupp

            elif line.startswith("sequence"):
                if not lcD[fields[1].rsplit(":")[0]].overlaps(int(fields[1].rsplit(":")[1][:-1]),
                                                              int(fields[2].rsplit(":")[1][:-1])):

                    ccn = float(fields[3])
                    if ccn > maxCN:
                        maxCN = ccn

    # just return 0 if there isn't enough support
    if fbEdges < 2:
        return 0, maxCN

    return fbCount / max(1.0, float(fbCount + nonFbCount)), maxCN

def nonbfb_cycles_are_ecdna(non_bfb_cycle_inds, cycleList, segSeqD, cycleCNs):
    for ind in non_bfb_cycle_inds:
        cycle = cycleList[ind]
        length = get_size(cycle, segSeqD)

        if length > 100000 and cycleCNs[ind] > 5:
            return True

    return False

# proportion of cycles with foldbacks
def cycles_file_bfb_props(cycleList, segSeqD, cycleCNs, graphf, add_chr_tag):
    FB_breaks = 0.0
    distal_breaks = 0.0
    lin_breaks = 0.0

    bfb_weight = 0.0
    non_bfb_cycle_weight = 0.0
    tot_bfb_supp_cycles = 0

    non_bfb_cycle_inds = []
    bfb_cycle_inds = []

    for ind, ocycle in enumerate(cycleList):
        cycle = copy.copy(ocycle)
        if cycle[0] != 0:
            cycle.append(cycle[0])

        hit = False
        isBFBelem = False
        illegalBFB = False
        for a, b in zip(cycle[:-1], cycle[1:]):
            # changes direction on same chrom
            diff = get_diff(a, b, segSeqD)
            aSize = get_size([a, ], segSeqD)
            bSize = get_size([b, ], segSeqD)
            if aSize < minCycleSize and bSize < minCycleSize:
                continue

            # check if front and back are connected via everted edge
            front_to_back_connection = amp_encompassed(cycle, segSeqD, graphf, add_chr_tag)
            if front_to_back_connection:
                # print("Cycle has front to back link", cycle)
                illegalBFB = True

            else:
                if a * b < 0 and segSeqD[abs(a)][0] == segSeqD[abs(b)][0]:
                    hit = True
                    if diff < 50000:
                        isBFBelem = True
                        FB_breaks += cycleCNs[ind]

                    else:
                        distal_breaks += cycleCNs[ind]

                elif diff > tot_min_del:
                    hit = True
                    distal_breaks += cycleCNs[ind]

                if segSeqD[abs(a)][0] != segSeqD[abs(b)][0] and not (a == 0 or b == 0):
                    illegalBFB = True

        if illegalBFB:
            isBFBelem = False

        if cycle[0] == 0 and not hit and get_size(cycle,segSeqD) > 10000:
            lin_breaks += cycleCNs[ind]

        if isBFBelem:
            tot_bfb_supp_cycles += 1
            bfb_weight += cycleCNs[ind]
            bfb_cycle_inds.append(ind)

        elif cycle[0] != 0 and get_size(cycle[:-1], segSeqD) > 30000:
            non_bfb_cycle_weight += cycleCNs[ind]
            non_bfb_cycle_inds.append(ind)

    hasEC = nonbfb_cycles_are_ecdna(non_bfb_cycle_inds, cycleList, segSeqD, cycleCNs)
    # if len(cycleList) >= 5:
    #     minBFBCyclesRequired = 2
    # else:
    minBFBCyclesRequired = 2
    if FB_breaks > 1.5 and tot_bfb_supp_cycles >= minBFBCyclesRequired:
        tot = float(FB_breaks + distal_breaks + lin_breaks)
        return FB_breaks / tot, distal_breaks / tot, bfb_weight / (non_bfb_cycle_weight + bfb_weight), hasEC, \
               non_bfb_cycle_inds, bfb_cycle_inds

    return 0, 0, 0, False, [], []

# ------------------------------------------------------------
# Classifications
def cycleIsNoAmpInvalid(cycle, cn, segSeqD, isSingleton, maxCN):
    # CN flow can be split across multiple amps
    if not isSingleton:
        scale = min(args.min_cn_flow, maxCN * decomposition_strictness)
    elif maxCN > 7:
        scale = min(3., maxCN / 8.)
    else:
        scale = 2.5

    # print(cycle)
    # print("decomp cutoff",scale)

    if (cn <= scale) or (maxCN < min_upper_cn):
        return True

    length = get_size(cycle, segSeqD)
    return length < minCycleSize

def classifyConnections(cycleSet1, cycleSet2, clfs):
    cycleSet1, cycleSet2 = sorted([cycleSet1, cycleSet2], key=lambda x: len(x), reverse=True)
    csets = []
    resultDict = defaultdict(float)
    if not cycleSet2:
        for c1 in cycleSet1:
            csets.append(frozenset([clfs[c1], ]))

    else:
        for c1 in cycleSet1:
            for c2 in cycleSet2:
                csets.append(frozenset([clfs[c1], clfs[c2]]))

    distributed_edge_value = 1.0 / len(csets) if csets else 0
    for cset in csets:
        resultDict[cset] += distributed_edge_value

    return resultDict

# categories = ["No amp/Invalid", "Linear amplification", "Trivial cycle", "Complex non-cyclic", "Complex cyclic"]
def classifyAmpliconProfile(amp_profile, rearr_e, totalCompCyclicCont, totCyclicCont, force=False):
    cycSig = amp_profile["Trivial cycle"] + amp_profile["Complex cyclic"]
    if (cycSig > cycCut or totalCompCyclicCont > compCycContCut) and totCyclicCont > 10000:
        return "Cyclic"

    elif amp_profile["Complex non-cyclic"] + cycSig > compCut:
        if rearr_e < 2:
            return "Linear amplification"

        return "Complex non-cyclic"

    else:
        if max(amp_profile.values()) == 0:
            return "No amp/Invalid"

        elif amp_profile["No amp/Invalid"] > 0:
            if amp_profile["Linear amplification"] / float(amp_profile["No amp/Invalid"]) > 0.25:
                if rearr_e >= 5:
                    return "Complex non-cyclic"

                return "Linear amplification"

        if force:
            del amp_profile["No amp/Invalid"]
            if cycSig > max(amp_profile.values()):
                return "Cyclic"

        maxCat = max(amp_profile.items(), key=operator.itemgetter(1))[0]
        return maxCat

def classifyBFB(fb, cyc_sig, nonbfb_sig, bfb_cyc_ratio, maxCN):
    if fb < min_score_for_bfb or cyc_sig < 0.295 or maxCN < 4:
        return None

    # dominated by non-classical BFB cycles
    elif nonbfb_sig > 0.5 and bfb_cyc_ratio < 0.6:
        return None

    # if bfb_cyc_ratio < 0.85:
    #     # if nonbfb_sig > 0.55:
    #     #     return None
    #
    #     if args.use_BFB_linked_cyclic_class:
    #         return "BFB-linked cyclic"

    return "BFB"

# ------------------------------------------------------------
# structure metanalysis

def check_max_cn(ec_cycle_inds, cycleList, segSeqD):
    for e_ind in ec_cycle_inds:
        for c_id in cycleList[e_ind]:
            chrom, l, r = segSeqD[abs(c_id)]
            if r - l < 1000:
                continue

            for i in graph_cns[chrom][l:r]:
                if i.data > min_upper_cn:
                    return True

    return False

def get_amount_sigamp(ec_cycle_inds, cycleList, segSeqD):
    used_content = defaultdict(set)
    for e_ind in ec_cycle_inds:
        for c_id in cycleList[e_ind]:
            chrom, l, r = segSeqD[abs(c_id)]
            if not chrom:
                continue
            seg_t = IntervalTree([Interval(l, r+1)])
            olapping_low_cns = [x for x in graph_cns[chrom][l:r] if x.data < 4]
            for x in olapping_low_cns:
                seg_t.chop(x.begin, x.end)
            for x in seg_t:
                used_content[chrom] |= set(range(x.begin, x.end))

    total_sigamp = 0
    for chrom, useset in used_content.items():
        total_sigamp += len(useset)

    return total_sigamp

def clusterECCycles(cycleList, cycleCNs, segSeqD, excludableCycleIndices=None):
    padding = 500000
    indices = [x for x in range(len(cycleList)) if cycleList[x][0] != 0 and x not in excludableCycleIndices]
    clusters = []
    seenSegs = set()
    total_EC_size = 0
    for ind in indices:
        cycle = cycleList[ind]
        csize = get_size(cycle, segSeqD)
        total_EC_size+=csize
        if cycleCNs[ind] < args.min_cn_flow and csize < minCycleSize:
            continue

        cIndsToMerge = set()
        s_set = set([segSeqD[abs(s_num)] for s_num in cycle])
        s_set -= seenSegs
        if not s_set:
            continue

        for c_ind, clust_dict in enumerate(clusters):
            for s in s_set:
                if clust_dict[s[0]][s[1] - padding:s[2] + padding]:
                    cIndsToMerge.add(c_ind)
                    break

        newClusters = []
        newClust = defaultdict(IntervalTree)
        for s in s_set:
            newClust[s[0]].addi(s[1], s[2] + 1, ind)

        for c_ind, currClust in enumerate(clusters):
            if c_ind in cIndsToMerge:
                for k, v in currClust.items():
                    for ival in v:
                        newClust[k].addi(ival.begin, ival.end, ival.data)

            else:
                newClusters.append(currClust)

        newClusters.append(newClust)
        clusters = newClusters
        seenSegs |= s_set

    indexClusters = []
    # extract only the cycle indices from each cluster and return
    for clust in clusters:
        currIndexSet = set()
        for k, v in clust.items():
            for ival in v:
                currIndexSet.add(ival.data)

        if get_amount_sigamp(currIndexSet, cycleList, segSeqD) > 10000:
            indexClusters.append(currIndexSet)

    # remove those where the max CN is below threshold
    indexClusters = [x for x in indexClusters if check_max_cn(x, cycleList, segSeqD)]

    return indexClusters

# ------------------------------------------------------------

'''
Amplicon Classes:
#if not invalid

1) No amp/Invalid
2) Linear amplification
3) Complex non-cyclic
4) BFB
5) BFB-linked cylic (ecBFB)
6) Cyclic (ecDNA)

Graph edge classes:
1) No amp/Invalid
2) Non-cyclic
3) Integration
4) Hybrid - joins amplified cyclic and non cyclic
5) Cyclic

'''

mixLookups = {
    frozenset(["No amp/Invalid", ]): "No amp/Invalid",
    frozenset(["No amp/Invalid", "Linear amplification"]): "Integration",
    frozenset(["No amp/Invalid", "Trivial cycle"]): "Integration",
    frozenset(["No amp/Invalid", "Complex non-cyclic"]): "Integration",
    frozenset(["No amp/Invalid", "Complex cyclic"]): "Integration",
    frozenset(["Linear amplification"]): "Non-cyclic",
    frozenset(["Linear amplification", "Trivial cycle"]): "Integration",
    frozenset(["Linear amplification", "Complex non-cyclic"]): "Non-cyclic",
    frozenset(["Linear amplification", "Complex cyclic"]): "Integration",
    frozenset(["Trivial cycle"]): "Cyclic",
    frozenset(["Trivial cycle", "Complex non-cyclic"]): "Hybrid",
    frozenset(["Trivial cycle", "Complex cyclic"]): "Cyclic",
    frozenset(["Complex non-cyclic"]): "Non-cyclic",
    frozenset(["Complex non-cyclic", "Complex cyclic"]): "Hybrid",
    frozenset(["Complex cyclic"]): "Cyclic",
}

# (circular,complex)
categories = ["No amp/Invalid", "Linear amplification", "Trivial cycle", "Complex non-cyclic", "Complex cyclic",
              "foldback_read_prop", "BFB_bwp", "Distal_bwp", "BFB_cwp", "Amp_entropy", "Amp_decomp_entropy",
              "Amp_nseg_entropy"]
mixing_cats = ["No amp/Invalid", "Non-cyclic", "Integration", "Hybrid", "Cyclic"]

ampDefs = {(False, False): "Linear amplification", (False, True): "Complex non-cyclic",
           (True, False): "Trivial cycle", (True, True): "Complex cyclic"}

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Classify AA amplicon type")
    parser.add_argument("-i", "--input", help="Path to list of files to use. Each line formatted as: "
                        "sample_name cycles.txt graph.txt. Give this argument if not using -c and -g.")
    parser.add_argument("-c", "--cycles", help="AA-formatted cycles file")
    parser.add_argument("-g", "--graph", help="AA-formatted graph file")
    parser.add_argument("--ref", help="Reference genome name used for alignment, one of hg19, GRCh37, or GRCh38.",
                        choices=["hg19", "GRCh37", "hg38", "GRCh38"], required=True)

    parser.add_argument("--min_cn_flow", type=float, help="Minimum CN flow to consider as amplification (1.0).",
                        default=1)
    parser.add_argument("--min_size", type=float, help="Minimum cycle size (in bp) to consider as valid amplicon "
                        "(5000).", default=5000)
    parser.add_argument("-o", help="Output filename prefix")
    parser.add_argument("--plotstyle", help="Type of visualizations to produce.",
                        choices=["grouped", "individual", "noplot"], default="noplot")
    parser.add_argument("--force", help="Disable No amp/Invalid class if possible", action='store_true')
    # parser.add_argument("--use_BFB_linked_cyclic_class", help="Include the \'BFB-linked cyclic\' class",
    #                     action='store_true')
    parser.add_argument("--add_chr_tag", help="Add \'chr\' to the beginning of chromosome names in input files.",
                        action='store_true')
    # parser.add_argument("--report_genes", help="Extract list of genes from amplicons with given classification.",
    #                     choices=["ecdna", "bfb", "other", "all"], nargs='+', default=[])
    parser.add_argument("--report_complexity", help="Compute a measure of amplicon entropy for each amplicon.",
                        action='store_true')
    parser.add_argument("--verbose_classification", help="Generate verbose output with raw classification scores.",
                        action='store_true')
    parser.add_argument("--annotate_cycles_file", help="Create an annotated cycles file, indicating the classification "
                        "of the paths and cycles present.", action='store_true')
    parser.add_argument("--no_LC_filter", help="Do not filter low-complexity cycles. Not recommended to set this flag.",
                        action='store_true', default=False)
    parser.add_argument("--decomposition_strictness", help="Value between 0 and 1 reflecting how strictly to filter "
                                                           "low CN decompositions (default = 0.1). Higher values "
                                                           "filter more of the low-weight decompositions.", type=float,
                        default=0.1)
    parser.add_argument("-v", "--version", action='version', version='amplicon_classifier {version} \n Author: Jens \
                        Luebeck (jluebeck [at] ucsd.edu)'.format(version=__version__))

    args = parser.parse_args()

    if not (args.cycles and args.graph) and not args.input:
        print("Need to specify (--cycles & --graph) or --input\n")
        sys.exit(1)

    if args.ref == "hg38": args.ref = "GRCh38"
    patch_links = read_patch_regions(args.ref)
    if 0 <= args.decomposition_strictness <= 1:
        decomposition_strictness= args.decomposition_strictness
    else:
        print("--decomposition_strictness must be a value between 0 and 1")
        sys.exit(1)

    # check if aa data repo set, construct LC datatabase
    try:
        AA_DATA_REPO = os.environ["AA_DATA_REPO"] + "/" + args.ref + "/"
        fDict = {}
        with open(AA_DATA_REPO + "file_list.txt") as infile:
            for line in infile:
                fields = line.strip().rsplit()
                fDict[fields[0]] = fields[1]

        lcPath = AA_DATA_REPO + fDict["mapability_exclude_filename"]
        lcD = defaultdict(IntervalTree)
        if not args.no_LC_filter:
            lcD = buildLCDatabase(lcPath)

    except KeyError:
        sys.stderr.write("$AA_DATA_REPO not set. Please see AA installation instructions.\n")
        sys.exit(1)

    # gene_lookup = {}
    ftgd_list = []  # store list of feature gene classifications
    # gene_file_location_lookup = {"hg19": "human_hg19_september_2011/Genes_July_2010_hg19.gff",
    #                              "GRCh38": "genes_hg38.gff",
    #                              "GRCh37": "human_hg19_september_2011/Genes_July_2010_hg19.gff"}
    #
    # refGeneFileLoc = AA_DATA_REPO + gene_file_location_lookup[args.ref]

    # read the gene list
    refGeneFileLoc = AA_DATA_REPO + fDict["gene_filename"]
    gene_lookup = get_genes.parse_genes(refGeneFileLoc)

    if not args.input:
        tempName = args.cycles.rsplit("/")[-1].rsplit(".")[0]
        flist = [[tempName, args.cycles, args.graph]]
        if not args.o:
            args.o = os.path.basename(args.cycles).rsplit("_cycles.txt")[0]

    else:
        flist = readFlist(args.input)
        if not args.o:
            args.o = os.path.basename(args.input).rsplit(".")[0]

    minCycleSize = args.min_size

    AMP_dvaluesList = []
    EDGE_dvaluesList = []
    AMP_classifications = []
    sampNames = []
    cyclesFiles = []
    featEntropyD = {}
    samp_to_ec_count = defaultdict(int)
    for fpair in flist:
        if len(fpair) > 2:
            sName, cyclesFile, graphFile = fpair
            sampNames.append(sName)
            cyclesFiles.append(cyclesFile)
            ampN = cyclesFile.rstrip("_cycles.txt").rsplit("_")[-1]
            print(sName, ampN)
            segSeqD, cycleList, cycleCNs = parseCycle(cyclesFile, graphFile, args.add_chr_tag, lcD, patch_links)

        else:
            print(fpair)
            sys.stderr.write("File list not properly formatted\n")
            sys.exit(1)

        cycleTypes = []
        cycleWeights = []
        invalidInds = []
        rearrCycleInds = set()
        graph_cns = get_graph_cns(graphFile, args.add_chr_tag)
        fb_prop, maxCN = compute_f_from_AA_graph(graphFile, args.add_chr_tag)
        rearr_e = tot_rearr_edges(graphFile, args.add_chr_tag)
        totalCompCyclicCont = 0
        totCyclicCont = 0
        for ind, cycle in enumerate(cycleList):
            hasNonCircLen1 = True if len(cycle) == 3 and cycle[0] == 0 else False
            oneCycle = (len(cycleList) == 1)
            isSingleton = hasNonCircLen1 or oneCycle
            if cycleIsNoAmpInvalid(cycle, cycleCNs[ind], segSeqD, isSingleton, maxCN) and not args.force:
                invalidInds.append(ind)
                cycleTypes.append("No amp/Invalid")

            else:
                circCyc = isCircular(cycle)
                compCyc = isRearranged(cycle, segSeqD)
                if compCyc:
                    rearrCycleInds.add(ind)
                    if circCyc:
                        totalCompCyclicCont += get_size(cycle, segSeqD)

                if circCyc:
                    totCyclicCont += get_size(cycle, segSeqD)

                cycleTypes.append(ampDefs[(circCyc, compCyc)])

            currWt = weightedCycleAmount(cycle, cycleCNs[ind], segSeqD)
            cycleWeights.append(currWt)

        totalWeight = max(sum(cycleWeights), 1)
        AMP_dvaluesDict = {x: 0.0 for x in categories}
        for i, wt in zip(cycleTypes, cycleWeights):
            AMP_dvaluesDict[i] += (wt / totalWeight)

        # anything stored in AMP_dvaluesDict prior to running classify will get used in classification
        # make sure you're not putting in other properties before here.
        ampClass = classifyAmpliconProfile(AMP_dvaluesDict, rearr_e, totalCompCyclicCont, totCyclicCont)

        # decomposition/amplicon complexity
        totalEnt, decompEnt, nEnt = decompositionComplexity(graphFile, cycleList, cycleCNs, segSeqD,
                                                            range(len(cycleList)), set(), args.add_chr_tag)
        AMP_dvaluesDict["Amp_entropy"] = totalEnt
        AMP_dvaluesDict["Amp_decomp_entropy"] = decompEnt
        AMP_dvaluesDict["Amp_nseg_entropy"] = nEnt

        # now layer on the bfb classification
        # first compute some properties
        fb_prop, maxCN = compute_f_from_AA_graph(graphFile, args.add_chr_tag)

        fb_bwp, nfb_bwp, bfb_cwp, bfbHasEC, non_bfb_cycle_inds, bfb_cycle_inds = cycles_file_bfb_props(cycleList,
                                                                        segSeqD, cycleCNs, graphFile, args.add_chr_tag)
        # "foldback_read_prop", "BFB_bwp", "Distal_bwp", "BFB_cwp"
        AMP_dvaluesDict["foldback_read_prop"] = fb_prop
        AMP_dvaluesDict["BFB_bwp"] = fb_bwp
        AMP_dvaluesDict["Distal_bwp"] = nfb_bwp
        AMP_dvaluesDict["BFB_cwp"] = bfb_cwp

        bfbClass = classifyBFB(fb_prop, fb_bwp, nfb_bwp, bfb_cwp, maxCN)

        ecStat = False
        bfbStat = False
        if ampClass == "Cyclic" and not bfbClass:
            ecStat = True
            bfb_cycle_inds = []

        elif bfbClass and ampClass != "No amp/Invalid":
            bfbStat = True
            if bfbHasEC:
                ecStat = True

        else:
            bfb_cycle_inds = []

        # determine number of ecDNA present
        ecIndexClusters = []
        if ecStat:
            excludableCycleIndices = set(bfb_cycle_inds + invalidInds)
            ecIndexClusters = clusterECCycles(cycleList, cycleCNs, segSeqD, excludableCycleIndices)
            ecAmpliconCount = max(len(ecIndexClusters), 1)

        else:
            ecAmpliconCount = 0

        samp_to_ec_count[sName] += ecAmpliconCount
        # write entropy for each feature
        ecEntropies = []
        if ecAmpliconCount == 1 and not ecIndexClusters:
            ecEntropies.append((totalEnt, decompEnt, nEnt))

        for ecCycleList in ecIndexClusters:
            c_ex_I = bfb_cycle_inds if bfbStat else set()
            totalEnt, decompEnt, nEnt = decompositionComplexity(graphFile, cycleList, cycleCNs, segSeqD,
                                                                ecCycleList, c_ex_I, args.add_chr_tag)
            ecEntropies.append((totalEnt, decompEnt, nEnt))

        for ind, etup in enumerate(ecEntropies):
            featEntropyD[(sName, ampN, "ecDNA_" + str(ind+1))] = etup

        if bfbStat:
            bfb_totalEnt, bfb_decompEnt, bfb_nEnt = decompositionComplexity(graphFile, cycleList, cycleCNs, segSeqD,
                                                                            bfb_cycle_inds, set(), args.add_chr_tag)
            featEntropyD[(sName, ampN, "BFB_1")] = (bfb_totalEnt, bfb_decompEnt, bfb_nEnt)

        # get genes
        feat_gene_truncs, feat_gene_cns = get_genes.extract_gene_list(sName, ampN, gene_lookup, cycleList, segSeqD,
                                                                      bfb_cycle_inds, ecIndexClusters, invalidInds,
                                                                      bfbStat, ecStat, ampClass, graphFile,
                                                                      args.add_chr_tag, args.o)

        ftgd_list.append([sName, ampN, feat_gene_truncs, feat_gene_cns])

        # store this additional information
        AMP_classifications.append((ampClass, ecStat, bfbStat, ecAmpliconCount))
        dvalues = [AMP_dvaluesDict[x] for x in categories]
        AMP_dvaluesList.append(dvalues)

        # edge classification
        edgeTypeCountD = defaultdict(float)
        if graphFile:
            posCycleLookup = buildPosCycleLookup(cycleList, segSeqD)
            bps = parseBPG(graphFile, args.add_chr_tag, lcD)
            for bp in bps:
                lCycles, rCycles = bpgEdgeToCycles(bp, posCycleLookup)
                # indices of left and right cycles on the discordant edges, and the index-ordered list of types
                resD = classifyConnections(lCycles, rCycles, cycleTypes)
                for k, v in resD.items():
                    edgeTypeCountD[mixLookups[k]] += v

            # norm the values
            eTCDSum = float(sum(edgeTypeCountD.values()))
            for k, v in edgeTypeCountD.items():
                edgeTypeCountD[k] = v / eTCDSum

        dvalues = [edgeTypeCountD[x] for x in mixing_cats]
        EDGE_dvaluesList.append(dvalues)

        #write the annotated cycles file
        if args.annotate_cycles_file:
            outname = os.path.basename(cyclesFile).rsplit("_cycles")[0] + "_annotated_cycles.txt"
            write_annotated_corrected_cycles_file(args.o, outname, cycleList, cycleCNs, segSeqD, bfb_cycle_inds,
                                                  ecIndexClusters, invalidInds, rearrCycleInds)

    # PLOTTING
    textCategories = ["No amp/Invalid", "Linear\namplification", "Trivial\ncycle", "Complex\nnon-cyclic",
                      "Complex\ncyclic", "BFB\nfoldback"]
    if args.plotstyle == "grouped":
        from radar_plotting import *

        print("plotting")
        make_classification_radar(textCategories, AMP_dvaluesList, args.o + "_amp_class", sampNames)
        make_classification_radar(mixing_cats, EDGE_dvaluesList, args.o + "_edge_class", sampNames)

    elif args.plotstyle == "individual":
        from radar_plotting import *

        print("plotting")
        for a, e, s in zip(AMP_dvaluesList, EDGE_dvaluesList, sampNames):
            print(textCategories, a)
            make_classification_radar(textCategories, [a[:len(textCategories)], ], args.o + "_" + s + "_amp_class",
                                      sampNames)
            make_classification_radar(mixing_cats, [e, ], args.o + "_" + s + "_edge_class", sampNames)

    #OUTPUT FILE WRITING
    print("writing output files")
    write_outputs(args, ftgd_list, featEntropyD, categories, sampNames, cyclesFiles, AMP_classifications,
                  AMP_dvaluesList, mixing_cats, EDGE_dvaluesList, samp_to_ec_count)

    print("done")


from glob import glob
from os.path import basename
from os.path import splitext

from setuptools import find_packages
from setuptools import setup

install_requires = [
      'pretty-json',
      'Pygments',
      'pygments-json',
      'pygments-solarized',
      'requests'
]

setup(name='playable',
      version='1.1',
      description='Playout assets tool',
      author='Alan So',
      author_email='alansoandso@gmail.com',
      packages=['playable'],
      include_package_data=True,
      entry_points={'console_scripts': ['playable = playable.tool:command_line_runner', ]},
      install_requires=install_requires
      )


from glob import glob
from config import EXAMPLE_PATH, MEDIA_FILE_HOST, CONTEXT_RANGE
import json
import string
from pathlib import Path

class Decks:
    def __init__(self, category="anime", path=EXAMPLE_PATH, has_image=True, has_sound=True):
        self.sentences = {}
        self.sentence_map = {}
        self.sentence_translation_map = {}
        self.category = category
        self.path = path
        self.has_image = has_image
        self.has_sound = has_sound

    def get_sentence(self, sentence_id):
        if sentence_id in self.sentences:
            return self.sentences[sentence_id]
        else:
            return None

    def get_sentence_map(self):
        return self.sentence_map

    def get_sentence_translation_map(self):
        return self.sentence_translation_map

    def load_decks(self):
        deck_folders = glob(str(self.path) + '/*/')
        for deck_folder in deck_folders:
            self.load_deck_by_path(deck_folder)

    def load_deck_by_path(self, path):
        sentences = []
        file = Path(path, 'data.json')
        with open(file, encoding='utf-8') as f:
            sentences = json.load(f)

        for index, sentence in enumerate(sentences):
            sentence = self.parse_sentence(sentence)
            pretext_sentences  = sentences[0:index] if index < CONTEXT_RANGE else sentences[index-CONTEXT_RANGE:index]
            posttext_sentences = []
            if index < len(sentences):
                posttext_sentences = sentences[index+1:len(sentences)] if index+CONTEXT_RANGE > len(sentences) else sentences[index+1:index+CONTEXT_RANGE]
            sentence["pretext"] = [sentence["id"] for sentence in pretext_sentences]
            sentence["posttext"] = [sentence["id"] for sentence in posttext_sentences]
            if 'word_base_list' in sentence:
                self.sentence_map = self.map_sentence(sentence['word_base_list'], sentence['id'], self.sentence_map)
            if 'translation_word_base_list' in sentence:
                self.sentence_translation_map = self.map_sentence(sentence['translation_word_base_list'], sentence['id'], self.sentence_translation_map)
            self.sentences[sentence["id"]] = sentence

    def parse_sentence(self, sentence):
        if (self.has_image):
            image_path = '{}/{}/{}/media/{}'.format(MEDIA_FILE_HOST, self.category, sentence['deck_name'], sentence['image'])
            sentence['image_url'] = image_path.replace(" ", "%20")

        if (self.has_sound):
            sound_path = '{}/{}/{}/media/{}'.format(MEDIA_FILE_HOST, self.category, sentence['deck_name'], sentence['sound'])
            sentence['sound_url'] = sound_path.replace(" ", "%20")
        return sentence

    def map_sentence(self, words, example_id, output_map):
        for (index, word) in enumerate(words):
            is_repeat = words.index(word) != index
            if is_repeat:
                continue
            if word in string.punctuation or word in '！？。、（）':
                continue
            if word not in output_map:
                output_map[word] = set()
            output_map[word].add(example_id)
        return output_map


class Chair:

    def __init__(self):
        self.color = ''
        self.weight_in_pounds = 0
        self.seats = 0
        self.number_of_legs = 0
        self.height_in_inches = 0
        self.maximum_height = 0
        self.minimum_height = 0

    def change_height(self, inches_to_raise):
        self.height_in_inches += inches_to_raise
        if self.height_in_inches > self.maximum_height:
            self.height_in_inches = self.maximum_height
        elif self.height_in_inches < self.minimum_height:
            self.height_in_inches = self.minimum_height

def which_chair_is_heavier(first_chair, second_chair):
    if first_chair.weight_in_pounds > second_chair.weight_in_pounds:
        print("The {} chair is heavier!".format(first_chair.color))
    else:
        print("The {} chair is heavier!".format(second_chair.color))

erics_chair = Chair()
erics_chair.color = 'Red'
erics_chair.weight_in_pounds = 5
erics_chair.seats = 1
erics_chair.number_of_legs = 5
erics_chair.height_in_inches = 22
erics_chair.maximum_height = 30
erics_chair.minimum_height = 20

jons_chair = Chair()
jons_chair.color = 'blue'
jons_chair.weight_in_pounds = 30
jons_chair.seats = 2
jons_chair.number_of_legs = 4
jons_chair.height_in_inches = 22
jons_chair.maximum_height = 22
jons_chair.minimum_height = 22

garrets_chair = Chair

print(erics_chair)
print(jons_chair)

which_chair_is_heavier(erics_chair, jons_chair)

print("eric's chair is {} inches tall".format(erics_chair.height_in_inches))
erics_chair.change_height(20)
print("eric's chair is {} inches tall".format(erics_chair.height_in_inches))


# @copyright@
# Copyright (c) 2006 - 2017 Teradata
# All rights reserved. Stacki(r) v5.x stacki.com
# https://github.com/Teradata/stacki/blob/master/LICENSE.txt
# @copyright@
#
# @rocks@
# Copyright (c) 2000 - 2010 The Regents of the University of California
# All rights reserved. Rocks(r) v5.4 www.rocksclusters.org
# https://github.com/Teradata/stacki/blob/master/LICENSE-ROCKS.txt
# @rocks@

import stack.commands

class Command(stack.commands.remove.host.command):
	"""
	Remove an alias from a host(s).

	<arg type='string' name='host' optional='1' repeat='1'>
	One hosts.
	</arg>

	<param type='string' name='alias'>
	The alias name that should be removed.
	</param>

	<example cmd='remove host alias backend-0-0 alias=c-0-0'>
	Removes the alias c-0-0 for host backend-0-0.
	</example>

	<example cmd='remove host alias backend-0-0'>
	Removes all aliases for backend-0-0.
	</example>
	"""

	def run(self, params, args):

		(alias, ) = self.fillParams([
			('alias', None)
			])

		for host in self.getHostnames(args):
			if not alias:
				self.db.execute("""
					delete from aliases where
					node = (select id from nodes where name='%s')
					""" % host)
			else:
				self.db.execute("""
					delete from aliases where
					node = (select id from nodes where name='%s')
					and name = '%s'
					""" % (host, alias))


CUSTOM_DICTIONARY = ['abstraction', 'airplane', 'aliquot', 'angular', 'antimony',
                     'bandage', 'biology', 'boardwalk', 'burgher',
                     'carbon', 'classification', 'countable', 'crouton',
                     'daunting', 'destruction', 'dynamic', 'dystrophy',
                     'earnings', 'enchanting', 'environmental', 'eulogy', 'evening',
                     'fathom', 'fibromyalgia', 'fluoride', 'frightening',
                     'gastronomic', 'gnome', 'guerrilla', 'gyromagnetic',
                     'halogen', 'historical', 'horticulture', 'hydrogen',
                     'ideological', 'ignition', 'indistinguishable',
                     'jaguar', 'jeopardize', 'jingoism', 'joule', 'jungle',
                     'kangaroo', 'kinematics', 'knitting',
                     'languid', 'lithium', 'longitudinal', 'lupine', 'lyrical',
                     'mitochondria', 'momentum', 'monopolize', 'myopic', 'mythology',
                     'nautical', 'neodymium', 'nucleolus',
                     'oncology', 'orthodontist', 'osteoporosis',
                     'potassium', 'pneumatic', 'premeditated', 'python',
                     'quantifiable', 'quiescent', 'quotient',
                     'radiology', 'resonance', 'reticent', 'routing',
                     'sanguine', 'serrated', 'sodium', 'sophomoric',
                     'tangential', 'tanner', 'thalidomide', 'toothbrush', 'turnkey',
                     'understudy', 'utilitarian', 'utterance',
                     'valedictorian', 'verify', 'vilify', 'viper',
                     'weariness', 'whimsical', 'wilderness', 'wunderkind',
                     'xerox', 'xylem', 'xylophone',
                     'yearbook', 'yearn', 'young',
                     'zebra', 'zoology']


#! /usr/bin/env python
# -*- coding: utf-8 -*-

"""Load the config file."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from os.path import join
import yaml
import shutil

def load_config(config_path, is_eval=False):
    """Load configure file.
    Args:
        config_path (string):
        is_eval (bool, option):
    Returns:
        params (dict):
    """
    with open(config_path, "r") as f:
        config = yaml.load(f)

        # Load the parent config file
        if 'parent' in config.keys():
            if not is_eval:
                with open(config['parent'], "r") as fp:
                    config_parent = yaml.load(fp)

            else:
                with open(config_path.replace('config.yml', 'config_parent.yml'), "r") as fp:
                    config_parent = yaml.load(fp)

            params = config_parent['param']

            # Override
            for key in config['param'].keys():
                params[key] = config['param'][key]

        else:
            params = config['param']

    return params

def save_config(config_path, save_path):
    """Save configure file.
    Args:
        config_path (string):
    Returns:
        save_path (string):
    """

    shutil.copyfile(config_path, join(save_path, 'config.yml'))

    # Save the parent config file
    with open(config_path, "r") as f:
        config = yaml.load(f)

        # Load the parent config file
        if 'parent' in config.keys():
            config_parent_path = config['parent']
            shutil.copyfile(config_parent_path, join(
                save_path, 'config_parent.yml'))


# -*- coding:utf-8 -*-
import numpy as np
import librosa

def to_mfcc(file, n_mfcc=12, rate=16000):
    ## -----*----- 音声データをMFCCに変換 -----*----- ##
    x, fs = librosa.load(file, sr=rate)
    mfcc = librosa.feature.mfcc(x, sr=fs, n_mfcc=n_mfcc)
    mfcc = np.reshape(mfcc, (mfcc.shape[0], mfcc.shape[1], 1))
    return np.array(mfcc, dtype=np.float32)

if __name__ == '__main__':
    wav_file = 'your .wav file path'
    to_mfcc(wav_file)


# Youtube

lista = [50,40,33,300,543,21,2]
tamanho_lista = len(lista)

for i in range(tamanho_lista):
    troca = False
    for j in range(1,tamanho_lista):

        if lista[j] < lista[j-1]:  # verifica se o numero sucessor é menor que o antecessor
            lista[j] , lista[j-1] = lista[j-1] , lista[j]
            troca = True

    if not troca:
        break

print(lista)


from settings import *

memberfile = open("clustering/membership.dat","r")
members = dict()
for line in memberfile:
  key = line.strip().split()[0]
  val = line.strip().split()[1]
  members[int(key)]=int(val)

memberfile.close()

for time in range(START_IDX,NUM_TIMES):
  infile = open("local_models/time-"+str(time)+".normalized","r")
  outfile = open("local_models/time-"+str(time)+".centroids","w")
  for line in infile:
    sums = [0.0]*GLOBAL_TOPICS
    normalizer = [0]*GLOBAL_TOPICS
    for nval,val in enumerate(line.strip().split()):
      if nval == 0:
        outfile.write(val) #vocab
      else:
        sums[members[nval-1 + LOCAL_TOPICS*time]] += float(val)
        normalizer[members[nval-1 + LOCAL_TOPICS*time]] += 1
    for index in range(GLOBAL_TOPICS):
      sums[index] = sums[index]/max(float(normalizer[index]),1) #don't divide by zero
    for element in sums:
      outfile.write(" "+str(element))
    outfile.write("\n")
  infile.close()
  outfile.close()


import os
import requests
from ratelimit import limits, sleep_and_retry
# import ruamel.yaml
import json
import yaml

BIO_TOOLS_API="https://bio.tools/api/tool/%s&format=json"
BIO_TOOLS_TOOL_API="https://bio.tools/api/tool/%s/?format=json"

FIFTEEN_MINUTES = 60

@sleep_and_retry
@limits(calls=1000, period=FIFTEEN_MINUTES)
def call_api(url):
    response = requests.get(url)

    if response.status_code != 200:
        raise Exception('API response: {}'.format(response.status_code))
    return response

class MyDumper(yaml.Dumper):

    def increase_indent(self, flow=False, indentless=False):
        return super(MyDumper, self).increase_indent(flow, False)

def sync_tool(tool):
    """
    This function allows to download the specific content for a tool.
    :param tool: tool
    :return:
    """
    id = tool['biotoolsID']
    if id is not None:
        if not os.path.exists(id):
            os.mkdir(id)
        tools_url = BIO_TOOLS_TOOL_API.replace("%s", id)
        try:
            response = call_api(tools_url)
            with open(id + '/descriptor.yaml', 'w') as stream:
               json_data = json.loads(response.content.decode('utf-8'))
               yaml.dump(json_data, stream, Dumper=MyDumper, default_flow_style=False)
               print("Processing -- " + id)
        except Exception:
            print("Error processing -- " + id)

def main():
    """
    This function enable to loop over all tools in bio.tools and write into a file ach representation.
    """
    current_page="?page=1"
    tools_url = BIO_TOOLS_API.replace("%s", current_page)
    response = requests.get(tools_url, headers={"Content-Type": "application/json"})

    data = response.json()
    count = 0;

    while(data['next'] != None):
        for index in range(len(data['list'])):
            sync_tool(data['list'][index])
        count = count + len(data['list'])
        print("Number of tools processed -- " + str(count))

        tools_url = BIO_TOOLS_API.replace("%s", data['next'])
        response = call_api(tools_url)
        data = response.json()

if __name__ == "__main__":
    main()


"""
expm.py - a module for all things e^M
"""

from autograd import jacobian
from autograd.extend import (defvjp as autograd_defvjp,
                             primitive as autograd_primitive)
import numpy as np
import scipy.linalg as la

from qoc.models.operationpolicy import OperationPolicy

@autograd_primitive
def expm(matrix, operation_policy=OperationPolicy.CPU):
    """
    Compute the matrix exponential of a matrix.
    Args:
    matrix :: numpy.ndarray - the matrix to exponentiate
    operation_policy :: qoc.OperationPolicy - what data type is
        used to perform the operation and with which method
    Returns:
    exp_matrix :: numpy.ndarray - the exponentiated matrix
    """
    if operation_policy == OperationPolicy.CPU:
        exp_matrix = la.expm(matrix)
    else:
        pass

    return exp_matrix

def _expm_vjp(exp_matrix, matrix, operation_policy=OperationPolicy.CPU):
    """
    Construct the left-multiplying vector jacobian product function
    for the matrix exponential.

    Intuition:
    `dfinal_dexpm` is the jacobian of `final` with respect to each element `expmij`
    of `exp_matrix`. `final` is the output of the first function in the
    backward differentiation series. It is also the output of the last
    function evaluated in the chain of functions that is being differentiated,
    i.e. the final cost function. The goal of `vjp_function` is to take
    `dfinal_dexpm` and yield `dfinal_dmatrix` which is the jacobian of
    `final` with respect to each element `mij` of `matrix`.
    To compute the frechet derivative of the matrix exponential with respect
    to each element `mij`, we use the approximation that
    dexpm_dmij = np.matmul(Eij, exp_matrix). Since Eij has a specific
    structure we don't need to do the full matrix multiplication and instead
    use some indexing tricks.

    Args:
    exp_matrix :: numpy.ndarray - the matrix exponential of matrix
    matrix :: numpy.ndarray - the matrix that was exponentiated
    operation_policy :: qoc.OperationPolicy - what data type is
        used to perform the operation and with which method

    Returns:
    vjp_function :: numpy.ndarray -> numpy.ndarray - the function that takes
        the jacobian of the final function with respect to `exp_matrix`
        to the jacobian of the final function with respect to `matrix`
    """
    if operation_policy == OperationPolicy.CPU:
        matrix_size = matrix.shape[0]

        def _expm_vjp_cpu(dfinal_dexpm):
            dfinal_dmatrix = np.zeros((matrix_size, matrix_size), dtype=np.complex128)

            # Compute a first order approximation of the frechet derivative of the matrix
            # exponential in every unit direction Eij.
            for i in range(matrix_size):
                for j in range(matrix_size):
                    dexpm_dmij_rowi = exp_matrix[j,:]
                    dfinal_dmatrix[i, j] = np.sum(np.multiply(dfinal_dexpm[i, :], dexpm_dmij_rowi))
                #ENDFOR
            #ENDFOR

            return dfinal_dmatrix
        #ENDDEF

        vjp_function = _expm_vjp_cpu
    else:
        pass

    return vjp_function

autograd_defvjp(expm, _expm_vjp)

### MODULE TESTS ###

_BIG = 30

def _get_skew_hermitian_matrix(matrix_size):
    """
    Args:
    matrix_size :: int - square matrix size

    Returns:
    skew_hermitian_matrix :: numpy.ndarray - a skew-hermitian matrix
        of `matrix_size`
    """
    matrix = (np.random.rand(matrix_size, matrix_size)
              + 1j * np.random.rand(matrix_size, matrix_size))

    return np.divide(matrix - conjugate_transpose(matrix), 2)

def _tests():
    """
    Run tests on the module.
    Args: none
    Returns: none
    """

    # Test that the end-to-end gradient of the matrix exponential is working.
    m = np.array([[1., 0.],
                  [0., 1.]])
    m_len = m.shape[0]
    exp_m = np.exp(m)
    dexpm_dm_expected = np.zeros((m_len, m_len, m_len, m_len), dtype=m.dtype)
    dexpm_dm_expected[0, 0, 0, 0] = exp_m[0, 0]
    dexpm_dm_expected[0, 1, 0, 1] = exp_m[0, 0]
    dexpm_dm_expected[1, 0, 1, 0] = exp_m[1, 1]
    dexpm_dm_expected[1, 1, 1, 1] = exp_m[1, 1]

    dexpm_dm = jacobian(expm, 0)(m)

    assert(np.allclose(dexpm_dm, dexpm_dm_expected))

if __name__ == "__main__":
    _tests()


# -*- coding: utf-8 -*-
"""
Created on Mon Dec 14 22:00:33 2020

@author: 张瑞娟
"""
import instance.Instance
import instance.Route
import copy

class WorstDestroy():

    @staticmethod
    def destroy(s, destroyNum, ins):     # 传入当前解决方案和destroy的点数
        removeCustomers = []
        solution = copy.deepcopy(s)
        distanceMatrix = ins.distaceMatrix

        while(len(removeCustomers) < destroyNum):
            bestFitness = float("inf")
            for r in solution.routes:
                for n in r.route[1:-1]:

                    #路径中只有一个顾客时，fitness为路径所有成本，否则fitnes为路径成本变化量
                    if(len(r.route) <= 3):
                        fitness = r.cost
                    else:
                        nodeIndex = r.route.index(n)
                        node0 = r.route[nodeIndex - 1]
                        #print('路径长度：', len(r.route))
                        #print('node:', nodeIndex)
                        node1 = r.route[nodeIndex + 1]
                        fitness = distanceMatrix[node0.id][node1.id] - distanceMatrix[node0.id][n.id]
                        - distanceMatrix[n.id][node1.id]

                    if fitness < bestFitness:
                        bestFitness = fitness
                        route = r
                        node = n

            removeCustomers.append(node)
            solution.removeRoute(route)
            route.removeNode(node)
            if(len(route.route) > 2):
                route.cost += ins.perCost * fitness
                solution.addRouteToRoutes(route)

        return removeCustomers, solution


# -*- coding: utf-8 -*-

import datetime
import sys

import numpy as np
import pandas as pd
from sklearn import preprocessing
from sklearn import tree
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import f1_score, classification_report
from sklearn.model_selection import StratifiedKFold

def load_data(filename):
    with open(filename, "rb") as fp:
        for i, line in enumerate(fp):
            if "Buchung" in str(line):
                skiprows = i
                break

    data = pd.read_csv(filename, skiprows=skiprows, sep=";", quotechar='"', encoding="Cp1252", decimal=",", thousands=".",
                       dtype={"Buchung": np.str, "Valuta": np.str}, parse_dates=True)
    data.Buchung = data.Buchung.apply(lambda x: datetime.datetime.strptime(x, "%d.%m.%Y"))
    data.Valuta = data.Valuta.apply(lambda x: datetime.datetime.strptime(x, "%d.%m.%Y"))
    return data

def get_filenames(folder="D:/Dropbox/Finanzen/Ing DIBA/Input/", extension="*.csv"):
    """
    This function gets all filenames of type "ending" in path "folder" and returns a list of all files.
    """
    filename_list = []
    import glob, os
    os.chdir(folder)
    for file in glob.glob(extension):
        filename_list.append(file)
    return filename_list

def merge_files(filename_list, drop_duplicates=True, drop_labels=["Buchung", "Betrag", "Buchungstext"]):
    """
    This function takes a list of files, loads the data from those files, concatenates the DataFrames, removes
    duplicates and returns the processed DataFrame.
    """
    data = pd.DataFrame()

    for filename in filename_list:
        account_number = filename.split("_")[1]
        df = load_data(filename=filename)
        df["account_number"] = account_number
        data = data.append(other=df)

    data = data.sort_values("Buchung", ascending=True)

    # Find intial Account level (Saldo) for each account and add the total (aggregated value of each account) as a
    # baseline account level.
    first_lines_per_account = data.groupby("account_number").first().iterrows()
    total_saldo_before_first_entry = 0
    for line in first_lines_per_account:
        line_saldo_before_first_entry = line[1].Saldo - line[1].Betrag
        total_saldo_before_first_entry += line_saldo_before_first_entry

    if drop_duplicates:
        data = data.drop_duplicates(keep="first", subset=drop_labels)

    data["Betrag_cumsum"] = data["Betrag"].cumsum()
    data["Betrag_cumsum"] += total_saldo_before_first_entry
    return data

def plot_account_data(giro_data, depot_data):
    from matplotlib import pyplot as plt
    import matplotlib.dates as mdates

    # Convert data to time series with daily values (asfreq "D").
    # Also add a comulated time series (giro+depot).
    first_day = min(giro_data["Buchung"].min(), depot_data["Buchung"].min())
    last_day = max(giro_data["Buchung"].max(), depot_data["Buchung"].max())
    ix = pd.DatetimeIndex(start=first_day, end=last_day, freq='D')

    giro_daily = giro_data.groupby("Buchung").Betrag.sum()
    giro_daily = giro_daily.asfreq("D").fillna(0)

    giro_cumulated_daily = giro_data.groupby("Buchung").Betrag_cumsum.last()
    giro_cumulated_daily = giro_cumulated_daily.reindex(ix).fillna(method="ffill")
    giro_cumulated_daily = giro_cumulated_daily.fillna(0)

    depot_cumulated_daily = depot_data["Betrag_cumsum"]
    depot_cumulated_daily = depot_cumulated_daily.reindex(ix).fillna(method="ffill")
    depot_cumulated_daily = depot_cumulated_daily.fillna(0)

    aggregated_cum_daily = giro_cumulated_daily.add(depot_cumulated_daily, fill_value=0)

    # Prepare figure and subplots
    f, axarr = plt.subplots(4, sharex=True)

    # Giro data changes
    axarr[0].set_title("Bookings and cumulated bookings over time")
    axarr[0].bar(giro_daily.index, giro_daily)
    axarr[0].set_ylabel("Giro account changes (daily)")

    # Giro data total
    axarr[1].step(giro_cumulated_daily.index, giro_cumulated_daily, color="blue")
    axarr[1].set_ylabel("Giro total")

    # Depot data total
    axarr[2].step(depot_cumulated_daily.index, depot_cumulated_daily, color="green")
    axarr[2].set_ylabel("Depot total")

    # Account plus depot total
    from sklearn.linear_model import LinearRegression
    axarr[3].step(aggregated_cum_daily.index, aggregated_cum_daily, color="red", label="Cumulated")
    x,y = zip(*enumerate(aggregated_cum_daily))
    fit = np.polyfit(x,y,1)
    fit_fn = np.poly1d(fit)
    axarr[3].plot(aggregated_cum_daily.index, fit_fn(x), "--k", label=u"total saving rate : {:3.0f}€/m".format(365./12*fit_fn.c[0]))

    x_365 = x[-365:]
    y_365 = y[-365:]
    fit_365 = np.polyfit(x_365, y_365, 1)
    fit_fn_365 = np.poly1d(fit_365)
    axarr[3].plot(aggregated_cum_daily.index[-365:], fit_fn_365(x_365), "--b", label=u"last year saving rate : {:3.0f}€/m".format(365./12*fit_fn_365.c[0]))
    axarr[3].set_ylabel("Giro + Depot (Daily)")
    axarr[3].set_xlabel("Date")
    axarr[3].legend(loc="best")

    plt.show()

def parse_depot_files():
    """
    This function browses a depot folder (a folder dedicated to contain only CSV related information).

    It parses each file for date and a DataFrame containing the complete stock.

    It appends all DataFrames plus a TimeSeries representing the total stock value at any parsing time.
    """
    folder = "D:/Dropbox/Finanzen/Ing DIBA/depot"
    filenames = get_filenames(folder=folder, extension="*.csv")
    data = pd.DataFrame()
    for filename in filenames:
        account_number = filename.split("_")[1]
        # Get the DataFrame
        df = pd.read_csv(filename, skiprows=5, sep=";", quotechar='"', encoding="Cp1252", decimal=",", thousands=".",
                         dtype={"Zeit": np.str, "Valuta": np.str}, parse_dates=True)

        # Drop any line that doesnt contain an ISIN (Stock ID)
        df = df[df.ISIN.notnull()]
        df["account_number"] = account_number

        data = data.append(df)
    data.Zeit = data.Zeit.apply(lambda x: datetime.datetime.strptime(x, "%d.%m.%Y"))
    data = data.sort_values("Zeit", ascending=True)

    data.drop_duplicates(keep="first", inplace=True)

    depot_cum_value = data.groupby("Zeit").Kurswert.sum()
    depot_cum_value.name = "Betrag_cumsum"

    # depot_cum_value.index.name = "Buchung"

    depot_cum_value = pd.DataFrame(depot_cum_value)
    depot_cum_value["Betrag"] = depot_cum_value.diff()
    depot_cum_value["Betrag"] = depot_cum_value["Betrag"].fillna(0)
    depot_cum_value["Buchung"] = depot_cum_value.index
    depot_cum_value = depot_cum_value.asfreq("D").fillna(method="ffill")
    return data, depot_cum_value

def get_features(dataset, vect=None, dropna=False):
    X = dataset[u"Auftraggeber/Empfänger"].str.strip() + " | " + dataset["Verwendungszweck"].str.strip() + " | " + dataset["Buchungstext"].str.strip()
    if dropna:
        X.dropna(inplace=True, axis=0)
    else:
        X.fillna("", inplace=True, axis=0)

    if not vect:
        vect = TfidfVectorizer(ngram_range=(1, 3), max_df=1.)
        X_tfidf = vect.fit_transform(X)
    else:
        X_tfidf = vect.transform(X)
    return pd.DataFrame(X, columns=["features"]), X_tfidf, vect

def get_labels(dataset):
    le = preprocessing.LabelEncoder()
    y1_ = le.fit_transform(dataset.Labels)
    return y1_, le

def create_classifier():
    # Predicting category based on receipient and comment

    # Load the labels data and reduce it to only approved rows.
    data = pd.read_excel("D:/Dropbox/Finanzen/Ing DIBA/output/labels.xlsx")
    data.dropna(axis=0, how="any", inplace=True)
    # data = pd.read_csv("D:/Dropbox/Finanzen/Ing DIBA/output/labels_.csv", sep=";")
    data_labelled = data[data["status"] == "approved"]
    data_new = data[data["status"] != "approved"]
    X = data_labelled["features"]
    y = data_labelled["label"]
    X_new = data_new["features"]

    # Use a td-idf vectorizor to convert the text data into machine readable format.
    vectorizor = TfidfVectorizer(ngram_range=(1, 3), max_df=1.)
    X_tfidf = vectorizor.fit_transform(X)

    # Use label enconder to transform text labels into machine readable format.
    label_encoder = preprocessing.LabelEncoder()
    y_encoded = label_encoder.fit_transform(y)
    labels = y.unique()
    labels_enc = label_encoder.transform(labels)

    # Classifier is a Decision Tree Classifier
    # clf = tree.DecisionTreeClassifier()
    clf = RandomForestClassifier(n_estimators=100, n_jobs=-1)

    # CROSSVALIDATION
    kf = StratifiedKFold(n_splits=3, shuffle=True)
    scores = []
    f1_scores = []
    for train, test in kf.split(X_tfidf, y_encoded):
        X_train_ = X_tfidf[train]
        y_train_ = y_encoded[train]

        X_test_ = X_tfidf[test]
        y_test_ = y_encoded[test]

        clf = clf.fit(X_train_, y_train_)
        y_pred_ = clf.predict(X_test_)

        scores.append(clf.score(X_test_, y_test_))
        f1_scores.append(f1_score(y_test_, y_pred_, average="weighted"))

    print("mean scores: %3.3f" % np.mean(scores))
    print("mean f1 scores: %3.3f" % np.mean(f1_scores))

    y_pred = label_encoder.inverse_transform(y_pred_)
    y_test = label_encoder.inverse_transform(y_test_)
    print("Classification report from last fold:")
    print(classification_report(y_test, y_pred))

    # PREDICT
    clf.fit(X_tfidf, y_encoded)

    if len(data_new)>0:
        X_new_tfidf = vectorizor.transform(X_new)
        y_new_enc = clf.predict(X_new_tfidf)
        y_new = label_encoder.inverse_transform(y_new_enc)
        data_new.ix[:,"label"] = y_new
        # data = data_labelled.append(other=data_new, ignore_index=True).reset_index(drop=True)
        """data.to_excel("D:/Dropbox/Finanzen/Ing DIBA/output/labels.xlsx", sheet_name='labels', na_rep='', header=True,
                      index=False, encoding=None)"""

    # data.to_csv("D:/Dropbox/Finanzen/Ing DIBA/output/labels.csv", sep=";", index=False)
    return clf, vectorizor, label_encoder

def classify(giro_data, clf, vectorizor, label_encoder):
    X, X_, _ = get_features(giro_data, vectorizor, dropna=False)
    y_ = clf.predict(X_)
    y = label_encoder.inverse_transform(y_)
    giro_data["Labels"] = y

    old_labels_df = pd.read_excel("D:/Dropbox/Finanzen/Ing DIBA/output/labels.xlsx")
    old_labels_df = old_labels_df[old_labels_df["status"] == "approved"]

    new_labels_df = X
    new_labels_df["label"]  = y
    new_labels_df["status"] = "new"

    old_labels_df = old_labels_df.append(new_labels_df).drop_duplicates()

    old_labels_df.to_csv("D:/Dropbox/Finanzen/Ing DIBA/output/labels_.csv", sep=";", index=False)
    old_labels_df.to_excel("D:/Dropbox/Finanzen/Ing DIBA/output/labels.xlsx",index=False)
    return giro_data

def more_info_on(data, label):
    subset = data[data["Labels"]==label]
    subset = subset[["year", "Betrag", "Buchung", "Labels", "Labels2", u"Auftraggeber/Empfänger", "Verwendungszweck"]].sort_values(["Buchung", "Betrag"])
    return subset

#def main():

    #TODO: Plot: Target Saving Rate
    #TODO: Plot: Aggregate per month

if __name__ == "__main__":
    #sys.exit(main())

    giro_data = merge_files(get_filenames(folder="D:/Dropbox/Finanzen/Ing DIBA/Input/", extension="*.csv"),
                            drop_duplicates=True)
    giro_data.to_csv("D:/Dropbox/Finanzen/Ing DIBA/output/aggregated_giro.csv", sep=";", quotechar='"',
                     encoding="Cp1252", decimal=",")
    depot, depot_cum_value = parse_depot_files()
    plot_account_data(giro_data, depot_cum_value)

    clf, vectorizor, label_encoder = create_classifier()
    giro_data_classified = classify(giro_data, clf, vectorizor, label_encoder)
    giro_data_classified["year"] = giro_data_classified["Buchung"].apply(lambda x: x.year)
    giro_data_classified.to_csv("D:/Dropbox/Finanzen/Ing DIBA/output/classified_giro.csv", sep=";", quotechar='"',
                                encoding="Cp1252", decimal=",")

    first_day = giro_data["Buchung"].min()
    last_day = giro_data["Buchung"].max()
    duration = last_day - first_day
    days = duration.days

    print("Average daily expensenses over all years:")
    grouped_all = giro_data_classified.groupby("Labels").sum().sort_values("Betrag")["Betrag"].div(days)
    print(grouped_all)
    print("Sum: ", grouped_all.sum())

    # Only 2016
    year = 2017
    current_year = giro_data_classified[giro_data_classified["year"] == year]
    first_day = current_year["Buchung"].min()
    last_day = current_year["Buchung"].max()
    duration = last_day - first_day
    days = duration.days
    print("\ncurrent year (%i):" % year)
    grouped_current = current_year.groupby("Labels").sum().sort_values("Betrag")["Betrag"].div(days)
    print(grouped_current)
    print("Sum: ", grouped_current.sum())

    print(giro_data_classified.groupby("year").sum().sort_index()["Betrag"])
    print(giro_data.pivot_table(index="Labels", columns="year", values="Betrag", aggfunc="sum", fill_value=0))

    grouped = pd.DataFrame(grouped_all).join(pd.DataFrame(grouped_current), rsuffix="_current" )
    grouped["diff_abs"] = grouped.diff(axis=1).ix[:,-1]
    grouped["diff_percentage"] = grouped["diff_abs"].div(grouped["Betrag"])*100

    print("\nComparison:")
    print(grouped.sort_values("diff_abs"))


import PyQt4.QtGui as qg
import PyQt4.QtCore as qc

import maya.cmds as mc

import os
import xml.etree.ElementTree as ET

import cg_inventor.sys.lib.qt.base as base
import cg_inventor.sys.lib.qt.widgets.scene_check as scene_check

from cg_inventor.maya.apps.toolbox import page, settings

import cg_inventor.maya.utils.generic    as util_generic
import cg_inventor.maya.utils.transforms as util_transforms
import cg_inventor.maya.utils.history    as util_history
import cg_inventor.maya.utils.mesh       as util_mesh
import cg_inventor.maya.utils.texture    as util_texture

callbackPlus    = util_generic.callbackPlus
callbackSetup   = util_generic.callbackSetup
callbackCounter = util_generic.callbackCounter

setup  = scene_check.setup
check  = scene_check.check
select = scene_check.select
fix    = scene_check.fix

MODE_SEL = scene_check.SceneCheck.MODE_SEL
MODE_ALL = scene_check.SceneCheck.MODE_ALL

# ------------------------------------------------------------------------------------------------ #
#                                      MODELLING CHECKS                                            #
# ------------------------------------------------------------------------------------------------ #

TITLE    = 'Scene Checker'
SUBTITLE = 'Modelling'

class ModellingChecks(page.Page):
    def __init__(self):
        page.Page.__init__(self)
        self.title    = TITLE
        self.subtitle = SUBTITLE

        self._all_sets    = []
        self._all_spacers = []

        self.settings_widget = ModellingChecksSettings

        self.loadXML("scene_checks_modelling.xml")

    def _addSceneCheckSet(self, title):
        new_set = scene_check.SceneCheckSet(title)
        self._all_sets.append(new_set)
        self.addElement(new_set)

        self.connect(new_set, qc.SIGNAL(base.Base.STATUS_NORMAL),  self.statusNormal)
        self.connect(new_set, qc.SIGNAL(base.Base.STATUS_SUCCESS), self.statusSuccess)
        self.connect(new_set, qc.SIGNAL(base.Base.STATUS_ERROR),   self.statusError)

        return new_set

    def clear(self):
        for check_set in self._all_sets:
            self.removeElement(check_set)
            check_set.deleteLater()

        for spacer in self._all_spacers:
            self.removeElement(spacer)

        self._all_sets    = []
        self._all_spacers = []

    def loadXML(self, xml_file):
        # get toolbox xml path
        #
        import cg_inventor.maya.apps.toolbox as toolbox
        xml_path = os.path.join(os.path.dirname(toolbox.__file__), 'xml', xml_file)

        # get the root element
        #
        tree = ET.parse(xml_path)
        root = tree.getroot()

        for check_set in root:
            set_title = check_set.attrib['title']
            new_set = self._addSceneCheckSet(set_title)

            for check in check_set:
                function_set = str(check.text).strip()
                new_check = scene_check.SceneCheck()
                new_check.loadFromLibrary(function_set)

                new_set.addSceneCheck(new_check)

            spacer = qg.QSpacerItem(20,20,qg.QSizePolicy.Fixed)
            self.addElement(spacer)
            self._all_spacers.append(spacer)

# ------------------------------------------------------------------------------------------------ #
#                                      MODELLING CHECKS                                            #
# ------------------------------------------------------------------------------------------------ #

class ModellingChecksSettings(settings.Settings):
    def __init__(self):
        settings.Settings.__init__(self)
        #self.setMinimumWidth(600)
        self.setFixedWidth(700)

        main_layout = qg.QHBoxLayout()
        main_layout.setContentsMargins(5,0,5,5)
        main_layout.setSpacing(5)
        self.addElement(main_layout)

        group_widget      = qg.QWidget()
        group_widget.setLayout(qg.QVBoxLayout())
        group_widget.layout().setContentsMargins(0,0,0,0)
        group_widget.layout().setSpacing(0)
        check_widget      = qg.QWidget()
        check_widget.setLayout(qg.QVBoxLayout())
        check_widget.layout().setContentsMargins(0,0,0,0)
        check_widget.layout().setSpacing(0)
        button_widget     = qg.QWidget()
        button_widget.setLayout(qg.QVBoxLayout())
        all_checks_widget = qg.QWidget()
        all_checks_widget.setLayout(qg.QVBoxLayout())
        all_checks_widget.layout().setContentsMargins(0,0,0,0)
        all_checks_widget.layout().setSpacing(0)
        description_widget = qg.QWidget()
        description_widget.setLayout(qg.QVBoxLayout())
        description_widget.layout().setContentsMargins(0,0,0,0)
        description_widget.layout().setSpacing(0)

        button_widget.setFixedWidth(30)

        main_layout.addWidget(group_widget)
        main_layout.addWidget(check_widget)
        main_layout.addWidget(button_widget)
        main_layout.addWidget(all_checks_widget)
        main_layout.addWidget(description_widget)

        title_font = qg.QFont()
        title_font.setBold(True)

        group_label        = qg.QLabel('GROUPS')
        group_layout       = qg.QHBoxLayout()
        checks_label       = qg.QLabel('CHECKS')
        checks_layout      = qg.QHBoxLayout()
        all_checks_label   = qg.QLabel('ALL CHECKS')
        all_checks_layout  = qg.QHBoxLayout()
        description_label  = qg.QLabel('DESCRIPTION')
        description_layout = qg.QHBoxLayout()

        group_layout.addSpacerItem(qg.QSpacerItem(5, 5, qg.QSizePolicy.Expanding))
        group_layout.addWidget(group_label)
        group_layout.addSpacerItem(qg.QSpacerItem(5, 5, qg.QSizePolicy.Expanding))

        checks_layout.addSpacerItem(qg.QSpacerItem(5, 5, qg.QSizePolicy.Expanding))
        checks_layout.addWidget(checks_label)
        checks_layout.addSpacerItem(qg.QSpacerItem(5, 5, qg.QSizePolicy.Expanding))

        all_checks_layout.addSpacerItem(qg.QSpacerItem(5, 5, qg.QSizePolicy.Expanding))
        all_checks_layout.addWidget(all_checks_label)
        all_checks_layout.addSpacerItem(qg.QSpacerItem(5, 5, qg.QSizePolicy.Expanding))

        description_layout.addSpacerItem(qg.QSpacerItem(5, 5, qg.QSizePolicy.Expanding))
        description_layout.addWidget(description_label)
        description_layout.addSpacerItem(qg.QSpacerItem(5, 5, qg.QSizePolicy.Expanding))

        group_label.setFont(title_font)
        checks_label.setFont(title_font)
        all_checks_label.setFont(title_font)
        description_label.setFont(title_font)

        group_label.setFixedHeight(18)
        checks_label.setFixedHeight(18)
        all_checks_label.setFixedHeight(18)
        description_label.setFixedHeight(18)

        self.group_list      = qg.QListWidget()
        self.check_list      = qg.QListWidget()
        self.all_checks_list = qg.QListWidget()
        self.description_te  = qg.QTextEdit()

        group_widget.layout().addLayout(group_layout)
        group_widget.layout().addWidget(self.group_list)
        check_widget.layout().addLayout(checks_layout)
        check_widget.layout().addWidget(self.check_list)
        all_checks_widget.layout().addLayout(all_checks_layout)
        all_checks_widget.layout().addWidget(self.all_checks_list)
        description_widget.layout().addLayout(description_layout)
        description_widget.layout().addWidget(self.description_te)

        self.all_checks_list.itemClicked.connect(self.showDescription)

        self._check_descriptions = {}

        self.populateAllChecks()

    def clearAllChecks(self):
        pass

    def populateAllChecks(self):
        self.clearAllChecks()

        all_names = []
        for check_name, setup_func in scene_check.SceneCheckLibrary.setup.items():
            title, _, description = setup_func()
            self._check_descriptions[title] = description
            all_names.append(title)

        for name in sorted(all_names):
            new_item = qg.QListWidgetItem()
            new_item.setText(name)
            self.all_checks_list.addItem(new_item)

    def showDescription(self, check_item):
        description = self._check_descriptions[str(check_item.text())]

        self.description_te.setText(description)

# ------------------------------------------------------------------------------------------------ #
#                                           CHECKS                                                 #
# ------------------------------------------------------------------------------------------------ #

@setup
def freezeTransforms():
    title       = 'Unfrozen Transforms'
    message     = 'Found $NUM Unfrozen Transform(s)'
    description = 'Finds all transforms with unfrozen values.'
    return title, message, description

@check
def freezeTransforms(selection_mode, callback=None):
    try:
        if selection_mode == MODE_SEL:
            transforms = mc.ls(sl=True)
        else:
            transforms = util_transforms.getAllTransforms()

        if not len(transforms): return

        return util_transforms.getUnfrozenTransforms(transforms, callback=callback)

    except util_transforms.TransformError as e:
        raise scene_check.SceneCheckError(e)

@select
def freezeTransforms(unfrozen_transforms):
    mc.select(mc.ls(unfrozen_transforms, type='transform'), r=True)

@fix
def freezeTransforms(transforms, callback=None):
    try:
        util_transforms.freezeTransforms(transforms, filter=True, callback=callback)
    except util_transforms.TransformError as e:
        raise scene_check.SceneCheckError(str(e))

#--------------------------------------------------------------------------------------------------#

@setup
def nonDeletedHistory():
    title       = 'Non-Deleted History'
    message     = 'Found $NUM Node(s) with History'
    description = 'Find any nodes with history.'
    return title, message, description

@check
def nonDeletedHistory(selection_mode, callback=None):
    try:
        if selection_mode == MODE_SEL:
            transforms = mc.ls(sl=True)
        else:
            transforms = util_transforms.getAllTransforms()

        if not len(transforms): return

        return util_history.getNodesWithHistory(transforms, callback=callback)
    except util_history.HistoryError as e:
        raise scene_check.SceneCheckError(str(e))

@select
def nonDeletedHistory(history_nodes):
    mc.select(mc.ls(history_nodes), r=True)

@fix
def nonDeletedHistory(history_nodes):
    try:
        util_history.deleteHistory(history_nodes)
    except util_history.HistoryError as e:
        raise scene_check.SceneCheckError(str(e))

#--------------------------------------------------------------------------------------------------#

@setup
def centeredPivots():
    title       = 'Non Centered Pivots'
    message     = 'Found $NUM Non Centered Pivot(s)'
    description = 'Find any geometry with uncentered pivots.'
    return title, message, description

@check
def centeredPivots(selection_mode, callback=None):
    try:
        if selection_mode == MODE_SEL:
            meshes = util_mesh.getMeshesFromSelection(mc.ls(sl=True))
        else:
            meshes = util_mesh.getAllMeshes()

        if not len(meshes): return

        if callback: counter, increment = callbackSetup(len(meshes), callback_range=30)

        transforms = set([])
        for mesh in meshes:
            if callback: callback(callbackCounter(counter, increment))

            if util_mesh._isMeshEmpty(mesh): continue
            transforms.add(util_mesh.getTransformFromShape(mesh))

        if callback: counter, increment = callbackSetup(len(transforms), callback_range=70)

        not_centred = []
        for transform in transforms:
            if callback: callback(30 + callbackCounter(counter, increment))

            if not util_transforms.isPivotCentered(transform):
                not_centred.append(transform)

        return not_centred

    except (util_transforms.TransformError, util_mesh.MeshError) as e:
        raise scene_check.SceneCheckError(str(e))

@select
def centeredPivots(transforms):
    mc.select(transforms, r=True)

@fix
def centeredPivots(transforms, callback=None):
    try:
        mc.xform(transforms, cp=True)
    except Exception as e:
        raise scene_check.SceneCheckError(str(e))

#--------------------------------------------------------------------------------------------------#

@setup
def unknownNodes():
    title       = "'Unknown' Nodes"
    message     = "Found $NUM 'Unknown' Node(s)"
    description = 'Find any geometry with uncentered pivots.'
    return title, message, description

@check
def unknownNodes(selection_mode):
    if selection_mode == MODE_SEL:
        return mc.ls(sl=True, type='unknown')
    else:
        return mc.ls(type='unknown')

@select
def unknownNodes(nodes):
    mc.select(mc.ls(nodes), r=True)

#--------------------------------------------------------------------------------------------------#

@setup
def triangles():
    title       = 'Triangles'
    message     = 'Found $NUM Triangle(s)'
    description = 'Find any geometry with uncentered pivots.'
    return title, message, description

@check
def triangles(selection_mode, callback=None):
    try:
        if selection_mode == MODE_SEL:
            meshes = util_mesh.getMeshesFromSelection(mc.ls(sl=True))
        else:
            meshes = util_mesh.getAllMeshes()

        if not len(meshes): return

        return util_mesh.checkMesh(meshes, flags=util_mesh.TRIANGLES, callback=callback)

    except util_mesh.MeshError as e:
        raise scene_check.SceneCheckError(str(e))

@select
def triangles(triangles):
    mc.select(triangles, r=True)

#--------------------------------------------------------------------------------------------------#

@setup
def nsided():
    title   = 'N-Sided Faces'
    message     = 'Found $NUM N-Sided Face(s)'
    description = 'Find any geometry with uncentered pivots.'
    return title, message, description

@check
def nsided(selection_mode, callback=None):
    try:
        if selection_mode == MODE_SEL:
            meshes = util_mesh.getMeshesFromSelection(mc.ls(sl=True))
        else:
            meshes = util_mesh.getAllMeshes()

        if not len(meshes): return

        return util_mesh.checkMesh(meshes, flags=util_mesh.NSIDED, callback=callback)

    except util_mesh.MeshError as e:
        raise scene_check.SceneCheckError(str(e))

@select
def nsided(faces):
    mc.select(faces, r=True)

#--------------------------------------------------------------------------------------------------#

@setup
def holed():
    title       = 'Holes'
    message     = 'Found $NUM Hole(s)'
    description = 'Find any geometry with uncentered pivots.'
    return title, message, description

@check
def holed(selection_mode, callback=None):
    try:
        if selection_mode == MODE_SEL:
            meshes = util_mesh.getMeshesFromSelection(mc.ls(sl=True))
        else:
            meshes = util_mesh.getAllMeshes()

        if not len(meshes): return

        if callback: counter, increment = callbackSetup(len(meshes), callback_range=30)

        holes = []
        for mesh in meshes:
            if callback: callback(callbackCounter(counter, increment))

            if util_mesh._isMeshEmpty(mesh): continue
            mesh_holes = util_mesh.getMeshHoles(mesh)
            holes.extend([(mesh, mesh_hole) for mesh_hole in mesh_holes])

        return holes

    except util_mesh.MeshError as e:
        raise scene_check.SceneCheckError(str(e))

@select
def holed(holes):
    edges = []
    for mesh, edge_indices in holes:
        edges.extend(['%s.e[%s]' %(mesh, edge_index) for edge_index in edge_indices])
    mc.select(mc.ls(edges), r=True)

#--------------------------------------------------------------------------------------------------#

@setup
def zeroLengthEdges():
    title       = 'Zero Length Edges'
    message     = 'Found $NUM Zero Length Edge(s)'
    description = 'Find any geometry with uncentered pivots.'
    return title, message, description

@check
def zeroLengthEdges(selection_mode, callback=None):
    try:
        if selection_mode == MODE_SEL:
            meshes = util_mesh.getMeshesFromSelection(mc.ls(sl=True))
        else:
            meshes = util_mesh.getAllMeshes()

        if not len(meshes): return

        return util_mesh.checkMesh(meshes, flags=util_mesh.ZERO_LENGTH_EDGES, callback=callback)

    except util_mesh.MeshError as e:
        raise scene_check.SceneCheckError(str(e))

@select
def zeroLengthEdges(edges):
    mc.select(mc.ls(edges), r=True)

#--------------------------------------------------------------------------------------------------#

@setup
def zeroAreaFaces():
    title       = 'Zero Area Faces'
    message     = 'Found $NUM Zero Area Face(s)'
    description = 'Find any geometry with uncentered pivots.'
    return title, message, description

@check
def zeroAreaFaces(selection_mode, callback=None):
    try:
        if selection_mode == MODE_SEL:
            meshes = util_mesh.getMeshesFromSelection(mc.ls(sl=True))
        else:
            meshes = util_mesh.getAllMeshes()

        if not len(meshes): return

        return util_mesh.checkMesh(meshes, flags=util_mesh.ZERO_AREA_FACES, callback=callback)

    except util_mesh.MeshError as e:
        raise scene_check.SceneCheckError(str(e))

@select
def zeroAreaFaces(faces):
    mc.select(mc.ls(faces), r=True)

#--------------------------------------------------------------------------------------------------#

@setup
def lockedNormals():
    title       = 'Locked Normals'
    message     = 'Found $NUM Locked Normal(s)'
    description = 'Find any geometry with uncentered pivots.'
    return title, message, description

@check
def lockedNormals(selection_mode, callback=None):
    try:
        if selection_mode == MODE_SEL:
            meshes = util_mesh.getMeshesFromSelection(mc.ls(sl=True))
        else:
            meshes = util_mesh.getAllMeshes()

        num_meshes = len(meshes)
        if not num_meshes: return

        if callback: counter, increment = callbackSetup(num_meshes, callback_range=100)

        locked_normals = []
        for mesh in meshes:
            if callback: callback(callbackCounter(counter, increment))

            if util_mesh._isMeshEmpty(mesh): continue
            indices = util_mesh.getLockedVertexNormals(mesh)
            vertices = ['%s.vtx[%s]' %(mesh, index) for index in indices]
            locked_normals.extend(vertices)

        return locked_normals

    except util_mesh.MeshError as e:
        raise scene_check.SceneCheckError(str(e))

@select
def lockedNormals(normals):
    mc.select(mc.ls(normals), r=True)

@fix
def lockedNormals(normals):
    try:
        mc.polyNormalPerVertex(normals, ufn=True)
    except Exception as e:
        raise scene_check.SceneCheckError(str(e))

#--------------------------------------------------------------------------------------------------#

@setup
def hardEdges():
    title       = 'Hard Edges'
    message     = 'Found $NUM Hard Edge(s)'
    description = 'Find any geometry with uncentered pivots.'
    return title, message, description

@check
def hardEdges(selection_mode, callback=None):
    try:
        if selection_mode == MODE_SEL:
            meshes = util_mesh.getMeshesFromSelection(mc.ls(sl=True))
        else:
            meshes = util_mesh.getAllMeshes()

        num_meshes = len(meshes)
        if not num_meshes: return

        if callback: counter, increment = callbackSetup(num_meshes, callback_range=100)

        hard_edges = []
        for mesh in meshes:
            if callback: callback(callbackCounter(counter, increment))

            if util_mesh._isMeshEmpty(mesh): continue
            hard_edges.extend([(mesh, edge) for edge in util_mesh.getHardEdges(mesh)])

        return hard_edges

    except util_mesh.MeshError as e:
        raise scene_check.SceneCheckError(str(e))

@select
def hardEdges(edges):
    edge_strings = []
    for mesh, edge in edges:
        edge_strings.append('%s.e[%s]' %(mesh, edge))
    mc.select(edge_strings, r=True)

@fix
def hardEdges(edges, callback=None):
    try:
        all_hard_edges = {}

        if callback:
            counter, increment = callbackSetup(len(edges), callback_range=20)

        for mesh, edge in edges:
            if callback: callback(callbackCounter(counter, increment))

            try:
                hard_edges = all_hard_edges[mesh]
            except KeyError:
                hard_edges = all_hard_edges[mesh] = []
            hard_edges.append('%s.e[%s]' %(mesh, edge))

        if callback:
            counter, increment = callbackSetup(len(all_hard_edges.keys()), callback_range=80)

        for mesh, hard_edges in all_hard_edges.items():
            if callback: callback(20 + callbackCounter(counter, increment))

            history = bool(mc.listHistory(mesh, il=2, pdo=True))
            mc.polySoftEdge(*hard_edges, a=180, ch=history)

    except Exception as e:
        raise scene_check.SceneCheckError(str(e))

#--------------------------------------------------------------------------------------------------#

@setup
def unconnectedIntermediates():
    title       = 'Unconnected Intermediates'
    message     = 'Found $NUM Unconnected Intermediate(s)'
    description = 'Find any geometry with uncentered pivots.'
    return title, message, description

@check
def unconnectedIntermediates(selection_mode):
    return util_mesh.getUnconnectedIntermediates()

@select
def unconnectedIntermediates(intermediates):
    mc.select(mc.ls(intermediates), r=True)

@fix
def unconnectedIntermediates(intermediates):
    intermediates = mc.ls(intermediates)
    if not intermediates: return

    mc.delete(intermediates)

#--------------------------------------------------------------------------------------------------#

@setup
def emptyMeshes():
    title       = 'Empty Meshes'
    message     = 'Found $NUM Empty Mesh(s)'
    description = 'Find any geometry with uncentered pivots.'
    return title, message, description

@check
def emptyMeshes(selection_mode, callback=None):
    try:
        if selection_mode == MODE_SEL:
            meshes = util_mesh.getMeshesFromSelection(mc.ls(sl=True))
        else:
            meshes = util_mesh.getEmptyMeshes()

        num_meshes = len(meshes)
        if not num_meshes: return

        if callback: counter, increment = callbackSetup(num_meshes, callback_range=100)

        empty_meshes = []
        for mesh in meshes:
            if callback: callback(callbackCounter(counter, increment))

            if util_mesh._isMeshEmpty(mesh):
                empty_meshes.append(mesh)

        return meshes

    except util_mesh.MeshError as e:
        raise scene_check.SceneCheckError(str(e))

@select
def emptyMeshes(meshes):
    mc.select(mc.ls(meshes), r=True)

@fix
def emptyMeshes(meshes):
    meshes = mc.ls(meshes)
    if not meshes: return

    try:
        mc.delete(meshes)
    except Exception as e:
        raise scene_check.SceneCheckError(str(e))

#--------------------------------------------------------------------------------------------------#

@setup
def unusedShaders():
    title       = 'Unused Shaders'
    message     = 'Found $NUM Unused Shader(s)'
    description = 'Find any geometry with uncentered pivots.'
    return title, message, description

@check
def unusedShaders(selection_mode):
    try:
        if selection_mode == MODE_SEL:
            materials = mc.ls(sl=True, mat=True)

            unused_materials = []
            for material in materials:
                if util_texture._getMaterialMembers(material) is None:
                    unused_materials.append(material)

        else:
            unused_materials = util_texture.getAllUnusedMaterials()

        return unused_materials

    except util_texture.TextureError as e:
        raise scene_check.SceneCheckError(str(e))

@select
def unusedShaders(materials):
    mc.select(materials, r=True)

@fix
def unusedShaders(materials, callback=None):
    try:
        if callback: counter, increment = callbackSetup(len(materials), callback_range=100)

        for material in materials:
            if callback: callback(callbackCounter(counter, increment))

            util_texture.deleteMaterial(material)

    except util_texture.TextureError as e:
        raise scene_check.SceneCheckError(str(e))

#--------------------------------------------------------------------------------------------------#

@setup
def uvsOutOfBounds():
    title       = 'UVs Out of Bounds'
    message     = 'Found $NUM UV(s) Out of Bounds'
    description = 'Find any geometry with uncentered pivots.'
    return title, message, description

@check
def uvsOutOfBounds(selection_mode, callback=None):
    try:
        if selection_mode == MODE_SEL:
            meshes = util_mesh.getMeshesFromSelection(mc.ls(sl=True))
        else:
            meshes = util_mesh.getAllMeshes()

        num_meshes = len(meshes)
        if not num_meshes: return

        if callback: counter, increment = callbackSetup(num_meshes, callback_range=100)

        all_uvs_oob = []
        for mesh in meshes:
            if callback: callback(callbackCounter(counter, increment))

            uvs_oob = util_texture.getAllUVsOutOfBounds(mesh)
            for uv_index in uvs_oob:
                all_uvs_oob.append('%s.map[%s]' %(mesh, uv_index))

        return all_uvs_oob

    except util_texture.TextureError as e:
        raise scene_check.SceneCheckError(str(e))

@select
def uvsOutOfBounds(uvs):
    mc.select(uvs, r=True)

#--------------------------------------------------------------------------------------------------#

@setup
def fivePointers():
    title       = 'Five+ Pointers'
    message     = 'Found $NUM Five+ Pointer(s)'
    description = 'Find any geometry with uncentered pivots.'
    return title, message, description

@check
def fivePointers(selection_mode, callback=None):
    try:
        if selection_mode == MODE_SEL:
            meshes = util_mesh.getMeshesFromSelection(mc.ls(sl=True))
        else:
            meshes = util_mesh.getAllMeshes()

        five_pointers = []
        for mesh in meshes:
            vertices = util_mesh.getFiveEdgeVerts(mesh)
            for vertex_index in vertices:
                five_pointers.append('%s.vtx[%s]' %(mesh, vertex_index))

        return five_pointers

    except Exception as e:
        raise scene_check.SceneCheckError(str(e))

@select
def fivePointers(verts):
    mc.select(verts, r=True)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Sep 28 21:15:03 2020.

@author: Odegov Ilya
"""
from sys import stdin

def process_line(line: str) -> int:
    """Process input line."""
    # check input line type
    price = int(line.strip().split(",")[-7])
    return price

def get_line(lines):
    """Generator for input line."""
    for line in lines:
        yield line

def process_data():
    """Process input data."""
    prices_sum, size = 0, 0
    lines = get_line(stdin)
    for line in lines:
        try:
            price = process_line(line)
        except ValueError:
            continue
        except IndexError:
            continue
        prices_sum += price
        size += 1
    # compute chunk size, mean
    mean = prices_sum / size
    print(f"chunk\t{size}, {mean}")

if __name__ == "__main__":
    process_data()