import sys, math, copy
import numpy
import matplotlib.pylab as plt

#---------- Useful stubs for plotting, writing PDB, etc ----------#

def writePDBcoordinates(PDBProfile, newFilename):
    fh = open(newFilename,'w')
    for i in xrange(len(PDBProfile)):
        fh.write("%-6s%5d  %-4s%3s %1s%4d%1s   %8.3f%8.3f%8.3f%6.2f%6.2f          %2s%2s\n" % ('ATOM', i+1, PDBProfile[i].AtomName, PDBProfile[i].ResName, 'A', PDBProfile[i].ResNum, '', PDBProfile[i].X, PDBProfile[i].Y, PDBProfile[i].Z, 1,0,'', ''))
    fh.write('TER')
    fh.close()

def plotContacts(matrix):
    x_min = min([k[0] for k in matrix.iterkeys()]); x_max = max([k[0] for k in matrix.iterkeys()])
    y_min = min([k[1] for k in matrix.iterkeys()]); y_max = max([k[1] for k in matrix.iterkeys()])
    pmin= min(x_min,y_min); pmax=max(x_max,y_max)
    m = [[0 for i in xrange(pmax-pmin+1)] for j in xrange(pmax-pmin+1)]
    for k,v in matrix.iteritems():
        m[k[0]][k[1]] = v; m[k[1]][k[0]] = v
    z = numpy.matrix(m)
    plt.imshow(z, extent=[pmin, pmax, pmax, pmin])
    plt.colorbar()
    plt.show()

def plotRMSDs(data, angle_steps=60):
    nx = math.sqrt(len(data))
    ny = math.sqrt(len(data))
    Z = numpy.matrix(data)
    Z = numpy.transpose(Z.reshape((nx,ny)))
    fig, ax = plt.subplots()
    plt.imshow(Z, interpolation='nearest', extent=[0,(nx-1)*angle_steps,(ny-1)*angle_steps,0])
    labels = range(0,360+1,angle_steps)
    ax.set_xticks(labels, minor=False)
    ax.set_yticks(labels, minor=False)
    labels = map(str, labels)
    ax.set_xticklabels(labels, minor=False)
    ax.set_yticklabels(labels, minor=False)
    ax.set_xlabel('Rotation on X axis')
    ax.set_ylabel('Rotation on Y axis')
    cbar = plt.colorbar()
    cbar.set_label('RMSD')
    plt.show()

def plotTorsionScatter(phipsi):
    phi = [p[0] for p in phipsi]
    psi = [p[1] for p in phipsi]
    fig, ax = plt.subplots()
    plt.scatter(phi,psi)
    plt.xlim(-180, +180)
    plt.ylim(-180, +180)
    ax.set_xlabel('Phi (degrees)')
    ax.set_ylabel('Psi (degrees)')
    plt.show()


from http.server import BaseHTTPRequestHandler
from socketserver import TCPServer

class RequestHandler(object):
    def __init__(self, router):
        self.render = router().render
        self.write = router().write
        self.redirect = router().redirect
        self.get()

    def get(self):
        pass

class Router(BaseHTTPRequestHandler):
    routes = {}

    template_path = None
    static_path = None
    @classmethod
    def set_template(cls, path):
        # TODO test path
        cls.template_path = path
    @classmethod
    def set_static(cls, path):
        # TODO test path
        cls.static_path = path

    def do_GET(self):
        handler = self.routes.get(self.path)
        if handler:
            handler(self)
        else:
            self.send_error(404)

    def write(self, message):
        self.send_response(200)
        self.send_header("Content-type", "text/plain")
        self.end_headers()
        m = message.encode('utf-8')
        self.wfile.write(m)

    def render(self, url):
        self.send_response(200)
        self.send_header("Content-type", "text/html")
        self.end_headers()
        if self.template_path:
            # FIXME /
            url = self.template_path + url
        f = open(url, 'r')
        m = f.read().encode('utf-8')
        self.wfile.write(m)

    def redirect(self, url):
        self.send_response(301)
        self.send_header("Location", url)
        self.end_headers()

    @classmethod
    def add_rule(cls, rule):
        if type(rule) is list:
            for r in rule:
                if type(r) is tuple and len(r) == 2:
                    pattern, handler = r
                    cls.add_handler(pattern, handler)
                else:
                    raise(ValueError)
        elif type(rule):
            raise(ValueError)

    @classmethod
    def add_handler(cls, pattern, handler):
        if type(pattern) is str and issubclass(handler, RequestHandler):
            cls.routes[pattern] = handler
        else:
            raise(ValueError)

    def __call__(self):
        return self

class Application(object):
    def __init__(self, rule=None, template_path=None,
                 static_path=None, router=Router):
        Router.set_template(template_path)
        Router.set_static(static_path)
        self.router = router
        self.router.add_rule(rule)

    def listen(self, host, port):
        try:
            httpd = TCPServer((host, port), self.router)
            print("Running server on %s %d" % (host, port))
            print("Quit the server with CONTROL-C")
            httpd.serve_forever()
        except KeyboardInterrupt:
            print("\nServer is stopping...")
            httpd.shutdown()
            print("Server is stopped!")


"""
actuators.py

Classes to control the motors and servos. These classes
are wrapped in a mixer class before being used in the drive loop.

"""

import time
import sys
import serial, atexit

def map_range(x, X_min, X_max, Y_min, Y_max):
    '''
    Linear mapping between two ranges of values
    '''
    X_range = X_max - X_min
    Y_range = Y_max - Y_min
    XY_ratio = X_range/Y_range

    y = ((x-X_min) / XY_ratio + Y_min) // 1

    return int(y)

class RasPiRobot_Controller:
    def __init__(self, driveLeft, driveRight):
        import rrb3
        rr = RRB3(9, 6)
        leftDir = 0
        rightDir = 0
        if driveLeft < 0:  # change direction if number is negative
            leftDir = 1
        if driveRight < 0:
            rightDir = 1
        rr.set_motors(abs(driveLeft), leftDir, abs(driveRight), rightDir)

class PCA9685_Controller:
    '''
    Adafruit PWM controler.
    This is used for most RC Cars
    '''
    def __init__(self, channel, frequency=60):
        import Adafruit_PCA9685
        # Initialise the PCA9685 using the default address (0x40).
        self.pwm = Adafruit_PCA9685.PCA9685()

        self.pwm.set_pwm_freq(frequency)
        self.channel = channel

    def set_pulse(self, pulse):
        self.pwm.set_pwm(self.channel, 0, pulse)

class PWMSteeringActuator:
    #max angle wheels can turn
    LEFT_ANGLE = -1
    RIGHT_ANGLE = 1

    def __init__(self, controller=None,
                       left_pulse=290,
                       right_pulse=490):

        self.controller = controller
        self.left_pulse = left_pulse
        self.right_pulse = right_pulse

    def update(self, angle):
        #map absolute angle to angle that vehicle can implement.
        pulse = map_range(angle,
                          self.LEFT_ANGLE, self.RIGHT_ANGLE,
                          self.left_pulse, self.right_pulse)

        self.controller.set_pulse(pulse)

class PWMThrottleActuator:

    MIN_THROTTLE = -1
    MAX_THROTTLE =  1

    def __init__(self, controller=None,
                       max_pulse=300,
                       min_pulse=490,
                       zero_pulse=350):

        #super().__init__(channel, frequency)
        self.controller = controller
        self.max_pulse = max_pulse
        self.min_pulse = min_pulse
        self.zero_pulse = zero_pulse
        self.calibrate()

    def calibrate(self):
        #Calibrate ESC (TODO: THIS DOES NOT WORK YET)
        print('center: %s' % self.zero_pulse)
        self.controller.set_pulse(self.zero_pulse)  #Set Max Throttle
        time.sleep(1)

    def update(self, throttle):
        if throttle > 0:
            pulse = map_range(throttle,
                              0, self.MAX_THROTTLE,
                              self.zero_pulse, self.max_pulse)
        else:
            pulse = map_range(throttle,
                              self.MIN_THROTTLE, 0,
                              self.min_pulse, self.zero_pulse)

        sys.stdout.flush()
        self.controller.set_pulse(pulse)
        return '123'

class SerialInterface:
    '''
    We need one instance of this class per device to control connection
    '''

    ser = None
    logfile = None

    def log_serial(self):
      '''
      We must read messages from serial device else we may get blocked io
      '''
      interrupted = False
      while not interrupted:
          try:
            if self.ser is None or self.ser.in_waiting <= 0:
              time.sleep(0.01)
            else:
              line=self.ser.readline()
              if len(line):
                if self.log:
                  self.logfile.write(line)
                else:
                  print(line)

          except (serial.SerialException, serial.serialutil.SerialException,
                  serial.SerialTimeoutException):
            time.sleep(0.01)
            pass   #Try again

          except KeyboardInterrupt:
            interrupted = True
            cleanup()
            raise

    #def __init__(self, device='/dev/ttyACM0', rate=115200, log=False):
    def __init__(self, device='/dev/serial0', rate=115200, log=False):
        import tempfile
        from threading import Thread

        self.log = log
        #Create a logfile
        if log and self.logfile is None:
            self.logfile = tempfile.NamedTemporaryFile(prefix='ser_', dir='.', delete=False)

        # Initialise the serial connection from RPi to its controller
        if self.ser is None:
            #Don't open the port yet
            self.ser = serial.Serial()
            self.ser.port = device
            self.ser.baudrate = rate
            self.ser.timeout = 0
            self.inWrite = False
            self.firstTime = True
            print("Serial device to open", device)
            self.openDevice()

        else:
          print("Serial is already open")

        atexit.register(self.cleanup)

    def openDevice(self):
        #devices = [self.ser.port, '/dev/ttyACM0', '/dev/ttyACM1']
        #for device in devices:
        try:
            #self.ser.port = device
            self.ser.open()
            #Wait for arduino to reset
            time.sleep(2)
            if not self.ser.is_open:
              self.ser.close()
            else:
              #Read everything already in pipe and ignore
              self.ser.flushInput()
              #self.ser.reset_input_buffer()
              return

        except serial.SerialException:
            print('Failed to open ' + device)

        # If we reached here, none of the ports was opened
        raise serial.SerialException

    def send_msg(self, msg):
      # Do this for only one channel
      if self.ser is None:
        print("Serial device doesn't exist")
        return

      #if not self.ser.is_open:
      #  print("Serial device is not open")
      #  self.openDevice()

      if not self.ser.is_open:
        print("Serial device could not be opened")
        return

      while self.inWrite:
        time.sleep(0.001)
      self.inWrite = True
      try:
        if self.firstTime:
          # We no longer have to do this
          # Ask serial to send debug messages
          #self.ser.write('d\n'.encode())
          #self.ser.flush()
          #self.ser.write('m=2\n'.encode())
          #self.ser.flush()
          #self.ser.write('n\n'.encode())
          #self.ser.flush()
          self.firstTime = False
        self.ser.write(msg.encode())
        self.ser.flush()

      except serial.SerialException:
        print("Failed to write to Serial device")

      self.inWrite = False

    def cleanup(self):
        if self.logfile is not None:
           self.logfile.close()
        self.ser.close()

class Differential_PassThrough_Controller:
    '''
    Generic Differential Pass Through Motor Controller
    For differential drive cars you need one controller for each side.
    PassThrough means the RPi will pass it to its controller via serial interface
    '''

    #This is shared between instances
    serialInterf = None

    #def __init__(self, motor_num, device='/dev/ttyACM0', rate=115200):
    def __init__(self, motor_num, device='/dev/serial0', rate=115200):
        # Initialise the serial connection from RPi to its controller
        if Differential_PassThrough_Controller.serialInterf is None:
            Differential_PassThrough_Controller.serialInterf = SerialInterface(device, rate)

        #Motor_num 0 for left, 1 for right
        if motor_num == 0  or motor_num == 1:
          self.motor_num = motor_num
        else:
          raise Exception("invalid motor number")

        self.throttle = 0
        atexit.register(self.set_pulse, pulse=0)

    def set_pulse(self, pulse):
        '''
        Required for caliberation
        '''
        self.turn(pulse)

    def turn(self, throttle):
        '''
        Update the speed of the motor
        '''

        # Direction can be inferred by + or - so use 8bit range
        #throttle = int(map_range(abs(speed), -1, 1, -127, 127))
        if throttle == self.throttle:
          return

        self.throttle = throttle
        if self.motor_num == 0:
          msg = "L="
        else:
          msg = "R="
        msg += str(self.throttle) + '\n'
        #print('\n' + msg)
        Differential_PassThrough_Controller.serialInterf.send_msg(msg)

    def test(self, seconds=.5):
        speeds = [-.5, -1, -.5, 0, .5, 1, 0]
        for s in speeds:
            self.turn(s)
            time.sleep(seconds)
            print('speed: %s, throttle=self.throttle' % (s, self.throttle))
        print('motor #%s test complete'% self.motor_num)

class PassThrough_Controller:
    '''
    Pass control over to motor controller over serial connection.
    '''

    serialInterf = None

    #def __init__(self, channel, device='/dev/ttyACM0', rate=115200):
    def __init__(self, channel, device='/dev/serial0', rate=115200):
        # Initialise the serial connection from RPi to its controller
        if PassThrough_Controller.serialInterf is None:
            PassThrough_Controller.serialInterf = SerialInterface(device, rate)

        if channel == 0  or channel == 1:
          self.channel = channel
        else:
          raise Exception("invalid channel")

        self.lastVal = 0

        #channel 0 for throttle and 1 for steering
        atexit.register(self.set_pulse, pulse=0)

    def set_pulse(self, pulse):
        #if pulse == self.lastVal:
        #  return
        if self.channel == 0:
          msg = "T="
        else:
          msg = "S="
        msg += str(pulse) + '\n'
        #print('\n' + msg)
        PassThrough_Controller.serialInterf.send_msg(msg)

class Adafruit_Motor_Hat_Controller:
    '''
    Adafruit DC Motor Controller
    For differential drive cars you need one controller for each motor.
    '''
    def __init__(self, motor_num):
        from Adafruit_MotorHAT import Adafruit_MotorHAT, Adafruit_DCMotor
        import atexit

        self.FORWARD = Adafruit_MotorHAT.FORWARD
        self.BACKWARD = Adafruit_MotorHAT.BACKWARD
        self.mh = Adafruit_MotorHAT(addr=0x60)

        self.motor = self.mh.getMotor(motor_num)
        self.motor_num = motor_num

        atexit.register(self.turn_off_motors)
        self.speed = 0
        self.throttle = 0

    def turn_off_motors(self):
        self.mh.getMotor(self.motor_num).run(Adafruit_MotorHAT.RELEASE)

    def turn(self, speed):
        '''
        Update the speed of the motor where 1 is full forward and
        -1 is full backwards.
        '''
        if speed > 1 or speed < -1:
            raise ValueError( "Speed must be between 1(forward) and -1(reverse)")

        self.speed = speed
        self.throttle = int(map_range(abs(speed), -1, 1, -255, 255))

        if speed > 0:
            self.motor.run(self.FORWARD)
        else:
            self.motor.run(self.BACKWARD)

        self.motor.setSpeed(self.throttle)

    def test(self, seconds=.5):
        speeds = [-.5, -1, -.5, 0, .5, 1, 0]
        for s in speeds:
            self.turn(s)
            time.sleep(seconds)
            print('speed: %s   throttle: %s' % (self.speed, self.throttle))
        print('motor #%s test complete'% self.motor_num)

{'tempfile': 'tempfile', 'Thread': 'threading.Thread'}


#! /usr/bin/python3
import os
import sys
import glob
import time
import shutil
import logging
import argparse
import subprocess
import pandas as pd
from pathlib import Path
from itertools import repeat
from multiprocessing import Pool
from pyamd.bbduk import QualCheck
from pyamd.alignment import Bwa
from pyamd.alignment import Bowtie
from pyamd.alignment import BBMap
from pyamd.alignment import Snap
from pyamd.samtools import Samtools
from pyamd.gatk import GenAnTK
from pyamd.gatk import Picard
from pyamd.kestrel import KestrelVar
#from pyamd.annotater import Annotate
from pyamd.kestrel import kes_runner
from pyamd.summarize import Summary
from pyamd.prepinputs import Prepper
from pyamd.parsers.vcf import Vcf

def main(arguments):
    bbduk_path = arguments[0]
    alinger_path = arguments[1]
    smt_path = arguments[2]
    bft_path = arguments[3]
    gatk_path = arguments[4]
    rone_path = arguments[5]
    rtwo_path = arguments[6]
    ref_path = arguments[7]
    adp_path = arguments[8]
    bed_path = arguments[9]
    out_dir = arguments[10]
    aligner = arguments[11]
    pic_path = arguments[12]
    sam_name = arguments[13]
    voi_path = arguments[14]
    java_path = arguments[15]
    #Setup logging
    #Get logger for main method
    main_logger = logging.getLogger('Kookaburra.{0}'.format(sam_name))
    #Check if files are present
    #sam_name = config[samples].sample
    #rone_path = config[samples].files[0]
    #rtwo_path = config[samples].files[1]
    out_path = '{0}/{1}'.format(os.path.abspath(out_dir), sam_name)
    if not os.path.exists(out_path):
        os.mkdir(out_path)
    #main_logger.info('Analyzing sample : {0}'.format(sam_name))

    if not os.path.exists(rone_path):
        raise FileNotFoundException('Forward read not found; Exiting MARs')
        sys.exit()

    if not os.path.exists(rtwo_path):
        raise FileNotFoundException('Reverse read not found; Exiting MARs')
        sys.exit()

    if not os.path.exists(ref_path):
        raise FileNotFoundException('Reference fasta file not found; Exiting MARs')
        sys.exit()

    if not os.path.exists(adp_path):
        raise FileNotFoundException('Adpater sequence not found; Exiting MARs')
        sys.exit()

    if not os.path.exists(out_path):
        os.mkdir(out_path)

    #Create completion folder
    completion_path = '{0}/completion'.format(out_path)
    if not os.path.exists(completion_path):
        os.mkdir(completion_path)

    #Call Bbduk
    main_logger.debug('Running BBDuk')
    if os.path.exists('{0}/bbduk.rt'.format(completion_path)):
        brone = os.path.splitext(os.path.basename(rone_path))[0]
        brtwo = os.path.splitext(os.path.basename(rtwo_path))[0]
        rone_path = '{0}/{1}/{2}_cleaned.fq'.format(out_path, 'CleanedFastq', brone)
        rtwo_path = '{0}/{1}/{2}_cleaned.fq'.format(out_path, 'CleanedFastq', brtwo)
        main_logger.debug('Skipping BBDuk')
        bret = 0
    else:
        bbduk = QualCheck(bbduk_path, adp_path, out_path, java_path)
        rone_path, rtwo_path, bret = bbduk.bbduk(rone_path, rtwo_path)
        if bret == 0:
            Path('{0}/bbduk.rt'.format(completion_path)).touch()
    if bret != 0:
        raise RuntimeError('BBDuk failed to complete; Exiting MARs')
    else:
        main_logger.debug('BBDuk completed')

    if aligner == 'bwa':
        #Call BWA
        main_logger.debug('Running BWA')
        if os.path.exists('{0}/align.rt'.format(completion_path)):
            sam_path = '{0}/alignments/output.sam'.format(out_path)
            mret = 0
            main_logger.debug('Skipping BWA')
        else:
            bwa = Bwa(alinger_path, out_path, ref_path)
            sam_path, mret = bwa.bwamem(rone_path, rtwo_path)
            if mret == 0:
                Path('{0}/align.rt'.format(completion_path)).touch()
        if mret != 0:
            raise RuntimeError('Bwa mem failed to complete; Exiting MARs')
        else:
            main_logger.debug('BWA completed')

    elif aligner == 'bowtie2':
        #Call Bowtie2
        main_logger.debug('Running Bowtie2')
        if os.path.exists('{0}/aling.rt'.format(completion_path)):
            sam_path = '{0}/alignments/output.sam'.format(out_path)
            mret = 0
            main_logger.debug('Skipping Bowtie2')
        else:
            bowtie = Bowtie(alinger_path, out_path, ref_path)
            sam_path, mret = bowtie.bowtie(rone_path, rtwo_path)
            if mret == 0:
                Path('{0}/align.rt'.format(completion_path)).touch()
        if mret != 0:
            raise RuntimeError('Bowtie2 failed to complete; Exiting MARs')
        else:
            main_logger.debug('Bowtie2 completed')

    elif aligner == 'snap':
        #Call Snap
        main_logger.debug('Running Snap')
        snap = Snap(alinger_path, out_path, ref_path)
        sam_path, mret = snap.snap(rone_path, rtwo_path)
        if mret != 0:
            raise RuntimeError('Snap failed to complete; Exiting MARs')
        else:
            main_logger.debug('Snap completed')

    elif aligner == 'bbmap':
        #Call Bbmap
        main_logger.debug('Running BBMap')
        if os.path.exists('{0}/aling.rt'.format(completion_path)):
            sam_path = '{0}/alignments/output.sam'.format(out_path)
            mret = 0
        else:
            bbmap = BBMap(alinger_path, out_path, ref_path)
            sam_path, mret = bbmap.bbmap(rone_path, rtwo_path)
            if mret == 0:
                Path('{0}/align.rt'.format(completion_path)).touch()
        if mret != 0:
            raise RuntimeError('BBMap failed to complete; Exitinign MARs')
        else:
            main_logger.debug('BBMap completed')

    #Fix mate information, sort files and add read groups
    varengine = Samtools(smt_path, bft_path, out_path)
    if os.path.exists('{0}/fixmate.rt'.format(completion_path)):
        base = os.path.splitext(os.path.basename(sam_path))[0]
        bam_path = '{0}/{1}_fixmate.bam'.format(out_path, base)
        fret = 0
        main_logger.debug('Skipping fixmate')
    else:
        bam_path, fret = varengine.fixmate(sam_path)
        if fret == 0:
            Path('{0}/fixmate.rt'.format(completion_path)).touch()
    main_logger.debug('Running Samtools fixmate')
    if fret != 0:
        raise RuntimeError('Samtools fixmate failed to complete; Exiting MARs')
    else:
        main_logger.debug('Samtools fixmate completed')

    if os.path.exists('{0}/sort.rt'.format(completion_path)):
        base = os.path.splitext(os.path.basename(bam_path))[0]
        bam_path = '{0}/{1}_sorted.bam'.format(out_path, base)
        sret = 0
        main_logger.debug('Skipping sort')
    else:
        bam_path, sret = varengine.sort(bam_path)
        if sret == 0:
            Path('{0}/sort.rt'.format(completion_path)).touch()
    main_logger.debug('Running Samtools sort')
    if sret != 0:
        raise RuntimeError('Samtools sort failed to complete; Exiting MARs')
    else:
        main_logger.debug('Samtools sort completed')

    rgadder = Picard(java_path, pic_path, out_path)
    if os.path.exists('{0}/readgroup.rt'.format(completion_path)):
        base = os.path.splitext(os.path.basename(bam_path))[0]
        bam_path = '{0}/{1}_RG.bam'.format(out_path, base)
        aret = 0
        main_logger.debug('Skipping add read group')
    else:
        bam_path, aret = rgadder.picard(bam_path, sam_name)
        main_logger.debug('Running Picard AddOrReplaceReadGroups')
        if aret == 0:
            Path('{0}/readgroup.rt'.format(completion_path)).touch()
    if aret != 0:
        raise RuntimeError('Picard AddOrReplaceReadGroups failed to complete; Exiting MARs')
    else:
        main_logger.debug('Picard AddOrReplaceReadGroups completed')

    #Run samtools mpileup, bcftools index, call and stats to generate VCF files
    if os.path.exists('{0}/pileup.rt'.format(completion_path)):
        bcf_path = '{0}/variants.bcf'.format(out_path)
        pret = 0
        main_logger.debug('Skipping Pileup')
    else:
        bcf_path, pret = varengine.pileup(ref_path, bam_path)
        main_logger.debug('Running Samtools mpileup')
        if pret == 0:
            Path('{0}/pileup.rt'.format(completion_path)).touch()
    if pret != 0:
        raise RuntimeError('Samtools mpileup failed to complete; Exiting MARs')
    else:
        main_logger.debug('Samtools mpileup completed')

    if os.path.exists('{0}/bcfindex.rt'.format(completion_path)):
        bret = 0
        main_logger.debug('Skipping Bcfindex')
    else:
        bret = varengine.bcfindex(bcf_path)
        main_logger.debug('Running Bcftools index')
        if bret ==0 :
            Path('{0}/bcfindex.rt'.format(completion_path)).touch()
    if bret != 0:
        raise RuntimeError('Bcftools index failed to complete; Exiting MARs')
    else:
        main_logger.debug('Bcftools index completed')

    if os.path.exists('{0}/bcfcall.rt'.format(completion_path)):
        vcf_path = '{0}/{1}_variants.vcf'.format(out_path, sam_name)
        bret = 0
        main_logger.debug('Skipping bcfcall')
    else:
        vcf_path, bret = varengine.bcftools(bcf_path, bed_path, sam_name)
        main_logger.debug('Running Bcftools call')
        if bret == 0:
            Path('{0}/bcfcall.rt'.format(completion_path)).touch()

    if bret != 0:
        raise RuntimeError('Bcftools call failed to complete; Exiting MARs')
    else:
        main_logger.debug('Bcftools call completed')

    #if os.path.exists('{0}/stats.rt'.format())
    #stats_path, bret = varengine.bcfstats(vcf_path, ref_path)
    #main_logger.debug('Running Bcftools stats')
    #if bret != 0:
    #    raise RuntimeError('Bcftools stats failed to complete; Exiting MARs')
    #else:
    #    main_logger.debug('Bcftools stats completed')

    #Call GATK HaplotypeCaller to generate VCF files
    varcaller = GenAnTK(gatk_path, out_path, java_path)
    main_logger.debug('Running GATK HaplotypeCaller')
    if os.path.exists('{0}/gatk.rt'.format(completion_path)):
        gvcf_path = '{0}/{1}_variants_gatk.vcf'.format(out_path, sam_name)
        gret = 0
        main_logger.debug('Skipping GATK')
    else:
        gvcf_path, gret = varcaller.hapCaller(bam_path, ref_path, sam_name)
        if gret == 0:
            Path('{0}/gatk.rt'.format(completion_path)).touch()
    if gret != 0:
        raise RuntimeError('GATK HaplotypeCaller failed to complete; Exiting MARs')
    else:
        main_logger.debug('GATK HaplotypeCaller stats completed')

    #Call Kestrel to generate VCF files
    #kestrel_path = 'lib/kestrel/kestrel.jar'
    #kanalyze_path = 'lib/kestrel/kanalyze.jar'
    #varcaller = KestrelVar(rone_path, rtwo_path, ref_path, kanalyze_path,
    #                        kestrel_path, out_path)
    #varcaller = GenAnTK(gatk_path, out_path, java_path)
    #main_logger.debug('Running Kestrel')
    #if os.path.exists('{0}/kestrel.rt'.format(completion_path)):
    #    kvcf_path = '{0}/vairants_kes.vcf'.format(out_path)
    #    kret = 0
    #    main_logger.debug('Skipping Kestrel')
    #else:
    #    kvcf_path, kret = varcaller.run_kestrel()
    #    if kret == 0:
    #        Path('{0}/kestrel.rt'.format(completion_path)).touch()
    #if kret != 0:
    #    raise RuntimeError('Kestrel failed to complete; Exiting MARs')
    #else:
    #    main_logger.debug('Kestrel stats completed')

    #Filer  and annotate variant calls
    main_logger.debug('Annotating variants')
    annotate = Vcf.Annotate()
    gvcf_path = annotate.getAnnotation(bed_path, gvcf_path, ref_path, out_path)
    vcf_path = annotate.getAnnotation(bed_path, vcf_path, ref_path, out_path)
    main_logger.debug('Filetering low quality variants and merging GATK and Samtools calls')
    gvcf_file = Vcf.Reader(gvcf_path)
    svcf_file = Vcf.Reader(vcf_path)
    merge_vcf = Vcf.Merge(gvcf_file, svcf_file)
    merged_vcf = merge_vcf.merge(out_path)
#    merged_vcf = annotate.iterVcf(bed_path, merged_vcf, sam_name, ref_path, 'merged'7)
#    gatk_vcf = annotate.iterVcf(bed_path, gvcf_path, sam_name, ref_path, 'gatk')
#    samtools_vcf = annotate.iterVcf(bed_path, vcf_path , sam_name, ref_path, 'samtools')
    summary = Summary(ref_path, bed_path, voi_path, out_dir)
    var_sum = summary.getVarStats(merged_vcf)
    main_logger.info('Total variants : {0}; Verified calls : {1}; Exonic : {2}; Intronic : {3}; Synonymous : {4}; Non Synonymous : {5}; Transition : {6}; Transversion : {7}'.format(
                        var_sum[0], var_sum[1], var_sum[2], var_sum[3], var_sum[4], var_sum[5], var_sum[6], var_sum[7]))
    return(merged_vcf, 0)

def marsBatch(bbduk_path, aligner_path, smt_path, bft_path, gatk_path,
              inp_path, ref_path, adp_path, bed_path, out_dir, aligner,
              pic_path, voi_path, java_path, sra_path):
    #Creating logger for pyamd
    logger = logging.getLogger('Kookaburra')
    logger.setLevel(logging.DEBUG)
    #Create output paths for the run
    if not os.path.exists(os.path.abspath(out_dir)):
        os.mkdir(os.path.abspath(out_dir))
    # Creating a file handler which logs even debug messages
    fh = logging.FileHandler('{0}/kookaburra.log'.format(os.path.abspath(out_dir)))
    fh.setLevel(logging.DEBUG)
    # Creating a console handler to log info messages
    ch = logging.StreamHandler()
    ch.setLevel(logging.INFO)
    # Create formatter and add it to the handlers
    formatter = logging.Formatter('{asctime} - {name} - {levelname} - {message}', style="{")
    fh.setFormatter(formatter)
    ch.setFormatter(formatter)
    # Add the handlers to the logger
    logger.addHandler(fh)
    logger.addHandler(ch)
    #Create file and console handlers for MaRS
    logger.info('Gathering input information from input path.')
    prep = Prepper(inp_path, sra_path)
    config = prep.prepInputs()
    logger.info('Running MaRS on {0} experiments'.format(len(config)))
    #summary = Summary(ref_path, bed_path, voi_path, out_dir)
    samples = config.keys()
    pools = Pool(4)
    rone_list = list()
    rtwo_list = list()
    name_list = list()
    for samples in config:
        name_list.append(config[samples].sample)
        rone_list.append(config[samples].files[0])
        rtwo_list.append(config[samples].files[1])

    vcf_list = pools.map(main, zip(repeat(bbduk_path), repeat(aligner_path),
                repeat(smt_path), repeat(bft_path), repeat(gatk_path),
                rone_list, rtwo_list, repeat(ref_path), repeat(adp_path),
                repeat(bed_path), repeat(out_dir), repeat(aligner),
                repeat(pic_path), name_list, repeat(voi_path),
                repeat(java_path)))

    logger.info('Summarizing variant calls from all {0} experiments'.format(len(config)))
    summary = Summary(ref_path, bed_path, voi_path, out_dir)
    #Sumarize variants of intrest
    exp_voi = summary.getRepSnps()
    exp_voi = summary.getDepthStats(exp_voi)
    exp_voi = exp_voi.reset_index(level=1)
    #exp_voi.drop_duplicates(subset='Variant', inplace=True)
    exp_voi[['Gene_name', 'RefAA_sym', 'AAPos_sort', 'AltAA_sym']] = exp_voi['Variant'].str.extract('(?P<Gene_name>[a-zA-Z0-9]+):(?P<RefAA_sym>[a-zA-Z]?)(?P<AAPos_sort>[0-9]+)(?P<AltAA_sym>[a-zA-Z]?)', expand=True)
    exp_voi['Sample_name'] = exp_voi.index
    exp_voi['AAPos_sort'] = pd.to_numeric(exp_voi['AAPos_sort'])
    exp_voi.sort_values(['Sample_name', 'Gene_name', 'AAPos_sort'], inplace=True)
    exp_voi.drop(labels=['Sample_name', 'Gene_name', 'RefAA_sym', 'AAPos_sort',
                  'AltAA_sym','Sample'], axis=1, inplace=True)
    exp_voi.to_csv('{0}/Study_variants.csv'.format(out_dir))

    exp_af = exp_voi.pivot(exp_voi.index, 'Variant')['AF'].transpose()
    exp_af['Variant'] = exp_af.index
    exp_af[['Gene_name', 'RefAA_sym', 'AAPos_sort', 'AltAA_sym']] = exp_af['Variant'].str.extract('(?P<Gene_name>[a-zA-Z0-9]+):(?P<RefAA_sym>[a-zA-Z]?)(?P<AAPos_sort>[0-9]+)(?P<AltAA_sym>[a-zA-Z]?)', expand=True)
    exp_af['AAPos_sort'] = pd.to_numeric(exp_af['AAPos_sort'])
    exp_af.sort_values(['Gene_name', 'AAPos_sort'], inplace=True)
    exp_af.drop(labels=['Variant', 'Gene_name', 'RefAA_sym', 'AAPos_sort',
                  'AltAA_sym'], axis=1, inplace=True)
    af_mask = exp_af.isnull()
    exp_af.to_csv('{0}/Study_reportable_variants_allele_frequency.csv'.format(out_dir))
#    summary.plotHeatMap(exp_af, 'voi_af', af_mask)
    exp_dp = exp_voi.pivot(exp_voi.index, 'Variant')['DP'].transpose()
    exp_dp['Variant'] = exp_dp.index
    exp_dp[['Gene_name', 'RefAA_sym', 'AAPos_sort', 'AltAA_sym']] = exp_dp['Variant'].str.extract('(?P<Gene_name>[a-zA-Z0-9]+):(?P<RefAA_sym>[a-zA-Z]?)(?P<AAPos_sort>[0-9]+)(?P<AltAA_sym>[a-zA-Z]?)', expand=True)
    exp_dp['AAPos_sort'] = pd.to_numeric(exp_dp['AAPos_sort'])
    exp_dp.sort_values(['Gene_name', 'AAPos_sort'], inplace=True)
    exp_dp.drop(labels=['Variant', 'Gene_name', 'RefAA_sym', 'AAPos_sort',
                  'AltAA_sym'], axis=1, inplace=True)
    dp_mask = exp_dp.isnull()
    exp_dp.to_csv('{0}/Study_reportable_variants_depth.csv'.format(out_dir))
#    summary.plotHeatMap(exp_dp, 'voi_dp', dp_mask)
#    summary.plotCountPlot(exp_af, 'voi')
    #Summarize novel variants
    exp_nov = summary.getNovSnps()
    exp_nov = summary.getNovDepthStats(exp_nov)
    exp_nov = exp_nov.reset_index(level=1)
    exp_nov[['Gene_name', 'RefAA_sym', 'AAPos_sort', 'AltAA_sym']] = exp_nov['Variant'].str.extract('(?P<Gene_name>[a-zA-Z0-9]+):(?P<RefAA_sym>[a-zA-Z]?)(?P<AAPos_sort>[0-9]+)(?P<AltAA_sym>[a-zA-Z]?)', expand=True)
    exp_nov['Sample_name'] = exp_nov.index
    exp_nov['AAPos_sort'] = pd.to_numeric(exp_nov['AAPos_sort'])
    exp_nov.sort_values(['Sample_name', 'Gene_name', 'AAPos_sort'], inplace=True)
    exp_nov.drop(labels=['Sample_name', 'Gene_name', 'RefAA_sym', 'AAPos_sort',
                  'AltAA_sym', 'Sample'], axis=1, inplace=True)
    exp_nov.to_csv('{0}/Study_novel_exonic_variants.csv'.format(out_dir))
    #Separate and capture Intron and exonic variants
    exp_nov_af = exp_nov.loc[:,['Variant', 'AF']]
    exp_nov_af[['Gene_name', 'RefAA_sym', 'AAPos_sort', 'AltAA_sym']] = exp_nov_af['Variant'].str.extract('(?P<Gene_name>[a-zA-Z0-9]+):(?P<RefAA_sym>[a-zA-Z]?)(?P<AAPos_sort>[0-9]+)(?P<AltAA_sym>[a-zA-Z]?)', expand=True)
    exp_nov_af['AAPos_sort'] = pd.to_numeric(exp_nov_af['AAPos_sort'])
    exp_nov_af.sort_values(['Gene_name', 'AAPos_sort'], inplace=True)
    exp_nov_af.drop(labels=['Variant', 'Gene_name', 'RefAA_sym', 'AAPos_sort',
                  'AltAA_sym'], axis=1, inplace=True)
    exp_nov_af.to_csv('{0}/Study_novel_variants_alele_frequency.csv'.format(out_dir))
    exp_nov_dp = exp_nov.loc[:,['Variant', 'DP']]
    exp_nov_dp['Variant'] = exp_nov_dp.index
    exp_nov_dp[['Gene_name', 'RefAA_sym', 'AAPos_sort', 'AltAA_sym']] = exp_nov_dp['Variant'].str.extract('(?P<Gene_name>[a-zA-Z0-9]+):(?P<RefAA_sym>[a-zA-Z]?)(?P<AAPos_sort>[0-9]+)(?P<AltAA_sym>[a-zA-Z]?)', expand=True)
    exp_nov_dp['AAPos_sort'] = pd.to_numeric(exp_nov_dp['AAPos_sort'])
    exp_nov_dp.sort_values(['Gene_name', 'AAPos_sort'], inplace=True)
    exp_nov_dp.drop(labels=['Variant', 'Gene_name', 'RefAA_sym', 'AAPos_sort',
                  'AltAA_sym'], axis=1, inplace=True)
    exp_nov_dp.to_csv('{0}/Study_novel_variants_depth.csv'.format(out_dir))
    exp_intron = summary.getIntronTables()
    exp_intron = exp_intron.reset_index()

    #print(exp_intron.index)
    #exp_intron.reset_index(level=1)
#    print(exp_intron.head())
    exp_intron[['Gene_name', 'RefAA_sym', 'AAPos_sort', 'AltAA_sym']] = exp_intron['Variant'].str.extract('(?P<Gene_name>[a-zA-Z0-9]+):(?P<RefAA_sym>[a-zA-Z]?)(?P<AAPos_sort>[0-9]+)(?P<AltAA_sym>[a-zA-Z]?)', expand=True)
    #exp_intron['']
    exp_intron['AAPos_sort'] = pd.to_numeric(exp_intron['AAPos_sort'])
    exp_intron.sort_values(['Sample', 'Gene_name', 'AAPos_sort'], inplace=True)
    exp_intron.drop(labels=['Gene_name', 'RefAA_sym', 'AAPos_sort',
                  'AltAA_sym' ], axis=1, inplace=True)
    exp_intron.sort_index().reset_index(drop=True).to_csv('{0}/Study_novel_intronic_variants.csv'.format(out_dir), index=False)
    # Plot using Rscript
    logger.info('Plotting Depth Per SNP')
    dcmd = ['Rscript', 'pyamd/Rscripts/DepthPerReportSNP.R', '-i',
            '{0}/Study_reportable_variants_depth.csv'.format(out_dir), '-o',
            '{0}/Study_depth.pdf'.format(out_dir)]
    drun = subprocess.Popen(dcmd, shell=False,
                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    drun.wait()
    if drun.returncode != 0:
        logger.error('Failed to execute DepthPerReportSNP.R')
        logger.error(' '.join(dcmd))

    logger.info('Plotting Reportable SNPs Frequency')
    acmd = ['Rscript', 'pyamd/Rscripts/reportableSNPsFreq.R', '-i',
            'Study_variants.csv'.format(out_dir), '-r',
            'ref/Reportable_SNPs.csv', '-o', '{0}/'.format(out_dir)]
    arun = subprocess.Popen(acmd, shell=False,
                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    arun.wait()
    if arun.returncode != 0:
        logger.error('Failed to execute reportableSNPsFreq.R')
        logger.error(' '.join(acmd))
    logger.info('Plotting Novel Exonic Non-Synonymous SNPs')
    nenscmd = ['Rscript', 'pyamd/Rscripts/NovelExonicNonSynSNPs.R', '-i',
            'Study_novel_exonic_variants.csv'.format(out_dir),
            '-o', '{0}/'.format(out_dir)]
    nensrun = subprocess.Popen(nenscmd, shell=False,
                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    nensrun.wait()
    if nensrun.returncode != 0:
        logger.error('Failed to execute NovelExonicNonSynSNPs.R')
        logger.error(' '.join(nenscmd))

    logger.info('Plotting Novel Exonic Synonymous SNPs')
    nescmd = ['Rscript', 'pyamd/Rscripts/NovelExonicSynSNPs.R', '-i',
            'Study_novel_exonic_variants.csv'.format(out_dir),
            '-o', '{0}/'.format(out_dir)]
    nesrun = subprocess.Popen(nescmd, shell=False,
                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    nesrun.wait()
    if nesrun.returncode != 0:
        logger.error('Failed to execute NovelExonicSynSNPs.R')
        logger.error(' '.join(acmd))

    logger.info('Plotting Novel Intronic SNPs')
    nicmd = ['Rscript', 'pyamd/Rscripts/NovelIntronicSNPs.R', '-i',
            'Study_novel_intronic_variants.csv'.format(out_dir),
            '-o', '{0}/'.format(out_dir)]
    nirun = subprocess.Popen(nicmd, shell=False,
                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    nirun.wait()
    if nirun.returncode != 0:
        logger.error('Failed to execute NovelIntronicSNPs.R')
        logger.error(' '.join(nicmd))

    #os.remove('{0}/Reportable_SNPs_Report.csv'.format(out_dir))
    os.remove('{0}/novel_SNPs_exonic_syn.csv'.format(out_dir))
    os.remove('{0}/novel_SNPs_intronic.csv'.format(out_dir))
    os.remove('{0}/novel_SNPs_exonic_nonsyn.csv'.format(out_dir))
    os.remove('{0}/Study_novel_exonic_variants_filtered.csv'.format(out_dir))
    os.remove('{0}/Study_novel_intronic_variants_filtered.csv'.format(out_dir))
    return(0)

if __name__ == '__main__':
    #Define deffault paths and aligner informations
    def_path = "{0}/lib".format(os.path.abspath(os.path.dirname(os.path.realpath(__file__))))
    ref_def_path = "{0}/ref".format(os.path.abspath(os.path.dirname(os.path.realpath(__file__))))
    bbduk_def = 'bbduk.sh' #"{0}/bbmap/bbduk.sh".format(def_path)
    bbmap_def = 'bbmap.sh' #"{0}/bbmap/bbmap.sh".format(def_path)
    bwa_def = 'bwa' #"{0}/bwa/bwa".format(def_path)
    bowtie_def = 'bowtie2' #"{0}/bowtie2/bowtie2".format(def_path)
    snap_def = 'snap-alinger' #"{0}/snap/snap-aligner".format(def_path)
    smt_def = 'samtools' #"{0}/samtools/samtools".format(def_path)
    bft_def = 'bcftools' #"{0}/bcftools/bcftools".format(def_path)
    gatk_def = 'gatk' #"{0}/GenomeAnalysisTK.jar".format(def_path)
    pic_def = 'picard' #"{0}/picard.jar".format(def_path)
    sra_def = 'fastq-dump' #'{0}/sratoolkit/bin/fastq-dump'.format(def_path)
    voi_def = '{0}/Reportable_SNPs.csv'.format(ref_def_path)
    #if 'java version "1.8.' in str(subprocess.check_output(["java", "-version"], stderr=subprocess.STDOUT).decode('UTF-8').split('\n')[0]):
    java_def = 'java'
    #else:
    #    java_def = "{0}/jdk/bin/java".format(def_path)
    aligner_def = {'bwa' : bwa_def, 'snap' : snap_def, 'bowtie2': bowtie_def, 'bbmap': bbmap_def}
    #Get arguments
    parser = argparse.ArgumentParser(prog='kookaburra')
    parser.add_argument('-i', '--inp_path', type=str,
                        help='Path to input directory (Specify only for batch mode)')
    parser.add_argument('-1', '--fwd', dest='rone_path', type=str,
                        help='Path to forward reads fastq', )
    parser.add_argument('-2', '--rev', dest='rtwo_path', type=str,
                        help='Path to reverse reads fastq')
    parser.add_argument('-r', '--ref', dest='ref_path', type=str,
                        help='Path to Reference fasta file', required=True)
    parser.add_argument('-a', '--adapter', dest='adp_path', type=str,
                        help='Path to Adpater fasta file', required=True)
    parser.add_argument('-b', '--bed', dest='bed_path', type=str,
                        help='Path to Bed file for MDR regions', required=True)
    parser.add_argument('-o', '--outpath', dest='out_path', type=str,
                        help='Path where all outputs will be stored', required=True)
    parser.add_argument('-n', '--sam_name', dest='sam_name', type=str,
                        help='Sample name', default=None)
    parser.add_argument('-m', '--mapper', dest='aligner', type=str,
                        choices=['bowtie2', 'bwa', 'bbmap', 'snap'],
                        default='bwa', help='The aligner to used by MARs')
    parser.add_argument('--bbduk', dest='bbduk_path', type=str, default=bbduk_def,
                        help='Path to BBduk executable')
    parser.add_argument('--aligner', dest='aligner_path', type=str, default=None,
                        help='Path to aligner executable')
    parser.add_argument('--samtools', dest='smt_path', type=str, default=smt_def,
                        help='Path to Samtools executable')
    parser.add_argument('--gatk', dest='gatk_path', type=str, default=gatk_def,
                        help='Path to GATK executable')
    parser.add_argument('--bcftools', dest='bft_path', type=str, default=bft_def,
                        help='Path to Bcftools executable')
    parser.add_argument('--picard', dest='pic_path', type=str, default=pic_def,
                        help='Path to Bcftools executable')
    parser.add_argument('--varofint', dest='voi_path', type=str, default=voi_def,
                        help='Path to variant of interest')
    args = parser.parse_args()

    #Validate parsed arguments
    if args.aligner_path is None:
        args.aligner_path = aligner_def[args.aligner]

    if not os.path.exists(args.out_path):
        os.mkdir(args.out_path)

    #Check if the run command is for batch mode analysis or single sample
    #analysis.
    #If inp_path is empty and rone_path is not, then the experiment is a
    #single sample experiment.
    status = marsBatch(args.bbduk_path, args.aligner_path, args.smt_path,
                args.bft_path, args.gatk_path, args.inp_path, args.ref_path,
                args.adp_path, args.bed_path, args.out_path, args.aligner,
                args.pic_path, args.voi_path, java_def, sra_def)


import argparse
import copy
import os
import os.path as osp
import pprint
import sys
import time
from pathlib import Path

import numpy as np
import yaml

class SemSegPredictor:
    def __init__(self):

        self.home_path = str(Path.home())
        self.base_path = self.home_path + "/dev/Open3D-ML"
        self.dateset_path = self.home_path + "/datasets/SmartLab"
        self.ckpt_path = self.base_path + "/mytests/logs/RandLANet_SmartLab_tf/checkpoint/ckpt-6"
        self.randlanet_smartlab_cfg = self.base_path + "/ml3d/configs/randlanet_smartlab.yml"
        self.kpconv_smartlab_cfg = self.base_path + "/ml3d/configs/kpconv_smartlab.yml"

        os.environ["OPEN3D_ML_ROOT"] = base_path
        import open3d.ml as _ml3d

        self._ml3d = _ml3d

        kwargs = {
            "framework": "tf",
            "device": "cuda",
            "dataset_path": datesetbase,
            "split": "test",
            "ckpt_path": ckpt_path,
            "cfg_file": randlanet_smartlab_cfg,
        }

        self.args = type("args", (object,), kwargs)()

        pprint.pprint(kwargs)

        if args.framework == "torch":
            import open3d.ml.torch as ml3d

            self.ml3d = ml3d
        else:
            import open3d.ml.tf as ml3d
            import tensorflow as tf

            self.ml3d = ml3d
            self.tf = tf

    def config_gpu(self):
        gpus = tf.config.experimental.list_physical_devices("GPU")
        print(gpus)

        if gpus:
            try:
                for gpu in gpus:
                    self.tf.config.experimental.set_memory_growth(gpu, True)
                if self.args.device == "cpu":
                    self.tf.config.set_visible_devices([], "GPU")
                elif self.args.device == "cuda":
                    self.tf.config.set_visible_devices(gpus[0], "GPU")
                else:
                    idx = self.args.device.split(":")[1]
                    self.tf.config.set_visible_devices(gpus[int(idx)], "GPU")
            except RuntimeError as e:
                print(e)

    @staticmethod
    def merge_cfg_file(cfg, args):
        if args.device is not None:
            cfg.pipeline.device = args.device
            cfg.model.device = args.device
        if args.split is not None:
            cfg.pipeline.split = args.split
        if args.dataset_path is not None:
            cfg.dataset.dataset_path = args.dataset_path
        if args.ckpt_path is not None:
            cfg.model.ckpt_path = args.ckpt_path

        return cfg.dataset, cfg.pipeline, cfg.model

    def setup_lut_colors(self):
        Unlabeled = [231, 87, 36]
        Floor = [188, 169, 26]
        Wall = [100, 244, 245]
        Robot = [150, 30, 140]
        Human = [0, 248, 26]
        AGV = [18, 35, 243]

        colors = {
            "Unlabeled": [x / 255.0 for x in Unlabeled],
            "Floor": [x / 255.0 for x in Floor],
            "Wall": [x / 255.0 for x in Wall],
            "Robot": [x / 255.0 for x in Robot],
            "Human": [x / 255.0 for x in Human],
            "AGV": [x / 255.0 for x in AGV],
        }

        self.colors = colors

    def config_pipeline(self):

        cfg = self._ml3d.utils.Config.load_from_file(self.args.cfg_file)

        Pipeline = self._ml3d.utils.get_module("pipeline", cfg.pipeline.name, self.args.framework)
        Model = self._ml3d.utils.get_module("model", cfg.model.name, self.args.framework)
        Dataset = self._ml3d.utils.get_module("dataset", cfg.dataset.name)

        cfg_dataset, cfg_pipeline, cfg_model = merge_cfg_file(cfg, self.args)

        self.dataset = Dataset(**cfg_dataset)
        self.model = Model(**cfg_model)
        self.pipeline = Pipeline(model, dataset, **cfg_pipeline)

        self.pipeline.cfg_tb = {
            "readme": "readme",
            "cmd_line": "cmd_line",
            "dataset": pprint.pformat(cfg_dataset, indent=2),
            "model": pprint.pformat(cfg_model, indent=2),
            "pipeline": pprint.pformat(cfg_pipeline, indent=2),
        }

    def run_pipeline_inference(dataframe):
        print(dataframe.name)
        results = self.pipeline.run_inference(dataframe.data)
        pred = (results["predict_labels"]).astype(np.int32)
        return pred

    def run_pipeline_traintest():
        if self.args.split == "test":
            self.pipeline.run_test()
        else:
            self.pipeline.run_train()

    def run_pipeline_test():
        self.pipeline.load_ckpt(ckpt_path=self.args.ckpt_path)
        test_split = self.dataset.get_split("test")

        vis_points = []
        times = []
        acc = []

        for idx in range(len(test_split)):
            # for idx in range(5):

            st = time.perf_counter()
            attr = test_split.get_attr(idx)
            data = test_split.get_data(idx)

            print(attr)
            results = self.pipeline.run_inference(data)

            pred = (results["predict_labels"]).astype(np.int32)
            # scores = results['predict_scores']

            label = data["label"]
            pts = data["point"]

            vis_d = {
                "name": attr["name"],
                "points": pts,
                "labels": label,
                "pred": pred,
            }

            vis_points.append(vis_d)
            et = time.perf_counter()
            times.append(et - st)

            correct = (pred == label).sum()
            print(f"Correct: " + str(correct) + " out of " + str(len(label)))
            accuracy = correct / len(label)
            acc.append(accuracy)
            print(f"Accuracy: " + str(accuracy))

        print("\n")
        print(times)
        print(f"Average time {np.mean(times):0.4f} seconds")
        overall_acc = np.asarray(acc).sum() / len(acc)
        print("Overall Accuracy: " + str(overall_acc))

        v = self.ml3d.vis.Visualizer()
        lut = self.ml3d.vis.LabelLUT()

        label_names = self.dataset.get_label_to_names()
        pprint.pprint(label_names)

        for (c, cv), (l, lv) in zip(colors.items(), label_names.items()):
            lut.add_label(lv, l, cv)

        v.set_lut("labels", lut)
        v.set_lut("pred", lut)

        v.visualize(vis_points, width=2600, height=2000)

def main(args):
    ps = pointcloud_segmenter()

if __name__ == "__main__":
    main(sys.argv)


import copy

import bumps.parameter
import numpy as np
import pytest
import refnx
import refl1d.experiment

from hogben.simulate import simulate
from hogben.utils import fisher
from refnx.reflect import SLD as SLD_refnx
from refl1d.material import SLD as SLD_refl1d
from unittest.mock import Mock, patch

QS = [np.array([0.1, 0.2, 0.4, 0.6, 0.8])]
COUNTS = [np.ones(len(QS[0])) * 100]

@pytest.fixture
def refl1d_model():
    """Define a bilayer sample, and return the associated refl1d model"""
    # Define sample
    air = SLD_refl1d(rho=0, name='Air')
    layer1 = SLD_refl1d(rho=4, name='Layer 1')(thickness=60, interface=8)
    layer2 = SLD_refl1d(rho=8, name='Layer 2')(thickness=150, interface=2)
    substrate = SLD_refl1d(rho=2.047, name='Substrate')(thickness=0,
                                                        interface=2)
    layer1.thickness.pm(10)
    layer2.thickness.pm(10)
    layer1.interface.pm(1)
    layer2.interface.pm(1)
    sample = substrate | layer2 | layer1 | air

    # Define model
    angle_times = [(0.7, 100, 5), (2.0, 100, 20)]  # (Angle, Points, Time)
    model, _ = simulate(sample, angle_times)
    model.xi = [layer1.interface, layer2.interface, layer1.thickness,
                layer2.thickness]
    return model

@pytest.fixture
def refnx_model():
    """Define a bilayer sample, and return the associated refnx model"""
    # Define sample
    air = SLD_refnx(0, name='Air')
    layer1 = SLD_refnx(4, name='Layer 1')(thick=60, rough=8)
    layer2 = SLD_refnx(8, name='Layer 2')(thick=150, rough=2)
    substrate = SLD_refnx(2.047, name='Substrate')(thick=0, rough=2)
    layer1.thick.bounds = (50, 70)
    layer2.thick.bounds = (140, 160)
    layer1.rough.bounds = (7, 9)
    layer2.rough.bounds = (1, 3)
    sample = air | layer1 | layer2 | substrate
    model = refnx.reflect.ReflectModel(sample, scale=1, bkg=5e-6, dq=2)
    # Define model
    model.xi = [layer1.rough, layer2.rough, layer1.thick, layer2.thick]
    return model

@pytest.fixture
def mock_refnx_model():
    """
    Create a mock of the refl1d model with a given set of parameters and their
    bounds
    """
    # Parameters described as tuples: (value, lower bound, upper bound)
    parameter_values = [(20, 15, 25), (50, 45, 55), (10, 7.5, 8.5),
                        (2, 1.5, 2.5)]

    # Fill parameter values and bounds
    parameters = [
        Mock(spec=refnx.analysis.Parameter, value=value,
             bounds=Mock(lb=lb, ub=ub))
        for value, lb, ub in parameter_values
    ]
    model = Mock(spec=refnx.reflect.ReflectModel, xi=parameters)
    model.xi = parameters
    return model

@pytest.fixture
def mock_refl1d_model():
    """
    Create a mock of the refl1d model with a given set of parameters and their
    bounds
    """
    # Parameters described as tuples: (value, lower bound, upper bound)
    parameter_values = [(20, 15, 25), (50, 45, 55), (10, 7.5, 8.5),
                        (2, 1.5, 2.5)]

    # Fill parameter values and bounds
    parameters = [
        Mock(spec=bumps.parameter.Parameter, value=value,
             bounds=Mock(limits=[lb, ub]))
        for value, lb, ub in parameter_values
    ]
    model = Mock(spec=refl1d.experiment.Experiment, xi=parameters)
    model.xi = parameters
    return model

def generate_reflectivity_data():
    """
    Generates predefined reflectivity.The reflectivity values are yielded
    alternatingly between two predefined lists of reflectivity values,
    simulating a change in reflectivity between two data points
    """
    r = [[1.0, 0.5, 0.4, 0.2, 0.1], [0.95, 0.45, 0.35, 0.15, 0.05]]
    while True:
        yield r[0]
        yield r[1]

def test_fisher_workflow_refnx(refnx_model):
    """
    Runs the entire fisher workflow for the refnx model, and checks that the
    corresponding results are consistent with the expected values
    """
    g = fisher(QS, refnx_model.xi, COUNTS, [refnx_model])
    expected_fisher = [
        [5.17704306e-06, 2.24179068e-06, -5.02221954e-07, -7.91886209e-07],
        [2.24179068e-06, 1.00559528e-06, -2.09433754e-07, -3.18583142e-07],
        [-5.02221954e-07, -2.09433754e-07, 5.75647233e-08, 1.03142100e-07],
        [-7.91886209e-07, -3.18583142e-07, 1.03142100e-07, 1.99470835e-07]
    ]
    np.testing.assert_allclose(g, expected_fisher, rtol=1e-08)

def test_fisher_workflow_refl1d(refl1d_model):
    """
    Runs the entire fisher workflow for the refl1d model, and checks that the
    corresponding results are consistent with the expected values
    """
    g = fisher(QS, refl1d_model.xi, COUNTS, [refl1d_model])
    expected_fisher = [
        [4.58294661e-06, 2.07712766e-06, -4.23068571e-07, -6.80596824e-07],
        [2.07712766e-06, 9.76175381e-07, -1.84017555e-07, -2.83513452e-07],
        [-4.23068571e-07, -1.84017555e-07, 4.51142562e-08, 8.21397190e-08],
        [-6.80596824e-07, -2.83513452e-07, 8.21397190e-08, 1.62625881e-07]
    ]
    np.testing.assert_allclose(g, expected_fisher, rtol=1e-08)

@patch('hogben.utils.reflectivity')
@pytest.mark.parametrize('model_class', ("mock_refl1d_model",
                                         "mock_refnx_model"))
def test_fisher_analytical_values(mock_reflectivity, model_class, request):
    """
    Tests that the values of the calculated Fisher information matrix (FIM)
    are calculated correctly when no importance scaling is given.

    The FIM is calculated using matrix multiplication given:
    g = J.T x M x J

    Where J describes the Jacobian of the reflectance with respect to the
    parameter value, and M describe the diagonal matrix of the incident count
    divided by the model reflectances. J.T describes the transpose of J.

    For this unit test the values for J and M are known:
    J = [-0.25, -0.1 , -0.5 ],
        [-0.25, -0.1 , -0.5 ],
        [-0.25, -0.1 , -0.5 ],
        [-0.25, -0.1 , -0.5 ],
        [-0.25, -0.1 , -0.5 ]

    M is given by by:
    M =  [ 100.,    0.,    0.,    0.,    0.],
         [   0.,  200.,    0.,    0.,    0.],
         [   0.,    0.,  250.,    0.,    0.],
         [   0.,    0.,    0.,  500.,    0.],
         [   0.,    0.,    0.,    0., 1000.]

    Resulting in g = J.T x M x J:
    g =  [128.125,  51.25 , 256.25 ],
       [ 51.25 ,  20.5  , 102.5  ],
       [256.25 , 102.5  , 512.5  ]

    After this, the elements are scaled to the bounds of each unit. Using:
    g_scaled = H.T * g *H
    Where H is a diagonal matrix where the diagonal elements for each
    parameter are given by H_ij = 1/(upper_bound - lower_bound), resulting in:
    H = [0.1, 0. , 0. ],
       [0. , 0.1, 0. ],
       [0. , 0. , 1. ]

    Which should finally result in the FIM matrix equal to:
    g = [1.28125, 0.5125, 25.625],
        [0.5125, 0.205, 10.25],
        [25.625, 10.25, 512.5]
    """
    model = request.getfixturevalue(model_class)
    xi = model.xi[:3]
    mock_reflectivity.side_effect = generate_reflectivity_data()
    g_correct = [
        [1.28125, 0.5125, 25.625],
        [0.5125, 0.205, 10.25],
        [25.625, 10.25, 512.5]
    ]
    g_reference = fisher(QS, xi, COUNTS, [model])
    np.testing.assert_allclose(g_reference, g_correct, rtol=1e-08)

@patch('hogben.utils.reflectivity')
@pytest.mark.parametrize('model_class', ("mock_refl1d_model",
                                         "mock_refnx_model"))
def test_fisher_importance_scaling(mock_reflectivity, model_class, request):
    """
    Tests that the values of the calculated Fisher information matrix
    are calculated correctly when an importance scaling is applied.

    The importance scaling is applied by scaling each parameter of the FIM
    to a given importance value using g_scaled = g * importance
    Where g is the unscaled FIM, and importance is a diagonal matrix with
    the importance scaling of each parameter on the diagonals. For this unit
    test the importance matrix is equal to:
    importance = [1, 0 , 0]
                 [0, 2, 0]
                [0, 0, 3]
    Yielding a FIM where every column should be scaled by the corresponding
    diagonal in the importance matrix:
    g = [1.28125, 1.025, 76.875],
        [0.5125, 0.41, 30.75],
        [25.625, 20.5, 1537.5]
    """
    model = request.getfixturevalue(model_class)
    xi = model.xi[:3]
    for index, param in enumerate(xi):
        param.importance = index + 1
    mock_reflectivity.side_effect = generate_reflectivity_data()
    g_correct = [
        [1.28125, 1.025, 76.875],
        [0.5125, 0.41, 30.75],
        [25.625, 20.5, 1537.5]
    ]
    g_reference = fisher(QS, xi, COUNTS, [model])
    np.testing.assert_allclose(g_reference, g_correct, rtol=1e-08)

@pytest.mark.parametrize('model_class', ("refl1d_model",
                                         "refnx_model"))
@pytest.mark.parametrize('step', (0.01, 0.0075, 0.0025, 0.001, 0.0001))
def test_fisher_consistent_steps(step, model_class, request):
    """
    Tests whether the Fisher information remains mostly consistent when
    changing step size using the refnx model
    """
    model = request.getfixturevalue(model_class)
    g_reference = fisher(QS, model.xi, COUNTS, [model], step=0.005)
    g_compare = fisher(QS, model.xi, COUNTS, [model], step=step)
    np.testing.assert_allclose(g_reference, g_compare, rtol=1e-02)

@patch('hogben.utils.reflectivity')
@pytest.mark.parametrize('model_class', ("mock_refl1d_model",
                                         "mock_refnx_model"))
@pytest.mark.parametrize('model_params', (1, 2, 3, 4))
def test_fisher_shape(mock_reflectivity, model_params, model_class, request):
    """
    Tests whether the shape of the Fisher information matrix remains
     correct when changing the amount of parameters
    """
    model = request.getfixturevalue(model_class)
    xi = model.xi[:model_params]

    mock_reflectivity.side_effect = generate_reflectivity_data()

    expected_shape = (model_params, model_params)
    g = fisher(QS, xi, COUNTS, [model])
    np.testing.assert_array_equal(g.shape, expected_shape)

@patch('hogben.utils.reflectivity')
@pytest.mark.parametrize('model_class', ("mock_refl1d_model",
                                         "mock_refnx_model"))
@pytest.mark.parametrize('qs',
                         (np.arange(0.001, 1.0, 0.25),
                          np.arange(0.001, 1.0, 0.10),
                          np.arange(0.001, 1.0, 0.05),
                          np.arange(0.001, 1.0, 0.01)))
def test_fisher_diagonal_non_negative(mock_reflectivity, qs, model_class,
                                   request):
    """Tests whether the diagonal values in the Fisher information matrix
     are all zero or greater"""
    model = request.getfixturevalue(model_class)
    mock_reflectivity.side_effect = (np.random.rand(len(qs)) for _ in range(9))
    counts = [np.ones(len(qs)) * 100]
    g = fisher([qs], model.xi, counts, [model])
    assert np.all(np.diag(g)) >= 0

@pytest.mark.parametrize('model_class', ("mock_refl1d_model",
                                         "mock_refnx_model"))
@pytest.mark.parametrize('model_params', (1, 2, 3, 4))
def test_fisher_no_data(model_params, model_class, request):
    """Tests whether a model with zero data points properly returns an empty
    matrix of the correct shape"""
    model = request.getfixturevalue(model_class)
    xi = model.xi[:model_params]
    g = fisher([], xi, COUNTS, [model])
    np.testing.assert_equal(g, np.zeros((len(xi), len(xi))))

@pytest.mark.parametrize('model_class', ("mock_refl1d_model",
                                         "mock_refnx_model"))
@patch('hogben.utils.reflectivity')
def test_fisher_no_parameters(mock_reflectivity, model_class, request):
    """Tests whether a model without any parameters properly returns a
    zero array"""
    model = request.getfixturevalue(model_class)
    mock_reflectivity.side_effect = generate_reflectivity_data()
    g = fisher(QS, [], COUNTS, [model])
    np.testing.assert_equal(g.shape, (0, 0))

@pytest.mark.parametrize('model_class', ("refnx_model",
                                         "refl1d_model"))
def test_fisher_doubling_with_two_identical_models(model_class, request):
    """
    Tests that using two identical models with the same q-points and counts
    correctly doubles the values on the elements in the Fisher information
    matrix
    """
    model = request.getfixturevalue(model_class)
    g_single = fisher(QS, model.xi, COUNTS, [model], 0.005)

    counts = [COUNTS[0], COUNTS[0]]
    qs = [QS[0], QS[0]]
    g_double = fisher(qs, model.xi, counts, [model, model], 0.005)
    np.testing.assert_allclose(g_double, g_single * 2, rtol=1e-08)

@pytest.mark.parametrize('model_class', ("refnx_model",
                                         "refl1d_model"))
def test_multiple_models_shape(model_class, request):
    """
    Tests that shape of the Fisher information matrix is equal to the total
    sum of parameters over all models.
    """
    model = request.getfixturevalue(model_class)
    model_2 = copy.deepcopy(model)
    model_2.xi = model_2.xi[:-1]
    xi = model.xi + model_2.xi
    xi_length = len(model.xi) + len(model_2.xi)
    counts = [COUNTS[0], COUNTS[0]]
    qs = [QS[0], QS[0]]
    g_double = fisher(qs, xi, counts, [model, model_2], 0.005)
    np.testing.assert_equal(g_double.shape, (xi_length, xi_length))


from django.core.management.base import BaseCommand
import numpy as np
import pickle
import os
import sys
import gc

sys.path.append(os.getcwd())
from web_app.models import Table
from clf_model.nb_model.naive_bayes import MultinomialNaiveBayes
from preprocess import word_segmentation, bags_of_words, split
from scrape.config import TOP_CATEGORY

MODEL_DUMP_PATH = "web_app/classification_dump"

class Command(BaseCommand):

    def add_arguments(self, parser):
        """
        seedsを指定する関数

        :param parser:
        :return:
        """
        parser.add_argument('-s'
                            '--seeds',
                            dest='seeds',
                            type=int,
                            default=None)

    def _dump_model(self, preprocess_obj, model_obj):
        """作成したモデルを保存する関数

        :param preprocess_obj:
        :param model_obj:
        :return:
        """
        with open(os.path.join(MODEL_DUMP_PATH, 'preprocess.pickle'),
                  mode='wb') as f:
            pickle.dump(preprocess_obj, f)

        with open(os.path.join(MODEL_DUMP_PATH, 'model.pickle'),
                  mode='wb') as f:
            pickle.dump(model_obj, f)

    def _create_model(self, SEEDS):
        """モデルを作成して保存する一連の関数

        :param SEEDS:
        :return:
        """
        print('わかち書き開始')
        x = np.array([word_segmentation.wakachi(data.content) for data in
                      Table.objects.all()])
        y = np.array(
            [TOP_CATEGORY[data.category] for data in Table.objects.all()])

        print('seeds = {}でデータを分割'.format(SEEDS))
        train_x, train_y, _, _ = split.split_train_test(x, y, seeds=SEEDS)

        # メモリ節約のため削除
        del x
        del y
        gc.collect()

        print('単語ベクトル学習開始')
        preprocess_obj = bags_of_words.Bow()
        processed_train_x = np.array(preprocess_obj.fit_transform(train_x),
                                     dtype=np.float32)

        # 　メモリ節約のため削除
        del train_x
        gc.collect()

        print('モデル学習開始')
        model_obj = MultinomialNaiveBayes()
        model_obj.fit(processed_train_x, train_y)

        print('モデルのdump開始')
        self._dump_model(preprocess_obj, model_obj)

        print('DONE')

    def handle(self, *args, **options):
        """カスタムコマンドが呼び出された時に動く関数

        :param args:
        :param options:
        :return:
        """
        print("START")
        SEEDS = options['seeds']
        self._create_model(SEEDS)


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Fri Jul 14 21:28:15 2017.

@author: spiros
"""

import numpy as np
import pickle
import sys
import os
import time
import scipy.ndimage.filters as flt
from functions_analysis import spike_map, binning

def analysis_path_cluster(ntrial, case, learning, inter):

    folder1 = 'data_analysis'

    os.system('mkdir -p '+folder1+'/figures_interneurons/')
    os.system('mkdir -p '+folder1+'/metrics_interneurons/')
    fdname1 = '/'+folder1+'/figures_interneurons/'
    fdname2 = '/'+folder1+'/metrics_interneurons/'

    cond = '../Simulation_Results/'+learning+'/'+case
    print("Analyse ... " + case + " trial "+ntrial + " " + learning)

    os.system('mkdir -p '+folder1+'/figures_interneurons/'+learning+'/')
    os.system('mkdir -p '+folder1+'/metrics_interneurons/'+learning+'/')
    # Define the map size!
    maindir = os.getcwd()

    # Give path dimensions
    # Give path dimensions
    npath_x = 200
    npath_y = 1
    # Number of pyramidal
    Ncells = 130
    Nbins = 100
    skernel = 3.0 / (npath_x/Nbins)
    runsAll = 5

    # Number of basket cells
    if inter == 'bcell':
        Ncells = 8
    elif inter == 'vipcck' or inter == 'vipcrnvm':
        Ncells = 1
    elif inter == 'vipcr':
        Ncells = 4
    elif inter == 'olm' or inter == 'aacell' or inter == 'bscell':
        Ncells = 2

    # 3-d matrix of all interneurons
    rateMaps = np.zeros((Ncells, Nbins, npath_y))
    rateMaps_unsmoothed = np.zeros((Ncells, Nbins, npath_y))
    time_array_in_bin = np.zeros((Ncells, Nbins, npath_y))

    # File location - pathfile
    fileload = folder1 + '/metrics_permutations/'+learning

    with open(fileload+'/path_all_trial_'+str(ntrial)+'.pkl', 'rb') as f:
        path_all = pickle.load(f)

    # Loop for all INs
    for ncell in range(Ncells):
        # A matrix for rate map
        Zall = np.zeros((Nbins, npath_y))
        time_array_all = np.zeros(Nbins*npath_y)

        for nrun in range(1, runsAll+1):

            # Load of path -- different for each run
            path = path_all[nrun-1]

            # make the time - space map
            time_array = np.bincount(path[:, 0])[1:]
            csum = np.cumsum(time_array)

            fileload = cond+'/Trial_' + \
                str(ntrial)+'/Run_'+str(nrun)+'/spiketimes_'+inter+'_.pkl'
            with open(fileload, 'rb') as f:
                spiketimes_all = pickle.load(f)
            spiketimes = spiketimes_all[ncell][1:]

            Z = spike_map(spiketimes, csum, npath_x, npath_y)
            # Take the sum over all runs given by total
            Zall += binning(Z, Nbins, 'summing')
            time_array_binned = binning(time_array, Nbins, 'summing').squeeze()

            # time spent in each bin in ms
            time_array_all += time_array_binned / 1000.0

        # Calculate the time spent in each bin
        time_array_smoothed = flt.gaussian_filter1d(
            time_array_all, sigma=skernel, mode='nearest', truncate=3.0)
        Zsmoothed = flt.gaussian_filter1d(
            Zall.squeeze(), sigma=skernel, mode='nearest', truncate=3.0)
        # convert to Hz, so divide with seconds
        # time ms/1000 (ms/sec) --> seconds
        Zmean = np.divide(Zsmoothed, time_array_smoothed)

        rateMaps_unsmoothed[int(ncell), :, :] = Zall
        rateMaps[int(ncell), :, :] = Zmean.reshape(-1, 1)
        time_array_in_bin[int(ncell), :, :] = time_array_all.reshape(-1, 1)

    print('\nDone with the rate maps')

    # ==============================================================================
    # ##################### RATE MAPS SAVING #################################
    # ==============================================================================
    filesave = maindir+fdname2+learning

    mydict = {}
    mydict['maps'] = rateMaps
    mydict['maps_unsmoothed'] = rateMaps_unsmoothed
    mydict['time_in_bin'] = time_array_in_bin
    if not os.path.exists(filesave):
        os.makedirs(filesave)

    with open(filesave+'/pickled_sn_'+inter+'_'+case+'_'+ntrial+'.pkl', 'wb') as handle:
        pickle.dump(mydict, handle, protocol=pickle.HIGHEST_PROTOCOL)

    print("\nDone with "+case+" analysis. Done with trial "+ntrial)

tic = time.time()
ntrial = sys.argv[1]
case = sys.argv[2]
learning = sys.argv[3]
inter = sys.argv[4]
results = analysis_path_cluster(ntrial, case, learning, inter)
toc = time.time()

print("\nTotal time: "+str(round(toc-tic, 3))+" seconds")


#exit 0 // OK
#exit 1 // Warning (if applicable)
#exit 2 // Critical error
#test

import requests
import sys,os
import crypto
sys.modules['Crypto'] = crypto
from Crypto.Cipher import AES  #pip install pycrypto, install crypto
import base64

class Crypt():
	def encryption(self, privateInfo, key):
		BLOCK_SIZE = 16
		PADDING = '{'
		pad = lambda s: s + (BLOCK_SIZE - len(s) % BLOCK_SIZE) * PADDING
		EncodeAES = lambda c, s: base64.b64encode(c.encrypt(pad(s)))
		key = str(key)
		if len(key) < 16:
			key = key + ("0" * (16 - len(key)))
		cipher = AES.new(key)
		encoded = EncodeAES(cipher, str(privateInfo))
		return encoded.decode("utf-8")

	def decryption(self, encryptedString, key):
		PADDING = b'{'
		DecodeAES = lambda c, e: c.decrypt(base64.b64decode(e)).rstrip(PADDING)
		encryption = str(encryptedString)
		key = str(key)
		if len(key) < 16:
			key = key + ("0" * (16 - len(key)))
		cipher = AES.new(key)
		decoded = DecodeAES(cipher, encryption)
		return decoded.decode("utf-8")

	def encrypt(self, privateInfo, key):
		if 'list' in str(privateInfo.__class__):
			rez = []
			for item in privateInfo:
				rez.append(self.encryption(item, key))
			return rez
		else:
			return self.encryption(privateInfo, key)

	def decrypt(self, privateInfo, key):
		if 'list' in str(privateInfo.__class__):
			rez = []
			for item in privateInfo:
				rez.append(self.decryption(item, key))
			return rez
		else:
			return self.decryption(privateInfo, key)

class Check():
	def __init__(self):
		self.allowedProducts=['utorrent','bittorrent','utmac','btmac','hydra-ut','hydra-bt']

	def usage(self):
		print('Usage: '+os.path.basename(__file__)+' product\n\tproduct: \n\t\t', end='')
		print(self.allowedProducts)
		print('\n\tkey\n\t\tThe secret key to decrypt credentials\n\n\tExample: '+os.path.basename(__file__)+' \'utorrent,bittorrent,btmac\' secretKey123')
		sys.exit(1)

	def unknown(self):
		print ('Unknown product: '+sys.argv[1]+'\n')
		self.usage()

	def parse(self):   # todo: use urllib.urlparser instead
		if len(sys.argv)<3:
			self.usage()
		products=sys.argv[1].replace("'",'').split(',')

		for product in products:
			print(product)
			if product not in self.allowedProducts:
				self.unknown()
		rez=[]
		for product in products:
			for url in self.getUrls(product):
				rez.append(url)
			if len(rez)==0:
				print("Failed to load rules for product")
				sys.exit(2)  #no rules was found for product
		return rez

	def req(self,product):
		headers = {'api-key':crypt.decrypt('qDLdfOQKGzHe7zgHO5NPngpIcJ0jjCntbHtZBSb8cj0=',sys.argv[2])}
		try:
			UTr=requests.get('https://cherryui.bittorrent.com/api/v2/ruleset/'+product+'/',headers=headers, timeout=3)
			UTdata=UTr.text
			return UTdata
		except:
			print('Error - request timeout.')
			sys.exit(1)

	def getUrls(self,product):
		r=self.req(product).split(',')
		z=0
		rez=[]
		for word in r:
			if word[1:5]=='uuid':
				z=z+1
				tmp=('http://download-01.utorrent.com/uuid/'+word[8:len(word)-1])
				rez.append(tmp)
				tmp=('http://download-02.utorrent.com/uuid/'+word[8:len(word)-1])
				rez.append(tmp)
		return rez[2:]  #Returns list or real urls for 'product' (cutting first 2 empty cherry links)

if __name__=='__main__':
	crypt=Crypt()
	check=Check()
	print(check.parse())
	sys.exit(0)


#!/usr/bin/python
import os
import argparse
from email.MIMEMultipart import MIMEMultipart
from email.MIMEBase import MIMEBase
from email import Encoders

def makemultipart(atomfile, package, outfile):
    atompart = MIMEBase('application', 'atom+xml')
    atompart.add_header('Content-Disposition', 'attachment; name=atom')
    atompart.set_payload(atomfile.read())

    payloadpart = MIMEBase('application', 'zip')
    payloadpart.add_header('Content-Disposition',
        'attachment; name=payload; filename=%s' % os.path.basename(
            package.name))
    payloadpart.set_payload(package.read())
    Encoders.encode_base64(payloadpart)

    message = MIMEMultipart('related')
    message.attach(atompart)
    message.attach(payloadpart)

    # print(message.as_string(unixfrom=False))
    outfile.write(message.as_string(unixfrom=False))
    outfile.close()

def main():
    parser = argparse.ArgumentParser(description='Create a multipart file from an atom entry and a package '
                                                 '(zip, word, odt)')
    parser.add_argument('atomfile', help='/path/to/atomfile', type=open)
    parser.add_argument('package', help='/path/to/package', type=open)
    parser.add_argument('outfile', help='name of output file', type=argparse.FileType('w + '))
    args = parser.parse_args()
    makemultipart(args.atomfile, args.package, args.outfile)

if __name__ == '__main__':
    main()


#--------------------------------------------------------------------------
# File and Version Information:
#  $Id$
#
# Description:
#  Module TemplateLoader...
#
#------------------------------------------------------------------------

"""Implementation of a Jinja template loader.

This loader looks for files in $SIT_DATA/psddl/templates directories.
One template file can contain multiple templates, each template is
preceeded by a line starting with "::::template::::" string followed
by spaces and the name of a template which is a single word, any words
following template name are ignored (can be used for comments). Lines
starting with colon are ignored.

This software was developed for the LCLS project.  If you use all or
part of it, please give an appropriate acknowledgment.

@version $Id$

@author Andy Salnikov
"""

#------------------------------
#  Module's version from CVS --
#------------------------------
__version__ = "$Revision$"
# $Source$

#--------------------------------
#  Imports of standard modules --
#--------------------------------
import sys
import os
import re

#---------------------------------
#  Imports of base class module --
#---------------------------------
import jinja2 as ji

#-----------------------------
# Imports for other modules --
#-----------------------------

#----------------------------------
# Local non-exported definitions --
#----------------------------------

#------------------------
# Exported definitions --
#------------------------

#---------------------
#  Class definition --
#---------------------
class TemplateLoader(ji.FileSystemLoader):

    #----------------
    #  Constructor --
    #----------------
    def __init__(self, package='psddl', templateSubDir='templates'):
        '''Loads templates for psddl. Similar to the jinja2 PackageLoader()
        function but adapted for packages within the SIT build system
        (as opposed to Python packages.)
        ARGS:
        templateSubdir -  a subdirectory to the data directory of the
        package.

        Due to an issue with jinja 2.8, we cache the templates ourselve, we
        expect the environment to be created with a cache_size of 0
        '''
        self.package=package
        self.templateSubDir=templateSubDir
        path = os.environ['SIT_DATA'].split(':')
        ji.FileSystemLoader.__init__(self, path)

        self.template_cache={}
    #-------------------
    #  Public methods --
    #-------------------

    def get_source(self, environment, template):
        if template in self.template_cache:
            return self.template_cache[template]
        orig_template_name = template
        # template name is the file name followed by "?template"
        fname, template = template.split('?')

        # prepend package/templateSubDir to path (defaults to psddl/templates)
        fname = os.path.join("%s/%s" % (self.package,self.templateSubDir), fname)

        # call base class method
        source, path, helper = ji.FileSystemLoader.get_source(self, environment, fname)

        # iterate over lines, find a template
        tmpl = []
        collect = False
        for line in source.splitlines(True):
            words = line.split()
            if words and words[0] == "::::template::::":
                if words[1] == template:
                    collect = True
                else:
                    collect = False
            elif line and line[0] == ':':
                pass
            elif collect:
                tmpl.append(line)
        templateSource = ''.join(tmpl)
        self.template_cache[orig_template_name] = (templateSource, path, helper)
        return ''.join(tmpl), path, helper

    #--------------------------------
    #  Static/class public methods --
    #--------------------------------

    #--------------------
    #  Private methods --
    #--------------------

#
#  In case someone decides to run this module
#
if __name__ == "__main__" :

    # In principle we can try to run test suite for this module,
    # have to think about it later. Right now just abort.
    sys.exit ( "Module is not supposed to be run as main module" )


# encoding: utf-8
# vim: sts=4 sw=4 fdm=marker
# Author: kakkyz <kakkyz81@gmail.com>
# License: MIT
import markdown
import xml.sax.saxutils
import re

# TODO: use vim indent length?
__INDENT__ = 4 * " "

class parserOption:  # {{{
    def __init__(self):
        self.a          = False
        self.ul         = 0
        self.ol         = 0
        self.li         = 0
        self.pre        = False
        self.code       = False
        self.p          = False
        self.blockquote = 0
        self.listCount = 0

    def __str__(self):
        return "a={0} ul={1} ol={2} li={3} pre={4} code={5} p={6} blockquote={7} listCount={8} ".format(
                self.a,
                self.ul,
                self.ol,
                self.li,
                self.pre,
                self.code,
                self.p,
                self.blockquote,
                self.listCount)
#}}}

removeheadercode = re.compile('^<code>')
removefootercode = re.compile('</code>$')

def parseENML(node, level=0, result='', option=parserOption()):  # {{{
# import html2text
#   return html2text.html2text(node.toxml())
#   print node.toxml()
#   print "{0}:{1}:{2}:{3}:{4}:{5}".format(
#           level ,
#           _getNodeType(node) ,
#           _getTagName(node),
#           _getAttribute(node),
#           _getData(node), option)
    if node.nodeType == node.ELEMENT_NODE:
        tag = _getTagName(node)
        if tag == "en-note":
            for child in node.childNodes:
                if child.nodeType == node.ELEMENT_NODE:
                    # Partition block
                    if len(result) == 0 or result[-2:] == '\n\n':
                        pass
                    elif result[-1:] == '\n':
                        result += '\n'
                    else:
                        result += '\n\n'
                    result += parseENML(child, level + 1, "", option) + '\n'
                elif node.nodeType == node.TEXT_NODE:
                    result += parseENML(child, level + 1, "", option)
        elif tag == "a":
            htmlhref = _getAttribute(node)
            option.a = True
            htmltext = "".join([parseENML(child, level + 1, "", option) for child in node.childNodes])
            option.a = False
#           result += '[{0}]({1})'.format(htmltext, htmlhref) # this code does not work multibyte!
            result += '[' + htmltext + '](' + htmlhref + ')'
        elif tag == "pre":
            option.pre = True
            result += "".join([parseENML(child, level + 1, result, option) for child in node.childNodes])
            option.pre = False
        elif tag == "code":
            option.code = True
            if option.pre == True:
                # precode = removeheadercode.sub('', xml.sax.saxutils.unescape(node.toxml()))
                precode = removeheadercode.sub('', _unescape(node.toxml()))
                precode = removefootercode.sub('', precode)
                for line in precode.splitlines():
                    result += __INDENT__ + "%s\n" % line.rstrip()
                result += "\n"
            else:
                # incode = removeheadercode.sub('`', xml.sax.saxutils.unescape(node.toxml()))
                incode = removeheadercode.sub('`', _unescape(node.toxml()))
                incode = removefootercode.sub('`', incode)
                result += incode
            option.code = False
        elif tag == "p":
            option.p = True
            result += "".join([parseENML(child, level + 1, "", option) for child in node.childNodes])
            result = re.compile(r'<br.*?/?>').sub('  ', result)
            result += '\n'
            option.p = False
        elif tag == "ul":
            option.ul += 1
            option.listCount = 0
            result += "".join([parseENML(child, level + 1, "", option) for child in node.childNodes])
            # print "'"+result+"'"
            option.ul -= 1
        elif tag == "ol":
            option.ol += 1
            option.listCount = 0
            result += "".join([parseENML(child, level + 1, "", option) for child in node.childNodes])
            option.ol -= 1
        elif tag == "li":
            option.li += 1
            option.listCount += 1
            listCount = option.listCount

            indent = __INDENT__ * (option.li - 1)
            if _getTagName(node.parentNode) == 'ul':
                result += indent + "* "
            elif _getTagName(node.parentNode) == 'ol':
                result += indent + str(option.listCount) + ". "

            for child in node.childNodes:
                cont = parseENML(child, level + 1, '', option)
                if cont.strip():
                    if child.nodeType == node.ELEMENT_NODE \
                            and _getTagName(child) in ['ul', 'ol']:
                        if result[-1] != '\n':
                            result += '\n'
                    else:
                        # ['strong', 'em']
                        if result[-1] != ' ':
                            result += ' '
                    result += cont
            if result[-1] != '\n':
                result += '\n'
            option.listCount = listCount
            option.li -= 1
        elif tag == "strong":
            result = "".join([parseENML(child, level + 1, "", option) for child in node.childNodes])
            result = '**' + result + '**'
        elif tag == "em":
            result = "".join([parseENML(child, level + 1, "", option) for child in node.childNodes])
            result = '_' + result + '_'
        elif tag in ["img", "br", "en-media", "en-todo", "en-crypt"]:  # 後で改行を除去して見やすくする？
            result += node.toxml()
            result += "\n"
        elif tag == "blockquote":
            option.blockquote += 1
            result += "".join([parseENML(child, level + 1, "", option) for child in node.childNodes])
            result = "\n".join(['> ' + line for line in result[:-1].split("\n")]) + "\n"
            option.blockquote -= 1
            if level == 0:
                result += "\n"
        elif tag == "hr":
            result += "-----"
        elif tag in ["h1", "h2", "h3", "h4", "h5", "h6"]:
            headerlv = tag[1:]
            result += ("#" * int(headerlv)) + " " + "".join([parseENML(child, level + 1, "", option) for child in node.childNodes])
        else:
            result += "".join([parseENML(child, level + 1, result, option) for child in node.childNodes])
    elif node.nodeType == node.TEXT_NODE:
        text = _getData(node)
        if text:
            result += text
        if not option.pre:
            result = _clearNeedlessSpace(result)
    return result
#}}}

def parseMarkdown(mkdtext):  # {{{
    m = markdown.markdown(mkdtext.decode('utf-8'))
    return m
#}}}

# ----- private methods

def _getTagName(node):  # {{{
    if node.nodeType == node.ELEMENT_NODE:
        return node.tagName
    return None
#}}}

def _getData(node):  # {{{
    """ return textdata """
    if node.nodeType == node.TEXT_NODE:
        return node.data.strip()
    return ""
#}}}

def _unescape(text): # {{{
    import HTMLParser
    return HTMLParser.HTMLParser().unescape(text)
#}}}

def _clearNeedlessSpace(text):
    text = re.compile(r'[ \t]*\n[ \t]*').sub('\n', text)
    return re.compile(r'[ \t]+').sub(' ', text)

def _getAttribute(node):  # {{{
    try:
        if _getTagName(node) == "a":
            return node.getAttribute("href")
    except:
        pass
    return None
#}}}

def _getNodeType(node):  # {{{
    """ return NodeType as String """
    if   node.nodeType == node.ELEMENT_NODE                    : return   "ELEMENT_NODE"
    elif node.nodeType == node.ATTRIBUTE_NODE                  : return   "ATTRIBUTE_NODE"
    elif node.nodeType == node.TEXT_NODE                       : return   "TEXT_NODE"
    elif node.nodeType == node.CDATA_SECTION_NODE              : return   "CDATA_SECTION_NODE"
    elif node.nodeType == node.ENTITY_NODE                     : return   "ENTITY_NODE"
    elif node.nodeType == node.PROCESSING_INSTRUCTION_NODE     : return   "PROCESSING_INSTRUCTION_NODE"
    elif node.nodeType == node.COMMENT_NODE                    : return   "COMMENT_NODE"
    elif node.nodeType == node.DOCUMENT_NODE                   : return   "DOCUMENT_NODE"
    elif node.nodeType == node.DOCUMENT_TYPE_NODE              : return   "DOCUMENT_TYPE_NODE"
    elif node.nodeType == node.NOTATION_NODE                   : return   "NOTATION_NODE"
    return "UKNOWN NODE"
#}}}

if __name__ == "__main__":
    import doctest
    doctest.testmod()


import os
from pathlib import Path

def empty_folders_before_run():
    """
    Empty folders and create folders for the subtitles and audio files if it does not exist.
    """

    folder_subtitles = Path('data/output/subtitles')

    # Create a folder if it does not exist.
    create_folder(folder_subtitles)

    for the_file in os.listdir(folder_subtitles):
        file_path = os.path.join(folder_subtitles, the_file)
        try:
            if os.path.isfile(file_path):
                os.unlink(file_path)
        except Exception as e:
            print(e)

    folder_audio = Path('data/output/audio')
    create_folder(folder_audio)

    for the_file in os.listdir(folder_audio):
        file_path = os.path.join(folder_audio, the_file)
        try:
            if os.path.isfile(file_path):
                os.unlink(file_path)
        except Exception as e:
            print(e)

def create_folder(directory):
    """
    Creates a folder.
    """
    try:
        if not os.path.exists(directory):
            os.makedirs(directory)
    except OSError:
        print ('Error: Creating directory. ' +  directory)


# import libraries

import urllib2
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.common.exceptions import NoAlertPresentException
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time
import traceback
import json
import re

# specify the url
paperPage = "https://www.jagran.com"
hdr = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
       'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
       'Accept-Encoding': 'none',
       'Accept-Language': 'en-US,en;q=0.8',
       'Connection': 'keep-alive'}

req = urllib2.Request(paperPage, headers=hdr)
page = urllib2.urlopen(req)

# parse the html using beautiful soup and store in variable `soup`
soup = BeautifulSoup(page, "html.parser")
# Take out the <div> of name and get its value
paperLink = soup.find('a', text="Epaper")
epaperLink = paperLink.get('href')

req = urllib2.Request(epaperLink, headers=hdr)
page = urllib2.urlopen(req)

# parse the html using beautiful soup and store in variable `soup`
soup = BeautifulSoup(page, "html.parser")

stateNameList = []
superCityState = [[]]
citiesFromSupercity = {}
paperLink = soup.find('ul', id="navbar").find_all(recursive=False)
for name in paperLink:
    findStateName = name.find('a', href="#")
    if findStateName is not None:
        stateName = findStateName.text.strip(";")
        stateNameList.append(stateName)

#capa = DesiredCapabilities.CHROME
#capa["pageLoadStrategy"] = "none"
driver = webdriver.Chrome()
wait = WebDriverWait(driver, 10)
driver.get("http://epaper.jagran.com/homepage.aspx")
dataEpaper = {}
dataEpaper['state'] = []
for stateVar in stateNameList:
    # select first state
    print("+++++++++++++++++++++++++")
    print(stateVar)
    print("@@@@@@@@@@@@@@@@@@@@@@@@@@@")
    #if (stateVar=="Delhi") | (stateVar=="UP") | (stateVar=="Haryana") | (stateVar=="Uttarakhand") | (stateVar=="Bihar") :
        #continue
    epaperLinkList = []
    dataEpaper['state'].append({
        'name':stateVar,
        'city':epaperLinkList
    })
    stateWebElement = driver.find_element_by_xpath('//ul[@id="navbar"]//a[text()="%s" and @href="#"]' %stateVar)
    hoverOverOnState = ActionChains(driver).move_to_element(stateWebElement).perform()
    superCityWebElement = stateWebElement.find_elements_by_xpath('./..//section/ul/li/a[@href="#" or text()="%s"]' %stateVar)

    print(len(superCityWebElement))
    superCityList = []
    for superCityWeb in superCityWebElement:
        print(superCityWeb)
        superCityList.append(superCityWeb.text)
        print(superCityWeb.text)
    print("-----------")
    for superCity in superCityList:
        #if (superCity == "Dhanbad") | (superCity == "Jamshedpur") : #| (superCity == "Allahabad") | (superCity == "Bareilly") | (superCity == "Gorakhpur") | (superCity == "Kanpur") | (superCity == "Jhansi") :
            #continue
        stateWebElementFirst = driver.find_element_by_xpath(
            '//ul[@id="navbar"]//a[text()="%s" and @href="#"]' % stateVar)
        superCityWebElementFirst = stateWebElementFirst.find_element_by_xpath(
            './..//section/ul/li/a[text()="%s"]' %superCity)
        hoverOverOnStateFirst = ActionChains(driver).move_to_element(stateWebElementFirst)
        hoverOverOnSuperCityFirst = hoverOverOnStateFirst.move_to_element(superCityWebElementFirst)
        hoverOverOnSuperCityFirst.perform()

        superCityName = superCity
        print("#################")
        print(superCity)
        print("#################")
        citiWebElement = superCityWebElementFirst.find_elements_by_xpath('./../ul//li')
        cityList = []
        for cityWeb in citiWebElement:
            cityList.append(cityWeb.text)
            print(cityWeb.text)
        print("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^")

        for city in cityList:
            cityObject = {}
            cityCode = ""
            staleElement = True
            while (staleElement):
                try:
                    wait.until(EC.visibility_of_element_located((By.ID, 'navbar')))
                    #driver.execute_script("window.stop();")
                    stateWebElementAgain = driver.find_element_by_xpath(
                        '//ul[@id="navbar"]//a[text()="%s" and @href="#"]' % stateVar)
                    superCityWebElementAgain = stateWebElementAgain.find_element_by_xpath(
                        './..//section/ul/li/a[text()="%s"]' % superCityName)
                    hoverOverOnStateAgain = ActionChains(driver).move_to_element(stateWebElementAgain)
                    hoverOverOnSuperCityAgain = hoverOverOnStateAgain.move_to_element(superCityWebElementAgain)
                    hoverOverOnSuperCityAgain.perform()
                    #cityName = city.text
                    print("#################")
                    print(city)
                    print("#################")
                    cityXPath = superCityWebElementAgain.find_element_by_xpath(
                        './..//li/a[text()="%s"]' % city)
                    #myxpath = ('//li/a[text()="%s"]' % city)
                    #element = wait.until(
                        #EC.element_to_be_clickable((By.XPATH, cityXPath)))
                    #time.sleep(1)
                    cityXPath.click()
                    #time.sleep(1)
                    wait.until(EC.visibility_of_element_located((By.ID, 'navbar')))
                    print("&&&&&&&&&&&&&&&&&&&&&&found &&&&&&&&&&&&&&&&&&&&&&&&&&&")
                    driver.execute_script("window.stop();")
                    elementPdfLink = wait.until(
                        EC.element_to_be_clickable((By.XPATH, '//a[@href="#" and @class="pdf"]')))
                    driver.find_element_by_xpath('//a[@href="#" and @class="pdf"]').send_keys(Keys.ESCAPE)#findElement((By.XPATH, '//a[@href="#" and @class="pdf"]')).sendKeys("Keys.ESCAPE");
                    elementPdfLink.click()
                    #time.sleep(1)
                    windowTab = driver.window_handles
                    for tab in windowTab:
                        if tab != windowTab[0]:
                            driver.switch_to.window(tab)
                            codeUrl = driver.current_url
                            print(codeUrl)
                            matchObj = re.match(r'http://epaper.jagran.com/epaperimages/\d+/.+/\d\d(.+)-pg.+', codeUrl, re.M | re.I)
                            cityCode = matchObj.group(1)
                            driver.close()
                    cityObject[city] = cityCode
                    epaperLinkList.append(cityObject)
                    driver.switch_to.window(windowTab[0])
                    print("coming till here")
                    json_data = json.dumps(dataEpaper)
                    print(json_data)
                    staleElement = False
                except Exception as e:
                    print("it is stale")
                    print(e)
                    cityObject[city] = "NOT Found"
                    for window in driver.window_handles:
                        driver.switch_to.window(window)
                        try:
                            driver.switch_to.alert.accept()
                        except NoAlertPresentException:
                            pass
                    windowTab = driver.window_handles
                    for tab in windowTab:
                        if tab != windowTab[0]:
                            driver.switch_to.window(tab)
                            codeUrl = driver.current_url
                            print(codeUrl)
                            driver.close()
                    driver.switch_to.window(windowTab[0])
                    traceback.print_exc()
                    staleElement = False

assert "No results found." not in driver.page_source
driver.close()

json_data = json.dumps(dataEpaper)
print(json_data)