UvmPinAsync / workloads /micro /run_micro_shared.py

lrh12580

first commit

5cb6c4b almost 3 years ago

14 kB

import os
import argparse
import sys
import subprocess
import psutil

import os
import collections
import csv

import numpy as np
import pandas as pd
import matplotlib
from matplotlib import pyplot as plt
import matplotlib.ticker as mticker
import xlsxwriter
import seaborn as sns

from matplotlib.ticker import FormatStrFormatter
from matplotlib.legend_handler import HandlerTuple

from subprocess import Popen, PIPE

from scipy.stats import gmean


config_super_list = ['standard', 'async', 'uvm', 'uvm_prefetch', 'uvm_prefetch_async']
workload_super_list = ['vector_seq']

thread_batch_list = [2, 4, 8, 16, 32, 64, 128]

def dict_to_list(input_dict):
    return_list = []
    for elemement in input_dict:
        return_list.append(elemement)
    return return_list

def addOptions(parser):
    parser.add_argument("-i", "--iterations", type=int, default=1,
                        help="Number of iterations")
    parser.add_argument("-c", "--csv", type=str, default='output.xlsx',
                        help="output trace log file")
    parser.add_argument("-f", "--figure", type=str, default='micro',
                        help="output pdf file")
    parser.add_argument("-p", "--profiling", action='store_true',
                        help="whether to run profiling or just parse results")


def get_config_list(root_directory):
    config_list = []
    for dict in os.listdir(root_directory):
        if os.path.isdir(dict) and dict in config_super_list:
            config_list.append(dict)
    return config_list


def get_workload_dict(root_directory, config_list):
    workload_list = []
    workload_dict = dict()
    for config in config_list:
        config_dir = root_directory + '/' + config
        
        for root, directories, files in os.walk(config_dir, topdown=False):
            for dir in directories:
                if dir in workload_super_list:
                    if dir not in workload_dict:
                        workload_dict[dir] = dict()
                    workload_dict[dir][config] = os.path.join(root, dir)    
                    if dir not in workload_list:
                        workload_list.append(dir)
                
    return workload_list, workload_dict

def execute_bashes(workload_dict, thread_batch_list, iterations):
    for workload in workload_dict:
        if workload in workload_super_list:
            for config in workload_dict[workload]:
                if config in config_super_list:
                    cur_dir = workload_dict[workload][config]
                    pwd = os.getcwd()
                    os.chdir(cur_dir)
                    os.system('make')
                    for thread_batch in thread_batch_list:
                        for i in range(0, iterations):
                            sh_file = './' + workload + '_' + str(thread_batch)
                            exe_cmd = sh_file + ' > ' + str(thread_batch) + '_' + str(i) + '.log'
                            os.system(exe_cmd)
                    os.chdir(pwd)


def process_file(log_file, config):
    result_dict = dict()
    text = open(log_file, "r")

    overlap = 0

    result_dict['gpu_kernel'] = 0
    result_dict['memcpy'] = 0
    result_dict['memcpy_HtoD'] = 0
    result_dict['memcpy_DtoH'] = 0
    result_dict['allocation'] = 0

    for line in text:
        line = line.replace(':', '')
        line = line.strip()
        words = line.split(',')
        
        if 'KERNEL' in words[0] and len(words) >= 4:
            result_dict['gpu_kernel'] += int(words[-1])
        elif 'MEMCPY' in words[0]:
            if 'HTOD' in words[0] or 'HtoD' in words[0]:
                result_dict['memcpy_HtoD'] += int(words[-1])
            else:
                result_dict['memcpy_DtoH'] += int(words[-1])
        elif 'cudaMalloc' in words[0]:
            result_dict['allocation'] += int(words[3])
        elif 'cudaFree' in words[0]:
            result_dict['allocation'] += int(words[3])
        
    return_dict = dict()
    
    if config == 'uvm':
        return_dict['gpu_kernel'] = result_dict['gpu_kernel'] - result_dict['memcpy_HtoD']
    else: 
        return_dict['gpu_kernel'] = result_dict['gpu_kernel']
    return_dict['memcpy'] = result_dict['memcpy_HtoD'] + result_dict['memcpy_DtoH']
    return_dict['allocation'] = result_dict['allocation']
        
    return return_dict


def process_results(workload_dict, thread_batch_list, iterations):
    result_dict = dict()
    for workload in workload_dict:
        if workload in workload_super_list:
            result_dict[workload] = dict()
            for thread_batch in thread_batch_list:
                thread_batch_str = str(thread_batch)
                result_dict[workload][thread_batch_str] = dict()
                for config in workload_dict[workload]:
                    if config in config_super_list:
                        result_dict[workload][thread_batch_str][config] = []
                        cur_dir = workload_dict[workload][config]

                        for i in range(0, iterations):
                            log_file =  cur_dir + '/' + thread_batch_str + '_' + str(i) + '.log'
                            result_dict[workload][thread_batch_str][config].append(process_file(log_file, config))
    return result_dict

def export_csv(result_dict, thread_batch_list, config_list, iterations):
    workload_list = dict_to_list(result_dict)
    
    csv_list = []
    for workload in workload_list:
        avg_dict = dict()
        for b in range(0, len(thread_batch_list)):
            thread_batch_str = str(thread_batch_list[b])
            avg_dict[thread_batch_str] = dict()
            for c in range(0, len(config_list)):
                avg_dict[thread_batch_str][config_list[c]] = dict()
                
                metric_list = dict_to_list(result_dict[workload][thread_batch_str][config_list[c]][0])
                
                for metric in metric_list:
                    avg_dict[thread_batch_str][config_list[c]][metric] = 0
                avg_dict[thread_batch_str][config_list[c]]['all'] = 0
                
                for i in range(0, iterations):
                    for metric in metric_list:
                        avg_dict[thread_batch_str][config_list[c]][metric] += result_dict[workload][thread_batch_str][config_list[c]][i][metric]
                        avg_dict[thread_batch_str][config_list[c]]['all'] += result_dict[workload][thread_batch_str][config_list[c]][i][metric]

            for c in range(0, len(config_list)):
                thread_batch_str_0 = str(thread_batch_list[0])
                normarlized_all = avg_dict[thread_batch_str][config_list[c]]['all'] / avg_dict[thread_batch_str_0]['standard']['all']
                for metric in metric_list:
                    avg_dict[thread_batch_str][config_list[c]][metric] = (avg_dict[thread_batch_str][config_list[c]][metric] / avg_dict[thread_batch_str][config_list[c]]['all']) * normarlized_all
        
        workload_csv_file = workload + '_sensitivity_shared.csv'

        csv_list.append(workload_csv_file)
        out = open(workload_csv_file, "w")

        out.write('group,,')
        for i in range(0, len(config_list)):
            out.write(config_list[i])
            if i != len(config_list) - 1:
                out.write(',')
            else:
                out.write(os.linesep)

        for thread_batch in thread_batch_list:
            thread_batch_str = str(thread_batch)
            for metric in metric_list:
                out.write(str(int (thread_batch * 256 * 4 / 1024)) + 'KB,' + metric + ',')
                for j in range(0, len(config_list)):
                    out.write(str(avg_dict[thread_batch_str][config_list[j]][metric]))
                    if j != len(config_list) - 1:
                        out.write(',')
                    else:
                        out.write(os.linesep)

        out.close()
        
    return csv_list

def plot_std_results(csv_file, output_file):
    df = pd.read_csv(csv_file, index_col=0)
            
    group_list = []
    for index in df.index:
        if index not in group_list:
            group_list.append(index)
    col_list = df.columns

    ngroups = len(group_list)
    x = np.arange(ngroups)
    nbars = len(col_list)
    width = (1 - 0.4) / (1.5 * nbars)  # the width of the bars

    matplotlib.rcParams["hatch.linewidth"] = 2

    # patterns = [ "|" , "/", "-", "", "x", "-", "\\", "+", "o", "O" ]
    # patterns = [ "|" , "/", "x", "*", ".", "-", "\\", "+", "o", "O" ]
    # patterns = ["//", "//", "//", "//", "//", "//", "//"]
    patterns = ["", "", "", "", "", ""]
    # color_tab = ['#D9D9D9', '#BFBFBF', '#A6A6A6', '#7F7F7F', '#7F7F7F', '#7F7F7F']
    color_tab = ['#ff0000', '#ff6d01','#46bdc6', '#4285f4', '#ea4335', '#34a853']
    edge_color_tab = ['#000000', '#000000', '#000000', '#000000', '#000000', '#000000']

    fig, ax = plt.subplots(figsize=[8.8, 2.8])

    rects = []

    for i in range(0, nbars):
        # height_cum = np.array([0.0] * ngroups)
        height_total = np.array([1 for g in group_list])  # y coo
        height_curr = np.array([float(df[col_list[i]][g]) for g in group_list])  # y coo
        rect_base = ax.bar(x - 0.3 + (3 * i + 1.5) * width / 2,          # x coo
                        height_curr / height_total,  # y coo
                        width, label=col_list[i],
                        color=color_tab[i],
                        edgecolor=edge_color_tab[i],
                        linewidth=0.5
                        )
        rects.append(rect_base)
        # height_cum += height_curr

    hdl_pair = [(rects[i]) for i in range(nbars)]
 
    ax.set_xticks(x)
    ax.set_xticklabels(group_list, rotation=0, weight='bold')
    ax.legend()
    # ax.yaxis.set_major_formatter(mticker.PercentFormatter(1.0))
    
    plt.xticks(fontsize=11)
    plt.yticks(fontsize=9, weight='bold')
    plt.grid(axis='y')
    plt.xlabel("")
    plt.ylabel("Standard deviation / Mean")
    plt.tight_layout()
    # plt.margins(x=0.01, y=0.01)

    plt.savefig(output_file + '_std.pdf', bbox_inches='tight')
    plt.close()
    

def plot_results(csv_file, output_file):
    df = pd.read_csv(csv_file, index_col=[0, 1])
    group_list = []
    subgrp_list = []
    for index in df.index:
        if index[0] not in group_list:
            group_list.append(index[0])
        if index[1] not in subgrp_list:
            subgrp_list.append(index[1])
    col_list = df.columns

    ngroups = len(group_list)
    nsubgrps = len(subgrp_list)
    x = np.arange(ngroups)
    nbars = len(col_list)
    width = (1 - 0.4) / (1.5 * nbars)  # the width of the bars

    matplotlib.rcParams["hatch.linewidth"] = 2

    patterns = ["", "-", "/", "|", "/", "-",  "x", "-", "\\", "+", "o", "O"]
    color_tab = ['#000000', '#0000ff', '#ff0000', '#ff6666', '#00ff00']

    fig, ax = plt.subplots(figsize=[8.8, 3.8])
    hdl_pair = []

    rects = []

    for i in range(0, nbars):
        height_cum = np.array([0.0] * ngroups)
        height_curr = np.array([df[col_list[i]][g][0]
                                for g in group_list])  # y coo
        rect_base = ax.bar(x - 0.3 + (3 * i + 1.5) * width / 2,          # x coo
                           height_curr,  # y coo
                           width,
                           label=col_list[i]+" "+subgrp_list[0],
                           color=color_tab[i],
                           edgecolor=color_tab[0],
                           linewidth=0.25
                           )
        rects.append(rect_base)
        height_cum += height_curr
        for j in range(1, 3):
            height_curr = np.array([df[col_list[i]][g][j]
                                    for g in group_list])
            rect = ax.bar(x - 0.3 + (3 * i + 1.5) * width / 2,          # x coo
                          height_curr,  # y coo
                          width,
                          label=col_list[i]+" "+subgrp_list[j],
                          bottom=height_cum,
                          color=color_tab[i],
                          edgecolor=color_tab[0],
                          linewidth=0.25,
                          alpha=0.25 * (4 - j)
                          )
            rects.append(rect)
            height_cum += height_curr

    hdl_pair = [(rects[i*nsubgrps], rects[i*nsubgrps+1],
                 rects[i*nsubgrps+2]) for i in range(nbars)]
    ax.set_xticks(x)
    ax.set_xticklabels(group_list, rotation=0)

    ax.legend(hdl_pair, col_list, loc='upper center', ncol=3, bbox_to_anchor=(0.5, 1.2), fontsize=14, handler_map={tuple: HandlerTuple(ndivide=None)})
    # ax.legend()
    # ax.yaxis.set_major_formatter(mticker.PercentFormatter(1.0))
    # ax.set_yscale('log')

    plt.xticks(fontsize=14, rotation=15)
    plt.yticks(fontsize=14)
    plt.grid(axis='y')
    plt.xlabel("")
    plt.ylabel("Normalized Time", fontsize=14)
    plt.xlabel("Shared Memory Capacity", fontsize=14)
    plt.tight_layout()
    # plt.margins(x=0.01, y=0.01)

    plt.savefig(output_file, bbox_inches='tight')
    plt.close()


def main():
    parser = argparse.ArgumentParser()
    addOptions(parser)
    
    options = parser.parse_args()
    
    iterations = options.iterations
    output_csv_file = options.csv
    output_figure_file = options.figure
    profiling = options.profiling
    
    root_directory = './'
    
    config_list = get_config_list(root_directory)
    print(config_list)
    workload_list, workload_dict = get_workload_dict(root_directory, config_list)
    print(workload_dict)

    if profiling:
        execute_bashes(workload_dict, thread_batch_list, iterations)
    
    result_dict = process_results(workload_dict, thread_batch_list, iterations)
    csv_list = export_csv(result_dict, thread_batch_list, config_super_list, iterations)
    for csv_file in csv_list:
        plot_results(csv_file, csv_file.replace(".csv", ".pdf"))


if __name__ == '__main__':
    main()

	import os
	import argparse
	import sys
	import subprocess
	import psutil

	import os
	import collections
	import csv

	import numpy as np
	import pandas as pd
	import matplotlib
	from matplotlib import pyplot as plt
	import matplotlib.ticker as mticker
	import xlsxwriter
	import seaborn as sns

	from matplotlib.ticker import FormatStrFormatter
	from matplotlib.legend_handler import HandlerTuple

	from subprocess import Popen, PIPE

	from scipy.stats import gmean


	config_super_list = ['standard', 'async', 'uvm', 'uvm_prefetch', 'uvm_prefetch_async']
	workload_super_list = ['vector_seq']

	thread_batch_list = [2, 4, 8, 16, 32, 64, 128]

	def dict_to_list(input_dict):
	return_list = []
	for elemement in input_dict:
	return_list.append(elemement)
	return return_list

	def addOptions(parser):
	parser.add_argument("-i", "--iterations", type=int, default=1,
	help="Number of iterations")
	parser.add_argument("-c", "--csv", type=str, default='output.xlsx',
	help="output trace log file")
	parser.add_argument("-f", "--figure", type=str, default='micro',
	help="output pdf file")
	parser.add_argument("-p", "--profiling", action='store_true',
	help="whether to run profiling or just parse results")


	def get_config_list(root_directory):
	config_list = []
	for dict in os.listdir(root_directory):
	if os.path.isdir(dict) and dict in config_super_list:
	config_list.append(dict)
	return config_list


	def get_workload_dict(root_directory, config_list):
	workload_list = []
	workload_dict = dict()
	for config in config_list:
	config_dir = root_directory + '/' + config

	for root, directories, files in os.walk(config_dir, topdown=False):
	for dir in directories:
	if dir in workload_super_list:
	if dir not in workload_dict:
	workload_dict[dir] = dict()
	workload_dict[dir][config] = os.path.join(root, dir)
	if dir not in workload_list:
	workload_list.append(dir)

	return workload_list, workload_dict

	def execute_bashes(workload_dict, thread_batch_list, iterations):
	for workload in workload_dict:
	if workload in workload_super_list:
	for config in workload_dict[workload]:
	if config in config_super_list:
	cur_dir = workload_dict[workload][config]
	pwd = os.getcwd()
	os.chdir(cur_dir)
	os.system('make')
	for thread_batch in thread_batch_list:
	for i in range(0, iterations):
	sh_file = './' + workload + '_' + str(thread_batch)
	exe_cmd = sh_file + ' > ' + str(thread_batch) + '_' + str(i) + '.log'
	os.system(exe_cmd)
	os.chdir(pwd)


	def process_file(log_file, config):
	result_dict = dict()
	text = open(log_file, "r")

	overlap = 0

	result_dict['gpu_kernel'] = 0
	result_dict['memcpy'] = 0
	result_dict['memcpy_HtoD'] = 0
	result_dict['memcpy_DtoH'] = 0
	result_dict['allocation'] = 0

	for line in text:
	line = line.replace(':', '')
	line = line.strip()
	words = line.split(',')

	if 'KERNEL' in words[0] and len(words) >= 4:
	result_dict['gpu_kernel'] += int(words[-1])
	elif 'MEMCPY' in words[0]:
	if 'HTOD' in words[0] or 'HtoD' in words[0]:
	result_dict['memcpy_HtoD'] += int(words[-1])
	else:
	result_dict['memcpy_DtoH'] += int(words[-1])
	elif 'cudaMalloc' in words[0]:
	result_dict['allocation'] += int(words[3])
	elif 'cudaFree' in words[0]:
	result_dict['allocation'] += int(words[3])

	return_dict = dict()

	if config == 'uvm':
	return_dict['gpu_kernel'] = result_dict['gpu_kernel'] - result_dict['memcpy_HtoD']
	else:
	return_dict['gpu_kernel'] = result_dict['gpu_kernel']
	return_dict['memcpy'] = result_dict['memcpy_HtoD'] + result_dict['memcpy_DtoH']
	return_dict['allocation'] = result_dict['allocation']

	return return_dict


	def process_results(workload_dict, thread_batch_list, iterations):
	result_dict = dict()
	for workload in workload_dict:
	if workload in workload_super_list:
	result_dict[workload] = dict()
	for thread_batch in thread_batch_list:
	thread_batch_str = str(thread_batch)
	result_dict[workload][thread_batch_str] = dict()
	for config in workload_dict[workload]:
	if config in config_super_list:
	result_dict[workload][thread_batch_str][config] = []
	cur_dir = workload_dict[workload][config]

	for i in range(0, iterations):
	log_file = cur_dir + '/' + thread_batch_str + '_' + str(i) + '.log'
	result_dict[workload][thread_batch_str][config].append(process_file(log_file, config))
	return result_dict

	def export_csv(result_dict, thread_batch_list, config_list, iterations):
	workload_list = dict_to_list(result_dict)

	csv_list = []
	for workload in workload_list:
	avg_dict = dict()
	for b in range(0, len(thread_batch_list)):
	thread_batch_str = str(thread_batch_list[b])
	avg_dict[thread_batch_str] = dict()
	for c in range(0, len(config_list)):
	avg_dict[thread_batch_str][config_list[c]] = dict()

	metric_list = dict_to_list(result_dict[workload][thread_batch_str][config_list[c]][0])

	for metric in metric_list:
	avg_dict[thread_batch_str][config_list[c]][metric] = 0
	avg_dict[thread_batch_str][config_list[c]]['all'] = 0

	for i in range(0, iterations):
	for metric in metric_list:
	avg_dict[thread_batch_str][config_list[c]][metric] += result_dict[workload][thread_batch_str][config_list[c]][i][metric]
	avg_dict[thread_batch_str][config_list[c]]['all'] += result_dict[workload][thread_batch_str][config_list[c]][i][metric]

	for c in range(0, len(config_list)):
	thread_batch_str_0 = str(thread_batch_list[0])
	normarlized_all = avg_dict[thread_batch_str][config_list[c]]['all'] / avg_dict[thread_batch_str_0]['standard']['all']
	for metric in metric_list:
	avg_dict[thread_batch_str][config_list[c]][metric] = (avg_dict[thread_batch_str][config_list[c]][metric] / avg_dict[thread_batch_str][config_list[c]]['all']) * normarlized_all

	workload_csv_file = workload + '_sensitivity_shared.csv'

	csv_list.append(workload_csv_file)
	out = open(workload_csv_file, "w")

	out.write('group,,')
	for i in range(0, len(config_list)):
	out.write(config_list[i])
	if i != len(config_list) - 1:
	out.write(',')
	else:
	out.write(os.linesep)

	for thread_batch in thread_batch_list:
	thread_batch_str = str(thread_batch)
	for metric in metric_list:
	out.write(str(int (thread_batch * 256 * 4 / 1024)) + 'KB,' + metric + ',')
	for j in range(0, len(config_list)):
	out.write(str(avg_dict[thread_batch_str][config_list[j]][metric]))
	if j != len(config_list) - 1:
	out.write(',')
	else:
	out.write(os.linesep)

	out.close()

	return csv_list

	def plot_std_results(csv_file, output_file):
	df = pd.read_csv(csv_file, index_col=0)

	group_list = []
	for index in df.index:
	if index not in group_list:
	group_list.append(index)
	col_list = df.columns

	ngroups = len(group_list)
	x = np.arange(ngroups)
	nbars = len(col_list)
	width = (1 - 0.4) / (1.5 * nbars) # the width of the bars

	matplotlib.rcParams["hatch.linewidth"] = 2

	# patterns = [ "\|" , "/", "-", "", "x", "-", "\\", "+", "o", "O" ]
	# patterns = [ "\|" , "/", "x", "*", ".", "-", "\\", "+", "o", "O" ]
	# patterns = ["//", "//", "//", "//", "//", "//", "//"]
	patterns = ["", "", "", "", "", ""]
	# color_tab = ['#D9D9D9', '#BFBFBF', '#A6A6A6', '#7F7F7F', '#7F7F7F', '#7F7F7F']
	color_tab = ['#ff0000', '#ff6d01','#46bdc6', '#4285f4', '#ea4335', '#34a853']
	edge_color_tab = ['#000000', '#000000', '#000000', '#000000', '#000000', '#000000']

	fig, ax = plt.subplots(figsize=[8.8, 2.8])

	rects = []

	for i in range(0, nbars):
	# height_cum = np.array([0.0] * ngroups)
	height_total = np.array([1 for g in group_list]) # y coo
	height_curr = np.array([float(df[col_list[i]][g]) for g in group_list]) # y coo
	rect_base = ax.bar(x - 0.3 + (3 * i + 1.5) * width / 2, # x coo
	height_curr / height_total, # y coo
	width, label=col_list[i],
	color=color_tab[i],
	edgecolor=edge_color_tab[i],
	linewidth=0.5
	)
	rects.append(rect_base)
	# height_cum += height_curr

	hdl_pair = [(rects[i]) for i in range(nbars)]

	ax.set_xticks(x)
	ax.set_xticklabels(group_list, rotation=0, weight='bold')
	ax.legend()
	# ax.yaxis.set_major_formatter(mticker.PercentFormatter(1.0))

	plt.xticks(fontsize=11)
	plt.yticks(fontsize=9, weight='bold')
	plt.grid(axis='y')
	plt.xlabel("")
	plt.ylabel("Standard deviation / Mean")
	plt.tight_layout()
	# plt.margins(x=0.01, y=0.01)

	plt.savefig(output_file + '_std.pdf', bbox_inches='tight')
	plt.close()


	def plot_results(csv_file, output_file):
	df = pd.read_csv(csv_file, index_col=[0, 1])
	group_list = []
	subgrp_list = []
	for index in df.index:
	if index[0] not in group_list:
	group_list.append(index[0])
	if index[1] not in subgrp_list:
	subgrp_list.append(index[1])
	col_list = df.columns

	ngroups = len(group_list)
	nsubgrps = len(subgrp_list)
	x = np.arange(ngroups)
	nbars = len(col_list)
	width = (1 - 0.4) / (1.5 * nbars) # the width of the bars

	matplotlib.rcParams["hatch.linewidth"] = 2

	patterns = ["", "-", "/", "\|", "/", "-", "x", "-", "\\", "+", "o", "O"]
	color_tab = ['#000000', '#0000ff', '#ff0000', '#ff6666', '#00ff00']

	fig, ax = plt.subplots(figsize=[8.8, 3.8])
	hdl_pair = []

	rects = []

	for i in range(0, nbars):
	height_cum = np.array([0.0] * ngroups)
	height_curr = np.array([df[col_list[i]][g][0]
	for g in group_list]) # y coo
	rect_base = ax.bar(x - 0.3 + (3 * i + 1.5) * width / 2, # x coo
	height_curr, # y coo
	width,
	label=col_list[i]+" "+subgrp_list[0],
	color=color_tab[i],
	edgecolor=color_tab[0],
	linewidth=0.25
	)
	rects.append(rect_base)
	height_cum += height_curr
	for j in range(1, 3):
	height_curr = np.array([df[col_list[i]][g][j]
	for g in group_list])
	rect = ax.bar(x - 0.3 + (3 * i + 1.5) * width / 2, # x coo
	height_curr, # y coo
	width,
	label=col_list[i]+" "+subgrp_list[j],
	bottom=height_cum,
	color=color_tab[i],
	edgecolor=color_tab[0],
	linewidth=0.25,
	alpha=0.25 * (4 - j)
	)
	rects.append(rect)
	height_cum += height_curr

	hdl_pair = [(rects[insubgrps], rects[insubgrps+1],
	rects[i*nsubgrps+2]) for i in range(nbars)]
	ax.set_xticks(x)
	ax.set_xticklabels(group_list, rotation=0)

	ax.legend(hdl_pair, col_list, loc='upper center', ncol=3, bbox_to_anchor=(0.5, 1.2), fontsize=14, handler_map={tuple: HandlerTuple(ndivide=None)})
	# ax.legend()
	# ax.yaxis.set_major_formatter(mticker.PercentFormatter(1.0))
	# ax.set_yscale('log')

	plt.xticks(fontsize=14, rotation=15)
	plt.yticks(fontsize=14)
	plt.grid(axis='y')
	plt.xlabel("")
	plt.ylabel("Normalized Time", fontsize=14)
	plt.xlabel("Shared Memory Capacity", fontsize=14)
	plt.tight_layout()
	# plt.margins(x=0.01, y=0.01)

	plt.savefig(output_file, bbox_inches='tight')
	plt.close()


	def main():
	parser = argparse.ArgumentParser()
	addOptions(parser)

	options = parser.parse_args()

	iterations = options.iterations
	output_csv_file = options.csv
	output_figure_file = options.figure
	profiling = options.profiling

	root_directory = './'

	config_list = get_config_list(root_directory)
	print(config_list)
	workload_list, workload_dict = get_workload_dict(root_directory, config_list)
	print(workload_dict)

	if profiling:
	execute_bashes(workload_dict, thread_batch_list, iterations)

	result_dict = process_results(workload_dict, thread_batch_list, iterations)
	csv_list = export_csv(result_dict, thread_batch_list, config_super_list, iterations)
	for csv_file in csv_list:
	plot_results(csv_file, csv_file.replace(".csv", ".pdf"))


	if __name__ == '__main__':
	main()