Add files using upload-large-folder tool

261228b verified 8 days ago

13.5 kB

#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# -----------------------------------------------------------------------------
# Copyright 2020-2022 Arm Limited
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# -----------------------------------------------------------------------------
"""
The ``astc_test_result_plot.py`` script consolidates all current sets of
reference results into a single graphical plot.
"""

import re
import os
import sys

import numpy as np
import matplotlib.pyplot as plt

import testlib.resultset as trs
from collections import defaultdict as ddict


def find_reference_results():
    """
    Scrape the Test/Images directory for result CSV files and return an
    mapping of the result sets.
    Returns:
        Returns a three deep tree of dictionaries, with the final dict
        pointing at a `ResultSet` object. The hierarchy is:
            imageSet => quality => encoder => result
    """
    scriptDir = os.path.dirname(__file__)
    imageDir = os.path.join(scriptDir, "Images")

    # Pattern for extracting useful data from the CSV file name
    filePat = re.compile(r"astc_reference-(.+)_(.+)_results\.csv")

    # Build a three level dictionary we can write into
    results = ddict(lambda: ddict(lambda: ddict()))

    # Final all CSVs, load them and store them in the dict tree
    for root, dirs, files in os.walk(imageDir):
        for name in files:
            match = filePat.match(name)
            if match:
                fullPath = os.path.join(root, name)

                encoder = match.group(1)
                quality = match.group(2)
                imageSet = os.path.basename(root)

                if imageSet not in ["Kodak", "Khronos", "HDRIHaven", "KodakSim"]:
                    continue

                testRef = trs.ResultSet(imageSet)
                testRef.load_from_file(fullPath)

                results[imageSet][quality]["ref-%s" % encoder] = testRef

    return results


def get_series(results, tgtEncoder, tgtQuality, resFilter=lambda x: True):
    psnrData = []
    mtsData = []
    marker = []
    records = []

    for imageSet, iResults in results.items():

        for quality, qResults in iResults.items():
            if quality != tgtQuality:
                continue

            for encoder, eResults in qResults.items():
                if encoder != tgtEncoder:
                    continue

                for record in eResults.records:
                    if resFilter(record):
                        records.append(record)
                        psnrData.append(record.psnr)
                        mtsData.append(record.cRate)

                        if "ldr-xy" in record.name:
                            marker.append('$N$')
                        elif "ldr-l" in record.name:
                            marker.append('$G$')
                        elif "ldr" in record.name:
                            marker.append('$L$')
                        elif "hdr" in record.name:
                            marker.append('$H$')
                        else:
                            marker.append('$?$')


    return mtsData, psnrData, marker, records


def get_series_rel(results, refEncoder, refQuality, tgtEncoder, tgtQuality, resFilter=lambda x: True):

    mts1, psnr1, marker1, rec1 = get_series(results, tgtEncoder, tgtQuality, resFilter)

    if refEncoder is None:
        refEncoder = tgtEncoder

    if refQuality is None:
        refQuality = tgtQuality

    mts2, psnr2, marker2, rec2 = get_series(results, refEncoder, refQuality, resFilter)

    mtsm  = [x/mts2[i] for i, x in enumerate(mts1)]
    psnrm = [x - psnr2[i] for i, x in enumerate(psnr1)]

    return mtsm, psnrm, marker1, rec1


def get_human_eq_name(encoder, quality):
    parts = encoder.split("-")
    if len(parts) == 2:
        return "astcenc %s -%s" % (parts[1], quality)
    else:
        return "astcenc-%s %s -%s" % (parts[2], parts[1], quality)


def get_human_e_name(encoder):
    parts = encoder.split("-")
    if len(parts) == 2:
        return "astcenc %s" % parts[1]
    else:
        return "astcenc-%s %s" % (parts[2], parts[1])


def get_human_q_name(quality):
    return "-%s" % quality


def plot(results, chartRows, chartCols, blockSizes,
         relative, pivotEncoder, pivotQuality, fileName, limits):

    fig, axs = plt.subplots(nrows=len(chartRows), ncols=len(chartCols),
                            sharex=True, sharey=True, figsize=(15, 8.43))

    for a in fig.axes:
        a.tick_params(
            axis="x", which="both",
            bottom=True, top=False, labelbottom=True)

        a.tick_params(
            axis="y", which="both",
            left=True, right=False, labelleft=True)

    for i, row in enumerate(chartRows):
        for j, col in enumerate(chartCols):
            if row == "fastest" and (("1.7" in col) or ("2.0" in col)):
                if len(chartCols) == 1:
                    fig.delaxes(axs[i])
                else:
                    fig.delaxes(axs[i][j])
                continue

            if len(chartRows) == 1 and len(chartCols) == 1:
                ax = axs
            elif len(chartCols) == 1:
                ax = axs[i]
            else:
                ax = axs[i, j]

            title = get_human_eq_name(col, row)

            if not relative:
                ax.set_title(title, y=0.97, backgroundcolor="white")
                ax.set_xlabel('Coding performance (MTex/s)')
                ax.set_ylabel('PSNR (dB)')
            else:
                if pivotEncoder and pivotQuality:
                    tag = get_human_eq_name(pivotEncoder, pivotQuality)
                elif pivotEncoder:
                    tag = get_human_e_name(pivotEncoder)
                else:
                    assert(pivotQuality)
                    tag = get_human_q_name(pivotQuality)

                ax.set_title("%s vs. %s" % (title, tag), y=0.97, backgroundcolor="white")
                ax.set_xlabel('Performance scaling')
                ax.set_ylabel('PSNR delta (dB)')

            for k, series in enumerate(blockSizes):
                fn = lambda x: x.blkSz == series

                if not relative:
                    x, y, m, r = get_series(results, col, row, fn)
                else:
                    x, y, m, r = get_series_rel(results, pivotEncoder, pivotQuality,
                                                col, row, fn)

                color = None
                label = "%s blocks" % series
                for xp, yp, mp in zip(x, y, m):
                    ax.scatter([xp],[yp], s=16, marker=mp,
                               color="C%u" % k, label=label)
                    label = None

            if i == 0 and j == 0:
                ax.legend(loc="lower right")

    for i, row in enumerate(chartRows):
        for j, col in enumerate(chartCols):

            if len(chartRows) == 1 and len(chartCols) == 1:
                ax = axs
            elif len(chartCols) == 1:
                ax = axs[i]
            else:
                ax = axs[i, j]

            ax.grid(ls=':')

            if limits and limits[0]:
                ax.set_xlim(left=limits[0][0], right=limits[0][1])
            if limits and limits[1]:
                ax.set_ylim(bottom=limits[1][0], top=limits[1][1])

    fig.tight_layout()
    fig.savefig(fileName)


def main():
    """
    The main function.
    Returns:
        int: The process return code.
    """
    absXMin = 0
    absXMax = 80
    absXLimits = (absXMin, absXMax)

    relXMin = 0.8
    relXMax = None
    relXLimits = (relXMin, relXMax)

    last1x = "1.7"
    last2x = "2.5"
    last3x = "3.7"
    prev4x = "4.3"
    last4x = "4.4"
    lastMain = "main"

    charts = [
        # --------------------------------------------------------
        # Latest in stable series charts
        [
            # Relative scores
            ["thorough", "medium", "fast"],
            [f"ref-{last2x}-avx2", f"ref-{last3x}-avx2", f"ref-{last4x}-avx2"],
            ["4x4", "6x6", "8x8"],
            True,
            f"ref-{last1x}",
            None,
            "results-relative-stable-series.png",
            (None, None)
        ], [
            # Absolute scores
            ["thorough", "medium", "fast"],
            [f"ref-{last1x}", f"ref-{last2x}-avx2", f"ref-{last3x}-avx2", f"ref-{last4x}-avx2"],
            ["4x4", "6x6", "8x8"],
            False,
            None,
            None,
            "results-absolute-stable-series.png",
            (absXLimits, None)
        ],
        # --------------------------------------------------------
        # Latest 2.x vs 1.x release charts
        [
            # Relative scores
            ["thorough", "medium", "fast"],
            [f"ref-{last2x}-avx2"],
            ["4x4", "6x6", "8x8"],
            True,
            f"ref-{last1x}",
            None,
            "results-relative-2.x-vs-1.x.png",
            (None, None)
        ],
        # --------------------------------------------------------
        # Latest 3.x vs 1.x release charts
        [
            # Relative scores
            ["thorough", "medium", "fast"],
            [f"ref-{last3x}-avx2"],
            ["4x4", "6x6", "8x8"],
            True,
            f"ref-{last1x}",
            None,
            "results-relative-3.x-vs-1.x.png",
            (None, None)
        ],
        # --------------------------------------------------------
        # Latest 4.x vs 1.x release charts
        [
            # Relative scores
            ["thorough", "medium", "fast"],
            [f"ref-{last4x}-avx2"],
            ["4x4", "6x6", "8x8"],
            True,
            f"ref-{last1x}",
            None,
            "results-relative-4.x-vs-1.x.png",
            (None, None)
        ],
        # --------------------------------------------------------
        # Latest 3.x vs 2.x release charts
        [
            # Relative scores
            ["thorough", "medium", "fast", "fastest"],
            [f"ref-{last3x}-avx2"],
            ["4x4", "6x6", "8x8"],
            True,
            f"ref-{last2x}-avx2",
            None,
            "results-relative-3.x-vs-2.x.png",
            (None, None)
        ],
        # --------------------------------------------------------
        # Latest 4.x vs 3.x release charts
        [
            # Relative scores
            ["thorough", "medium", "fast", "fastest"],
            [f"ref-{last4x}-avx2"],
            ["4x4", "6x6", "8x8"],
            True,
            f"ref-{last3x}-avx2",
            None,
            "results-relative-4.x-vs-3.x.png",
            (relXLimits, None),
        ], [
            # Relative ISAs of latest
            ["thorough", "medium", "fast", "fastest"],
            [f"ref-{last4x}-sse4.1", f"ref-{last4x}-avx2"],
            ["4x4", "6x6", "8x8"],
            True,
            f"ref-{last4x}-sse2",
            None,
            "results-relative-4.x-isa.png",
            (None, None)
        ], [
            # Relative quality of latest
            ["medium", "fast", "fastest"],
            [f"ref-{last4x}-avx2"],
            ["4x4", "6x6", "8x8"],
            True,
            None,
            "thorough",
            "results-relative-4.x-quality.png",
            (None, None)
        ],
        # --------------------------------------------------------
        # Latest 4.x vs previous 4.x release charts
        [
            # Relative scores
            ["thorough", "medium", "fast", "fastest"],
            [f"ref-{last4x}-avx2"],
            ["4x4", "6x6", "8x8"],
            True,
            f"ref-{prev4x}-avx2",
            None,
            "results-relative-4.x-vs-4.x.png",
            (relXLimits, None)
        ],
        # --------------------------------------------------------
        # Latest 4.x vs previous 4.x release charts
        [
            # Relative scores
            ["thorough", "medium", "fast", "fastest"],
            [f"ref-{lastMain}-avx2"],
            ["4x4", "6x6", "8x8"],
            True,
            f"ref-{last4x}-avx2",
            None,
            "results-relative-main-vs-4.x.png",
            (relXLimits, None)
        ]
    ]

    results = find_reference_results()

    # Force select is triggered by adding a trailing entry to the argument list
    # of the charts that you want rendered; designed for debugging use cases
    maxIndex = 0
    expectedLength = 8
    for chart in charts:
        maxIndex = max(maxIndex, len(chart))

    for chart in charts:
        # If force select is enabled then only keep the forced ones
        if len(chart) != maxIndex:
            print("Skipping %s" % chart[6])
            continue
        else:
            print("Generating %s" % chart[6])

        # If force select is enabled then strip the dummy force option
        if maxIndex != expectedLength:
            chart = chart[:expectedLength]

        plot(results, *chart)

    return 0


if __name__ == "__main__":
    sys.exit(main())

	#!/usr/bin/env python3
	# SPDX-License-Identifier: Apache-2.0
	# -----------------------------------------------------------------------------
	# Copyright 2020-2022 Arm Limited
	#
	# Licensed under the Apache License, Version 2.0 (the "License"); you may not
	# use this file except in compliance with the License. You may obtain a copy
	# of the License at:
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
	# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
	# License for the specific language governing permissions and limitations
	# under the License.
	# -----------------------------------------------------------------------------
	"""
	The ``astc_test_result_plot.py`` script consolidates all current sets of
	reference results into a single graphical plot.
	"""

	import re
	import os
	import sys

	import numpy as np
	import matplotlib.pyplot as plt

	import testlib.resultset as trs
	from collections import defaultdict as ddict


	def find_reference_results():
	"""
	Scrape the Test/Images directory for result CSV files and return an
	mapping of the result sets.

	Returns:
	Returns a three deep tree of dictionaries, with the final dict
	pointing at a `ResultSet` object. The hierarchy is:

	imageSet => quality => encoder => result
	"""
	scriptDir = os.path.dirname(__file__)
	imageDir = os.path.join(scriptDir, "Images")

	# Pattern for extracting useful data from the CSV file name
	filePat = re.compile(r"astc_reference-(.+)_(.+)_results\.csv")

	# Build a three level dictionary we can write into
	results = ddict(lambda: ddict(lambda: ddict()))

	# Final all CSVs, load them and store them in the dict tree
	for root, dirs, files in os.walk(imageDir):
	for name in files:
	match = filePat.match(name)
	if match:
	fullPath = os.path.join(root, name)

	encoder = match.group(1)
	quality = match.group(2)
	imageSet = os.path.basename(root)

	if imageSet not in ["Kodak", "Khronos", "HDRIHaven", "KodakSim"]:
	continue

	testRef = trs.ResultSet(imageSet)
	testRef.load_from_file(fullPath)

	results[imageSet][quality]["ref-%s" % encoder] = testRef

	return results


	def get_series(results, tgtEncoder, tgtQuality, resFilter=lambda x: True):
	psnrData = []
	mtsData = []
	marker = []
	records = []

	for imageSet, iResults in results.items():

	for quality, qResults in iResults.items():
	if quality != tgtQuality:
	continue

	for encoder, eResults in qResults.items():
	if encoder != tgtEncoder:
	continue

	for record in eResults.records:
	if resFilter(record):
	records.append(record)
	psnrData.append(record.psnr)
	mtsData.append(record.cRate)

	if "ldr-xy" in record.name:
	marker.append('$N$')
	elif "ldr-l" in record.name:
	marker.append('$G$')
	elif "ldr" in record.name:
	marker.append('$L$')
	elif "hdr" in record.name:
	marker.append('$H$')
	else:
	marker.append('$?$')


	return mtsData, psnrData, marker, records


	def get_series_rel(results, refEncoder, refQuality, tgtEncoder, tgtQuality, resFilter=lambda x: True):

	mts1, psnr1, marker1, rec1 = get_series(results, tgtEncoder, tgtQuality, resFilter)

	if refEncoder is None:
	refEncoder = tgtEncoder

	if refQuality is None:
	refQuality = tgtQuality

	mts2, psnr2, marker2, rec2 = get_series(results, refEncoder, refQuality, resFilter)

	mtsm = [x/mts2[i] for i, x in enumerate(mts1)]
	psnrm = [x - psnr2[i] for i, x in enumerate(psnr1)]

	return mtsm, psnrm, marker1, rec1


	def get_human_eq_name(encoder, quality):
	parts = encoder.split("-")
	if len(parts) == 2:
	return "astcenc %s -%s" % (parts[1], quality)
	else:
	return "astcenc-%s %s -%s" % (parts[2], parts[1], quality)


	def get_human_e_name(encoder):
	parts = encoder.split("-")
	if len(parts) == 2:
	return "astcenc %s" % parts[1]
	else:
	return "astcenc-%s %s" % (parts[2], parts[1])


	def get_human_q_name(quality):
	return "-%s" % quality


	def plot(results, chartRows, chartCols, blockSizes,
	relative, pivotEncoder, pivotQuality, fileName, limits):

	fig, axs = plt.subplots(nrows=len(chartRows), ncols=len(chartCols),
	sharex=True, sharey=True, figsize=(15, 8.43))

	for a in fig.axes:
	a.tick_params(
	axis="x", which="both",
	bottom=True, top=False, labelbottom=True)

	a.tick_params(
	axis="y", which="both",
	left=True, right=False, labelleft=True)

	for i, row in enumerate(chartRows):
	for j, col in enumerate(chartCols):
	if row == "fastest" and (("1.7" in col) or ("2.0" in col)):
	if len(chartCols) == 1:
	fig.delaxes(axs[i])
	else:
	fig.delaxes(axs[i][j])
	continue

	if len(chartRows) == 1 and len(chartCols) == 1:
	ax = axs
	elif len(chartCols) == 1:
	ax = axs[i]
	else:
	ax = axs[i, j]

	title = get_human_eq_name(col, row)

	if not relative:
	ax.set_title(title, y=0.97, backgroundcolor="white")
	ax.set_xlabel('Coding performance (MTex/s)')
	ax.set_ylabel('PSNR (dB)')
	else:
	if pivotEncoder and pivotQuality:
	tag = get_human_eq_name(pivotEncoder, pivotQuality)
	elif pivotEncoder:
	tag = get_human_e_name(pivotEncoder)
	else:
	assert(pivotQuality)
	tag = get_human_q_name(pivotQuality)

	ax.set_title("%s vs. %s" % (title, tag), y=0.97, backgroundcolor="white")
	ax.set_xlabel('Performance scaling')
	ax.set_ylabel('PSNR delta (dB)')

	for k, series in enumerate(blockSizes):
	fn = lambda x: x.blkSz == series

	if not relative:
	x, y, m, r = get_series(results, col, row, fn)
	else:
	x, y, m, r = get_series_rel(results, pivotEncoder, pivotQuality,
	col, row, fn)

	color = None
	label = "%s blocks" % series
	for xp, yp, mp in zip(x, y, m):
	ax.scatter([xp],[yp], s=16, marker=mp,
	color="C%u" % k, label=label)
	label = None

	if i == 0 and j == 0:
	ax.legend(loc="lower right")

	for i, row in enumerate(chartRows):
	for j, col in enumerate(chartCols):

	if len(chartRows) == 1 and len(chartCols) == 1:
	ax = axs
	elif len(chartCols) == 1:
	ax = axs[i]
	else:
	ax = axs[i, j]

	ax.grid(ls=':')

	if limits and limits[0]:
	ax.set_xlim(left=limits[0][0], right=limits[0][1])
	if limits and limits[1]:
	ax.set_ylim(bottom=limits[1][0], top=limits[1][1])

	fig.tight_layout()
	fig.savefig(fileName)


	def main():
	"""
	The main function.

	Returns:
	int: The process return code.
	"""
	absXMin = 0
	absXMax = 80
	absXLimits = (absXMin, absXMax)

	relXMin = 0.8
	relXMax = None
	relXLimits = (relXMin, relXMax)

	last1x = "1.7"
	last2x = "2.5"
	last3x = "3.7"
	prev4x = "4.3"
	last4x = "4.4"
	lastMain = "main"

	charts = [
	# --------------------------------------------------------
	# Latest in stable series charts
	[
	# Relative scores
	["thorough", "medium", "fast"],
	[f"ref-{last2x}-avx2", f"ref-{last3x}-avx2", f"ref-{last4x}-avx2"],
	["4x4", "6x6", "8x8"],
	True,
	f"ref-{last1x}",
	None,
	"results-relative-stable-series.png",
	(None, None)
	], [
	# Absolute scores
	["thorough", "medium", "fast"],
	[f"ref-{last1x}", f"ref-{last2x}-avx2", f"ref-{last3x}-avx2", f"ref-{last4x}-avx2"],
	["4x4", "6x6", "8x8"],
	False,
	None,
	None,
	"results-absolute-stable-series.png",
	(absXLimits, None)
	],
	# --------------------------------------------------------
	# Latest 2.x vs 1.x release charts
	[
	# Relative scores
	["thorough", "medium", "fast"],
	[f"ref-{last2x}-avx2"],
	["4x4", "6x6", "8x8"],
	True,
	f"ref-{last1x}",
	None,
	"results-relative-2.x-vs-1.x.png",
	(None, None)
	],
	# --------------------------------------------------------
	# Latest 3.x vs 1.x release charts
	[
	# Relative scores
	["thorough", "medium", "fast"],
	[f"ref-{last3x}-avx2"],
	["4x4", "6x6", "8x8"],
	True,
	f"ref-{last1x}",
	None,
	"results-relative-3.x-vs-1.x.png",
	(None, None)
	],
	# --------------------------------------------------------
	# Latest 4.x vs 1.x release charts
	[
	# Relative scores
	["thorough", "medium", "fast"],
	[f"ref-{last4x}-avx2"],
	["4x4", "6x6", "8x8"],
	True,
	f"ref-{last1x}",
	None,
	"results-relative-4.x-vs-1.x.png",
	(None, None)
	],
	# --------------------------------------------------------
	# Latest 3.x vs 2.x release charts
	[
	# Relative scores
	["thorough", "medium", "fast", "fastest"],
	[f"ref-{last3x}-avx2"],
	["4x4", "6x6", "8x8"],
	True,
	f"ref-{last2x}-avx2",
	None,
	"results-relative-3.x-vs-2.x.png",
	(None, None)
	],
	# --------------------------------------------------------
	# Latest 4.x vs 3.x release charts
	[
	# Relative scores
	["thorough", "medium", "fast", "fastest"],
	[f"ref-{last4x}-avx2"],
	["4x4", "6x6", "8x8"],
	True,
	f"ref-{last3x}-avx2",
	None,
	"results-relative-4.x-vs-3.x.png",
	(relXLimits, None),
	], [
	# Relative ISAs of latest
	["thorough", "medium", "fast", "fastest"],
	[f"ref-{last4x}-sse4.1", f"ref-{last4x}-avx2"],
	["4x4", "6x6", "8x8"],
	True,
	f"ref-{last4x}-sse2",
	None,
	"results-relative-4.x-isa.png",
	(None, None)
	], [
	# Relative quality of latest
	["medium", "fast", "fastest"],
	[f"ref-{last4x}-avx2"],
	["4x4", "6x6", "8x8"],
	True,
	None,
	"thorough",
	"results-relative-4.x-quality.png",
	(None, None)
	],
	# --------------------------------------------------------
	# Latest 4.x vs previous 4.x release charts
	[
	# Relative scores
	["thorough", "medium", "fast", "fastest"],
	[f"ref-{last4x}-avx2"],
	["4x4", "6x6", "8x8"],
	True,
	f"ref-{prev4x}-avx2",
	None,
	"results-relative-4.x-vs-4.x.png",
	(relXLimits, None)
	],
	# --------------------------------------------------------
	# Latest 4.x vs previous 4.x release charts
	[
	# Relative scores
	["thorough", "medium", "fast", "fastest"],
	[f"ref-{lastMain}-avx2"],
	["4x4", "6x6", "8x8"],
	True,
	f"ref-{last4x}-avx2",
	None,
	"results-relative-main-vs-4.x.png",
	(relXLimits, None)
	]
	]

	results = find_reference_results()

	# Force select is triggered by adding a trailing entry to the argument list
	# of the charts that you want rendered; designed for debugging use cases
	maxIndex = 0
	expectedLength = 8
	for chart in charts:
	maxIndex = max(maxIndex, len(chart))

	for chart in charts:
	# If force select is enabled then only keep the forced ones
	if len(chart) != maxIndex:
	print("Skipping %s" % chart[6])
	continue
	else:
	print("Generating %s" % chart[6])

	# If force select is enabled then strip the dummy force option
	if maxIndex != expectedLength:
	chart = chart[:expectedLength]

	plot(results, *chart)

	return 0


	if __name__ == "__main__":
	sys.exit(main())