Download third_party/ktx-src/tools/ktx/command_compare.cpp from kimjunhan/vkgs-sim: direct link, hf CLI and curl.
- Browser
- Download file 118 kB
-
https://huggingface.co/kimjunhan/vkgs-sim/resolve/main/third_party/ktx-src/tools/ktx/command_compare.cpp
- Command line
-
hf download hf://kimjunhan/vkgs-sim/third_party/ktx-src/tools/ktx/command_compare.cpp
-
curl -L -o command_compare.cpp https://huggingface.co/kimjunhan/vkgs-sim/resolve/main/third_party/ktx-src/tools/ktx/command_compare.cpp
118 kB
// Copyright 2023-2024 The Khronos Group Inc.
// Copyright 2023-2024 RasterGrid Kft.
// SPDX-License-Identifier: Apache-2.0
#include "command.h"
#include "sbufstream.h"
#include "stdafx.h"
#include "utility.h"
#include "validate.h"
#include "formats.h"
#include "format_descriptor.h"
#include "imagecodec.hpp"
#include "imagespan.hpp"
#include "basis_sgd.h"
#include <fstream>
#include <iostream>
#include <sstream>
#include <utility>
#include <iomanip>
#include <map>
#include <cxxopts.hpp>
#include <fmt/printf.h>
#include <ktx.h>
#include <ktxint.h>
// -------------------------------------------------------------------------------------------------
namespace ktx {
extern "C" {
const char* ktxBUImageFlagsBitString(ktx_uint32_t bit_index, bool bit_value);
}
template <typename T>
struct DiffBase {
DiffBase(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const std::optional<T>& value1, const std::optional<T>& value2)
: textHeader(textHeaderIn), jsonPath(jsonPathIn), values(), different(value1 != value2) {
values[0] = value1;
values[1] = value2;
}
DiffBase(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const T& value1, const T& value2)
: textHeader(textHeaderIn), jsonPath(jsonPathIn), values(), different(value1 != value2) {
values[0] = value1;
values[1] = value2;
}
const std::string_view textHeader;
const std::string_view jsonPath;
bool isDifferent() const {
return different;
}
bool hasValue(std::size_t index) const {
return values[index].has_value();
}
virtual std::string value(std::size_t index, OutputFormat format) const = 0;
protected:
const T& rawValue(std::size_t index) const {
return *values[index];
}
private:
std::optional<T> values[2];
const bool different;
};
template <typename T>
struct Diff : public DiffBase<T> {
Diff(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const std::optional<T>& value1, const std::optional<T>& value2)
: DiffBase<T>(textHeaderIn, jsonPathIn, value1, value2) {}
Diff(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const T& value1, const T& value2)
: DiffBase<T>(textHeaderIn, jsonPathIn, value1, value2) {}
virtual std::string value(std::size_t index, OutputFormat) const override {
return fmt::format("{}", DiffBase<T>::rawValue(index));
}
};
struct DiffIdentifier : public DiffBase<std::array<uint8_t, sizeof(KTX_header2::identifier)>> {
DiffIdentifier(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const KTX_header2& value1, const KTX_header2& value2)
: DiffBase<std::array<uint8_t, sizeof(KTX_header2::identifier)>>(textHeaderIn, jsonPathIn,
to_array(value1), to_array(value2)) {
}
virtual std::string value(std::size_t index, OutputFormat format) const override {
// Convert identifier for better display.
auto identifier = rawValue(index);
uint32_t idlen = 0;
char u8identifier[30];
for (uint32_t i = 0; i < 12 && idlen < sizeof(u8identifier); i++, idlen++) {
// Convert the angle brackets to utf-8 for better printing. The
// conversion below only works for characters whose msb's are 10.
if (identifier[i] == U'\xAB') {
u8identifier[idlen++] = '\xc2';
u8identifier[idlen] = identifier[i];
} else if (identifier[i] == U'\xBB') {
u8identifier[idlen++] = '\xc2';
u8identifier[idlen] = identifier[i];
} else if (identifier[i] < '\x20') {
uint32_t nchars;
switch (identifier[i]) {
case '\n':
u8identifier[idlen++] = '\\';
u8identifier[idlen] = 'n';
break;
case '\r':
u8identifier[idlen++] = '\\';
u8identifier[idlen] = 'r';
break;
default:
nchars = snprintf(&u8identifier[idlen],
sizeof(u8identifier) - idlen,
format == OutputFormat::text ? "\\x%02X" : "\\u%04X",
identifier[i]);
idlen += nchars - 1;
}
} else {
u8identifier[idlen] = identifier[i];
}
}
if (format == OutputFormat::text) {
return std::string(u8identifier, idlen);
} else {
return fmt::format("\"{}\"", std::string(u8identifier, idlen));
}
}
private:
static std::array<uint8_t, sizeof(KTX_header2::identifier)> to_array(const KTX_header2& header) {
std::array<uint8_t, sizeof(KTX_header2::identifier)> arr{};
for (std::size_t i = 0; i < sizeof(KTX_header2::identifier); ++i) {
arr[i] = header.identifier[i];
}
return arr;
}
};
template <typename T>
struct DiffHex : public DiffBase<T> {
DiffHex(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const std::optional<T>& value1, const std::optional<T>& value2)
: DiffBase<T>(textHeaderIn, jsonPathIn, value1, value2) {}
DiffHex(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const T& value1, const T& value2)
: DiffBase<T>(textHeaderIn, jsonPathIn, value1, value2) {}
virtual std::string value(std::size_t index, OutputFormat format) const override {
if (format == OutputFormat::text) {
return fmt::format("0x{:x}", DiffBase<T>::rawValue(index));
} else {
return fmt::format("{}", DiffBase<T>::rawValue(index));
}
}
};
template <typename T>
struct DiffHexFixedWidth : public DiffBase<T> {
DiffHexFixedWidth(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const std::optional<T>& value1, const std::optional<T>& value2)
: DiffBase<T>(textHeaderIn, jsonPathIn, value1, value2) {}
DiffHexFixedWidth(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const T& value1, const T& value2)
: DiffBase<T>(textHeaderIn, jsonPathIn, value1, value2) {}
virtual std::string value(std::size_t index, OutputFormat format) const override {
if (format == OutputFormat::text) {
return fmt::format(fmt::format("0x{{:0{}x}}", sizeof(T) << 1), DiffBase<T>::rawValue(index));
} else {
return fmt::format("{}", DiffBase<T>::rawValue(index));
}
}
};
template <typename T>
struct DiffEnum : public DiffBase<T> {
DiffEnum(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const std::optional<int32_t>& value1, const std::optional<int32_t>& value2,
const std::function<const char*(std::size_t)>& strFunc)
: DiffBase<T>(textHeaderIn, jsonPathIn,
value1.has_value() ? T(*value1) : std::optional<T>(),
value2.has_value() ? T(*value2) : std::optional<T>()) {
if (value1.has_value()) enumNames[0] = strFunc(0);
if (value2.has_value()) enumNames[1] = strFunc(1);
}
DiffEnum(const std::string_view textHeaderIn, const std::string_view jsonPathIn, const int32_t& value1, const int32_t& value2,
const std::function<const char*(std::size_t)>& strFunc)
: DiffBase<T>(textHeaderIn, jsonPathIn, T(value1), T(value2)) {
enumNames[0] = strFunc(0);
enumNames[1] = strFunc(1);
}
virtual std::string value(std::size_t index, OutputFormat format) const override {
if (format == OutputFormat::text) {
if (enumNames[index]) {
if (hexInText) {
return fmt::format("0x{:x} ({})", uint32_t(DiffBase<T>::rawValue(index)), enumNames[index]);
} else {
return enumNames[index];
}
} else {
return fmt::format("0x{:x}", uint32_t(DiffBase<T>::rawValue(index)));
}
} else {
if (enumNames[index]) {
return fmt::format("\"{}\"", enumNames[index]);
} else {
return fmt::format("{}", int32_t(DiffBase<T>::rawValue(index)));
}
}
}
DiffEnum<T>& outputHexInText() {
hexInText = true;
return *this;
}
private:
bool hexInText = false;
const char* enumNames[2] = { nullptr, nullptr };
};
template <>
struct DiffEnum<ktxSupercmpScheme> : public DiffBase<ktxSupercmpScheme> {
DiffEnum(const std::string_view textHeaderIn, const std::string_view jsonPathIn, const uint32_t& value1, const uint32_t& value2)
: DiffBase<ktxSupercmpScheme>(textHeaderIn, jsonPathIn, ktxSupercmpScheme(value1), ktxSupercmpScheme(value2)) {
enumNames[0] = ktxSupercompressionSchemeString(ktxSupercmpScheme(value1));
enumNames[1] = ktxSupercompressionSchemeString(ktxSupercmpScheme(value2));
}
virtual std::string value(std::size_t index, OutputFormat format) const override {
bool invalidScheme = (strcmp(enumNames[index], "Invalid scheme value") == 0);
bool vendorOrReservedScheme = (strcmp(enumNames[index], "Vendor or reserved scheme") == 0);
if (format == OutputFormat::text) {
if (enumNames[index]) {
if (invalidScheme)
return fmt::format("Invalid scheme (0x{:x})", uint32_t(rawValue(index)));
else if (vendorOrReservedScheme)
return fmt::format("Vendor or reserved scheme (0x{:x})", uint32_t(rawValue(index)));
else
return enumNames[index];
} else {
return fmt::format("0x{:x}", uint32_t(rawValue(index)));
}
} else {
if (enumNames[index] && !invalidScheme && !vendorOrReservedScheme) {
return fmt::format("\"{}\"", enumNames[index]);
} else {
return fmt::format("{}", int32_t(rawValue(index)));
}
}
}
private:
const char* enumNames[2] = { nullptr, nullptr };
};
struct DiffFlags : DiffBase<uint32_t> {
DiffFlags(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const std::optional<uint32_t>& value1, const std::optional<uint32_t>& value2,
const char*(*bitToString)(uint32_t, bool))
: DiffBase<uint32_t>(textHeaderIn, jsonPathIn, value1, value2), toStringFn(bitToString) {}
DiffFlags(const std::string_view textHeaderIn, const std::string_view jsonPathIn, const uint32_t& value1, const uint32_t& value2,
const char*(*bitToString)(uint32_t, bool))
: DiffBase<uint32_t>(textHeaderIn, jsonPathIn, value1, value2), toStringFn(bitToString) {}
virtual std::string value(std::size_t index, OutputFormat format) const override {
const auto space = format != OutputFormat::json_mini ? " " : "";
const auto quote = format == OutputFormat::text ? "" : "\"";
std::stringstream formattedValue;
bool first = true;
std::string comma = "";
for (uint32_t bitIndex = 0; bitIndex < 32; ++bitIndex) {
uint32_t bitMask = 1u << bitIndex;
bool bitValue = (bitMask & DiffBase<uint32_t>::rawValue(index)) != 0;
const char* bitStr = toStringFn(bitIndex, bitValue);
if (bitStr) {
fmt::print(formattedValue, "{}{}{}{}", comma, quote, bitStr, quote);
} else if (bitValue) {
fmt::print(formattedValue, "{}{}", comma, bitMask);
}
if (bitValue && first) {
first = false;
comma = fmt::format(",{}", space);
}
}
if (format == OutputFormat::text) {
return fmt::format("0x{:x} ({})", DiffBase<uint32_t>::rawValue(index), formattedValue.str());
} else {
if (formattedValue.str().empty()) {
return "[]";
} else {
return fmt::format("[{}{}{}]", space, formattedValue.str(), space);
}
}
}
private:
const char*(*toStringFn)(uint32_t, bool);
};
template <typename T, std::size_t N>
struct DiffArray : public DiffBase<std::array<T, N>> {
DiffArray(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const std::optional<std::array<T, N>>& value1, const std::optional<std::array<T, N>>& value2)
: DiffBase<std::array<T, N>>(textHeaderIn, jsonPathIn, value1, value2) {}
DiffArray(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const std::array<T, N>& value1, const std::array<T, N>& value2)
: DiffBase<std::array<T, N>>(textHeaderIn, jsonPathIn, value1, value2) {}
virtual std::string value(std::size_t index, OutputFormat format) const override {
const auto space = format != OutputFormat::json_mini ? " " : "";
std::stringstream formattedValue;
bool first = true;
for (std::size_t arrayIndex = 0; arrayIndex < N; ++arrayIndex) {
const auto comma = std::exchange(first, false) ? "" : fmt::format(",{}", space);
fmt::print(formattedValue, "{}{}", comma, DiffBase<std::array<T, N>>::rawValue(index)[arrayIndex]);
}
if (format == OutputFormat::text) {
return formattedValue.str();
} else {
if (formattedValue.str().empty()) {
return "[]";
} else {
return fmt::format("[{}{}{}]", space, formattedValue.str(), space);
}
}
}
};
struct DiffRawBytes : public DiffBase<std::vector<uint8_t>> {
DiffRawBytes(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const std::optional<std::vector<uint8_t>>& value1, const std::optional<std::vector<uint8_t>>& value2)
: DiffBase<std::vector<uint8_t>>(textHeaderIn, jsonPathIn, value1, value2) {}
DiffRawBytes(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const std::vector<uint8_t>& value1, const std::vector<uint8_t>& value2)
: DiffBase<std::vector<uint8_t>>(textHeaderIn, jsonPathIn, value1, value2) {}
virtual std::string value(std::size_t index, OutputFormat format) const override {
const auto space = format != OutputFormat::json_mini ? " " : "";
std::stringstream formattedValue;
bool first = true;
for (std::size_t arrayIndex = 0; arrayIndex < rawValue(index).size(); ++arrayIndex) {
const auto comma = std::exchange(first, false) ? "" : fmt::format(",{}", space);
if (format == OutputFormat::text) {
fmt::print(formattedValue, "{}0x{:x}", comma, rawValue(index)[arrayIndex]);
} else {
fmt::print(formattedValue, "{}{}", comma, rawValue(index)[arrayIndex]);
}
}
if (format == OutputFormat::text) {
return fmt::format("[{}]", formattedValue.str());
} else {
if (formattedValue.str().empty()) {
return "[]";
} else {
return fmt::format("[{}{}{}]", space, formattedValue.str(), space);
}
}
}
};
template <typename T>
struct DiffComplex {
DiffComplex(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const std::optional<T>& value1, const std::optional<T>& value2)
: textHeader(textHeaderIn), jsonPath(jsonPathIn), values(),
different((value1.has_value() && value2.has_value())
? value1->isDifferent(*value2)
: (!value1.has_value() || !value2.has_value()) ? true : false) {
values[0] = value1;
values[1] = value2;
}
DiffComplex(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const T& value1, const T& value2)
: textHeader(textHeaderIn), jsonPath(jsonPathIn), values(), different(value1.different(value2)) {
values[0] = value1;
values[1] = value2;
}
const std::string_view textHeader;
const std::string_view jsonPath;
bool isDifferent() const {
return different;
}
bool hasValue(std::size_t index) const {
return values[index].has_value();
}
void printText(std::size_t index, PrintIndent& out, const char* prefix) const {
values[index]->printText(out, prefix);
}
void printJson(std::size_t index, PrintIndent& out, int indent, const char* space, const char* nl) const {
values[index]->printJson(out, indent, space, nl);
}
private:
std::optional<T> values[2];
const bool different;
};
struct DiffTextCustom {
DiffTextCustom(const std::optional<std::string>& text1, const std::optional<std::string>& text2)
: texts(), different(text1 != text2) {
texts[0] = text1;
texts[1] = text2;
}
DiffTextCustom(std::string&& text1, std::string&& text2)
: texts(), different(text1 != text2) {
texts[0] = text1;
texts[1] = text2;
}
bool isDifferent() const {
return different;
}
bool hasText(std::size_t index) const {
return texts[index].has_value();
}
const std::string& text(std::size_t index) const {
return *texts[index];
}
private:
std::optional<std::string> texts[2];
const bool different;
};
// Helper used to report a mismatch without actual values to include in the report
struct DiffMismatch {
DiffMismatch(const std::string_view textMsgIn, const std::string_view jsonPathIn)
: textMsg(textMsgIn), jsonPath(jsonPathIn) {}
const std::string_view textMsg;
const std::string_view jsonPath;
};
struct DiffImage {
using TexelBlockPairList = std::vector<std::pair<ImageSpan::TexelBlockPtr<>, ImageSpan::TexelBlockPtr<>>>;
DiffImage(const std::string_view textHeaderIn, const std::string_view fragmentUriIn,
const std::optional<std::size_t>& fileOffset1, const std::optional<std::size_t>& fileOffset2,
const TexelBlockPairList& texelBlockPairListIn)
: textHeader(textHeaderIn), fragmentUri(fragmentUriIn), texelBlockPairList(texelBlockPairListIn) {
fileOffsets[0] = fileOffset1;
fileOffsets[1] = fileOffset2;
}
const std::string_view textHeader;
const std::string_view fragmentUri;
const TexelBlockPairList& texelBlockPairList;
std::optional<std::size_t> fileOffsets[2];
};
class PrintDiff {
PrintIndent& printIndent;
OutputFormat outputFormat;
std::optional<std::string> jsonSection = {};
bool jsonSectionEmpty = false;
bool firstContext = true;
std::vector<std::string> context = {};
bool different = false;
void printContext() {
if (!context.empty()) {
if (!std::exchange(firstContext, false))
printIndent(0, "\n");
for (const auto& ctx : context)
printIndent(0, ctx);
context.clear();
}
}
void beginJsonOutput() {
assert(jsonSection.has_value());
const auto space = outputFormat != OutputFormat::json_mini ? " " : "";
const auto nl = outputFormat != OutputFormat::json_mini ? "\n" : "";
printIndent(0, ",{}", nl);
if (std::exchange(jsonSectionEmpty, false))
printIndent(1, "\"{}\":{}{{{}", *jsonSection, space, nl);
}
public:
PrintDiff(PrintIndent& output, OutputFormat format)
: printIndent(output), outputFormat(format) {}
bool isDifferent() const {
return different;
}
void beginJsonSection(std::string&& section) {
jsonSection = section;
jsonSectionEmpty = true;
}
void endJsonSection() {
if (jsonSection.has_value()) {
const auto nl = outputFormat != OutputFormat::json_mini ? "\n" : "";
if (!jsonSectionEmpty) {
printIndent(0, "{}", nl);
printIndent(1, "}}");
}
jsonSection.reset();
}
}
void setContext(std::string&& ctx) {
context.clear();
context.emplace_back(std::move(ctx));
}
void addContext(std::string&& ctx) {
context.emplace_back(std::move(ctx));
}
void updateContext(std::string&& ctx) {
if (!context.empty()) context.pop_back();
context.emplace_back(std::move(ctx));
}
template <typename T>
void operator<<(const DiffBase<T>& diff) {
if (!diff.isDifferent()) return;
different = true;
const auto space = outputFormat != OutputFormat::json_mini ? " " : "";
const auto nl = outputFormat != OutputFormat::json_mini ? "\n" : "";
if (outputFormat == OutputFormat::text) {
printContext();
if (diff.hasValue(0))
printIndent(0, "-{}: {}\n", diff.textHeader, diff.value(0, outputFormat));
if (diff.hasValue(1))
printIndent(0, "+{}: {}\n", diff.textHeader, diff.value(1, outputFormat));
} else {
beginJsonOutput();
printIndent(2, "\"{}\":{}[{}", diff.jsonPath, space, nl);
if (diff.hasValue(0))
printIndent(3, "{},{}", diff.value(0, outputFormat), nl);
else
printIndent(3, "null,{}", nl);
if (diff.hasValue(1))
printIndent(3, "{}{}", diff.value(1, outputFormat), nl);
else
printIndent(3, "null{}", nl);
printIndent(2, "]");
}
}
template <typename T>
void operator<<(const DiffComplex<T>& diff) {
if (!diff.isDifferent()) return;
different = true;
const auto space = outputFormat != OutputFormat::json_mini ? " " : "";
const auto nl = outputFormat != OutputFormat::json_mini ? "\n" : "";
if (outputFormat == OutputFormat::text) {
printContext();
if (diff.hasValue(0)) {
printIndent(0, "-{}:", diff.textHeader);
diff.printText(0, printIndent, "-");
}
if (diff.hasValue(1)) {
printIndent(0, "+{}:", diff.textHeader);
diff.printText(1, printIndent, "+");
}
} else {
beginJsonOutput();
printIndent(2, "\"{}\":{}[{}", diff.jsonPath, space, nl);
if (diff.hasValue(0)) {
diff.printJson(0, printIndent, 3, space, nl);
printIndent(0, ",{}", nl);
} else
printIndent(3, "null,{}", nl);
if (diff.hasValue(1)) {
diff.printJson(1, printIndent, 3, space, nl);
printIndent(0, "{}", nl);
} else
printIndent(3, "null{}", nl);
printIndent(2, "]");
}
}
void operator<<(const DiffTextCustom& diff) {
if (!diff.isDifferent()) return;
different = true;
assert(outputFormat == OutputFormat::text);
if (diff.hasText(0))
printIndent(0, "-{}\n", diff.text(0));
if (diff.hasText(1))
printIndent(0, "+{}\n", diff.text(1));
}
void operator<<(const DiffMismatch& diff) {
different = true;
const auto space = outputFormat != OutputFormat::json_mini ? " " : "";
if (outputFormat == OutputFormat::text) {
printIndent(0, "+{}\n", diff.textMsg);
} else {
beginJsonOutput();
printIndent(2, "\"{}\":{}[]", diff.jsonPath, space);
}
}
void operator<<(const DiffImage& diff) {
different = true;
const auto space = outputFormat != OutputFormat::json_mini ? " " : "";
const auto nl = outputFormat != OutputFormat::json_mini ? "\n" : "";
auto formatOptionalFileOffset = [](const std::optional<std::size_t>& fileOffset, std::size_t imageOffset, bool json) {
if (fileOffset.has_value())
return fmt::format(json ? "{}" : "0x{:x}", *fileOffset + imageOffset);
else
return fmt::format(json ? "null" : "N/A");
};
auto formatPacked = [=](const ImageSpan::TexelBlockPtr<>& texelBlock, bool json) {
const uint32_t hexDigits = texelBlock.getPackedElementByteSize() << 1;
const auto quote = json ? "\"" : "";
std::stringstream formattedValue;
bool first = true;
for (uint32_t elementIdx = 0; elementIdx < texelBlock.getPackedElementCount(); ++elementIdx) {
const uint32_t element = texelBlock.getPackedElement(elementIdx);
const auto comma = std::exchange(first, false) ? "" : fmt::format(",{}", space);
fmt::print(formattedValue, "{}{}0x{:0{}x}{}", comma, quote, element, hexDigits, quote);
}
return formattedValue.str();
};
auto formatChannels = [=](const ImageSpan::TexelBlockPtr<>& texelBlock, bool json) {
// If decodable channels are not available (e.g. block compressed), then this should not be called
assert(texelBlock.getChannelCount() != 0);
const auto quote = json ? "\"" : "";
std::stringstream formattedValue;
bool first = true;
// Prefer to decode to integer (e.g. UNORM will be output as list of integer values instead of float values)
if (texelBlock.canDecodeUINT()) {
// Unsigned integer channels
const glm::uvec4 channels = texelBlock.decodeUINT();
for (uint32_t channelIdx = 0; channelIdx < texelBlock.getChannelCount(); ++channelIdx) {
const auto comma = std::exchange(first, false) ? "" : fmt::format(",{}", space);
fmt::print(formattedValue, "{}{}", comma, channels[channelIdx]);
}
} else if (texelBlock.canDecodeSINT()) {
// Signed integer channels
const glm::ivec4 channels = texelBlock.decodeSINT();
for (uint32_t channelIdx = 0; channelIdx < texelBlock.getChannelCount(); ++channelIdx) {
const auto comma = std::exchange(first, false) ? "" : fmt::format(",{}", space);
fmt::print(formattedValue, "{}{}", comma, channels[channelIdx]);
}
} else if (texelBlock.canDecodeFLOAT()) {
// Floating point channels
const glm::vec4 channels = texelBlock.decodeFLOAT();
for (uint32_t channelIdx = 0; channelIdx < texelBlock.getChannelCount(); ++channelIdx) {
const auto comma = std::exchange(first, false) ? "" : fmt::format(",{}", space);
float channel = channels[channelIdx];
if (std::isinf(channel)) {
// Output signed infinity (optionally quoted)
const auto sign = std::signbit(channel) ? "-" : "+";
fmt::print(formattedValue, "{}{}{}inf{}", comma, quote, sign, quote);
} else if (std::isnan(channel)) {
// Output signed not-a-number (optionally quoted)
fmt::print(formattedValue, "{}{}nan{}", comma, quote, quote);
} else {
// Output signed value (explicitly handle sign bit to differentiate between +0.0 and -0.0)
// However, we only output sign for negative values because JSON does not allow leading "+"
const auto sign = std::signbit(channel) ? "-" : "";
fmt::print(formattedValue, "{}{}{}", comma, sign, std::abs(channel));
}
}
} else {
// Unexpected format
assert(false);
return std::string("");
}
return formattedValue.str();
};
if (outputFormat == OutputFormat::text) {
printContext();
printIndent(0, "+{}\n", diff.textHeader);
for (const auto& texelBlockPair : diff.texelBlockPairList) {
const auto pixelCoords = texelBlockPair.first.getPixelLocation();
// Currently we only compare texel blocks with the same coordinates
assert(pixelCoords == texelBlockPair.second.getPixelLocation());
printIndent(0, " Coordinates: {}, {}, {}\n", pixelCoords.x, pixelCoords.y, pixelCoords.z);
const auto imageByteOffset = texelBlockPair.first.getTexelBlockByteOffset();
// Currently we only compare matching formats, hence image byte offsets should match
assert(imageByteOffset == texelBlockPair.second.getTexelBlockByteOffset());
printIndent(0, " Image byte offset: 0x{:x}\n", imageByteOffset);
auto printDiff = [=](const char* textHeader, const std::string firstValue, const std::string secondValue) {
if (firstValue == secondValue) {
printIndent(0, " {}: {}\n", textHeader, firstValue);
} else {
printIndent(0, "- {}: {}\n", textHeader, firstValue);
printIndent(0, "+ {}: {}\n", textHeader, secondValue);
}
};
printDiff("File byte offset",
formatOptionalFileOffset(diff.fileOffsets[0], imageByteOffset, false),
formatOptionalFileOffset(diff.fileOffsets[1], imageByteOffset, false));
printDiff("Packed", formatPacked(texelBlockPair.first, false), formatPacked(texelBlockPair.second, false));
// Only output channels if not block-compressed
if (!texelBlockPair.first.isBlockCompressed())
printDiff("Channels", formatChannels(texelBlockPair.first, false), formatChannels(texelBlockPair.second, false));
}
} else {
beginJsonOutput();
printIndent(2, "\"{}\":{}[", diff.fragmentUri, space);
bool first = true;
for (const auto& texelBlockPair : diff.texelBlockPairList) {
const auto comma = std::exchange(first, false) ? "" : fmt::format(",{}", space);
printIndent(0, "{}{}", comma, nl);
printIndent(3, "{{{}", nl);
const auto pixelCoords = texelBlockPair.first.getPixelLocation();
// Currently we only compare texel blocks with the same coordinates
assert(pixelCoords == texelBlockPair.second.getPixelLocation());
printIndent(4, "\"coordinates\":{}[{}{},{}{},{}{}{}],{}", space, space,
pixelCoords.x, space, pixelCoords.y, space, pixelCoords.z, space, nl);
const auto imageByteOffset = texelBlockPair.first.getTexelBlockByteOffset();
// Currently we only compare matching formats, hence image byte offsets should match
assert(imageByteOffset == texelBlockPair.second.getTexelBlockByteOffset());
printIndent(4, "\"imageByteOffset\":{}[{}{},{}{}{}],{}", space, space,
imageByteOffset, space, imageByteOffset, space, nl);
printIndent(4, "\"fileByteOffset\":{}[{}{},{}{}{}],{}", space, space,
formatOptionalFileOffset(diff.fileOffsets[0], imageByteOffset, true), space,
formatOptionalFileOffset(diff.fileOffsets[1], imageByteOffset, true), space, nl);
printIndent(4, "\"packed\":{}[{}", space, nl);
printIndent(5, "[{}{}{}],{}", space, formatPacked(texelBlockPair.first, true), space, nl);
printIndent(5, "[{}{}{}]{}", space, formatPacked(texelBlockPair.second, true), space, nl);
printIndent(4, "]");
// Only output channels if not block-compressed
if (!texelBlockPair.first.isBlockCompressed()) {
printIndent(0, ",{}", nl);
printIndent(4, "\"channels\":{}[{}", space, nl);
printIndent(5, "[{}{}{}],{}", space, formatChannels(texelBlockPair.first, true), space, nl);
printIndent(5, "[{}{}{}]{}", space, formatChannels(texelBlockPair.second, true), space, nl);
printIndent(4, "]");
}
printIndent(0, "{}", nl);
printIndent(3, "}}");
}
if (diff.texelBlockPairList.empty()) {
printIndent(0, "]");
} else {
printIndent(0, "{}", nl);
printIndent(2, "]");
}
}
}
};
// -------------------------------------------------------------------------------------------------
/** @page ktx_compare ktx compare
@~English
Compare two KTX2 files.
@section ktx_compare_synopsis SYNOPSIS
ktx compare [option...] @e input-file1 @e input-file2
@section ktx_compare_description DESCRIPTION
@b ktx @b compare compares the two KTX2 files specified as the @e input-file1 and @e input-file2
arguments and outputs any mismatch in texture information and/or image data.
The command implicitly calls @ref ktx_validate "validate" and prints any found errors
and warnings to stdout.
If any of the specified input files are invalid then comparison is done based on best effort
and may be incomplete.
The JSON output formats conform to the https://schema.khronos.org/ktx/compare_v0.json
schema even if the input file is invalid and certain information cannot be parsed or
displayed.
Additionally, for JSON outputs the KTX file identifier is printed using "\u001A" instead of
"\x1A" as an unescaped "\x1A" sequence inside a JSON string breaks nearly every JSON tool.
Note that this does not change the value of the string only its representation.
@note @b ktx @b compare prints using UTF-8 encoding. If your console is not
set for UTF-8 you will see incorrect characters in output of the file
identifier on each side of the "KTX nn".
@section ktx\_compare\_options OPTIONS
The following options are available:
@snippet{doc} ktx/command.h command options_format
<dl>
<dt>\--content raw | image | ignore</dt>
<dd>Controls how image content is compared. Possible options are: <br />
@b raw - Encoded image data is compared verbatim, as it appears in the file. <br />
@b image - Effective image data is compared per texel block. <br />
@b ignore - Ignore image contents. <br />
The default mode is @b raw, meaning that the encoded image data must match exactly.
Note: When the mode is set to @b image, the effective image data of individual mip
levels, layers, and faces is compared texel block by texel block after applying
one or more of the following transformations, if needed: <br />
\- If the texture is supercompressed (e.g. using Zstandard or ZLIB), the images
are deflated before comparison. <br />
\- If the texture is encoded using BasisLZ, the images are transcoded to
R8G8B8A_UNORM before comparison. <br />
For block compressed textures and textures encoded using UASTC the texel blocks are
the individual compressed blocks, while for other formats the texel blocks are the
individual pixels of the image. <br />
When comparing files that use different supercompression schemes, or otherwise
different encoding, additional options may need to be used to avoid unexpected
differences to be reported related to the meta information of the files.
For example: <br />
\- The supercompression scheme can be ignored with --ignore-supercomp. <br />
\- Compressed byte length and other index section differences can be ignored
with --ignore-index all or --ignore-index level. <br />
\- DFD section differences can be ignored with --ignore-dfd all or --ignore-dfd
all-except-color-space. <br />
\- BDFD bytesPlane value differences can be ignored with --ignore-bdfd-bytesplane. <br />
\- SGD section differences can be ignored with --ignore-sgd all or --ignore-sgd
payload. <br />
</dd>
<dt>\--per-pixel-output all | <number> | none</dt>
<dd>Controls whether per pixel / texel block difference output is generated when
--content is set to @b image: <br />
@b all - Every single difference is output (may result in a very large output). <br />
@b <number> - At most the specified number of differences are output. <br />
@b none - No per pixel / texel block differences are output. <br />
The default mode is @b none to limit the verbosity of the output.
<dt>\--allow-invalid-input</dt>
<dd>Perform best effort comparison even if any of the input files are invalid.</dd>
<dt>\--ignore-format-header<dt>
<dd>Ignore the vkFormat and typeSize fields in the file header. <br />
Note: useful when comparing textures with and without BasisLZ encoding, respectively.</dd>
<dt>\--ignore-supercomp</dt>
<dd>Ignore supercompression scheme in the file header. <br />
Note: use the --ignore-sgd option to also ignore the SGD section, if needed.</dd>
<dt>\--ignore-index all | level | none</dt>
<dd>Controls the comparison of index entries in the file headers. Possible options are: <br />
@b all - Ignore all index entries. <br />
@b level - Ignore level index entries only. <br />
@b none - Do not ignore any index entries. <br />
The default mode is @b none, meaning that all index entries will be compared.
</dd>
<dt>\--ignore-dfd all | all-except-color-space | unknown | extended | none</dt>
<dd>Controls the comparison of DFD blocks. Possible options are: <br />
@b all - Ignore all DFD blocks. <br />
@b all-except-color-space: Ignore all DFD blocks except the color space information
in the basic DFD block. <br />
@b unknown - Ignore any unrecognized DFD blocks. <br />
@b extended - Ignore all DFD blocks except the basic DFD block. <br />
@b none - Do not ignore any DFD blocks. <br />
The default mode is @b none, meaning that all DFD entries will be compared.
</dd>
<dt>\--ignore-bdfd-bytesplane</dt>
<dd>Ignore BDFD bytesPlane values. Useful when comparing pre-spec-2.0.4 supercompressed
files with 2.0.4+ supercompressed or non-supercompressed ones, as bytesPlane is set to
zero in older supercompressed files.</dd>
<dt>\--ignore-metadata all | <key>[,...] | none</dt>
<dd>Controls the comparison of metadata (KVD) entries. Possible options are: <br />
@b all - Ignore all metadata entries. <br />
@b <key>[,...] - Ignore the specified comma separated list of metadata keys. <br />
@b none - Do not ignore any metadata entries. <br />
The default mode is @b none, meaning that all metadata entries will be compared.
</dd>
<dt>\--ignore-sgd all | unknown | payload | none</dt>
<dd>Controls the comparison of the SGD section. Possible options are: <br />
@b all - Ignore the SGD section. <br />
@b unknown - Ignore any unrecognized SGD section. <br />
@b payload - Ignore any unrecognized SGD section and the payload of any known SGD section. <br />
@b none - Do not ignore the SGD section. <br />
The default mode is @b none, meaning that SGD sections will be always compared.
Note: --ignore-sgd payload can be used to compare BasisLZ SGD headers without
expecting an exact match for the individual SGD payload sections.
</dd>
</dl>
@snippet{doc} ktx/command.h command options_generic
@note Reports about image content mismatches when --content is set to @b raw or @b image
include the following information: <br />
- The level, layer, and face index of the image where the difference was found.
@note Reports about individual texel block differences when --content is set to @b image
include the following additional information: <br />
- The texel coordinates within the image where the difference was found. These
coordinates are the raw coordinates of the image and are not affected by metadata
such as @b KTXorientation.
- The packed elements of the texel block output as an array of hexadecimal values.
Each packed element has a byte size equal to the @b typeSize value of the image.
- The channel values of the texel block output as an array of integer (for integer
and normalized formats) or floating point (for floating point formats) values.
In case of floating point formats @b NaN, @b +inf, and @b -inf are used to indicate
values that are not a number or represent infinities, respectively.
The output order of the channels always follow the order they appear in the BDFD.
@section ktx_compare_exitstatus EXIT STATUS
@snippet{doc} ktx/command.h command exitstatus
- 7 - Input files are different
@section ktx_compare_history HISTORY
@par Version 4.0
- Initial version
@section ktx_compare_author AUTHOR
- Daniel Rákos, RasterGrid www.rastergrid.com
*/
class CommandCompare : public Command {
enum class ContentMode {
raw,
image,
ignore
};
enum class IgnoreIndex {
all,
level,
none
};
enum class IgnoreDFD {
all,
all_except_color_space,
unknown,
extended,
none
};
enum class IgnoreSGD {
all,
unknown,
payload,
none
};
struct OptionsCompare {
inline static const char* kContent = "content";
inline static const char* kPerPixelOutput = "per-pixel-output";
inline static const char* kAllowInvalidInput = "allow-invalid-input";
inline static const char* kIgnoreFormatHeader = "ignore-format-header";
inline static const char* kIgnoreSupercomp = "ignore-supercomp";
inline static const char* kIgnoreIndex = "ignore-index";
inline static const char* kIgnoreDFD = "ignore-dfd";
inline static const char* kIgnoreBDFDBytesPlane = "ignore-bdfd-bytesplane";
inline static const char* kIgnoreMetadata = "ignore-metadata";
inline static const char* kIgnoreSGD = "ignore-sgd";
std::array<std::string, 2> inputFilepaths;
ContentMode contentMode = ContentMode::raw;
std::size_t perPixelOutputLimit = 0;
bool allowInvalidInput = false;
bool ignoreFormatHeader = false;
bool ignoreSupercomp = false;
IgnoreIndex ignoreIndex = IgnoreIndex::none;
IgnoreDFD ignoreDFD = IgnoreDFD::none;
bool ignoreBDFDBytesPlane = false;
bool ignoreAllMetadata = false;
std::unordered_set<std::string> ignoreMetadataKeys = {};
IgnoreSGD ignoreSGD = IgnoreSGD::none;
void init(cxxopts::Options& opts) {
opts.add_options()
("input-file1", "The first input file to compare.", cxxopts::value<std::string>(), "filepath")
("input-file2", "The second input file to compare.", cxxopts::value<std::string>(), "filepath")
(kContent, "Controls how image content is compared. Possible values are:\n"
" raw: Encoded image data is compared verbatim, as it appears in the file\n"
" image: Effective image data is compared per texel block\n"
" ignore: Ignore image contents\n"
"Note: When the mode is set to @b image, the effective image data of individual mip "
"levels, layers, and faces is compared texel block by texel block after applying "
"one or more of the following transformations, if needed:\n"
"- If the texture is supercompressed (e.g. using Zstandard or ZLIB), the images "
"are deflated before comparison.\n"
"- If the texture is encoded using BasisLZ, the images are transcoded to "
"R8G8B8A_UNORM before comparison.\n"
"For block compressed textures and textures encoded using UASTC the texel blocks are "
"the individual compressed blocks, while for other formats the texel blocks are the "
"individual pixels of the image.\n"
"When comparing files that use different supercompression schemes, or otherwise "
"different encoding, additional options may need to be used to avoid unexpected "
"differences to be reported related to the meta information of the files. "
"For example:\n"
"- The supercompression scheme can be ignored with --ignore-supercomp\n"
"- Compressed byte length and other index section differences can be ignored "
"with --ignore-index all or --ignore-index level\n"
"- DFD section differences can be ignored with --ignore-dfd all or --ignore-dfd "
"all-except-color-space\n"
"- SGD section differences can be ignored with --ignore-sgd all or --ignore-sgd "
"payload\n",
cxxopts::value<std::string>()->default_value("raw"), "raw|image|ignore")
(kPerPixelOutput, "Controls whether per pixel / texel block difference output is generated when "
"--content is set to image:\n"
" all - Every single difference is output (may result in a very large output)\n"
" <number> - At most the specified number of differences are output\n"
" none - No per pixel / texel block differences are output\n",
cxxopts::value<std::string>()->default_value("none"), "all|<number>|none")
(kAllowInvalidInput, "Perform best effort comparison even if any of the input files are invalid.")
(kIgnoreFormatHeader, "Ignore the vkFormat and typeSize fields in the file header.\n"
"Note: useful when comparing textures with and without BasisLZ encoding, respectively.")
(kIgnoreSupercomp, "Ignore supercompression scheme in the file header.\n"
"Note: use the --ignore-sgd option to also ignore the SGD section, if needed.")
(kIgnoreIndex, "Controls the comparison of index entries in the file headers. Possible options are:\n"
" all: Ignore all index entries\n"
" level: Ignore level index entries only\n"
" none: Do not ignore any index entries\n",
cxxopts::value<std::string>()->default_value("none"), "all|level|none")
(kIgnoreDFD, "Controls the comparison of DFD blocks. Possible options are:\n"
" all: Ignore all DFD blocks\n"
" all-except-color-space: Ignore all DFD blocks except the color space information "
"in the basic DFD block\n"
" unknown: Ignore any unrecognized DFD blocks\n"
" extended: Ignore all DFD blocks except the basic DFD block\n"
" none: Do not ignore any DFD blocks\n",
cxxopts::value<std::string>()->default_value("none"),
"all|all-except-color-space|unknown|extended|none")
(kIgnoreBDFDBytesPlane, "Ignore BDFD bytesPlane values. Useful when comparing pre-spec-2.0.4 "
"supercompressed files with 2.0.4+ supercompressed or non-supercompressed ones, as bytesPlane "
"is set to zero in older supercompressed files.")
(kIgnoreMetadata, "Controls the comparison of metadata (KVD) entries. Possible options are:\n"
" all: Ignore all metadata entries\n"
" <key>[,...]: Ignore the specified comma separated list of metadata keys\n"
" none: Do not ignore any metadata entries\n",
cxxopts::value<std::string>()->default_value("none"), "all|<key>[,...]|none")
(kIgnoreSGD, "Controls the comparison of the SGD section. Possible options are:\n"
" all: Ignore the SGD section\n"
" unknown: Ignore any unrecognized SGD section\n"
" payload: Ignore any unrecognized SGD section and the payload of any known SGD section\n"
" none: Do not ignore the SGD section\n"
"Note: --ignore-sgd payload can be used to compare BasisLZ SGD headers without "
"expecting an exact match for the individual SGD payload sections.",
cxxopts::value<std::string>()->default_value("none"), "all|unknown|payload|none");
opts.parse_positional("input-file1", "input-file2");
opts.positional_help("<input-file1> <input-file2>");
}
void process(cxxopts::Options&, cxxopts::ParseResult& args, Reporter& report) {
if (args.count("input-file1") == 0)
report.fatal_usage("Missing input files.");
if (args.count("input-file2") == 0)
report.fatal_usage("Missing second input file.");
inputFilepaths[0] = args["input-file1"].as<std::string>();
inputFilepaths[1] = args["input-file2"].as<std::string>();
if (args[kContent].count()) {
static std::unordered_map<std::string, ContentMode> contentModeMapping{
{"raw", ContentMode::raw},
{"image", ContentMode::image},
{"ignore", ContentMode::ignore}
};
const auto contentModeStr = to_lower_copy(args[kContent].as<std::string>());
const auto it = contentModeMapping.find(contentModeStr);
if (it == contentModeMapping.end())
report.fatal_usage("Invalid --content argument: \"{}\".", contentModeStr);
contentMode = it->second;
}
if (args[kPerPixelOutput].count()) {
if (contentMode != ContentMode::image)
report.fatal_usage("--per-pixel-output is specified but --content was not set to \"image\".");
const auto perPixelOutputStr = to_lower_copy(args[kPerPixelOutput].as<std::string>());
if (perPixelOutputStr == "all") {
perPixelOutputLimit = SIZE_MAX;
} else if (perPixelOutputStr == "none") {
perPixelOutputLimit = 0;
} else {
std::size_t parsedCharacters = 0;
try {
perPixelOutputLimit = static_cast<std::size_t>(std::stoull(perPixelOutputStr, &parsedCharacters));
} catch (std::exception&) {}
if (parsedCharacters != perPixelOutputStr.length())
report.fatal_usage("Invalid --per-pixel-output arugment: \"{}\".", perPixelOutputStr);
}
}
allowInvalidInput = args[kAllowInvalidInput].as<bool>();
ignoreFormatHeader = args[kIgnoreFormatHeader].as<bool>();
ignoreSupercomp = args[kIgnoreSupercomp].as<bool>();
if (args[kIgnoreIndex].count()) {
static std::unordered_map<std::string, IgnoreIndex> ignoreIndexMapping{
{"all", IgnoreIndex::all},
{"level", IgnoreIndex::level},
{"none", IgnoreIndex::none}
};
const auto ignoreIndexStr = to_lower_copy(args[kIgnoreIndex].as<std::string>());
const auto it = ignoreIndexMapping.find(ignoreIndexStr);
if (it == ignoreIndexMapping.end())
report.fatal_usage("Invalid --ignore-index argument: \"{}\".", ignoreIndexStr);
ignoreIndex = it->second;
}
if (args[kIgnoreDFD].count()) {
static std::unordered_map<std::string, IgnoreDFD> ignoreDFDMapping{
{"all", IgnoreDFD::all},
{"all-except-color-space", IgnoreDFD::all_except_color_space},
{"unknown", IgnoreDFD::unknown},
{"extended", IgnoreDFD::extended},
{"none", IgnoreDFD::none}
};
const auto ignoreDFDStr = to_lower_copy(args[kIgnoreDFD].as<std::string>());
const auto it = ignoreDFDMapping.find(ignoreDFDStr);
if (it == ignoreDFDMapping.end())
report.fatal_usage("Invalid --ignore-dfd argument: \"{}\".", ignoreDFDStr);
ignoreDFD = it->second;
}
ignoreBDFDBytesPlane = args[kIgnoreBDFDBytesPlane].as<bool>();
if (args[kIgnoreMetadata].count()) {
static std::unordered_map<std::string, bool> ignoreMetadataMapping{
{"all", true},
{"none", false}
};
const auto ignoreMetadataStr = to_lower_copy(args[kIgnoreMetadata].as<std::string>());
const auto it = ignoreMetadataMapping.find(ignoreMetadataStr);
if (it == ignoreMetadataMapping.end()) {
// Comma separated list of keys
std::stringstream stream(args[kIgnoreMetadata].as<std::string>());
std::string key;
while (std::getline(stream, key, ','))
ignoreMetadataKeys.emplace(std::move(key));
} else {
ignoreAllMetadata = it->second;
}
}
if (args[kIgnoreSGD].count()) {
static std::unordered_map<std::string, IgnoreSGD> ignoreSGDMapping{
{"all", IgnoreSGD::all},
{"unknown", IgnoreSGD::unknown},
{"payload", IgnoreSGD::payload},
{"none", IgnoreSGD::none}
};
const auto ignoreSGDStr = to_lower_copy(args[kIgnoreSGD].as<std::string>());
const auto it = ignoreSGDMapping.find(ignoreSGDStr);
if (it == ignoreSGDMapping.end())
report.fatal_usage("Invalid --ignore-sgd argument: \"{}\".", ignoreSGDStr);
ignoreSGD = it->second;
}
}
};
Combine<OptionsFormat, OptionsCompare, OptionsGeneric> options;
using InputStreams = std::array<InputStream, 2>;
std::vector<KTX_header2> headers;
public:
virtual int main(int argc, char* argv[]) override;
virtual void initOptions(cxxopts::Options& opts) override;
virtual void processOptions(cxxopts::Options& opts, cxxopts::ParseResult& args) override;
private:
void executeCompare();
void compareHeader(PrintDiff& diff, InputStreams& streams);
void compareLevelIndex(PrintDiff& diff, InputStreams& streams);
void compareDFD(PrintDiff& diff, InputStreams& streams);
void compareDFDBasic(PrintDiff& diff, uint32_t blockIndex,
std::optional<BDFD> bdfds[2], std::optional<std::vector<SampleType>> bdfdSamples[2]);
void compareKVD(PrintDiff& diff, InputStreams& streams);
void compareKVEntry(PrintDiff& diff, const std::string_view& key,
ktxHashListEntry* entry1, ktxHashListEntry* entry2);
void compareSGD(PrintDiff& diff, InputStreams& streams);
void compareImages(PrintDiff& diff, InputStreams& streams);
void compareImagesRaw(PrintDiff& diff, InputStreams& streams);
void compareImagesPerPixel(PrintDiff& diff, InputStreams& streams);
void read(InputStream& stream, std::size_t offset, void* readDst, std::size_t readSize, std::string_view what) {
stream->seekg(offset);
if (stream->fail())
fatal(rc::IO_FAILURE, "Failed to seek file to {} \"{}\": {}.", what, stream.str(), errnoMessage());
stream->read(reinterpret_cast<char*>(readDst), readSize);
if (stream->eof()) {
fatal(rc::IO_FAILURE, "Unexpected end of file reading {} from file \"{}\".", what, stream.str());
} else if (stream->fail()) {
fatal(rc::IO_FAILURE, "Failed to read {} from file \"{}\": {}.", what, stream.str(), errnoMessage());
}
}
template <typename T>
std::vector<T> read(InputStreams& streams, std::size_t offset, std::string_view what) {
std::vector<T> result(streams.size());
for (std::size_t i = 0; i < streams.size(); ++i) {
read(streams[i], offset, &result[i], sizeof(T), what);
}
return result;
}
typedef bool (*CompareFunc)(ImageSpan::TexelBlockPtr<> texelBlocks[2]);
static bool compareTexelBlocksPacked(ImageSpan::TexelBlockPtr<> texelBlocks[2]) {
// Expect number of packed elements and byte size to match
assert(texelBlocks[0].imageCodec().getPackedElementCount() == texelBlocks[1].imageCodec().getPackedElementCount());
assert(texelBlocks[0].imageCodec().getPackedElementByteSize() == texelBlocks[1].imageCodec().getPackedElementByteSize());
for (uint32_t elementIdx = 0; elementIdx < texelBlocks[0].imageCodec().getPackedElementCount(); ++elementIdx) {
if (texelBlocks[0].getPackedElement(elementIdx) != texelBlocks[1].getPackedElement(elementIdx)){
return false;
}
}
return true;
}
};
// -------------------------------------------------------------------------------------------------
int CommandCompare::main(int argc, char* argv[]) {
try {
parseCommandLine("ktx compare",
"Compares the two KTX files specified as the input-file1 and input-file2 arguments.\n"
" The command implicitly calls validate and prints any found errors\n"
" and warnings to stdout.",
argc, argv);
executeCompare();
return to_underlying(rc::SUCCESS);
} catch (const FatalError& error) {
return +error.returnCode;
} catch (const std::exception& e) {
fmt::print(std::cerr, "{} fatal: {}\n", commandName, e.what());
return +rc::RUNTIME_ERROR;
}
}
void CommandCompare::initOptions(cxxopts::Options& opts) {
options.init(opts);
}
void CommandCompare::processOptions(cxxopts::Options& opts, cxxopts::ParseResult& args) {
options.process(opts, args, *this);
}
void CommandCompare::executeCompare() {
InputStreams inputStreams{
InputStream(options.inputFilepaths[0], *this),
InputStream(options.inputFilepaths[1], *this),
};
std::vector<std::string> validationMessages;
std::vector<int> validationResults;
switch (options.format) {
case OutputFormat::text: {
for (std::size_t i = 0; i < inputStreams.size(); ++i) {
std::ostringstream messagesOS;
validationResults.emplace_back(validateIOStream(inputStreams[i], fmtInFile(options.inputFilepaths[i]),
false, false, [&](const ValidationReport& issue) {
fmt::print(messagesOS, "{}-{:04}: {}\n", toString(issue.type), issue.id, issue.message);
fmt::print(messagesOS, " {}\n", issue.details);
}));
validationMessages.emplace_back(std::move(messagesOS).str());
}
bool hasValidationMessages = false;
for (std::size_t i = 0; i < inputStreams.size(); ++i) {
if (!validationMessages[i].empty()) {
if (std::exchange(hasValidationMessages, true))
fmt::print("\n");
fmt::print("Validation {} for '{}'\n", validationResults[i] == 0 ? "successful" : "failed",
options.inputFilepaths[i]);
fmt::print("\n");
fmt::print("{}", validationMessages[i]);
}
}
for (std::size_t i = 0; i < inputStreams.size(); ++i) {
if (validationResults[i] != 0) {
if (validationResults[i] != +rc::INVALID_FILE || !options.allowInvalidInput)
throw FatalError(ReturnCode{validationResults[i]});
// Image comparison is only supported for valid input files
options.contentMode = ContentMode::ignore;
}
}
if (hasValidationMessages)
fmt::print("\n");
PrintIndent out{std::cout};
PrintDiff diff(out, options.format);
compareHeader(diff, inputStreams);
compareLevelIndex(diff, inputStreams);
compareDFD(diff, inputStreams);
compareKVD(diff, inputStreams);
compareSGD(diff, inputStreams);
compareImages(diff, inputStreams);
if (diff.isDifferent())
throw FatalError(rc::DIFFERENCE_FOUND);
break;
}
case OutputFormat::json: [[fallthrough]];
case OutputFormat::json_mini: {
int fatalValidationError = 0;
const auto baseIndent = options.format == OutputFormat::json ? +0 : 0;
const auto indentWidth = options.format == OutputFormat::json ? 4 : 0;
const auto space = options.format == OutputFormat::json ? " " : "";
const auto nl = options.format == OutputFormat::json ? "\n" : "";
PrintIndent out{std::cout, baseIndent, indentWidth};
out(0, "{{{}", nl);
out(1, "\"$schema\":{}\"https://schema.khronos.org/ktx/compare_v0.json\",{}", space, nl);
for (std::size_t i = 0; i < inputStreams.size(); ++i) {
std::ostringstream messagesOS;
PrintIndent pi{messagesOS, baseIndent, indentWidth};
bool first = true;
validationResults.emplace_back(validateIOStream(inputStreams[i], fmtInFile(options.inputFilepaths[i]),
false, false, [&](const ValidationReport& issue) {
if (!std::exchange(first, false)) {
pi(3, "}},{}", nl);
}
pi(3, "{{{}", nl);
pi(4, "\"id\":{}{},{}", space, issue.id, nl);
pi(4, "\"type\":{}\"{}\",{}", space, toString(issue.type), nl);
pi(4, "\"message\":{}\"{}\",{}", space, escape_json_copy(issue.message), nl);
pi(4, "\"details\":{}\"{}\"{}", space, escape_json_copy(issue.details), nl);
}));
validationMessages.emplace_back(std::move(messagesOS).str());
}
out(1, "\"valid\":{}[{}", space, nl);
for (std::size_t i = 0; i < inputStreams.size(); ++i) {
bool last = (i == inputStreams.size() - 1);
out(2, "{}{}{}", validationResults[i] == 0, last ? "" : ",", nl);
}
out(1, "],{}", nl);
out(1, "\"messages\":{}[{}", space, nl);
for (std::size_t i = 0; i < inputStreams.size(); ++i) {
bool last = (i == inputStreams.size() - 1);
if (!validationMessages[i].empty()) {
out(2, "[{}", nl);
fmt::print("{}", validationMessages[i]);
out(3, "}}{}", nl);
out(2, "]{}{}", last ? "" : ",", nl);
} else {
out(2, "[]{}{}", last ? "" : ",", nl);
}
}
out(1, "]");
for (std::size_t i = 0; i < inputStreams.size(); ++i) {
if (validationResults[i] != 0) {
if (validationResults[i] != +rc::INVALID_FILE || !options.allowInvalidInput) {
fatalValidationError = validationResults[i];
break;
}
// Image comparison is only supported for valid input files
options.contentMode = ContentMode::ignore;
}
}
PrintDiff diff(out, options.format);
try {
if (fatalValidationError)
throw FatalError(ReturnCode{fatalValidationError});
diff.beginJsonSection("info");
compareHeader(diff, inputStreams);
compareLevelIndex(diff, inputStreams);
compareDFD(diff, inputStreams);
compareKVD(diff, inputStreams);
compareSGD(diff, inputStreams);
diff.endJsonSection();
diff.beginJsonSection("image");
compareImages(diff, inputStreams);
diff.endJsonSection();
if (diff.isDifferent())
throw FatalError(rc::DIFFERENCE_FOUND);
} catch (...) {
diff.endJsonSection();
fmt::print("{}}}{}", nl, nl);
throw;
}
fmt::print("{}}}{}", nl, nl);
break;
}
}
}
// Helper macros to access the fields of optional structures
#define OPT_FIELD(struct, member) ((struct).has_value() ? \
std::optional<decltype((struct)->member)>((struct)->member) : std::optional<decltype((struct)->member)>() )
#define OPT_BITFIELD(struct, member) ((struct).has_value() ? \
std::optional<int32_t>(static_cast<int32_t>((struct)->member)) : std::optional<int32_t>() )
#define OPT_UINT4(struct, memberPrefix) ((struct).has_value() ? \
std::optional<std::array<uint32_t, 4>>(std::array<uint32_t, 4>{ \
(struct)->memberPrefix##0, \
(struct)->memberPrefix##1, \
(struct)->memberPrefix##2, \
(struct)->memberPrefix##3, \
}) : std::optional<std::array<uint32_t, 4>>())
// Helper macros to access the fields of an optional structure array with 2 values
#define OPT_FIELDS(structArr, member) OPT_FIELD((structArr)[0], member), OPT_FIELD((structArr)[1], member)
#define OPT_BITFIELDS(structArr, member) OPT_BITFIELD((structArr)[0], member), OPT_BITFIELD((structArr)[1], member)
#define OPT_UINT4S(structArr, memberPrefix) OPT_UINT4((structArr)[0], memberPrefix), OPT_UINT4((structArr)[1], memberPrefix)
void CommandCompare::compareHeader(PrintDiff& diff, InputStreams& streams) {
diff.setContext("Header\n\n");
headers = read<KTX_header2>(streams, 0, "header");
diff << DiffIdentifier("identifier", "/header/identifier", headers[0], headers[1]);
if (!options.ignoreFormatHeader) {
diff << DiffEnum<VkFormat>("vkFormat", "/header/vkFormat", headers[0].vkFormat, headers[1].vkFormat,
[&](auto i) { return vkFormatString(VkFormat(headers[i].vkFormat)); });
diff << Diff("typeSize", "/header/typeSize", headers[0].typeSize, headers[1].typeSize);
}
diff << Diff("pixelWidth", "/header/pixelWidth", headers[0].pixelWidth, headers[1].pixelWidth);
diff << Diff("pixelHeight", "/header/pixelHeight", headers[0].pixelHeight, headers[1].pixelHeight);
diff << Diff("pixelDepth", "/header/pixelDepth", headers[0].pixelDepth, headers[1].pixelDepth);
diff << Diff("layerCount", "/header/layerCount", headers[0].layerCount, headers[1].layerCount);
diff << Diff("faceCount", "/header/faceCount", headers[0].faceCount, headers[1].faceCount);
diff << Diff("levelCount", "/header/levelCount", headers[0].levelCount, headers[1].levelCount);
if (!options.ignoreSupercomp)
diff << DiffEnum<ktxSupercmpScheme>("supercompressionScheme", "/header/supercompressionScheme",
headers[0].supercompressionScheme, headers[1].supercompressionScheme);
if (options.ignoreIndex != IgnoreIndex::all) {
diff << DiffHex("dataFormatDescriptor.byteOffset", "/index/dataFormatDescriptor/byteOffset",
headers[0].dataFormatDescriptor.byteOffset, headers[1].dataFormatDescriptor.byteOffset);
diff << Diff("dataFormatDescriptor.byteLength", "/index/dataFormatDescriptor/byteLength",
headers[0].dataFormatDescriptor.byteLength, headers[1].dataFormatDescriptor.byteLength);
diff << DiffHex("keyValueData.byteOffset", "/index/keyValueData/byteOffset",
headers[0].keyValueData.byteOffset, headers[1].keyValueData.byteOffset);
diff << Diff("keyValueData.byteLength", "/index/keyValueData/byteLength",
headers[0].keyValueData.byteLength, headers[1].keyValueData.byteLength);
diff << DiffHex("supercompressionGlobalData.byteOffset", "/index/supercompressionGlobalData/byteOffset",
headers[0].supercompressionGlobalData.byteOffset, headers[1].supercompressionGlobalData.byteOffset);
diff << Diff("supercompressionGlobalData.byteLength", "/index/supercompressionGlobalData/byteLength",
headers[0].supercompressionGlobalData.byteLength, headers[1].supercompressionGlobalData.byteLength);
}
}
void CommandCompare::compareLevelIndex(PrintDiff& diff, InputStreams& streams) {
if (options.ignoreIndex != IgnoreIndex::none) return;
diff.setContext("Level Index\n\n");
const uint32_t numLevels[] = {
std::max(1u, headers[0].levelCount),
std::max(1u, headers[1].levelCount)
};
const uint32_t maxNumLevels = std::max(numLevels[0], numLevels[1]);
for (uint32_t level = 0; level < maxNumLevels; ++level) {
const auto levelIndexEntryOffset = sizeof(KTX_header2) + level * sizeof(ktxLevelIndexEntry);
std::optional<ktxLevelIndexEntry> levelIndexEntry[2];
for (std::size_t i = 0; i < streams.size(); ++i)
if (level < numLevels[i]) {
ktxLevelIndexEntry entry;
read(streams[i], levelIndexEntryOffset, &entry, sizeof(entry), "the level index");
levelIndexEntry[i] = entry;
}
diff << DiffHex(fmt::format("Level{}.byteOffset", level),
fmt::format("/index/levels/{}/byteOffset", level),
OPT_FIELDS(levelIndexEntry, byteOffset));
diff << Diff(fmt::format("Level{}.byteLength", level),
fmt::format("/index/levels/{}/byteLength", level),
OPT_FIELDS(levelIndexEntry, byteLength));
diff << Diff(fmt::format("Level{}.uncompressedByteLength", level),
fmt::format("/index/levels/{}/uncompressedByteLength", level),
OPT_FIELDS(levelIndexEntry, uncompressedByteLength));
}
}
void CommandCompare::compareDFD(PrintDiff& diff, InputStreams& streams) {
if (options.ignoreDFD == IgnoreDFD::all) return;
diff.setContext("Data Format Descriptor\n\n");
std::unique_ptr<uint8_t[]> buffers[] = {
std::make_unique<uint8_t[]>(headers[0].dataFormatDescriptor.byteLength),
std::make_unique<uint8_t[]>(headers[1].dataFormatDescriptor.byteLength)
};
for (std::size_t i = 0; i < streams.size(); ++i)
read(streams[i], headers[i].dataFormatDescriptor.byteOffset, buffers[i].get(),
headers[i].dataFormatDescriptor.byteLength, "the DFD blocks");
const uint8_t* ptrDFD[] = { buffers[0].get(), buffers[1].get() };
const uint8_t* ptrDFDEnd[] = {
ptrDFD[0] + headers[0].dataFormatDescriptor.byteLength,
ptrDFD[1] + headers[1].dataFormatDescriptor.byteLength,
};
const uint8_t* ptrDFDIt[] = { ptrDFD[0], ptrDFD[1] };
uint32_t dfdTotalSize[2];
for (std::size_t i = 0; i < streams.size(); ++i) {
std::memcpy(&dfdTotalSize[i], ptrDFDIt[i], sizeof(uint32_t));
ptrDFDIt[i] += sizeof(uint32_t);
}
if (options.ignoreDFD != IgnoreDFD::all_except_color_space)
if (dfdTotalSize[0] != dfdTotalSize[1])
diff << Diff("DFD total bytes", "/dataFormatDescriptor/totalSize", dfdTotalSize[0], dfdTotalSize[1]);
uint32_t blockIndex = 0;
while ((ptrDFDIt[0] < ptrDFDEnd[0]) || (ptrDFDIt[1] < ptrDFDEnd[1])) {
const std::size_t remainingDFDBytes[] = {
static_cast<std::size_t>(ptrDFDEnd[0] - ptrDFDIt[0]),
static_cast<std::size_t>(ptrDFDEnd[1] - ptrDFDIt[1]),
};
std::optional<DFDHeader> blockHeaders[2] = {};
if (remainingDFDBytes[0] < sizeof(DFDHeader) && remainingDFDBytes[1] < sizeof(DFDHeader))
break;
for (std::size_t i = 0; i < streams.size(); ++i)
if (remainingDFDBytes[i] >= sizeof(DFDHeader)) {
DFDHeader blockHeader;
std::memcpy(&blockHeader, ptrDFDIt[i], sizeof(blockHeader));
blockHeaders[i] = blockHeader;
}
bool dfdKnown[2] = { false, false };
bool dfdBasic[2] = { false, false };
for (std::size_t i = 0; i < streams.size(); ++i) {
if (blockHeaders[i].has_value() &&
blockHeaders[i]->vendorId == KHR_DF_VENDORID_KHRONOS &&
blockHeaders[i]->descriptorType == KHR_DF_KHR_DESCRIPTORTYPE_BASICFORMAT) {
dfdKnown[i] = dfdBasic[i] = true;
}
}
// Consider the ignore-dfd option before comparing the headers
bool compareDFDs = true;
switch (options.ignoreDFD) {
case IgnoreDFD::unknown:
// Only compare the DFDs if at least one of them is known
compareDFDs = dfdKnown[0] || dfdKnown[1];
break;
case IgnoreDFD::all_except_color_space: [[fallthrough]];
case IgnoreDFD::extended:
// Only compare the DFDs if at least one of them is basic
compareDFDs = dfdBasic[0] || dfdBasic[1];
break;
default:
break;
}
if (compareDFDs) {
if (options.ignoreDFD != IgnoreDFD::all_except_color_space) {
diff << DiffEnum<khr_df_vendorid_e>("Vendor ID",
fmt::format("/dataFormatDescriptor/blocks/{}/vendorId", blockIndex),
OPT_BITFIELDS(blockHeaders, vendorId),
[&](auto i) { return dfdToStringVendorID(khr_df_vendorid_e(blockHeaders[i]->vendorId)); });
diff << DiffEnum<khr_df_khr_descriptortype_e>("Descriptor type",
fmt::format("/dataFormatDescriptor/blocks/{}/descriptorType", blockIndex),
OPT_BITFIELDS(blockHeaders, descriptorType),
[&](auto i) {
return (blockHeaders[i]->vendorId == KHR_DF_VENDORID_KHRONOS)
? dfdToStringDescriptorType(khr_df_khr_descriptortype_e(blockHeaders[i]->descriptorType))
: nullptr;
});
diff << DiffEnum<khr_df_versionnumber_e>("Version",
fmt::format("/dataFormatDescriptor/blocks/{}/versionNumber", blockIndex),
OPT_BITFIELDS(blockHeaders, versionNumber),
[&](auto i) { return dfdToStringVersionNumber(khr_df_versionnumber_e(blockHeaders[i]->versionNumber)); });
diff << Diff("Descriptor block size",
fmt::format("/dataFormatDescriptor/blocks/{}/descriptorBlockSize", blockIndex),
OPT_BITFIELDS(blockHeaders, descriptorBlockSize));
}
// Compare basic DFD data if possible
if (dfdBasic[0] || dfdBasic[1]) {
std::optional<BDFD> bdfds[2] = {};
std::optional<std::vector<SampleType>> bdfdSamples[2] = {};
for (std::size_t i = 0; i < streams.size(); ++i)
if (blockHeaders[i].has_value() &&
blockHeaders[i]->vendorId == KHR_DF_VENDORID_KHRONOS &&
blockHeaders[i]->descriptorType == KHR_DF_KHR_DESCRIPTORTYPE_BASICFORMAT) {
BDFD bdfd;
std::memcpy(&bdfd, ptrDFDIt[i], sizeof(bdfd));
bdfds[i] = std::move(bdfd);
std::vector<SampleType> samples(std::min(MAX_NUM_BDFD_SAMPLES, (blockHeaders[i]->descriptorBlockSize - 24u) / 16u));
std::memcpy(samples.data(), ptrDFDIt[i] + sizeof(BDFD), samples.size() * sizeof(SampleType));
bdfdSamples[i] = std::move(samples);
}
assert(bdfds[0].has_value() || bdfds[1].has_value());
compareDFDBasic(diff, blockIndex, bdfds, bdfdSamples);
}
// Compare any unrecognized DFD data as raw payload
if (options.ignoreDFD != IgnoreDFD::all_except_color_space)
if (!dfdKnown[0] || !dfdKnown[1]) {
std::optional<std::vector<uint8_t>> rawPayloads[2];
for (std::size_t i = 0; i < streams.size(); ++i) {
if (blockHeaders[i].has_value()) {
rawPayloads[i] = std::vector<uint8_t>(blockHeaders[i]->descriptorBlockSize - sizeof(DFDHeader));
std::memcpy(rawPayloads[i]->data(), ptrDFDIt[i], rawPayloads[i]->size());
diff << DiffRawBytes("Raw payload",
fmt::format("/dataFormatDescriptor/blocks/{}/rawPayload", blockIndex),
rawPayloads[0], rawPayloads[1]);
}
}
}
}
if (++blockIndex >= MAX_NUM_DFD_BLOCKS)
return;
for (std::size_t i = 0; i < streams.size(); ++i)
if (blockHeaders[i].has_value())
ptrDFDIt[i] += std::max(blockHeaders[i]->descriptorBlockSize, 8u);
}
}
void CommandCompare::compareDFDBasic(PrintDiff& diff, uint32_t blockIndex,
std::optional<BDFD> bdfds[2], std::optional<std::vector<SampleType>> bdfdSamples[2]) {
for (std::size_t i = 0; i < 2; ++i)
assert(bdfds[i].has_value() == bdfdSamples[i].has_value());
if (options.ignoreDFD != IgnoreDFD::all_except_color_space)
diff << DiffFlags("Flags",
fmt::format("/dataFormatDescriptor/blocks/{}/flags", blockIndex),
OPT_BITFIELDS(bdfds, flags), dfdToStringFlagsBit);
diff << DiffEnum<khr_df_transfer_e>("Transfer",
fmt::format("/dataFormatDescriptor/blocks/{}/transferFunction", blockIndex),
OPT_BITFIELDS(bdfds, transfer),
[&](auto i) { return dfdToStringTransferFunction(khr_df_transfer_e(bdfds[i]->transfer)); });
diff << DiffEnum<khr_df_primaries_e>("Primaries",
fmt::format("/dataFormatDescriptor/blocks/{}/colorPrimaries", blockIndex),
OPT_BITFIELDS(bdfds, primaries),
[&](auto i) { return dfdToStringColorPrimaries(khr_df_primaries_e(bdfds[i]->primaries)); });
// Do not compare the remainder of the BDFD if everything but color space information is ignored
if (options.ignoreDFD == IgnoreDFD::all_except_color_space)
return;
diff << DiffEnum<khr_df_model_e>("Model",
fmt::format("/dataFormatDescriptor/blocks/{}/colorModel", blockIndex),
OPT_BITFIELDS(bdfds, model),
[&](auto i) { return dfdToStringColorModel(khr_df_model_e(bdfds[i]->model)); });
diff << DiffArray("Dimensions",
fmt::format("/dataFormatDescriptor/blocks/{}/texelBlockDimension", blockIndex),
OPT_UINT4S(bdfds, texelBlockDimension));
if (!options.ignoreBDFDBytesPlane)
diff << DiffArray("Plane bytes",
fmt::format("/dataFormatDescriptor/blocks/{}/bytesPlane", blockIndex),
OPT_FIELDS(bdfds, bytesPlanes));
diff.addContext("Sample <i>:\n");
std::size_t maxNumSamples = std::max(
bdfdSamples[0].has_value() ? bdfdSamples[0]->size() : 0,
bdfdSamples[1].has_value() ? bdfdSamples[1]->size() : 0
);
for (std::size_t sampleIndex = 0; sampleIndex < maxNumSamples; ++sampleIndex) {
diff.updateContext(fmt::format("Sample {}:\n", sampleIndex));
std::optional<SampleType> samples[2] = {};
for (std::size_t i = 0; i < 2; ++i)
if (bdfdSamples[i].has_value() && sampleIndex < bdfdSamples[i]->size())
samples[i] = (*bdfdSamples[i])[sampleIndex];
std::optional<uint32_t> qualifierFlags[2] = {};
for (std::size_t i = 0; i < 2; ++i)
if (samples[i].has_value()) {
uint32_t flags = 0;
flags |= samples[i]->qualifierLinear ? KHR_DF_SAMPLE_DATATYPE_LINEAR : 0;
flags |= samples[i]->qualifierExponent ? KHR_DF_SAMPLE_DATATYPE_EXPONENT : 0;
flags |= samples[i]->qualifierSigned ? KHR_DF_SAMPLE_DATATYPE_SIGNED : 0;
flags |= samples[i]->qualifierFloat ? KHR_DF_SAMPLE_DATATYPE_FLOAT : 0;
qualifierFlags[i] = flags;
}
diff << DiffFlags(" Qualifiers",
fmt::format("/dataFormatDescriptor/blocks/{}/samples/{}/qualifiers", blockIndex, sampleIndex),
qualifierFlags[0], qualifierFlags[1], dfdToStringSampleDatatypeQualifiersBit);
diff << DiffEnum<khr_df_model_channels_e>(" Channel Type",
fmt::format("/dataFormatDescriptor/blocks/{}/samples/{}/channelType", blockIndex, sampleIndex),
OPT_BITFIELDS(samples, channelType),
[&](auto i) {
return dfdToStringChannelId(khr_df_model_e(bdfds[i]->model),
khr_df_model_channels_e(samples[i]->channelType));
})
.outputHexInText();
// Text output combines length and offset so we have to special-case here
if (options.format == OutputFormat::text) {
std::optional<std::string> lengthAndOffset[2] = {};
for (std::size_t i = 0; i < 2; ++i)
if (samples[i].has_value())
lengthAndOffset[i] = fmt::format(" Length: {} bits Offset: {}",
static_cast<uint32_t>(samples[i]->bitLength + 1),
static_cast<uint32_t>(samples[i]->bitOffset));
diff << DiffTextCustom(lengthAndOffset[0], lengthAndOffset[1]);
} else {
diff << Diff({},
fmt::format("/dataFormatDescriptor/blocks/{}/samples/{}/bitLength", blockIndex, sampleIndex),
OPT_BITFIELDS(samples, bitLength));
diff << Diff({},
fmt::format("/dataFormatDescriptor/blocks/{}/samples/{}/bitOffset", blockIndex, sampleIndex),
OPT_BITFIELDS(samples, bitOffset));
}
diff << DiffArray(" Position",
fmt::format("/dataFormatDescriptor/blocks/{}/samples/{}/samplePosition", blockIndex, sampleIndex),
OPT_UINT4S(samples, samplePosition));
diff << DiffHexFixedWidth(" Lower",
fmt::format("/dataFormatDescriptor/blocks/{}/samples/{}/sampleLower", blockIndex, sampleIndex),
OPT_FIELDS(samples, lower));
diff << DiffHexFixedWidth(" Upper",
fmt::format("/dataFormatDescriptor/blocks/{}/samples/{}/sampleUpper", blockIndex, sampleIndex),
OPT_FIELDS(samples, upper));
}
}
void CommandCompare::compareKVD(PrintDiff& diff, InputStreams& streams) {
if (options.ignoreAllMetadata) return;
diff.setContext("Key/Value Data\n\n");
std::vector<uint8_t> keyValueStores[2] = {};
std::map<std::string_view, ktxHashListEntry*> keys[2] = {};
for (std::size_t i = 0; i < streams.size(); ++i) {
if (headers[i].keyValueData.byteLength == 0) continue;
keyValueStores[i].resize(headers[i].keyValueData.byteLength);
read(streams[i], headers[i].keyValueData.byteOffset, keyValueStores[i].data(),
headers[i].keyValueData.byteLength, "the KVD");
ktxHashList kvDataHead = nullptr;
KTX_error_code result = ktxHashList_Deserialize(&kvDataHead,
headers[i].keyValueData.byteLength, keyValueStores[i].data());
if (result != KTX_SUCCESS)
fatal(rc::KTX_FAILURE, "Failed to parse KVD in file \"{}\".", streams[i].str());
if (kvDataHead == nullptr)
continue;
uint32_t entryIndex = 0;
ktxHashListEntry* entry = kvDataHead;
for (; entry != NULL && entryIndex < MAX_NUM_KV_ENTRIES; entry = ktxHashList_Next(entry), ++entryIndex) {
char* key;
ktx_uint32_t keyLen;
ktxHashListEntry_GetKey(entry, &keyLen, &key);
keys[i].emplace(key, entry);
}
}
std::map<std::string_view, ktxHashListEntry*>::iterator it[2] = { keys[0].begin(), keys[1].begin() };
while (it[0] != keys[0].end() || it[1] != keys[1].end()) {
bool ignoreKey = false;
for (std::size_t i = 0; i < streams.size(); ++i)
if ((it[i] != keys[i].end()) &&
(options.ignoreMetadataKeys.find(std::string(it[i]->first)) != options.ignoreMetadataKeys.end())) {
// Key is on the ignore list, skip it
it[i]++;
ignoreKey = true;
break;
}
if (ignoreKey) continue;
if ((it[0] != keys[0].end()) && ((it[1] == keys[1].end()) || (it[0]->first < it[1]->first))) {
// First stream has a key that is not present in the second stream
compareKVEntry(diff, it[0]->first, it[0]->second, nullptr);
it[0]++;
} else if ((it[1] != keys[1].end()) && ((it[0] == keys[0].end()) || (it[0]->first > it[1]->first))) {
// Second stream has a key that is not present in the first stream
compareKVEntry(diff, it[1]->first, nullptr, it[1]->second);
it[1]++;
} else {
// Both streams have the key
compareKVEntry(diff, it[0]->first, it[0]->second, it[1]->second);
it[0]++;
it[1]++;
}
}
}
struct KVEntry {
char* value;
ktx_uint32_t valueLen;
// shut up clang's complaints about derived classes
// that have virtual functions but non-virtual destructor
virtual ~KVEntry() {}
template <typename T>
static std::optional<T> load(ktxHashListEntry* entry) {
if (entry) {
T newEntry;
ktxHashListEntry_GetValue(entry, &newEntry.valueLen, reinterpret_cast<void**>(&newEntry.value));
if (newEntry.value)
return std::optional<T>(std::move(newEntry));
}
return std::optional<T>();
}
virtual bool isDifferent(const KVEntry& other) const {
return valueLen != other.valueLen || memcmp(value, other.value, valueLen) != 0;
}
template <typename T>
T extract(std::size_t offset = 0) const {
return (valueLen >= offset + sizeof(T)) ? T(*reinterpret_cast<const T*>(value + offset)) : T();
}
std::string extractRawBytes(bool text, const char* space = " ") const {
std::stringstream formattedValue;
bool first = true;
for (ktx_uint32_t arrayIndex = 0; arrayIndex < valueLen; ++arrayIndex) {
const auto comma = std::exchange(first, false) ? "" : fmt::format(",{}", space);
if (text) {
fmt::print(formattedValue, "{}0x{:x}", comma, (uint8_t)value[arrayIndex]);
} else {
fmt::print(formattedValue, "{}{}", comma, (uint8_t)value[arrayIndex]);
}
}
if (text) {
return fmt::format("[{}]", formattedValue.str());
} else {
if (formattedValue.str().empty()) {
return "[]";
} else {
return fmt::format("[{}{}{}]", space, formattedValue.str(), space);
}
}
}
};
void CommandCompare::compareKVEntry(PrintDiff& diff, const std::string_view& key,
ktxHashListEntry* entry1, ktxHashListEntry* entry2) {
const std::unordered_set<std::string_view> keysWithUint32Values = {
"KTXdxgiFormat__",
"KTXmetalPixelFormat"
};
const std::unordered_set<std::string_view> keysWithStringValues = {
"KTXorientation",
"KTXswizzle",
"KTXwriter",
"KTXwriterScParams",
"KTXastcDecodeMode"
};
if (keysWithUint32Values.find(key) != keysWithUint32Values.end()) {
struct KVEntryUint32 : public KVEntry {
bool isValid() const {
return valueLen == sizeof(ktx_uint32_t);
}
void printText(PrintIndent& out, const char*) const {
if (isValid()) {
out(0, " {}\n", extract<ktx_uint32_t>());
} else {
out(0, " {}\n", extractRawBytes(true));
}
}
void printJson(PrintIndent& out, int indent, const char* space, const char*) const {
if (isValid()) {
out(indent, "{}", extract<ktx_uint32_t>());
} else {
out(indent, "{}", extractRawBytes(false, space));
}
}
};
diff << DiffComplex(key, fmt::format("/keyValueData/{}", key),
KVEntry::load<KVEntryUint32>(entry1), KVEntry::load<KVEntryUint32>(entry2));
} else if (keysWithStringValues.find(key) != keysWithStringValues.end()) {
struct KVEntryString : public KVEntry {
bool isValid() const {
return value[valueLen - 1] == '\0';
}
void printText(PrintIndent& out, const char*) const {
if (isValid()) {
out(0, " {}\n", value);
} else {
out(0, " {}\n", extractRawBytes(true));
}
}
void printJson(PrintIndent& out, int indent, const char* space, const char*) const {
if (isValid()) {
out(indent, "\"{}\"", value);
} else {
out(indent, "{}", extractRawBytes(false, space));
}
}
};
diff << DiffComplex(key, fmt::format("/keyValueData/{}", key),
KVEntry::load<KVEntryString>(entry1), KVEntry::load<KVEntryString>(entry2));
} else if (key == "KTXglFormat") {
struct KTXglFormat : public KVEntry {
bool isValid() const {
return valueLen == 3 * sizeof(ktx_uint32_t);
}
void printText(PrintIndent& out, const char* prefix) const {
if (isValid()) {
out(0, "\n");
out(0, "{} glInternalformat: 0x{:08X}\n", prefix, extract<ktx_uint32_t>(0));
out(0, "{} glFormat: 0x{:08X}\n", prefix, extract<ktx_uint32_t>(4));
out(0, "{} glType: 0x{:08X}\n", prefix, extract<ktx_uint32_t>(8));
} else {
out(0, " {}\n", extractRawBytes(true));
}
}
void printJson(PrintIndent& out, int indent, const char* space, const char* nl) const {
if (isValid()) {
out(indent, "{{{}", nl);
out(indent + 1, "\"glInternalformat\":{}{},{}", space, extract<ktx_uint32_t>(0), nl);
out(indent + 1, "\"glFormat\":{}{},{}", space, extract<ktx_uint32_t>(4), nl);
out(indent + 1, "\"glType\":{}{}{}", space, extract<ktx_uint32_t>(8), nl);
out(indent, "}}");
} else {
out(indent, "{}", extractRawBytes(false, space));
}
}
};
diff << DiffComplex("KTXglFormat", "/keyValueData/KTXglFormat",
KVEntry::load<KTXglFormat>(entry1), KVEntry::load<KTXglFormat>(entry2));
} else if (key == "KTXanimData") {
struct KTXanimData : public KVEntry {
bool isValid() const {
return valueLen == 3 * sizeof(ktx_uint32_t);
}
void printText(PrintIndent& out, const char* prefix) const {
if (isValid()) {
out(0, "\n");
out(0, "{} duration: {}\n", prefix, extract<ktx_uint32_t>(0));
out(0, "{} timescale: {}\n", prefix, extract<ktx_uint32_t>(4));
out(0, "{} loopCount: {}\n", prefix, extract<ktx_uint32_t>(8));
} else {
out(0, " {}\n", extractRawBytes(true));
}
}
void printJson(PrintIndent& out, int indent, const char* space, const char* nl) const {
if (isValid()) {
out(indent, "{{{}", nl);
out(indent + 1, "\"duration\":{}{},{}", space, extract<ktx_uint32_t>(0), nl);
out(indent + 1, "\"timescale\":{}{},{}", space, extract<ktx_uint32_t>(4), nl);
out(indent + 1, "\"loopCount\":{}{}{}", space, extract<ktx_uint32_t>(8), nl);
out(indent, "}}");
} else {
out(indent, "{}", extractRawBytes(false, space));
}
}
};
diff << DiffComplex("KTXanimData", "/keyValueData/KTXanimData",
KVEntry::load<KTXanimData>(entry1), KVEntry::load<KTXanimData>(entry2));
} else if (key == "KTXcubemapIncomplete") {
struct KTXcubemapIncomplete : public KVEntry {
bool isValid() const {
return valueLen == sizeof(ktx_uint8_t);
}
const char* bitValue(uint8_t bitIndex) const {
assert(isValid());
ktx_uint8_t faces = *reinterpret_cast<ktx_uint8_t*>(value);
return (faces & (1u << bitIndex)) ? "true" : "false;";
}
void printText(PrintIndent& out, const char* prefix) const {
if (isValid()) {
out(0, "\n");
out(0, "{} positiveX: {}\n", prefix, bitValue(0));
out(0, "{} negativeX: {}\n", prefix, bitValue(1));
out(0, "{} positiveY: {}\n", prefix, bitValue(2));
out(0, "{} negativeY: {}\n", prefix, bitValue(3));
out(0, "{} positiveZ: {}\n", prefix, bitValue(4));
out(0, "{} negativeZ: {}\n", prefix, bitValue(5));
} else {
out(0, " {}\n", extractRawBytes(true));
}
}
void printJson(PrintIndent& out, int indent, const char* space, const char* nl) const {
if (isValid()) {
out(indent, "{{{}", nl);
out(indent + 1, "\"positiveX\":{}{},{}", space, bitValue(0), nl);
out(indent + 1, "\"negativeX\":{}{},{}", space, bitValue(1), nl);
out(indent + 1, "\"positiveY\":{}{},{}", space, bitValue(2), nl);
out(indent + 1, "\"negativeY\":{}{},{}", space, bitValue(3), nl);
out(indent + 1, "\"positiveZ\":{}{},{}", space, bitValue(4), nl);
out(indent + 1, "\"negativeZ\":{}{},{}", space, bitValue(5), nl);
out(indent, "}}");
} else {
out(indent, "{}", extractRawBytes(false, space));
}
}
};
diff << DiffComplex("KTXcubemapIncomplete", "/keyValueData/KTXcubemapIncomplete",
KVEntry::load<KTXcubemapIncomplete>(entry1), KVEntry::load<KTXcubemapIncomplete>(entry2));
} else {
struct KVEntryUnknown : public KVEntry {
void printText(PrintIndent& out, const char*) const {
out(0, " {}\n", extractRawBytes(true));
}
void printJson(PrintIndent& out, int indent, const char* space, const char*) const {
out(indent, "{}", extractRawBytes(false, space));
}
};
diff << DiffComplex(key, fmt::format("/keyValueData/{}", key),
KVEntry::load<KVEntryUnknown>(entry1), KVEntry::load<KVEntryUnknown>(entry2));
}
}
void CommandCompare::compareSGD(PrintDiff& diff, InputStreams& streams) {
if (options.ignoreSGD == IgnoreSGD::all) return;
auto sgdTypeBasisLZ = [&](std::size_t i) { return headers[i].supercompressionScheme == KTX_SS_BASIS_LZ; };
std::unique_ptr<uint8_t[]> buffers[] = {
std::make_unique<uint8_t[]>(headers[0].supercompressionGlobalData.byteLength),
std::make_unique<uint8_t[]>(headers[1].supercompressionGlobalData.byteLength)
};
for (std::size_t i = 0; i < streams.size(); ++i)
read(streams[i], headers[i].supercompressionGlobalData.byteOffset, buffers[i].get(),
headers[i].supercompressionGlobalData.byteLength, "the SGD");
// BasisLZ specific parameters
struct SGDBasisLZ {
std::optional<uint16_t> endpointCount;
std::optional<uint16_t> selectorCount;
std::optional<uint64_t> endpointsByteLength;
std::optional<uint64_t> selectorsByteLength;
std::optional<uint64_t> tablesByteLength;
std::optional<uint64_t> extendedByteLength;
std::vector<std::optional<uint32_t>> imageFlags;
std::vector<std::optional<uint32_t>> rgbSliceByteOffset;
std::vector<std::optional<uint32_t>> rgbSliceByteLength;
std::vector<std::optional<uint32_t>> alphaSliceByteOffset;
std::vector<std::optional<uint32_t>> alphaSliceByteLength;
std::optional<uint64_t> endpointsByteOffset;
std::optional<uint64_t> selectorsByteOffset;
std::optional<uint64_t> tablesByteOffset;
std::optional<uint64_t> extendedByteOffset;
} basisLZ[2] = {};
for (std::size_t i = 0; i < streams.size(); ++i) {
const auto sgdByteLength = headers[i].supercompressionGlobalData.byteLength;
if (sgdByteLength == 0) continue;
read(streams[i], headers[i].supercompressionGlobalData.byteOffset, buffers[i].get(), sgdByteLength, "the SGD");
if (sgdTypeBasisLZ(i)) {
uint32_t imageCount = 0;
for (uint32_t level = 0; level < std::max(headers[i].levelCount, 1u); ++level)
// numFaces * depth is only reasonable because they can't both be > 1. There are no 3D cubemaps
imageCount += std::max(headers[i].layerCount, 1u) * headers[i].faceCount * std::max(headers[i].pixelDepth >> level, 1u);
if (sgdByteLength < sizeof(ktxBasisLzGlobalHeader))
continue;
const ktxBasisLzGlobalHeader& bgh = *reinterpret_cast<const ktxBasisLzGlobalHeader*>(buffers[i].get());
basisLZ[i].endpointCount = bgh.endpointCount;
basisLZ[i].selectorCount = bgh.selectorCount;
basisLZ[i].endpointsByteLength = bgh.endpointsByteLength;
basisLZ[i].selectorsByteLength = bgh.selectorsByteLength;
basisLZ[i].tablesByteLength = bgh.tablesByteLength;
basisLZ[i].extendedByteLength = bgh.extendedByteLength;
if (sgdByteLength < sizeof(ktxBasisLzGlobalHeader) + sizeof(ktxBasisLzEtc1sImageDesc) * imageCount)
continue;
const ktxBasisLzEtc1sImageDesc* imageDesc = BGD_ETC1S_IMAGE_DESCS(buffers[i].get());
for (uint32_t level = 0; level < std::max(headers[i].levelCount, 1u); ++level) {
for (uint32_t layer = 0; layer < std::max(headers[i].layerCount, 1u); ++layer) {
for (uint32_t face = 0; face < headers[i].faceCount; ++face) {
for (uint32_t zSlice = 0; zSlice < std::max(headers[i].pixelDepth >> level, 1u); ++zSlice) {
basisLZ[i].imageFlags.push_back(imageDesc->imageFlags);
basisLZ[i].rgbSliceByteOffset.push_back(imageDesc->rgbSliceByteOffset);
basisLZ[i].rgbSliceByteLength.push_back(imageDesc->rgbSliceByteLength);
basisLZ[i].alphaSliceByteLength.push_back(imageDesc->alphaSliceByteLength);
basisLZ[i].alphaSliceByteOffset.push_back(imageDesc->alphaSliceByteOffset);
imageDesc++;
}
}
}
}
// Calculate payload offsets
basisLZ[i].endpointsByteOffset = sizeof(ktxBasisLzGlobalHeader) + sizeof(ktxBasisLzEtc1sImageDesc) * imageCount;
basisLZ[i].selectorsByteOffset = *basisLZ[i].endpointsByteOffset + *basisLZ[i].endpointsByteLength;
basisLZ[i].tablesByteOffset = *basisLZ[i].selectorsByteOffset + *basisLZ[i].selectorsByteLength;
basisLZ[i].extendedByteOffset = *basisLZ[i].tablesByteOffset + *basisLZ[i].tablesByteLength;
}
}
// Helper for comparing SGD payloads
auto compareSGDPayload = [&](const char* textName, const char* jsonPath,
std::optional<uint64_t> offset1, std::optional<uint64_t> length1,
std::optional<uint64_t> offset2, std::optional<uint64_t> length2) {
bool mismatch = false;
// If SGD is not present in both files then consider that a mismatch
if (!offset1.has_value() || !length1.has_value() || !offset2.has_value() || !length2.has_value())
mismatch = true;
if (!mismatch)
// If we have an out of bounds situation then consider that a mismatch
if ((*offset1 + *length1 > headers[0].supercompressionGlobalData.byteLength) ||
(*offset2 + *length2 > headers[1].supercompressionGlobalData.byteLength))
mismatch = true;
if (!mismatch) {
if (length1 != length2) {
mismatch = true;
} else {
mismatch = (memcmp(buffers[0].get() + *offset1, buffers[1].get() + *offset2, *length1) != 0);
}
}
if (mismatch)
diff << DiffMismatch(fmt::format("{} mismatch", textName), jsonPath);
};
if (sgdTypeBasisLZ(0) || sgdTypeBasisLZ(1)) {
diff.setContext("Basis Supercompression Global Data\n\n");
// Supercompression global data type is only needed in JSON format
if (options.format != OutputFormat::text)
diff << DiffEnum<ktxSupercmpScheme>("supercompressionScheme", "/supercompressionGlobalData/type",
headers[0].supercompressionScheme, headers[1].supercompressionScheme);
diff << Diff("endpointCount", "/supercompressionGlobalData/endpointCount",
basisLZ[0].endpointCount, basisLZ[1].endpointCount);
diff << Diff("selectorCount", "/supercompressionGlobalData/selectorCount",
basisLZ[0].selectorCount, basisLZ[1].selectorCount);
diff << Diff("endpointsByteLength", "/supercompressionGlobalData/endpointsByteLength",
basisLZ[0].endpointsByteLength, basisLZ[1].endpointsByteLength);
diff << Diff("selectorsByteLength", "/supercompressionGlobalData/selectorsByteLength",
basisLZ[0].selectorsByteLength, basisLZ[1].selectorsByteLength);
diff << Diff("tablesByteLength", "/supercompressionGlobalData/tablesByteLength",
basisLZ[0].tablesByteLength, basisLZ[1].tablesByteLength);
diff << Diff("extendedByteLength", "/supercompressionGlobalData/extendedByteLength",
basisLZ[0].extendedByteLength, basisLZ[1].extendedByteLength);
// Make the per image arrays the same size for easier diffing
std::size_t maxImageCount = std::max(basisLZ[0].imageFlags.size(), basisLZ[1].imageFlags.size());
for (std::size_t i = 0; i < streams.size(); ++i) {
basisLZ[i].imageFlags.resize(maxImageCount);
basisLZ[i].rgbSliceByteOffset.resize(maxImageCount);
basisLZ[i].rgbSliceByteLength.resize(maxImageCount);
basisLZ[i].alphaSliceByteOffset.resize(maxImageCount);
basisLZ[i].alphaSliceByteLength.resize(maxImageCount);
}
for (std::size_t imageIndex = 0; imageIndex < maxImageCount; ++imageIndex) {
diff << DiffFlags(fmt::format("Image{}.imageFlags", imageIndex),
fmt::format("/supercompressionGlobalData/images/{}/imageFlags", imageIndex),
basisLZ[0].imageFlags[imageIndex], basisLZ[1].imageFlags[imageIndex], ktxBUImageFlagsBitString);
diff << Diff(fmt::format("Image{}.rgbSliceByteLength", imageIndex),
fmt::format("/supercompressionGlobalData/images/{}/rgbSliceByteLength", imageIndex),
basisLZ[0].rgbSliceByteLength[imageIndex], basisLZ[1].rgbSliceByteLength[imageIndex]);
diff << Diff(fmt::format("Image{}.rgbSliceByteOffset", imageIndex),
fmt::format("/supercompressionGlobalData/images/{}/rgbSliceByteOffset", imageIndex),
basisLZ[0].rgbSliceByteOffset[imageIndex], basisLZ[1].rgbSliceByteOffset[imageIndex]);
diff << Diff(fmt::format("Image{}.alphaSliceByteLength", imageIndex),
fmt::format("/supercompressionGlobalData/images/{}/alphaSliceByteLength", imageIndex),
basisLZ[0].alphaSliceByteLength[imageIndex], basisLZ[1].alphaSliceByteLength[imageIndex]);
diff << Diff(fmt::format("Image{}.alphaSliceByteOffset", imageIndex),
fmt::format("/supercompressionGlobalData/images/{}/alphaSliceByteOffset", imageIndex),
basisLZ[0].alphaSliceByteOffset[imageIndex], basisLZ[1].alphaSliceByteOffset[imageIndex]);
}
if (options.ignoreSGD != IgnoreSGD::payload) {
compareSGDPayload("endpointsData", "/supercompressionGlobalData/endpointsData",
basisLZ[0].endpointsByteOffset, basisLZ[0].endpointsByteLength,
basisLZ[1].endpointsByteOffset, basisLZ[1].endpointsByteLength);
compareSGDPayload("selectorsData", "/supercompressionGlobalData/selectorsData",
basisLZ[0].selectorsByteOffset, basisLZ[0].selectorsByteLength,
basisLZ[1].selectorsByteOffset, basisLZ[1].selectorsByteLength);
compareSGDPayload("tablesData", "/supercompressionGlobalData/tablesData",
basisLZ[0].tablesByteOffset, basisLZ[0].tablesByteLength,
basisLZ[1].tablesByteOffset, basisLZ[1].tablesByteLength);
compareSGDPayload("extendedData", "/supercompressionGlobalData/extendedData",
basisLZ[0].extendedByteOffset, basisLZ[0].extendedByteLength,
basisLZ[1].extendedByteOffset, basisLZ[1].extendedByteLength);
}
} else if (options.ignoreSGD == IgnoreSGD::none) {
diff.setContext("Unrecognized Supercompression Global Data\n\n");
// Just compare raw payloads of the SGDs
compareSGDPayload("SGD", "/supercompressionGlobalData/rawPayload",
0, headers[0].supercompressionGlobalData.byteLength,
0, headers[1].supercompressionGlobalData.byteLength);
}
}
void CommandCompare::compareImages(PrintDiff& diff, InputStreams& streams) {
switch (options.contentMode) {
case ContentMode::ignore:
// Nothing to do
break;
case ContentMode::raw:
compareImagesRaw(diff, streams);
break;
case ContentMode::image:
compareImagesPerPixel(diff, streams);
break;
}
}
void CommandCompare::compareImagesRaw(PrintDiff& diff, InputStreams& streams) {
diff.setContext("Image Data\n\n");
const uint32_t numLevels[] = {
std::max(1u, headers[0].levelCount),
std::max(1u, headers[1].levelCount)
};
const uint32_t maxNumLevels = std::max(numLevels[0], numLevels[1]);
for (uint32_t level = 0; level < maxNumLevels; ++level) {
const auto levelIndexEntryOffset = sizeof(KTX_header2) + level * sizeof(ktxLevelIndexEntry);
std::optional<ktxLevelIndexEntry> levelIndexEntry[2];
for (std::size_t i = 0; i < streams.size(); ++i)
if (level < numLevels[i]) {
ktxLevelIndexEntry entry;
read(streams[i], levelIndexEntryOffset, &entry, sizeof(entry), "the level index");
levelIndexEntry[i] = entry;
}
bool mismatch = false;
// Missing levels are always considered a mismatch
if (!levelIndexEntry[0].has_value() || !levelIndexEntry[1].has_value())
mismatch = true;
// Mismatching level data sizes are always considered a mismatch
if (!mismatch && levelIndexEntry[0]->byteLength != levelIndexEntry[1]->byteLength)
mismatch = true;
if (!mismatch) {
// If so far so good then load the level data and compare them
std::unique_ptr<uint8_t[]> buffers[] = {
std::make_unique<uint8_t[]>(levelIndexEntry[0]->byteLength),
std::make_unique<uint8_t[]>(levelIndexEntry[1]->byteLength)
};
for (std::size_t i = 0; i < streams.size(); ++i)
read(streams[i], levelIndexEntry[i]->byteOffset, buffers[i].get(),
levelIndexEntry[i]->byteLength, fmt::format("level {} data", level));
if (std::memcmp(buffers[0].get(), buffers[1].get(), levelIndexEntry[0]->byteLength) != 0)
mismatch = true;
}
if (mismatch)
diff << DiffMismatch(fmt::format("Mismatch in level {} data", level), fmt::format("m={}", level));
}
}
void CommandCompare::compareImagesPerPixel(PrintDiff& diff, InputStreams& streams) {
diff.setContext("Image Data\n\n");
// Reset the streams to the beginning of the files
streams[0]->seekg(0);
streams[1]->seekg(0);
// Load texture data, setup format descriptors and image codecs
KTXTexture2 textures[2] = {KTXTexture2(nullptr), KTXTexture2(nullptr)};
StreambufStream<std::streambuf*> ktx2Streams[2] = {
StreambufStream<std::streambuf*>(streams[0]->rdbuf(), std::ios::in | std::ios::binary),
StreambufStream<std::streambuf*>(streams[1]->rdbuf(), std::ios::in | std::ios::binary)
};
FormatDescriptor formatDesc[2] = {};
ImageCodec imageCodecs[2] = {};
bool fileOffsetsValid[2] = {true, true};
for (std::size_t i = 0; i < streams.size(); ++i) {
auto ret = ktxTexture2_CreateFromStream(ktx2Streams[i].stream(), KTX_TEXTURE_CREATE_LOAD_IMAGE_DATA_BIT, textures[i].pHandle());
if (ret != KTX_SUCCESS)
fatal(rc::INVALID_FILE, "Failed to create KTX2 texture from file \"{}\": {}", streams[i].str(), ktxErrorString(ret));
formatDesc[i] = createFormatDescriptor(textures[i]->pDfd);
imageCodecs[i] = ImageCodec(static_cast<VkFormat>(headers[i].vkFormat), headers[i].typeSize, textures[i]->pDfd);
if (formatDesc[i].model() == KHR_DF_MODEL_ETC1S) {
// Transcode BasisLZ textures to RGBA8 before comparison
ret = ktxTexture2_TranscodeBasis(textures[i], KTX_TTF_RGBA32, 0);
if (ret != KTX_SUCCESS)
fatal(rc::INVALID_FILE, "Failed to transcode KTX2 texture from file \"{}\": {}", streams[i].str(), ktxErrorString(ret));
// Update format descriptor and image codec after transcoding
formatDesc[i] = createFormatDescriptor(textures[i]->pDfd);
imageCodecs[i] = ImageCodec(VK_FORMAT_R8G8B8A8_UNORM, 1, textures[i]->pDfd);
}
// If the image data was supercompressed then file offsets of texel blocks cannot be calculated
if (headers[i].supercompressionScheme != KTX_SS_NONE)
fileOffsetsValid[i] = false;
}
// Currently, we only support comparing images with matching dimensions
if (textures[0]->numDimensions != textures[1]->numDimensions ||
textures[0]->baseWidth != textures[1]->baseWidth ||
textures[0]->baseHeight != textures[1]->baseHeight ||
textures[0]->baseDepth != textures[1]->baseDepth ||
imageCodecs[0].getTexelBlockDimensions() != imageCodecs[1].getTexelBlockDimensions())
fatal(rc::INVALID_ARGUMENTS, "Comparison requires matching texture and texel block dimensions.");
// Currently, we only support comparing images with matching formats (barring BasisLZ decoding)
if (textures[0]->vkFormat != textures[1]->vkFormat)
fatal(rc::INVALID_ARGUMENTS, "Comparison requires matching texture formats (BasisLZ is treated as R8G8B8A8_UNORM).");
// Currently, we only support comparing raw packed elements, but this can be extended
// in the future with color value comparison (with or without tolerance), for example
CompareFunc compareFunc = compareTexelBlocksPacked;
const uint32_t maxNumLevels = std::max(textures[0]->numLevels, textures[1]->numLevels);
const uint32_t maxNumFaces = std::max(textures[0]->numFaces, textures[1]->numFaces);
const uint32_t maxNumLayers = std::max(textures[0]->numLayers, textures[1]->numLayers);
std::size_t texelBlockDifferences = 0;
for (uint32_t level = 0; level < maxNumLevels; ++level) {
// Calculate base file offset of level from level data
const auto levelIndexEntryOffset = sizeof(KTX_header2) + level * sizeof(ktxLevelIndexEntry);
std::optional<std::size_t> levelFileOffsets[2];
for (std::size_t i = 0; i < streams.size(); ++i)
if (fileOffsetsValid[i] && level < textures[i]->numLevels) {
ktxLevelIndexEntry entry;
read(streams[i], levelIndexEntryOffset, &entry, sizeof(entry), "the level index");
levelFileOffsets[i] = entry.byteOffset;
}
const auto imageWidth = std::max(1u, textures[0]->baseWidth >> level);
const auto imageHeight = std::max(1u, textures[0]->baseHeight >> level);
const auto imageDepth = std::max(1u, textures[0]->baseDepth >> level);
const auto texelBlockDims = imageCodecs[0].pixelToTexelBlockSize(glm::uvec4(imageWidth, imageHeight, imageDepth, 1));
// Size returned by libktx is only for a single layer/face/slice
const std::size_t imageSizes[2] = {
ktxTexture_GetImageSize(textures[0], level) * texelBlockDims.z,
ktxTexture_GetImageSize(textures[1], level) * texelBlockDims.z
};
for (uint32_t layer = 0; layer < maxNumLayers; ++layer) {
for (uint32_t face = 0; face < maxNumFaces; ++face) {
// Handle when image is missing from one of the files
bool missingImage = false;
for (std::size_t i = 0; i < streams.size(); ++i)
if (level >= textures[i]->numLevels || face >= textures[i]->numFaces || layer >= textures[i]->numLayers)
missingImage = true;
if (missingImage) {
diff << DiffImage(fmt::format("Mismatch in level {}, layer {}, face {}", level, layer, face),
fmt::format("m={},a={},f={}", level, layer, face), 0, 0, {});
continue;
}
// Calculate image file offset
std::optional<std::size_t> imageFileOffsets[2];
for (std::size_t i = 0; i < streams.size(); ++i)
if (levelFileOffsets[i].has_value())
imageFileOffsets[i] = *levelFileOffsets[i] + (face + layer * textures[i]->numFaces) * imageSizes[i];
// Get image data pointers and create image spans from them
char* imageDataPtr[2];
for (std::size_t i = 0; i < streams.size(); ++i) {
ktx_size_t imageOffset;
ktxTexture_GetImageOffset(textures[i], level, layer, face, &imageOffset);
imageDataPtr[i] = reinterpret_cast<char*>(textures[i]->pData) + imageOffset;
}
ImageSpan images[2] = {
ImageSpan(imageWidth, imageHeight, imageDepth, imageDataPtr[0], imageCodecs[0]),
ImageSpan(imageWidth, imageHeight, imageDepth, imageDataPtr[1], imageCodecs[1]),
};
// Loop through texel blocks
bool imageMismatch = false;
std::vector<std::pair<ImageSpan::TexelBlockPtr<>, ImageSpan::TexelBlockPtr<>>> mismatchingBlocks = {};
for (uint32_t blockZ = 0; blockZ < texelBlockDims.z; ++blockZ) {
for (uint32_t blockY = 0; blockY < texelBlockDims.y; ++blockY) {
for (uint32_t blockX = 0; blockX < texelBlockDims.x; ++blockX) {
ImageSpan::TexelBlockPtr<> texelBlocks[2] = {
images[0].at(blockX, blockY, blockZ),
images[1].at(blockX, blockY, blockZ)
};
if (!compareFunc(texelBlocks)) {
imageMismatch = true;
texelBlockDifferences += 1;
if (texelBlockDifferences <= options.perPixelOutputLimit) {
mismatchingBlocks.push_back(std::make_pair(texelBlocks[0], texelBlocks[1]));
}
}
}
}
}
if (imageMismatch) {
diff << DiffImage(fmt::format("Mismatch in level {} layer {} face {}", level, layer, face),
fmt::format("m={},a={},f={}", level, layer, face), imageFileOffsets[0], imageFileOffsets[1],
mismatchingBlocks);
}
}
}
}
}
} // namespace ktx
KTX_COMMAND_ENTRY_POINT(ktxCompare, ktx::CommandCompare)
// Copyright 2023-2024 The Khronos Group Inc.
// Copyright 2023-2024 RasterGrid Kft.
// SPDX-License-Identifier: Apache-2.0
// -------------------------------------------------------------------------------------------------
namespace ktx {
extern "C" {
const char* ktxBUImageFlagsBitString(ktx_uint32_t bit_index, bool bit_value);
}
template <typename T>
struct DiffBase {
DiffBase(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const std::optional<T>& value1, const std::optional<T>& value2)
: textHeader(textHeaderIn), jsonPath(jsonPathIn), values(), different(value1 != value2) {
values[0] = value1;
values[1] = value2;
}
DiffBase(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const T& value1, const T& value2)
: textHeader(textHeaderIn), jsonPath(jsonPathIn), values(), different(value1 != value2) {
values[0] = value1;
values[1] = value2;
}
const std::string_view textHeader;
const std::string_view jsonPath;
bool isDifferent() const {
return different;
}
bool hasValue(std::size_t index) const {
return values[index].has_value();
}
virtual std::string value(std::size_t index, OutputFormat format) const = 0;
protected:
const T& rawValue(std::size_t index) const {
return *values[index];
}
private:
std::optional<T> values[2];
const bool different;
};
template <typename T>
struct Diff : public DiffBase<T> {
Diff(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const std::optional<T>& value1, const std::optional<T>& value2)
: DiffBase<T>(textHeaderIn, jsonPathIn, value1, value2) {}
Diff(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const T& value1, const T& value2)
: DiffBase<T>(textHeaderIn, jsonPathIn, value1, value2) {}
virtual std::string value(std::size_t index, OutputFormat) const override {
return fmt::format("{}", DiffBase<T>::rawValue(index));
}
};
struct DiffIdentifier : public DiffBase<std::array<uint8_t, sizeof(KTX_header2::identifier)>> {
DiffIdentifier(const std::string_view textHeaderIn, const std::string_view jsonPathIn,
const KTX_header2& value1, const KTX_header2& value2)
: DiffBase<std::array<uint8_t, sizeof(KTX_header2::identifier)>>(textHeaderIn, jsonPathIn,
to_array(value1), to_array(value2)) {
}
virtual std::string value(std::size_t index, OutputFormat format) const override {
// Convert identifier for better display.
auto identifier = rawValue(index);
uint32_t idlen = 0;