diff --git a/third_party/tracy/.github/FUNDING.yml b/third_party/tracy/.github/FUNDING.yml new file mode 100644 index 0000000000000000000000000000000000000000..976135c82dddd81053237b52ebde5e04ea7626e8 --- /dev/null +++ b/third_party/tracy/.github/FUNDING.yml @@ -0,0 +1 @@ +github: wolfpld diff --git a/third_party/tracy/.github/sponsor.png b/third_party/tracy/.github/sponsor.png new file mode 100644 index 0000000000000000000000000000000000000000..ce689b149a50b86be270cbec5f62732f2c5600f2 Binary files /dev/null and b/third_party/tracy/.github/sponsor.png differ diff --git a/third_party/tracy/.github/workflows/build.yml b/third_party/tracy/.github/workflows/build.yml new file mode 100644 index 0000000000000000000000000000000000000000..ff00fac0c0a9b95adc8bf0dc317c91267dfdcfeb --- /dev/null +++ b/third_party/tracy/.github/workflows/build.yml @@ -0,0 +1,78 @@ +name: build + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + strategy: + matrix: + os: [ windows-latest, macos-latest ] + + runs-on: ${{ matrix.os }} + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - if: startsWith(matrix.os, 'windows') + uses: microsoft/setup-msbuild@v2 + - if: startsWith(matrix.os, 'windows') + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - if: startsWith(matrix.os, 'windows') + run: pip install meson ninja + - if: startsWith(matrix.os, 'macos') + name: Install macos dependencies + run: brew install pkg-config glfw meson + - name: Profiler GUI + run: | + cmake -B profiler/build -S profiler -DCMAKE_BUILD_TYPE=Release + cmake --build profiler/build --parallel --config Release + - name: Update utility + run: | + cmake -B update/build -S update -DCMAKE_BUILD_TYPE=Release + cmake --build update/build --parallel --config Release + - name: Capture utility + run: | + cmake -B capture/build -S capture -DCMAKE_BUILD_TYPE=Release + cmake --build capture/build --parallel --config Release + - name: Csvexport utility + run: | + cmake -B csvexport/build -S csvexport -DCMAKE_BUILD_TYPE=Release + cmake --build csvexport/build --parallel --config Release + - name: Import utilities + run: | + cmake -B import/build -S import -DCMAKE_BUILD_TYPE=Release + cmake --build import/build --parallel --config Release + - if: ${{ !startsWith(matrix.os, 'windows') }} + name: Library + run: meson setup -Dprefix=$GITHUB_WORKSPACE/bin/lib build && meson compile -C build && meson install -C build + - if: ${{ !startsWith(matrix.os, 'windows') }} + name: Find Artifacts + id: find_artifacts + run: | + mkdir -p bin + cp profiler/build/tracy-profiler bin + cp update/build/tracy-update bin + cp capture/build/tracy-capture bin + cp csvexport/build/tracy-csvexport bin + cp import/build/tracy-import-chrome bin + cp import/build/tracy-import-fuchsia bin + - if: startsWith(matrix.os, 'windows') + name: Find Artifacts + id: find_artifacts_windows + run: | + mkdir bin + copy profiler\build\Release\tracy-profiler.exe bin + copy update\build\Release\tracy-update.exe bin + copy capture\build\Release\tracy-capture.exe bin + copy csvexport\build\Release\tracy-csvexport.exe bin + copy import\build\Release\tracy-import-chrome.exe bin + copy import\build\Release\tracy-import-fuchsia.exe bin + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }} + path: bin diff --git a/third_party/tracy/.github/workflows/latex.yml b/third_party/tracy/.github/workflows/latex.yml new file mode 100644 index 0000000000000000000000000000000000000000..8482baa8592bc972a0352f6242b132784e4dee24 --- /dev/null +++ b/third_party/tracy/.github/workflows/latex.yml @@ -0,0 +1,27 @@ +name: Manual + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Fix stupidity + run: | + cp LICENSE LICENSE. + - name: Compile LaTeX + uses: xu-cheng/latex-action@v3 + with: + working_directory: manual + root_file: tracy.tex + - uses: actions/upload-artifact@v4 + with: + name: manual + path: manual/tracy.pdf diff --git a/third_party/tracy/.github/workflows/linux.yml b/third_party/tracy/.github/workflows/linux.yml new file mode 100644 index 0000000000000000000000000000000000000000..b0ed2592656bf3a0cd1eaddf6582acbc5b608315 --- /dev/null +++ b/third_party/tracy/.github/workflows/linux.yml @@ -0,0 +1,75 @@ +name: linux + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + runs-on: ubuntu-latest + container: archlinux:base-devel + steps: + - name: Install dependencies + run: pacman -Syu --noconfirm && pacman -S --noconfirm --needed freetype2 tbb debuginfod wayland dbus libxkbcommon libglvnd meson cmake git wayland-protocols nodejs + - uses: actions/checkout@v4 + - name: Profiler GUI + run: | + cmake -B profiler/build -S profiler -DCMAKE_BUILD_TYPE=Release + cmake --build profiler/build --parallel + - name: Update utility + run: | + cmake -B update/build -S update -DCMAKE_BUILD_TYPE=Release + cmake --build update/build --parallel + - name: Capture utility + run: | + cmake -B capture/build -S capture -DCMAKE_BUILD_TYPE=Release + cmake --build capture/build --parallel + - name: Csvexport utility + run: | + cmake -B csvexport/build -S csvexport -DCMAKE_BUILD_TYPE=Release + cmake --build csvexport/build --parallel + - name: Import utilities + run: | + cmake -B import/build -S import -DCMAKE_BUILD_TYPE=Release + cmake --build import/build --parallel + - name: Library + run: meson setup -Dprefix=$GITHUB_WORKSPACE/bin/lib build && meson compile -C build && meson install -C build + - name: Test application + run: | + # test compilation with different flags + # we clean the build folder to reset cached variables between runs + cmake -B test/build -S test -DCMAKE_BUILD_TYPE=Release + cmake --build test/build --parallel + rm -rf test/build + + # same with TRACY_ON_DEMAND + cmake -B test/build -S test -DCMAKE_BUILD_TYPE=Release -DTRACY_ON_DEMAND=ON . + cmake --build test/build --parallel + rm -rf test/build + + # same with TRACY_DELAYED_INIT TRACY_MANUAL_LIFETIME + cmake -B test/build -S test -DCMAKE_BUILD_TYPE=Release -DTRACY_DELAYED_INIT=ON -DTRACY_MANUAL_LIFETIME=ON . + cmake --build test/build --parallel + rm -rf test/build + + # same with TRACY_DEMANGLE + cmake -B test/build -S test -DCMAKE_BUILD_TYPE=Release -DTRACY_DEMANGLE=ON . + cmake --build test/build --parallel + rm -rf test/build + - name: Find Artifacts + id: find_artifacts + run: | + mkdir -p bin + cp profiler/build/tracy-profiler bin + cp update/build/tracy-update bin + cp capture/build/tracy-capture bin + cp csvexport/build/tracy-csvexport bin + cp import/build/tracy-import-chrome bin + cp import/build/tracy-import-fuchsia bin + strip bin/tracy-* + - uses: actions/upload-artifact@v4 + with: + name: arch-linux + path: bin diff --git a/third_party/tracy/cmake/CPM.cmake b/third_party/tracy/cmake/CPM.cmake new file mode 100644 index 0000000000000000000000000000000000000000..8269a8bf6553529b4a10f8fe42dfa62f9805c1a4 --- /dev/null +++ b/third_party/tracy/cmake/CPM.cmake @@ -0,0 +1,1269 @@ +# CPM.cmake - CMake's missing package manager +# =========================================== +# See https://github.com/cpm-cmake/CPM.cmake for usage and update instructions. +# +# MIT License +# ----------- +#[[ + Copyright (c) 2019-2023 Lars Melchior and contributors + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +]] + +cmake_minimum_required(VERSION 3.14 FATAL_ERROR) + +# Initialize logging prefix +if(NOT CPM_INDENT) + set(CPM_INDENT + "CPM:" + CACHE INTERNAL "" + ) +endif() + +if(NOT COMMAND cpm_message) + function(cpm_message) + message(${ARGV}) + endfunction() +endif() + +set(CURRENT_CPM_VERSION 0.40.2) + +get_filename_component(CPM_CURRENT_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}" REALPATH) +if(CPM_DIRECTORY) + if(NOT CPM_DIRECTORY STREQUAL CPM_CURRENT_DIRECTORY) + if(CPM_VERSION VERSION_LESS CURRENT_CPM_VERSION) + message( + AUTHOR_WARNING + "${CPM_INDENT} \ +A dependency is using a more recent CPM version (${CURRENT_CPM_VERSION}) than the current project (${CPM_VERSION}). \ +It is recommended to upgrade CPM to the most recent version. \ +See https://github.com/cpm-cmake/CPM.cmake for more information." + ) + endif() + if(${CMAKE_VERSION} VERSION_LESS "3.17.0") + include(FetchContent) + endif() + return() + endif() + + get_property( + CPM_INITIALIZED GLOBAL "" + PROPERTY CPM_INITIALIZED + SET + ) + if(CPM_INITIALIZED) + return() + endif() +endif() + +if(CURRENT_CPM_VERSION MATCHES "development-version") + message( + WARNING "${CPM_INDENT} Your project is using an unstable development version of CPM.cmake. \ +Please update to a recent release if possible. \ +See https://github.com/cpm-cmake/CPM.cmake for details." + ) +endif() + +set_property(GLOBAL PROPERTY CPM_INITIALIZED true) + +macro(cpm_set_policies) + # the policy allows us to change options without caching + cmake_policy(SET CMP0077 NEW) + set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) + + # the policy allows us to change set(CACHE) without caching + if(POLICY CMP0126) + cmake_policy(SET CMP0126 NEW) + set(CMAKE_POLICY_DEFAULT_CMP0126 NEW) + endif() + + # The policy uses the download time for timestamp, instead of the timestamp in the archive. This + # allows for proper rebuilds when a projects url changes + if(POLICY CMP0135) + cmake_policy(SET CMP0135 NEW) + set(CMAKE_POLICY_DEFAULT_CMP0135 NEW) + endif() + + # treat relative git repository paths as being relative to the parent project's remote + if(POLICY CMP0150) + cmake_policy(SET CMP0150 NEW) + set(CMAKE_POLICY_DEFAULT_CMP0150 NEW) + endif() +endmacro() +cpm_set_policies() + +option(CPM_USE_LOCAL_PACKAGES "Always try to use `find_package` to get dependencies" + $ENV{CPM_USE_LOCAL_PACKAGES} +) +option(CPM_LOCAL_PACKAGES_ONLY "Only use `find_package` to get dependencies" + $ENV{CPM_LOCAL_PACKAGES_ONLY} +) +option(CPM_DOWNLOAD_ALL "Always download dependencies from source" $ENV{CPM_DOWNLOAD_ALL}) +option(CPM_DONT_UPDATE_MODULE_PATH "Don't update the module path to allow using find_package" + $ENV{CPM_DONT_UPDATE_MODULE_PATH} +) +option(CPM_DONT_CREATE_PACKAGE_LOCK "Don't create a package lock file in the binary path" + $ENV{CPM_DONT_CREATE_PACKAGE_LOCK} +) +option(CPM_INCLUDE_ALL_IN_PACKAGE_LOCK + "Add all packages added through CPM.cmake to the package lock" + $ENV{CPM_INCLUDE_ALL_IN_PACKAGE_LOCK} +) +option(CPM_USE_NAMED_CACHE_DIRECTORIES + "Use additional directory of package name in cache on the most nested level." + $ENV{CPM_USE_NAMED_CACHE_DIRECTORIES} +) + +set(CPM_VERSION + ${CURRENT_CPM_VERSION} + CACHE INTERNAL "" +) +set(CPM_DIRECTORY + ${CPM_CURRENT_DIRECTORY} + CACHE INTERNAL "" +) +set(CPM_FILE + ${CMAKE_CURRENT_LIST_FILE} + CACHE INTERNAL "" +) +set(CPM_PACKAGES + "" + CACHE INTERNAL "" +) +set(CPM_DRY_RUN + OFF + CACHE INTERNAL "Don't download or configure dependencies (for testing)" +) + +if(DEFINED ENV{CPM_SOURCE_CACHE}) + set(CPM_SOURCE_CACHE_DEFAULT $ENV{CPM_SOURCE_CACHE}) +else() + set(CPM_SOURCE_CACHE_DEFAULT OFF) +endif() + +set(CPM_SOURCE_CACHE + ${CPM_SOURCE_CACHE_DEFAULT} + CACHE PATH "Directory to download CPM dependencies" +) + +if(NOT CPM_DONT_UPDATE_MODULE_PATH) + set(CPM_MODULE_PATH + "${CMAKE_BINARY_DIR}/CPM_modules" + CACHE INTERNAL "" + ) + # remove old modules + file(REMOVE_RECURSE ${CPM_MODULE_PATH}) + file(MAKE_DIRECTORY ${CPM_MODULE_PATH}) + # locally added CPM modules should override global packages + set(CMAKE_MODULE_PATH "${CPM_MODULE_PATH};${CMAKE_MODULE_PATH}") +endif() + +if(NOT CPM_DONT_CREATE_PACKAGE_LOCK) + set(CPM_PACKAGE_LOCK_FILE + "${CMAKE_BINARY_DIR}/cpm-package-lock.cmake" + CACHE INTERNAL "" + ) + file(WRITE ${CPM_PACKAGE_LOCK_FILE} + "# CPM Package Lock\n# This file should be committed to version control\n\n" + ) +endif() + +include(FetchContent) + +# Try to infer package name from git repository uri (path or url) +function(cpm_package_name_from_git_uri URI RESULT) + if("${URI}" MATCHES "([^/:]+)/?.git/?$") + set(${RESULT} + ${CMAKE_MATCH_1} + PARENT_SCOPE + ) + else() + unset(${RESULT} PARENT_SCOPE) + endif() +endfunction() + +# Try to infer package name and version from a url +function(cpm_package_name_and_ver_from_url url outName outVer) + if(url MATCHES "[/\\?]([a-zA-Z0-9_\\.-]+)\\.(tar|tar\\.gz|tar\\.bz2|zip|ZIP)(\\?|/|$)") + # We matched an archive + set(filename "${CMAKE_MATCH_1}") + + if(filename MATCHES "([a-zA-Z0-9_\\.-]+)[_-]v?(([0-9]+\\.)*[0-9]+[a-zA-Z0-9]*)") + # We matched - (ie foo-1.2.3) + set(${outName} + "${CMAKE_MATCH_1}" + PARENT_SCOPE + ) + set(${outVer} + "${CMAKE_MATCH_2}" + PARENT_SCOPE + ) + elseif(filename MATCHES "(([0-9]+\\.)+[0-9]+[a-zA-Z0-9]*)") + # We couldn't find a name, but we found a version + # + # In many cases (which we don't handle here) the url would look something like + # `irrelevant/ACTUAL_PACKAGE_NAME/irrelevant/1.2.3.zip`. In such a case we can't possibly + # distinguish the package name from the irrelevant bits. Moreover if we try to match the + # package name from the filename, we'd get bogus at best. + unset(${outName} PARENT_SCOPE) + set(${outVer} + "${CMAKE_MATCH_1}" + PARENT_SCOPE + ) + else() + # Boldly assume that the file name is the package name. + # + # Yes, something like `irrelevant/ACTUAL_NAME/irrelevant/download.zip` will ruin our day, but + # such cases should be quite rare. No popular service does this... we think. + set(${outName} + "${filename}" + PARENT_SCOPE + ) + unset(${outVer} PARENT_SCOPE) + endif() + else() + # No ideas yet what to do with non-archives + unset(${outName} PARENT_SCOPE) + unset(${outVer} PARENT_SCOPE) + endif() +endfunction() + +function(cpm_find_package NAME VERSION) + string(REPLACE " " ";" EXTRA_ARGS "${ARGN}") + find_package(${NAME} ${VERSION} ${EXTRA_ARGS} QUIET) + if(${CPM_ARGS_NAME}_FOUND) + if(DEFINED ${CPM_ARGS_NAME}_VERSION) + set(VERSION ${${CPM_ARGS_NAME}_VERSION}) + endif() + cpm_message(STATUS "${CPM_INDENT} Using local package ${CPM_ARGS_NAME}@${VERSION}") + CPMRegisterPackage(${CPM_ARGS_NAME} "${VERSION}") + set(CPM_PACKAGE_FOUND + YES + PARENT_SCOPE + ) + else() + set(CPM_PACKAGE_FOUND + NO + PARENT_SCOPE + ) + endif() +endfunction() + +# Create a custom FindXXX.cmake module for a CPM package This prevents `find_package(NAME)` from +# finding the system library +function(cpm_create_module_file Name) + if(NOT CPM_DONT_UPDATE_MODULE_PATH) + # erase any previous modules + file(WRITE ${CPM_MODULE_PATH}/Find${Name}.cmake + "include(\"${CPM_FILE}\")\n${ARGN}\nset(${Name}_FOUND TRUE)" + ) + endif() +endfunction() + +# Find a package locally or fallback to CPMAddPackage +function(CPMFindPackage) + set(oneValueArgs NAME VERSION GIT_TAG FIND_PACKAGE_ARGUMENTS) + + cmake_parse_arguments(CPM_ARGS "" "${oneValueArgs}" "" ${ARGN}) + + if(NOT DEFINED CPM_ARGS_VERSION) + if(DEFINED CPM_ARGS_GIT_TAG) + cpm_get_version_from_git_tag("${CPM_ARGS_GIT_TAG}" CPM_ARGS_VERSION) + endif() + endif() + + set(downloadPackage ${CPM_DOWNLOAD_ALL}) + if(DEFINED CPM_DOWNLOAD_${CPM_ARGS_NAME}) + set(downloadPackage ${CPM_DOWNLOAD_${CPM_ARGS_NAME}}) + elseif(DEFINED ENV{CPM_DOWNLOAD_${CPM_ARGS_NAME}}) + set(downloadPackage $ENV{CPM_DOWNLOAD_${CPM_ARGS_NAME}}) + endif() + if(downloadPackage) + CPMAddPackage(${ARGN}) + cpm_export_variables(${CPM_ARGS_NAME}) + return() + endif() + + cpm_find_package(${CPM_ARGS_NAME} "${CPM_ARGS_VERSION}" ${CPM_ARGS_FIND_PACKAGE_ARGUMENTS}) + + if(NOT CPM_PACKAGE_FOUND) + CPMAddPackage(${ARGN}) + cpm_export_variables(${CPM_ARGS_NAME}) + endif() + +endfunction() + +# checks if a package has been added before +function(cpm_check_if_package_already_added CPM_ARGS_NAME CPM_ARGS_VERSION) + if("${CPM_ARGS_NAME}" IN_LIST CPM_PACKAGES) + CPMGetPackageVersion(${CPM_ARGS_NAME} CPM_PACKAGE_VERSION) + if("${CPM_PACKAGE_VERSION}" VERSION_LESS "${CPM_ARGS_VERSION}") + message( + WARNING + "${CPM_INDENT} Requires a newer version of ${CPM_ARGS_NAME} (${CPM_ARGS_VERSION}) than currently included (${CPM_PACKAGE_VERSION})." + ) + endif() + cpm_get_fetch_properties(${CPM_ARGS_NAME}) + set(${CPM_ARGS_NAME}_ADDED NO) + set(CPM_PACKAGE_ALREADY_ADDED + YES + PARENT_SCOPE + ) + cpm_export_variables(${CPM_ARGS_NAME}) + else() + set(CPM_PACKAGE_ALREADY_ADDED + NO + PARENT_SCOPE + ) + endif() +endfunction() + +# Parse the argument of CPMAddPackage in case a single one was provided and convert it to a list of +# arguments which can then be parsed idiomatically. For example gh:foo/bar@1.2.3 will be converted +# to: GITHUB_REPOSITORY;foo/bar;VERSION;1.2.3 +function(cpm_parse_add_package_single_arg arg outArgs) + # Look for a scheme + if("${arg}" MATCHES "^([a-zA-Z]+):(.+)$") + string(TOLOWER "${CMAKE_MATCH_1}" scheme) + set(uri "${CMAKE_MATCH_2}") + + # Check for CPM-specific schemes + if(scheme STREQUAL "gh") + set(out "GITHUB_REPOSITORY;${uri}") + set(packageType "git") + elseif(scheme STREQUAL "gl") + set(out "GITLAB_REPOSITORY;${uri}") + set(packageType "git") + elseif(scheme STREQUAL "bb") + set(out "BITBUCKET_REPOSITORY;${uri}") + set(packageType "git") + # A CPM-specific scheme was not found. Looks like this is a generic URL so try to determine + # type + elseif(arg MATCHES ".git/?(@|#|$)") + set(out "GIT_REPOSITORY;${arg}") + set(packageType "git") + else() + # Fall back to a URL + set(out "URL;${arg}") + set(packageType "archive") + + # We could also check for SVN since FetchContent supports it, but SVN is so rare these days. + # We just won't bother with the additional complexity it will induce in this function. SVN is + # done by multi-arg + endif() + else() + if(arg MATCHES ".git/?(@|#|$)") + set(out "GIT_REPOSITORY;${arg}") + set(packageType "git") + else() + # Give up + message(FATAL_ERROR "${CPM_INDENT} Can't determine package type of '${arg}'") + endif() + endif() + + # For all packages we interpret @... as version. Only replace the last occurrence. Thus URIs + # containing '@' can be used + string(REGEX REPLACE "@([^@]+)$" ";VERSION;\\1" out "${out}") + + # Parse the rest according to package type + if(packageType STREQUAL "git") + # For git repos we interpret #... as a tag or branch or commit hash + string(REGEX REPLACE "#([^#]+)$" ";GIT_TAG;\\1" out "${out}") + elseif(packageType STREQUAL "archive") + # For archives we interpret #... as a URL hash. + string(REGEX REPLACE "#([^#]+)$" ";URL_HASH;\\1" out "${out}") + # We don't try to parse the version if it's not provided explicitly. cpm_get_version_from_url + # should do this at a later point + else() + # We should never get here. This is an assertion and hitting it means there's a problem with the + # code above. A packageType was set, but not handled by this if-else. + message(FATAL_ERROR "${CPM_INDENT} Unsupported package type '${packageType}' of '${arg}'") + endif() + + set(${outArgs} + ${out} + PARENT_SCOPE + ) +endfunction() + +# Check that the working directory for a git repo is clean +function(cpm_check_git_working_dir_is_clean repoPath gitTag isClean) + + find_package(Git REQUIRED) + + if(NOT GIT_EXECUTABLE) + # No git executable, assume directory is clean + set(${isClean} + TRUE + PARENT_SCOPE + ) + return() + endif() + + # check for uncommitted changes + execute_process( + COMMAND ${GIT_EXECUTABLE} status --porcelain + RESULT_VARIABLE resultGitStatus + OUTPUT_VARIABLE repoStatus + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET + WORKING_DIRECTORY ${repoPath} + ) + if(resultGitStatus) + # not supposed to happen, assume clean anyway + message(WARNING "${CPM_INDENT} Calling git status on folder ${repoPath} failed") + set(${isClean} + TRUE + PARENT_SCOPE + ) + return() + endif() + + if(NOT "${repoStatus}" STREQUAL "") + set(${isClean} + FALSE + PARENT_SCOPE + ) + return() + endif() + + # check for committed changes + execute_process( + COMMAND ${GIT_EXECUTABLE} diff -s --exit-code ${gitTag} + RESULT_VARIABLE resultGitDiff + OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_QUIET + WORKING_DIRECTORY ${repoPath} + ) + + if(${resultGitDiff} EQUAL 0) + set(${isClean} + TRUE + PARENT_SCOPE + ) + else() + set(${isClean} + FALSE + PARENT_SCOPE + ) + endif() + +endfunction() + +# Add PATCH_COMMAND to CPM_ARGS_UNPARSED_ARGUMENTS. This method consumes a list of files in ARGN +# then generates a `PATCH_COMMAND` appropriate for `ExternalProject_Add()`. This command is appended +# to the parent scope's `CPM_ARGS_UNPARSED_ARGUMENTS`. +function(cpm_add_patches) + # Return if no patch files are supplied. + if(NOT ARGN) + return() + endif() + + # Find the patch program. + find_program(PATCH_EXECUTABLE patch) + if(WIN32 AND NOT PATCH_EXECUTABLE) + # The Windows git executable is distributed with patch.exe. Find the path to the executable, if + # it exists, then search `../usr/bin` and `../../usr/bin` for patch.exe. + find_package(Git QUIET) + if(GIT_EXECUTABLE) + get_filename_component(extra_search_path ${GIT_EXECUTABLE} DIRECTORY) + get_filename_component(extra_search_path_1up ${extra_search_path} DIRECTORY) + get_filename_component(extra_search_path_2up ${extra_search_path_1up} DIRECTORY) + find_program( + PATCH_EXECUTABLE patch HINTS "${extra_search_path_1up}/usr/bin" + "${extra_search_path_2up}/usr/bin" + ) + endif() + endif() + if(NOT PATCH_EXECUTABLE) + message(FATAL_ERROR "Couldn't find `patch` executable to use with PATCHES keyword.") + endif() + + # Create a temporary + set(temp_list ${CPM_ARGS_UNPARSED_ARGUMENTS}) + + # Ensure each file exists (or error out) and add it to the list. + set(first_item True) + foreach(PATCH_FILE ${ARGN}) + # Make sure the patch file exists, if we can't find it, try again in the current directory. + if(NOT EXISTS "${PATCH_FILE}") + if(NOT EXISTS "${CMAKE_CURRENT_LIST_DIR}/${PATCH_FILE}") + message(FATAL_ERROR "Couldn't find patch file: '${PATCH_FILE}'") + endif() + set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/${PATCH_FILE}") + endif() + + # Convert to absolute path for use with patch file command. + get_filename_component(PATCH_FILE "${PATCH_FILE}" ABSOLUTE) + + # The first patch entry must be preceded by "PATCH_COMMAND" while the following items are + # preceded by "&&". + if(first_item) + set(first_item False) + list(APPEND temp_list "PATCH_COMMAND") + else() + list(APPEND temp_list "&&") + endif() + # Add the patch command to the list + list(APPEND temp_list "${PATCH_EXECUTABLE}" "-p1" "<" "${PATCH_FILE}") + endforeach() + + # Move temp out into parent scope. + set(CPM_ARGS_UNPARSED_ARGUMENTS + ${temp_list} + PARENT_SCOPE + ) + +endfunction() + +# method to overwrite internal FetchContent properties, to allow using CPM.cmake to overload +# FetchContent calls. As these are internal cmake properties, this method should be used carefully +# and may need modification in future CMake versions. Source: +# https://github.com/Kitware/CMake/blob/dc3d0b5a0a7d26d43d6cfeb511e224533b5d188f/Modules/FetchContent.cmake#L1152 +function(cpm_override_fetchcontent contentName) + cmake_parse_arguments(PARSE_ARGV 1 arg "" "SOURCE_DIR;BINARY_DIR" "") + if(NOT "${arg_UNPARSED_ARGUMENTS}" STREQUAL "") + message(FATAL_ERROR "${CPM_INDENT} Unsupported arguments: ${arg_UNPARSED_ARGUMENTS}") + endif() + + string(TOLOWER ${contentName} contentNameLower) + set(prefix "_FetchContent_${contentNameLower}") + + set(propertyName "${prefix}_sourceDir") + define_property( + GLOBAL + PROPERTY ${propertyName} + BRIEF_DOCS "Internal implementation detail of FetchContent_Populate()" + FULL_DOCS "Details used by FetchContent_Populate() for ${contentName}" + ) + set_property(GLOBAL PROPERTY ${propertyName} "${arg_SOURCE_DIR}") + + set(propertyName "${prefix}_binaryDir") + define_property( + GLOBAL + PROPERTY ${propertyName} + BRIEF_DOCS "Internal implementation detail of FetchContent_Populate()" + FULL_DOCS "Details used by FetchContent_Populate() for ${contentName}" + ) + set_property(GLOBAL PROPERTY ${propertyName} "${arg_BINARY_DIR}") + + set(propertyName "${prefix}_populated") + define_property( + GLOBAL + PROPERTY ${propertyName} + BRIEF_DOCS "Internal implementation detail of FetchContent_Populate()" + FULL_DOCS "Details used by FetchContent_Populate() for ${contentName}" + ) + set_property(GLOBAL PROPERTY ${propertyName} TRUE) +endfunction() + +# Download and add a package from source +function(CPMAddPackage) + cpm_set_policies() + + list(LENGTH ARGN argnLength) + if(argnLength EQUAL 1) + cpm_parse_add_package_single_arg("${ARGN}" ARGN) + + # The shorthand syntax implies EXCLUDE_FROM_ALL and SYSTEM + set(ARGN "${ARGN};EXCLUDE_FROM_ALL;YES;SYSTEM;YES;") + endif() + + set(oneValueArgs + NAME + FORCE + VERSION + GIT_TAG + DOWNLOAD_ONLY + GITHUB_REPOSITORY + GITLAB_REPOSITORY + BITBUCKET_REPOSITORY + GIT_REPOSITORY + SOURCE_DIR + FIND_PACKAGE_ARGUMENTS + NO_CACHE + SYSTEM + GIT_SHALLOW + EXCLUDE_FROM_ALL + SOURCE_SUBDIR + CUSTOM_CACHE_KEY + ) + + set(multiValueArgs URL OPTIONS DOWNLOAD_COMMAND PATCHES) + + cmake_parse_arguments(CPM_ARGS "" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") + + # Set default values for arguments + + if(NOT DEFINED CPM_ARGS_VERSION) + if(DEFINED CPM_ARGS_GIT_TAG) + cpm_get_version_from_git_tag("${CPM_ARGS_GIT_TAG}" CPM_ARGS_VERSION) + endif() + endif() + + if(CPM_ARGS_DOWNLOAD_ONLY) + set(DOWNLOAD_ONLY ${CPM_ARGS_DOWNLOAD_ONLY}) + else() + set(DOWNLOAD_ONLY NO) + endif() + + if(DEFINED CPM_ARGS_GITHUB_REPOSITORY) + set(CPM_ARGS_GIT_REPOSITORY "https://github.com/${CPM_ARGS_GITHUB_REPOSITORY}.git") + elseif(DEFINED CPM_ARGS_GITLAB_REPOSITORY) + set(CPM_ARGS_GIT_REPOSITORY "https://gitlab.com/${CPM_ARGS_GITLAB_REPOSITORY}.git") + elseif(DEFINED CPM_ARGS_BITBUCKET_REPOSITORY) + set(CPM_ARGS_GIT_REPOSITORY "https://bitbucket.org/${CPM_ARGS_BITBUCKET_REPOSITORY}.git") + endif() + + if(DEFINED CPM_ARGS_GIT_REPOSITORY) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS GIT_REPOSITORY ${CPM_ARGS_GIT_REPOSITORY}) + if(NOT DEFINED CPM_ARGS_GIT_TAG) + set(CPM_ARGS_GIT_TAG v${CPM_ARGS_VERSION}) + endif() + + # If a name wasn't provided, try to infer it from the git repo + if(NOT DEFINED CPM_ARGS_NAME) + cpm_package_name_from_git_uri(${CPM_ARGS_GIT_REPOSITORY} CPM_ARGS_NAME) + endif() + endif() + + set(CPM_SKIP_FETCH FALSE) + + if(DEFINED CPM_ARGS_GIT_TAG) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS GIT_TAG ${CPM_ARGS_GIT_TAG}) + # If GIT_SHALLOW is explicitly specified, honor the value. + if(DEFINED CPM_ARGS_GIT_SHALLOW) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS GIT_SHALLOW ${CPM_ARGS_GIT_SHALLOW}) + endif() + endif() + + if(DEFINED CPM_ARGS_URL) + # If a name or version aren't provided, try to infer them from the URL + list(GET CPM_ARGS_URL 0 firstUrl) + cpm_package_name_and_ver_from_url(${firstUrl} nameFromUrl verFromUrl) + # If we fail to obtain name and version from the first URL, we could try other URLs if any. + # However multiple URLs are expected to be quite rare, so for now we won't bother. + + # If the caller provided their own name and version, they trump the inferred ones. + if(NOT DEFINED CPM_ARGS_NAME) + set(CPM_ARGS_NAME ${nameFromUrl}) + endif() + if(NOT DEFINED CPM_ARGS_VERSION) + set(CPM_ARGS_VERSION ${verFromUrl}) + endif() + + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS URL "${CPM_ARGS_URL}") + endif() + + # Check for required arguments + + if(NOT DEFINED CPM_ARGS_NAME) + message( + FATAL_ERROR + "${CPM_INDENT} 'NAME' was not provided and couldn't be automatically inferred for package added with arguments: '${ARGN}'" + ) + endif() + + # Check if package has been added before + cpm_check_if_package_already_added(${CPM_ARGS_NAME} "${CPM_ARGS_VERSION}") + if(CPM_PACKAGE_ALREADY_ADDED) + cpm_export_variables(${CPM_ARGS_NAME}) + return() + endif() + + # Check for manual overrides + if(NOT CPM_ARGS_FORCE AND NOT "${CPM_${CPM_ARGS_NAME}_SOURCE}" STREQUAL "") + set(PACKAGE_SOURCE ${CPM_${CPM_ARGS_NAME}_SOURCE}) + set(CPM_${CPM_ARGS_NAME}_SOURCE "") + CPMAddPackage( + NAME "${CPM_ARGS_NAME}" + SOURCE_DIR "${PACKAGE_SOURCE}" + EXCLUDE_FROM_ALL "${CPM_ARGS_EXCLUDE_FROM_ALL}" + SYSTEM "${CPM_ARGS_SYSTEM}" + PATCHES "${CPM_ARGS_PATCHES}" + OPTIONS "${CPM_ARGS_OPTIONS}" + SOURCE_SUBDIR "${CPM_ARGS_SOURCE_SUBDIR}" + DOWNLOAD_ONLY "${DOWNLOAD_ONLY}" + FORCE True + ) + cpm_export_variables(${CPM_ARGS_NAME}) + return() + endif() + + # Check for available declaration + if(NOT CPM_ARGS_FORCE AND NOT "${CPM_DECLARATION_${CPM_ARGS_NAME}}" STREQUAL "") + set(declaration ${CPM_DECLARATION_${CPM_ARGS_NAME}}) + set(CPM_DECLARATION_${CPM_ARGS_NAME} "") + CPMAddPackage(${declaration}) + cpm_export_variables(${CPM_ARGS_NAME}) + # checking again to ensure version and option compatibility + cpm_check_if_package_already_added(${CPM_ARGS_NAME} "${CPM_ARGS_VERSION}") + return() + endif() + + if(NOT CPM_ARGS_FORCE) + if(CPM_USE_LOCAL_PACKAGES OR CPM_LOCAL_PACKAGES_ONLY) + cpm_find_package(${CPM_ARGS_NAME} "${CPM_ARGS_VERSION}" ${CPM_ARGS_FIND_PACKAGE_ARGUMENTS}) + + if(CPM_PACKAGE_FOUND) + cpm_export_variables(${CPM_ARGS_NAME}) + return() + endif() + + if(CPM_LOCAL_PACKAGES_ONLY) + message( + SEND_ERROR + "${CPM_INDENT} ${CPM_ARGS_NAME} not found via find_package(${CPM_ARGS_NAME} ${CPM_ARGS_VERSION})" + ) + endif() + endif() + endif() + + CPMRegisterPackage("${CPM_ARGS_NAME}" "${CPM_ARGS_VERSION}") + + if(DEFINED CPM_ARGS_GIT_TAG) + set(PACKAGE_INFO "${CPM_ARGS_GIT_TAG}") + elseif(DEFINED CPM_ARGS_SOURCE_DIR) + set(PACKAGE_INFO "${CPM_ARGS_SOURCE_DIR}") + else() + set(PACKAGE_INFO "${CPM_ARGS_VERSION}") + endif() + + if(DEFINED FETCHCONTENT_BASE_DIR) + # respect user's FETCHCONTENT_BASE_DIR if set + set(CPM_FETCHCONTENT_BASE_DIR ${FETCHCONTENT_BASE_DIR}) + else() + set(CPM_FETCHCONTENT_BASE_DIR ${CMAKE_BINARY_DIR}/_deps) + endif() + + cpm_add_patches(${CPM_ARGS_PATCHES}) + + if(DEFINED CPM_ARGS_DOWNLOAD_COMMAND) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS DOWNLOAD_COMMAND ${CPM_ARGS_DOWNLOAD_COMMAND}) + elseif(DEFINED CPM_ARGS_SOURCE_DIR) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS SOURCE_DIR ${CPM_ARGS_SOURCE_DIR}) + if(NOT IS_ABSOLUTE ${CPM_ARGS_SOURCE_DIR}) + # Expand `CPM_ARGS_SOURCE_DIR` relative path. This is important because EXISTS doesn't work + # for relative paths. + get_filename_component( + source_directory ${CPM_ARGS_SOURCE_DIR} REALPATH BASE_DIR ${CMAKE_CURRENT_BINARY_DIR} + ) + else() + set(source_directory ${CPM_ARGS_SOURCE_DIR}) + endif() + if(NOT EXISTS ${source_directory}) + string(TOLOWER ${CPM_ARGS_NAME} lower_case_name) + # remove timestamps so CMake will re-download the dependency + file(REMOVE_RECURSE "${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-subbuild") + endif() + elseif(CPM_SOURCE_CACHE AND NOT CPM_ARGS_NO_CACHE) + string(TOLOWER ${CPM_ARGS_NAME} lower_case_name) + set(origin_parameters ${CPM_ARGS_UNPARSED_ARGUMENTS}) + list(SORT origin_parameters) + if(CPM_ARGS_CUSTOM_CACHE_KEY) + # Application set a custom unique directory name + set(download_directory ${CPM_SOURCE_CACHE}/${lower_case_name}/${CPM_ARGS_CUSTOM_CACHE_KEY}) + elseif(CPM_USE_NAMED_CACHE_DIRECTORIES) + string(SHA1 origin_hash "${origin_parameters};NEW_CACHE_STRUCTURE_TAG") + set(download_directory ${CPM_SOURCE_CACHE}/${lower_case_name}/${origin_hash}/${CPM_ARGS_NAME}) + else() + string(SHA1 origin_hash "${origin_parameters}") + set(download_directory ${CPM_SOURCE_CACHE}/${lower_case_name}/${origin_hash}) + endif() + # Expand `download_directory` relative path. This is important because EXISTS doesn't work for + # relative paths. + get_filename_component(download_directory ${download_directory} ABSOLUTE) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS SOURCE_DIR ${download_directory}) + + if(CPM_SOURCE_CACHE) + file(LOCK ${download_directory}/../cmake.lock) + endif() + + if(EXISTS ${download_directory}) + if(CPM_SOURCE_CACHE) + file(LOCK ${download_directory}/../cmake.lock RELEASE) + endif() + + cpm_store_fetch_properties( + ${CPM_ARGS_NAME} "${download_directory}" + "${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-build" + ) + cpm_get_fetch_properties("${CPM_ARGS_NAME}") + + if(DEFINED CPM_ARGS_GIT_TAG AND NOT (PATCH_COMMAND IN_LIST CPM_ARGS_UNPARSED_ARGUMENTS)) + # warn if cache has been changed since checkout + cpm_check_git_working_dir_is_clean(${download_directory} ${CPM_ARGS_GIT_TAG} IS_CLEAN) + if(NOT ${IS_CLEAN}) + message( + WARNING "${CPM_INDENT} Cache for ${CPM_ARGS_NAME} (${download_directory}) is dirty" + ) + endif() + endif() + + cpm_add_subdirectory( + "${CPM_ARGS_NAME}" + "${DOWNLOAD_ONLY}" + "${${CPM_ARGS_NAME}_SOURCE_DIR}/${CPM_ARGS_SOURCE_SUBDIR}" + "${${CPM_ARGS_NAME}_BINARY_DIR}" + "${CPM_ARGS_EXCLUDE_FROM_ALL}" + "${CPM_ARGS_SYSTEM}" + "${CPM_ARGS_OPTIONS}" + ) + set(PACKAGE_INFO "${PACKAGE_INFO} at ${download_directory}") + + # As the source dir is already cached/populated, we override the call to FetchContent. + set(CPM_SKIP_FETCH TRUE) + cpm_override_fetchcontent( + "${lower_case_name}" SOURCE_DIR "${${CPM_ARGS_NAME}_SOURCE_DIR}/${CPM_ARGS_SOURCE_SUBDIR}" + BINARY_DIR "${${CPM_ARGS_NAME}_BINARY_DIR}" + ) + + else() + # Enable shallow clone when GIT_TAG is not a commit hash. Our guess may not be accurate, but + # it should guarantee no commit hash get mis-detected. + if(NOT DEFINED CPM_ARGS_GIT_SHALLOW) + cpm_is_git_tag_commit_hash("${CPM_ARGS_GIT_TAG}" IS_HASH) + if(NOT ${IS_HASH}) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS GIT_SHALLOW TRUE) + endif() + endif() + + # remove timestamps so CMake will re-download the dependency + file(REMOVE_RECURSE ${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-subbuild) + set(PACKAGE_INFO "${PACKAGE_INFO} to ${download_directory}") + endif() + endif() + + cpm_create_module_file(${CPM_ARGS_NAME} "CPMAddPackage(\"${ARGN}\")") + + if(CPM_PACKAGE_LOCK_ENABLED) + if((CPM_ARGS_VERSION AND NOT CPM_ARGS_SOURCE_DIR) OR CPM_INCLUDE_ALL_IN_PACKAGE_LOCK) + cpm_add_to_package_lock(${CPM_ARGS_NAME} "${ARGN}") + elseif(CPM_ARGS_SOURCE_DIR) + cpm_add_comment_to_package_lock(${CPM_ARGS_NAME} "local directory") + else() + cpm_add_comment_to_package_lock(${CPM_ARGS_NAME} "${ARGN}") + endif() + endif() + + cpm_message( + STATUS "${CPM_INDENT} Adding package ${CPM_ARGS_NAME}@${CPM_ARGS_VERSION} (${PACKAGE_INFO})" + ) + + if(NOT CPM_SKIP_FETCH) + # CMake 3.28 added EXCLUDE, SYSTEM (3.25), and SOURCE_SUBDIR (3.18) to FetchContent_Declare. + # Calling FetchContent_MakeAvailable will then internally forward these options to + # add_subdirectory. Up until these changes, we had to call FetchContent_Populate and + # add_subdirectory separately, which is no longer necessary and has been deprecated as of 3.30. + set(fetchContentDeclareExtraArgs "") + if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.28.0") + if(${CPM_ARGS_EXCLUDE_FROM_ALL}) + list(APPEND fetchContentDeclareExtraArgs EXCLUDE_FROM_ALL) + endif() + if(${CPM_ARGS_SYSTEM}) + list(APPEND fetchContentDeclareExtraArgs SYSTEM) + endif() + if(DEFINED CPM_ARGS_SOURCE_SUBDIR) + list(APPEND fetchContentDeclareExtraArgs SOURCE_SUBDIR ${CPM_ARGS_SOURCE_SUBDIR}) + endif() + # For CMake version <3.28 OPTIONS are parsed in cpm_add_subdirectory + if(CPM_ARGS_OPTIONS AND NOT DOWNLOAD_ONLY) + foreach(OPTION ${CPM_ARGS_OPTIONS}) + cpm_parse_option("${OPTION}") + set(${OPTION_KEY} "${OPTION_VALUE}") + endforeach() + endif() + endif() + cpm_declare_fetch( + "${CPM_ARGS_NAME}" ${fetchContentDeclareExtraArgs} "${CPM_ARGS_UNPARSED_ARGUMENTS}" + ) + + cpm_fetch_package("${CPM_ARGS_NAME}" ${DOWNLOAD_ONLY} populated ${CPM_ARGS_UNPARSED_ARGUMENTS}) + if(CPM_SOURCE_CACHE AND download_directory) + file(LOCK ${download_directory}/../cmake.lock RELEASE) + endif() + if(${populated} AND ${CMAKE_VERSION} VERSION_LESS "3.28.0") + cpm_add_subdirectory( + "${CPM_ARGS_NAME}" + "${DOWNLOAD_ONLY}" + "${${CPM_ARGS_NAME}_SOURCE_DIR}/${CPM_ARGS_SOURCE_SUBDIR}" + "${${CPM_ARGS_NAME}_BINARY_DIR}" + "${CPM_ARGS_EXCLUDE_FROM_ALL}" + "${CPM_ARGS_SYSTEM}" + "${CPM_ARGS_OPTIONS}" + ) + endif() + cpm_get_fetch_properties("${CPM_ARGS_NAME}") + endif() + + set(${CPM_ARGS_NAME}_ADDED YES) + cpm_export_variables("${CPM_ARGS_NAME}") +endfunction() + +# Fetch a previously declared package +macro(CPMGetPackage Name) + if(DEFINED "CPM_DECLARATION_${Name}") + CPMAddPackage(NAME ${Name}) + else() + message(SEND_ERROR "${CPM_INDENT} Cannot retrieve package ${Name}: no declaration available") + endif() +endmacro() + +# export variables available to the caller to the parent scope expects ${CPM_ARGS_NAME} to be set +macro(cpm_export_variables name) + set(${name}_SOURCE_DIR + "${${name}_SOURCE_DIR}" + PARENT_SCOPE + ) + set(${name}_BINARY_DIR + "${${name}_BINARY_DIR}" + PARENT_SCOPE + ) + set(${name}_ADDED + "${${name}_ADDED}" + PARENT_SCOPE + ) + set(CPM_LAST_PACKAGE_NAME + "${name}" + PARENT_SCOPE + ) +endmacro() + +# declares a package, so that any call to CPMAddPackage for the package name will use these +# arguments instead. Previous declarations will not be overridden. +macro(CPMDeclarePackage Name) + if(NOT DEFINED "CPM_DECLARATION_${Name}") + set("CPM_DECLARATION_${Name}" "${ARGN}") + endif() +endmacro() + +function(cpm_add_to_package_lock Name) + if(NOT CPM_DONT_CREATE_PACKAGE_LOCK) + cpm_prettify_package_arguments(PRETTY_ARGN false ${ARGN}) + file(APPEND ${CPM_PACKAGE_LOCK_FILE} "# ${Name}\nCPMDeclarePackage(${Name}\n${PRETTY_ARGN})\n") + endif() +endfunction() + +function(cpm_add_comment_to_package_lock Name) + if(NOT CPM_DONT_CREATE_PACKAGE_LOCK) + cpm_prettify_package_arguments(PRETTY_ARGN true ${ARGN}) + file(APPEND ${CPM_PACKAGE_LOCK_FILE} + "# ${Name} (unversioned)\n# CPMDeclarePackage(${Name}\n${PRETTY_ARGN}#)\n" + ) + endif() +endfunction() + +# includes the package lock file if it exists and creates a target `cpm-update-package-lock` to +# update it +macro(CPMUsePackageLock file) + if(NOT CPM_DONT_CREATE_PACKAGE_LOCK) + get_filename_component(CPM_ABSOLUTE_PACKAGE_LOCK_PATH ${file} ABSOLUTE) + if(EXISTS ${CPM_ABSOLUTE_PACKAGE_LOCK_PATH}) + include(${CPM_ABSOLUTE_PACKAGE_LOCK_PATH}) + endif() + if(NOT TARGET cpm-update-package-lock) + add_custom_target( + cpm-update-package-lock COMMAND ${CMAKE_COMMAND} -E copy ${CPM_PACKAGE_LOCK_FILE} + ${CPM_ABSOLUTE_PACKAGE_LOCK_PATH} + ) + endif() + set(CPM_PACKAGE_LOCK_ENABLED true) + endif() +endmacro() + +# registers a package that has been added to CPM +function(CPMRegisterPackage PACKAGE VERSION) + list(APPEND CPM_PACKAGES ${PACKAGE}) + set(CPM_PACKAGES + ${CPM_PACKAGES} + CACHE INTERNAL "" + ) + set("CPM_PACKAGE_${PACKAGE}_VERSION" + ${VERSION} + CACHE INTERNAL "" + ) +endfunction() + +# retrieve the current version of the package to ${OUTPUT} +function(CPMGetPackageVersion PACKAGE OUTPUT) + set(${OUTPUT} + "${CPM_PACKAGE_${PACKAGE}_VERSION}" + PARENT_SCOPE + ) +endfunction() + +# declares a package in FetchContent_Declare +function(cpm_declare_fetch PACKAGE) + if(${CPM_DRY_RUN}) + cpm_message(STATUS "${CPM_INDENT} Package not declared (dry run)") + return() + endif() + + FetchContent_Declare(${PACKAGE} ${ARGN}) +endfunction() + +# returns properties for a package previously defined by cpm_declare_fetch +function(cpm_get_fetch_properties PACKAGE) + if(${CPM_DRY_RUN}) + return() + endif() + + set(${PACKAGE}_SOURCE_DIR + "${CPM_PACKAGE_${PACKAGE}_SOURCE_DIR}" + PARENT_SCOPE + ) + set(${PACKAGE}_BINARY_DIR + "${CPM_PACKAGE_${PACKAGE}_BINARY_DIR}" + PARENT_SCOPE + ) +endfunction() + +function(cpm_store_fetch_properties PACKAGE source_dir binary_dir) + if(${CPM_DRY_RUN}) + return() + endif() + + set(CPM_PACKAGE_${PACKAGE}_SOURCE_DIR + "${source_dir}" + CACHE INTERNAL "" + ) + set(CPM_PACKAGE_${PACKAGE}_BINARY_DIR + "${binary_dir}" + CACHE INTERNAL "" + ) +endfunction() + +# adds a package as a subdirectory if viable, according to provided options +function( + cpm_add_subdirectory + PACKAGE + DOWNLOAD_ONLY + SOURCE_DIR + BINARY_DIR + EXCLUDE + SYSTEM + OPTIONS +) + + if(NOT DOWNLOAD_ONLY AND EXISTS ${SOURCE_DIR}/CMakeLists.txt) + set(addSubdirectoryExtraArgs "") + if(EXCLUDE) + list(APPEND addSubdirectoryExtraArgs EXCLUDE_FROM_ALL) + endif() + if("${SYSTEM}" AND "${CMAKE_VERSION}" VERSION_GREATER_EQUAL "3.25") + # https://cmake.org/cmake/help/latest/prop_dir/SYSTEM.html#prop_dir:SYSTEM + list(APPEND addSubdirectoryExtraArgs SYSTEM) + endif() + if(OPTIONS) + foreach(OPTION ${OPTIONS}) + cpm_parse_option("${OPTION}") + set(${OPTION_KEY} "${OPTION_VALUE}") + endforeach() + endif() + set(CPM_OLD_INDENT "${CPM_INDENT}") + set(CPM_INDENT "${CPM_INDENT} ${PACKAGE}:") + add_subdirectory(${SOURCE_DIR} ${BINARY_DIR} ${addSubdirectoryExtraArgs}) + set(CPM_INDENT "${CPM_OLD_INDENT}") + endif() +endfunction() + +# downloads a previously declared package via FetchContent and exports the variables +# `${PACKAGE}_SOURCE_DIR` and `${PACKAGE}_BINARY_DIR` to the parent scope +function(cpm_fetch_package PACKAGE DOWNLOAD_ONLY populated) + set(${populated} + FALSE + PARENT_SCOPE + ) + if(${CPM_DRY_RUN}) + cpm_message(STATUS "${CPM_INDENT} Package ${PACKAGE} not fetched (dry run)") + return() + endif() + + FetchContent_GetProperties(${PACKAGE}) + + string(TOLOWER "${PACKAGE}" lower_case_name) + + if(NOT ${lower_case_name}_POPULATED) + if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.28.0") + if(DOWNLOAD_ONLY) + # MakeAvailable will call add_subdirectory internally which is not what we want when + # DOWNLOAD_ONLY is set. Populate will only download the dependency without adding it to the + # build + FetchContent_Populate( + ${PACKAGE} + SOURCE_DIR "${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-src" + BINARY_DIR "${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-build" + SUBBUILD_DIR "${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-subbuild" + ${ARGN} + ) + else() + FetchContent_MakeAvailable(${PACKAGE}) + endif() + else() + FetchContent_Populate(${PACKAGE}) + endif() + set(${populated} + TRUE + PARENT_SCOPE + ) + endif() + + cpm_store_fetch_properties( + ${CPM_ARGS_NAME} ${${lower_case_name}_SOURCE_DIR} ${${lower_case_name}_BINARY_DIR} + ) + + set(${PACKAGE}_SOURCE_DIR + ${${lower_case_name}_SOURCE_DIR} + PARENT_SCOPE + ) + set(${PACKAGE}_BINARY_DIR + ${${lower_case_name}_BINARY_DIR} + PARENT_SCOPE + ) +endfunction() + +# splits a package option +function(cpm_parse_option OPTION) + string(REGEX MATCH "^[^ ]+" OPTION_KEY "${OPTION}") + string(LENGTH "${OPTION}" OPTION_LENGTH) + string(LENGTH "${OPTION_KEY}" OPTION_KEY_LENGTH) + if(OPTION_KEY_LENGTH STREQUAL OPTION_LENGTH) + # no value for key provided, assume user wants to set option to "ON" + set(OPTION_VALUE "ON") + else() + math(EXPR OPTION_KEY_LENGTH "${OPTION_KEY_LENGTH}+1") + string(SUBSTRING "${OPTION}" "${OPTION_KEY_LENGTH}" "-1" OPTION_VALUE) + endif() + set(OPTION_KEY + "${OPTION_KEY}" + PARENT_SCOPE + ) + set(OPTION_VALUE + "${OPTION_VALUE}" + PARENT_SCOPE + ) +endfunction() + +# guesses the package version from a git tag +function(cpm_get_version_from_git_tag GIT_TAG RESULT) + string(LENGTH ${GIT_TAG} length) + if(length EQUAL 40) + # GIT_TAG is probably a git hash + set(${RESULT} + 0 + PARENT_SCOPE + ) + else() + string(REGEX MATCH "v?([0123456789.]*).*" _ ${GIT_TAG}) + set(${RESULT} + ${CMAKE_MATCH_1} + PARENT_SCOPE + ) + endif() +endfunction() + +# guesses if the git tag is a commit hash or an actual tag or a branch name. +function(cpm_is_git_tag_commit_hash GIT_TAG RESULT) + string(LENGTH "${GIT_TAG}" length) + # full hash has 40 characters, and short hash has at least 7 characters. + if(length LESS 7 OR length GREATER 40) + set(${RESULT} + 0 + PARENT_SCOPE + ) + else() + if(${GIT_TAG} MATCHES "^[a-fA-F0-9]+$") + set(${RESULT} + 1 + PARENT_SCOPE + ) + else() + set(${RESULT} + 0 + PARENT_SCOPE + ) + endif() + endif() +endfunction() + +function(cpm_prettify_package_arguments OUT_VAR IS_IN_COMMENT) + set(oneValueArgs + NAME + FORCE + VERSION + GIT_TAG + DOWNLOAD_ONLY + GITHUB_REPOSITORY + GITLAB_REPOSITORY + BITBUCKET_REPOSITORY + GIT_REPOSITORY + SOURCE_DIR + FIND_PACKAGE_ARGUMENTS + NO_CACHE + SYSTEM + GIT_SHALLOW + EXCLUDE_FROM_ALL + SOURCE_SUBDIR + ) + set(multiValueArgs URL OPTIONS DOWNLOAD_COMMAND) + cmake_parse_arguments(CPM_ARGS "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + foreach(oneArgName ${oneValueArgs}) + if(DEFINED CPM_ARGS_${oneArgName}) + if(${IS_IN_COMMENT}) + string(APPEND PRETTY_OUT_VAR "#") + endif() + if(${oneArgName} STREQUAL "SOURCE_DIR") + string(REPLACE ${CMAKE_SOURCE_DIR} "\${CMAKE_SOURCE_DIR}" CPM_ARGS_${oneArgName} + ${CPM_ARGS_${oneArgName}} + ) + endif() + string(APPEND PRETTY_OUT_VAR " ${oneArgName} ${CPM_ARGS_${oneArgName}}\n") + endif() + endforeach() + foreach(multiArgName ${multiValueArgs}) + if(DEFINED CPM_ARGS_${multiArgName}) + if(${IS_IN_COMMENT}) + string(APPEND PRETTY_OUT_VAR "#") + endif() + string(APPEND PRETTY_OUT_VAR " ${multiArgName}\n") + foreach(singleOption ${CPM_ARGS_${multiArgName}}) + if(${IS_IN_COMMENT}) + string(APPEND PRETTY_OUT_VAR "#") + endif() + string(APPEND PRETTY_OUT_VAR " \"${singleOption}\"\n") + endforeach() + endif() + endforeach() + + if(NOT "${CPM_ARGS_UNPARSED_ARGUMENTS}" STREQUAL "") + if(${IS_IN_COMMENT}) + string(APPEND PRETTY_OUT_VAR "#") + endif() + string(APPEND PRETTY_OUT_VAR " ") + foreach(CPM_ARGS_UNPARSED_ARGUMENT ${CPM_ARGS_UNPARSED_ARGUMENTS}) + string(APPEND PRETTY_OUT_VAR " ${CPM_ARGS_UNPARSED_ARGUMENT}") + endforeach() + string(APPEND PRETTY_OUT_VAR "\n") + endif() + + set(${OUT_VAR} + ${PRETTY_OUT_VAR} + PARENT_SCOPE + ) + +endfunction() diff --git a/third_party/tracy/cmake/ECMFindModuleHelpers.cmake b/third_party/tracy/cmake/ECMFindModuleHelpers.cmake new file mode 100644 index 0000000000000000000000000000000000000000..a5dabb1074b672c671bf0b5fa06b271bbc1b4ca9 --- /dev/null +++ b/third_party/tracy/cmake/ECMFindModuleHelpers.cmake @@ -0,0 +1,300 @@ +#.rst: +# ECMFindModuleHelpers +# -------------------- +# +# Helper macros for find modules: ecm_find_package_version_check(), +# ecm_find_package_parse_components() and +# ecm_find_package_handle_library_components(). +# +# :: +# +# ecm_find_package_version_check() +# +# Prints warnings if the CMake version or the project's required CMake version +# is older than that required by extra-cmake-modules. +# +# :: +# +# ecm_find_package_parse_components( +# RESULT_VAR +# KNOWN_COMPONENTS [ [...]] +# [SKIP_DEPENDENCY_HANDLING]) +# +# This macro will populate with a list of components found in +# _FIND_COMPONENTS, after checking that all those components are in the +# list of KNOWN_COMPONENTS; if there are any unknown components, it will print +# an error or warning (depending on the value of _FIND_REQUIRED) and call +# return(). +# +# The order of components in is guaranteed to match the order they +# are listed in the KNOWN_COMPONENTS argument. +# +# If SKIP_DEPENDENCY_HANDLING is not set, for each component the variable +# __component_deps will be checked for dependent components. +# If is listed in _FIND_COMPONENTS, then all its (transitive) +# dependencies will also be added to . +# +# :: +# +# ecm_find_package_handle_library_components( +# COMPONENTS [ [...]] +# [SKIP_DEPENDENCY_HANDLING]) +# [SKIP_PKG_CONFIG]) +# +# Creates an imported library target for each component. The operation of this +# macro depends on the presence of a number of CMake variables. +# +# The __lib variable should contain the name of this library, +# and __header variable should contain the name of a header +# file associated with it (whatever relative path is normally passed to +# '#include'). __header_subdir variable can be used to specify +# which subdirectory of the include path the headers will be found in. +# ecm_find_package_components() will then search for the library +# and include directory (creating appropriate cache variables) and create an +# imported library target named ::. +# +# Additional variables can be used to provide additional information: +# +# If SKIP_PKG_CONFIG, the __pkg_config variable is set, and +# pkg-config is found, the pkg-config module given by +# __pkg_config will be searched for and used to help locate the +# library and header file. It will also be used to set +# __VERSION. +# +# Note that if version information is found via pkg-config, +# __FIND_VERSION can be set to require a particular version +# for each component. +# +# If SKIP_DEPENDENCY_HANDLING is not set, the INTERFACE_LINK_LIBRARIES property +# of the imported target for will be set to contain the imported +# targets for the components listed in __component_deps. +# _FOUND will also be set to false if any of the components in +# __component_deps are not found. This requires the components +# in __component_deps to be listed before in the +# COMPONENTS argument. +# +# The following variables will be set: +# +# ``_TARGETS`` +# the imported targets +# ``_LIBRARIES`` +# the found libraries +# ``_INCLUDE_DIRS`` +# the combined required include directories for the components +# ``_DEFINITIONS`` +# the "other" CFLAGS provided by pkg-config, if any +# ``_VERSION`` +# the value of ``__VERSION`` for the first component that +# has this variable set (note that components are searched for in the order +# they are passed to the macro), although if it is already set, it will not +# be altered +# +# Note that these variables are never cleared, so if +# ecm_find_package_handle_library_components() is called multiple times with +# different components (typically because of multiple find_package() calls) then +# ``_TARGETS``, for example, will contain all the targets found in any +# call (although no duplicates). +# +# Since pre-1.0.0. + +#============================================================================= +# Copyright 2014 Alex Merry +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. The name of the author may not be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +macro(ecm_find_package_version_check module_name) + if(CMAKE_VERSION VERSION_LESS 2.8.12) + message(FATAL_ERROR "CMake 2.8.12 is required by Find${module_name}.cmake") + endif() + if(CMAKE_MINIMUM_REQUIRED_VERSION VERSION_LESS 2.8.12) + message(AUTHOR_WARNING "Your project should require at least CMake 2.8.12 to use Find${module_name}.cmake") + endif() +endmacro() + +macro(ecm_find_package_parse_components module_name) + set(ecm_fppc_options SKIP_DEPENDENCY_HANDLING) + set(ecm_fppc_oneValueArgs RESULT_VAR) + set(ecm_fppc_multiValueArgs KNOWN_COMPONENTS DEFAULT_COMPONENTS) + cmake_parse_arguments(ECM_FPPC "${ecm_fppc_options}" "${ecm_fppc_oneValueArgs}" "${ecm_fppc_multiValueArgs}" ${ARGN}) + + if(ECM_FPPC_UNPARSED_ARGUMENTS) + message(FATAL_ERROR "Unexpected arguments to ecm_find_package_parse_components: ${ECM_FPPC_UNPARSED_ARGUMENTS}") + endif() + if(NOT ECM_FPPC_RESULT_VAR) + message(FATAL_ERROR "Missing RESULT_VAR argument to ecm_find_package_parse_components") + endif() + if(NOT ECM_FPPC_KNOWN_COMPONENTS) + message(FATAL_ERROR "Missing KNOWN_COMPONENTS argument to ecm_find_package_parse_components") + endif() + if(NOT ECM_FPPC_DEFAULT_COMPONENTS) + set(ECM_FPPC_DEFAULT_COMPONENTS ${ECM_FPPC_KNOWN_COMPONENTS}) + endif() + + if(${module_name}_FIND_COMPONENTS) + set(ecm_fppc_requestedComps ${${module_name}_FIND_COMPONENTS}) + + if(NOT ECM_FPPC_SKIP_DEPENDENCY_HANDLING) + # Make sure deps are included + foreach(ecm_fppc_comp ${ecm_fppc_requestedComps}) + foreach(ecm_fppc_dep_comp ${${module_name}_${ecm_fppc_comp}_component_deps}) + list(FIND ecm_fppc_requestedComps "${ecm_fppc_dep_comp}" ecm_fppc_index) + if("${ecm_fppc_index}" STREQUAL "-1") + if(NOT ${module_name}_FIND_QUIETLY) + message(STATUS "${module_name}: ${ecm_fppc_comp} requires ${${module_name}_${ecm_fppc_comp}_component_deps}") + endif() + list(APPEND ecm_fppc_requestedComps "${ecm_fppc_dep_comp}") + endif() + endforeach() + endforeach() + else() + message(STATUS "Skipping dependency handling for ${module_name}") + endif() + list(REMOVE_DUPLICATES ecm_fppc_requestedComps) + + # This makes sure components are listed in the same order as + # KNOWN_COMPONENTS (potentially important for inter-dependencies) + set(${ECM_FPPC_RESULT_VAR}) + foreach(ecm_fppc_comp ${ECM_FPPC_KNOWN_COMPONENTS}) + list(FIND ecm_fppc_requestedComps "${ecm_fppc_comp}" ecm_fppc_index) + if(NOT "${ecm_fppc_index}" STREQUAL "-1") + list(APPEND ${ECM_FPPC_RESULT_VAR} "${ecm_fppc_comp}") + list(REMOVE_AT ecm_fppc_requestedComps ${ecm_fppc_index}) + endif() + endforeach() + # if there are any left, they are unknown components + if(ecm_fppc_requestedComps) + set(ecm_fppc_msgType STATUS) + if(${module_name}_FIND_REQUIRED) + set(ecm_fppc_msgType FATAL_ERROR) + endif() + if(NOT ${module_name}_FIND_QUIETLY) + message(${ecm_fppc_msgType} "${module_name}: requested unknown components ${ecm_fppc_requestedComps}") + endif() + return() + endif() + else() + set(${ECM_FPPC_RESULT_VAR} ${ECM_FPPC_DEFAULT_COMPONENTS}) + endif() +endmacro() + +macro(ecm_find_package_handle_library_components module_name) + set(ecm_fpwc_options SKIP_PKG_CONFIG SKIP_DEPENDENCY_HANDLING) + set(ecm_fpwc_oneValueArgs) + set(ecm_fpwc_multiValueArgs COMPONENTS) + cmake_parse_arguments(ECM_FPWC "${ecm_fpwc_options}" "${ecm_fpwc_oneValueArgs}" "${ecm_fpwc_multiValueArgs}" ${ARGN}) + + if(ECM_FPWC_UNPARSED_ARGUMENTS) + message(FATAL_ERROR "Unexpected arguments to ecm_find_package_handle_components: ${ECM_FPWC_UNPARSED_ARGUMENTS}") + endif() + if(NOT ECM_FPWC_COMPONENTS) + message(FATAL_ERROR "Missing COMPONENTS argument to ecm_find_package_handle_components") + endif() + + include(FindPackageHandleStandardArgs) + find_package(PkgConfig QUIET) + foreach(ecm_fpwc_comp ${ECM_FPWC_COMPONENTS}) + set(ecm_fpwc_dep_vars) + set(ecm_fpwc_dep_targets) + if(NOT SKIP_DEPENDENCY_HANDLING) + foreach(ecm_fpwc_dep ${${module_name}_${ecm_fpwc_comp}_component_deps}) + list(APPEND ecm_fpwc_dep_vars "${module_name}_${ecm_fpwc_dep}_FOUND") + list(APPEND ecm_fpwc_dep_targets "${module_name}::${ecm_fpwc_dep}") + endforeach() + endif() + + if(NOT ECM_FPWC_SKIP_PKG_CONFIG AND ${module_name}_${ecm_fpwc_comp}_pkg_config) + pkg_check_modules(PKG_${module_name}_${ecm_fpwc_comp} QUIET + ${${module_name}_${ecm_fpwc_comp}_pkg_config}) + endif() + + find_path(${module_name}_${ecm_fpwc_comp}_INCLUDE_DIR + NAMES ${${module_name}_${ecm_fpwc_comp}_header} + HINTS ${PKG_${module_name}_${ecm_fpwc_comp}_INCLUDE_DIRS} + PATH_SUFFIXES ${${module_name}_${ecm_fpwc_comp}_header_subdir} + ) + find_library(${module_name}_${ecm_fpwc_comp}_LIBRARY + NAMES ${${module_name}_${ecm_fpwc_comp}_lib} + HINTS ${PKG_${module_name}_${ecm_fpwc_comp}_LIBRARY_DIRS} + ) + + set(${module_name}_${ecm_fpwc_comp}_VERSION "${PKG_${module_name}_${ecm_fpwc_comp}_VERSION}") + if(NOT ${module_name}_VERSION) + set(${module_name}_VERSION ${${module_name}_${ecm_fpwc_comp}_VERSION}) + endif() + + set(_name_mismatched_arg) + if(NOT CMAKE_VERSION VERSION_LESS 3.17) + set(_name_mismatched_arg NAME_MISMATCHED) + endif() + find_package_handle_standard_args(${module_name}_${ecm_fpwc_comp} + FOUND_VAR + ${module_name}_${ecm_fpwc_comp}_FOUND + REQUIRED_VARS + ${module_name}_${ecm_fpwc_comp}_LIBRARY + ${module_name}_${ecm_fpwc_comp}_INCLUDE_DIR + ${ecm_fpwc_dep_vars} + VERSION_VAR + ${module_name}_${ecm_fpwc_comp}_VERSION + ${_name_mismatched_arg} + ) + + mark_as_advanced( + ${module_name}_${ecm_fpwc_comp}_LIBRARY + ${module_name}_${ecm_fpwc_comp}_INCLUDE_DIR + ) + + if(${module_name}_${ecm_fpwc_comp}_FOUND) + list(APPEND ${module_name}_LIBRARIES + "${${module_name}_${ecm_fpwc_comp}_LIBRARY}") + list(APPEND ${module_name}_INCLUDE_DIRS + "${${module_name}_${ecm_fpwc_comp}_INCLUDE_DIR}") + set(${module_name}_DEFINITIONS + ${${module_name}_DEFINITIONS} + ${PKG_${module_name}_${ecm_fpwc_comp}_DEFINITIONS}) + if(NOT TARGET ${module_name}::${ecm_fpwc_comp}) + add_library(${module_name}::${ecm_fpwc_comp} UNKNOWN IMPORTED) + set_target_properties(${module_name}::${ecm_fpwc_comp} PROPERTIES + IMPORTED_LOCATION "${${module_name}_${ecm_fpwc_comp}_LIBRARY}" + INTERFACE_COMPILE_OPTIONS "${PKG_${module_name}_${ecm_fpwc_comp}_DEFINITIONS}" + INTERFACE_INCLUDE_DIRECTORIES "${${module_name}_${ecm_fpwc_comp}_INCLUDE_DIR}" + INTERFACE_LINK_LIBRARIES "${ecm_fpwc_dep_targets}" + ) + endif() + list(APPEND ${module_name}_TARGETS + "${module_name}::${ecm_fpwc_comp}") + endif() + endforeach() + if(${module_name}_LIBRARIES) + list(REMOVE_DUPLICATES ${module_name}_LIBRARIES) + endif() + if(${module_name}_INCLUDE_DIRS) + list(REMOVE_DUPLICATES ${module_name}_INCLUDE_DIRS) + endif() + if(${module_name}_DEFINITIONS) + list(REMOVE_DUPLICATES ${module_name}_DEFINITIONS) + endif() + if(${module_name}_TARGETS) + list(REMOVE_DUPLICATES ${module_name}_TARGETS) + endif() +endmacro() diff --git a/third_party/tracy/cmake/FindWaylandScanner.cmake b/third_party/tracy/cmake/FindWaylandScanner.cmake new file mode 100644 index 0000000000000000000000000000000000000000..e164a639ff6b819f3da6d36289887fba5d5bb3ef --- /dev/null +++ b/third_party/tracy/cmake/FindWaylandScanner.cmake @@ -0,0 +1,170 @@ +#.rst: +# FindWaylandScanner +# ------------------ +# +# Try to find wayland-scanner. +# +# If the wayland-scanner executable is not in your PATH, you can provide +# an alternative name or full path location with the ``WaylandScanner_EXECUTABLE`` +# variable. +# +# This will define the following variables: +# +# ``WaylandScanner_FOUND`` +# True if wayland-scanner is available. +# +# ``WaylandScanner_EXECUTABLE`` +# The wayland-scanner executable. +# +# If ``WaylandScanner_FOUND`` is TRUE, it will also define the following imported +# target: +# +# ``Wayland::Scanner`` +# The wayland-scanner executable. +# +# This module provides the following functions to generate C protocol +# implementations: +# +# - ``ecm_add_wayland_client_protocol`` +# - ``ecm_add_wayland_server_protocol`` +# +# :: +# +# ecm_add_wayland_client_protocol( +# PROTOCOL +# BASENAME ) +# +# Generate Wayland client protocol files from ```` XML +# definition for the ```` interface and append those files +# to ````. +# +# :: +# +# ecm_add_wayland_server_protocol( +# PROTOCOL +# BASENAME ) +# +# Generate Wayland server protocol files from ```` XML +# definition for the ```` interface and append those files +# to ````. +# +# Since 1.4.0. + +#============================================================================= +# Copyright 2012-2014 Pier Luigi Fiorini +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. The name of the author may not be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#============================================================================= + +include(${CMAKE_CURRENT_LIST_DIR}/ECMFindModuleHelpers.cmake) + +ecm_find_package_version_check(WaylandScanner) + +# Find wayland-scanner +find_program(WaylandScanner_EXECUTABLE NAMES wayland-scanner) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(WaylandScanner + FOUND_VAR + WaylandScanner_FOUND + REQUIRED_VARS + WaylandScanner_EXECUTABLE +) + +mark_as_advanced(WaylandScanner_EXECUTABLE) + +if(NOT TARGET Wayland::Scanner AND WaylandScanner_FOUND) + add_executable(Wayland::Scanner IMPORTED) + set_target_properties(Wayland::Scanner PROPERTIES + IMPORTED_LOCATION "${WaylandScanner_EXECUTABLE}" + ) +endif() + +include(FeatureSummary) +set_package_properties(WaylandScanner PROPERTIES + URL "https://wayland.freedesktop.org/" + DESCRIPTION "Executable that converts XML protocol files to C code" +) + +function(ecm_add_wayland_client_protocol out_var) + # Parse arguments + set(oneValueArgs PROTOCOL BASENAME) + cmake_parse_arguments(ARGS "" "${oneValueArgs}" "" ${ARGN}) + + if(ARGS_UNPARSED_ARGUMENTS) + message(FATAL_ERROR "Unknown keywords given to ecm_add_wayland_client_protocol(): \"${ARGS_UNPARSED_ARGUMENTS}\"") + endif() + + get_filename_component(_infile ${ARGS_PROTOCOL} ABSOLUTE) + set(_client_header "${CMAKE_CURRENT_BINARY_DIR}/wayland-${ARGS_BASENAME}-client-protocol.h") + set(_code "${CMAKE_CURRENT_BINARY_DIR}/wayland-${ARGS_BASENAME}-protocol.c") + + set_source_files_properties(${_client_header} GENERATED) + set_source_files_properties(${_code} GENERATED) + set_property(SOURCE ${_client_header} PROPERTY SKIP_AUTOMOC ON) + + add_custom_command(OUTPUT "${_client_header}" + COMMAND ${WaylandScanner_EXECUTABLE} client-header ${_infile} ${_client_header} + DEPENDS ${WaylandScanner_EXECUTABLE} ${_infile} + VERBATIM + ) + + add_custom_command(OUTPUT "${_code}" + COMMAND ${WaylandScanner_EXECUTABLE} private-code ${_infile} ${_code} + DEPENDS ${WaylandScanner_EXECUTABLE} ${_infile} ${_client_header} + VERBATIM + ) + + list(APPEND ${out_var} "${_client_header}" "${_code}") + set(${out_var} ${${out_var}} PARENT_SCOPE) +endfunction() + + +function(ecm_add_wayland_server_protocol out_var) + # Parse arguments + set(oneValueArgs PROTOCOL BASENAME) + cmake_parse_arguments(ARGS "" "${oneValueArgs}" "" ${ARGN}) + + if(ARGS_UNPARSED_ARGUMENTS) + message(FATAL_ERROR "Unknown keywords given to ecm_add_wayland_server_protocol(): \"${ARGS_UNPARSED_ARGUMENTS}\"") + endif() + + ecm_add_wayland_client_protocol(${out_var} + PROTOCOL ${ARGS_PROTOCOL} + BASENAME ${ARGS_BASENAME}) + + get_filename_component(_infile ${ARGS_PROTOCOL} ABSOLUTE) + set(_server_header "${CMAKE_CURRENT_BINARY_DIR}/wayland-${ARGS_BASENAME}-server-protocol.h") + set_property(SOURCE ${_server_header} PROPERTY SKIP_AUTOMOC ON) + set_source_files_properties(${_server_header} GENERATED) + + add_custom_command(OUTPUT "${_server_header}" + COMMAND ${WaylandScanner_EXECUTABLE} server-header ${_infile} ${_server_header} + DEPENDS ${WaylandScanner_EXECUTABLE} ${_infile} + VERBATIM + ) + + list(APPEND ${out_var} "${_server_header}") + set(${out_var} ${${out_var}} PARENT_SCOPE) +endfunction() diff --git a/third_party/tracy/cmake/config.cmake b/third_party/tracy/cmake/config.cmake new file mode 100644 index 0000000000000000000000000000000000000000..6a6e15832f6edc4aaacf49fdd329d46db7a6843d --- /dev/null +++ b/third_party/tracy/cmake/config.cmake @@ -0,0 +1,56 @@ +if (NOT NO_ISA_EXTENSIONS) + include(CheckCXXCompilerFlag) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") + CHECK_CXX_COMPILER_FLAG("-mcpu=native" COMPILER_SUPPORTS_MCPU_NATIVE) + if(COMPILER_SUPPORTS_MARCH_NATIVE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=native") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=native") + endif() + else() + CHECK_CXX_COMPILER_FLAG("-march=native" COMPILER_SUPPORTS_MARCH_NATIVE) + if(COMPILER_SUPPORTS_MARCH_NATIVE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=native") + endif() + endif() + if(WIN32) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /arch:AVX2") + endif() +endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND NOT LEGACY) + set(USE_WAYLAND ON) +else() + set(USE_WAYLAND OFF) +endif() + +if(WIN32) + add_definitions(-DNOMINMAX -DWIN32_LEAN_AND_MEAN) + add_compile_options(/MP) +else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always") +endif() + +if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug" AND NOT EMSCRIPTEN) + set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON) +endif() + +if(EMSCRIPTEN) + add_compile_options(-pthread) + add_link_options(-pthread) +endif() + +if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND NOT EMSCRIPTEN) + find_program(MOLD_LINKER mold) + if(MOLD_LINKER) + set(CMAKE_LINKER_TYPE "MOLD") + endif() + if (CMAKE_BUILD_TYPE STREQUAL "Debug") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-eliminate-unused-debug-types") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-eliminate-unused-debug-types") + endif() +endif() + +file(GENERATE OUTPUT .gitignore CONTENT "*") diff --git a/third_party/tracy/cmake/server.cmake b/third_party/tracy/cmake/server.cmake new file mode 100644 index 0000000000000000000000000000000000000000..c12a340842a79fcb9f00737c6033af7fff37f720 --- /dev/null +++ b/third_party/tracy/cmake/server.cmake @@ -0,0 +1,39 @@ +set(TRACY_COMMON_DIR ${CMAKE_CURRENT_LIST_DIR}/../public/common) + +set(TRACY_COMMON_SOURCES + tracy_lz4.cpp + tracy_lz4hc.cpp + TracySocket.cpp + TracyStackFrames.cpp + TracySystem.cpp +) + +list(TRANSFORM TRACY_COMMON_SOURCES PREPEND "${TRACY_COMMON_DIR}/") + + +set(TRACY_SERVER_DIR ${CMAKE_CURRENT_LIST_DIR}/../server) + +set(TRACY_SERVER_SOURCES + TracyMemory.cpp + TracyMmap.cpp + TracyPrint.cpp + TracySysUtil.cpp + TracyTaskDispatch.cpp + TracyTextureCompression.cpp + TracyThreadCompress.cpp + TracyWorker.cpp +) + +list(TRANSFORM TRACY_SERVER_SOURCES PREPEND "${TRACY_SERVER_DIR}/") + + +add_library(TracyServer STATIC ${TRACY_COMMON_SOURCES} ${TRACY_SERVER_SOURCES}) +target_include_directories(TracyServer PUBLIC ${TRACY_COMMON_DIR} ${TRACY_SERVER_DIR}) +target_link_libraries(TracyServer PUBLIC TracyCapstone TracyZstd) +if(NO_STATISTICS) + target_compile_definitions(TracyServer PUBLIC TRACY_NO_STATISTICS) +endif() + +if(NOT NO_PARALLEL_STL AND UNIX AND NOT APPLE AND NOT EMSCRIPTEN) + target_link_libraries(TracyServer PRIVATE TracyTbb) +endif() diff --git a/third_party/tracy/cmake/vendor.cmake b/third_party/tracy/cmake/vendor.cmake new file mode 100644 index 0000000000000000000000000000000000000000..29f12cfada7589e91272a21c46190e998fc20423 --- /dev/null +++ b/third_party/tracy/cmake/vendor.cmake @@ -0,0 +1,238 @@ +# Vendor Specific CMake +# The Tracy project keeps most vendor source locally + +set (ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}/../") + +# Dependencies are taken from the system first and if not found, they are pulled with CPM and built from source + +include(FindPkgConfig) +include(${CMAKE_CURRENT_LIST_DIR}/CPM.cmake) + +option(DOWNLOAD_CAPSTONE "Force download capstone" ON) +option(DOWNLOAD_GLFW "Force download glfw" OFF) +option(DOWNLOAD_FREETYPE "Force download freetype" OFF) + +# capstone + +pkg_check_modules(CAPSTONE capstone) +if(CAPSTONE_FOUND AND NOT DOWNLOAD_CAPSTONE) + message(STATUS "Capstone found: ${CAPSTONE}") + add_library(TracyCapstone INTERFACE) + target_include_directories(TracyCapstone INTERFACE ${CAPSTONE_INCLUDE_DIRS}) + target_link_libraries(TracyCapstone INTERFACE ${CAPSTONE_LINK_LIBRARIES}) +else() + CPMAddPackage( + NAME capstone + GITHUB_REPOSITORY capstone-engine/capstone + GIT_TAG 5.0.3 + ) + add_library(TracyCapstone INTERFACE) + target_include_directories(TracyCapstone INTERFACE ${capstone_SOURCE_DIR}/include/capstone) + target_link_libraries(TracyCapstone INTERFACE capstone) +endif() + +# GLFW + +if(NOT USE_WAYLAND AND NOT EMSCRIPTEN) + pkg_check_modules(GLFW glfw3) + if (GLFW_FOUND AND NOT DOWNLOAD_GLFW) + add_library(TracyGlfw3 INTERFACE) + target_include_directories(TracyGlfw3 INTERFACE ${GLFW_INCLUDE_DIRS}) + target_link_libraries(TracyGlfw3 INTERFACE ${GLFW_LINK_LIBRARIES}) + else() + CPMAddPackage( + NAME glfw + GITHUB_REPOSITORY glfw/glfw + GIT_TAG 3.4 + OPTIONS + "GLFW_BUILD_EXAMPLES OFF" + "GLFW_BUILD_TESTS OFF" + "GLFW_BUILD_DOCS OFF" + "GLFW_INSTALL OFF" + ) + add_library(TracyGlfw3 INTERFACE) + target_link_libraries(TracyGlfw3 INTERFACE glfw) + endif() +endif() + +# freetype + +pkg_check_modules(FREETYPE freetype2) +if (FREETYPE_FOUND AND NOT DOWNLOAD_FREETYPE) + add_library(TracyFreetype INTERFACE) + target_include_directories(TracyFreetype INTERFACE ${FREETYPE_INCLUDE_DIRS}) + target_link_libraries(TracyFreetype INTERFACE ${FREETYPE_LINK_LIBRARIES}) +else() + CPMAddPackage( + NAME freetype + GITHUB_REPOSITORY freetype/freetype + GIT_TAG VER-2-13-2 + OPTIONS + "FT_DISABLE_HARFBUZZ ON" + "FT_WITH_HARFBUZZ OFF" + ) + add_library(TracyFreetype INTERFACE) + target_link_libraries(TracyFreetype INTERFACE freetype) +endif() + +# zstd + +set(ZSTD_DIR "${ROOT_DIR}/zstd") + +set(ZSTD_SOURCES + decompress/zstd_ddict.c + decompress/zstd_decompress_block.c + decompress/huf_decompress.c + decompress/zstd_decompress.c + common/zstd_common.c + common/error_private.c + common/xxhash.c + common/entropy_common.c + common/debug.c + common/threading.c + common/pool.c + common/fse_decompress.c + compress/zstd_ldm.c + compress/zstd_compress_superblock.c + compress/zstd_opt.c + compress/zstd_compress_sequences.c + compress/fse_compress.c + compress/zstd_double_fast.c + compress/zstd_compress.c + compress/zstd_compress_literals.c + compress/hist.c + compress/zstdmt_compress.c + compress/zstd_lazy.c + compress/huf_compress.c + compress/zstd_fast.c + dictBuilder/zdict.c + dictBuilder/cover.c + dictBuilder/divsufsort.c + dictBuilder/fastcover.c +) + +list(TRANSFORM ZSTD_SOURCES PREPEND "${ZSTD_DIR}/") + +set_property(SOURCE ${ZSTD_DIR}/decompress/huf_decompress_amd64.S APPEND PROPERTY COMPILE_OPTIONS "-x" "assembler-with-cpp") + +add_library(TracyZstd STATIC ${ZSTD_SOURCES}) +target_include_directories(TracyZstd PUBLIC ${ZSTD_DIR}) +target_compile_definitions(TracyZstd PRIVATE ZSTD_DISABLE_ASM) + + +# Diff Template Library + +set(DTL_DIR "${ROOT_DIR}/dtl") +file(GLOB_RECURSE DTL_HEADERS CONFIGURE_DEPENDS RELATIVE ${DTL_DIR} "*.hpp") +add_library(TracyDtl INTERFACE) +target_sources(TracyDtl INTERFACE ${DTL_HEADERS}) +target_include_directories(TracyDtl INTERFACE ${DTL_DIR}) + + +# Get Opt + +set(GETOPT_DIR "${ROOT_DIR}/getopt") +set(GETOPT_SOURCES ${GETOPT_DIR}/getopt.c) +set(GETOPT_HEADERS ${GETOPT_DIR}/getopt.h) +add_library(TracyGetOpt STATIC ${GETOPT_SOURCES} ${GETOPT_HEADERS}) +target_include_directories(TracyGetOpt PUBLIC ${GETOPT_DIR}) + + +# ImGui + +set(IMGUI_DIR "${ROOT_DIR}/imgui") + +set(IMGUI_SOURCES + imgui_widgets.cpp + imgui_draw.cpp + imgui_demo.cpp + imgui.cpp + imgui_tables.cpp + misc/freetype/imgui_freetype.cpp +) + +list(TRANSFORM IMGUI_SOURCES PREPEND "${IMGUI_DIR}/") + +add_definitions(-DIMGUI_ENABLE_FREETYPE) + +add_library(TracyImGui STATIC ${IMGUI_SOURCES}) +target_include_directories(TracyImGui PUBLIC ${IMGUI_DIR}) +target_link_libraries(TracyImGui PUBLIC TracyFreetype) + +# NFD + +if (NOT NO_FILESELECTOR AND NOT EMSCRIPTEN) + set(NFD_DIR "${ROOT_DIR}/nfd") + + if (WIN32) + set(NFD_SOURCES "${NFD_DIR}/nfd_win.cpp") + elseif (APPLE) + set(NFD_SOURCES "${NFD_DIR}/nfd_cocoa.m") + else() + if (GTK_FILESELECTOR) + set(NFD_SOURCES "${NFD_DIR}/nfd_gtk.cpp") + else() + set(NFD_SOURCES "${NFD_DIR}/nfd_portal.cpp") + endif() + endif() + + file(GLOB_RECURSE NFD_HEADERS CONFIGURE_DEPENDS RELATIVE ${NFD_DIR} "*.h") + add_library(TracyNfd STATIC ${NFD_SOURCES} ${NFD_HEADERS}) + target_include_directories(TracyNfd PUBLIC ${NFD_DIR}) + + if (APPLE) + find_library(APPKIT_LIBRARY AppKit) + find_library(UNIFORMTYPEIDENTIFIERS_LIBRARY UniformTypeIdentifiers) + target_link_libraries(TracyNfd PUBLIC ${APPKIT_LIBRARY} ${UNIFORMTYPEIDENTIFIERS_LIBRARY}) + elseif (UNIX) + if (GTK_FILESELECTOR) + pkg_check_modules(GTK3 gtk+-3.0) + if (NOT GTK3_FOUND) + message(FATAL_ERROR "GTK3 not found. Please install it or set TRACY_GTK_FILESELECTOR to OFF.") + endif() + add_library(TracyGtk3 INTERFACE) + target_include_directories(TracyGtk3 INTERFACE ${GTK3_INCLUDE_DIRS}) + target_link_libraries(TracyGtk3 INTERFACE ${GTK3_LINK_LIBRARIES}) + target_link_libraries(TracyNfd PUBLIC TracyGtk3) + else() + pkg_check_modules(DBUS dbus-1) + if (NOT DBUS_FOUND) + message(FATAL_ERROR "D-Bus not found. Please install it or set TRACY_GTK_FILESELECTOR to ON.") + endif() + add_library(TracyDbus INTERFACE) + target_include_directories(TracyDbus INTERFACE ${DBUS_INCLUDE_DIRS}) + target_link_libraries(TracyDbus INTERFACE ${DBUS_LINK_LIBRARIES}) + target_link_libraries(TracyNfd PUBLIC TracyDbus) + endif() + endif() +endif() + +# TBB +if (NO_PARALLEL_STL) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNO_PARALLEL_SORT") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DNO_PARALLEL_SORT") +else() + if (UNIX AND NOT APPLE AND NOT EMSCRIPTEN) + # Tracy does not use TBB directly, but the implementation of parallel algorithms + # in some versions of libstdc++ depends on TBB. When it does, you must + # explicitly link against -ltbb. + # + # Some distributions have pgk-config files for TBB, others don't. + + pkg_check_modules(TBB tbb) + if (TBB_FOUND) + add_library(TracyTbb INTERFACE) + target_include_directories(TracyTbb INTERFACE ${TBB_INCLUDE_DIRS}) + target_link_libraries(TracyTbb INTERFACE ${TBB_LINK_LIBRARIES}) + else() + CPMAddPackage( + NAME tbb + GITHUB_REPOSITORY oneapi-src/oneTBB + GIT_TAG v2021.12.0-rc2 + OPTIONS "TBB_TEST OFF" + ) + add_library(TracyTbb INTERFACE) + target_link_libraries(TracyTbb INTERFACE tbb) + endif() + endif() +endif() diff --git a/third_party/tracy/cmake/version.cmake b/third_party/tracy/cmake/version.cmake new file mode 100644 index 0000000000000000000000000000000000000000..03813b444e654be586c3358ba2c462a75b5efaa4 --- /dev/null +++ b/third_party/tracy/cmake/version.cmake @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 3.10) + +message("Parsing public/common/TracyVersion.hpp file") + +file(READ "${CMAKE_CURRENT_LIST_DIR}/../public/common/TracyVersion.hpp" version) + +# Note: This looks for a specific pattern in TracyVersion.hpp, if it changes +# this needs updating. +string(REGEX MATCH "Major = ([0-9]+)" _ ${version}) + +# This works do to the above () subexpression selection. See +# https://cmake.org/cmake/help/latest/command/string.html#regex-match for more +# details +set(TRACY_VERSION_MAJOR ${CMAKE_MATCH_1}) + +string(REGEX MATCH "Minor = ([0-9]+)" _ ${version}) +set(TRACY_VERSION_MINOR ${CMAKE_MATCH_1}) + +string(REGEX MATCH "Patch = ([0-9]+)" _ ${version}) +set(TRACY_VERSION_PATCH ${CMAKE_MATCH_1}) + +set(TRACY_VERSION_STRING "${TRACY_VERSION_MAJOR}.${TRACY_VERSION_MINOR}.${TRACY_VERSION_PATCH}") + +message("VERSION ${TRACY_VERSION_STRING}") diff --git a/third_party/tracy/csvexport/CMakeLists.txt b/third_party/tracy/csvexport/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..a69c03a8586434ac18f3f916a738a444307f6ab6 --- /dev/null +++ b/third_party/tracy/csvexport/CMakeLists.txt @@ -0,0 +1,28 @@ +cmake_minimum_required(VERSION 3.16) + +option(NO_ISA_EXTENSIONS "Disable ISA extensions (don't pass -march=native or -mcpu=native to the compiler)" OFF) +option(NO_PARALLEL_STL "Disable parallel STL" OFF) + +set(NO_STATISTICS OFF) + +include(${CMAKE_CURRENT_LIST_DIR}/../cmake/version.cmake) + +set(CMAKE_CXX_STANDARD 20) + +project( + tracy-csvexport + LANGUAGES C CXX + VERSION ${TRACY_VERSION_STRING} +) + +include(${CMAKE_CURRENT_LIST_DIR}/../cmake/config.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/../cmake/vendor.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/../cmake/server.cmake) + +set(PROGRAM_FILES + src/csvexport.cpp +) + +add_executable(${PROJECT_NAME} ${PROGRAM_FILES} ${COMMON_FILES} ${SERVER_FILES}) +target_link_libraries(${PROJECT_NAME} PRIVATE TracyServer TracyGetOpt) +set_property(DIRECTORY ${CMAKE_CURRENT_LIST_DIR} PROPERTY VS_STARTUP_PROJECT ${PROJECT_NAME}) diff --git a/third_party/tracy/csvexport/src/csvexport.cpp b/third_party/tracy/csvexport/src/csvexport.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e4435e492dd9542ecd51eaa68761bc525d14509e --- /dev/null +++ b/third_party/tracy/csvexport/src/csvexport.cpp @@ -0,0 +1,353 @@ +#ifdef _WIN32 +# include +#endif + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "../../server/TracyFileRead.hpp" +#include "../../server/TracyWorker.hpp" +#include "../../getopt/getopt.h" + +void print_usage_exit(int e) +{ + fprintf(stderr, "Extract statistics from a trace to a CSV format\n"); + fprintf(stderr, "Usage:\n"); + fprintf(stderr, " extract [OPTION...] \n"); + fprintf(stderr, "\n"); + fprintf(stderr, " -h, --help Print usage\n"); + fprintf(stderr, " -f, --filter arg Filter zone names (default: "")\n"); + fprintf(stderr, " -s, --sep arg CSV separator (default: ,)\n"); + fprintf(stderr, " -c, --case Case sensitive filtering\n"); + fprintf(stderr, " -e, --self Get self times\n"); + fprintf(stderr, " -u, --unwrap Report each zone event\n"); + fprintf(stderr, " -m, --messages Report only messages\n"); + + exit(e); +} + +struct Args { + const char* filter; + const char* separator; + const char* trace_file; + bool case_sensitive; + bool self_time; + bool unwrap; + bool unwrapMessages; +}; + +Args parse_args(int argc, char** argv) +{ + if (argc == 1) + { + print_usage_exit(1); + } + + Args args = { "", ",", "", false, false, false, false }; + + struct option long_opts[] = { + { "help", no_argument, NULL, 'h' }, + { "filter", optional_argument, NULL, 'f' }, + { "sep", optional_argument, NULL, 's' }, + { "case", no_argument, NULL, 'c' }, + { "self", no_argument, NULL, 'e' }, + { "unwrap", no_argument, NULL, 'u' }, + { "messages", no_argument, NULL, 'm' }, + { NULL, 0, NULL, 0 } + }; + + int c; + while ((c = getopt_long(argc, argv, "hf:s:ceum", long_opts, NULL)) != -1) + { + switch (c) + { + case 'h': + print_usage_exit(0); + break; + case 'f': + args.filter = optarg; + break; + case 's': + args.separator = optarg; + break; + case 'c': + args.case_sensitive = true; + break; + case 'e': + args.self_time = true; + break; + case 'u': + args.unwrap = true; + break; + case 'm': + args.unwrapMessages = true; + break; + default: + print_usage_exit(1); + break; + } + } + + if (argc != optind + 1) + { + print_usage_exit(1); + } + + args.trace_file = argv[optind]; + + return args; +} + +bool is_substring( + const char* term, + const char* s, + bool case_sensitive = false +){ + auto new_term = std::string(term); + auto new_s = std::string(s); + + if (!case_sensitive) { + std::transform( + new_term.begin(), + new_term.end(), + new_term.begin(), + [](unsigned char c){ return std::tolower(c); } + ); + + std::transform( + new_s.begin(), + new_s.end(), + new_s.begin(), + [](unsigned char c){ return std::tolower(c); } + ); + } + + return new_s.find(new_term) != std::string::npos; +} + +const char* get_name(int32_t id, const tracy::Worker& worker) +{ + auto& srcloc = worker.GetSourceLocation(id); + return worker.GetString(srcloc.name.active ? srcloc.name : srcloc.function); +} + +template +std::string join(const T& v, const char* sep) { + std::ostringstream s; + for (const auto& i : v) { + if (&i != &v[0]) { + s << sep; + } + s << i; + } + return s.str(); +} + +// From TracyView.cpp +int64_t GetZoneChildTimeFast( + const tracy::Worker& worker, + const tracy::ZoneEvent& zone +){ + int64_t time = 0; + if( zone.HasChildren() ) + { + auto& children = worker.GetZoneChildren( zone.Child() ); + if( children.is_magic() ) + { + auto& vec = *(tracy::Vector*)&children; + for( auto& v : vec ) + { + assert( v.IsEndValid() ); + time += v.End() - v.Start(); + } + } + else + { + for( auto& v : children ) + { + assert( v->IsEndValid() ); + time += v->End() - v->Start(); + } + } + } + return time; +} + +int main(int argc, char** argv) +{ +#ifdef _WIN32 + if (!AttachConsole(ATTACH_PARENT_PROCESS)) + { + AllocConsole(); + SetConsoleMode(GetStdHandle(STD_OUTPUT_HANDLE), 0x07); + } +#endif + + Args args = parse_args(argc, argv); + + auto f = std::unique_ptr( + tracy::FileRead::Open(args.trace_file) + ); + if (!f) + { + fprintf(stderr, "Could not open file %s\n", args.trace_file); + return 1; + } + + auto worker = tracy::Worker(*f); + + if (args.unwrapMessages) + { + const auto& msgs = worker.GetMessages(); + + if (msgs.size() > 0) + { + std::vector columnsForMessages; + columnsForMessages = { + "MessageName", "total_ns" + }; + std::string headerForMessages = join(columnsForMessages, args.separator); + printf("%s\n", headerForMessages.data()); + + for(auto& it : msgs) + { + std::vector values(columnsForMessages.size()); + + values[0] = worker.GetString(it->ref); + values[1] = std::to_string(it->time); + + std::string row = join(values, args.separator); + printf("%s\n", row.data()); + } + } + else + { + printf("There are currently no messages!\n"); + } + + return 0; + } + + while (!worker.AreSourceLocationZonesReady()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + + auto& slz = worker.GetSourceLocationZones(); + tracy::Vector slz_selected; + slz_selected.reserve(slz.size()); + + uint32_t total_cnt = 0; + for(auto it = slz.begin(); it != slz.end(); ++it) + { + if(it->second.total != 0) + { + ++total_cnt; + if(args.filter[0] == '\0') + { + slz_selected.push_back_no_space_check(it); + } + else + { + auto name = get_name(it->first, worker); + if(is_substring(args.filter, name, args.case_sensitive)) + { + slz_selected.push_back_no_space_check(it); + } + } + } + } + + std::vector columns; + if (args.unwrap) + { + columns = { + "name", "src_file", "src_line", "ns_since_start", "exec_time_ns", "thread" + }; + } + else + { + columns = { + "name", "src_file", "src_line", "total_ns", "total_perc", + "counts", "mean_ns", "min_ns", "max_ns", "std_ns" + }; + } + std::string header = join(columns, args.separator); + printf("%s\n", header.data()); + + const auto last_time = worker.GetLastTime(); + for(auto& it : slz_selected) + { + std::vector values(columns.size()); + + values[0] = get_name(it->first, worker); + + const auto& srcloc = worker.GetSourceLocation(it->first); + values[1] = worker.GetString(srcloc.file); + values[2] = std::to_string(srcloc.line); + + const auto& zone_data = it->second; + + if (args.unwrap) + { + int i = 0; + for (const auto& zone_thread_data : zone_data.zones) { + const auto zone_event = zone_thread_data.Zone(); + const auto tId = zone_thread_data.Thread(); + const auto start = zone_event->Start(); + const auto end = zone_event->End(); + + values[3] = std::to_string(start); + + auto timespan = end - start; + if (args.self_time) { + timespan -= GetZoneChildTimeFast(worker, *zone_event); + } + values[4] = std::to_string(timespan); + values[5] = std::to_string(tId); + + std::string row = join(values, args.separator); + printf("%s\n", row.data()); + } + } + else + { + const auto time = args.self_time ? zone_data.selfTotal : zone_data.total; + values[3] = std::to_string(time); + values[4] = std::to_string(100. * time / last_time); + + values[5] = std::to_string(zone_data.zones.size()); + + const auto avg = (args.self_time ? zone_data.selfTotal : zone_data.total) + / zone_data.zones.size(); + values[6] = std::to_string(avg); + + const auto tmin = args.self_time ? zone_data.selfMin : zone_data.min; + const auto tmax = args.self_time ? zone_data.selfMax : zone_data.max; + values[7] = std::to_string(tmin); + values[8] = std::to_string(tmax); + + const auto sz = zone_data.zones.size(); + const auto ss = zone_data.sumSq + - 2. * zone_data.total * avg + + avg * avg * sz; + double std = 0; + if( sz > 1 ) + std = sqrt(ss / (sz - 1)); + values[9] = std::to_string(std); + + std::string row = join(values, args.separator); + printf("%s\n", row.data()); + } + } + + return 0; +} diff --git a/third_party/tracy/examples/OpenCLVectorAdd/CMakeLists.txt b/third_party/tracy/examples/OpenCLVectorAdd/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..e982013358acdc96a050c079a27c2135336bde95 --- /dev/null +++ b/third_party/tracy/examples/OpenCLVectorAdd/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 3.10) + +project(OpenCLVectorAdd) + +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + +find_package(OpenCL REQUIRED) + +add_executable(OpenCLVectorAdd OpenCLVectorAdd.cpp) + +add_library(TracyClient STATIC ../../public/TracyClient.cpp + ../../public/tracy/TracyOpenCL.hpp) +target_include_directories(TracyClient PUBLIC ../../public/tracy) +target_compile_definitions(TracyClient PUBLIC TRACY_ENABLE=1) + +target_link_libraries(OpenCLVectorAdd PUBLIC OpenCL::OpenCL TracyClient ${CMAKE_DL_LIBS} Threads::Threads) diff --git a/third_party/tracy/examples/OpenCLVectorAdd/OpenCLVectorAdd.cpp b/third_party/tracy/examples/OpenCLVectorAdd/OpenCLVectorAdd.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a2560eaef36be3a3109e4f4872d000312971e14e --- /dev/null +++ b/third_party/tracy/examples/OpenCLVectorAdd/OpenCLVectorAdd.cpp @@ -0,0 +1,220 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#define CL_ASSERT(err) \ + if((err) != CL_SUCCESS) \ + { \ + std::cerr << "OpenCL Call Returned " << err << std::endl; \ + assert(false); \ + } + +const char kernelSource[] = +" void __kernel vectorAdd(global float* C, global float* A, global float* B, int N) " +" { " +" int i = get_global_id(0); " +" if (i < N) { " +" C[i] = A[i] + B[i]; " +" } " +" } "; + +int main() +{ + cl_platform_id platform; + cl_device_id device; + cl_context context; + cl_command_queue commandQueue; + cl_kernel vectorAddKernel; + cl_program program; + cl_int err; + cl_mem bufferA, bufferB, bufferC; + + TracyCLCtx tracyCLCtx; + + { + ZoneScopedN("OpenCL Init"); + + cl_uint numPlatforms = 0; + CL_ASSERT(clGetPlatformIDs(0, nullptr, &numPlatforms)); + + if (numPlatforms == 0) + { + std::cerr << "Cannot find OpenCL platform to run this application" << std::endl; + return 1; + } + + CL_ASSERT(clGetPlatformIDs(1, &platform, nullptr)); + + size_t platformNameBufferSize = 0; + CL_ASSERT(clGetPlatformInfo(platform, CL_PLATFORM_NAME, 0, nullptr, &platformNameBufferSize)); + std::string platformName(platformNameBufferSize, '\0'); + CL_ASSERT(clGetPlatformInfo(platform, CL_PLATFORM_NAME, platformNameBufferSize, &platformName[0], nullptr)); + + std::cout << "OpenCL Platform: " << platformName << std::endl; + + CL_ASSERT(clGetDeviceIDs(platform, CL_DEVICE_TYPE_ALL, 1, &device, nullptr)); + size_t deviceNameBufferSize = 0; + CL_ASSERT(clGetDeviceInfo(device, CL_DEVICE_NAME, 0, nullptr, &deviceNameBufferSize)); + std::string deviceName(deviceNameBufferSize, '\0'); + CL_ASSERT(clGetDeviceInfo(device, CL_DEVICE_NAME, deviceNameBufferSize, &deviceName[0], nullptr)); + + std::cout << "OpenCL Device: " << deviceName << std::endl; + + err = CL_SUCCESS; + context = clCreateContext(nullptr, 1, &device, nullptr, nullptr, &err); + CL_ASSERT(err); + + size_t kernelSourceLength = sizeof(kernelSource); + const char* kernelSourceArray = { kernelSource }; + program = clCreateProgramWithSource(context, 1, &kernelSourceArray, &kernelSourceLength, &err); + CL_ASSERT(err); + + if (clBuildProgram(program, 1, &device, nullptr, nullptr, nullptr) != CL_SUCCESS) + { + size_t programBuildLogBufferSize = 0; + CL_ASSERT(clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, 0, nullptr, &programBuildLogBufferSize)); + std::string programBuildLog(programBuildLogBufferSize, '\0'); + CL_ASSERT(clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, programBuildLogBufferSize, &programBuildLog[0], nullptr)); + std::clog << programBuildLog << std::endl; + return 1; + } + + vectorAddKernel = clCreateKernel(program, "vectorAdd", &err); + CL_ASSERT(err); + + commandQueue = clCreateCommandQueue(context, device, CL_QUEUE_PROFILING_ENABLE, &err); + CL_ASSERT(err); + } + + tracyCLCtx = TracyCLContext(context, device); + + size_t N = 10 * 1024 * 1024 / sizeof(float); // 10MB of floats + std::vector hostA, hostB, hostC; + + { + ZoneScopedN("Host Data Init"); + hostA.resize(N); + hostB.resize(N); + hostC.resize(N); + + std::iota(std::begin(hostA), std::end(hostA), 0.0f); + std::iota(std::begin(hostB), std::end(hostB), 0.0f); + } + + { + ZoneScopedN("Host to Device Memory Copy"); + + bufferA = clCreateBuffer(context, CL_MEM_READ_WRITE, N * sizeof(float), nullptr, &err); + CL_ASSERT(err); + bufferB = clCreateBuffer(context, CL_MEM_READ_WRITE, N * sizeof(float), nullptr, &err); + CL_ASSERT(err); + bufferC = clCreateBuffer(context, CL_MEM_READ_WRITE, N * sizeof(float), nullptr, &err); + CL_ASSERT(err); + + cl_event writeBufferAEvent, writeBufferBEvent; + { + ZoneScopedN("Write Buffer A"); + TracyCLZoneS(tracyCLCtx, "Write BufferA", 5); + + CL_ASSERT(clEnqueueWriteBuffer(commandQueue, bufferA, CL_FALSE, 0, N * sizeof(float), hostA.data(), 0, nullptr, &writeBufferAEvent)); + + TracyCLZoneSetEvent(writeBufferAEvent); + } + { + ZoneScopedN("Write Buffer B"); + TracyCLZone(tracyCLCtx, "Write BufferB"); + + CL_ASSERT(clEnqueueWriteBuffer(commandQueue, bufferB, CL_FALSE, 0, N * sizeof(float), hostB.data(), 0, nullptr, &writeBufferBEvent)); + + TracyCLZoneSetEvent(writeBufferBEvent); + } + } + + cl_int clN = static_cast(N); + const int numFrames = 10; + const int launchsPerFrame = 10; + constexpr int numLaunchs = numFrames * launchsPerFrame; + std::vector kernelLaunchEvts; + kernelLaunchEvts.reserve(numLaunchs); + for (int i = 0; i < numFrames; ++i) + { + FrameMark; + for (int j = 0; j < launchsPerFrame; ++j) { + ZoneScopedN("VectorAdd Kernel Launch"); + TracyCLZoneC(tracyCLCtx, "VectorAdd Kernel", tracy::Color::Blue4); + + CL_ASSERT(clSetKernelArg(vectorAddKernel, 0, sizeof(cl_mem), &bufferC)); + CL_ASSERT(clSetKernelArg(vectorAddKernel, 1, sizeof(cl_mem), &bufferA)); + CL_ASSERT(clSetKernelArg(vectorAddKernel, 2, sizeof(cl_mem), &bufferB)); + CL_ASSERT(clSetKernelArg(vectorAddKernel, 3, sizeof(cl_int), &clN)); + + cl_event vectorAddKernelEvent; + CL_ASSERT(clEnqueueNDRangeKernel(commandQueue, vectorAddKernel, 1, nullptr, &N, nullptr, 0, nullptr, &vectorAddKernelEvent)); + TracyCLZoneSetEvent(vectorAddKernelEvent); + CL_ASSERT(clRetainEvent(vectorAddKernelEvent)); + kernelLaunchEvts.push_back(vectorAddKernelEvent); + std::cout << "VectorAdd Kernel Enqueued" << std::endl; + } + { + // Wait frame events to be finished + ZoneScopedN("clFinish"); + CL_ASSERT(clFinish(commandQueue)); + } + // You should collect on each 'frame' ends, so that streaming can be achieved. + TracyCLCollect(tracyCLCtx); + } + + { + ZoneScopedN("Device to Host Memory Copy"); + TracyCLZone(tracyCLCtx, "Read Buffer C"); + + cl_event readbufferCEvent; + CL_ASSERT(clEnqueueReadBuffer(commandQueue, bufferC, CL_TRUE, 0, N * sizeof(float), hostC.data(), 0, nullptr, &readbufferCEvent)); + TracyCLZoneSetEvent(readbufferCEvent); + } + + CL_ASSERT(clFinish(commandQueue)); + std::vector durations(kernelLaunchEvts.size()); + for (int i=0; i(durations[i]) << "us" << std::endl; + }; + float avg = std::accumulate(durations.cbegin(), durations.cend(), 0.0f) / durations.size(); + float stddev2 = std::accumulate(durations.cbegin(), durations.cend(), 0.0f, [avg](const float& acc, const float& v) { + auto d = v - avg; + return acc + d*d; + }) / (durations.size() - 1.0f); + std::cout << "VectorAdd runtime avg: " << avg << "us, std: " << sqrt(stddev2) << "us over " << numLaunchs << " runs." << std::endl; + + // User should ensure all events are finished, in this case, collect after the clFinish will do the trick. + TracyCLCollect(tracyCLCtx); + + { + ZoneScopedN("Checking results"); + + for (int i = 0; i < N; ++i) + { + assert(hostC[i] == hostA[i] + hostB[i]); + } + } + + std::cout << "Results are correct!" << std::endl; + + TracyCLDestroy(tracyCLCtx); + + return 0; +} diff --git a/third_party/tracy/examples/ToyPathTracer/.gitignore b/third_party/tracy/examples/ToyPathTracer/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..5bb5351d363bbad0f57fc718c76841d490028bc2 --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/.gitignore @@ -0,0 +1 @@ +Windows/Compiled*Shader.h diff --git a/third_party/tracy/examples/ToyPathTracer/README b/third_party/tracy/examples/ToyPathTracer/README new file mode 100644 index 0000000000000000000000000000000000000000..62c5051f989dbc8544ca71571310e0c7833a19ee --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/README @@ -0,0 +1,4 @@ +https://github.com/aras-p/ToyPathTracer + +Modified to render only 10 frames. Client part requires 12 GB, server part +requires 6.4 GB. diff --git a/third_party/tracy/examples/ToyPathTracer/Source/Config.h b/third_party/tracy/examples/ToyPathTracer/Source/Config.h new file mode 100644 index 0000000000000000000000000000000000000000..9feede1b251e75ae6f1ab5e1ae00529e521b8058 --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Source/Config.h @@ -0,0 +1,33 @@ + +#if defined(__APPLE__) && !defined(__METAL_VERSION__) +#include +#endif + +#define kBackbufferWidth 1280 +#define kBackbufferHeight 720 + +#if defined(__EMSCRIPTEN__) +#define CPU_CAN_DO_SIMD 0 +#define CPU_CAN_DO_THREADS 0 +#else +#define CPU_CAN_DO_SIMD 1 +#define CPU_CAN_DO_THREADS 1 +#endif + + +#define DO_SAMPLES_PER_PIXEL 4 +#define DO_ANIMATE_SMOOTHING 0.9f +#define DO_LIGHT_SAMPLING 1 +#define DO_MITSUBA_COMPARE 0 + +// Should path tracing be done on the GPU with a compute shader? +#define DO_COMPUTE_GPU 0 +#define kCSGroupSizeX 8 +#define kCSGroupSizeY 8 +#define kCSMaxObjects 64 + +// Should float3 struct use SSE/NEON? +#define DO_FLOAT3_WITH_SIMD (!(DO_COMPUTE_GPU) && CPU_CAN_DO_SIMD && 1) + +// Should HitSpheres function use SSE/NEON? +#define DO_HIT_SPHERES_SIMD (CPU_CAN_DO_SIMD && 1) diff --git a/third_party/tracy/examples/ToyPathTracer/Source/MathSimd.h b/third_party/tracy/examples/ToyPathTracer/Source/MathSimd.h new file mode 100644 index 0000000000000000000000000000000000000000..3e0a5cbe9a296de98f1bb4dcd7a1349002073437 --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Source/MathSimd.h @@ -0,0 +1,192 @@ +#pragma once + +#if defined(_MSC_VER) +#define VM_INLINE __forceinline +#else +#define VM_INLINE __attribute__((unused, always_inline, nodebug)) inline +#endif + +#define kSimdWidth 4 + +#if !defined(__arm__) && !defined(__arm64__) && !defined(__EMSCRIPTEN__) + +// ---- SSE implementation + +#include +#include +#include + +#define SHUFFLE4(V, X,Y,Z,W) float4(_mm_shuffle_ps((V).m, (V).m, _MM_SHUFFLE(W,Z,Y,X))) + +struct float4 +{ + VM_INLINE float4() {} + VM_INLINE explicit float4(const float *p) { m = _mm_loadu_ps(p); } + VM_INLINE explicit float4(float x, float y, float z, float w) { m = _mm_set_ps(w, z, y, x); } + VM_INLINE explicit float4(float v) { m = _mm_set_ps1(v); } + VM_INLINE explicit float4(__m128 v) { m = v; } + + VM_INLINE float getX() const { return _mm_cvtss_f32(m); } + VM_INLINE float getY() const { return _mm_cvtss_f32(_mm_shuffle_ps(m, m, _MM_SHUFFLE(1, 1, 1, 1))); } + VM_INLINE float getZ() const { return _mm_cvtss_f32(_mm_shuffle_ps(m, m, _MM_SHUFFLE(2, 2, 2, 2))); } + VM_INLINE float getW() const { return _mm_cvtss_f32(_mm_shuffle_ps(m, m, _MM_SHUFFLE(3, 3, 3, 3))); } + + __m128 m; +}; + +typedef float4 bool4; + +VM_INLINE float4 operator+ (float4 a, float4 b) { a.m = _mm_add_ps(a.m, b.m); return a; } +VM_INLINE float4 operator- (float4 a, float4 b) { a.m = _mm_sub_ps(a.m, b.m); return a; } +VM_INLINE float4 operator* (float4 a, float4 b) { a.m = _mm_mul_ps(a.m, b.m); return a; } +VM_INLINE bool4 operator==(float4 a, float4 b) { a.m = _mm_cmpeq_ps(a.m, b.m); return a; } +VM_INLINE bool4 operator!=(float4 a, float4 b) { a.m = _mm_cmpneq_ps(a.m, b.m); return a; } +VM_INLINE bool4 operator< (float4 a, float4 b) { a.m = _mm_cmplt_ps(a.m, b.m); return a; } +VM_INLINE bool4 operator> (float4 a, float4 b) { a.m = _mm_cmpgt_ps(a.m, b.m); return a; } +VM_INLINE bool4 operator<=(float4 a, float4 b) { a.m = _mm_cmple_ps(a.m, b.m); return a; } +VM_INLINE bool4 operator>=(float4 a, float4 b) { a.m = _mm_cmpge_ps(a.m, b.m); return a; } +VM_INLINE bool4 operator&(bool4 a, bool4 b) { a.m = _mm_and_ps(a.m, b.m); return a; } +VM_INLINE bool4 operator|(bool4 a, bool4 b) { a.m = _mm_or_ps(a.m, b.m); return a; } +VM_INLINE float4 operator- (float4 a) { a.m = _mm_xor_ps(a.m, _mm_set1_ps(-0.0f)); return a; } +VM_INLINE float4 min(float4 a, float4 b) { a.m = _mm_min_ps(a.m, b.m); return a; } +VM_INLINE float4 max(float4 a, float4 b) { a.m = _mm_max_ps(a.m, b.m); return a; } + +VM_INLINE float hmin(float4 v) +{ + v = min(v, SHUFFLE4(v, 2, 3, 0, 0)); + v = min(v, SHUFFLE4(v, 1, 0, 0, 0)); + return v.getX(); +} + +// Returns a 4-bit code where bit0..bit3 is X..W +VM_INLINE unsigned mask(float4 v) { return _mm_movemask_ps(v.m); } +// Once we have a comparison, we can branch based on its results: +VM_INLINE bool any(bool4 v) { return mask(v) != 0; } +VM_INLINE bool all(bool4 v) { return mask(v) == 15; } + +// "select", i.e. hibit(cond) ? b : a +// on SSE4.1 and up this can be done easily via "blend" instruction; +// on older SSEs has to do a bunch of hoops, see +// https://fgiesen.wordpress.com/2016/04/03/sse-mind-the-gap/ + +VM_INLINE float4 select(float4 a, float4 b, bool4 cond) +{ +#if defined(__SSE4_1__) || defined(_MSC_VER) // on windows assume we always have SSE4.1 + a.m = _mm_blendv_ps(a.m, b.m, cond.m); +#else + __m128 d = _mm_castsi128_ps(_mm_srai_epi32(_mm_castps_si128(cond.m), 31)); + a.m = _mm_or_ps(_mm_and_ps(d, b.m), _mm_andnot_ps(d, a.m)); +#endif + return a; +} +VM_INLINE __m128i select(__m128i a, __m128i b, bool4 cond) +{ +#if defined(__SSE4_1__) || defined(_MSC_VER) // on windows assume we always have SSE4.1 + return _mm_blendv_epi8(a, b, _mm_castps_si128(cond.m)); +#else + __m128i d = _mm_srai_epi32(_mm_castps_si128(cond.m), 31); + return _mm_or_si128(_mm_and_si128(d, b), _mm_andnot_si128(d, a)); +#endif +} + +VM_INLINE float4 sqrtf(float4 v) { return float4(_mm_sqrt_ps(v.m)); } + +#elif !defined(__EMSCRIPTEN__) + +// ---- NEON implementation + +#define USE_NEON 1 +#include + +struct float4 +{ + VM_INLINE float4() {} + VM_INLINE explicit float4(const float *p) { m = vld1q_f32(p); } + VM_INLINE explicit float4(float x, float y, float z, float w) { float v[4] = {x, y, z, w}; m = vld1q_f32(v); } + VM_INLINE explicit float4(float v) { m = vdupq_n_f32(v); } + VM_INLINE explicit float4(float32x4_t v) { m = v; } + + VM_INLINE float getX() const { return vgetq_lane_f32(m, 0); } + VM_INLINE float getY() const { return vgetq_lane_f32(m, 1); } + VM_INLINE float getZ() const { return vgetq_lane_f32(m, 2); } + VM_INLINE float getW() const { return vgetq_lane_f32(m, 3); } + + float32x4_t m; +}; + +typedef float4 bool4; + +VM_INLINE float4 operator+ (float4 a, float4 b) { a.m = vaddq_f32(a.m, b.m); return a; } +VM_INLINE float4 operator- (float4 a, float4 b) { a.m = vsubq_f32(a.m, b.m); return a; } +VM_INLINE float4 operator* (float4 a, float4 b) { a.m = vmulq_f32(a.m, b.m); return a; } +VM_INLINE bool4 operator==(float4 a, float4 b) { a.m = vceqq_f32(a.m, b.m); return a; } +VM_INLINE bool4 operator!=(float4 a, float4 b) { a.m = a.m = vmvnq_u32(vceqq_f32(a.m, b.m)); return a; } +VM_INLINE bool4 operator< (float4 a, float4 b) { a.m = vcltq_f32(a.m, b.m); return a; } +VM_INLINE bool4 operator> (float4 a, float4 b) { a.m = vcgtq_f32(a.m, b.m); return a; } +VM_INLINE bool4 operator<=(float4 a, float4 b) { a.m = vcleq_f32(a.m, b.m); return a; } +VM_INLINE bool4 operator>=(float4 a, float4 b) { a.m = vcgeq_f32(a.m, b.m); return a; } +VM_INLINE bool4 operator&(bool4 a, bool4 b) { a.m = vandq_u32(a.m, b.m); return a; } +VM_INLINE bool4 operator|(bool4 a, bool4 b) { a.m = vorrq_u32(a.m, b.m); return a; } +VM_INLINE float4 operator- (float4 a) { a.m = vnegq_f32(a.m); return a; } +VM_INLINE float4 min(float4 a, float4 b) { a.m = vminq_f32(a.m, b.m); return a; } +VM_INLINE float4 max(float4 a, float4 b) { a.m = vmaxq_f32(a.m, b.m); return a; } + +VM_INLINE float hmin(float4 v) +{ + float32x2_t minOfHalfs = vpmin_f32(vget_low_f32(v.m), vget_high_f32(v.m)); + float32x2_t minOfMinOfHalfs = vpmin_f32(minOfHalfs, minOfHalfs); + return vget_lane_f32(minOfMinOfHalfs, 0); +} + +// Returns a 4-bit code where bit0..bit3 is X..W +VM_INLINE unsigned mask(float4 v) +{ + static const uint32x4_t movemask = { 1, 2, 4, 8 }; + static const uint32x4_t highbit = { 0x80000000, 0x80000000, 0x80000000, 0x80000000 }; + uint32x4_t t0 = vreinterpretq_u32_f32(v.m); + uint32x4_t t1 = vtstq_u32(t0, highbit); + uint32x4_t t2 = vandq_u32(t1, movemask); + uint32x2_t t3 = vorr_u32(vget_low_u32(t2), vget_high_u32(t2)); + return vget_lane_u32(t3, 0) | vget_lane_u32(t3, 1); +} +// Once we have a comparison, we can branch based on its results: +VM_INLINE bool any(bool4 v) { return mask(v) != 0; } +VM_INLINE bool all(bool4 v) { return mask(v) == 15; } + +// "select", i.e. hibit(cond) ? b : a +// on SSE4.1 and up this can be done easily via "blend" instruction; +// on older SSEs has to do a bunch of hoops, see +// https://fgiesen.wordpress.com/2016/04/03/sse-mind-the-gap/ + +VM_INLINE float4 select(float4 a, float4 b, bool4 cond) +{ + a.m = vbslq_f32(cond.m, b.m, a.m); + return a; +} +VM_INLINE int32x4_t select(int32x4_t a, int32x4_t b, bool4 cond) +{ + return vbslq_f32(cond.m, b, a); +} + +VM_INLINE float4 sqrtf(float4 v) +{ + float32x4_t V = v.m; + float32x4_t S0 = vrsqrteq_f32(V); + float32x4_t P0 = vmulq_f32( V, S0 ); + float32x4_t R0 = vrsqrtsq_f32( P0, S0 ); + float32x4_t S1 = vmulq_f32( S0, R0 ); + float32x4_t P1 = vmulq_f32( V, S1 ); + float32x4_t R1 = vrsqrtsq_f32( P1, S1 ); + float32x4_t S2 = vmulq_f32( S1, R1 ); + float32x4_t P2 = vmulq_f32( V, S2 ); + float32x4_t R2 = vrsqrtsq_f32( P2, S2 ); + float32x4_t S3 = vmulq_f32( S2, R2 ); + return float4(vmulq_f32(V, S3)); +} + +VM_INLINE float4 splatX(float32x4_t v) { return float4(vdupq_lane_f32(vget_low_f32(v), 0)); } +VM_INLINE float4 splatY(float32x4_t v) { return float4(vdupq_lane_f32(vget_low_f32(v), 1)); } +VM_INLINE float4 splatZ(float32x4_t v) { return float4(vdupq_lane_f32(vget_high_f32(v), 0)); } +VM_INLINE float4 splatW(float32x4_t v) { return float4(vdupq_lane_f32(vget_high_f32(v), 1)); } + +#endif diff --git a/third_party/tracy/examples/ToyPathTracer/Source/Maths.cpp b/third_party/tracy/examples/ToyPathTracer/Source/Maths.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ca8751399b37656c9423b78bb516f2a6a0b8ea8c --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Source/Maths.cpp @@ -0,0 +1,203 @@ +#include "Maths.h" +#include +#include + +static uint32_t XorShift32(uint32_t& state) +{ + uint32_t x = state; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 15; + state = x; + return x; +} + +float RandomFloat01(uint32_t& state) +{ + return (XorShift32(state) & 0xFFFFFF) / 16777216.0f; +} + +float3 RandomInUnitDisk(uint32_t& state) +{ + float3 p; + do + { + p = 2.0 * float3(RandomFloat01(state),RandomFloat01(state),0) - float3(1,1,0); + } while (dot(p,p) >= 1.0); + return p; +} + +float3 RandomInUnitSphere(uint32_t& state) +{ + float3 p; + do { + p = 2.0*float3(RandomFloat01(state),RandomFloat01(state),RandomFloat01(state)) - float3(1,1,1); + } while (sqLength(p) >= 1.0); + return p; +} + +float3 RandomUnitVector(uint32_t& state) +{ + float z = RandomFloat01(state) * 2.0f - 1.0f; + float a = RandomFloat01(state) * 2.0f * kPI; + float r = sqrtf(1.0f - z * z); + float x = r * cosf(a); + float y = r * sinf(a); + return float3(x, y, z); +} + + +int HitSpheres(const Ray& r, const SpheresSoA& spheres, float tMin, float tMax, Hit& outHit) +{ +#if DO_HIT_SPHERES_SIMD + float4 hitT = float4(tMax); +#if USE_NEON + int32x4_t id = vdupq_n_s32(-1); +#else + __m128i id = _mm_set1_epi32(-1); +#endif + +#if DO_FLOAT3_WITH_SIMD && !USE_NEON + float4 rOrigX = SHUFFLE4(r.orig, 0, 0, 0, 0); + float4 rOrigY = SHUFFLE4(r.orig, 1, 1, 1, 1); + float4 rOrigZ = SHUFFLE4(r.orig, 2, 2, 2, 2); + float4 rDirX = SHUFFLE4(r.dir, 0, 0, 0, 0); + float4 rDirY = SHUFFLE4(r.dir, 1, 1, 1, 1); + float4 rDirZ = SHUFFLE4(r.dir, 2, 2, 2, 2); +#elif DO_FLOAT3_WITH_SIMD + float4 rOrigX = splatX(r.orig.m); + float4 rOrigY = splatY(r.orig.m); + float4 rOrigZ = splatZ(r.orig.m); + float4 rDirX = splatX(r.dir.m); + float4 rDirY = splatY(r.dir.m); + float4 rDirZ = splatZ(r.dir.m); +#else + float4 rOrigX = float4(r.orig.x); + float4 rOrigY = float4(r.orig.y); + float4 rOrigZ = float4(r.orig.z); + float4 rDirX = float4(r.dir.x); + float4 rDirY = float4(r.dir.y); + float4 rDirZ = float4(r.dir.z); +#endif + float4 tMin4 = float4(tMin); +#if USE_NEON + int32x4_t curId = vcombine_u32(vcreate_u32(0ULL | (1ULL<<32)), vcreate_u32(2ULL | (3ULL<<32))); +#else + __m128i curId = _mm_set_epi32(3, 2, 1, 0); +#endif + // process 4 spheres at once + for (int i = 0; i < spheres.simdCount; i += kSimdWidth) + { + // load data for 4 spheres + float4 sCenterX = float4(spheres.centerX + i); + float4 sCenterY = float4(spheres.centerY + i); + float4 sCenterZ = float4(spheres.centerZ + i); + float4 sSqRadius = float4(spheres.sqRadius + i); + // note: we flip this vector and calculate -b (nb) since that happens to be slightly preferable computationally + float4 coX = sCenterX - rOrigX; + float4 coY = sCenterY - rOrigY; + float4 coZ = sCenterZ - rOrigZ; + float4 nb = coX * rDirX + coY * rDirY + coZ * rDirZ; + float4 c = coX * coX + coY * coY + coZ * coZ - sSqRadius; + float4 discr = nb * nb - c; + bool4 discrPos = discr > float4(0.0f); + // if ray hits any of the 4 spheres + if (any(discrPos)) + { + float4 discrSq = sqrtf(discr); + + // ray could hit spheres at t0 & t1 + float4 t0 = nb - discrSq; + float4 t1 = nb + discrSq; + + float4 t = select(t1, t0, t0 > tMin4); // if t0 is above min, take it (since it's the earlier hit); else try t1. + bool4 msk = discrPos & (t > tMin4) & (t < hitT); + // if hit, take it + id = select(id, curId, msk); + hitT = select(hitT, t, msk); + } +#if USE_NEON + curId = vaddq_s32(curId, vdupq_n_s32(kSimdWidth)); +#else + curId = _mm_add_epi32(curId, _mm_set1_epi32(kSimdWidth)); +#endif + } + // now we have up to 4 hits, find and return closest one + float minT = hmin(hitT); + if (minT < tMax) // any actual hits? + { + int minMask = mask(hitT == float4(minT)); + if (minMask != 0) + { + int id_scalar[4]; + float hitT_scalar[4]; +#if USE_NEON + vst1q_s32(id_scalar, id); + vst1q_f32(hitT_scalar, hitT.m); +#else + _mm_storeu_si128((__m128i *)id_scalar, id); + _mm_storeu_ps(hitT_scalar, hitT.m); +#endif + + // In general, you would do this with a bit scan (first set/trailing zero count). + // But who cares, it's only 16 options. + static const int laneId[16] = + { + 0, 0, 1, 0, // 00xx + 2, 0, 1, 0, // 01xx + 3, 0, 1, 0, // 10xx + 2, 0, 1, 0, // 11xx + }; + + int lane = laneId[minMask]; + int hitId = id_scalar[lane]; + float finalHitT = hitT_scalar[lane]; + + outHit.pos = r.pointAt(finalHitT); + outHit.normal = (outHit.pos - float3(spheres.centerX[hitId], spheres.centerY[hitId], spheres.centerZ[hitId])) * spheres.invRadius[hitId]; + outHit.t = finalHitT; + return hitId; + } + } + + return -1; + +#else // #if DO_HIT_SPHERES_SIMD + + float hitT = tMax; + int id = -1; + for (int i = 0; i < spheres.count; ++i) + { + float coX = spheres.centerX[i] - r.orig.getX(); + float coY = spheres.centerY[i] - r.orig.getY(); + float coZ = spheres.centerZ[i] - r.orig.getZ(); + float nb = coX * r.dir.getX() + coY * r.dir.getY() + coZ * r.dir.getZ(); + float c = coX * coX + coY * coY + coZ * coZ - spheres.sqRadius[i]; + float discr = nb * nb - c; + if (discr > 0) + { + float discrSq = sqrtf(discr); + + // Try earlier t + float t = nb - discrSq; + if (t <= tMin) // before min, try later t! + t = nb + discrSq; + + if (t > tMin && t < hitT) + { + id = i; + hitT = t; + } + } + } + if (id != -1) + { + outHit.pos = r.pointAt(hitT); + outHit.normal = (outHit.pos - float3(spheres.centerX[id], spheres.centerY[id], spheres.centerZ[id])) * spheres.invRadius[id]; + outHit.t = hitT; + return id; + } + else + return -1; +#endif // #else of #if DO_HIT_SPHERES_SIMD +} diff --git a/third_party/tracy/examples/ToyPathTracer/Source/Maths.h b/third_party/tracy/examples/ToyPathTracer/Source/Maths.h new file mode 100644 index 0000000000000000000000000000000000000000..b587715feaffe12e1b86fd42b9abe6232ebc61e0 --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Source/Maths.h @@ -0,0 +1,436 @@ +#pragma once + +#include +#include +#include +#include "Config.h" +#include "MathSimd.h" + +#define kPI 3.1415926f + +// SSE/SIMD vector largely based on http://www.codersnotes.com/notes/maths-lib-2016/ +#if DO_FLOAT3_WITH_SIMD + + +#if !defined(__arm__) && !defined(__arm64__) + +// ---- SSE implementation + +// SHUFFLE3(v, 0,1,2) leaves the vector unchanged (v.xyz). +// SHUFFLE3(v, 0,0,0) splats the X (v.xxx). +#define SHUFFLE3(V, X,Y,Z) float3(_mm_shuffle_ps((V).m, (V).m, _MM_SHUFFLE(Z,Z,Y,X))) + +struct float3 +{ + VM_INLINE float3() {} + VM_INLINE explicit float3(const float *p) { m = _mm_set_ps(p[2], p[2], p[1], p[0]); } + VM_INLINE explicit float3(float x, float y, float z) { m = _mm_set_ps(z, z, y, x); } + VM_INLINE explicit float3(float v) { m = _mm_set1_ps(v); } + VM_INLINE explicit float3(__m128 v) { m = v; } + + VM_INLINE float getX() const { return _mm_cvtss_f32(m); } + VM_INLINE float getY() const { return _mm_cvtss_f32(_mm_shuffle_ps(m, m, _MM_SHUFFLE(1, 1, 1, 1))); } + VM_INLINE float getZ() const { return _mm_cvtss_f32(_mm_shuffle_ps(m, m, _MM_SHUFFLE(2, 2, 2, 2))); } + + VM_INLINE float3 yzx() const { return SHUFFLE3(*this, 1, 2, 0); } + VM_INLINE float3 zxy() const { return SHUFFLE3(*this, 2, 0, 1); } + + VM_INLINE void store(float *p) const { p[0] = getX(); p[1] = getY(); p[2] = getZ(); } + + void setX(float x) + { + m = _mm_move_ss(m, _mm_set_ss(x)); + } + void setY(float y) + { + __m128 t = _mm_move_ss(m, _mm_set_ss(y)); + t = _mm_shuffle_ps(t, t, _MM_SHUFFLE(3, 2, 0, 0)); + m = _mm_move_ss(t, m); + } + void setZ(float z) + { + __m128 t = _mm_move_ss(m, _mm_set_ss(z)); + t = _mm_shuffle_ps(t, t, _MM_SHUFFLE(3, 0, 1, 0)); + m = _mm_move_ss(t, m); + } + + __m128 m; +}; + +typedef float3 bool3; + +VM_INLINE float3 operator+ (float3 a, float3 b) { a.m = _mm_add_ps(a.m, b.m); return a; } +VM_INLINE float3 operator- (float3 a, float3 b) { a.m = _mm_sub_ps(a.m, b.m); return a; } +VM_INLINE float3 operator* (float3 a, float3 b) { a.m = _mm_mul_ps(a.m, b.m); return a; } +VM_INLINE float3 operator/ (float3 a, float3 b) { a.m = _mm_div_ps(a.m, b.m); return a; } +VM_INLINE float3 operator* (float3 a, float b) { a.m = _mm_mul_ps(a.m, _mm_set1_ps(b)); return a; } +VM_INLINE float3 operator/ (float3 a, float b) { a.m = _mm_div_ps(a.m, _mm_set1_ps(b)); return a; } +VM_INLINE float3 operator* (float a, float3 b) { b.m = _mm_mul_ps(_mm_set1_ps(a), b.m); return b; } +VM_INLINE float3 operator/ (float a, float3 b) { b.m = _mm_div_ps(_mm_set1_ps(a), b.m); return b; } +VM_INLINE float3& operator+= (float3 &a, float3 b) { a = a + b; return a; } +VM_INLINE float3& operator-= (float3 &a, float3 b) { a = a - b; return a; } +VM_INLINE float3& operator*= (float3 &a, float3 b) { a = a * b; return a; } +VM_INLINE float3& operator/= (float3 &a, float3 b) { a = a / b; return a; } +VM_INLINE float3& operator*= (float3 &a, float b) { a = a * b; return a; } +VM_INLINE float3& operator/= (float3 &a, float b) { a = a / b; return a; } +VM_INLINE bool3 operator==(float3 a, float3 b) { a.m = _mm_cmpeq_ps(a.m, b.m); return a; } +VM_INLINE bool3 operator!=(float3 a, float3 b) { a.m = _mm_cmpneq_ps(a.m, b.m); return a; } +VM_INLINE bool3 operator< (float3 a, float3 b) { a.m = _mm_cmplt_ps(a.m, b.m); return a; } +VM_INLINE bool3 operator> (float3 a, float3 b) { a.m = _mm_cmpgt_ps(a.m, b.m); return a; } +VM_INLINE bool3 operator<=(float3 a, float3 b) { a.m = _mm_cmple_ps(a.m, b.m); return a; } +VM_INLINE bool3 operator>=(float3 a, float3 b) { a.m = _mm_cmpge_ps(a.m, b.m); return a; } +VM_INLINE float3 min(float3 a, float3 b) { a.m = _mm_min_ps(a.m, b.m); return a; } +VM_INLINE float3 max(float3 a, float3 b) { a.m = _mm_max_ps(a.m, b.m); return a; } + +VM_INLINE float3 operator- (float3 a) { return float3(_mm_setzero_ps()) - a; } + +VM_INLINE float hmin(float3 v) +{ + v = min(v, SHUFFLE3(v, 1, 0, 2)); + return min(v, SHUFFLE3(v, 2, 0, 1)).getX(); +} +VM_INLINE float hmax(float3 v) +{ + v = max(v, SHUFFLE3(v, 1, 0, 2)); + return max(v, SHUFFLE3(v, 2, 0, 1)).getX(); +} + +VM_INLINE float3 cross(float3 a, float3 b) +{ + // x <- a.y*b.z - a.z*b.y + // y <- a.z*b.x - a.x*b.z + // z <- a.x*b.y - a.y*b.x + // We can save a shuffle by grouping it in this wacky order: + return (a.zxy()*b - a*b.zxy()).zxy(); +} + +// Returns a 3-bit code where bit0..bit2 is X..Z +VM_INLINE unsigned mask(float3 v) { return _mm_movemask_ps(v.m) & 7; } +// Once we have a comparison, we can branch based on its results: +VM_INLINE bool any(bool3 v) { return mask(v) != 0; } +VM_INLINE bool all(bool3 v) { return mask(v) == 7; } + +VM_INLINE float3 clamp(float3 t, float3 a, float3 b) { return min(max(t, a), b); } +VM_INLINE float sum(float3 v) { return v.getX() + v.getY() + v.getZ(); } +VM_INLINE float dot(float3 a, float3 b) { return sum(a*b); } + +#else // #if !defined(__arm__) && !defined(__arm64__) + +// ---- NEON implementation + +#include + +struct float3 +{ + VM_INLINE float3() {} + VM_INLINE explicit float3(const float *p) { float v[4] = {p[0], p[1], p[2], 0}; m = vld1q_f32(v); } + VM_INLINE explicit float3(float x, float y, float z) { float v[4] = {x, y, z, 0}; m = vld1q_f32(v); } + VM_INLINE explicit float3(float v) { m = vdupq_n_f32(v); } + VM_INLINE explicit float3(float32x4_t v) { m = v; } + + VM_INLINE float getX() const { return vgetq_lane_f32(m, 0); } + VM_INLINE float getY() const { return vgetq_lane_f32(m, 1); } + VM_INLINE float getZ() const { return vgetq_lane_f32(m, 2); } + + VM_INLINE float3 yzx() const + { + float32x2_t low = vget_low_f32(m); + float32x4_t yzx = vcombine_f32(vext_f32(low, vget_high_f32(m), 1), low); + return float3(yzx); + } + VM_INLINE float3 zxy() const + { + float32x4_t p = m; + p = vuzpq_f32(vreinterpretq_f32_s32(vextq_s32(vreinterpretq_s32_f32(p), vreinterpretq_s32_f32(p), 1)), p).val[1]; + return float3(p); + } + + VM_INLINE void store(float *p) const { p[0] = getX(); p[1] = getY(); p[2] = getZ(); } + + void setX(float x) + { + m = vsetq_lane_f32(x, m, 0); + } + void setY(float y) + { + m = vsetq_lane_f32(y, m, 1); + } + void setZ(float z) + { + m = vsetq_lane_f32(z, m, 2); + } + + float32x4_t m; +}; + +typedef float3 bool3; + +VM_INLINE float32x4_t rcp_2(float32x4_t v) +{ + float32x4_t e = vrecpeq_f32(v); + e = vmulq_f32(vrecpsq_f32(e, v), e); + e = vmulq_f32(vrecpsq_f32(e, v), e); + return e; +} + +VM_INLINE float3 operator+ (float3 a, float3 b) { a.m = vaddq_f32(a.m, b.m); return a; } +VM_INLINE float3 operator- (float3 a, float3 b) { a.m = vsubq_f32(a.m, b.m); return a; } +VM_INLINE float3 operator* (float3 a, float3 b) { a.m = vmulq_f32(a.m, b.m); return a; } +VM_INLINE float3 operator/ (float3 a, float3 b) { float32x4_t recip = rcp_2(b.m); a.m = vmulq_f32(a.m, recip); return a; } +VM_INLINE float3 operator* (float3 a, float b) { a.m = vmulq_f32(a.m, vdupq_n_f32(b)); return a; } +VM_INLINE float3 operator/ (float3 a, float b) { float32x4_t recip = rcp_2(vdupq_n_f32(b)); a.m = vmulq_f32(a.m, recip); return a; } +VM_INLINE float3 operator* (float a, float3 b) { b.m = vmulq_f32(vdupq_n_f32(a), b.m); return b; } +VM_INLINE float3 operator/ (float a, float3 b) { float32x4_t recip = rcp_2(b.m); b.m = vmulq_f32(vdupq_n_f32(a), recip); return b; } +VM_INLINE float3& operator+= (float3 &a, float3 b) { a = a + b; return a; } +VM_INLINE float3& operator-= (float3 &a, float3 b) { a = a - b; return a; } +VM_INLINE float3& operator*= (float3 &a, float3 b) { a = a * b; return a; } +VM_INLINE float3& operator/= (float3 &a, float3 b) { a = a / b; return a; } +VM_INLINE float3& operator*= (float3 &a, float b) { a = a * b; return a; } +VM_INLINE float3& operator/= (float3 &a, float b) { a = a / b; return a; } +VM_INLINE bool3 operator==(float3 a, float3 b) { a.m = vceqq_f32(a.m, b.m); return a; } +VM_INLINE bool3 operator!=(float3 a, float3 b) { a.m = vmvnq_u32(vceqq_f32(a.m, b.m)); return a; } +VM_INLINE bool3 operator< (float3 a, float3 b) { a.m = vcltq_f32(a.m, b.m); return a; } +VM_INLINE bool3 operator> (float3 a, float3 b) { a.m = vcgtq_f32(a.m, b.m); return a; } +VM_INLINE bool3 operator<=(float3 a, float3 b) { a.m = vcleq_f32(a.m, b.m); return a; } +VM_INLINE bool3 operator>=(float3 a, float3 b) { a.m = vcgeq_f32(a.m, b.m); return a; } +VM_INLINE float3 min(float3 a, float3 b) { a.m = vminq_f32(a.m, b.m); return a; } +VM_INLINE float3 max(float3 a, float3 b) { a.m = vmaxq_f32(a.m, b.m); return a; } + +VM_INLINE float3 operator- (float3 a) { a.m = vnegq_f32(a.m); return a; } + +VM_INLINE float hmin(float3 v) +{ + float32x2_t minOfHalfs = vpmin_f32(vget_low_f32(v.m), vget_high_f32(v.m)); + float32x2_t minOfMinOfHalfs = vpmin_f32(minOfHalfs, minOfHalfs); + return vget_lane_f32(minOfMinOfHalfs, 0); +} +VM_INLINE float hmax(float3 v) +{ + float32x2_t maxOfHalfs = vpmax_f32(vget_low_f32(v.m), vget_high_f32(v.m)); + float32x2_t maxOfMaxOfHalfs = vpmax_f32(maxOfHalfs, maxOfHalfs); + return vget_lane_f32(maxOfMaxOfHalfs, 0); +} + +VM_INLINE float3 cross(float3 a, float3 b) +{ + // x <- a.y*b.z - a.z*b.y + // y <- a.z*b.x - a.x*b.z + // z <- a.x*b.y - a.y*b.x + // We can save a shuffle by grouping it in this wacky order: + return (a.zxy()*b - a*b.zxy()).zxy(); +} + +// Returns a 3-bit code where bit0..bit2 is X..Z +VM_INLINE unsigned mask(float3 v) +{ + static const uint32x4_t movemask = { 1, 2, 4, 8 }; + static const uint32x4_t highbit = { 0x80000000, 0x80000000, 0x80000000, 0x80000000 }; + uint32x4_t t0 = vreinterpretq_u32_f32(v.m); + uint32x4_t t1 = vtstq_u32(t0, highbit); + uint32x4_t t2 = vandq_u32(t1, movemask); + uint32x2_t t3 = vorr_u32(vget_low_u32(t2), vget_high_u32(t2)); + return vget_lane_u32(t3, 0) | vget_lane_u32(t3, 1); +} +// Once we have a comparison, we can branch based on its results: +VM_INLINE bool any(bool3 v) { return mask(v) != 0; } +VM_INLINE bool all(bool3 v) { return mask(v) == 7; } + +VM_INLINE float3 clamp(float3 t, float3 a, float3 b) { return min(max(t, a), b); } +VM_INLINE float sum(float3 v) { return v.getX() + v.getY() + v.getZ(); } +VM_INLINE float dot(float3 a, float3 b) { return sum(a*b); } + + +#endif // #else of #if !defined(__arm__) && !defined(__arm64__) + +#else // #if DO_FLOAT3_WITH_SIMD + +// ---- Simple scalar C implementation + + +struct float3 +{ + float3() : x(0), y(0), z(0) {} + float3(float x_, float y_, float z_) : x(x_), y(y_), z(z_) {} + + float3 operator-() const { return float3(-x, -y, -z); } + float3& operator+=(const float3& o) { x+=o.x; y+=o.y; z+=o.z; return *this; } + float3& operator-=(const float3& o) { x-=o.x; y-=o.y; z-=o.z; return *this; } + float3& operator*=(const float3& o) { x*=o.x; y*=o.y; z*=o.z; return *this; } + float3& operator*=(float o) { x*=o; y*=o; z*=o; return *this; } + + VM_INLINE float getX() const { return x; } + VM_INLINE float getY() const { return y; } + VM_INLINE float getZ() const { return z; } + VM_INLINE void setX(float x_) { x = x_; } + VM_INLINE void setY(float y_) { y = y_; } + VM_INLINE void setZ(float z_) { z = z_; } + VM_INLINE void store(float *p) const { p[0] = getX(); p[1] = getY(); p[2] = getZ(); } + + float x, y, z; +}; + +VM_INLINE float3 operator+(const float3& a, const float3& b) { return float3(a.x+b.x,a.y+b.y,a.z+b.z); } +VM_INLINE float3 operator-(const float3& a, const float3& b) { return float3(a.x-b.x,a.y-b.y,a.z-b.z); } +VM_INLINE float3 operator*(const float3& a, const float3& b) { return float3(a.x*b.x,a.y*b.y,a.z*b.z); } +VM_INLINE float3 operator*(const float3& a, float b) { return float3(a.x*b,a.y*b,a.z*b); } +VM_INLINE float3 operator*(float a, const float3& b) { return float3(a*b.x,a*b.y,a*b.z); } +VM_INLINE float dot(const float3& a, const float3& b) { return a.x*b.x+a.y*b.y+a.z*b.z; } +VM_INLINE float3 cross(const float3& a, const float3& b) +{ + return float3( + a.y*b.z - a.z*b.y, + -(a.x*b.z - a.z*b.x), + a.x*b.y - a.y*b.x + ); +} +#endif // #else of #if DO_FLOAT3_WITH_SIMD + +VM_INLINE float length(float3 v) { return sqrtf(dot(v, v)); } +VM_INLINE float sqLength(float3 v) { return dot(v, v); } +VM_INLINE float3 normalize(float3 v) { return v * (1.0f / length(v)); } +VM_INLINE float3 lerp(float3 a, float3 b, float t) { return a + (b-a)*t; } + + +inline void AssertUnit(float3 v) +{ + assert(fabsf(sqLength(v) - 1.0f) < 0.01f); +} + +inline float3 reflect(float3 v, float3 n) +{ + return v - 2*dot(v,n)*n; +} + +inline bool refract(float3 v, float3 n, float nint, float3& outRefracted) +{ + AssertUnit(v); + float dt = dot(v, n); + float discr = 1.0f - nint*nint*(1-dt*dt); + if (discr > 0) + { + outRefracted = nint * (v - n*dt) - n*sqrtf(discr); + return true; + } + return false; +} +inline float schlick(float cosine, float ri) +{ + float r0 = (1-ri) / (1+ri); + r0 = r0*r0; + return r0 + (1-r0)*powf(1-cosine, 5); +} + +struct Ray +{ + Ray() {} + Ray(float3 orig_, float3 dir_) : orig(orig_), dir(dir_) { AssertUnit(dir); } + + float3 pointAt(float t) const { return orig + dir * t; } + + float3 orig; + float3 dir; +}; + + +struct Hit +{ + float3 pos; + float3 normal; + float t; +}; + + +struct Sphere +{ + Sphere() : radius(1.0f), invRadius(0.0f) {} + Sphere(float3 center_, float radius_) : center(center_), radius(radius_), invRadius(0.0f) {} + + void UpdateDerivedData() { invRadius = 1.0f/radius; } + + float3 center; + float radius; + float invRadius; +}; + + +// data for all spheres in a "structure of arrays" layout +struct SpheresSoA +{ + SpheresSoA(int c) + { + count = c; + // we'll be processing spheres in kSimdWidth chunks, so make sure to allocate + // enough space + simdCount = (c + (kSimdWidth - 1)) / kSimdWidth * kSimdWidth; + centerX = new float[simdCount]; + centerY = new float[simdCount]; + centerZ = new float[simdCount]; + sqRadius = new float[simdCount]; + invRadius = new float[simdCount]; + // set all data to "impossible sphere" state + for (int i = count; i < simdCount; ++i) + { + centerX[i] = centerY[i] = centerZ[i] = 10000.0f; + sqRadius[i] = 0.0f; + invRadius[i] = 0.0f; + } + } + ~SpheresSoA() + { + delete[] centerX; + delete[] centerY; + delete[] centerZ; + delete[] sqRadius; + delete[] invRadius; + } + float* centerX; + float* centerY; + float* centerZ; + float* sqRadius; + float* invRadius; + int simdCount; + int count; +}; + + +int HitSpheres(const Ray& r, const SpheresSoA& spheres, float tMin, float tMax, Hit& outHit); + +float RandomFloat01(uint32_t& state); +float3 RandomInUnitDisk(uint32_t& state); +float3 RandomInUnitSphere(uint32_t& state); +float3 RandomUnitVector(uint32_t& state); + +struct Camera +{ + Camera() {} + // vfov is top to bottom in degrees + Camera(const float3& lookFrom, const float3& lookAt, const float3& vup, float vfov, float aspect, float aperture, float focusDist) + { + lensRadius = aperture / 2; + float theta = vfov*kPI/180; + float halfHeight = tanf(theta/2); + float halfWidth = aspect * halfHeight; + origin = lookFrom; + w = normalize(lookFrom - lookAt); + u = normalize(cross(vup, w)); + v = cross(w, u); + lowerLeftCorner = origin - halfWidth*focusDist*u - halfHeight*focusDist*v - focusDist*w; + horizontal = 2*halfWidth*focusDist*u; + vertical = 2*halfHeight*focusDist*v; + } + + Ray GetRay(float s, float t, uint32_t& state) const + { + float3 rd = lensRadius * RandomInUnitDisk(state); + float3 offset = u * rd.getX() + v * rd.getY(); + return Ray(origin + offset, normalize(lowerLeftCorner + s*horizontal + t*vertical - origin - offset)); + } + + float3 origin; + float3 lowerLeftCorner; + float3 horizontal; + float3 vertical; + float3 u, v, w; + float lensRadius; +}; + diff --git a/third_party/tracy/examples/ToyPathTracer/Source/Test.cpp b/third_party/tracy/examples/ToyPathTracer/Source/Test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bf4a02f969a25c923258e454c469204a14c74a59 --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Source/Test.cpp @@ -0,0 +1,392 @@ +#include "Config.h" +#include "Test.h" +#include "Maths.h" +#include +#if CPU_CAN_DO_THREADS +#include "enkiTS/TaskScheduler_c.h" +#include +#endif +#include + +#include "../../../public/tracy/Tracy.hpp" + +// 46 spheres (2 emissive) when enabled; 9 spheres (1 emissive) when disabled +#define DO_BIG_SCENE 1 + +static Sphere s_Spheres[] = +{ + {float3(0,-100.5,-1), 100}, + {float3(2,0,-1), 0.5f}, + {float3(0,0,-1), 0.5f}, + {float3(-2,0,-1), 0.5f}, + {float3(2,0,1), 0.5f}, + {float3(0,0,1), 0.5f}, + {float3(-2,0,1), 0.5f}, + {float3(0.5f,1,0.5f), 0.5f}, + {float3(-1.5f,1.5f,0.f), 0.3f}, +#if DO_BIG_SCENE + {float3(4,0,-3), 0.5f}, {float3(3,0,-3), 0.5f}, {float3(2,0,-3), 0.5f}, {float3(1,0,-3), 0.5f}, {float3(0,0,-3), 0.5f}, {float3(-1,0,-3), 0.5f}, {float3(-2,0,-3), 0.5f}, {float3(-3,0,-3), 0.5f}, {float3(-4,0,-3), 0.5f}, + {float3(4,0,-4), 0.5f}, {float3(3,0,-4), 0.5f}, {float3(2,0,-4), 0.5f}, {float3(1,0,-4), 0.5f}, {float3(0,0,-4), 0.5f}, {float3(-1,0,-4), 0.5f}, {float3(-2,0,-4), 0.5f}, {float3(-3,0,-4), 0.5f}, {float3(-4,0,-4), 0.5f}, + {float3(4,0,-5), 0.5f}, {float3(3,0,-5), 0.5f}, {float3(2,0,-5), 0.5f}, {float3(1,0,-5), 0.5f}, {float3(0,0,-5), 0.5f}, {float3(-1,0,-5), 0.5f}, {float3(-2,0,-5), 0.5f}, {float3(-3,0,-5), 0.5f}, {float3(-4,0,-5), 0.5f}, + {float3(4,0,-6), 0.5f}, {float3(3,0,-6), 0.5f}, {float3(2,0,-6), 0.5f}, {float3(1,0,-6), 0.5f}, {float3(0,0,-6), 0.5f}, {float3(-1,0,-6), 0.5f}, {float3(-2,0,-6), 0.5f}, {float3(-3,0,-6), 0.5f}, {float3(-4,0,-6), 0.5f}, + {float3(1.5f,1.5f,-2), 0.3f}, +#endif // #if DO_BIG_SCENE +}; +const int kSphereCount = sizeof(s_Spheres) / sizeof(s_Spheres[0]); + +static SpheresSoA s_SpheresSoA(kSphereCount); + +struct Material +{ + enum Type { Lambert, Metal, Dielectric }; + Type type; + float3 albedo; + float3 emissive; + float roughness; + float ri; +}; + +static Material s_SphereMats[kSphereCount] = +{ + { Material::Lambert, float3(0.8f, 0.8f, 0.8f), float3(0,0,0), 0, 0, }, + { Material::Lambert, float3(0.8f, 0.4f, 0.4f), float3(0,0,0), 0, 0, }, + { Material::Lambert, float3(0.4f, 0.8f, 0.4f), float3(0,0,0), 0, 0, }, + { Material::Metal, float3(0.4f, 0.4f, 0.8f), float3(0,0,0), 0, 0 }, + { Material::Metal, float3(0.4f, 0.8f, 0.4f), float3(0,0,0), 0, 0 }, + { Material::Metal, float3(0.4f, 0.8f, 0.4f), float3(0,0,0), 0.2f, 0 }, + { Material::Metal, float3(0.4f, 0.8f, 0.4f), float3(0,0,0), 0.6f, 0 }, + { Material::Dielectric, float3(0.4f, 0.4f, 0.4f), float3(0,0,0), 0, 1.5f }, + { Material::Lambert, float3(0.8f, 0.6f, 0.2f), float3(30,25,15), 0, 0 }, +#if DO_BIG_SCENE + { Material::Lambert, float3(0.1f, 0.1f, 0.1f), float3(0,0,0), 0, 0, }, { Material::Lambert, float3(0.2f, 0.2f, 0.2f), float3(0,0,0), 0, 0, }, { Material::Lambert, float3(0.3f, 0.3f, 0.3f), float3(0,0,0), 0, 0, }, { Material::Lambert, float3(0.4f, 0.4f, 0.4f), float3(0,0,0), 0, 0, }, { Material::Lambert, float3(0.5f, 0.5f, 0.5f), float3(0,0,0), 0, 0, }, { Material::Lambert, float3(0.6f, 0.6f, 0.6f), float3(0,0,0), 0, 0, }, { Material::Lambert, float3(0.7f, 0.7f, 0.7f), float3(0,0,0), 0, 0, }, { Material::Lambert, float3(0.8f, 0.8f, 0.8f), float3(0,0,0), 0, 0, }, { Material::Lambert, float3(0.9f, 0.9f, 0.9f), float3(0,0,0), 0, 0, }, + { Material::Metal, float3(0.1f, 0.1f, 0.1f), float3(0,0,0), 0, 0, }, { Material::Metal, float3(0.2f, 0.2f, 0.2f), float3(0,0,0), 0, 0, }, { Material::Metal, float3(0.3f, 0.3f, 0.3f), float3(0,0,0), 0, 0, }, { Material::Metal, float3(0.4f, 0.4f, 0.4f), float3(0,0,0), 0, 0, }, { Material::Metal, float3(0.5f, 0.5f, 0.5f), float3(0,0,0), 0, 0, }, { Material::Metal, float3(0.6f, 0.6f, 0.6f), float3(0,0,0), 0, 0, }, { Material::Metal, float3(0.7f, 0.7f, 0.7f), float3(0,0,0), 0, 0, }, { Material::Metal, float3(0.8f, 0.8f, 0.8f), float3(0,0,0), 0, 0, }, { Material::Metal, float3(0.9f, 0.9f, 0.9f), float3(0,0,0), 0, 0, }, + { Material::Metal, float3(0.8f, 0.1f, 0.1f), float3(0,0,0), 0, 0, }, { Material::Metal, float3(0.8f, 0.5f, 0.1f), float3(0,0,0), 0, 0, }, { Material::Metal, float3(0.8f, 0.8f, 0.1f), float3(0,0,0), 0, 0, }, { Material::Metal, float3(0.4f, 0.8f, 0.1f), float3(0,0,0), 0, 0, }, { Material::Metal, float3(0.1f, 0.8f, 0.1f), float3(0,0,0), 0, 0, }, { Material::Metal, float3(0.1f, 0.8f, 0.5f), float3(0,0,0), 0, 0, }, { Material::Metal, float3(0.1f, 0.8f, 0.8f), float3(0,0,0), 0, 0, }, { Material::Metal, float3(0.1f, 0.1f, 0.8f), float3(0,0,0), 0, 0, }, { Material::Metal, float3(0.5f, 0.1f, 0.8f), float3(0,0,0), 0, 0, }, + { Material::Lambert, float3(0.8f, 0.1f, 0.1f), float3(0,0,0), 0, 0, }, { Material::Lambert, float3(0.8f, 0.5f, 0.1f), float3(0,0,0), 0, 0, }, { Material::Lambert, float3(0.8f, 0.8f, 0.1f), float3(0,0,0), 0, 0, }, { Material::Lambert, float3(0.4f, 0.8f, 0.1f), float3(0,0,0), 0, 0, }, { Material::Lambert, float3(0.1f, 0.8f, 0.1f), float3(0,0,0), 0, 0, }, { Material::Lambert, float3(0.1f, 0.8f, 0.5f), float3(0,0,0), 0, 0, }, { Material::Lambert, float3(0.1f, 0.8f, 0.8f), float3(0,0,0), 0, 0, }, { Material::Lambert, float3(0.1f, 0.1f, 0.8f), float3(0,0,0), 0, 0, }, { Material::Metal, float3(0.5f, 0.1f, 0.8f), float3(0,0,0), 0, 0, }, + { Material::Lambert, float3(0.1f, 0.2f, 0.5f), float3(3,10,20), 0, 0 }, +#endif +}; + +static int s_EmissiveSpheres[kSphereCount]; +static int s_EmissiveSphereCount; + +static Camera s_Cam; + +const float kMinT = 0.001f; +const float kMaxT = 1.0e7f; +const int kMaxDepth = 10; + + +bool HitWorld(const Ray& r, float tMin, float tMax, Hit& outHit, int& outID) +{ + outID = HitSpheres(r, s_SpheresSoA, tMin, tMax, outHit); + return outID != -1; +} + + +static bool Scatter(const Material& mat, const Ray& r_in, const Hit& rec, float3& attenuation, Ray& scattered, float3& outLightE, int& inoutRayCount, uint32_t& state) +{ + ZoneScoped; + outLightE = float3(0,0,0); + if (mat.type == Material::Lambert) + { + // random point on unit sphere that is tangent to the hit point + float3 target = rec.pos + rec.normal + RandomUnitVector(state); + scattered = Ray(rec.pos, normalize(target - rec.pos)); + attenuation = mat.albedo; + + // sample lights +#if DO_LIGHT_SAMPLING + for (int j = 0; j < s_EmissiveSphereCount; ++j) + { + int i = s_EmissiveSpheres[j]; + const Material& smat = s_SphereMats[i]; + if (&mat == &smat) + continue; // skip self + const Sphere& s = s_Spheres[i]; + + // create a random direction towards sphere + // coord system for sampling: sw, su, sv + float3 sw = normalize(s.center - rec.pos); + float3 su = normalize(cross(fabs(sw.getX())>0.01f ? float3(0,1,0):float3(1,0,0), sw)); + float3 sv = cross(sw, su); + // sample sphere by solid angle + float cosAMax = sqrtf(1.0f - s.radius*s.radius / sqLength(rec.pos-s.center)); + float eps1 = RandomFloat01(state), eps2 = RandomFloat01(state); + float cosA = 1.0f - eps1 + eps1 * cosAMax; + float sinA = sqrtf(1.0f - cosA*cosA); + float phi = 2 * kPI * eps2; + float3 l = su * (cosf(phi) * sinA) + sv * (sinf(phi) * sinA) + sw * cosA; + //l = normalize(l); // NOTE(fg): This is already normalized, by construction. + + // shoot shadow ray + Hit lightHit; + int hitID; + ++inoutRayCount; + if (HitWorld(Ray(rec.pos, l), kMinT, kMaxT, lightHit, hitID) && hitID == i) + { + float omega = 2 * kPI * (1-cosAMax); + + float3 rdir = r_in.dir; + AssertUnit(rdir); + float3 nl = dot(rec.normal, rdir) < 0 ? rec.normal : -rec.normal; + outLightE += (mat.albedo * smat.emissive) * (std::max(0.0f, dot(l, nl)) * omega / kPI); + } + } +#endif + return true; + } + else if (mat.type == Material::Metal) + { + AssertUnit(r_in.dir); AssertUnit(rec.normal); + float3 refl = reflect(r_in.dir, rec.normal); + // reflected ray, and random inside of sphere based on roughness + float roughness = mat.roughness; +#if DO_MITSUBA_COMPARE + roughness = 0; // until we get better BRDF for metals +#endif + scattered = Ray(rec.pos, normalize(refl + roughness*RandomInUnitSphere(state))); + attenuation = mat.albedo; + return dot(scattered.dir, rec.normal) > 0; + } + else if (mat.type == Material::Dielectric) + { + AssertUnit(r_in.dir); AssertUnit(rec.normal); + float3 outwardN; + float3 rdir = r_in.dir; + float3 refl = reflect(rdir, rec.normal); + float nint; + attenuation = float3(1,1,1); + float3 refr; + float reflProb; + float cosine; + if (dot(rdir, rec.normal) > 0) + { + outwardN = -rec.normal; + nint = mat.ri; + cosine = mat.ri * dot(rdir, rec.normal); + } + else + { + outwardN = rec.normal; + nint = 1.0f / mat.ri; + cosine = -dot(rdir, rec.normal); + } + if (refract(rdir, outwardN, nint, refr)) + { + reflProb = schlick(cosine, mat.ri); + } + else + { + reflProb = 1; + } + if (RandomFloat01(state) < reflProb) + scattered = Ray(rec.pos, normalize(refl)); + else + scattered = Ray(rec.pos, normalize(refr)); + } + else + { + attenuation = float3(1,0,1); + return false; + } + return true; +} + +static float3 Trace(const Ray& r, int depth, int& inoutRayCount, uint32_t& state, bool doMaterialE = true) +{ + ZoneScoped; + Hit rec; + int id = 0; + ++inoutRayCount; + if (HitWorld(r, kMinT, kMaxT, rec, id)) + { + Ray scattered; + float3 attenuation; + float3 lightE; + const Material& mat = s_SphereMats[id]; + float3 matE = mat.emissive; + if (depth < kMaxDepth && Scatter(mat, r, rec, attenuation, scattered, lightE, inoutRayCount, state)) + { +#if DO_LIGHT_SAMPLING + if (!doMaterialE) matE = float3(0,0,0); // don't add material emission if told so + // dor Lambert materials, we just did explicit light (emissive) sampling and already + // for their contribution, so if next ray bounce hits the light again, don't add + // emission + doMaterialE = (mat.type != Material::Lambert); +#endif + return matE + lightE + attenuation * Trace(scattered, depth+1, inoutRayCount, state, doMaterialE); + } + else + { + return matE; + } + } + else + { + // sky +#if DO_MITSUBA_COMPARE + return float3(0.15f,0.21f,0.3f); // easier compare with Mitsuba's constant environment light +#else + float3 unitDir = r.dir; + float t = 0.5f*(unitDir.getY() + 1.0f); + return ((1.0f-t)*float3(1.0f, 1.0f, 1.0f) + t*float3(0.5f, 0.7f, 1.0f)) * 0.3f; +#endif + } +} + +#if CPU_CAN_DO_THREADS +static enkiTaskScheduler* g_TS; +#endif + +void InitializeTest() +{ + ZoneScoped; + #if CPU_CAN_DO_THREADS + g_TS = enkiNewTaskScheduler(); + enkiInitTaskSchedulerNumThreads(g_TS, std::max( 2, std::thread::hardware_concurrency() - 2)); + #endif +} + +void ShutdownTest() +{ + ZoneScoped; + #if CPU_CAN_DO_THREADS + enkiDeleteTaskScheduler(g_TS); + #endif +} + +struct JobData +{ + float time; + int frameCount; + int screenWidth, screenHeight; + float* backbuffer; + Camera* cam; + std::atomic rayCount; + unsigned testFlags; +}; + +static void TraceRowJob(uint32_t start, uint32_t end, uint32_t threadnum, void* data_) +{ + ZoneScoped; + JobData& data = *(JobData*)data_; + float* backbuffer = data.backbuffer + start * data.screenWidth * 4; + float invWidth = 1.0f / data.screenWidth; + float invHeight = 1.0f / data.screenHeight; + float lerpFac = float(data.frameCount) / float(data.frameCount+1); + if (data.testFlags & kFlagAnimate) + lerpFac *= DO_ANIMATE_SMOOTHING; + if (!(data.testFlags & kFlagProgressive)) + lerpFac = 0; + int rayCount = 0; + for (uint32_t y = start; y < end; ++y) + { + uint32_t state = (y * 9781 + data.frameCount * 6271) | 1; + for (int x = 0; x < data.screenWidth; ++x) + { + float3 col(0, 0, 0); + for (int s = 0; s < DO_SAMPLES_PER_PIXEL; s++) + { + float u = float(x + RandomFloat01(state)) * invWidth; + float v = float(y + RandomFloat01(state)) * invHeight; + Ray r = data.cam->GetRay(u, v, state); + col += Trace(r, 0, rayCount, state); + } + col *= 1.0f / float(DO_SAMPLES_PER_PIXEL); + + float3 prev(backbuffer[0], backbuffer[1], backbuffer[2]); + col = prev * lerpFac + col * (1-lerpFac); + col.store(backbuffer); + backbuffer += 4; + } + } + data.rayCount += rayCount; +} + +void UpdateTest(float time, int frameCount, int screenWidth, int screenHeight, unsigned testFlags) +{ + ZoneScoped; + if (testFlags & kFlagAnimate) + { + s_Spheres[1].center.setY(cosf(time) + 1.0f); + s_Spheres[8].center.setZ(sinf(time)*0.3f); + } + float3 lookfrom(0, 2, 3); + float3 lookat(0, 0, 0); + float distToFocus = 3; +#if DO_MITSUBA_COMPARE + float aperture = 0.0f; +#else + float aperture = 0.1f; +#endif +#if DO_BIG_SCENE + aperture *= 0.2f; +#endif + + s_EmissiveSphereCount = 0; + for (int i = 0; i < kSphereCount; ++i) + { + Sphere& s = s_Spheres[i]; + s.UpdateDerivedData(); + s_SpheresSoA.centerX[i] = s.center.getX(); + s_SpheresSoA.centerY[i] = s.center.getY(); + s_SpheresSoA.centerZ[i] = s.center.getZ(); + s_SpheresSoA.sqRadius[i] = s.radius * s.radius; + s_SpheresSoA.invRadius[i] = s.invRadius; + + // Remember IDs of emissive spheres (light sources) + const Material& smat = s_SphereMats[i]; + if (smat.emissive.getX() > 0 || smat.emissive.getY() > 0 || smat.emissive.getZ() > 0) + { + s_EmissiveSpheres[s_EmissiveSphereCount] = i; + s_EmissiveSphereCount++; + } + } + + s_Cam = Camera(lookfrom, lookat, float3(0, 1, 0), 60, float(screenWidth) / float(screenHeight), aperture, distToFocus); +} + +void DrawTest(float time, int frameCount, int screenWidth, int screenHeight, float* backbuffer, int& outRayCount, unsigned testFlags) +{ + ZoneScoped; + JobData args; + args.time = time; + args.frameCount = frameCount; + args.screenWidth = screenWidth; + args.screenHeight = screenHeight; + args.backbuffer = backbuffer; + args.cam = &s_Cam; + args.testFlags = testFlags; + args.rayCount = 0; + + #if CPU_CAN_DO_THREADS + enkiTaskSet* task = enkiCreateTaskSet(g_TS, TraceRowJob); + bool threaded = true; + enkiAddTaskSetToPipeMinRange(g_TS, task, &args, screenHeight, threaded ? 4 : screenHeight); + enkiWaitForTaskSet(g_TS, task); + enkiDeleteTaskSet(task); + #else + TraceRowJob(0, screenHeight, 0, &args); + #endif + + outRayCount = args.rayCount; +} + +void GetObjectCount(int& outCount, int& outObjectSize, int& outMaterialSize, int& outCamSize) +{ + ZoneScoped; + outCount = kSphereCount; + outObjectSize = sizeof(Sphere); + outMaterialSize = sizeof(Material); + outCamSize = sizeof(Camera); +} + +void GetSceneDesc(void* outObjects, void* outMaterials, void* outCam, void* outEmissives, int* outEmissiveCount) +{ + ZoneScoped; + memcpy(outObjects, s_Spheres, kSphereCount * sizeof(s_Spheres[0])); + memcpy(outMaterials, s_SphereMats, kSphereCount * sizeof(s_SphereMats[0])); + memcpy(outCam, &s_Cam, sizeof(s_Cam)); + memcpy(outEmissives, s_EmissiveSpheres, s_EmissiveSphereCount * sizeof(s_EmissiveSpheres[0])); + *outEmissiveCount = s_EmissiveSphereCount; +} diff --git a/third_party/tracy/examples/ToyPathTracer/Source/Test.h b/third_party/tracy/examples/ToyPathTracer/Source/Test.h new file mode 100644 index 0000000000000000000000000000000000000000..c085b5d5cc9f02f10bb18a8a11664fe4c9a18564 --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Source/Test.h @@ -0,0 +1,17 @@ +#pragma once +#include + +enum TestFlags +{ + kFlagAnimate = (1 << 0), + kFlagProgressive = (1 << 1), +}; + +void InitializeTest(); +void ShutdownTest(); + +void UpdateTest(float time, int frameCount, int screenWidth, int screenHeight, unsigned testFlags); +void DrawTest(float time, int frameCount, int screenWidth, int screenHeight, float* backbuffer, int& outRayCount, unsigned testFlags); + +void GetObjectCount(int& outCount, int& outObjectSize, int& outMaterialSize, int& outCamSize); +void GetSceneDesc(void* outObjects, void* outMaterials, void* outCam, void* outEmissives, int* outEmissiveCount); diff --git a/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/Atomics.h b/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/Atomics.h new file mode 100644 index 0000000000000000000000000000000000000000..878572ba8cd694cdd19411d65e128f7f2653bd13 --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/Atomics.h @@ -0,0 +1,79 @@ +// Copyright (c) 2013 Doug Binks +// +// This software is provided 'as-is', without any express or implied +// warranty. In no event will the authors be held liable for any damages +// arising from the use of this software. +// +// Permission is granted to anyone to use this software for any purpose, +// including commercial applications, and to alter it and redistribute it +// freely, subject to the following restrictions: +// +// 1. The origin of this software must not be misrepresented; you must not +// claim that you wrote the original software. If you use this software +// in a product, an acknowledgement in the product documentation would be +// appreciated but is not required. +// 2. Altered source versions must be plainly marked as such, and must not be +// misrepresented as being the original software. +// 3. This notice may not be removed or altered from any source distribution. + +#pragma once + +#include + +#ifdef _WIN32 + #define WIN32_LEAN_AND_MEAN + #include + #undef GetObject + #include + + extern "C" void _ReadWriteBarrier(); + #pragma intrinsic(_ReadWriteBarrier) + #pragma intrinsic(_InterlockedCompareExchange) + #pragma intrinsic(_InterlockedExchangeAdd) + + // Memory Barriers to prevent CPU and Compiler re-ordering + #define BASE_MEMORYBARRIER_ACQUIRE() _ReadWriteBarrier() + #define BASE_MEMORYBARRIER_RELEASE() _ReadWriteBarrier() + #define BASE_ALIGN(x) __declspec( align( x ) ) + +#else + #define BASE_MEMORYBARRIER_ACQUIRE() __asm__ __volatile__("": : :"memory") + #define BASE_MEMORYBARRIER_RELEASE() __asm__ __volatile__("": : :"memory") + #define BASE_ALIGN(x) __attribute__ ((aligned( x ))) +#endif + +namespace enki +{ + // Atomically performs: if( *pDest == compareWith ) { *pDest = swapTo; } + // returns old *pDest (so if successfull, returns compareWith) + inline uint32_t AtomicCompareAndSwap( volatile uint32_t* pDest, uint32_t swapTo, uint32_t compareWith ) + { + #ifdef _WIN32 + // assumes two's complement - unsigned / signed conversion leads to same bit pattern + return _InterlockedCompareExchange( (volatile long*)pDest,swapTo, compareWith ); + #else + return __sync_val_compare_and_swap( pDest, compareWith, swapTo ); + #endif + } + + inline uint64_t AtomicCompareAndSwap( volatile uint64_t* pDest, uint64_t swapTo, uint64_t compareWith ) + { + #ifdef _WIN32 + // assumes two's complement - unsigned / signed conversion leads to same bit pattern + return _InterlockedCompareExchange64( (__int64 volatile*)pDest, swapTo, compareWith ); + #else + return __sync_val_compare_and_swap( pDest, compareWith, swapTo ); + #endif + } + + // Atomically performs: tmp = *pDest; *pDest += value; return tmp; + inline int32_t AtomicAdd( volatile int32_t* pDest, int32_t value ) + { + #ifdef _WIN32 + return _InterlockedExchangeAdd( (long*)pDest, value ); + #else + return __sync_fetch_and_add( pDest, value ); + #endif + } + +} \ No newline at end of file diff --git a/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/LockLessMultiReadPipe.h b/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/LockLessMultiReadPipe.h new file mode 100644 index 0000000000000000000000000000000000000000..7439d09cedccdc94e0595deb2c69653117099260 --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/LockLessMultiReadPipe.h @@ -0,0 +1,240 @@ +// Copyright (c) 2013 Doug Binks +// +// This software is provided 'as-is', without any express or implied +// warranty. In no event will the authors be held liable for any damages +// arising from the use of this software. +// +// Permission is granted to anyone to use this software for any purpose, +// including commercial applications, and to alter it and redistribute it +// freely, subject to the following restrictions: +// +// 1. The origin of this software must not be misrepresented; you must not +// claim that you wrote the original software. If you use this software +// in a product, an acknowledgement in the product documentation would be +// appreciated but is not required. +// 2. Altered source versions must be plainly marked as such, and must not be +// misrepresented as being the original software. +// 3. This notice may not be removed or altered from any source distribution. + +#pragma once + +#include +#include + +#include "Atomics.h" +#include + + +namespace enki +{ + // LockLessMultiReadPipe - Single writer, multiple reader thread safe pipe using (semi) lockless programming + // Readers can only read from the back of the pipe + // The single writer can write to the front of the pipe, and read from both ends (a writer can be a reader) + // for many of the principles used here, see http://msdn.microsoft.com/en-us/library/windows/desktop/ee418650(v=vs.85).aspx + // Note: using log2 sizes so we do not need to clamp (multi-operation) + // T is the contained type + // Note this is not true lockless as the use of flags as a form of lock state. + template class LockLessMultiReadPipe + { + public: + LockLessMultiReadPipe(); + ~LockLessMultiReadPipe() {} + + // ReaderTryReadBack returns false if we were unable to read + // This is thread safe for both multiple readers and the writer + bool ReaderTryReadBack( T* pOut ); + + // WriterTryReadFront returns false if we were unable to read + // This is thread safe for the single writer, but should not be called by readers + bool WriterTryReadFront( T* pOut ); + + // WriterTryWriteFront returns false if we were unable to write + // This is thread safe for the single writer, but should not be called by readers + bool WriterTryWriteFront( const T& in ); + + // IsPipeEmpty() is a utility function, not intended for general use + // Should only be used very prudently. + bool IsPipeEmpty() const + { + return 0 == m_WriteIndex - m_ReadCount; + } + + void Clear() + { + m_WriteIndex = 0; + m_ReadIndex = 0; + m_ReadCount = 0; + memset( (void*)m_Flags, 0, sizeof( m_Flags ) ); + } + + private: + const static uint32_t ms_cSize = ( 1 << cSizeLog2 ); + const static uint32_t ms_cIndexMask = ms_cSize - 1; + const static uint32_t FLAG_INVALID = 0xFFFFFFFF; // 32bit for CAS + const static uint32_t FLAG_CAN_WRITE = 0x00000000; // 32bit for CAS + const static uint32_t FLAG_CAN_READ = 0x11111111; // 32bit for CAS + + T m_Buffer[ ms_cSize ]; + + // read and write indexes allow fast access to the pipe, but actual access + // controlled by the access flags. + volatile uint32_t BASE_ALIGN(4) m_WriteIndex; + volatile uint32_t BASE_ALIGN(4) m_ReadCount; + volatile uint32_t m_Flags[ ms_cSize ]; + volatile uint32_t BASE_ALIGN(4) m_ReadIndex; + }; + + template inline + LockLessMultiReadPipe::LockLessMultiReadPipe() + : m_WriteIndex(0) + , m_ReadIndex(0) + , m_ReadCount(0) + { + assert( cSizeLog2 < 32 ); + memset( (void*)m_Flags, 0, sizeof( m_Flags ) ); + } + + template inline + bool LockLessMultiReadPipe::ReaderTryReadBack( T* pOut ) + { + + uint32_t actualReadIndex; + + uint32_t readCount = m_ReadCount; + + // We get hold of read index for consistency, + // and do first pass starting at read count + uint32_t readIndexToUse = readCount; + + + while(true) + { + + uint32_t writeIndex = m_WriteIndex; + // power of two sizes ensures we can use a simple calc without modulus + uint32_t numInPipe = writeIndex - readCount; + if( 0 == numInPipe ) + { + return false; + } + if( readIndexToUse >= writeIndex ) + { + // move back to start + readIndexToUse = m_ReadIndex; + } + + + // power of two sizes ensures we can perform AND for a modulus + actualReadIndex = readIndexToUse & ms_cIndexMask; + + // Multiple potential readers mean we should check if the data is valid, + // using an atomic compare exchange + uint32_t previous = AtomicCompareAndSwap( &m_Flags[ actualReadIndex ], FLAG_INVALID, FLAG_CAN_READ ); + if( FLAG_CAN_READ == previous ) + { + break; + } + ++readIndexToUse; + + //update known readcount + readCount = m_ReadCount; + } + + // we update the read index using an atomic add, as we've only read one piece of data. + // this ensure consistency of the read index, and the above loop ensures readers + // only read from unread data + AtomicAdd( (volatile int32_t*)&m_ReadCount, 1 ); + + BASE_MEMORYBARRIER_ACQUIRE(); + // now read data, ensuring we do so after above reads & CAS + *pOut = m_Buffer[ actualReadIndex ]; + + m_Flags[ actualReadIndex ] = FLAG_CAN_WRITE; + + return true; + } + + template inline + bool LockLessMultiReadPipe::WriterTryReadFront( T* pOut ) + { + uint32_t writeIndex = m_WriteIndex; + uint32_t frontReadIndex = writeIndex; + + // Multiple potential readers mean we should check if the data is valid, + // using an atomic compare exchange - which acts as a form of lock (so not quite lockless really). + uint32_t previous = FLAG_INVALID; + uint32_t actualReadIndex = 0; + while( true ) + { + // power of two sizes ensures we can use a simple calc without modulus + uint32_t readCount = m_ReadCount; + uint32_t numInPipe = writeIndex - readCount; + if( 0 == numInPipe || 0 == frontReadIndex ) + { + // frontReadIndex can get to 0 here if that item was just being read by another thread. + m_ReadIndex = readCount; + return false; + } + --frontReadIndex; + actualReadIndex = frontReadIndex & ms_cIndexMask; + previous = AtomicCompareAndSwap( &m_Flags[ actualReadIndex ], FLAG_INVALID, FLAG_CAN_READ ); + if( FLAG_CAN_READ == previous ) + { + break; + } + else if( m_ReadIndex >= frontReadIndex ) + { + return false; + } + } + + // now read data, ensuring we do so after above reads & CAS + *pOut = m_Buffer[ actualReadIndex ]; + + m_Flags[ actualReadIndex ] = FLAG_CAN_WRITE; + + BASE_MEMORYBARRIER_RELEASE(); + + // 32-bit aligned stores are atomic, and writer owns the write index + // we only move one back as this is as many as we have read, not where we have read from. + --m_WriteIndex; + return true; + } + + + template inline + bool LockLessMultiReadPipe::WriterTryWriteFront( const T& in ) + { + // The writer 'owns' the write index, and readers can only reduce + // the amount of data in the pipe. + // We get hold of both values for consistency and to reduce false sharing + // impacting more than one access + uint32_t writeIndex = m_WriteIndex; + + + // power of two sizes ensures we can perform AND for a modulus + uint32_t actualWriteIndex = writeIndex & ms_cIndexMask; + + // a reader may still be reading this item, as there are multiple readers + if( m_Flags[ actualWriteIndex ] != FLAG_CAN_WRITE ) + { + return false; // still being read, so have caught up with tail. + } + + + // as we are the only writer we can update the data without atomics + // whilst the write index has not been updated + m_Buffer[ actualWriteIndex ] = in; + m_Flags[ actualWriteIndex ] = FLAG_CAN_READ; + + // We need to ensure the above writes occur prior to updating the write index, + // otherwise another thread might read before it's finished + BASE_MEMORYBARRIER_RELEASE(); + + // 32-bit aligned stores are atomic, and the writer controls the write index + ++writeIndex; + m_WriteIndex = writeIndex; + return true; + } + +} diff --git a/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/TaskScheduler.cpp b/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/TaskScheduler.cpp new file mode 100644 index 0000000000000000000000000000000000000000..187673adc76289e9d19a36d6d455fe30d746ffbb --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/TaskScheduler.cpp @@ -0,0 +1,437 @@ +// Copyright (c) 2013 Doug Binks +// +// This software is provided 'as-is', without any express or implied +// warranty. In no event will the authors be held liable for any damages +// arising from the use of this software. +// +// Permission is granted to anyone to use this software for any purpose, +// including commercial applications, and to alter it and redistribute it +// freely, subject to the following restrictions: +// +// 1. The origin of this software must not be misrepresented; you must not +// claim that you wrote the original software. If you use this software +// in a product, an acknowledgement in the product documentation would be +// appreciated but is not required. +// 2. Altered source versions must be plainly marked as such, and must not be +// misrepresented as being the original software. +// 3. This notice may not be removed or altered from any source distribution. + +#include + +#include "TaskScheduler.h" +#include "LockLessMultiReadPipe.h" + + + +using namespace enki; + + +static const uint32_t PIPESIZE_LOG2 = 8; +static const uint32_t SPIN_COUNT = 100; +static const uint32_t SPIN_BACKOFF_MULTIPLIER = 10; +static const uint32_t MAX_NUM_INITIAL_PARTITIONS = 8; + +// each software thread gets it's own copy of gtl_threadNum, so this is safe to use as a static variable +static THREAD_LOCAL uint32_t gtl_threadNum = 0; + +namespace enki +{ + struct SubTaskSet + { + ITaskSet* pTask; + TaskSetPartition partition; + }; + + // we derive class TaskPipe rather than typedef to get forward declaration working easily + class TaskPipe : public LockLessMultiReadPipe {}; + + struct ThreadArgs + { + uint32_t threadNum; + TaskScheduler* pTaskScheduler; + }; +} + +namespace +{ + SubTaskSet SplitTask( SubTaskSet& subTask_, uint32_t rangeToSplit_ ) + { + SubTaskSet splitTask = subTask_; + uint32_t rangeLeft = subTask_.partition.end - subTask_.partition.start; + + if( rangeToSplit_ > rangeLeft ) + { + rangeToSplit_ = rangeLeft; + } + splitTask.partition.end = subTask_.partition.start + rangeToSplit_; + subTask_.partition.start = splitTask.partition.end; + return splitTask; + } + + #if defined _WIN32 + #if defined _M_IX86 || defined _M_X64 + #pragma intrinsic(_mm_pause) + inline void Pause() { _mm_pause(); } + #endif + #elif defined __i386__ || defined __x86_64__ + inline void Pause() { __asm__ __volatile__("pause;"); } + #else + inline void Pause() { ;} // may have NOP or yield equiv + #endif +} + + +static void SafeCallback(ProfilerCallbackFunc func_, uint32_t threadnum_) +{ + if( func_ ) + { + func_(threadnum_); + } +} + +ProfilerCallbacks* TaskScheduler::GetProfilerCallbacks() +{ + return &m_ProfilerCallbacks; +} + +THREADFUNC_DECL TaskScheduler::TaskingThreadFunction( void* pArgs ) +{ + ThreadArgs args = *(ThreadArgs*)pArgs; + uint32_t threadNum = args.threadNum; + TaskScheduler* pTS = args.pTaskScheduler; + gtl_threadNum = threadNum; + + SafeCallback( pTS->m_ProfilerCallbacks.threadStart, threadNum ); + + uint32_t spinCount = 0; + uint32_t hintPipeToCheck_io = threadNum + 1; // does not need to be clamped. + while( pTS->m_bRunning ) + { + if(!pTS->TryRunTask( threadNum, hintPipeToCheck_io ) ) + { + // no tasks, will spin then wait + ++spinCount; + if( spinCount > SPIN_COUNT ) + { + pTS->WaitForTasks( threadNum ); + spinCount = 0; + } + else + { + uint32_t spinBackoffCount = spinCount * SPIN_BACKOFF_MULTIPLIER; + while( spinBackoffCount ) + { + Pause(); + --spinBackoffCount; + } + } + } + else + { + spinCount = 0; + } + } + + AtomicAdd( &pTS->m_NumThreadsRunning, -1 ); + SafeCallback( pTS->m_ProfilerCallbacks.threadStop, threadNum ); + + return 0; +} + + +void TaskScheduler::StartThreads() +{ + if( m_bHaveThreads ) + { + return; + } + m_bRunning = true; + + SemaphoreCreate( m_NewTaskSemaphore ); + + // we create one less thread than m_NumThreads as the main thread counts as one + m_pThreadNumStore = new ThreadArgs[m_NumThreads]; + m_pThreadIDs = new threadid_t[m_NumThreads]; + m_pThreadNumStore[0].threadNum = 0; + m_pThreadNumStore[0].pTaskScheduler = this; + m_pThreadIDs[0] = 0; + m_NumThreadsWaiting = 0; + m_NumThreadsRunning = 1;// acount for main thread + for( uint32_t thread = 1; thread < m_NumThreads; ++thread ) + { + m_pThreadNumStore[thread].threadNum = thread; + m_pThreadNumStore[thread].pTaskScheduler = this; + ThreadCreate( &m_pThreadIDs[thread], TaskingThreadFunction, &m_pThreadNumStore[thread] ); + ++m_NumThreadsRunning; + } + + // ensure we have sufficient tasks to equally fill either all threads including main + // or just the threads we've launched, this is outside the firstinit as we want to be able + // to runtime change it + if( 1 == m_NumThreads ) + { + m_NumPartitions = 1; + m_NumInitialPartitions = 1; + } + else + { + m_NumPartitions = m_NumThreads * (m_NumThreads - 1); + m_NumInitialPartitions = m_NumThreads - 1; + if( m_NumInitialPartitions > MAX_NUM_INITIAL_PARTITIONS ) + { + m_NumInitialPartitions = MAX_NUM_INITIAL_PARTITIONS; + } + } + + m_bHaveThreads = true; +} + +void TaskScheduler::StopThreads( bool bWait_ ) +{ + if( m_bHaveThreads ) + { + // wait for them threads quit before deleting data + m_bRunning = false; + while( bWait_ && m_NumThreadsRunning > 1 ) + { + // keep firing event to ensure all threads pick up state of m_bRunning + SemaphoreSignal( m_NewTaskSemaphore, m_NumThreadsRunning ); + } + + for( uint32_t thread = 1; thread < m_NumThreads; ++thread ) + { + ThreadTerminate( m_pThreadIDs[thread] ); + } + + m_NumThreads = 0; + delete[] m_pThreadNumStore; + delete[] m_pThreadIDs; + m_pThreadNumStore = 0; + m_pThreadIDs = 0; + SemaphoreClose( m_NewTaskSemaphore ); + + m_bHaveThreads = false; + m_NumThreadsWaiting = 0; + m_NumThreadsRunning = 0; + } +} + +bool TaskScheduler::TryRunTask( uint32_t threadNum, uint32_t& hintPipeToCheck_io_ ) +{ + // check for tasks + SubTaskSet subTask; + bool bHaveTask = m_pPipesPerThread[ threadNum ].WriterTryReadFront( &subTask ); + + uint32_t threadToCheck = hintPipeToCheck_io_; + uint32_t checkCount = 0; + while( !bHaveTask && checkCount < m_NumThreads ) + { + threadToCheck = ( hintPipeToCheck_io_ + checkCount ) % m_NumThreads; + if( threadToCheck != threadNum ) + { + bHaveTask = m_pPipesPerThread[ threadToCheck ].ReaderTryReadBack( &subTask ); + } + ++checkCount; + } + + if( bHaveTask ) + { + // update hint, will preserve value unless actually got task from another thread. + hintPipeToCheck_io_ = threadToCheck; + + uint32_t partitionSize = subTask.partition.end - subTask.partition.start; + if( subTask.pTask->m_RangeToRun < partitionSize ) + { + SubTaskSet taskToRun = SplitTask( subTask, subTask.pTask->m_RangeToRun ); + SplitAndAddTask( gtl_threadNum, subTask, subTask.pTask->m_RangeToRun, 0 ); + taskToRun.pTask->ExecuteRange( taskToRun.partition, threadNum ); + AtomicAdd( &taskToRun.pTask->m_RunningCount, -1 ); + } + else + { + + // the task has already been divided up by AddTaskSetToPipe, so just run it + subTask.pTask->ExecuteRange( subTask.partition, threadNum ); + AtomicAdd( &subTask.pTask->m_RunningCount, -1 ); + } + } + + return bHaveTask; + +} + +void TaskScheduler::WaitForTasks( uint32_t threadNum ) +{ + // We incrememt the number of threads waiting here in order + // to ensure that the check for tasks occurs after the increment + // to prevent a task being added after a check, then the thread waiting. + // This will occasionally result in threads being mistakenly awoken, + // but they will then go back to sleep. + AtomicAdd( &m_NumThreadsWaiting, 1 ); + + bool bHaveTasks = false; + for( uint32_t thread = 0; thread < m_NumThreads; ++thread ) + { + if( !m_pPipesPerThread[ thread ].IsPipeEmpty() ) + { + bHaveTasks = true; + break; + } + } + if( !bHaveTasks ) + { + SafeCallback( m_ProfilerCallbacks.waitStart, threadNum ); + SemaphoreWait( m_NewTaskSemaphore ); + SafeCallback( m_ProfilerCallbacks.waitStop, threadNum ); + } + + int32_t prev = AtomicAdd( &m_NumThreadsWaiting, -1 ); + assert( prev != 0 ); +} + +void TaskScheduler::WakeThreads() +{ + SemaphoreSignal( m_NewTaskSemaphore, m_NumThreadsWaiting ); +} + +void TaskScheduler::SplitAndAddTask( uint32_t threadNum_, SubTaskSet subTask_, + uint32_t rangeToSplit_, int32_t runningCountOffset_ ) +{ + int32_t numAdded = 0; + while( subTask_.partition.start != subTask_.partition.end ) + { + SubTaskSet taskToAdd = SplitTask( subTask_, rangeToSplit_ ); + + // add the partition to the pipe + ++numAdded; + if( !m_pPipesPerThread[ gtl_threadNum ].WriterTryWriteFront( taskToAdd ) ) + { + if( numAdded > 1 ) + { + WakeThreads(); + } + // alter range to run the appropriate fraction + if( taskToAdd.pTask->m_RangeToRun < rangeToSplit_ ) + { + taskToAdd.partition.end = taskToAdd.partition.start + taskToAdd.pTask->m_RangeToRun; + subTask_.partition.start = taskToAdd.partition.end; + } + taskToAdd.pTask->ExecuteRange( taskToAdd.partition, threadNum_ ); + --numAdded; + } + } + + // increment running count by number added + AtomicAdd( &subTask_.pTask->m_RunningCount, numAdded + runningCountOffset_ ); + + WakeThreads(); +} + +void TaskScheduler::AddTaskSetToPipe( ITaskSet* pTaskSet ) +{ + // set running count to -1 to guarantee it won't be found complete until all subtasks added + pTaskSet->m_RunningCount = -1; + + // divide task up and add to pipe + pTaskSet->m_RangeToRun = pTaskSet->m_SetSize / m_NumPartitions; + if( pTaskSet->m_RangeToRun < pTaskSet->m_MinRange ) { pTaskSet->m_RangeToRun = pTaskSet->m_MinRange; } + + uint32_t rangeToSplit = pTaskSet->m_SetSize / m_NumInitialPartitions; + if( rangeToSplit < pTaskSet->m_MinRange ) { rangeToSplit = pTaskSet->m_MinRange; } + + SubTaskSet subTask; + subTask.pTask = pTaskSet; + subTask.partition.start = 0; + subTask.partition.end = pTaskSet->m_SetSize; + SplitAndAddTask( gtl_threadNum, subTask, rangeToSplit, 1 ); +} + +void TaskScheduler::WaitforTaskSet( const ITaskSet* pTaskSet ) +{ + uint32_t hintPipeToCheck_io = gtl_threadNum + 1; // does not need to be clamped. + if( pTaskSet ) + { + while( pTaskSet->m_RunningCount ) + { + TryRunTask( gtl_threadNum, hintPipeToCheck_io ); + // should add a spin then wait for task completion event. + } + } + else + { + TryRunTask( gtl_threadNum, hintPipeToCheck_io ); + } +} + +void TaskScheduler::WaitforAll() +{ + bool bHaveTasks = true; + uint32_t hintPipeToCheck_io = gtl_threadNum + 1; // does not need to be clamped. + int32_t threadsRunning = m_NumThreadsRunning - 1; + while( bHaveTasks || m_NumThreadsWaiting < threadsRunning ) + { + TryRunTask( gtl_threadNum, hintPipeToCheck_io ); + bHaveTasks = false; + for( uint32_t thread = 0; thread < m_NumThreads; ++thread ) + { + if( !m_pPipesPerThread[ thread ].IsPipeEmpty() ) + { + bHaveTasks = true; + break; + } + } + } +} + +void TaskScheduler::WaitforAllAndShutdown() +{ + WaitforAll(); + StopThreads(true); + delete[] m_pPipesPerThread; + m_pPipesPerThread = 0; +} + +uint32_t TaskScheduler::GetNumTaskThreads() const +{ + return m_NumThreads; +} + +TaskScheduler::TaskScheduler() + : m_pPipesPerThread(NULL) + , m_NumThreads(0) + , m_pThreadNumStore(NULL) + , m_pThreadIDs(NULL) + , m_bRunning(false) + , m_NumThreadsRunning(0) + , m_NumThreadsWaiting(0) + , m_NumPartitions(0) + , m_bHaveThreads(false) +{ + memset(&m_ProfilerCallbacks, 0, sizeof(m_ProfilerCallbacks)); +} + +TaskScheduler::~TaskScheduler() +{ + StopThreads( true ); // Stops threads, waiting for them. + + delete[] m_pPipesPerThread; + m_pPipesPerThread = 0; +} + +void TaskScheduler::Initialize( uint32_t numThreads_ ) +{ + assert( numThreads_ ); + StopThreads( true ); // Stops threads, waiting for them. + delete[] m_pPipesPerThread; + + m_NumThreads = numThreads_; + + m_pPipesPerThread = new TaskPipe[ m_NumThreads ]; + + StartThreads(); +} + +void TaskScheduler::Initialize() +{ + Initialize( GetNumHardwareThreads() ); +} \ No newline at end of file diff --git a/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/TaskScheduler.h b/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/TaskScheduler.h new file mode 100644 index 0000000000000000000000000000000000000000..74f9376ccc2852c6f8899cd018526af9a403636b --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/TaskScheduler.h @@ -0,0 +1,177 @@ +// Copyright (c) 2013 Doug Binks +// +// This software is provided 'as-is', without any express or implied +// warranty. In no event will the authors be held liable for any damages +// arising from the use of this software. +// +// Permission is granted to anyone to use this software for any purpose, +// including commercial applications, and to alter it and redistribute it +// freely, subject to the following restrictions: +// +// 1. The origin of this software must not be misrepresented; you must not +// claim that you wrote the original software. If you use this software +// in a product, an acknowledgement in the product documentation would be +// appreciated but is not required. +// 2. Altered source versions must be plainly marked as such, and must not be +// misrepresented as being the original software. +// 3. This notice may not be removed or altered from any source distribution. + +#pragma once + +#include +#include "Threads.h" + +namespace enki +{ + + struct TaskSetPartition + { + uint32_t start; + uint32_t end; + }; + + class TaskScheduler; + class TaskPipe; + struct ThreadArgs; + struct SubTaskSet; + + // Subclass ITaskSet to create tasks. + // TaskSets can be re-used, but check + class ITaskSet + { + public: + ITaskSet() + : m_SetSize(1) + , m_MinRange(1) + , m_RunningCount(0) + , m_RangeToRun(1) + {} + + ITaskSet( uint32_t setSize_ ) + : m_SetSize( setSize_ ) + , m_MinRange(1) + , m_RunningCount(0) + , m_RangeToRun(1) + {} + + ITaskSet( uint32_t setSize_, uint32_t minRange_ ) + : m_SetSize( setSize_ ) + , m_MinRange( minRange_ ) + , m_RunningCount(0) + , m_RangeToRun(minRange_) + {} + + // Execute range should be overloaded to process tasks. It will be called with a + // range_ where range.start >= 0; range.start < range.end; and range.end < m_SetSize; + // The range values should be mapped so that linearly processing them in order is cache friendly + // i.e. neighbouring values should be close together. + // threadnum should not be used for changing processing of data, it's intended purpose + // is to allow per-thread data buckets for output. + virtual void ExecuteRange( TaskSetPartition range, uint32_t threadnum ) = 0; + + // Size of set - usually the number of data items to be processed, see ExecuteRange. Defaults to 1 + uint32_t m_SetSize; + + // Minimum size of of TaskSetPartition range when splitting a task set into partitions. + // This should be set to a value which results in computation effort of at least 10k + // clock cycles to minimize tast scheduler overhead. + // NOTE: The last partition will be smaller than m_MinRange if m_SetSize is not a multiple + // of m_MinRange. + // Also known as grain size in literature. + uint32_t m_MinRange; + + bool GetIsComplete() + { + return 0 == m_RunningCount; + } + private: + friend class TaskScheduler; + volatile int32_t m_RunningCount; + uint32_t m_RangeToRun; + }; + + // TaskScheduler implements several callbacks intended for profilers + typedef void (*ProfilerCallbackFunc)( uint32_t threadnum_ ); + struct ProfilerCallbacks + { + ProfilerCallbackFunc threadStart; + ProfilerCallbackFunc threadStop; + ProfilerCallbackFunc waitStart; + ProfilerCallbackFunc waitStop; + }; + + class TaskScheduler + { + public: + TaskScheduler(); + ~TaskScheduler(); + + // Call either Initialize() or Initialize( numThreads_ ) before adding tasks. + + // Initialize() will create GetNumHardwareThreads()-1 threads, which is + // sufficient to fill the system when including the main thread. + // Initialize can be called multiple times - it will wait for completion + // before re-initializing. + void Initialize(); + + // Initialize( numThreads_ ) - numThreads_ (must be > 0) + // will create numThreads_-1 threads, as thread 0 is + // the thread on which the initialize was called. + void Initialize( uint32_t numThreads_ ); + + + // Adds the TaskSet to pipe and returns if the pipe is not full. + // If the pipe is full, pTaskSet is run. + // should only be called from main thread, or within a task + void AddTaskSetToPipe( ITaskSet* pTaskSet ); + + // Runs the TaskSets in pipe until true == pTaskSet->GetIsComplete(); + // should only be called from thread which created the taskscheduler , or within a task + // if called with 0 it will try to run tasks, and return if none available. + void WaitforTaskSet( const ITaskSet* pTaskSet ); + + // Waits for all task sets to complete - not guaranteed to work unless we know we + // are in a situation where tasks aren't being continuosly added. + void WaitforAll(); + + // Waits for all task sets to complete and shutdown threads - not guaranteed to work unless we know we + // are in a situation where tasks aren't being continuosly added. + void WaitforAllAndShutdown(); + + // Returns the number of threads created for running tasks + 1 + // to account for the main thread. + uint32_t GetNumTaskThreads() const; + + // Returns the ProfilerCallbacks structure so that it can be modified to + // set the callbacks. + ProfilerCallbacks* GetProfilerCallbacks(); + + private: + static THREADFUNC_DECL TaskingThreadFunction( void* pArgs ); + void WaitForTasks( uint32_t threadNum ); + bool TryRunTask( uint32_t threadNum, uint32_t& hintPipeToCheck_io_ ); + void StartThreads(); + void StopThreads( bool bWait_ ); + void SplitAndAddTask( uint32_t threadNum_, SubTaskSet subTask_, + uint32_t rangeToSplit_, int32_t runningCountOffset_ ); + void WakeThreads(); + + TaskPipe* m_pPipesPerThread; + + uint32_t m_NumThreads; + ThreadArgs* m_pThreadNumStore; + threadid_t* m_pThreadIDs; + volatile bool m_bRunning; + volatile int32_t m_NumThreadsRunning; + volatile int32_t m_NumThreadsWaiting; + uint32_t m_NumPartitions; + uint32_t m_NumInitialPartitions; + semaphoreid_t m_NewTaskSemaphore; + bool m_bHaveThreads; + ProfilerCallbacks m_ProfilerCallbacks; + + TaskScheduler( const TaskScheduler& nocopy ); + TaskScheduler& operator=( const TaskScheduler& nocopy ); + }; + +} \ No newline at end of file diff --git a/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/TaskScheduler_c.cpp b/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/TaskScheduler_c.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a7d25b49f07c10c3e35ffa5c81726d88dc780c30 --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/TaskScheduler_c.cpp @@ -0,0 +1,122 @@ +// Copyright (c) 2013 Doug Binks +// +// This software is provided 'as-is', without any express or implied +// warranty. In no event will the authors be held liable for any damages +// arising from the use of this software. +// +// Permission is granted to anyone to use this software for any purpose, +// including commercial applications, and to alter it and redistribute it +// freely, subject to the following restrictions: +// +// 1. The origin of this software must not be misrepresented; you must not +// claim that you wrote the original software. If you use this software +// in a product, an acknowledgement in the product documentation would be +// appreciated but is not required. +// 2. Altered source versions must be plainly marked as such, and must not be +// misrepresented as being the original software. +// 3. This notice may not be removed or altered from any source distribution. + +#include "TaskScheduler_c.h" +#include "TaskScheduler.h" + +#include + +using namespace enki; + +struct enkiTaskScheduler : TaskScheduler +{ +}; + +struct enkiTaskSet : ITaskSet +{ + enkiTaskSet( enkiTaskExecuteRange taskFun_ ) : taskFun(taskFun_), pArgs(NULL) {} + + virtual void ExecuteRange( TaskSetPartition range, uint32_t threadnum ) + { + taskFun( range.start, range.end, threadnum, pArgs ); + } + + enkiTaskExecuteRange taskFun; + void* pArgs; +}; + +enkiTaskScheduler* enkiNewTaskScheduler() +{ + enkiTaskScheduler* pETS = new enkiTaskScheduler(); + return pETS; +} + +void enkiInitTaskScheduler( enkiTaskScheduler* pETS_ ) +{ + pETS_->Initialize(); +} + +void enkiInitTaskSchedulerNumThreads( enkiTaskScheduler* pETS_, uint32_t numThreads_ ) +{ + pETS_->Initialize( numThreads_ ); +} + +void enkiDeleteTaskScheduler( enkiTaskScheduler* pETS_ ) +{ + delete pETS_; +} + +enkiTaskSet* enkiCreateTaskSet( enkiTaskScheduler* pETS_, enkiTaskExecuteRange taskFunc_ ) +{ + return new enkiTaskSet( taskFunc_ ); +} + +void enkiDeleteTaskSet( enkiTaskSet* pTaskSet_ ) +{ + delete pTaskSet_; +} + +void enkiAddTaskSetToPipe( enkiTaskScheduler* pETS_, enkiTaskSet* pTaskSet_, void* pArgs_, uint32_t setSize_ ) +{ + assert( pTaskSet_ ); + assert( pTaskSet_->taskFun ); + + pTaskSet_->m_SetSize = setSize_; + pTaskSet_->pArgs = pArgs_; + pETS_->AddTaskSetToPipe( pTaskSet_ ); +} + +void enkiAddTaskSetToPipeMinRange(enkiTaskScheduler * pETS_, enkiTaskSet * pTaskSet_, void * pArgs_, uint32_t setSize_, uint32_t minRange_) +{ + assert( pTaskSet_ ); + assert( pTaskSet_->taskFun ); + + pTaskSet_->m_SetSize = setSize_; + pTaskSet_->m_MinRange = minRange_; + pTaskSet_->pArgs = pArgs_; + pETS_->AddTaskSetToPipe( pTaskSet_ ); +} + +int enkiIsTaskSetComplete( enkiTaskScheduler* pETS_, enkiTaskSet* pTaskSet_ ) +{ + assert( pTaskSet_ ); + return ( pTaskSet_->GetIsComplete() ) ? 1 : 0; +} + +void enkiWaitForTaskSet( enkiTaskScheduler* pETS_, enkiTaskSet* pTaskSet_ ) +{ + pETS_->WaitforTaskSet( pTaskSet_ ); +} + +void enkiWaitForAll( enkiTaskScheduler* pETS_ ) +{ + pETS_->WaitforAll(); +} + + +uint32_t enkiGetNumTaskThreads( enkiTaskScheduler* pETS_ ) +{ + return pETS_->GetNumTaskThreads(); +} + +enkiProfilerCallbacks* enkiGetProfilerCallbacks( enkiTaskScheduler* pETS_ ) +{ + assert( sizeof(enkiProfilerCallbacks) == sizeof(enki::ProfilerCallbacks) ); + return (enkiProfilerCallbacks*)pETS_->GetProfilerCallbacks(); +} + diff --git a/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/TaskScheduler_c.h b/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/TaskScheduler_c.h new file mode 100644 index 0000000000000000000000000000000000000000..912771cff4950a0cc2048667d610fced0141406a --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/TaskScheduler_c.h @@ -0,0 +1,104 @@ +// Copyright (c) 2013 Doug Binks +// +// This software is provided 'as-is', without any express or implied +// warranty. In no event will the authors be held liable for any damages +// arising from the use of this software. +// +// Permission is granted to anyone to use this software for any purpose, +// including commercial applications, and to alter it and redistribute it +// freely, subject to the following restrictions: +// +// 1. The origin of this software must not be misrepresented; you must not +// claim that you wrote the original software. If you use this software +// in a product, an acknowledgement in the product documentation would be +// appreciated but is not required. +// 2. Altered source versions must be plainly marked as such, and must not be +// misrepresented as being the original software. +// 3. This notice may not be removed or altered from any source distribution. + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +typedef struct enkiTaskScheduler enkiTaskScheduler; +typedef struct enkiTaskSet enkiTaskSet; + +typedef void (* enkiTaskExecuteRange)( uint32_t start_, uint32_t end, uint32_t threadnum_, void* pArgs_ ); + + +// Create a new task scheduler +enkiTaskScheduler* enkiNewTaskScheduler(); + +// Initialize task scheduler - will create GetNumHardwareThreads()-1 threads, which is +// sufficient to fill the system when including the main thread. +// Initialize can be called multiple times - it will wait for completion +// before re-initializing. +void enkiInitTaskScheduler( enkiTaskScheduler* pETS_ ); + +// Initialize a task scheduler with numThreads_ (must be > 0) +// will create numThreads_-1 threads, as thread 0 is +// the thread on which the initialize was called. +void enkiInitTaskSchedulerNumThreads( enkiTaskScheduler* pETS_, uint32_t numThreads_ ); + + +// Delete a task scheduler +void enkiDeleteTaskScheduler( enkiTaskScheduler* pETS_ ); + +// Create a task set. +enkiTaskSet* enkiCreateTaskSet( enkiTaskScheduler* pETS_, enkiTaskExecuteRange taskFunc_ ); + +// Delete a task set. +void enkiDeleteTaskSet( enkiTaskSet* pTaskSet_ ); + +// Schedule the task +void enkiAddTaskSetToPipe( enkiTaskScheduler* pETS_, enkiTaskSet* pTaskSet_, + void* pArgs_, uint32_t setSize_ ); + +// Schedule the task with a minimum range. +// This should be set to a value which results in computation effort of at least 10k +// clock cycles to minimize tast scheduler overhead. +// NOTE: The last partition will be smaller than m_MinRange if m_SetSize is not a multiple +// of m_MinRange. +// Also known as grain size in literature. +void enkiAddTaskSetToPipeMinRange( enkiTaskScheduler* pETS_, enkiTaskSet* pTaskSet_, + void* pArgs_, uint32_t setSize_, uint32_t minRange_ ); + + +// Check if TaskSet is complete. Doesn't wait. Returns 1 if complete, 0 if not. +int enkiIsTaskSetComplete( enkiTaskScheduler* pETS_, enkiTaskSet* pTaskSet_ ); + + +// Wait for a given task. +// should only be called from thread which created the taskscheduler , or within a task +// if called with 0 it will try to run tasks, and return if none available. +void enkiWaitForTaskSet( enkiTaskScheduler* pETS_, enkiTaskSet* pTaskSet_ ); + + +// Waits for all task sets to complete - not guaranteed to work unless we know we +// are in a situation where tasks aren't being continuosly added. +void enkiWaitForAll( enkiTaskScheduler* pETS_ ); + + +// get number of threads +uint32_t enkiGetNumTaskThreads( enkiTaskScheduler* pETS_ ); + +// TaskScheduler implements several callbacks intended for profilers +typedef void (*enkiProfilerCallbackFunc)( uint32_t threadnum_ ); +struct enkiProfilerCallbacks +{ + enkiProfilerCallbackFunc threadStart; + enkiProfilerCallbackFunc threadStop; + enkiProfilerCallbackFunc waitStart; + enkiProfilerCallbackFunc waitStop; +}; + +// Get the callback structure so it can be set +struct enkiProfilerCallbacks* enkiGetProfilerCallbacks( enkiTaskScheduler* pETS_ ); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/Threads.h b/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/Threads.h new file mode 100644 index 0000000000000000000000000000000000000000..575a22447775f061db4f3e3f3aefc1cede55fce7 --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Source/enkiTS/Threads.h @@ -0,0 +1,210 @@ +// Copyright (c) 2013 Doug Binks +// +// This software is provided 'as-is', without any express or implied +// warranty. In no event will the authors be held liable for any damages +// arising from the use of this software. +// +// Permission is granted to anyone to use this software for any purpose, +// including commercial applications, and to alter it and redistribute it +// freely, subject to the following restrictions: +// +// 1. The origin of this software must not be misrepresented; you must not +// claim that you wrote the original software. If you use this software +// in a product, an acknowledgement in the product documentation would be +// appreciated but is not required. +// 2. Altered source versions must be plainly marked as such, and must not be +// misrepresented as being the original software. +// 3. This notice may not be removed or altered from any source distribution. + +#pragma once + +#include +#include + +#ifdef _WIN32 + + #include "Atomics.h" + + #define WIN32_LEAN_AND_MEAN + #include + + #define THREADFUNC_DECL DWORD WINAPI + #define THREAD_LOCAL __declspec( thread ) + +namespace enki +{ + typedef HANDLE threadid_t; + + // declare the thread start function as: + // THREADFUNC_DECL MyThreadStart( void* pArg ); + inline bool ThreadCreate( threadid_t* returnid, DWORD ( WINAPI *StartFunc) (void* ), void* pArg ) + { + // posix equiv pthread_create + DWORD threadid; + *returnid = CreateThread( 0, 0, StartFunc, pArg, 0, &threadid ); + return *returnid != NULL; + } + + inline bool ThreadTerminate( threadid_t threadid ) + { + // posix equiv pthread_cancel + return CloseHandle( threadid ) == 0; + } + + inline uint32_t GetNumHardwareThreads() + { + SYSTEM_INFO sysInfo; + GetSystemInfo(&sysInfo); + return sysInfo.dwNumberOfProcessors; + } +} + +#else // posix + + #include + #include + #define THREADFUNC_DECL void* + #define THREAD_LOCAL __thread + +namespace enki +{ + typedef pthread_t threadid_t; + + // declare the thread start function as: + // THREADFUNC_DECL MyThreadStart( void* pArg ); + inline bool ThreadCreate( threadid_t* returnid, void* ( *StartFunc) (void* ), void* pArg ) + { + // posix equiv pthread_create + int32_t retval = pthread_create( returnid, NULL, StartFunc, pArg ); + + return retval == 0; + } + + inline bool ThreadTerminate( threadid_t threadid ) + { + // posix equiv pthread_cancel + return pthread_cancel( threadid ) == 0; + } + + inline uint32_t GetNumHardwareThreads() + { + return (uint32_t)sysconf( _SC_NPROCESSORS_ONLN ); + } +} + +#endif // posix + + +// Semaphore implementation +#ifdef _WIN32 + +namespace enki +{ + struct semaphoreid_t + { + HANDLE sem; + }; + + inline void SemaphoreCreate( semaphoreid_t& semaphoreid ) + { + semaphoreid.sem = CreateSemaphore(NULL, 0, MAXLONG, NULL ); + } + + inline void SemaphoreClose( semaphoreid_t& semaphoreid ) + { + CloseHandle( semaphoreid.sem ); + } + + inline void SemaphoreWait( semaphoreid_t& semaphoreid ) + { + DWORD retval = WaitForSingleObject( semaphoreid.sem, INFINITE ); + + assert( retval != WAIT_FAILED ); + } + + inline void SemaphoreSignal( semaphoreid_t& semaphoreid, int32_t countWaiting ) + { + if( countWaiting ) + { + ReleaseSemaphore( semaphoreid.sem, countWaiting, NULL ); + } + } +} +#elif defined(__MACH__) + +// OS X does not have POSIX semaphores +// see https://developer.apple.com/library/content/documentation/Darwin/Conceptual/KernelProgramming/synchronization/synchronization.html +#include + +namespace enki +{ + + struct semaphoreid_t + { + semaphore_t sem; + }; + + inline void SemaphoreCreate( semaphoreid_t& semaphoreid ) + { + semaphore_create( mach_task_self(), &semaphoreid.sem, SYNC_POLICY_FIFO, 0 ); + } + + inline void SemaphoreClose( semaphoreid_t& semaphoreid ) + { + semaphore_destroy( mach_task_self(), semaphoreid.sem ); + } + + inline void SemaphoreWait( semaphoreid_t& semaphoreid ) + { + semaphore_wait( semaphoreid.sem ); + } + + inline void SemaphoreSignal( semaphoreid_t& semaphoreid, int32_t countWaiting ) + { + while( countWaiting-- > 0 ) + { + semaphore_signal( semaphoreid.sem ); + } + } +} + +#else // POSIX + +#include + +namespace enki +{ + + struct semaphoreid_t + { + sem_t sem; + }; + + inline void SemaphoreCreate( semaphoreid_t& semaphoreid ) + { + int err = sem_init( &semaphoreid.sem, 0, 0 ); + assert( err == 0 ); + } + + inline void SemaphoreClose( semaphoreid_t& semaphoreid ) + { + sem_destroy( &semaphoreid.sem ); + } + + inline void SemaphoreWait( semaphoreid_t& semaphoreid ) + { + int err = sem_wait( &semaphoreid.sem ); + assert( err == 0 ); + } + + inline void SemaphoreSignal( semaphoreid_t& semaphoreid, int32_t countWaiting ) + { + while( countWaiting-- > 0 ) + { + sem_post( &semaphoreid.sem ); + } + } +} +#endif + + diff --git a/third_party/tracy/examples/ToyPathTracer/Windows/ComputeShader.hlsl b/third_party/tracy/examples/ToyPathTracer/Windows/ComputeShader.hlsl new file mode 100644 index 0000000000000000000000000000000000000000..ed6ef03fe85a2265bf34c0410a828c9831a52416 --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Windows/ComputeShader.hlsl @@ -0,0 +1,395 @@ +#include "../Source/Config.h" + +inline uint RNG(inout uint state) +{ + uint x = state; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 15; + state = x; + return x; +} + +float RandomFloat01(inout uint state) +{ + return (RNG(state) & 0xFFFFFF) / 16777216.0f; +} + +float3 RandomInUnitDisk(inout uint state) +{ + float a = RandomFloat01(state) * 2.0f * 3.1415926f; + float2 xy = float2(cos(a), sin(a)); + xy *= sqrt(RandomFloat01(state)); + return float3(xy, 0); +} +float3 RandomInUnitSphere(inout uint state) +{ + float z = RandomFloat01(state) * 2.0f - 1.0f; + float t = RandomFloat01(state) * 2.0f * 3.1415926f; + float r = sqrt(max(0.0, 1.0f - z * z)); + float x = r * cos(t); + float y = r * sin(t); + float3 res = float3(x, y, z); + res *= pow(RandomFloat01(state), 1.0 / 3.0); + return res; +} +float3 RandomUnitVector(inout uint state) +{ + float z = RandomFloat01(state) * 2.0f - 1.0f; + float a = RandomFloat01(state) * 2.0f * 3.1415926f; + float r = sqrt(1.0f - z * z); + float x = r * cos(a); + float y = r * sin(a); + return float3(x, y, z); +} + + + +struct Ray +{ + float3 orig; + float3 dir; +}; +Ray MakeRay(float3 orig_, float3 dir_) { Ray r; r.orig = orig_; r.dir = dir_; return r; } +float3 RayPointAt(Ray r, float t) { return r.orig + r.dir * t; } + + +inline bool refract(float3 v, float3 n, float nint, out float3 outRefracted) +{ + float dt = dot(v, n); + float discr = 1.0f - nint * nint*(1 - dt * dt); + if (discr > 0) + { + outRefracted = nint * (v - n * dt) - n * sqrt(discr); + return true; + } + return false; +} +inline float schlick(float cosine, float ri) +{ + float r0 = (1 - ri) / (1 + ri); + r0 = r0 * r0; + // note: saturate to guard against possible tiny negative numbers + return r0 + (1 - r0)*pow(saturate(1 - cosine), 5); +} + +struct Hit +{ + float3 pos; + float3 normal; + float t; +}; + +struct Sphere +{ + float3 center; + float radius; + float invRadius; +}; + +#define MatLambert 0 +#define MatMetal 1 +#define MatDielectric 2 + +struct Material +{ + int type; + float3 albedo; + float3 emissive; + float roughness; + float ri; +}; + +groupshared Sphere s_GroupSpheres[kCSMaxObjects]; +groupshared Material s_GroupMaterials[kCSMaxObjects]; +groupshared int s_GroupEmissives[kCSMaxObjects]; + + +struct Camera +{ + float3 origin; + float3 lowerLeftCorner; + float3 horizontal; + float3 vertical; + float3 u, v, w; + float lensRadius; +}; + +Ray CameraGetRay(Camera cam, float s, float t, inout uint state) +{ + float3 rd = cam.lensRadius * RandomInUnitDisk(state); + float3 offset = cam.u * rd.x + cam.v * rd.y; + return MakeRay(cam.origin + offset, normalize(cam.lowerLeftCorner + s * cam.horizontal + t * cam.vertical - cam.origin - offset)); +} + + +int HitSpheres(Ray r, int sphereCount, float tMin, float tMax, inout Hit outHit) +{ + float hitT = tMax; + int id = -1; + for (int i = 0; i < sphereCount; ++i) + { + Sphere s = s_GroupSpheres[i]; + float3 co = s.center - r.orig; + float nb = dot(co, r.dir); + float c = dot(co, co) - s.radius*s.radius; + float discr = nb * nb - c; + if (discr > 0) + { + float discrSq = sqrt(discr); + + // Try earlier t + float t = nb - discrSq; + if (t <= tMin) // before min, try later t! + t = nb + discrSq; + + if (t > tMin && t < hitT) + { + id = i; + hitT = t; + } + } + } + + if (id != -1) + { + outHit.pos = RayPointAt(r, hitT); + outHit.normal = (outHit.pos - s_GroupSpheres[id].center) * s_GroupSpheres[id].invRadius; + outHit.t = hitT; + } + return id; +} + +struct Params +{ + Camera cam; + int sphereCount; + int screenWidth; + int screenHeight; + int frames; + float invWidth; + float invHeight; + float lerpFac; + int emissiveCount; +}; + + +#define kMinT 0.001f +#define kMaxT 1.0e7f +#define kMaxDepth 10 + + +static int HitWorld(int sphereCount, Ray r, float tMin, float tMax, inout Hit outHit) +{ + return HitSpheres(r, sphereCount, tMin, tMax, outHit); +} + + +static bool Scatter(int sphereCount, int emissiveCount, int matID, Ray r_in, Hit rec, out float3 attenuation, out Ray scattered, out float3 outLightE, inout int inoutRayCount, inout uint state) +{ + outLightE = float3(0, 0, 0); + Material mat = s_GroupMaterials[matID]; + if (mat.type == MatLambert) + { + // random point on unit sphere that is tangent to the hit point + float3 target = rec.pos + rec.normal + RandomUnitVector(state); + scattered = MakeRay(rec.pos, normalize(target - rec.pos)); + attenuation = mat.albedo; + + // sample lights +#if DO_LIGHT_SAMPLING + for (int j = 0; j < emissiveCount; ++j) + { + int i = s_GroupEmissives[j]; + if (matID == i) + continue; // skip self + Material smat = s_GroupMaterials[i]; + Sphere s = s_GroupSpheres[i]; + + // create a random direction towards sphere + // coord system for sampling: sw, su, sv + float3 sw = normalize(s.center - rec.pos); + float3 su = normalize(cross(abs(sw.x)>0.01f ? float3(0, 1, 0) : float3(1, 0, 0), sw)); + float3 sv = cross(sw, su); + // sample sphere by solid angle + float cosAMax = sqrt(1.0f - s.radius*s.radius / dot(rec.pos - s.center, rec.pos - s.center)); + float eps1 = RandomFloat01(state), eps2 = RandomFloat01(state); + float cosA = 1.0f - eps1 + eps1 * cosAMax; + float sinA = sqrt(1.0f - cosA * cosA); + float phi = 2 * 3.1415926 * eps2; + float3 l = su * cos(phi) * sinA + sv * sin(phi) * sinA + sw * cosA; + + // shoot shadow ray + Hit lightHit; + ++inoutRayCount; + int hitID = HitWorld(sphereCount, MakeRay(rec.pos, l), kMinT, kMaxT, lightHit); + if (hitID == i) + { + float omega = 2 * 3.1415926 * (1 - cosAMax); + + float3 rdir = r_in.dir; + float3 nl = dot(rec.normal, rdir) < 0 ? rec.normal : -rec.normal; + outLightE += (mat.albedo * smat.emissive) * (max(0.0f, dot(l, nl)) * omega / 3.1415926); + } + } +#endif + return true; + } + else if (mat.type == MatMetal) + { + float3 refl = reflect(r_in.dir, rec.normal); + // reflected ray, and random inside of sphere based on roughness + float roughness = mat.roughness; +#if DO_MITSUBA_COMPARE + roughness = 0; // until we get better BRDF for metals +#endif + scattered = MakeRay(rec.pos, normalize(refl + roughness*RandomInUnitSphere(state))); + attenuation = mat.albedo; + return dot(scattered.dir, rec.normal) > 0; + } + else if (mat.type == MatDielectric) + { + float3 outwardN; + float3 rdir = r_in.dir; + float3 refl = reflect(rdir, rec.normal); + float nint; + attenuation = float3(1, 1, 1); + float3 refr; + float reflProb; + float cosine; + if (dot(rdir, rec.normal) > 0) + { + outwardN = -rec.normal; + nint = mat.ri; + cosine = mat.ri * dot(rdir, rec.normal); + } + else + { + outwardN = rec.normal; + nint = 1.0f / mat.ri; + cosine = -dot(rdir, rec.normal); + } + if (refract(rdir, outwardN, nint, refr)) + { + reflProb = schlick(cosine, mat.ri); + } + else + { + reflProb = 1; + } + if (RandomFloat01(state) < reflProb) + scattered = MakeRay(rec.pos, normalize(refl)); + else + scattered = MakeRay(rec.pos, normalize(refr)); + } + else + { + attenuation = float3(1, 0, 1); + scattered = MakeRay(float3(0,0,0), float3(0, 0, 1)); + return false; + } + return true; +} + +static float3 Trace(int sphereCount, int emissiveCount, Ray r, inout int inoutRayCount, inout uint state) +{ + float3 col = 0; + float3 curAtten = 1; + bool doMaterialE = true; + // GPUs don't support recursion, so do tracing iterations in a loop up to max depth + for (int depth = 0; depth < kMaxDepth; ++depth) + { + Hit rec; + ++inoutRayCount; + int id = HitWorld(sphereCount, r, kMinT, kMaxT, rec); + if (id >= 0) + { + Ray scattered; + float3 attenuation; + float3 lightE; + Material mat = s_GroupMaterials[id]; + float3 matE = mat.emissive; + if (Scatter(sphereCount, emissiveCount, id, r, rec, attenuation, scattered, lightE, inoutRayCount, state)) + { +#if DO_LIGHT_SAMPLING + if (!doMaterialE) matE = 0; + doMaterialE = (mat.type != MatLambert); +#endif + col += curAtten * (matE + lightE); + curAtten *= attenuation; + r = scattered; + } + else + { + col += curAtten * matE; + break; + } + } + else + { + // sky +#if DO_MITSUBA_COMPARE + col += curAtten * float3(0.15f, 0.21f, 0.3f); // easier compare with Mitsuba's constant environment light +#else + float3 unitDir = r.dir; + float t = 0.5f*(unitDir.y + 1.0f); + float3 skyCol = ((1.0f - t)*float3(1.0f, 1.0f, 1.0f) + t * float3(0.5f, 0.7f, 1.0f)) * 0.3f; + col += curAtten * skyCol; +#endif + break; + } + } + return col; +} + +Texture2D srcImage : register(t0); +RWTexture2D dstImage : register(u0); +StructuredBuffer g_Spheres : register(t1); +StructuredBuffer g_Materials : register(t2); +StructuredBuffer g_Params : register(t3); +StructuredBuffer g_Emissives : register(t4); +RWByteAddressBuffer g_OutRayCount : register(u1); + +[numthreads(kCSGroupSizeX, kCSGroupSizeY, 1)] +void main(uint3 gid : SV_DispatchThreadID, uint3 tid : SV_GroupThreadID) +{ + // First, move scene data (spheres, materials, emissive indices) into group shared + // memory. Do this in parallel; each thread in group copies its own chunk of data. + uint threadID = tid.y * kCSGroupSizeX + tid.x; + uint groupSize = kCSGroupSizeX * kCSGroupSizeY; + uint objCount = g_Params[0].sphereCount; + uint myObjCount = (objCount + groupSize - 1) / groupSize; + uint myObjStart = threadID * myObjCount; + for (uint io = myObjStart; io < myObjStart + myObjCount; ++io) + { + if (io < objCount) + { + s_GroupSpheres[io] = g_Spheres[io]; + s_GroupMaterials[io] = g_Materials[io]; + } + if (io < g_Params[0].emissiveCount) + { + s_GroupEmissives[io] = g_Emissives[io]; + } + } + GroupMemoryBarrierWithGroupSync(); + + int rayCount = 0; + float3 col = 0; + Params params = g_Params[0]; + uint rngState = (gid.x * 1973 + gid.y * 9277 + params.frames * 26699) | 1; + for (int s = 0; s < DO_SAMPLES_PER_PIXEL; s++) + { + float u = float(gid.x + RandomFloat01(rngState)) * params.invWidth; + float v = float(gid.y + RandomFloat01(rngState)) * params.invHeight; + Ray r = CameraGetRay(params.cam, u, v, rngState); + col += Trace(params.sphereCount, params.emissiveCount, r, rayCount, rngState); + } + col *= 1.0f / float(DO_SAMPLES_PER_PIXEL); + + float3 prev = srcImage.Load(int3(gid.xy,0)).rgb; + col = lerp(col, prev, params.lerpFac); + dstImage[gid.xy] = float4(col, 1); + + g_OutRayCount.InterlockedAdd(0, rayCount); +} diff --git a/third_party/tracy/examples/ToyPathTracer/Windows/PixelShader.hlsl b/third_party/tracy/examples/ToyPathTracer/Windows/PixelShader.hlsl new file mode 100644 index 0000000000000000000000000000000000000000..41864cccdbc63178a24d5a109203fb314f0c074c --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Windows/PixelShader.hlsl @@ -0,0 +1,15 @@ +float3 LinearToSRGB(float3 rgb) +{ + rgb = max(rgb, float3(0, 0, 0)); + return max(1.055 * pow(rgb, 0.416666667) - 0.055, 0.0); +} + +Texture2D tex : register(t0); +SamplerState smp : register(s0); + +float4 main(float2 uv : TEXCOORD0) : SV_Target +{ + float3 col = tex.Sample(smp, uv).rgb; + col = LinearToSRGB(col); + return float4(col, 1.0f); +} diff --git a/third_party/tracy/examples/ToyPathTracer/Windows/TestCpu.sln b/third_party/tracy/examples/ToyPathTracer/Windows/TestCpu.sln new file mode 100644 index 0000000000000000000000000000000000000000..b4a52c70d4e652bb54fc28d4c756782c7f537f54 --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Windows/TestCpu.sln @@ -0,0 +1,31 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.30907.101 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "TestCpu", "TestCpu.vcxproj", "{4F84B756-87F5-4B92-827B-DA087DAE1900}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4F84B756-87F5-4B92-827B-DA087DAE1900}.Debug|x64.ActiveCfg = Debug|x64 + {4F84B756-87F5-4B92-827B-DA087DAE1900}.Debug|x64.Build.0 = Debug|x64 + {4F84B756-87F5-4B92-827B-DA087DAE1900}.Debug|x86.ActiveCfg = Debug|Win32 + {4F84B756-87F5-4B92-827B-DA087DAE1900}.Debug|x86.Build.0 = Debug|Win32 + {4F84B756-87F5-4B92-827B-DA087DAE1900}.Release|x64.ActiveCfg = Release|x64 + {4F84B756-87F5-4B92-827B-DA087DAE1900}.Release|x64.Build.0 = Release|x64 + {4F84B756-87F5-4B92-827B-DA087DAE1900}.Release|x86.ActiveCfg = Release|Win32 + {4F84B756-87F5-4B92-827B-DA087DAE1900}.Release|x86.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {067FB780-37B8-465E-AD7E-E7B238B9C04F} + EndGlobalSection +EndGlobal diff --git a/third_party/tracy/examples/ToyPathTracer/Windows/TestCpu.vcxproj b/third_party/tracy/examples/ToyPathTracer/Windows/TestCpu.vcxproj new file mode 100644 index 0000000000000000000000000000000000000000..c71cccf9961d4cb2807c5358d9a421ecb831fbfa --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Windows/TestCpu.vcxproj @@ -0,0 +1,245 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + 15.0 + {4F84B756-87F5-4B92-827B-DA087DAE1900} + Win32Proj + TestCpu + 10.0 + + + + Application + true + v143 + Unicode + + + Application + false + v143 + true + Unicode + + + Application + true + v143 + Unicode + + + Application + false + v143 + true + Unicode + + + + + + + + + + + + + + + + + + + + + true + + + true + + + false + + + false + + + + Level3 + Disabled + true + WIN32;_DEBUG;_WINDOWS;%(PreprocessorDefinitions) + true + VectorCall + Fast + ProgramDatabase + + + Windows + true + d3d11.lib;kernel32.lib;user32.lib;gdi32.lib;%(AdditionalDependencies) + + + + + Level3 + Disabled + true + TRACY_ENABLE;_DEBUG;_WINDOWS;%(PreprocessorDefinitions) + true + VectorCall + Fast + ProgramDatabase + + + Windows + true + d3d11.lib;kernel32.lib;user32.lib;gdi32.lib;%(AdditionalDependencies) + + + + + Level3 + MaxSpeed + true + true + WIN32;NDEBUG;_WINDOWS;%(PreprocessorDefinitions) + true + false + MultiThreaded + false + VectorCall + Fast + + + Windows + true + true + true + d3d11.lib;kernel32.lib;user32.lib;gdi32.lib;%(AdditionalDependencies) + + + + + Level3 + MaxSpeed + true + true + TRACY_ENABLE;NDEBUG;_WINDOWS;%(PreprocessorDefinitions) + true + false + MultiThreaded + false + VectorCall + Fast + + + Windows + true + true + true + d3d11.lib;kernel32.lib;user32.lib;gdi32.lib;%(AdditionalDependencies) + + + + + + + + + + + + + + + + + + + + + + + + + + + + Compute + 5.0 + Compute + 5.0 + Compute + 5.0 + Compute + 5.0 + g_CSBytecode + CompiledComputeShader.h + g_CSBytecode + CompiledComputeShader.h + g_CSBytecode + CompiledComputeShader.h + g_CSBytecode + CompiledComputeShader.h + + + Pixel + Pixel + Pixel + Pixel + 5.0 + 5.0 + 5.0 + 5.0 + CompiledPixelShader.h + CompiledPixelShader.h + CompiledPixelShader.h + CompiledPixelShader.h + g_PSBytecode + g_PSBytecode + g_PSBytecode + g_PSBytecode + + + Vertex + Vertex + Vertex + Vertex + 5.0 + 5.0 + 5.0 + 5.0 + CompiledVertexShader.h + CompiledVertexShader.h + CompiledVertexShader.h + CompiledVertexShader.h + g_VSBytecode + g_VSBytecode + g_VSBytecode + g_VSBytecode + + + + + + \ No newline at end of file diff --git a/third_party/tracy/examples/ToyPathTracer/Windows/TestCpu.vcxproj.filters b/third_party/tracy/examples/ToyPathTracer/Windows/TestCpu.vcxproj.filters new file mode 100644 index 0000000000000000000000000000000000000000..f447ce77a26feb1a2ab604edf43945454ac38b97 --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Windows/TestCpu.vcxproj.filters @@ -0,0 +1,67 @@ + + + + + + Source + + + Source\enkiTS + + + Source\enkiTS + + + Source + + + + + + {5f19f217-c1c7-4eeb-be61-8b986fee9375} + + + {38c448a8-1dcc-4116-9410-a9f8d068caff} + + + + + Source + + + Source + + + Source\enkiTS + + + Source\enkiTS + + + Source\enkiTS + + + Source\enkiTS + + + Source\enkiTS + + + Source + + + Source + + + Source + + + + + + + + + + + \ No newline at end of file diff --git a/third_party/tracy/examples/ToyPathTracer/Windows/TestWin.cpp b/third_party/tracy/examples/ToyPathTracer/Windows/TestWin.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bfb4ad426b5639d9ec6702191c39f9a42312462e --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Windows/TestWin.cpp @@ -0,0 +1,567 @@ +#include +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include +#include + +#include +#include +#include +#include + +#include "../Source/Config.h" +#include "../Source/Maths.h" +#include "../Source/Test.h" +#include "CompiledVertexShader.h" +#include "CompiledPixelShader.h" + +#include "../../../public/tracy/Tracy.hpp" +#include "../../../public/tracy/TracyD3D11.hpp" + +static HINSTANCE g_HInstance; +static HWND g_Wnd; + +ATOM MyRegisterClass(HINSTANCE hInstance); +BOOL InitInstance(HINSTANCE, int); +LRESULT CALLBACK WndProc(HWND, UINT, WPARAM, LPARAM); +INT_PTR CALLBACK About(HWND, UINT, WPARAM, LPARAM); + +static HRESULT InitD3DDevice(); +static void ShutdownD3DDevice(); +static void RenderFrame(); + +static float* g_Backbuffer; + +static D3D_FEATURE_LEVEL g_D3D11FeatureLevel = D3D_FEATURE_LEVEL_11_0; +static ID3D11Device* g_D3D11Device = nullptr; +static ID3D11DeviceContext* g_D3D11Ctx = nullptr; +static IDXGISwapChain* g_D3D11SwapChain = nullptr; +static ID3D11RenderTargetView* g_D3D11RenderTarget = nullptr; +static ID3D11VertexShader* g_VertexShader; +static ID3D11PixelShader* g_PixelShader; +static ID3D11Texture2D *g_BackbufferTexture, *g_BackbufferTexture2; +static ID3D11ShaderResourceView *g_BackbufferSRV, *g_BackbufferSRV2; +static ID3D11UnorderedAccessView *g_BackbufferUAV, *g_BackbufferUAV2; +static ID3D11SamplerState* g_SamplerLinear; +static ID3D11RasterizerState* g_RasterState; +static int g_BackbufferIndex; +static tracy::D3D11Ctx *g_tracyCtx; + + +#if DO_COMPUTE_GPU +#include "CompiledComputeShader.h" +struct ComputeParams +{ + Camera cam; + int sphereCount; + int screenWidth; + int screenHeight; + int frames; + float invWidth; + float invHeight; + float lerpFac; + int emissiveCount; +}; +static ID3D11ComputeShader* g_ComputeShader; +static ID3D11Buffer* g_DataSpheres; static ID3D11ShaderResourceView* g_SRVSpheres; +static ID3D11Buffer* g_DataMaterials; static ID3D11ShaderResourceView* g_SRVMaterials; +static ID3D11Buffer* g_DataParams; static ID3D11ShaderResourceView* g_SRVParams; +static ID3D11Buffer* g_DataEmissives; static ID3D11ShaderResourceView* g_SRVEmissives; +static ID3D11Buffer* g_DataCounter; static ID3D11UnorderedAccessView* g_UAVCounter; +static int g_SphereCount, g_ObjSize, g_MatSize; +static ID3D11Query *g_QueryBegin, *g_QueryEnd, *g_QueryDisjoint; +#endif // #if DO_COMPUTE_GPU + +int APIENTRY wWinMain(_In_ HINSTANCE hInstance, _In_opt_ HINSTANCE, _In_ LPWSTR, _In_ int nCmdShow) +{ + g_Backbuffer = new float[kBackbufferWidth * kBackbufferHeight * 4]; + memset(g_Backbuffer, 0, kBackbufferWidth * kBackbufferHeight * 4 * sizeof(g_Backbuffer[0])); + + InitializeTest(); + + MyRegisterClass(hInstance); + if (!InitInstance (hInstance, nCmdShow)) + { + return FALSE; + } + + if (FAILED(InitD3DDevice())) + { + ShutdownD3DDevice(); + return 0; + } + + g_D3D11Device->CreateVertexShader(g_VSBytecode, ARRAYSIZE(g_VSBytecode), NULL, &g_VertexShader); + g_D3D11Device->CreatePixelShader(g_PSBytecode, ARRAYSIZE(g_PSBytecode), NULL, &g_PixelShader); +#if DO_COMPUTE_GPU + g_D3D11Device->CreateComputeShader(g_CSBytecode, ARRAYSIZE(g_CSBytecode), NULL, &g_ComputeShader); +#endif + + D3D11_TEXTURE2D_DESC texDesc = {}; + texDesc.Width = kBackbufferWidth; + texDesc.Height = kBackbufferHeight; + texDesc.MipLevels = 1; + texDesc.ArraySize = 1; + texDesc.Format = DXGI_FORMAT_R32G32B32A32_FLOAT; + texDesc.SampleDesc.Count = 1; + texDesc.SampleDesc.Quality = 0; +#if DO_COMPUTE_GPU + texDesc.Usage = D3D11_USAGE_DEFAULT; + texDesc.BindFlags = D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_UNORDERED_ACCESS; + texDesc.CPUAccessFlags = 0; +#else + texDesc.Usage = D3D11_USAGE_DYNAMIC; + texDesc.BindFlags = D3D11_BIND_SHADER_RESOURCE; + texDesc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE; +#endif + texDesc.MiscFlags = 0; + g_D3D11Device->CreateTexture2D(&texDesc, NULL, &g_BackbufferTexture); + g_D3D11Device->CreateTexture2D(&texDesc, NULL, &g_BackbufferTexture2); + + D3D11_SHADER_RESOURCE_VIEW_DESC srvDesc = {}; + srvDesc.Format = texDesc.Format; + srvDesc.ViewDimension = D3D11_SRV_DIMENSION_TEXTURE2D; + srvDesc.Texture2D.MipLevels = 1; + srvDesc.Texture2D.MostDetailedMip = 0; + g_D3D11Device->CreateShaderResourceView(g_BackbufferTexture, &srvDesc, &g_BackbufferSRV); + g_D3D11Device->CreateShaderResourceView(g_BackbufferTexture2, &srvDesc, &g_BackbufferSRV2); + + D3D11_SAMPLER_DESC smpDesc = {}; + smpDesc.Filter = D3D11_FILTER_MIN_MAG_LINEAR_MIP_POINT; + smpDesc.AddressU = smpDesc.AddressV = smpDesc.AddressW = D3D11_TEXTURE_ADDRESS_CLAMP; + g_D3D11Device->CreateSamplerState(&smpDesc, &g_SamplerLinear); + + D3D11_RASTERIZER_DESC rasterDesc = {}; + rasterDesc.FillMode = D3D11_FILL_SOLID; + rasterDesc.CullMode = D3D11_CULL_NONE; + g_D3D11Device->CreateRasterizerState(&rasterDesc, &g_RasterState); + +#if DO_COMPUTE_GPU + D3D11_UNORDERED_ACCESS_VIEW_DESC uavDesc = {}; + + int camSize; + GetObjectCount(g_SphereCount, g_ObjSize, g_MatSize, camSize); + assert(g_ObjSize == 20); + assert(g_MatSize == 36); + assert(camSize == 88); + D3D11_BUFFER_DESC bdesc = {}; + bdesc.ByteWidth = g_SphereCount * g_ObjSize; + bdesc.Usage = D3D11_USAGE_DEFAULT; + bdesc.BindFlags = D3D11_BIND_SHADER_RESOURCE; + bdesc.CPUAccessFlags = 0; + bdesc.MiscFlags = D3D11_RESOURCE_MISC_BUFFER_STRUCTURED; + bdesc.StructureByteStride = g_ObjSize; + g_D3D11Device->CreateBuffer(&bdesc, NULL, &g_DataSpheres); + srvDesc.Format = DXGI_FORMAT_UNKNOWN; + srvDesc.ViewDimension = D3D11_SRV_DIMENSION_BUFFER; + srvDesc.Buffer.FirstElement = 0; + srvDesc.Buffer.NumElements = g_SphereCount; + g_D3D11Device->CreateShaderResourceView(g_DataSpheres, &srvDesc, &g_SRVSpheres); + + bdesc.ByteWidth = g_SphereCount * g_MatSize; + bdesc.StructureByteStride = g_MatSize; + g_D3D11Device->CreateBuffer(&bdesc, NULL, &g_DataMaterials); + srvDesc.Buffer.NumElements = g_SphereCount; + g_D3D11Device->CreateShaderResourceView(g_DataMaterials, &srvDesc, &g_SRVMaterials); + + bdesc.ByteWidth = sizeof(ComputeParams); + bdesc.StructureByteStride = sizeof(ComputeParams); + g_D3D11Device->CreateBuffer(&bdesc, NULL, &g_DataParams); + srvDesc.Buffer.NumElements = 1; + g_D3D11Device->CreateShaderResourceView(g_DataParams, &srvDesc, &g_SRVParams); + + bdesc.ByteWidth = g_SphereCount * 4; + bdesc.StructureByteStride = 4; + g_D3D11Device->CreateBuffer(&bdesc, NULL, &g_DataEmissives); + srvDesc.Buffer.NumElements = g_SphereCount; + g_D3D11Device->CreateShaderResourceView(g_DataEmissives, &srvDesc, &g_SRVEmissives); + + bdesc.ByteWidth = 4; + bdesc.BindFlags |= D3D11_BIND_UNORDERED_ACCESS; + bdesc.MiscFlags = D3D11_RESOURCE_MISC_BUFFER_ALLOW_RAW_VIEWS; + bdesc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + g_D3D11Device->CreateBuffer(&bdesc, NULL, &g_DataCounter); + uavDesc.Format = DXGI_FORMAT_R32_TYPELESS; + uavDesc.ViewDimension = D3D11_UAV_DIMENSION_BUFFER; + uavDesc.Buffer.FirstElement = 0; + uavDesc.Buffer.NumElements = 1; + uavDesc.Buffer.Flags = D3D11_BUFFER_UAV_FLAG_RAW; + g_D3D11Device->CreateUnorderedAccessView(g_DataCounter, &uavDesc, &g_UAVCounter); + + uavDesc.Format = DXGI_FORMAT_R32G32B32A32_FLOAT; + uavDesc.ViewDimension = D3D11_UAV_DIMENSION_TEXTURE2D; + uavDesc.Texture2D.MipSlice = 0; + g_D3D11Device->CreateUnorderedAccessView(g_BackbufferTexture, &uavDesc, &g_BackbufferUAV); + g_D3D11Device->CreateUnorderedAccessView(g_BackbufferTexture2, &uavDesc, &g_BackbufferUAV2); + + D3D11_QUERY_DESC qDesc = {}; + qDesc.Query = D3D11_QUERY_TIMESTAMP; + g_D3D11Device->CreateQuery(&qDesc, &g_QueryBegin); + g_D3D11Device->CreateQuery(&qDesc, &g_QueryEnd); + qDesc.Query = D3D11_QUERY_TIMESTAMP_DISJOINT; + g_D3D11Device->CreateQuery(&qDesc, &g_QueryDisjoint); +#endif // #if DO_COMPUTE_GPU + + + static int framesLeft = 10; + + // Main message loop + MSG msg = { 0 }; + while (msg.message != WM_QUIT) + { + if (PeekMessage(&msg, NULL, 0U, 0U, PM_REMOVE)) + { + TranslateMessage(&msg); + DispatchMessage(&msg); + } + else + { + RenderFrame(); + TracyD3D11Collect(g_tracyCtx); + if( --framesLeft == 0 ) break; + } + } + + ShutdownTest(); + ShutdownD3DDevice(); + + return (int) msg.wParam; +} + + +ATOM MyRegisterClass(HINSTANCE hInstance) +{ + ZoneScoped; + + WNDCLASSEXW wcex; + memset(&wcex, 0, sizeof(wcex)); + wcex.cbSize = sizeof(WNDCLASSEX); + wcex.style = CS_HREDRAW | CS_VREDRAW; + wcex.lpfnWndProc = WndProc; + wcex.cbClsExtra = 0; + wcex.cbWndExtra = 0; + wcex.hInstance = hInstance; + wcex.hCursor = LoadCursor(nullptr, IDC_ARROW); + wcex.hbrBackground = (HBRUSH)(COLOR_WINDOW+1); + wcex.lpszClassName = L"TestClass"; + return RegisterClassExW(&wcex); +} + +BOOL InitInstance(HINSTANCE hInstance, int nCmdShow) +{ + ZoneScoped; + + g_HInstance = hInstance; + RECT rc = { 0, 0, kBackbufferWidth, kBackbufferHeight }; + DWORD style = WS_OVERLAPPED | WS_CAPTION | WS_SYSMENU | WS_MINIMIZEBOX; + AdjustWindowRect(&rc, style, FALSE); + HWND hWnd = CreateWindowW(L"TestClass", L"Test", style, CW_USEDEFAULT, CW_USEDEFAULT, rc.right-rc.left, rc.bottom-rc.top, nullptr, nullptr, hInstance, nullptr); + if (!hWnd) + return FALSE; + g_Wnd = hWnd; + ShowWindow(hWnd, nCmdShow); + UpdateWindow(hWnd); + return TRUE; +} + +static uint64_t s_Time; +static int s_Count; +static char s_Buffer[200]; +static unsigned s_Flags = kFlagProgressive; +static int s_FrameCount = 0; + + +static void RenderFrame() +{ + ZoneScoped; + TracyD3D11Zone(g_tracyCtx, "RenderFrame"); + + LARGE_INTEGER time1; + +#if DO_COMPUTE_GPU + QueryPerformanceCounter(&time1); + float t = float(clock()) / CLOCKS_PER_SEC; + UpdateTest(t, s_FrameCount, kBackbufferWidth, kBackbufferHeight, s_Flags); + + g_BackbufferIndex = 1 - g_BackbufferIndex; + void* dataSpheres = alloca(g_SphereCount * g_ObjSize); + void* dataMaterials = alloca(g_SphereCount * g_MatSize); + void* dataEmissives = alloca(g_SphereCount * 4); + ComputeParams dataParams; + GetSceneDesc(dataSpheres, dataMaterials, &dataParams.cam, dataEmissives, &dataParams.emissiveCount); + + dataParams.sphereCount = g_SphereCount; + dataParams.screenWidth = kBackbufferWidth; + dataParams.screenHeight = kBackbufferHeight; + dataParams.frames = s_FrameCount; + dataParams.invWidth = 1.0f / kBackbufferWidth; + dataParams.invHeight = 1.0f / kBackbufferHeight; + float lerpFac = float(s_FrameCount) / float(s_FrameCount + 1); + if (s_Flags & kFlagAnimate) + lerpFac *= DO_ANIMATE_SMOOTHING; + if (!(s_Flags & kFlagProgressive)) + lerpFac = 0; + dataParams.lerpFac = lerpFac; + + g_D3D11Ctx->UpdateSubresource(g_DataSpheres, 0, NULL, dataSpheres, 0, 0); + g_D3D11Ctx->UpdateSubresource(g_DataMaterials, 0, NULL, dataMaterials, 0, 0); + g_D3D11Ctx->UpdateSubresource(g_DataParams, 0, NULL, &dataParams, 0, 0); + g_D3D11Ctx->UpdateSubresource(g_DataEmissives, 0, NULL, dataEmissives, 0, 0); + + ID3D11ShaderResourceView* srvs[] = { + g_BackbufferIndex == 0 ? g_BackbufferSRV2 : g_BackbufferSRV, + g_SRVSpheres, + g_SRVMaterials, + g_SRVParams, + g_SRVEmissives + }; + g_D3D11Ctx->CSSetShaderResources(0, ARRAYSIZE(srvs), srvs); + ID3D11UnorderedAccessView* uavs[] = { + g_BackbufferIndex == 0 ? g_BackbufferUAV : g_BackbufferUAV2, + g_UAVCounter + }; + g_D3D11Ctx->CSSetUnorderedAccessViews(0, ARRAYSIZE(uavs), uavs, NULL); + g_D3D11Ctx->CSSetShader(g_ComputeShader, NULL, 0); + g_D3D11Ctx->Begin(g_QueryDisjoint); + g_D3D11Ctx->End(g_QueryBegin); + g_D3D11Ctx->Dispatch(kBackbufferWidth/kCSGroupSizeX, kBackbufferHeight/kCSGroupSizeY, 1); + g_D3D11Ctx->End(g_QueryEnd); + uavs[0] = NULL; + g_D3D11Ctx->CSSetUnorderedAccessViews(0, ARRAYSIZE(uavs), uavs, NULL); + ++s_FrameCount; + +#else + QueryPerformanceCounter(&time1); + float t = float(clock()) / CLOCKS_PER_SEC; + static size_t s_RayCounter = 0; + int rayCount; + UpdateTest(t, s_FrameCount, kBackbufferWidth, kBackbufferHeight, s_Flags); + DrawTest(t, s_FrameCount, kBackbufferWidth, kBackbufferHeight, g_Backbuffer, rayCount, s_Flags); + s_FrameCount++; + s_RayCounter += rayCount; + LARGE_INTEGER time2; + QueryPerformanceCounter(&time2); + uint64_t dt = time2.QuadPart - time1.QuadPart; + ++s_Count; + s_Time += dt; + if (s_Count > 10) + { + LARGE_INTEGER frequency; + QueryPerformanceFrequency(&frequency); + + double s = double(s_Time) / double(frequency.QuadPart) / s_Count; + sprintf_s(s_Buffer, sizeof(s_Buffer), "%.2fms (%.1f FPS) %.1fMrays/s %.2fMrays/frame frames %i\n", s * 1000.0f, 1.f / s, s_RayCounter / s_Count / s * 1.0e-6f, s_RayCounter / s_Count * 1.0e-6f, s_FrameCount); + SetWindowTextA(g_Wnd, s_Buffer); + OutputDebugStringA(s_Buffer); + s_Count = 0; + s_Time = 0; + s_RayCounter = 0; + } + + D3D11_MAPPED_SUBRESOURCE mapped; + g_D3D11Ctx->Map(g_BackbufferTexture, 0, D3D11_MAP_WRITE_DISCARD, 0, &mapped); + const uint8_t* src = (const uint8_t*)g_Backbuffer; + uint8_t* dst = (uint8_t*)mapped.pData; + for (int y = 0; y < kBackbufferHeight; ++y) + { + memcpy(dst, src, kBackbufferWidth * 16); + src += kBackbufferWidth * 16; + dst += mapped.RowPitch; + } + g_D3D11Ctx->Unmap(g_BackbufferTexture, 0); +#endif + + g_D3D11Ctx->VSSetShader(g_VertexShader, NULL, 0); + g_D3D11Ctx->PSSetShader(g_PixelShader, NULL, 0); + g_D3D11Ctx->PSSetShaderResources(0, 1, g_BackbufferIndex == 0 ? &g_BackbufferSRV : &g_BackbufferSRV2); + g_D3D11Ctx->PSSetSamplers(0, 1, &g_SamplerLinear); + g_D3D11Ctx->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST); + g_D3D11Ctx->RSSetState(g_RasterState); + g_D3D11Ctx->Draw(3, 0); + g_D3D11SwapChain->Present(0, 0); + + FrameMark; + +#if DO_COMPUTE_GPU + g_D3D11Ctx->End(g_QueryDisjoint); + + // get GPU times + while (g_D3D11Ctx->GetData(g_QueryDisjoint, NULL, 0, 0) == S_FALSE) { Sleep(0); } + D3D10_QUERY_DATA_TIMESTAMP_DISJOINT tsDisjoint; + g_D3D11Ctx->GetData(g_QueryDisjoint, &tsDisjoint, sizeof(tsDisjoint), 0); + if (!tsDisjoint.Disjoint) + { + UINT64 tsBegin, tsEnd; + // Note: on some GPUs/drivers, even when the disjoint query above already said "yeah I have data", + // might still not return "I have data" for timestamp queries before it. + while (g_D3D11Ctx->GetData(g_QueryBegin, &tsBegin, sizeof(tsBegin), 0) == S_FALSE) { Sleep(0); } + while (g_D3D11Ctx->GetData(g_QueryEnd, &tsEnd, sizeof(tsEnd), 0) == S_FALSE) { Sleep(0); } + + float s = float(tsEnd - tsBegin) / float(tsDisjoint.Frequency); + + static uint64_t s_RayCounter; + D3D11_MAPPED_SUBRESOURCE mapped; + g_D3D11Ctx->Map(g_DataCounter, 0, D3D11_MAP_READ, 0, &mapped); + s_RayCounter += *(const int*)mapped.pData; + g_D3D11Ctx->Unmap(g_DataCounter, 0); + int zeroCount = 0; + g_D3D11Ctx->UpdateSubresource(g_DataCounter, 0, NULL, &zeroCount, 0, 0); + + static float s_Time; + ++s_Count; + s_Time += s; + if (s_Count > 150) + { + s = s_Time / s_Count; + sprintf_s(s_Buffer, sizeof(s_Buffer), "%.2fms (%.1f FPS) %.1fMrays/s %.2fMrays/frame frames %i\n", s * 1000.0f, 1.f / s, s_RayCounter / s_Count / s * 1.0e-6f, s_RayCounter / s_Count * 1.0e-6f, s_FrameCount); + SetWindowTextA(g_Wnd, s_Buffer); + s_Count = 0; + s_Time = 0; + s_RayCounter = 0; + } + + } +#endif // #if DO_COMPUTE_GPU +} + + +LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam) +{ + switch (message) + { + case WM_PAINT: + { + PAINTSTRUCT ps; + HDC hdc = BeginPaint(hWnd, &ps); + EndPaint(hWnd, &ps); + } + break; + case WM_DESTROY: + PostQuitMessage(0); + break; + case WM_CHAR: + if (wParam == 'a') + s_Flags = s_Flags ^ kFlagAnimate; + if (wParam == 'p') + { + s_Flags = s_Flags ^ kFlagProgressive; + s_FrameCount = 0; + } + break; + default: + return DefWindowProc(hWnd, message, wParam, lParam); + } + return 0; +} + + +static HRESULT InitD3DDevice() +{ + ZoneScoped; + + HRESULT hr = S_OK; + + RECT rc; + GetClientRect(g_Wnd, &rc); + UINT width = rc.right - rc.left; + UINT height = rc.bottom - rc.top; + + UINT createDeviceFlags = 0; +#ifdef _DEBUG + createDeviceFlags |= D3D11_CREATE_DEVICE_DEBUG; +#endif + + D3D_FEATURE_LEVEL featureLevels[] = + { + D3D_FEATURE_LEVEL_11_0, + }; + UINT numFeatureLevels = ARRAYSIZE(featureLevels); + hr = D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, createDeviceFlags, featureLevels, numFeatureLevels, D3D11_SDK_VERSION, &g_D3D11Device, &g_D3D11FeatureLevel, &g_D3D11Ctx); + if (FAILED(hr)) + return hr; + + // Get DXGI factory + IDXGIFactory1* dxgiFactory = nullptr; + { + IDXGIDevice* dxgiDevice = nullptr; + hr = g_D3D11Device->QueryInterface(__uuidof(IDXGIDevice), reinterpret_cast(&dxgiDevice)); + if (SUCCEEDED(hr)) + { + IDXGIAdapter* adapter = nullptr; + hr = dxgiDevice->GetAdapter(&adapter); + if (SUCCEEDED(hr)) + { + hr = adapter->GetParent(__uuidof(IDXGIFactory1), reinterpret_cast(&dxgiFactory)); + adapter->Release(); + } + dxgiDevice->Release(); + } + } + if (FAILED(hr)) + return hr; + + // Create swap chain + DXGI_SWAP_CHAIN_DESC sd; + ZeroMemory(&sd, sizeof(sd)); + sd.BufferCount = 1; + sd.BufferDesc.Width = width; + sd.BufferDesc.Height = height; + sd.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM; + sd.BufferDesc.RefreshRate.Numerator = 60; + sd.BufferDesc.RefreshRate.Denominator = 1; + sd.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT; + sd.OutputWindow = g_Wnd; + sd.SampleDesc.Count = 1; + sd.SampleDesc.Quality = 0; + sd.Windowed = TRUE; + hr = dxgiFactory->CreateSwapChain(g_D3D11Device, &sd, &g_D3D11SwapChain); + + // Prevent Alt-Enter + dxgiFactory->MakeWindowAssociation(g_Wnd, DXGI_MWA_NO_ALT_ENTER); + dxgiFactory->Release(); + + if (FAILED(hr)) + return hr; + + // RTV + ID3D11Texture2D* pBackBuffer = nullptr; + hr = g_D3D11SwapChain->GetBuffer(0, __uuidof(ID3D11Texture2D), reinterpret_cast(&pBackBuffer)); + if (FAILED(hr)) + return hr; + hr = g_D3D11Device->CreateRenderTargetView(pBackBuffer, nullptr, &g_D3D11RenderTarget); + pBackBuffer->Release(); + if (FAILED(hr)) + return hr; + + g_D3D11Ctx->OMSetRenderTargets(1, &g_D3D11RenderTarget, nullptr); + + // Viewport + D3D11_VIEWPORT vp; + vp.Width = (float)width; + vp.Height = (float)height; + vp.MinDepth = 0.0f; + vp.MaxDepth = 1.0f; + vp.TopLeftX = 0; + vp.TopLeftY = 0; + g_D3D11Ctx->RSSetViewports(1, &vp); + + g_tracyCtx = TracyD3D11Context(g_D3D11Device, g_D3D11Ctx); + const char* tracyD3D11CtxName = "D3D11"; + TracyD3D11ContextName(g_tracyCtx, tracyD3D11CtxName, (uint16_t)strlen(tracyD3D11CtxName)); + + return S_OK; +} + +static void ShutdownD3DDevice() +{ + ZoneScoped; + + if (g_tracyCtx) TracyD3D11Destroy(g_tracyCtx); + + if (g_D3D11Ctx) g_D3D11Ctx->ClearState(); + + if (g_D3D11RenderTarget) g_D3D11RenderTarget->Release(); + if (g_D3D11SwapChain) g_D3D11SwapChain->Release(); + if (g_D3D11Ctx) g_D3D11Ctx->Release(); + if (g_D3D11Device) g_D3D11Device->Release(); +} diff --git a/third_party/tracy/examples/ToyPathTracer/Windows/VertexShader.hlsl b/third_party/tracy/examples/ToyPathTracer/Windows/VertexShader.hlsl new file mode 100644 index 0000000000000000000000000000000000000000..956f4d569416d8e74ef359151b6b97f5203b924c --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/Windows/VertexShader.hlsl @@ -0,0 +1,13 @@ +struct vs2ps +{ + float2 uv : TEXCOORD0; + float4 pos : SV_Position; +}; + +vs2ps main(uint vid : SV_VertexID) +{ + vs2ps o; + o.uv = float2((vid << 1) & 2, vid & 2); + o.pos = float4(o.uv * float2(2, 2) + float2(-1, -1), 0, 1); + return o; +} diff --git a/third_party/tracy/examples/ToyPathTracer/license.md b/third_party/tracy/examples/ToyPathTracer/license.md new file mode 100644 index 0000000000000000000000000000000000000000..cf1ab25da0349f84a3fdd40032f0ce99db813b8b --- /dev/null +++ b/third_party/tracy/examples/ToyPathTracer/license.md @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/third_party/tracy/examples/fibers.cpp b/third_party/tracy/examples/fibers.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f416c14c11d4024c35cadcf02d078aea29c8f1ea --- /dev/null +++ b/third_party/tracy/examples/fibers.cpp @@ -0,0 +1,30 @@ +// g++ fibers.cpp ../public/TracyClient.cpp -I../public/tracy -DTRACY_ENABLE -DTRACY_FIBERS -lpthread -ldl + +#include +#include + +#include "Tracy.hpp" +#include "TracyC.h" + +const char* fiber = "job1"; +TracyCZoneCtx zone; + +int main() +{ + std::thread t1( [] { + TracyFiberEnter( fiber ); + TracyCZone( ctx, 1 ); + zone = ctx; + sleep( 1 ); + TracyFiberLeave; + }); + t1.join(); + + std::thread t2( [] { + TracyFiberEnter( fiber ); + sleep( 1 ); + TracyCZoneEnd( zone ); + TracyFiberLeave; + }); + t2.join(); +} diff --git a/third_party/tracy/public/TracyClient.cpp b/third_party/tracy/public/TracyClient.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6224f48bfe750f4f750219ebed53cc07312c15d3 --- /dev/null +++ b/third_party/tracy/public/TracyClient.cpp @@ -0,0 +1,61 @@ +// +// Tracy profiler +// ---------------- +// +// For fast integration, compile and +// link with this source file (and none +// other) in your executable (or in the +// main DLL / shared object on multi-DLL +// projects). +// + +// Define TRACY_ENABLE to enable profiler. + +#include "common/TracySystem.cpp" + +#ifdef TRACY_ENABLE + +#ifdef _MSC_VER +# pragma warning(push, 0) +#endif + +#include "common/tracy_lz4.cpp" +#include "client/TracyProfiler.cpp" +#include "client/TracyCallstack.cpp" +#include "client/TracySysPower.cpp" +#include "client/TracySysTime.cpp" +#include "client/TracySysTrace.cpp" +#include "common/TracySocket.cpp" +#include "client/tracy_rpmalloc.cpp" +#include "client/TracyDxt1.cpp" +#include "client/TracyAlloc.cpp" +#include "client/TracyOverride.cpp" +#include "client/TracyKCore.cpp" + +#if defined(TRACY_HAS_CALLSTACK) +# if TRACY_HAS_CALLSTACK == 2 || TRACY_HAS_CALLSTACK == 3 || TRACY_HAS_CALLSTACK == 4 || TRACY_HAS_CALLSTACK == 6 +# include "libbacktrace/alloc.cpp" +# include "libbacktrace/dwarf.cpp" +# include "libbacktrace/fileline.cpp" +# include "libbacktrace/mmapio.cpp" +# include "libbacktrace/posix.cpp" +# include "libbacktrace/sort.cpp" +# include "libbacktrace/state.cpp" +# if TRACY_HAS_CALLSTACK == 4 +# include "libbacktrace/macho.cpp" +# else +# include "libbacktrace/elf.cpp" +# endif +# include "common/TracyStackFrames.cpp" +# endif +#endif + +#ifdef _MSC_VER +# pragma comment(lib, "ws2_32.lib") +# pragma comment(lib, "dbghelp.lib") +# pragma comment(lib, "advapi32.lib") +# pragma comment(lib, "user32.lib") +# pragma warning(pop) +#endif + +#endif diff --git a/third_party/tracy/public/libbacktrace/alloc.cpp b/third_party/tracy/public/libbacktrace/alloc.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a365a4860ac827574c75fdcc2146230ab4a39bac --- /dev/null +++ b/third_party/tracy/public/libbacktrace/alloc.cpp @@ -0,0 +1,174 @@ +/* alloc.c -- Memory allocation without mmap. + Copyright (C) 2012-2021 Free Software Foundation, Inc. + Written by Ian Lance Taylor, Google. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + (1) Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + (2) Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + (3) The name of the author may not be used to + endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. */ + +#include "config.h" + +#include +#include +#include + +#include "backtrace.hpp" +#include "internal.hpp" + +#include "../common/TracyAlloc.hpp" + +namespace tracy +{ + +/* Allocation routines to use on systems that do not support anonymous + mmap. This implementation just uses malloc, which means that the + backtrace functions may not be safely invoked from a signal + handler. */ + +/* Allocate memory like malloc. If ERROR_CALLBACK is NULL, don't + report an error. */ + +void * +backtrace_alloc (struct backtrace_state *state ATTRIBUTE_UNUSED, + size_t size, backtrace_error_callback error_callback, + void *data) +{ + void *ret; + + ret = tracy_malloc (size); + if (ret == NULL) + { + if (error_callback) + error_callback (data, "malloc", errno); + } + return ret; +} + +/* Free memory. */ + +void +backtrace_free (struct backtrace_state *state ATTRIBUTE_UNUSED, + void *p, size_t size ATTRIBUTE_UNUSED, + backtrace_error_callback error_callback ATTRIBUTE_UNUSED, + void *data ATTRIBUTE_UNUSED) +{ + tracy_free (p); +} + +/* Grow VEC by SIZE bytes. */ + +void * +backtrace_vector_grow (struct backtrace_state *state ATTRIBUTE_UNUSED, + size_t size, backtrace_error_callback error_callback, + void *data, struct backtrace_vector *vec) +{ + void *ret; + + if (size > vec->alc) + { + size_t alc; + void *base; + + if (vec->size == 0) + alc = 32 * size; + else if (vec->size >= 4096) + alc = vec->size + 4096; + else + alc = 2 * vec->size; + + if (alc < vec->size + size) + alc = vec->size + size; + + base = tracy_realloc (vec->base, alc); + if (base == NULL) + { + error_callback (data, "realloc", errno); + return NULL; + } + + vec->base = base; + vec->alc = alc - vec->size; + } + + ret = (char *) vec->base + vec->size; + vec->size += size; + vec->alc -= size; + return ret; +} + +/* Finish the current allocation on VEC. */ + +void * +backtrace_vector_finish (struct backtrace_state *state, + struct backtrace_vector *vec, + backtrace_error_callback error_callback, + void *data) +{ + void *ret; + + /* With this allocator we call realloc in backtrace_vector_grow, + which means we can't easily reuse the memory here. So just + release it. */ + if (!backtrace_vector_release (state, vec, error_callback, data)) + return NULL; + ret = vec->base; + vec->base = NULL; + vec->size = 0; + vec->alc = 0; + return ret; +} + +/* Release any extra space allocated for VEC. */ + +int +backtrace_vector_release (struct backtrace_state *state ATTRIBUTE_UNUSED, + struct backtrace_vector *vec, + backtrace_error_callback error_callback, + void *data) +{ + vec->alc = 0; + + if (vec->size == 0) + { + /* As of C17, realloc with size 0 is marked as an obsolescent feature, use + free instead. */ + tracy_free (vec->base); + vec->base = NULL; + return 1; + } + + vec->base = tracy_realloc (vec->base, vec->size); + if (vec->base == NULL) + { + error_callback (data, "realloc", errno); + return 0; + } + + return 1; +} + +} diff --git a/third_party/tracy/public/libbacktrace/dwarf.cpp b/third_party/tracy/public/libbacktrace/dwarf.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b6d681aa9cbc30582f2251f1727d796bae5982b4 --- /dev/null +++ b/third_party/tracy/public/libbacktrace/dwarf.cpp @@ -0,0 +1,4448 @@ +/* dwarf.c -- Get file/line information from DWARF for backtraces. + Copyright (C) 2012-2021 Free Software Foundation, Inc. + Written by Ian Lance Taylor, Google. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + (1) Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + (2) Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + (3) The name of the author may not be used to + endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. */ + +#include "config.h" + +#include +#include +#include +#include + +#include "filenames.hpp" + +#include "backtrace.hpp" +#include "internal.hpp" + +namespace tracy +{ + +/* DWARF constants. */ + +enum dwarf_tag { + DW_TAG_entry_point = 0x3, + DW_TAG_compile_unit = 0x11, + DW_TAG_inlined_subroutine = 0x1d, + DW_TAG_subprogram = 0x2e, + DW_TAG_skeleton_unit = 0x4a, +}; + +enum dwarf_form { + DW_FORM_addr = 0x01, + DW_FORM_block2 = 0x03, + DW_FORM_block4 = 0x04, + DW_FORM_data2 = 0x05, + DW_FORM_data4 = 0x06, + DW_FORM_data8 = 0x07, + DW_FORM_string = 0x08, + DW_FORM_block = 0x09, + DW_FORM_block1 = 0x0a, + DW_FORM_data1 = 0x0b, + DW_FORM_flag = 0x0c, + DW_FORM_sdata = 0x0d, + DW_FORM_strp = 0x0e, + DW_FORM_udata = 0x0f, + DW_FORM_ref_addr = 0x10, + DW_FORM_ref1 = 0x11, + DW_FORM_ref2 = 0x12, + DW_FORM_ref4 = 0x13, + DW_FORM_ref8 = 0x14, + DW_FORM_ref_udata = 0x15, + DW_FORM_indirect = 0x16, + DW_FORM_sec_offset = 0x17, + DW_FORM_exprloc = 0x18, + DW_FORM_flag_present = 0x19, + DW_FORM_ref_sig8 = 0x20, + DW_FORM_strx = 0x1a, + DW_FORM_addrx = 0x1b, + DW_FORM_ref_sup4 = 0x1c, + DW_FORM_strp_sup = 0x1d, + DW_FORM_data16 = 0x1e, + DW_FORM_line_strp = 0x1f, + DW_FORM_implicit_const = 0x21, + DW_FORM_loclistx = 0x22, + DW_FORM_rnglistx = 0x23, + DW_FORM_ref_sup8 = 0x24, + DW_FORM_strx1 = 0x25, + DW_FORM_strx2 = 0x26, + DW_FORM_strx3 = 0x27, + DW_FORM_strx4 = 0x28, + DW_FORM_addrx1 = 0x29, + DW_FORM_addrx2 = 0x2a, + DW_FORM_addrx3 = 0x2b, + DW_FORM_addrx4 = 0x2c, + DW_FORM_GNU_addr_index = 0x1f01, + DW_FORM_GNU_str_index = 0x1f02, + DW_FORM_GNU_ref_alt = 0x1f20, + DW_FORM_GNU_strp_alt = 0x1f21 +}; + +enum dwarf_attribute { + DW_AT_sibling = 0x01, + DW_AT_location = 0x02, + DW_AT_name = 0x03, + DW_AT_ordering = 0x09, + DW_AT_subscr_data = 0x0a, + DW_AT_byte_size = 0x0b, + DW_AT_bit_offset = 0x0c, + DW_AT_bit_size = 0x0d, + DW_AT_element_list = 0x0f, + DW_AT_stmt_list = 0x10, + DW_AT_low_pc = 0x11, + DW_AT_high_pc = 0x12, + DW_AT_language = 0x13, + DW_AT_member = 0x14, + DW_AT_discr = 0x15, + DW_AT_discr_value = 0x16, + DW_AT_visibility = 0x17, + DW_AT_import = 0x18, + DW_AT_string_length = 0x19, + DW_AT_common_reference = 0x1a, + DW_AT_comp_dir = 0x1b, + DW_AT_const_value = 0x1c, + DW_AT_containing_type = 0x1d, + DW_AT_default_value = 0x1e, + DW_AT_inline = 0x20, + DW_AT_is_optional = 0x21, + DW_AT_lower_bound = 0x22, + DW_AT_producer = 0x25, + DW_AT_prototyped = 0x27, + DW_AT_return_addr = 0x2a, + DW_AT_start_scope = 0x2c, + DW_AT_bit_stride = 0x2e, + DW_AT_upper_bound = 0x2f, + DW_AT_abstract_origin = 0x31, + DW_AT_accessibility = 0x32, + DW_AT_address_class = 0x33, + DW_AT_artificial = 0x34, + DW_AT_base_types = 0x35, + DW_AT_calling_convention = 0x36, + DW_AT_count = 0x37, + DW_AT_data_member_location = 0x38, + DW_AT_decl_column = 0x39, + DW_AT_decl_file = 0x3a, + DW_AT_decl_line = 0x3b, + DW_AT_declaration = 0x3c, + DW_AT_discr_list = 0x3d, + DW_AT_encoding = 0x3e, + DW_AT_external = 0x3f, + DW_AT_frame_base = 0x40, + DW_AT_friend = 0x41, + DW_AT_identifier_case = 0x42, + DW_AT_macro_info = 0x43, + DW_AT_namelist_items = 0x44, + DW_AT_priority = 0x45, + DW_AT_segment = 0x46, + DW_AT_specification = 0x47, + DW_AT_static_link = 0x48, + DW_AT_type = 0x49, + DW_AT_use_location = 0x4a, + DW_AT_variable_parameter = 0x4b, + DW_AT_virtuality = 0x4c, + DW_AT_vtable_elem_location = 0x4d, + DW_AT_allocated = 0x4e, + DW_AT_associated = 0x4f, + DW_AT_data_location = 0x50, + DW_AT_byte_stride = 0x51, + DW_AT_entry_pc = 0x52, + DW_AT_use_UTF8 = 0x53, + DW_AT_extension = 0x54, + DW_AT_ranges = 0x55, + DW_AT_trampoline = 0x56, + DW_AT_call_column = 0x57, + DW_AT_call_file = 0x58, + DW_AT_call_line = 0x59, + DW_AT_description = 0x5a, + DW_AT_binary_scale = 0x5b, + DW_AT_decimal_scale = 0x5c, + DW_AT_small = 0x5d, + DW_AT_decimal_sign = 0x5e, + DW_AT_digit_count = 0x5f, + DW_AT_picture_string = 0x60, + DW_AT_mutable = 0x61, + DW_AT_threads_scaled = 0x62, + DW_AT_explicit = 0x63, + DW_AT_object_pointer = 0x64, + DW_AT_endianity = 0x65, + DW_AT_elemental = 0x66, + DW_AT_pure = 0x67, + DW_AT_recursive = 0x68, + DW_AT_signature = 0x69, + DW_AT_main_subprogram = 0x6a, + DW_AT_data_bit_offset = 0x6b, + DW_AT_const_expr = 0x6c, + DW_AT_enum_class = 0x6d, + DW_AT_linkage_name = 0x6e, + DW_AT_string_length_bit_size = 0x6f, + DW_AT_string_length_byte_size = 0x70, + DW_AT_rank = 0x71, + DW_AT_str_offsets_base = 0x72, + DW_AT_addr_base = 0x73, + DW_AT_rnglists_base = 0x74, + DW_AT_dwo_name = 0x76, + DW_AT_reference = 0x77, + DW_AT_rvalue_reference = 0x78, + DW_AT_macros = 0x79, + DW_AT_call_all_calls = 0x7a, + DW_AT_call_all_source_calls = 0x7b, + DW_AT_call_all_tail_calls = 0x7c, + DW_AT_call_return_pc = 0x7d, + DW_AT_call_value = 0x7e, + DW_AT_call_origin = 0x7f, + DW_AT_call_parameter = 0x80, + DW_AT_call_pc = 0x81, + DW_AT_call_tail_call = 0x82, + DW_AT_call_target = 0x83, + DW_AT_call_target_clobbered = 0x84, + DW_AT_call_data_location = 0x85, + DW_AT_call_data_value = 0x86, + DW_AT_noreturn = 0x87, + DW_AT_alignment = 0x88, + DW_AT_export_symbols = 0x89, + DW_AT_deleted = 0x8a, + DW_AT_defaulted = 0x8b, + DW_AT_loclists_base = 0x8c, + DW_AT_lo_user = 0x2000, + DW_AT_hi_user = 0x3fff, + DW_AT_MIPS_fde = 0x2001, + DW_AT_MIPS_loop_begin = 0x2002, + DW_AT_MIPS_tail_loop_begin = 0x2003, + DW_AT_MIPS_epilog_begin = 0x2004, + DW_AT_MIPS_loop_unroll_factor = 0x2005, + DW_AT_MIPS_software_pipeline_depth = 0x2006, + DW_AT_MIPS_linkage_name = 0x2007, + DW_AT_MIPS_stride = 0x2008, + DW_AT_MIPS_abstract_name = 0x2009, + DW_AT_MIPS_clone_origin = 0x200a, + DW_AT_MIPS_has_inlines = 0x200b, + DW_AT_HP_block_index = 0x2000, + DW_AT_HP_unmodifiable = 0x2001, + DW_AT_HP_prologue = 0x2005, + DW_AT_HP_epilogue = 0x2008, + DW_AT_HP_actuals_stmt_list = 0x2010, + DW_AT_HP_proc_per_section = 0x2011, + DW_AT_HP_raw_data_ptr = 0x2012, + DW_AT_HP_pass_by_reference = 0x2013, + DW_AT_HP_opt_level = 0x2014, + DW_AT_HP_prof_version_id = 0x2015, + DW_AT_HP_opt_flags = 0x2016, + DW_AT_HP_cold_region_low_pc = 0x2017, + DW_AT_HP_cold_region_high_pc = 0x2018, + DW_AT_HP_all_variables_modifiable = 0x2019, + DW_AT_HP_linkage_name = 0x201a, + DW_AT_HP_prof_flags = 0x201b, + DW_AT_HP_unit_name = 0x201f, + DW_AT_HP_unit_size = 0x2020, + DW_AT_HP_widened_byte_size = 0x2021, + DW_AT_HP_definition_points = 0x2022, + DW_AT_HP_default_location = 0x2023, + DW_AT_HP_is_result_param = 0x2029, + DW_AT_sf_names = 0x2101, + DW_AT_src_info = 0x2102, + DW_AT_mac_info = 0x2103, + DW_AT_src_coords = 0x2104, + DW_AT_body_begin = 0x2105, + DW_AT_body_end = 0x2106, + DW_AT_GNU_vector = 0x2107, + DW_AT_GNU_guarded_by = 0x2108, + DW_AT_GNU_pt_guarded_by = 0x2109, + DW_AT_GNU_guarded = 0x210a, + DW_AT_GNU_pt_guarded = 0x210b, + DW_AT_GNU_locks_excluded = 0x210c, + DW_AT_GNU_exclusive_locks_required = 0x210d, + DW_AT_GNU_shared_locks_required = 0x210e, + DW_AT_GNU_odr_signature = 0x210f, + DW_AT_GNU_template_name = 0x2110, + DW_AT_GNU_call_site_value = 0x2111, + DW_AT_GNU_call_site_data_value = 0x2112, + DW_AT_GNU_call_site_target = 0x2113, + DW_AT_GNU_call_site_target_clobbered = 0x2114, + DW_AT_GNU_tail_call = 0x2115, + DW_AT_GNU_all_tail_call_sites = 0x2116, + DW_AT_GNU_all_call_sites = 0x2117, + DW_AT_GNU_all_source_call_sites = 0x2118, + DW_AT_GNU_macros = 0x2119, + DW_AT_GNU_deleted = 0x211a, + DW_AT_GNU_dwo_name = 0x2130, + DW_AT_GNU_dwo_id = 0x2131, + DW_AT_GNU_ranges_base = 0x2132, + DW_AT_GNU_addr_base = 0x2133, + DW_AT_GNU_pubnames = 0x2134, + DW_AT_GNU_pubtypes = 0x2135, + DW_AT_GNU_discriminator = 0x2136, + DW_AT_GNU_locviews = 0x2137, + DW_AT_GNU_entry_view = 0x2138, + DW_AT_VMS_rtnbeg_pd_address = 0x2201, + DW_AT_use_GNAT_descriptive_type = 0x2301, + DW_AT_GNAT_descriptive_type = 0x2302, + DW_AT_GNU_numerator = 0x2303, + DW_AT_GNU_denominator = 0x2304, + DW_AT_GNU_bias = 0x2305, + DW_AT_upc_threads_scaled = 0x3210, + DW_AT_PGI_lbase = 0x3a00, + DW_AT_PGI_soffset = 0x3a01, + DW_AT_PGI_lstride = 0x3a02, + DW_AT_APPLE_optimized = 0x3fe1, + DW_AT_APPLE_flags = 0x3fe2, + DW_AT_APPLE_isa = 0x3fe3, + DW_AT_APPLE_block = 0x3fe4, + DW_AT_APPLE_major_runtime_vers = 0x3fe5, + DW_AT_APPLE_runtime_class = 0x3fe6, + DW_AT_APPLE_omit_frame_ptr = 0x3fe7, + DW_AT_APPLE_property_name = 0x3fe8, + DW_AT_APPLE_property_getter = 0x3fe9, + DW_AT_APPLE_property_setter = 0x3fea, + DW_AT_APPLE_property_attribute = 0x3feb, + DW_AT_APPLE_objc_complete_type = 0x3fec, + DW_AT_APPLE_property = 0x3fed +}; + +enum dwarf_line_number_op { + DW_LNS_extended_op = 0x0, + DW_LNS_copy = 0x1, + DW_LNS_advance_pc = 0x2, + DW_LNS_advance_line = 0x3, + DW_LNS_set_file = 0x4, + DW_LNS_set_column = 0x5, + DW_LNS_negate_stmt = 0x6, + DW_LNS_set_basic_block = 0x7, + DW_LNS_const_add_pc = 0x8, + DW_LNS_fixed_advance_pc = 0x9, + DW_LNS_set_prologue_end = 0xa, + DW_LNS_set_epilogue_begin = 0xb, + DW_LNS_set_isa = 0xc, +}; + +enum dwarf_extended_line_number_op { + DW_LNE_end_sequence = 0x1, + DW_LNE_set_address = 0x2, + DW_LNE_define_file = 0x3, + DW_LNE_set_discriminator = 0x4, +}; + +enum dwarf_line_number_content_type { + DW_LNCT_path = 0x1, + DW_LNCT_directory_index = 0x2, + DW_LNCT_timestamp = 0x3, + DW_LNCT_size = 0x4, + DW_LNCT_MD5 = 0x5, + DW_LNCT_lo_user = 0x2000, + DW_LNCT_hi_user = 0x3fff +}; + +enum dwarf_range_list_entry { + DW_RLE_end_of_list = 0x00, + DW_RLE_base_addressx = 0x01, + DW_RLE_startx_endx = 0x02, + DW_RLE_startx_length = 0x03, + DW_RLE_offset_pair = 0x04, + DW_RLE_base_address = 0x05, + DW_RLE_start_end = 0x06, + DW_RLE_start_length = 0x07 +}; + +enum dwarf_unit_type { + DW_UT_compile = 0x01, + DW_UT_type = 0x02, + DW_UT_partial = 0x03, + DW_UT_skeleton = 0x04, + DW_UT_split_compile = 0x05, + DW_UT_split_type = 0x06, + DW_UT_lo_user = 0x80, + DW_UT_hi_user = 0xff +}; + +#if !defined(HAVE_DECL_STRNLEN) || !HAVE_DECL_STRNLEN + +/* If strnlen is not declared, provide our own version. */ + +static size_t +xstrnlen (const char *s, size_t maxlen) +{ + size_t i; + + for (i = 0; i < maxlen; ++i) + if (s[i] == '\0') + break; + return i; +} + +#define strnlen xstrnlen + +#endif + +/* A buffer to read DWARF info. */ + +struct dwarf_buf +{ + /* Buffer name for error messages. */ + const char *name; + /* Start of the buffer. */ + const unsigned char *start; + /* Next byte to read. */ + const unsigned char *buf; + /* The number of bytes remaining. */ + size_t left; + /* Whether the data is big-endian. */ + int is_bigendian; + /* Error callback routine. */ + backtrace_error_callback error_callback; + /* Data for error_callback. */ + void *data; + /* Non-zero if we've reported an underflow error. */ + int reported_underflow; +}; + +/* A single attribute in a DWARF abbreviation. */ + +struct attr +{ + /* The attribute name. */ + enum dwarf_attribute name; + /* The attribute form. */ + enum dwarf_form form; + /* The attribute value, for DW_FORM_implicit_const. */ + int64_t val; +}; + +/* A single DWARF abbreviation. */ + +struct abbrev +{ + /* The abbrev code--the number used to refer to the abbrev. */ + uint64_t code; + /* The entry tag. */ + enum dwarf_tag tag; + /* Non-zero if this abbrev has child entries. */ + int has_children; + /* The number of attributes. */ + size_t num_attrs; + /* The attributes. */ + struct attr *attrs; +}; + +/* The DWARF abbreviations for a compilation unit. This structure + only exists while reading the compilation unit. Most DWARF readers + seem to a hash table to map abbrev ID's to abbrev entries. + However, we primarily care about GCC, and GCC simply issues ID's in + numerical order starting at 1. So we simply keep a sorted vector, + and try to just look up the code. */ + +struct abbrevs +{ + /* The number of abbrevs in the vector. */ + size_t num_abbrevs; + /* The abbrevs, sorted by the code field. */ + struct abbrev *abbrevs; +}; + +/* The different kinds of attribute values. */ + +enum attr_val_encoding +{ + /* No attribute value. */ + ATTR_VAL_NONE, + /* An address. */ + ATTR_VAL_ADDRESS, + /* An index into the .debug_addr section, whose value is relative to + the DW_AT_addr_base attribute of the compilation unit. */ + ATTR_VAL_ADDRESS_INDEX, + /* A unsigned integer. */ + ATTR_VAL_UINT, + /* A sigd integer. */ + ATTR_VAL_SINT, + /* A string. */ + ATTR_VAL_STRING, + /* An index into the .debug_str_offsets section. */ + ATTR_VAL_STRING_INDEX, + /* An offset to other data in the containing unit. */ + ATTR_VAL_REF_UNIT, + /* An offset to other data within the .debug_info section. */ + ATTR_VAL_REF_INFO, + /* An offset to other data within the alt .debug_info section. */ + ATTR_VAL_REF_ALT_INFO, + /* An offset to data in some other section. */ + ATTR_VAL_REF_SECTION, + /* A type signature. */ + ATTR_VAL_REF_TYPE, + /* An index into the .debug_rnglists section. */ + ATTR_VAL_RNGLISTS_INDEX, + /* A block of data (not represented). */ + ATTR_VAL_BLOCK, + /* An expression (not represented). */ + ATTR_VAL_EXPR, +}; + +/* An attribute value. */ + +struct attr_val +{ + /* How the value is stored in the field u. */ + enum attr_val_encoding encoding; + union + { + /* ATTR_VAL_ADDRESS*, ATTR_VAL_UINT, ATTR_VAL_REF*. */ + uint64_t uint; + /* ATTR_VAL_SINT. */ + int64_t sint; + /* ATTR_VAL_STRING. */ + const char *string; + /* ATTR_VAL_BLOCK not stored. */ + } u; +}; + +/* The line number program header. */ + +struct line_header +{ + /* The version of the line number information. */ + int version; + /* Address size. */ + int addrsize; + /* The minimum instruction length. */ + unsigned int min_insn_len; + /* The maximum number of ops per instruction. */ + unsigned int max_ops_per_insn; + /* The line base for special opcodes. */ + int line_base; + /* The line range for special opcodes. */ + unsigned int line_range; + /* The opcode base--the first special opcode. */ + unsigned int opcode_base; + /* Opcode lengths, indexed by opcode - 1. */ + const unsigned char *opcode_lengths; + /* The number of directory entries. */ + size_t dirs_count; + /* The directory entries. */ + const char **dirs; + /* The number of filenames. */ + size_t filenames_count; + /* The filenames. */ + const char **filenames; +}; + +/* A format description from a line header. */ + +struct line_header_format +{ + int lnct; /* LNCT code. */ + enum dwarf_form form; /* Form of entry data. */ +}; + +/* Map a single PC value to a file/line. We will keep a vector of + these sorted by PC value. Each file/line will be correct from the + PC up to the PC of the next entry if there is one. We allocate one + extra entry at the end so that we can use bsearch. */ + +struct line +{ + /* PC. */ + uintptr_t pc; + /* File name. Many entries in the array are expected to point to + the same file name. */ + const char *filename; + /* Line number. */ + int lineno; + /* Index of the object in the original array read from the DWARF + section, before it has been sorted. The index makes it possible + to use Quicksort and maintain stability. */ + int idx; +}; + +/* A growable vector of line number information. This is used while + reading the line numbers. */ + +struct line_vector +{ + /* Memory. This is an array of struct line. */ + struct backtrace_vector vec; + /* Number of valid mappings. */ + size_t count; +}; + +/* A function described in the debug info. */ + +struct function +{ + /* The name of the function. */ + const char *name; + /* If this is an inlined function, the filename of the call + site. */ + const char *caller_filename; + /* If this is an inlined function, the line number of the call + site. */ + int caller_lineno; + /* Map PC ranges to inlined functions. */ + struct function_addrs *function_addrs; + size_t function_addrs_count; +}; + +/* An address range for a function. This maps a PC value to a + specific function. */ + +struct function_addrs +{ + /* Range is LOW <= PC < HIGH. */ + uintptr_t low; + uintptr_t high; + /* Function for this address range. */ + struct function *function; +}; + +/* A growable vector of function address ranges. */ + +struct function_vector +{ + /* Memory. This is an array of struct function_addrs. */ + struct backtrace_vector vec; + /* Number of address ranges present. */ + size_t count; +}; + +/* A DWARF compilation unit. This only holds the information we need + to map a PC to a file and line. */ + +struct unit +{ + /* The first entry for this compilation unit. */ + const unsigned char *unit_data; + /* The length of the data for this compilation unit. */ + size_t unit_data_len; + /* The offset of UNIT_DATA from the start of the information for + this compilation unit. */ + size_t unit_data_offset; + /* Offset of the start of the compilation unit from the start of the + .debug_info section. */ + size_t low_offset; + /* Offset of the end of the compilation unit from the start of the + .debug_info section. */ + size_t high_offset; + /* DWARF version. */ + int version; + /* Whether unit is DWARF64. */ + int is_dwarf64; + /* Address size. */ + int addrsize; + /* Offset into line number information. */ + off_t lineoff; + /* Offset of compilation unit in .debug_str_offsets. */ + uint64_t str_offsets_base; + /* Offset of compilation unit in .debug_addr. */ + uint64_t addr_base; + /* Offset of compilation unit in .debug_rnglists. */ + uint64_t rnglists_base; + /* Primary source file. */ + const char *filename; + /* Compilation command working directory. */ + const char *comp_dir; + /* Absolute file name, only set if needed. */ + const char *abs_filename; + /* The abbreviations for this unit. */ + struct abbrevs abbrevs; + + /* The fields above this point are read in during initialization and + may be accessed freely. The fields below this point are read in + as needed, and therefore require care, as different threads may + try to initialize them simultaneously. */ + + /* PC to line number mapping. This is NULL if the values have not + been read. This is (struct line *) -1 if there was an error + reading the values. */ + struct line *lines; + /* Number of entries in lines. */ + size_t lines_count; + /* PC ranges to function. */ + struct function_addrs *function_addrs; + size_t function_addrs_count; +}; + +/* An address range for a compilation unit. This maps a PC value to a + specific compilation unit. Note that we invert the representation + in DWARF: instead of listing the units and attaching a list of + ranges, we list the ranges and have each one point to the unit. + This lets us do a binary search to find the unit. */ + +struct unit_addrs +{ + /* Range is LOW <= PC < HIGH. */ + uintptr_t low; + uintptr_t high; + /* Compilation unit for this address range. */ + struct unit *u; +}; + +/* A growable vector of compilation unit address ranges. */ + +struct unit_addrs_vector +{ + /* Memory. This is an array of struct unit_addrs. */ + struct backtrace_vector vec; + /* Number of address ranges present. */ + size_t count; +}; + +/* A growable vector of compilation unit pointer. */ + +struct unit_vector +{ + struct backtrace_vector vec; + size_t count; +}; + +/* The information we need to map a PC to a file and line. */ + +struct dwarf_data +{ + /* The data for the next file we know about. */ + struct dwarf_data *next; + /* The data for .gnu_debugaltlink. */ + struct dwarf_data *altlink; + /* The base address for this file. */ + uintptr_t base_address; + /* A sorted list of address ranges. */ + struct unit_addrs *addrs; + /* Number of address ranges in list. */ + size_t addrs_count; + /* A sorted list of units. */ + struct unit **units; + /* Number of units in the list. */ + size_t units_count; + /* The unparsed DWARF debug data. */ + struct dwarf_sections dwarf_sections; + /* Whether the data is big-endian or not. */ + int is_bigendian; + /* A vector used for function addresses. We keep this here so that + we can grow the vector as we read more functions. */ + struct function_vector fvec; +}; + +/* Report an error for a DWARF buffer. */ + +static void +dwarf_buf_error (struct dwarf_buf *buf, const char *msg, int errnum) +{ + char b[200]; + + snprintf (b, sizeof b, "%s in %s at %d", + msg, buf->name, (int) (buf->buf - buf->start)); + buf->error_callback (buf->data, b, errnum); +} + +/* Require at least COUNT bytes in BUF. Return 1 if all is well, 0 on + error. */ + +static int +require (struct dwarf_buf *buf, size_t count) +{ + if (buf->left >= count) + return 1; + + if (!buf->reported_underflow) + { + dwarf_buf_error (buf, "DWARF underflow", 0); + buf->reported_underflow = 1; + } + + return 0; +} + +/* Advance COUNT bytes in BUF. Return 1 if all is well, 0 on + error. */ + +static int +advance (struct dwarf_buf *buf, size_t count) +{ + if (!require (buf, count)) + return 0; + buf->buf += count; + buf->left -= count; + return 1; +} + +/* Read one zero-terminated string from BUF and advance past the string. */ + +static const char * +read_string (struct dwarf_buf *buf) +{ + const char *p = (const char *)buf->buf; + size_t len = strnlen (p, buf->left); + + /* - If len == left, we ran out of buffer before finding the zero terminator. + Generate an error by advancing len + 1. + - If len < left, advance by len + 1 to skip past the zero terminator. */ + size_t count = len + 1; + + if (!advance (buf, count)) + return NULL; + + return p; +} + +/* Read one byte from BUF and advance 1 byte. */ + +static unsigned char +read_byte (struct dwarf_buf *buf) +{ + const unsigned char *p = buf->buf; + + if (!advance (buf, 1)) + return 0; + return p[0]; +} + +/* Read a signed char from BUF and advance 1 byte. */ + +static signed char +read_sbyte (struct dwarf_buf *buf) +{ + const unsigned char *p = buf->buf; + + if (!advance (buf, 1)) + return 0; + return (*p ^ 0x80) - 0x80; +} + +/* Read a uint16 from BUF and advance 2 bytes. */ + +static uint16_t +read_uint16 (struct dwarf_buf *buf) +{ + const unsigned char *p = buf->buf; + + if (!advance (buf, 2)) + return 0; + if (buf->is_bigendian) + return ((uint16_t) p[0] << 8) | (uint16_t) p[1]; + else + return ((uint16_t) p[1] << 8) | (uint16_t) p[0]; +} + +/* Read a 24 bit value from BUF and advance 3 bytes. */ + +static uint32_t +read_uint24 (struct dwarf_buf *buf) +{ + const unsigned char *p = buf->buf; + + if (!advance (buf, 3)) + return 0; + if (buf->is_bigendian) + return (((uint32_t) p[0] << 16) | ((uint32_t) p[1] << 8) + | (uint32_t) p[2]); + else + return (((uint32_t) p[2] << 16) | ((uint32_t) p[1] << 8) + | (uint32_t) p[0]); +} + +/* Read a uint32 from BUF and advance 4 bytes. */ + +static uint32_t +read_uint32 (struct dwarf_buf *buf) +{ + const unsigned char *p = buf->buf; + + if (!advance (buf, 4)) + return 0; + if (buf->is_bigendian) + return (((uint32_t) p[0] << 24) | ((uint32_t) p[1] << 16) + | ((uint32_t) p[2] << 8) | (uint32_t) p[3]); + else + return (((uint32_t) p[3] << 24) | ((uint32_t) p[2] << 16) + | ((uint32_t) p[1] << 8) | (uint32_t) p[0]); +} + +/* Read a uint64 from BUF and advance 8 bytes. */ + +static uint64_t +read_uint64 (struct dwarf_buf *buf) +{ + const unsigned char *p = buf->buf; + + if (!advance (buf, 8)) + return 0; + if (buf->is_bigendian) + return (((uint64_t) p[0] << 56) | ((uint64_t) p[1] << 48) + | ((uint64_t) p[2] << 40) | ((uint64_t) p[3] << 32) + | ((uint64_t) p[4] << 24) | ((uint64_t) p[5] << 16) + | ((uint64_t) p[6] << 8) | (uint64_t) p[7]); + else + return (((uint64_t) p[7] << 56) | ((uint64_t) p[6] << 48) + | ((uint64_t) p[5] << 40) | ((uint64_t) p[4] << 32) + | ((uint64_t) p[3] << 24) | ((uint64_t) p[2] << 16) + | ((uint64_t) p[1] << 8) | (uint64_t) p[0]); +} + +/* Read an offset from BUF and advance the appropriate number of + bytes. */ + +static uint64_t +read_offset (struct dwarf_buf *buf, int is_dwarf64) +{ + if (is_dwarf64) + return read_uint64 (buf); + else + return read_uint32 (buf); +} + +/* Read an address from BUF and advance the appropriate number of + bytes. */ + +static uint64_t +read_address (struct dwarf_buf *buf, int addrsize) +{ + switch (addrsize) + { + case 1: + return read_byte (buf); + case 2: + return read_uint16 (buf); + case 4: + return read_uint32 (buf); + case 8: + return read_uint64 (buf); + default: + dwarf_buf_error (buf, "unrecognized address size", 0); + return 0; + } +} + +/* Return whether a value is the highest possible address, given the + address size. */ + +static int +is_highest_address (uint64_t address, int addrsize) +{ + switch (addrsize) + { + case 1: + return address == (unsigned char) -1; + case 2: + return address == (uint16_t) -1; + case 4: + return address == (uint32_t) -1; + case 8: + return address == (uint64_t) -1; + default: + return 0; + } +} + +/* Read an unsigned LEB128 number. */ + +static uint64_t +read_uleb128 (struct dwarf_buf *buf) +{ + uint64_t ret; + unsigned int shift; + int overflow; + unsigned char b; + + ret = 0; + shift = 0; + overflow = 0; + do + { + const unsigned char *p; + + p = buf->buf; + if (!advance (buf, 1)) + return 0; + b = *p; + if (shift < 64) + ret |= ((uint64_t) (b & 0x7f)) << shift; + else if (!overflow) + { + dwarf_buf_error (buf, "LEB128 overflows uint64_t", 0); + overflow = 1; + } + shift += 7; + } + while ((b & 0x80) != 0); + + return ret; +} + +/* Read a signed LEB128 number. */ + +static int64_t +read_sleb128 (struct dwarf_buf *buf) +{ + uint64_t val; + unsigned int shift; + int overflow; + unsigned char b; + + val = 0; + shift = 0; + overflow = 0; + do + { + const unsigned char *p; + + p = buf->buf; + if (!advance (buf, 1)) + return 0; + b = *p; + if (shift < 64) + val |= ((uint64_t) (b & 0x7f)) << shift; + else if (!overflow) + { + dwarf_buf_error (buf, "signed LEB128 overflows uint64_t", 0); + overflow = 1; + } + shift += 7; + } + while ((b & 0x80) != 0); + + if ((b & 0x40) != 0 && shift < 64) + val |= ((uint64_t) -1) << shift; + + return (int64_t) val; +} + +/* Return the length of an LEB128 number. */ + +static size_t +leb128_len (const unsigned char *p) +{ + size_t ret; + + ret = 1; + while ((*p & 0x80) != 0) + { + ++p; + ++ret; + } + return ret; +} + +/* Read initial_length from BUF and advance the appropriate number of bytes. */ + +static uint64_t +read_initial_length (struct dwarf_buf *buf, int *is_dwarf64) +{ + uint64_t len; + + len = read_uint32 (buf); + if (len == 0xffffffff) + { + len = read_uint64 (buf); + *is_dwarf64 = 1; + } + else + *is_dwarf64 = 0; + + return len; +} + +/* Free an abbreviations structure. */ + +static void +free_abbrevs (struct backtrace_state *state, struct abbrevs *abbrevs, + backtrace_error_callback error_callback, void *data) +{ + size_t i; + + for (i = 0; i < abbrevs->num_abbrevs; ++i) + backtrace_free (state, abbrevs->abbrevs[i].attrs, + abbrevs->abbrevs[i].num_attrs * sizeof (struct attr), + error_callback, data); + backtrace_free (state, abbrevs->abbrevs, + abbrevs->num_abbrevs * sizeof (struct abbrev), + error_callback, data); + abbrevs->num_abbrevs = 0; + abbrevs->abbrevs = NULL; +} + +/* Read an attribute value. Returns 1 on success, 0 on failure. If + the value can be represented as a uint64_t, sets *VAL and sets + *IS_VALID to 1. We don't try to store the value of other attribute + forms, because we don't care about them. */ + +static int +read_attribute (enum dwarf_form form, uint64_t implicit_val, + struct dwarf_buf *buf, int is_dwarf64, int version, + int addrsize, const struct dwarf_sections *dwarf_sections, + struct dwarf_data *altlink, struct attr_val *val) +{ + /* Avoid warnings about val.u.FIELD may be used uninitialized if + this function is inlined. The warnings aren't valid but can + occur because the different fields are set and used + conditionally. */ + memset (val, 0, sizeof *val); + + switch (form) + { + case DW_FORM_addr: + val->encoding = ATTR_VAL_ADDRESS; + val->u.uint = read_address (buf, addrsize); + return 1; + case DW_FORM_block2: + val->encoding = ATTR_VAL_BLOCK; + return advance (buf, read_uint16 (buf)); + case DW_FORM_block4: + val->encoding = ATTR_VAL_BLOCK; + return advance (buf, read_uint32 (buf)); + case DW_FORM_data2: + val->encoding = ATTR_VAL_UINT; + val->u.uint = read_uint16 (buf); + return 1; + case DW_FORM_data4: + val->encoding = ATTR_VAL_UINT; + val->u.uint = read_uint32 (buf); + return 1; + case DW_FORM_data8: + val->encoding = ATTR_VAL_UINT; + val->u.uint = read_uint64 (buf); + return 1; + case DW_FORM_data16: + val->encoding = ATTR_VAL_BLOCK; + return advance (buf, 16); + case DW_FORM_string: + val->encoding = ATTR_VAL_STRING; + val->u.string = read_string (buf); + return val->u.string == NULL ? 0 : 1; + case DW_FORM_block: + val->encoding = ATTR_VAL_BLOCK; + return advance (buf, read_uleb128 (buf)); + case DW_FORM_block1: + val->encoding = ATTR_VAL_BLOCK; + return advance (buf, read_byte (buf)); + case DW_FORM_data1: + val->encoding = ATTR_VAL_UINT; + val->u.uint = read_byte (buf); + return 1; + case DW_FORM_flag: + val->encoding = ATTR_VAL_UINT; + val->u.uint = read_byte (buf); + return 1; + case DW_FORM_sdata: + val->encoding = ATTR_VAL_SINT; + val->u.sint = read_sleb128 (buf); + return 1; + case DW_FORM_strp: + { + uint64_t offset; + + offset = read_offset (buf, is_dwarf64); + if (offset >= dwarf_sections->size[DEBUG_STR]) + { + dwarf_buf_error (buf, "DW_FORM_strp out of range", 0); + return 0; + } + val->encoding = ATTR_VAL_STRING; + val->u.string = + (const char *) dwarf_sections->data[DEBUG_STR] + offset; + return 1; + } + case DW_FORM_line_strp: + { + uint64_t offset; + + offset = read_offset (buf, is_dwarf64); + if (offset >= dwarf_sections->size[DEBUG_LINE_STR]) + { + dwarf_buf_error (buf, "DW_FORM_line_strp out of range", 0); + return 0; + } + val->encoding = ATTR_VAL_STRING; + val->u.string = + (const char *) dwarf_sections->data[DEBUG_LINE_STR] + offset; + return 1; + } + case DW_FORM_udata: + val->encoding = ATTR_VAL_UINT; + val->u.uint = read_uleb128 (buf); + return 1; + case DW_FORM_ref_addr: + val->encoding = ATTR_VAL_REF_INFO; + if (version == 2) + val->u.uint = read_address (buf, addrsize); + else + val->u.uint = read_offset (buf, is_dwarf64); + return 1; + case DW_FORM_ref1: + val->encoding = ATTR_VAL_REF_UNIT; + val->u.uint = read_byte (buf); + return 1; + case DW_FORM_ref2: + val->encoding = ATTR_VAL_REF_UNIT; + val->u.uint = read_uint16 (buf); + return 1; + case DW_FORM_ref4: + val->encoding = ATTR_VAL_REF_UNIT; + val->u.uint = read_uint32 (buf); + return 1; + case DW_FORM_ref8: + val->encoding = ATTR_VAL_REF_UNIT; + val->u.uint = read_uint64 (buf); + return 1; + case DW_FORM_ref_udata: + val->encoding = ATTR_VAL_REF_UNIT; + val->u.uint = read_uleb128 (buf); + return 1; + case DW_FORM_indirect: + { + uint64_t form; + + form = read_uleb128 (buf); + if (form == DW_FORM_implicit_const) + { + dwarf_buf_error (buf, + "DW_FORM_indirect to DW_FORM_implicit_const", + 0); + return 0; + } + return read_attribute ((enum dwarf_form) form, 0, buf, is_dwarf64, + version, addrsize, dwarf_sections, altlink, + val); + } + case DW_FORM_sec_offset: + val->encoding = ATTR_VAL_REF_SECTION; + val->u.uint = read_offset (buf, is_dwarf64); + return 1; + case DW_FORM_exprloc: + val->encoding = ATTR_VAL_EXPR; + return advance (buf, read_uleb128 (buf)); + case DW_FORM_flag_present: + val->encoding = ATTR_VAL_UINT; + val->u.uint = 1; + return 1; + case DW_FORM_ref_sig8: + val->encoding = ATTR_VAL_REF_TYPE; + val->u.uint = read_uint64 (buf); + return 1; + case DW_FORM_strx: case DW_FORM_strx1: case DW_FORM_strx2: + case DW_FORM_strx3: case DW_FORM_strx4: + { + uint64_t offset; + + switch (form) + { + case DW_FORM_strx: + offset = read_uleb128 (buf); + break; + case DW_FORM_strx1: + offset = read_byte (buf); + break; + case DW_FORM_strx2: + offset = read_uint16 (buf); + break; + case DW_FORM_strx3: + offset = read_uint24 (buf); + break; + case DW_FORM_strx4: + offset = read_uint32 (buf); + break; + default: + /* This case can't happen. */ + return 0; + } + val->encoding = ATTR_VAL_STRING_INDEX; + val->u.uint = offset; + return 1; + } + case DW_FORM_addrx: case DW_FORM_addrx1: case DW_FORM_addrx2: + case DW_FORM_addrx3: case DW_FORM_addrx4: + { + uint64_t offset; + + switch (form) + { + case DW_FORM_addrx: + offset = read_uleb128 (buf); + break; + case DW_FORM_addrx1: + offset = read_byte (buf); + break; + case DW_FORM_addrx2: + offset = read_uint16 (buf); + break; + case DW_FORM_addrx3: + offset = read_uint24 (buf); + break; + case DW_FORM_addrx4: + offset = read_uint32 (buf); + break; + default: + /* This case can't happen. */ + return 0; + } + val->encoding = ATTR_VAL_ADDRESS_INDEX; + val->u.uint = offset; + return 1; + } + case DW_FORM_ref_sup4: + val->encoding = ATTR_VAL_REF_SECTION; + val->u.uint = read_uint32 (buf); + return 1; + case DW_FORM_ref_sup8: + val->encoding = ATTR_VAL_REF_SECTION; + val->u.uint = read_uint64 (buf); + return 1; + case DW_FORM_implicit_const: + val->encoding = ATTR_VAL_UINT; + val->u.uint = implicit_val; + return 1; + case DW_FORM_loclistx: + /* We don't distinguish this from DW_FORM_sec_offset. It + * shouldn't matter since we don't care about loclists. */ + val->encoding = ATTR_VAL_REF_SECTION; + val->u.uint = read_uleb128 (buf); + return 1; + case DW_FORM_rnglistx: + val->encoding = ATTR_VAL_RNGLISTS_INDEX; + val->u.uint = read_uleb128 (buf); + return 1; + case DW_FORM_GNU_addr_index: + val->encoding = ATTR_VAL_REF_SECTION; + val->u.uint = read_uleb128 (buf); + return 1; + case DW_FORM_GNU_str_index: + val->encoding = ATTR_VAL_REF_SECTION; + val->u.uint = read_uleb128 (buf); + return 1; + case DW_FORM_GNU_ref_alt: + val->u.uint = read_offset (buf, is_dwarf64); + if (altlink == NULL) + { + val->encoding = ATTR_VAL_NONE; + return 1; + } + val->encoding = ATTR_VAL_REF_ALT_INFO; + return 1; + case DW_FORM_strp_sup: case DW_FORM_GNU_strp_alt: + { + uint64_t offset; + + offset = read_offset (buf, is_dwarf64); + if (altlink == NULL) + { + val->encoding = ATTR_VAL_NONE; + return 1; + } + if (offset >= altlink->dwarf_sections.size[DEBUG_STR]) + { + dwarf_buf_error (buf, "DW_FORM_strp_sup out of range", 0); + return 0; + } + val->encoding = ATTR_VAL_STRING; + val->u.string = + (const char *) altlink->dwarf_sections.data[DEBUG_STR] + offset; + return 1; + } + default: + dwarf_buf_error (buf, "unrecognized DWARF form", -1); + return 0; + } +} + +/* If we can determine the value of a string attribute, set *STRING to + point to the string. Return 1 on success, 0 on error. If we don't + know the value, we consider that a success, and we don't change + *STRING. An error is only reported for some sort of out of range + offset. */ + +static int +resolve_string (const struct dwarf_sections *dwarf_sections, int is_dwarf64, + int is_bigendian, uint64_t str_offsets_base, + const struct attr_val *val, + backtrace_error_callback error_callback, void *data, + const char **string) +{ + switch (val->encoding) + { + case ATTR_VAL_STRING: + *string = val->u.string; + return 1; + + case ATTR_VAL_STRING_INDEX: + { + uint64_t offset; + struct dwarf_buf offset_buf; + + offset = val->u.uint * (is_dwarf64 ? 8 : 4) + str_offsets_base; + if (offset + (is_dwarf64 ? 8 : 4) + > dwarf_sections->size[DEBUG_STR_OFFSETS]) + { + error_callback (data, "DW_FORM_strx value out of range", 0); + return 0; + } + + offset_buf.name = ".debug_str_offsets"; + offset_buf.start = dwarf_sections->data[DEBUG_STR_OFFSETS]; + offset_buf.buf = dwarf_sections->data[DEBUG_STR_OFFSETS] + offset; + offset_buf.left = dwarf_sections->size[DEBUG_STR_OFFSETS] - offset; + offset_buf.is_bigendian = is_bigendian; + offset_buf.error_callback = error_callback; + offset_buf.data = data; + offset_buf.reported_underflow = 0; + + offset = read_offset (&offset_buf, is_dwarf64); + if (offset >= dwarf_sections->size[DEBUG_STR]) + { + dwarf_buf_error (&offset_buf, + "DW_FORM_strx offset out of range", + 0); + return 0; + } + *string = (const char *) dwarf_sections->data[DEBUG_STR] + offset; + return 1; + } + + default: + return 1; + } +} + +/* Set *ADDRESS to the real address for a ATTR_VAL_ADDRESS_INDEX. + Return 1 on success, 0 on error. */ + +static int +resolve_addr_index (const struct dwarf_sections *dwarf_sections, + uint64_t addr_base, int addrsize, int is_bigendian, + uint64_t addr_index, + backtrace_error_callback error_callback, void *data, + uintptr_t *address) +{ + uint64_t offset; + struct dwarf_buf addr_buf; + + offset = addr_index * addrsize + addr_base; + if (offset + addrsize > dwarf_sections->size[DEBUG_ADDR]) + { + error_callback (data, "DW_FORM_addrx value out of range", 0); + return 0; + } + + addr_buf.name = ".debug_addr"; + addr_buf.start = dwarf_sections->data[DEBUG_ADDR]; + addr_buf.buf = dwarf_sections->data[DEBUG_ADDR] + offset; + addr_buf.left = dwarf_sections->size[DEBUG_ADDR] - offset; + addr_buf.is_bigendian = is_bigendian; + addr_buf.error_callback = error_callback; + addr_buf.data = data; + addr_buf.reported_underflow = 0; + + *address = (uintptr_t) read_address (&addr_buf, addrsize); + return 1; +} + +/* Compare a unit offset against a unit for bsearch. */ + +static int +units_search (const void *vkey, const void *ventry) +{ + const size_t *key = (const size_t *) vkey; + const struct unit *entry = *((const struct unit *const *) ventry); + size_t offset; + + offset = *key; + if (offset < entry->low_offset) + return -1; + else if (offset >= entry->high_offset) + return 1; + else + return 0; +} + +/* Find a unit in PU containing OFFSET. */ + +static struct unit * +find_unit (struct unit **pu, size_t units_count, size_t offset) +{ + struct unit **u; + u = (struct unit**)bsearch (&offset, pu, units_count, sizeof (struct unit *), units_search); + return u == NULL ? NULL : *u; +} + +/* Compare function_addrs for qsort. When ranges are nested, make the + smallest one sort last. */ + +static int +function_addrs_compare (const void *v1, const void *v2) +{ + const struct function_addrs *a1 = (const struct function_addrs *) v1; + const struct function_addrs *a2 = (const struct function_addrs *) v2; + + if (a1->low < a2->low) + return -1; + if (a1->low > a2->low) + return 1; + if (a1->high < a2->high) + return 1; + if (a1->high > a2->high) + return -1; + return strcmp (a1->function->name, a2->function->name); +} + +/* Compare a PC against a function_addrs for bsearch. We always + allocate an entra entry at the end of the vector, so that this + routine can safely look at the next entry. Note that if there are + multiple ranges containing PC, which one will be returned is + unpredictable. We compensate for that in dwarf_fileline. */ + +static int +function_addrs_search (const void *vkey, const void *ventry) +{ + const uintptr_t *key = (const uintptr_t *) vkey; + const struct function_addrs *entry = (const struct function_addrs *) ventry; + uintptr_t pc; + + pc = *key; + if (pc < entry->low) + return -1; + else if (pc > (entry + 1)->low) + return 1; + else + return 0; +} + +/* Add a new compilation unit address range to a vector. This is + called via add_ranges. Returns 1 on success, 0 on failure. */ + +static int +add_unit_addr (struct backtrace_state *state, void *rdata, + uintptr_t lowpc, uintptr_t highpc, + backtrace_error_callback error_callback, void *data, + void *pvec) +{ + struct unit *u = (struct unit *) rdata; + struct unit_addrs_vector *vec = (struct unit_addrs_vector *) pvec; + struct unit_addrs *p; + + /* Try to merge with the last entry. */ + if (vec->count > 0) + { + p = (struct unit_addrs *) vec->vec.base + (vec->count - 1); + if ((lowpc == p->high || lowpc == p->high + 1) + && u == p->u) + { + if (highpc > p->high) + p->high = highpc; + return 1; + } + } + + p = ((struct unit_addrs *) + backtrace_vector_grow (state, sizeof (struct unit_addrs), + error_callback, data, &vec->vec)); + if (p == NULL) + return 0; + + p->low = lowpc; + p->high = highpc; + p->u = u; + + ++vec->count; + + return 1; +} + +/* Compare unit_addrs for qsort. When ranges are nested, make the + smallest one sort last. */ + +static int +unit_addrs_compare (const void *v1, const void *v2) +{ + const struct unit_addrs *a1 = (const struct unit_addrs *) v1; + const struct unit_addrs *a2 = (const struct unit_addrs *) v2; + + if (a1->low < a2->low) + return -1; + if (a1->low > a2->low) + return 1; + if (a1->high < a2->high) + return 1; + if (a1->high > a2->high) + return -1; + if (a1->u->lineoff < a2->u->lineoff) + return -1; + if (a1->u->lineoff > a2->u->lineoff) + return 1; + return 0; +} + +/* Compare a PC against a unit_addrs for bsearch. We always allocate + an entry entry at the end of the vector, so that this routine can + safely look at the next entry. Note that if there are multiple + ranges containing PC, which one will be returned is unpredictable. + We compensate for that in dwarf_fileline. */ + +static int +unit_addrs_search (const void *vkey, const void *ventry) +{ + const uintptr_t *key = (const uintptr_t *) vkey; + const struct unit_addrs *entry = (const struct unit_addrs *) ventry; + uintptr_t pc; + + pc = *key; + if (pc < entry->low) + return -1; + else if (pc > (entry + 1)->low) + return 1; + else + return 0; +} + +/* Sort the line vector by PC. We want a stable sort here to maintain + the order of lines for the same PC values. Since the sequence is + being sorted in place, their addresses cannot be relied on to + maintain stability. That is the purpose of the index member. */ + +static int +line_compare (const void *v1, const void *v2) +{ + const struct line *ln1 = (const struct line *) v1; + const struct line *ln2 = (const struct line *) v2; + + if (ln1->pc < ln2->pc) + return -1; + else if (ln1->pc > ln2->pc) + return 1; + else if (ln1->idx < ln2->idx) + return -1; + else if (ln1->idx > ln2->idx) + return 1; + else + return 0; +} + +/* Find a PC in a line vector. We always allocate an extra entry at + the end of the lines vector, so that this routine can safely look + at the next entry. Note that when there are multiple mappings for + the same PC value, this will return the last one. */ + +static int +line_search (const void *vkey, const void *ventry) +{ + const uintptr_t *key = (const uintptr_t *) vkey; + const struct line *entry = (const struct line *) ventry; + uintptr_t pc; + + pc = *key; + if (pc < entry->pc) + return -1; + else if (pc >= (entry + 1)->pc) + return 1; + else + return 0; +} + +/* Sort the abbrevs by the abbrev code. This function is passed to + both qsort and bsearch. */ + +static int +abbrev_compare (const void *v1, const void *v2) +{ + const struct abbrev *a1 = (const struct abbrev *) v1; + const struct abbrev *a2 = (const struct abbrev *) v2; + + if (a1->code < a2->code) + return -1; + else if (a1->code > a2->code) + return 1; + else + { + /* This really shouldn't happen. It means there are two + different abbrevs with the same code, and that means we don't + know which one lookup_abbrev should return. */ + return 0; + } +} + +/* Read the abbreviation table for a compilation unit. Returns 1 on + success, 0 on failure. */ + +static int +read_abbrevs (struct backtrace_state *state, uint64_t abbrev_offset, + const unsigned char *dwarf_abbrev, size_t dwarf_abbrev_size, + int is_bigendian, backtrace_error_callback error_callback, + void *data, struct abbrevs *abbrevs) +{ + struct dwarf_buf abbrev_buf; + struct dwarf_buf count_buf; + size_t num_abbrevs; + + abbrevs->num_abbrevs = 0; + abbrevs->abbrevs = NULL; + + if (abbrev_offset >= dwarf_abbrev_size) + { + error_callback (data, "abbrev offset out of range", 0); + return 0; + } + + abbrev_buf.name = ".debug_abbrev"; + abbrev_buf.start = dwarf_abbrev; + abbrev_buf.buf = dwarf_abbrev + abbrev_offset; + abbrev_buf.left = dwarf_abbrev_size - abbrev_offset; + abbrev_buf.is_bigendian = is_bigendian; + abbrev_buf.error_callback = error_callback; + abbrev_buf.data = data; + abbrev_buf.reported_underflow = 0; + + /* Count the number of abbrevs in this list. */ + + count_buf = abbrev_buf; + num_abbrevs = 0; + while (read_uleb128 (&count_buf) != 0) + { + if (count_buf.reported_underflow) + return 0; + ++num_abbrevs; + // Skip tag. + read_uleb128 (&count_buf); + // Skip has_children. + read_byte (&count_buf); + // Skip attributes. + while (read_uleb128 (&count_buf) != 0) + { + uint64_t form; + + form = read_uleb128 (&count_buf); + if ((enum dwarf_form) form == DW_FORM_implicit_const) + read_sleb128 (&count_buf); + } + // Skip form of last attribute. + read_uleb128 (&count_buf); + } + + if (count_buf.reported_underflow) + return 0; + + if (num_abbrevs == 0) + return 1; + + abbrevs->abbrevs = ((struct abbrev *) + backtrace_alloc (state, + num_abbrevs * sizeof (struct abbrev), + error_callback, data)); + if (abbrevs->abbrevs == NULL) + return 0; + abbrevs->num_abbrevs = num_abbrevs; + memset (abbrevs->abbrevs, 0, num_abbrevs * sizeof (struct abbrev)); + + num_abbrevs = 0; + while (1) + { + uint64_t code; + struct abbrev a; + size_t num_attrs; + struct attr *attrs; + + if (abbrev_buf.reported_underflow) + goto fail; + + code = read_uleb128 (&abbrev_buf); + if (code == 0) + break; + + a.code = code; + a.tag = (enum dwarf_tag) read_uleb128 (&abbrev_buf); + a.has_children = read_byte (&abbrev_buf); + + count_buf = abbrev_buf; + num_attrs = 0; + while (read_uleb128 (&count_buf) != 0) + { + uint64_t form; + + ++num_attrs; + form = read_uleb128 (&count_buf); + if ((enum dwarf_form) form == DW_FORM_implicit_const) + read_sleb128 (&count_buf); + } + + if (num_attrs == 0) + { + attrs = NULL; + read_uleb128 (&abbrev_buf); + read_uleb128 (&abbrev_buf); + } + else + { + attrs = ((struct attr *) + backtrace_alloc (state, num_attrs * sizeof *attrs, + error_callback, data)); + if (attrs == NULL) + goto fail; + num_attrs = 0; + while (1) + { + uint64_t name; + uint64_t form; + + name = read_uleb128 (&abbrev_buf); + form = read_uleb128 (&abbrev_buf); + if (name == 0) + break; + attrs[num_attrs].name = (enum dwarf_attribute) name; + attrs[num_attrs].form = (enum dwarf_form) form; + if ((enum dwarf_form) form == DW_FORM_implicit_const) + attrs[num_attrs].val = read_sleb128 (&abbrev_buf); + else + attrs[num_attrs].val = 0; + ++num_attrs; + } + } + + a.num_attrs = num_attrs; + a.attrs = attrs; + + abbrevs->abbrevs[num_abbrevs] = a; + ++num_abbrevs; + } + + backtrace_qsort (abbrevs->abbrevs, abbrevs->num_abbrevs, + sizeof (struct abbrev), abbrev_compare); + + return 1; + + fail: + free_abbrevs (state, abbrevs, error_callback, data); + return 0; +} + +/* Return the abbrev information for an abbrev code. */ + +static const struct abbrev * +lookup_abbrev (struct abbrevs *abbrevs, uint64_t code, + backtrace_error_callback error_callback, void *data) +{ + struct abbrev key; + void *p; + + /* With GCC, where abbrevs are simply numbered in order, we should + be able to just look up the entry. */ + if (code - 1 < abbrevs->num_abbrevs + && abbrevs->abbrevs[code - 1].code == code) + return &abbrevs->abbrevs[code - 1]; + + /* Otherwise we have to search. */ + memset (&key, 0, sizeof key); + key.code = code; + p = bsearch (&key, abbrevs->abbrevs, abbrevs->num_abbrevs, + sizeof (struct abbrev), abbrev_compare); + if (p == NULL) + { + error_callback (data, "invalid abbreviation code", 0); + return NULL; + } + return (const struct abbrev *) p; +} + +/* This struct is used to gather address range information while + reading attributes. We use this while building a mapping from + address ranges to compilation units and then again while mapping + from address ranges to function entries. Normally either + lowpc/highpc is set or ranges is set. */ + +struct pcrange { + uintptr_t lowpc; /* The low PC value. */ + int have_lowpc; /* Whether a low PC value was found. */ + int lowpc_is_addr_index; /* Whether lowpc is in .debug_addr. */ + uintptr_t highpc; /* The high PC value. */ + int have_highpc; /* Whether a high PC value was found. */ + int highpc_is_relative; /* Whether highpc is relative to lowpc. */ + int highpc_is_addr_index; /* Whether highpc is in .debug_addr. */ + uint64_t ranges; /* Offset in ranges section. */ + int have_ranges; /* Whether ranges is valid. */ + int ranges_is_index; /* Whether ranges is DW_FORM_rnglistx. */ +}; + +/* Update PCRANGE from an attribute value. */ + +static void +update_pcrange (const struct attr* attr, const struct attr_val* val, + struct pcrange *pcrange) +{ + switch (attr->name) + { + case DW_AT_low_pc: + if (val->encoding == ATTR_VAL_ADDRESS) + { + pcrange->lowpc = (uintptr_t) val->u.uint; + pcrange->have_lowpc = 1; + } + else if (val->encoding == ATTR_VAL_ADDRESS_INDEX) + { + pcrange->lowpc = (uintptr_t) val->u.uint; + pcrange->have_lowpc = 1; + pcrange->lowpc_is_addr_index = 1; + } + break; + + case DW_AT_high_pc: + if (val->encoding == ATTR_VAL_ADDRESS) + { + pcrange->highpc = (uintptr_t) val->u.uint; + pcrange->have_highpc = 1; + } + else if (val->encoding == ATTR_VAL_UINT) + { + pcrange->highpc = (uintptr_t) val->u.uint; + pcrange->have_highpc = 1; + pcrange->highpc_is_relative = 1; + } + else if (val->encoding == ATTR_VAL_ADDRESS_INDEX) + { + pcrange->highpc = (uintptr_t) val->u.uint; + pcrange->have_highpc = 1; + pcrange->highpc_is_addr_index = 1; + } + break; + + case DW_AT_ranges: + if (val->encoding == ATTR_VAL_UINT + || val->encoding == ATTR_VAL_REF_SECTION) + { + pcrange->ranges = val->u.uint; + pcrange->have_ranges = 1; + } + else if (val->encoding == ATTR_VAL_RNGLISTS_INDEX) + { + pcrange->ranges = val->u.uint; + pcrange->have_ranges = 1; + pcrange->ranges_is_index = 1; + } + break; + + default: + break; + } +} + +/* Call ADD_RANGE for a low/high PC pair. Returns 1 on success, 0 on + error. */ + +static int +add_low_high_range (struct backtrace_state *state, + const struct dwarf_sections *dwarf_sections, + uintptr_t base_address, int is_bigendian, + struct unit *u, const struct pcrange *pcrange, + int (*add_range) (struct backtrace_state *state, + void *rdata, uintptr_t lowpc, + uintptr_t highpc, + backtrace_error_callback error_callback, + void *data, void *vec), + void *rdata, + backtrace_error_callback error_callback, void *data, + void *vec) +{ + uintptr_t lowpc; + uintptr_t highpc; + + lowpc = pcrange->lowpc; + if (pcrange->lowpc_is_addr_index) + { + if (!resolve_addr_index (dwarf_sections, u->addr_base, u->addrsize, + is_bigendian, lowpc, error_callback, data, + &lowpc)) + return 0; + } + + highpc = pcrange->highpc; + if (pcrange->highpc_is_addr_index) + { + if (!resolve_addr_index (dwarf_sections, u->addr_base, u->addrsize, + is_bigendian, highpc, error_callback, data, + &highpc)) + return 0; + } + if (pcrange->highpc_is_relative) + highpc += lowpc; + + /* Add in the base address of the module when recording PC values, + so that we can look up the PC directly. */ + lowpc += base_address; + highpc += base_address; + + return add_range (state, rdata, lowpc, highpc, error_callback, data, vec); +} + +/* Call ADD_RANGE for each range read from .debug_ranges, as used in + DWARF versions 2 through 4. */ + +static int +add_ranges_from_ranges ( + struct backtrace_state *state, + const struct dwarf_sections *dwarf_sections, + uintptr_t base_address, int is_bigendian, + struct unit *u, uintptr_t base, + const struct pcrange *pcrange, + int (*add_range) (struct backtrace_state *state, void *rdata, + uintptr_t lowpc, uintptr_t highpc, + backtrace_error_callback error_callback, void *data, + void *vec), + void *rdata, + backtrace_error_callback error_callback, void *data, + void *vec) +{ + struct dwarf_buf ranges_buf; + + if (pcrange->ranges >= dwarf_sections->size[DEBUG_RANGES]) + { + error_callback (data, "ranges offset out of range", 0); + return 0; + } + + ranges_buf.name = ".debug_ranges"; + ranges_buf.start = dwarf_sections->data[DEBUG_RANGES]; + ranges_buf.buf = dwarf_sections->data[DEBUG_RANGES] + pcrange->ranges; + ranges_buf.left = dwarf_sections->size[DEBUG_RANGES] - pcrange->ranges; + ranges_buf.is_bigendian = is_bigendian; + ranges_buf.error_callback = error_callback; + ranges_buf.data = data; + ranges_buf.reported_underflow = 0; + + while (1) + { + uint64_t low; + uint64_t high; + + if (ranges_buf.reported_underflow) + return 0; + + low = read_address (&ranges_buf, u->addrsize); + high = read_address (&ranges_buf, u->addrsize); + + if (low == 0 && high == 0) + break; + + if (is_highest_address (low, u->addrsize)) + base = (uintptr_t) high; + else + { + if (!add_range (state, rdata, + (uintptr_t) low + base + base_address, + (uintptr_t) high + base + base_address, + error_callback, data, vec)) + return 0; + } + } + + if (ranges_buf.reported_underflow) + return 0; + + return 1; +} + +/* Call ADD_RANGE for each range read from .debug_rnglists, as used in + DWARF version 5. */ + +static int +add_ranges_from_rnglists ( + struct backtrace_state *state, + const struct dwarf_sections *dwarf_sections, + uintptr_t base_address, int is_bigendian, + struct unit *u, uintptr_t base, + const struct pcrange *pcrange, + int (*add_range) (struct backtrace_state *state, void *rdata, + uintptr_t lowpc, uintptr_t highpc, + backtrace_error_callback error_callback, void *data, + void *vec), + void *rdata, + backtrace_error_callback error_callback, void *data, + void *vec) +{ + uint64_t offset; + struct dwarf_buf rnglists_buf; + + if (!pcrange->ranges_is_index) + offset = pcrange->ranges; + else + offset = u->rnglists_base + pcrange->ranges * (u->is_dwarf64 ? 8 : 4); + if (offset >= dwarf_sections->size[DEBUG_RNGLISTS]) + { + error_callback (data, "rnglists offset out of range", 0); + return 0; + } + + rnglists_buf.name = ".debug_rnglists"; + rnglists_buf.start = dwarf_sections->data[DEBUG_RNGLISTS]; + rnglists_buf.buf = dwarf_sections->data[DEBUG_RNGLISTS] + offset; + rnglists_buf.left = dwarf_sections->size[DEBUG_RNGLISTS] - offset; + rnglists_buf.is_bigendian = is_bigendian; + rnglists_buf.error_callback = error_callback; + rnglists_buf.data = data; + rnglists_buf.reported_underflow = 0; + + if (pcrange->ranges_is_index) + { + offset = read_offset (&rnglists_buf, u->is_dwarf64); + offset += u->rnglists_base; + if (offset >= dwarf_sections->size[DEBUG_RNGLISTS]) + { + error_callback (data, "rnglists index offset out of range", 0); + return 0; + } + rnglists_buf.buf = dwarf_sections->data[DEBUG_RNGLISTS] + offset; + rnglists_buf.left = dwarf_sections->size[DEBUG_RNGLISTS] - offset; + } + + while (1) + { + unsigned char rle; + + rle = read_byte (&rnglists_buf); + if (rle == DW_RLE_end_of_list) + break; + switch (rle) + { + case DW_RLE_base_addressx: + { + uint64_t index; + + index = read_uleb128 (&rnglists_buf); + if (!resolve_addr_index (dwarf_sections, u->addr_base, + u->addrsize, is_bigendian, index, + error_callback, data, &base)) + return 0; + } + break; + + case DW_RLE_startx_endx: + { + uint64_t index; + uintptr_t low; + uintptr_t high; + + index = read_uleb128 (&rnglists_buf); + if (!resolve_addr_index (dwarf_sections, u->addr_base, + u->addrsize, is_bigendian, index, + error_callback, data, &low)) + return 0; + index = read_uleb128 (&rnglists_buf); + if (!resolve_addr_index (dwarf_sections, u->addr_base, + u->addrsize, is_bigendian, index, + error_callback, data, &high)) + return 0; + if (!add_range (state, rdata, low + base_address, + high + base_address, error_callback, data, + vec)) + return 0; + } + break; + + case DW_RLE_startx_length: + { + uint64_t index; + uintptr_t low; + uintptr_t length; + + index = read_uleb128 (&rnglists_buf); + if (!resolve_addr_index (dwarf_sections, u->addr_base, + u->addrsize, is_bigendian, index, + error_callback, data, &low)) + return 0; + length = read_uleb128 (&rnglists_buf); + low += base_address; + if (!add_range (state, rdata, low, low + length, + error_callback, data, vec)) + return 0; + } + break; + + case DW_RLE_offset_pair: + { + uint64_t low; + uint64_t high; + + low = read_uleb128 (&rnglists_buf); + high = read_uleb128 (&rnglists_buf); + if (!add_range (state, rdata, low + base + base_address, + high + base + base_address, + error_callback, data, vec)) + return 0; + } + break; + + case DW_RLE_base_address: + base = (uintptr_t) read_address (&rnglists_buf, u->addrsize); + break; + + case DW_RLE_start_end: + { + uintptr_t low; + uintptr_t high; + + low = (uintptr_t) read_address (&rnglists_buf, u->addrsize); + high = (uintptr_t) read_address (&rnglists_buf, u->addrsize); + if (!add_range (state, rdata, low + base_address, + high + base_address, error_callback, data, + vec)) + return 0; + } + break; + + case DW_RLE_start_length: + { + uintptr_t low; + uintptr_t length; + + low = (uintptr_t) read_address (&rnglists_buf, u->addrsize); + length = (uintptr_t) read_uleb128 (&rnglists_buf); + low += base_address; + if (!add_range (state, rdata, low, low + length, + error_callback, data, vec)) + return 0; + } + break; + + default: + dwarf_buf_error (&rnglists_buf, "unrecognized DW_RLE value", -1); + return 0; + } + } + + if (rnglists_buf.reported_underflow) + return 0; + + return 1; +} + +/* Call ADD_RANGE for each lowpc/highpc pair in PCRANGE. RDATA is + passed to ADD_RANGE, and is either a struct unit * or a struct + function *. VEC is the vector we are adding ranges to, and is + either a struct unit_addrs_vector * or a struct function_vector *. + Returns 1 on success, 0 on error. */ + +static int +add_ranges (struct backtrace_state *state, + const struct dwarf_sections *dwarf_sections, + uintptr_t base_address, int is_bigendian, + struct unit *u, uintptr_t base, const struct pcrange *pcrange, + int (*add_range) (struct backtrace_state *state, void *rdata, + uintptr_t lowpc, uintptr_t highpc, + backtrace_error_callback error_callback, + void *data, void *vec), + void *rdata, + backtrace_error_callback error_callback, void *data, + void *vec) +{ + if (pcrange->have_lowpc && pcrange->have_highpc) + return add_low_high_range (state, dwarf_sections, base_address, + is_bigendian, u, pcrange, add_range, rdata, + error_callback, data, vec); + + if (!pcrange->have_ranges) + { + /* Did not find any address ranges to add. */ + return 1; + } + + if (u->version < 5) + return add_ranges_from_ranges (state, dwarf_sections, base_address, + is_bigendian, u, base, pcrange, add_range, + rdata, error_callback, data, vec); + else + return add_ranges_from_rnglists (state, dwarf_sections, base_address, + is_bigendian, u, base, pcrange, add_range, + rdata, error_callback, data, vec); +} + +/* Find the address range covered by a compilation unit, reading from + UNIT_BUF and adding values to U. Returns 1 if all data could be + read, 0 if there is some error. */ + +static int +find_address_ranges (struct backtrace_state *state, uintptr_t base_address, + struct dwarf_buf *unit_buf, + const struct dwarf_sections *dwarf_sections, + int is_bigendian, struct dwarf_data *altlink, + backtrace_error_callback error_callback, void *data, + struct unit *u, struct unit_addrs_vector *addrs, + enum dwarf_tag *unit_tag) +{ + while (unit_buf->left > 0) + { + uint64_t code; + const struct abbrev *abbrev; + struct pcrange pcrange; + struct attr_val name_val; + int have_name_val; + struct attr_val comp_dir_val; + int have_comp_dir_val; + size_t i; + + code = read_uleb128 (unit_buf); + if (code == 0) + return 1; + + abbrev = lookup_abbrev (&u->abbrevs, code, error_callback, data); + if (abbrev == NULL) + return 0; + + if (unit_tag != NULL) + *unit_tag = abbrev->tag; + + memset (&pcrange, 0, sizeof pcrange); + memset (&name_val, 0, sizeof name_val); + have_name_val = 0; + memset (&comp_dir_val, 0, sizeof comp_dir_val); + have_comp_dir_val = 0; + for (i = 0; i < abbrev->num_attrs; ++i) + { + struct attr_val val; + + if (!read_attribute (abbrev->attrs[i].form, abbrev->attrs[i].val, + unit_buf, u->is_dwarf64, u->version, + u->addrsize, dwarf_sections, altlink, &val)) + return 0; + + switch (abbrev->attrs[i].name) + { + case DW_AT_low_pc: case DW_AT_high_pc: case DW_AT_ranges: + update_pcrange (&abbrev->attrs[i], &val, &pcrange); + break; + + case DW_AT_stmt_list: + if ((abbrev->tag == DW_TAG_compile_unit + || abbrev->tag == DW_TAG_skeleton_unit) + && (val.encoding == ATTR_VAL_UINT + || val.encoding == ATTR_VAL_REF_SECTION)) + u->lineoff = val.u.uint; + break; + + case DW_AT_name: + if (abbrev->tag == DW_TAG_compile_unit + || abbrev->tag == DW_TAG_skeleton_unit) + { + name_val = val; + have_name_val = 1; + } + break; + + case DW_AT_comp_dir: + if (abbrev->tag == DW_TAG_compile_unit + || abbrev->tag == DW_TAG_skeleton_unit) + { + comp_dir_val = val; + have_comp_dir_val = 1; + } + break; + + case DW_AT_str_offsets_base: + if ((abbrev->tag == DW_TAG_compile_unit + || abbrev->tag == DW_TAG_skeleton_unit) + && val.encoding == ATTR_VAL_REF_SECTION) + u->str_offsets_base = val.u.uint; + break; + + case DW_AT_addr_base: + if ((abbrev->tag == DW_TAG_compile_unit + || abbrev->tag == DW_TAG_skeleton_unit) + && val.encoding == ATTR_VAL_REF_SECTION) + u->addr_base = val.u.uint; + break; + + case DW_AT_rnglists_base: + if ((abbrev->tag == DW_TAG_compile_unit + || abbrev->tag == DW_TAG_skeleton_unit) + && val.encoding == ATTR_VAL_REF_SECTION) + u->rnglists_base = val.u.uint; + break; + + default: + break; + } + } + + // Resolve strings after we're sure that we have seen + // DW_AT_str_offsets_base. + if (have_name_val) + { + if (!resolve_string (dwarf_sections, u->is_dwarf64, is_bigendian, + u->str_offsets_base, &name_val, + error_callback, data, &u->filename)) + return 0; + } + if (have_comp_dir_val) + { + if (!resolve_string (dwarf_sections, u->is_dwarf64, is_bigendian, + u->str_offsets_base, &comp_dir_val, + error_callback, data, &u->comp_dir)) + return 0; + } + + if (abbrev->tag == DW_TAG_compile_unit + || abbrev->tag == DW_TAG_subprogram + || abbrev->tag == DW_TAG_skeleton_unit) + { + if (!add_ranges (state, dwarf_sections, base_address, + is_bigendian, u, pcrange.lowpc, &pcrange, + add_unit_addr, (void *) u, error_callback, data, + (void *) addrs)) + return 0; + + /* If we found the PC range in the DW_TAG_compile_unit or + DW_TAG_skeleton_unit, we can stop now. */ + if ((abbrev->tag == DW_TAG_compile_unit + || abbrev->tag == DW_TAG_skeleton_unit) + && (pcrange.have_ranges + || (pcrange.have_lowpc && pcrange.have_highpc))) + return 1; + } + + if (abbrev->has_children) + { + if (!find_address_ranges (state, base_address, unit_buf, + dwarf_sections, is_bigendian, altlink, + error_callback, data, u, addrs, NULL)) + return 0; + } + } + + return 1; +} + +/* Build a mapping from address ranges to the compilation units where + the line number information for that range can be found. Returns 1 + on success, 0 on failure. */ + +static int +build_address_map (struct backtrace_state *state, uintptr_t base_address, + const struct dwarf_sections *dwarf_sections, + int is_bigendian, struct dwarf_data *altlink, + backtrace_error_callback error_callback, void *data, + struct unit_addrs_vector *addrs, + struct unit_vector *unit_vec) +{ + struct dwarf_buf info; + struct backtrace_vector units; + size_t units_count; + size_t i; + struct unit **pu; + size_t unit_offset = 0; + struct unit_addrs *pa; + + memset (&addrs->vec, 0, sizeof addrs->vec); + memset (&unit_vec->vec, 0, sizeof unit_vec->vec); + addrs->count = 0; + unit_vec->count = 0; + + /* Read through the .debug_info section. FIXME: Should we use the + .debug_aranges section? gdb and addr2line don't use it, but I'm + not sure why. */ + + info.name = ".debug_info"; + info.start = dwarf_sections->data[DEBUG_INFO]; + info.buf = info.start; + info.left = dwarf_sections->size[DEBUG_INFO]; + info.is_bigendian = is_bigendian; + info.error_callback = error_callback; + info.data = data; + info.reported_underflow = 0; + + memset (&units, 0, sizeof units); + units_count = 0; + + while (info.left > 0) + { + const unsigned char *unit_data_start; + uint64_t len; + int is_dwarf64; + struct dwarf_buf unit_buf; + int version; + int unit_type; + uint64_t abbrev_offset; + int addrsize; + struct unit *u; + enum dwarf_tag unit_tag; + + if (info.reported_underflow) + goto fail; + + unit_data_start = info.buf; + + len = read_initial_length (&info, &is_dwarf64); + unit_buf = info; + unit_buf.left = len; + + if (!advance (&info, len)) + goto fail; + + version = read_uint16 (&unit_buf); + if (version < 2 || version > 5) + { + dwarf_buf_error (&unit_buf, "unrecognized DWARF version", -1); + goto fail; + } + + if (version < 5) + unit_type = 0; + else + { + unit_type = read_byte (&unit_buf); + if (unit_type == DW_UT_type || unit_type == DW_UT_split_type) + { + /* This unit doesn't have anything we need. */ + continue; + } + } + + pu = ((struct unit **) + backtrace_vector_grow (state, sizeof (struct unit *), + error_callback, data, &units)); + if (pu == NULL) + goto fail; + + u = ((struct unit *) + backtrace_alloc (state, sizeof *u, error_callback, data)); + if (u == NULL) + goto fail; + + *pu = u; + ++units_count; + + if (version < 5) + addrsize = 0; /* Set below. */ + else + addrsize = read_byte (&unit_buf); + + memset (&u->abbrevs, 0, sizeof u->abbrevs); + abbrev_offset = read_offset (&unit_buf, is_dwarf64); + if (!read_abbrevs (state, abbrev_offset, + dwarf_sections->data[DEBUG_ABBREV], + dwarf_sections->size[DEBUG_ABBREV], + is_bigendian, error_callback, data, &u->abbrevs)) + goto fail; + + if (version < 5) + addrsize = read_byte (&unit_buf); + + switch (unit_type) + { + case 0: + break; + case DW_UT_compile: case DW_UT_partial: + break; + case DW_UT_skeleton: case DW_UT_split_compile: + read_uint64 (&unit_buf); /* dwo_id */ + break; + default: + break; + } + + u->low_offset = unit_offset; + unit_offset += len + (is_dwarf64 ? 12 : 4); + u->high_offset = unit_offset; + u->unit_data = unit_buf.buf; + u->unit_data_len = unit_buf.left; + u->unit_data_offset = unit_buf.buf - unit_data_start; + u->version = version; + u->is_dwarf64 = is_dwarf64; + u->addrsize = addrsize; + u->filename = NULL; + u->comp_dir = NULL; + u->abs_filename = NULL; + u->lineoff = 0; + u->str_offsets_base = 0; + u->addr_base = 0; + u->rnglists_base = 0; + + /* The actual line number mappings will be read as needed. */ + u->lines = NULL; + u->lines_count = 0; + u->function_addrs = NULL; + u->function_addrs_count = 0; + + if (!find_address_ranges (state, base_address, &unit_buf, dwarf_sections, + is_bigendian, altlink, error_callback, data, + u, addrs, &unit_tag)) + goto fail; + + if (unit_buf.reported_underflow) + goto fail; + } + if (info.reported_underflow) + goto fail; + + /* Add a trailing addrs entry, but don't include it in addrs->count. */ + pa = ((struct unit_addrs *) + backtrace_vector_grow (state, sizeof (struct unit_addrs), + error_callback, data, &addrs->vec)); + if (pa == NULL) + goto fail; + pa->low = 0; + --pa->low; + pa->high = pa->low; + pa->u = NULL; + + unit_vec->vec = units; + unit_vec->count = units_count; + return 1; + + fail: + if (units_count > 0) + { + pu = (struct unit **) units.base; + for (i = 0; i < units_count; i++) + { + free_abbrevs (state, &pu[i]->abbrevs, error_callback, data); + backtrace_free (state, pu[i], sizeof **pu, error_callback, data); + } + backtrace_vector_free (state, &units, error_callback, data); + } + if (addrs->count > 0) + { + backtrace_vector_free (state, &addrs->vec, error_callback, data); + addrs->count = 0; + } + return 0; +} + +/* Add a new mapping to the vector of line mappings that we are + building. Returns 1 on success, 0 on failure. */ + +static int +add_line (struct backtrace_state *state, struct dwarf_data *ddata, + uintptr_t pc, const char *filename, int lineno, + backtrace_error_callback error_callback, void *data, + struct line_vector *vec) +{ + struct line *ln; + + /* If we are adding the same mapping, ignore it. This can happen + when using discriminators. */ + if (vec->count > 0) + { + ln = (struct line *) vec->vec.base + (vec->count - 1); + if (pc == ln->pc && filename == ln->filename && lineno == ln->lineno) + return 1; + } + + ln = ((struct line *) + backtrace_vector_grow (state, sizeof (struct line), error_callback, + data, &vec->vec)); + if (ln == NULL) + return 0; + + /* Add in the base address here, so that we can look up the PC + directly. */ + ln->pc = pc + ddata->base_address; + + ln->filename = filename; + ln->lineno = lineno; + ln->idx = vec->count; + + ++vec->count; + + return 1; +} + +/* Free the line header information. */ + +static void +free_line_header (struct backtrace_state *state, struct line_header *hdr, + backtrace_error_callback error_callback, void *data) +{ + if (hdr->dirs_count != 0) + backtrace_free (state, hdr->dirs, hdr->dirs_count * sizeof (const char *), + error_callback, data); + backtrace_free (state, hdr->filenames, + hdr->filenames_count * sizeof (char *), + error_callback, data); +} + +/* Read the directories and file names for a line header for version + 2, setting fields in HDR. Return 1 on success, 0 on failure. */ + +static int +read_v2_paths (struct backtrace_state *state, struct unit *u, + struct dwarf_buf *hdr_buf, struct line_header *hdr) +{ + const unsigned char *p; + const unsigned char *pend; + size_t i; + + /* Count the number of directory entries. */ + hdr->dirs_count = 0; + p = hdr_buf->buf; + pend = p + hdr_buf->left; + while (p < pend && *p != '\0') + { + p += strnlen((const char *) p, pend - p) + 1; + ++hdr->dirs_count; + } + + /* The index of the first entry in the list of directories is 1. Index 0 is + used for the current directory of the compilation. To simplify index + handling, we set entry 0 to the compilation unit directory. */ + ++hdr->dirs_count; + hdr->dirs = ((const char **) + backtrace_alloc (state, + hdr->dirs_count * sizeof (const char *), + hdr_buf->error_callback, + hdr_buf->data)); + if (hdr->dirs == NULL) + return 0; + + hdr->dirs[0] = u->comp_dir; + i = 1; + while (*hdr_buf->buf != '\0') + { + if (hdr_buf->reported_underflow) + return 0; + + hdr->dirs[i] = read_string (hdr_buf); + if (hdr->dirs[i] == NULL) + return 0; + ++i; + } + if (!advance (hdr_buf, 1)) + return 0; + + /* Count the number of file entries. */ + hdr->filenames_count = 0; + p = hdr_buf->buf; + pend = p + hdr_buf->left; + while (p < pend && *p != '\0') + { + p += strnlen ((const char *) p, pend - p) + 1; + p += leb128_len (p); + p += leb128_len (p); + p += leb128_len (p); + ++hdr->filenames_count; + } + + /* The index of the first entry in the list of file names is 1. Index 0 is + used for the DW_AT_name of the compilation unit. To simplify index + handling, we set entry 0 to the compilation unit file name. */ + ++hdr->filenames_count; + hdr->filenames = ((const char **) + backtrace_alloc (state, + hdr->filenames_count * sizeof (char *), + hdr_buf->error_callback, + hdr_buf->data)); + if (hdr->filenames == NULL) + return 0; + hdr->filenames[0] = u->filename; + i = 1; + while (*hdr_buf->buf != '\0') + { + const char *filename; + uint64_t dir_index; + + if (hdr_buf->reported_underflow) + return 0; + + filename = read_string (hdr_buf); + if (filename == NULL) + return 0; + dir_index = read_uleb128 (hdr_buf); + if (IS_ABSOLUTE_PATH (filename) + || (dir_index < hdr->dirs_count && hdr->dirs[dir_index] == NULL)) + hdr->filenames[i] = filename; + else + { + const char *dir; + size_t dir_len; + size_t filename_len; + char *s; + + if (dir_index < hdr->dirs_count) + dir = hdr->dirs[dir_index]; + else + { + dwarf_buf_error (hdr_buf, + ("invalid directory index in " + "line number program header"), + 0); + return 0; + } + dir_len = strlen (dir); + filename_len = strlen (filename); + s = ((char *) backtrace_alloc (state, dir_len + filename_len + 2, + hdr_buf->error_callback, + hdr_buf->data)); + if (s == NULL) + return 0; + memcpy (s, dir, dir_len); + /* FIXME: If we are on a DOS-based file system, and the + directory or the file name use backslashes, then we + should use a backslash here. */ + s[dir_len] = '/'; + memcpy (s + dir_len + 1, filename, filename_len + 1); + hdr->filenames[i] = s; + } + + /* Ignore the modification time and size. */ + read_uleb128 (hdr_buf); + read_uleb128 (hdr_buf); + + ++i; + } + + return 1; +} + +/* Read a single version 5 LNCT entry for a directory or file name in a + line header. Sets *STRING to the resulting name, ignoring other + data. Return 1 on success, 0 on failure. */ + +static int +read_lnct (struct backtrace_state *state, struct dwarf_data *ddata, + struct unit *u, struct dwarf_buf *hdr_buf, + const struct line_header *hdr, size_t formats_count, + const struct line_header_format *formats, const char **string) +{ + size_t i; + const char *dir; + const char *path; + + dir = NULL; + path = NULL; + for (i = 0; i < formats_count; i++) + { + struct attr_val val; + + if (!read_attribute (formats[i].form, 0, hdr_buf, u->is_dwarf64, + u->version, hdr->addrsize, &ddata->dwarf_sections, + ddata->altlink, &val)) + return 0; + switch (formats[i].lnct) + { + case DW_LNCT_path: + if (!resolve_string (&ddata->dwarf_sections, u->is_dwarf64, + ddata->is_bigendian, u->str_offsets_base, + &val, hdr_buf->error_callback, hdr_buf->data, + &path)) + return 0; + break; + case DW_LNCT_directory_index: + if (val.encoding == ATTR_VAL_UINT) + { + if (val.u.uint >= hdr->dirs_count) + { + dwarf_buf_error (hdr_buf, + ("invalid directory index in " + "line number program header"), + 0); + return 0; + } + dir = hdr->dirs[val.u.uint]; + } + break; + default: + /* We don't care about timestamps or sizes or hashes. */ + break; + } + } + + if (path == NULL) + { + dwarf_buf_error (hdr_buf, + "missing file name in line number program header", + 0); + return 0; + } + + if (dir == NULL) + *string = path; + else + { + size_t dir_len; + size_t path_len; + char *s; + + dir_len = strlen (dir); + path_len = strlen (path); + s = (char *) backtrace_alloc (state, dir_len + path_len + 2, + hdr_buf->error_callback, hdr_buf->data); + if (s == NULL) + return 0; + memcpy (s, dir, dir_len); + /* FIXME: If we are on a DOS-based file system, and the + directory or the path name use backslashes, then we should + use a backslash here. */ + s[dir_len] = '/'; + memcpy (s + dir_len + 1, path, path_len + 1); + *string = s; + } + + return 1; +} + +/* Read a set of DWARF 5 line header format entries, setting *PCOUNT + and *PPATHS. Return 1 on success, 0 on failure. */ + +static int +read_line_header_format_entries (struct backtrace_state *state, + struct dwarf_data *ddata, + struct unit *u, + struct dwarf_buf *hdr_buf, + struct line_header *hdr, + size_t *pcount, + const char ***ppaths) +{ + size_t formats_count; + struct line_header_format *formats; + size_t paths_count; + const char **paths; + size_t i; + int ret; + + formats_count = read_byte (hdr_buf); + if (formats_count == 0) + formats = NULL; + else + { + formats = ((struct line_header_format *) + backtrace_alloc (state, + (formats_count + * sizeof (struct line_header_format)), + hdr_buf->error_callback, + hdr_buf->data)); + if (formats == NULL) + return 0; + + for (i = 0; i < formats_count; i++) + { + formats[i].lnct = (int) read_uleb128(hdr_buf); + formats[i].form = (enum dwarf_form) read_uleb128 (hdr_buf); + } + } + + paths_count = read_uleb128 (hdr_buf); + if (paths_count == 0) + { + *pcount = 0; + *ppaths = NULL; + ret = 1; + goto exit; + } + + paths = ((const char **) + backtrace_alloc (state, paths_count * sizeof (const char *), + hdr_buf->error_callback, hdr_buf->data)); + if (paths == NULL) + { + ret = 0; + goto exit; + } + for (i = 0; i < paths_count; i++) + { + if (!read_lnct (state, ddata, u, hdr_buf, hdr, formats_count, + formats, &paths[i])) + { + backtrace_free (state, paths, + paths_count * sizeof (const char *), + hdr_buf->error_callback, hdr_buf->data); + ret = 0; + goto exit; + } + } + + *pcount = paths_count; + *ppaths = paths; + + ret = 1; + + exit: + if (formats != NULL) + backtrace_free (state, formats, + formats_count * sizeof (struct line_header_format), + hdr_buf->error_callback, hdr_buf->data); + + return ret; +} + +/* Read the line header. Return 1 on success, 0 on failure. */ + +static int +read_line_header (struct backtrace_state *state, struct dwarf_data *ddata, + struct unit *u, int is_dwarf64, struct dwarf_buf *line_buf, + struct line_header *hdr) +{ + uint64_t hdrlen; + struct dwarf_buf hdr_buf; + + hdr->version = read_uint16 (line_buf); + if (hdr->version < 2 || hdr->version > 5) + { + dwarf_buf_error (line_buf, "unsupported line number version", -1); + return 0; + } + + if (hdr->version < 5) + hdr->addrsize = u->addrsize; + else + { + hdr->addrsize = read_byte (line_buf); + /* We could support a non-zero segment_selector_size but I doubt + we'll ever see it. */ + if (read_byte (line_buf) != 0) + { + dwarf_buf_error (line_buf, + "non-zero segment_selector_size not supported", + -1); + return 0; + } + } + + hdrlen = read_offset (line_buf, is_dwarf64); + + hdr_buf = *line_buf; + hdr_buf.left = hdrlen; + + if (!advance (line_buf, hdrlen)) + return 0; + + hdr->min_insn_len = read_byte (&hdr_buf); + if (hdr->version < 4) + hdr->max_ops_per_insn = 1; + else + hdr->max_ops_per_insn = read_byte (&hdr_buf); + + /* We don't care about default_is_stmt. */ + read_byte (&hdr_buf); + + hdr->line_base = read_sbyte (&hdr_buf); + hdr->line_range = read_byte (&hdr_buf); + + hdr->opcode_base = read_byte (&hdr_buf); + hdr->opcode_lengths = hdr_buf.buf; + if (!advance (&hdr_buf, hdr->opcode_base - 1)) + return 0; + + if (hdr->version < 5) + { + if (!read_v2_paths (state, u, &hdr_buf, hdr)) + return 0; + } + else + { + if (!read_line_header_format_entries (state, ddata, u, &hdr_buf, hdr, + &hdr->dirs_count, + &hdr->dirs)) + return 0; + if (!read_line_header_format_entries (state, ddata, u, &hdr_buf, hdr, + &hdr->filenames_count, + &hdr->filenames)) + return 0; + } + + if (hdr_buf.reported_underflow) + return 0; + + return 1; +} + +/* Read the line program, adding line mappings to VEC. Return 1 on + success, 0 on failure. */ + +static int +read_line_program (struct backtrace_state *state, struct dwarf_data *ddata, + const struct line_header *hdr, struct dwarf_buf *line_buf, + struct line_vector *vec) +{ + uint64_t address; + unsigned int op_index; + const char *reset_filename; + const char *filename; + int lineno; + + address = 0; + op_index = 0; + if (hdr->filenames_count > 1) + reset_filename = hdr->filenames[1]; + else + reset_filename = ""; + filename = reset_filename; + lineno = 1; + while (line_buf->left > 0) + { + unsigned int op; + + op = read_byte (line_buf); + if (op >= hdr->opcode_base) + { + unsigned int advance; + + /* Special opcode. */ + op -= hdr->opcode_base; + advance = op / hdr->line_range; + address += (hdr->min_insn_len * (op_index + advance) + / hdr->max_ops_per_insn); + op_index = (op_index + advance) % hdr->max_ops_per_insn; + lineno += hdr->line_base + (int) (op % hdr->line_range); + add_line (state, ddata, address, filename, lineno, + line_buf->error_callback, line_buf->data, vec); + } + else if (op == DW_LNS_extended_op) + { + uint64_t len; + + len = read_uleb128 (line_buf); + op = read_byte (line_buf); + switch (op) + { + case DW_LNE_end_sequence: + /* FIXME: Should we mark the high PC here? It seems + that we already have that information from the + compilation unit. */ + address = 0; + op_index = 0; + filename = reset_filename; + lineno = 1; + break; + case DW_LNE_set_address: + address = read_address (line_buf, hdr->addrsize); + break; + case DW_LNE_define_file: + { + const char *f; + unsigned int dir_index; + + f = read_string (line_buf); + if (f == NULL) + return 0; + dir_index = read_uleb128 (line_buf); + /* Ignore that time and length. */ + read_uleb128 (line_buf); + read_uleb128 (line_buf); + if (IS_ABSOLUTE_PATH (f)) + filename = f; + else + { + const char *dir; + size_t dir_len; + size_t f_len; + char *p; + + if (dir_index < hdr->dirs_count) + dir = hdr->dirs[dir_index]; + else + { + dwarf_buf_error (line_buf, + ("invalid directory index " + "in line number program"), + 0); + return 0; + } + dir_len = strlen (dir); + f_len = strlen (f); + p = ((char *) + backtrace_alloc (state, dir_len + f_len + 2, + line_buf->error_callback, + line_buf->data)); + if (p == NULL) + return 0; + memcpy (p, dir, dir_len); + /* FIXME: If we are on a DOS-based file system, + and the directory or the file name use + backslashes, then we should use a backslash + here. */ + p[dir_len] = '/'; + memcpy (p + dir_len + 1, f, f_len + 1); + filename = p; + } + } + break; + case DW_LNE_set_discriminator: + /* We don't care about discriminators. */ + read_uleb128 (line_buf); + break; + default: + if (!advance (line_buf, len - 1)) + return 0; + break; + } + } + else + { + switch (op) + { + case DW_LNS_copy: + add_line (state, ddata, address, filename, lineno, + line_buf->error_callback, line_buf->data, vec); + break; + case DW_LNS_advance_pc: + { + uint64_t advance; + + advance = read_uleb128 (line_buf); + address += (hdr->min_insn_len * (op_index + advance) + / hdr->max_ops_per_insn); + op_index = (op_index + advance) % hdr->max_ops_per_insn; + } + break; + case DW_LNS_advance_line: + lineno += (int) read_sleb128 (line_buf); + break; + case DW_LNS_set_file: + { + uint64_t fileno; + + fileno = read_uleb128 (line_buf); + if (fileno >= hdr->filenames_count) + { + dwarf_buf_error (line_buf, + ("invalid file number in " + "line number program"), + 0); + return 0; + } + filename = hdr->filenames[fileno]; + } + break; + case DW_LNS_set_column: + read_uleb128 (line_buf); + break; + case DW_LNS_negate_stmt: + break; + case DW_LNS_set_basic_block: + break; + case DW_LNS_const_add_pc: + { + unsigned int advance; + + op = 255 - hdr->opcode_base; + advance = op / hdr->line_range; + address += (hdr->min_insn_len * (op_index + advance) + / hdr->max_ops_per_insn); + op_index = (op_index + advance) % hdr->max_ops_per_insn; + } + break; + case DW_LNS_fixed_advance_pc: + address += read_uint16 (line_buf); + op_index = 0; + break; + case DW_LNS_set_prologue_end: + break; + case DW_LNS_set_epilogue_begin: + break; + case DW_LNS_set_isa: + read_uleb128 (line_buf); + break; + default: + { + unsigned int i; + + for (i = hdr->opcode_lengths[op - 1]; i > 0; --i) + read_uleb128 (line_buf); + } + break; + } + } + } + + return 1; +} + +/* Read the line number information for a compilation unit. Returns 1 + on success, 0 on failure. */ + +static int +read_line_info (struct backtrace_state *state, struct dwarf_data *ddata, + backtrace_error_callback error_callback, void *data, + struct unit *u, struct line_header *hdr, struct line **lines, + size_t *lines_count) +{ + struct line_vector vec; + struct dwarf_buf line_buf; + uint64_t len; + int is_dwarf64; + struct line *ln; + + memset (&vec.vec, 0, sizeof vec.vec); + vec.count = 0; + + memset (hdr, 0, sizeof *hdr); + + if (u->lineoff != (off_t) (size_t) u->lineoff + || (size_t) u->lineoff >= ddata->dwarf_sections.size[DEBUG_LINE]) + { + error_callback (data, "unit line offset out of range", 0); + goto fail; + } + + line_buf.name = ".debug_line"; + line_buf.start = ddata->dwarf_sections.data[DEBUG_LINE]; + line_buf.buf = ddata->dwarf_sections.data[DEBUG_LINE] + u->lineoff; + line_buf.left = ddata->dwarf_sections.size[DEBUG_LINE] - u->lineoff; + line_buf.is_bigendian = ddata->is_bigendian; + line_buf.error_callback = error_callback; + line_buf.data = data; + line_buf.reported_underflow = 0; + + len = read_initial_length (&line_buf, &is_dwarf64); + line_buf.left = len; + + if (!read_line_header (state, ddata, u, is_dwarf64, &line_buf, hdr)) + goto fail; + + if (!read_line_program (state, ddata, hdr, &line_buf, &vec)) + goto fail; + + if (line_buf.reported_underflow) + goto fail; + + if (vec.count == 0) + { + /* This is not a failure in the sense of a generating an error, + but it is a failure in that sense that we have no useful + information. */ + goto fail; + } + + /* Allocate one extra entry at the end. */ + ln = ((struct line *) + backtrace_vector_grow (state, sizeof (struct line), error_callback, + data, &vec.vec)); + if (ln == NULL) + goto fail; + ln->pc = (uintptr_t) -1; + ln->filename = NULL; + ln->lineno = 0; + ln->idx = 0; + + if (!backtrace_vector_release (state, &vec.vec, error_callback, data)) + goto fail; + + ln = (struct line *) vec.vec.base; + backtrace_qsort (ln, vec.count, sizeof (struct line), line_compare); + + *lines = ln; + *lines_count = vec.count; + + return 1; + + fail: + backtrace_vector_free (state, &vec.vec, error_callback, data); + free_line_header (state, hdr, error_callback, data); + *lines = (struct line *) (uintptr_t) -1; + *lines_count = 0; + return 0; +} + +static const char *read_referenced_name (struct dwarf_data *, struct unit *, + uint64_t, backtrace_error_callback, + void *); + +/* Read the name of a function from a DIE referenced by ATTR with VAL. */ + +static const char * +read_referenced_name_from_attr (struct dwarf_data *ddata, struct unit *u, + struct attr *attr, struct attr_val *val, + backtrace_error_callback error_callback, + void *data) +{ + switch (attr->name) + { + case DW_AT_abstract_origin: + case DW_AT_specification: + break; + default: + return NULL; + } + + if (attr->form == DW_FORM_ref_sig8) + return NULL; + + if (val->encoding == ATTR_VAL_REF_INFO) + { + struct unit *unit + = find_unit (ddata->units, ddata->units_count, + val->u.uint); + if (unit == NULL) + return NULL; + + uint64_t offset = val->u.uint - unit->low_offset; + return read_referenced_name (ddata, unit, offset, error_callback, data); + } + + if (val->encoding == ATTR_VAL_UINT + || val->encoding == ATTR_VAL_REF_UNIT) + return read_referenced_name (ddata, u, val->u.uint, error_callback, data); + + if (val->encoding == ATTR_VAL_REF_ALT_INFO) + { + struct unit *alt_unit + = find_unit (ddata->altlink->units, ddata->altlink->units_count, + val->u.uint); + if (alt_unit == NULL) + return NULL; + + uint64_t offset = val->u.uint - alt_unit->low_offset; + return read_referenced_name (ddata->altlink, alt_unit, offset, + error_callback, data); + } + + return NULL; +} + +/* Read the name of a function from a DIE referenced by a + DW_AT_abstract_origin or DW_AT_specification tag. OFFSET is within + the same compilation unit. */ + +static const char * +read_referenced_name (struct dwarf_data *ddata, struct unit *u, + uint64_t offset, backtrace_error_callback error_callback, + void *data) +{ + struct dwarf_buf unit_buf; + uint64_t code; + const struct abbrev *abbrev; + const char *ret; + size_t i; + + /* OFFSET is from the start of the data for this compilation unit. + U->unit_data is the data, but it starts U->unit_data_offset bytes + from the beginning. */ + + if (offset < u->unit_data_offset + || offset - u->unit_data_offset >= u->unit_data_len) + { + error_callback (data, + "abstract origin or specification out of range", + 0); + return NULL; + } + + offset -= u->unit_data_offset; + + unit_buf.name = ".debug_info"; + unit_buf.start = ddata->dwarf_sections.data[DEBUG_INFO]; + unit_buf.buf = u->unit_data + offset; + unit_buf.left = u->unit_data_len - offset; + unit_buf.is_bigendian = ddata->is_bigendian; + unit_buf.error_callback = error_callback; + unit_buf.data = data; + unit_buf.reported_underflow = 0; + + code = read_uleb128 (&unit_buf); + if (code == 0) + { + dwarf_buf_error (&unit_buf, + "invalid abstract origin or specification", + 0); + return NULL; + } + + abbrev = lookup_abbrev (&u->abbrevs, code, error_callback, data); + if (abbrev == NULL) + return NULL; + + ret = NULL; + for (i = 0; i < abbrev->num_attrs; ++i) + { + struct attr_val val; + + if (!read_attribute (abbrev->attrs[i].form, abbrev->attrs[i].val, + &unit_buf, u->is_dwarf64, u->version, u->addrsize, + &ddata->dwarf_sections, ddata->altlink, &val)) + return NULL; + + switch (abbrev->attrs[i].name) + { + case DW_AT_name: + /* Third name preference: don't override. A name we found in some + other way, will normally be more useful -- e.g., this name is + normally not mangled. */ + if (ret != NULL) + break; + if (!resolve_string (&ddata->dwarf_sections, u->is_dwarf64, + ddata->is_bigendian, u->str_offsets_base, + &val, error_callback, data, &ret)) + return NULL; + break; + + case DW_AT_linkage_name: + case DW_AT_MIPS_linkage_name: + /* First name preference: override all. */ + { + const char *s; + + s = NULL; + if (!resolve_string (&ddata->dwarf_sections, u->is_dwarf64, + ddata->is_bigendian, u->str_offsets_base, + &val, error_callback, data, &s)) + return NULL; + if (s != NULL) + return s; + } + break; + + case DW_AT_specification: + /* Second name preference: override DW_AT_name, don't override + DW_AT_linkage_name. */ + { + const char *name; + + name = read_referenced_name_from_attr (ddata, u, &abbrev->attrs[i], + &val, error_callback, data); + if (name != NULL) + ret = name; + } + break; + + default: + break; + } + } + + return ret; +} + +/* Add a range to a unit that maps to a function. This is called via + add_ranges. Returns 1 on success, 0 on error. */ + +static int +add_function_range (struct backtrace_state *state, void *rdata, + uintptr_t lowpc, uintptr_t highpc, + backtrace_error_callback error_callback, void *data, + void *pvec) +{ + struct function *function = (struct function *) rdata; + struct function_vector *vec = (struct function_vector *) pvec; + struct function_addrs *p; + + if (vec->count > 0) + { + p = (struct function_addrs *) vec->vec.base + (vec->count - 1); + if ((lowpc == p->high || lowpc == p->high + 1) + && function == p->function) + { + if (highpc > p->high) + p->high = highpc; + return 1; + } + } + + p = ((struct function_addrs *) + backtrace_vector_grow (state, sizeof (struct function_addrs), + error_callback, data, &vec->vec)); + if (p == NULL) + return 0; + + p->low = lowpc; + p->high = highpc; + p->function = function; + + ++vec->count; + + return 1; +} + +/* Read one entry plus all its children. Add function addresses to + VEC. Returns 1 on success, 0 on error. */ + +static int +read_function_entry (struct backtrace_state *state, struct dwarf_data *ddata, + struct unit *u, uintptr_t base, struct dwarf_buf *unit_buf, + const struct line_header *lhdr, + backtrace_error_callback error_callback, void *data, + struct function_vector *vec_function, + struct function_vector *vec_inlined) +{ + while (unit_buf->left > 0) + { + uint64_t code; + const struct abbrev *abbrev; + int is_function; + struct function *function; + struct function_vector *vec; + size_t i; + struct pcrange pcrange; + int have_linkage_name; + + code = read_uleb128 (unit_buf); + if (code == 0) + return 1; + + abbrev = lookup_abbrev (&u->abbrevs, code, error_callback, data); + if (abbrev == NULL) + return 0; + + is_function = (abbrev->tag == DW_TAG_subprogram + || abbrev->tag == DW_TAG_entry_point + || abbrev->tag == DW_TAG_inlined_subroutine); + + if (abbrev->tag == DW_TAG_inlined_subroutine) + vec = vec_inlined; + else + vec = vec_function; + + function = NULL; + if (is_function) + { + function = ((struct function *) + backtrace_alloc (state, sizeof *function, + error_callback, data)); + if (function == NULL) + return 0; + memset (function, 0, sizeof *function); + } + + memset (&pcrange, 0, sizeof pcrange); + have_linkage_name = 0; + for (i = 0; i < abbrev->num_attrs; ++i) + { + struct attr_val val; + + if (!read_attribute (abbrev->attrs[i].form, abbrev->attrs[i].val, + unit_buf, u->is_dwarf64, u->version, + u->addrsize, &ddata->dwarf_sections, + ddata->altlink, &val)) + return 0; + + /* The compile unit sets the base address for any address + ranges in the function entries. */ + if ((abbrev->tag == DW_TAG_compile_unit + || abbrev->tag == DW_TAG_skeleton_unit) + && abbrev->attrs[i].name == DW_AT_low_pc) + { + if (val.encoding == ATTR_VAL_ADDRESS) + base = (uintptr_t) val.u.uint; + else if (val.encoding == ATTR_VAL_ADDRESS_INDEX) + { + if (!resolve_addr_index (&ddata->dwarf_sections, + u->addr_base, u->addrsize, + ddata->is_bigendian, val.u.uint, + error_callback, data, &base)) + return 0; + } + } + + if (is_function) + { + switch (abbrev->attrs[i].name) + { + case DW_AT_call_file: + if (val.encoding == ATTR_VAL_UINT) + { + if (val.u.uint >= lhdr->filenames_count) + { + dwarf_buf_error (unit_buf, + ("invalid file number in " + "DW_AT_call_file attribute"), + 0); + return 0; + } + function->caller_filename = lhdr->filenames[val.u.uint]; + } + break; + + case DW_AT_call_line: + if (val.encoding == ATTR_VAL_UINT) + function->caller_lineno = val.u.uint; + break; + + case DW_AT_abstract_origin: + case DW_AT_specification: + /* Second name preference: override DW_AT_name, don't override + DW_AT_linkage_name. */ + if (have_linkage_name) + break; + { + const char *name; + + name + = read_referenced_name_from_attr (ddata, u, + &abbrev->attrs[i], &val, + error_callback, data); + if (name != NULL) + function->name = name; + } + break; + + case DW_AT_name: + /* Third name preference: don't override. */ + if (function->name != NULL) + break; + if (!resolve_string (&ddata->dwarf_sections, u->is_dwarf64, + ddata->is_bigendian, + u->str_offsets_base, &val, + error_callback, data, &function->name)) + return 0; + break; + + case DW_AT_linkage_name: + case DW_AT_MIPS_linkage_name: + /* First name preference: override all. */ + { + const char *s; + + s = NULL; + if (!resolve_string (&ddata->dwarf_sections, u->is_dwarf64, + ddata->is_bigendian, + u->str_offsets_base, &val, + error_callback, data, &s)) + return 0; + if (s != NULL) + { + function->name = s; + have_linkage_name = 1; + } + } + break; + + case DW_AT_low_pc: case DW_AT_high_pc: case DW_AT_ranges: + update_pcrange (&abbrev->attrs[i], &val, &pcrange); + break; + + default: + break; + } + } + } + + /* If we couldn't find a name for the function, we have no use + for it. */ + if (is_function && function->name == NULL) + { + backtrace_free (state, function, sizeof *function, + error_callback, data); + is_function = 0; + } + + if (is_function) + { + if (pcrange.have_ranges + || (pcrange.have_lowpc && pcrange.have_highpc)) + { + if (!add_ranges (state, &ddata->dwarf_sections, + ddata->base_address, ddata->is_bigendian, + u, base, &pcrange, add_function_range, + (void *) function, error_callback, data, + (void *) vec)) + return 0; + } + else + { + backtrace_free (state, function, sizeof *function, + error_callback, data); + is_function = 0; + } + } + + if (abbrev->has_children) + { + if (!is_function) + { + if (!read_function_entry (state, ddata, u, base, unit_buf, lhdr, + error_callback, data, vec_function, + vec_inlined)) + return 0; + } + else + { + struct function_vector fvec; + + /* Gather any information for inlined functions in + FVEC. */ + + memset (&fvec, 0, sizeof fvec); + + if (!read_function_entry (state, ddata, u, base, unit_buf, lhdr, + error_callback, data, vec_function, + &fvec)) + return 0; + + if (fvec.count > 0) + { + struct function_addrs *p; + struct function_addrs *faddrs; + + /* Allocate a trailing entry, but don't include it + in fvec.count. */ + p = ((struct function_addrs *) + backtrace_vector_grow (state, + sizeof (struct function_addrs), + error_callback, data, + &fvec.vec)); + if (p == NULL) + return 0; + p->low = 0; + --p->low; + p->high = p->low; + p->function = NULL; + + if (!backtrace_vector_release (state, &fvec.vec, + error_callback, data)) + return 0; + + faddrs = (struct function_addrs *) fvec.vec.base; + backtrace_qsort (faddrs, fvec.count, + sizeof (struct function_addrs), + function_addrs_compare); + + function->function_addrs = faddrs; + function->function_addrs_count = fvec.count; + } + } + } + } + + return 1; +} + +/* Read function name information for a compilation unit. We look + through the whole unit looking for function tags. */ + +static void +read_function_info (struct backtrace_state *state, struct dwarf_data *ddata, + const struct line_header *lhdr, + backtrace_error_callback error_callback, void *data, + struct unit *u, struct function_vector *fvec, + struct function_addrs **ret_addrs, + size_t *ret_addrs_count) +{ + struct function_vector lvec; + struct function_vector *pfvec; + struct dwarf_buf unit_buf; + struct function_addrs *p; + struct function_addrs *addrs; + size_t addrs_count; + + /* Use FVEC if it is not NULL. Otherwise use our own vector. */ + if (fvec != NULL) + pfvec = fvec; + else + { + memset (&lvec, 0, sizeof lvec); + pfvec = &lvec; + } + + unit_buf.name = ".debug_info"; + unit_buf.start = ddata->dwarf_sections.data[DEBUG_INFO]; + unit_buf.buf = u->unit_data; + unit_buf.left = u->unit_data_len; + unit_buf.is_bigendian = ddata->is_bigendian; + unit_buf.error_callback = error_callback; + unit_buf.data = data; + unit_buf.reported_underflow = 0; + + while (unit_buf.left > 0) + { + if (!read_function_entry (state, ddata, u, 0, &unit_buf, lhdr, + error_callback, data, pfvec, pfvec)) + return; + } + + if (pfvec->count == 0) + return; + + /* Allocate a trailing entry, but don't include it in + pfvec->count. */ + p = ((struct function_addrs *) + backtrace_vector_grow (state, sizeof (struct function_addrs), + error_callback, data, &pfvec->vec)); + if (p == NULL) + return; + p->low = 0; + --p->low; + p->high = p->low; + p->function = NULL; + + addrs_count = pfvec->count; + + if (fvec == NULL) + { + if (!backtrace_vector_release (state, &lvec.vec, error_callback, data)) + return; + addrs = (struct function_addrs *) pfvec->vec.base; + } + else + { + /* Finish this list of addresses, but leave the remaining space in + the vector available for the next function unit. */ + addrs = ((struct function_addrs *) + backtrace_vector_finish (state, &fvec->vec, + error_callback, data)); + if (addrs == NULL) + return; + fvec->count = 0; + } + + backtrace_qsort (addrs, addrs_count, sizeof (struct function_addrs), + function_addrs_compare); + + *ret_addrs = addrs; + *ret_addrs_count = addrs_count; +} + +/* See if PC is inlined in FUNCTION. If it is, print out the inlined + information, and update FILENAME and LINENO for the caller. + Returns whatever CALLBACK returns, or 0 to keep going. */ + +static int +report_inlined_functions (uintptr_t pc, struct function *function, const char* comp_dir, + backtrace_full_callback callback, void *data, + const char **filename, int *lineno) +{ + struct function_addrs *p; + struct function_addrs *match; + struct function *inlined; + int ret; + + if (function->function_addrs_count == 0) + return 0; + + /* Our search isn't safe if pc == -1, as that is the sentinel + value. */ + if (pc + 1 == 0) + return 0; + + p = ((struct function_addrs *) + bsearch (&pc, function->function_addrs, + function->function_addrs_count, + sizeof (struct function_addrs), + function_addrs_search)); + if (p == NULL) + return 0; + + /* Here pc >= p->low && pc < (p + 1)->low. The function_addrs are + sorted by low, so if pc > p->low we are at the end of a range of + function_addrs with the same low value. If pc == p->low walk + forward to the end of the range with that low value. Then walk + backward and use the first range that includes pc. */ + while (pc == (p + 1)->low) + ++p; + match = NULL; + while (1) + { + if (pc < p->high) + { + match = p; + break; + } + if (p == function->function_addrs) + break; + if ((p - 1)->low < p->low) + break; + --p; + } + if (match == NULL) + return 0; + + /* We found an inlined call. */ + + inlined = match->function; + + /* Report any calls inlined into this one. */ + ret = report_inlined_functions (pc, inlined, comp_dir, callback, data, + filename, lineno); + if (ret != 0) + return ret; + + /* Report this inlined call. */ + if (*filename[0] != '/' && comp_dir) + { + char buf[1024]; + snprintf (buf, 1024, "%s/%s", comp_dir, *filename); + ret = callback (data, pc, match->low, buf, *lineno, inlined->name); + } + else + { + ret = callback (data, pc, match->low, *filename, *lineno, inlined->name); + } + if (ret != 0) + return ret; + + /* Our caller will report the caller of the inlined function; tell + it the appropriate filename and line number. */ + *filename = inlined->caller_filename; + *lineno = inlined->caller_lineno; + + return 0; +} + +/* Look for a PC in the DWARF mapping for one module. On success, + call CALLBACK and return whatever it returns. On error, call + ERROR_CALLBACK and return 0. Sets *FOUND to 1 if the PC is found, + 0 if not. */ + +static int +dwarf_lookup_pc (struct backtrace_state *state, struct dwarf_data *ddata, + uintptr_t pc, backtrace_full_callback callback, + backtrace_error_callback error_callback, void *data, + int *found) +{ + struct unit_addrs *entry; + int found_entry; + struct unit *u; + int new_data; + struct line *lines; + struct line *ln; + struct function_addrs *p; + struct function_addrs *fmatch; + struct function *function; + const char *filename; + int lineno; + int ret; + + *found = 1; + + /* Find an address range that includes PC. Our search isn't safe if + PC == -1, as we use that as a sentinel value, so skip the search + in that case. */ + entry = (ddata->addrs_count == 0 || pc + 1 == 0 + ? NULL + : (struct unit_addrs*)bsearch (&pc, ddata->addrs, ddata->addrs_count, + sizeof (struct unit_addrs), unit_addrs_search)); + + if (entry == NULL) + { + *found = 0; + return 0; + } + + /* Here pc >= entry->low && pc < (entry + 1)->low. The unit_addrs + are sorted by low, so if pc > p->low we are at the end of a range + of unit_addrs with the same low value. If pc == p->low walk + forward to the end of the range with that low value. Then walk + backward and use the first range that includes pc. */ + while (pc == (entry + 1)->low) + ++entry; + found_entry = 0; + while (1) + { + if (pc < entry->high) + { + found_entry = 1; + break; + } + if (entry == ddata->addrs) + break; + if ((entry - 1)->low < entry->low) + break; + --entry; + } + if (!found_entry) + { + *found = 0; + return 0; + } + + /* We need the lines, lines_count, function_addrs, + function_addrs_count fields of u. If they are not set, we need + to set them. When running in threaded mode, we need to allow for + the possibility that some other thread is setting them + simultaneously. */ + + u = entry->u; + lines = u->lines; + + /* Skip units with no useful line number information by walking + backward. Useless line number information is marked by setting + lines == -1. */ + while (entry > ddata->addrs + && pc >= (entry - 1)->low + && pc < (entry - 1)->high) + { + if (state->threaded) + lines = (struct line *) backtrace_atomic_load_pointer (&u->lines); + + if (lines != (struct line *) (uintptr_t) -1) + break; + + --entry; + + u = entry->u; + lines = u->lines; + } + + if (state->threaded) + lines = backtrace_atomic_load_pointer (&u->lines); + + new_data = 0; + if (lines == NULL) + { + struct function_addrs *function_addrs; + size_t function_addrs_count; + struct line_header lhdr; + size_t count; + + /* We have never read the line information for this unit. Read + it now. */ + + function_addrs = NULL; + function_addrs_count = 0; + if (read_line_info (state, ddata, error_callback, data, entry->u, &lhdr, + &lines, &count)) + { + struct function_vector *pfvec; + + /* If not threaded, reuse DDATA->FVEC for better memory + consumption. */ + if (state->threaded) + pfvec = NULL; + else + pfvec = &ddata->fvec; + read_function_info (state, ddata, &lhdr, error_callback, data, + entry->u, pfvec, &function_addrs, + &function_addrs_count); + free_line_header (state, &lhdr, error_callback, data); + new_data = 1; + } + + /* Atomically store the information we just read into the unit. + If another thread is simultaneously writing, it presumably + read the same information, and we don't care which one we + wind up with; we just leak the other one. We do have to + write the lines field last, so that the acquire-loads above + ensure that the other fields are set. */ + + if (!state->threaded) + { + u->lines_count = count; + u->function_addrs = function_addrs; + u->function_addrs_count = function_addrs_count; + u->lines = lines; + } + else + { + backtrace_atomic_store_size_t (&u->lines_count, count); + backtrace_atomic_store_pointer (&u->function_addrs, function_addrs); + backtrace_atomic_store_size_t (&u->function_addrs_count, + function_addrs_count); + backtrace_atomic_store_pointer (&u->lines, lines); + } + } + + /* Now all fields of U have been initialized. */ + + if (lines == (struct line *) (uintptr_t) -1) + { + /* If reading the line number information failed in some way, + try again to see if there is a better compilation unit for + this PC. */ + if (new_data) + return dwarf_lookup_pc (state, ddata, pc, callback, error_callback, + data, found); + return callback (data, pc, 0, NULL, 0, NULL); + } + + /* Search for PC within this unit. */ + + ln = (struct line *) bsearch (&pc, lines, entry->u->lines_count, + sizeof (struct line), line_search); + if (ln == NULL) + { + /* The PC is between the low_pc and high_pc attributes of the + compilation unit, but no entry in the line table covers it. + This implies that the start of the compilation unit has no + line number information. */ + + if (entry->u->abs_filename == NULL) + { + const char *filename; + + filename = entry->u->filename; + if (filename != NULL + && !IS_ABSOLUTE_PATH (filename) + && entry->u->comp_dir != NULL) + { + size_t filename_len; + const char *dir; + size_t dir_len; + char *s; + + filename_len = strlen (filename); + dir = entry->u->comp_dir; + dir_len = strlen (dir); + s = (char *) backtrace_alloc (state, dir_len + filename_len + 2, + error_callback, data); + if (s == NULL) + { + *found = 0; + return 0; + } + memcpy (s, dir, dir_len); + /* FIXME: Should use backslash if DOS file system. */ + s[dir_len] = '/'; + memcpy (s + dir_len + 1, filename, filename_len + 1); + filename = s; + } + entry->u->abs_filename = filename; + } + + return callback (data, pc, 0, entry->u->abs_filename, 0, NULL); + } + + /* Search for function name within this unit. */ + + if (entry->u->function_addrs_count == 0) + return callback (data, pc, 0, ln->filename, ln->lineno, NULL); + + p = ((struct function_addrs *) + bsearch (&pc, entry->u->function_addrs, + entry->u->function_addrs_count, + sizeof (struct function_addrs), + function_addrs_search)); + if (p == NULL) + return callback (data, pc, 0, ln->filename, ln->lineno, NULL); + + /* Here pc >= p->low && pc < (p + 1)->low. The function_addrs are + sorted by low, so if pc > p->low we are at the end of a range of + function_addrs with the same low value. If pc == p->low walk + forward to the end of the range with that low value. Then walk + backward and use the first range that includes pc. */ + while (pc == (p + 1)->low) + ++p; + fmatch = NULL; + while (1) + { + if (pc < p->high) + { + fmatch = p; + break; + } + if (p == entry->u->function_addrs) + break; + if ((p - 1)->low < p->low) + break; + --p; + } + if (fmatch == NULL) + return callback (data, pc, 0, ln->filename, ln->lineno, NULL); + + function = fmatch->function; + + filename = ln->filename; + lineno = ln->lineno; + + ret = report_inlined_functions (pc, function, entry->u->comp_dir, callback, data, + &filename, &lineno); + if (ret != 0) + return ret; + + if (filename[0] != '/' && entry->u->comp_dir) + { + char buf[1024]; + snprintf (buf, 1024, "%s/%s", entry->u->comp_dir, filename); + return callback (data, pc, fmatch->low, buf, lineno, function->name); + } + else + { + return callback (data, pc, fmatch->low, filename, lineno, function->name); + } +} + +bool dwarf_fileline_dwarf_lookup_pc_in_all_entries(struct backtrace_state *state, uintptr_t pc, + backtrace_full_callback callback, backtrace_error_callback error_callback, void *data, + int& found, int ret) +{ + for (struct dwarf_data* ddata = (struct dwarf_data *)state->fileline_data; + ddata != NULL; + ddata = ddata->next) + { + ret = dwarf_lookup_pc(state, ddata, pc, callback, error_callback, data, &found); + if (ret != 0 || found) return true; + } + return false; +} + +/* Return the file/line information for a PC using the DWARF mapping + we built earlier. */ + +static int +dwarf_fileline (struct backtrace_state *state, uintptr_t pc, + backtrace_full_callback callback, + backtrace_error_callback error_callback, void *data) +{ + struct dwarf_data *ddata; + int found; + int ret = 0; + + if (!state->threaded) + { + if (dwarf_fileline_dwarf_lookup_pc_in_all_entries(state, pc, callback, error_callback, data, found, ret)) + { + return ret; + } + + // if we failed to obtain an entry in range, it can mean that the address map has been changed and new entries + // have been loaded in the meantime. Request a refresh and try again. + if (state->request_known_address_ranges_refresh_fn) + { + int new_range_count = state->request_known_address_ranges_refresh_fn(state, pc); + if (new_range_count > 0) + { + if (dwarf_fileline_dwarf_lookup_pc_in_all_entries(state, pc, callback, error_callback, data, found, ret)) + { + return ret; + } + } + } + + } + else + { + struct dwarf_data **pp; + + pp = (struct dwarf_data **) (void *) &state->fileline_data; + while (1) + { + ddata = backtrace_atomic_load_pointer (pp); + if (ddata == NULL) + break; + + ret = dwarf_lookup_pc (state, ddata, pc, callback, error_callback, + data, &found); + if (ret != 0 || found) + return ret; + + pp = &ddata->next; + } + } + + /* FIXME: See if any libraries have been dlopen'ed. */ + + return callback (data, pc, 0, NULL, 0, NULL); +} + +/* Initialize our data structures from the DWARF debug info for a + file. Return NULL on failure. */ + +static struct dwarf_data * +build_dwarf_data (struct backtrace_state *state, + uintptr_t base_address, + const struct dwarf_sections *dwarf_sections, + int is_bigendian, + struct dwarf_data *altlink, + backtrace_error_callback error_callback, + void *data) +{ + struct unit_addrs_vector addrs_vec; + struct unit_addrs *addrs; + size_t addrs_count; + struct unit_vector units_vec; + struct unit **units; + size_t units_count; + struct dwarf_data *fdata; + + if (!build_address_map (state, base_address, dwarf_sections, is_bigendian, + altlink, error_callback, data, &addrs_vec, + &units_vec)) + return NULL; + + if (!backtrace_vector_release (state, &addrs_vec.vec, error_callback, data)) + return NULL; + if (!backtrace_vector_release (state, &units_vec.vec, error_callback, data)) + return NULL; + addrs = (struct unit_addrs *) addrs_vec.vec.base; + units = (struct unit **) units_vec.vec.base; + addrs_count = addrs_vec.count; + units_count = units_vec.count; + backtrace_qsort (addrs, addrs_count, sizeof (struct unit_addrs), + unit_addrs_compare); + /* No qsort for units required, already sorted. */ + + fdata = ((struct dwarf_data *) + backtrace_alloc (state, sizeof (struct dwarf_data), + error_callback, data)); + if (fdata == NULL) + return NULL; + + fdata->next = NULL; + fdata->altlink = altlink; + fdata->base_address = base_address; + fdata->addrs = addrs; + fdata->addrs_count = addrs_count; + fdata->units = units; + fdata->units_count = units_count; + fdata->dwarf_sections = *dwarf_sections; + fdata->is_bigendian = is_bigendian; + memset (&fdata->fvec, 0, sizeof fdata->fvec); + + return fdata; +} + +/* Build our data structures from the DWARF sections for a module. + Set FILELINE_FN and STATE->FILELINE_DATA. Return 1 on success, 0 + on failure. */ + +int +backtrace_dwarf_add (struct backtrace_state *state, + uintptr_t base_address, + const struct dwarf_sections *dwarf_sections, + int is_bigendian, + struct dwarf_data *fileline_altlink, + backtrace_error_callback error_callback, + void *data, fileline *fileline_fn, + struct dwarf_data **fileline_entry) +{ + struct dwarf_data *fdata; + + fdata = build_dwarf_data (state, base_address, dwarf_sections, is_bigendian, + fileline_altlink, error_callback, data); + if (fdata == NULL) + return 0; + + if (fileline_entry != NULL) + *fileline_entry = fdata; + + if (!state->threaded) + { + struct dwarf_data **pp; + + for (pp = (struct dwarf_data **) (void *) &state->fileline_data; + *pp != NULL; + pp = &(*pp)->next) + ; + *pp = fdata; + } + else + { + while (1) + { + struct dwarf_data **pp; + + pp = (struct dwarf_data **) (void *) &state->fileline_data; + + while (1) + { + struct dwarf_data *p; + + p = backtrace_atomic_load_pointer (pp); + + if (p == NULL) + break; + + pp = &p->next; + } + + if (__sync_bool_compare_and_swap (pp, NULL, fdata)) + break; + } + } + + *fileline_fn = dwarf_fileline; + + return 1; +} + +} diff --git a/third_party/tracy/public/libbacktrace/elf.cpp b/third_party/tracy/public/libbacktrace/elf.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e88a33b08a9202d44246eb0b5daaee45f2725794 --- /dev/null +++ b/third_party/tracy/public/libbacktrace/elf.cpp @@ -0,0 +1,7597 @@ +/* elf.c -- Get debug data from an ELF file for backtraces. + Copyright (C) 2012-2021 Free Software Foundation, Inc. + Written by Ian Lance Taylor, Google. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + (1) Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + (2) Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + (3) The name of the author may not be used to + endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_DL_ITERATE_PHDR +#include +#endif + +#include "backtrace.hpp" +#include "internal.hpp" + +#include "../client/TracyFastVector.hpp" +#include "../common/TracyAlloc.hpp" + +#ifndef S_ISLNK + #ifndef S_IFLNK + #define S_IFLNK 0120000 + #endif + #ifndef S_IFMT + #define S_IFMT 0170000 + #endif + #define S_ISLNK(m) (((m) & S_IFMT) == S_IFLNK) +#endif + +#ifndef __GNUC__ +#define __builtin_prefetch(p, r, l) +#ifndef unlikely +#define unlikely(x) (x) +#endif +#else +#ifndef unlikely +#define unlikely(x) __builtin_expect(!!(x), 0) +#endif +#endif + +namespace tracy +{ + +#ifdef TRACY_DEBUGINFOD +int GetDebugInfoDescriptor( const char* buildid_data, size_t buildid_size ); +#endif + +#if !defined(HAVE_DECL_STRNLEN) || !HAVE_DECL_STRNLEN + +/* If strnlen is not declared, provide our own version. */ + +static size_t +xstrnlen (const char *s, size_t maxlen) +{ + size_t i; + + for (i = 0; i < maxlen; ++i) + if (s[i] == '\0') + break; + return i; +} + +#define strnlen xstrnlen + +#endif + +#ifndef HAVE_LSTAT + +/* Dummy version of lstat for systems that don't have it. */ + +static int +xlstat (const char *path ATTRIBUTE_UNUSED, struct stat *st ATTRIBUTE_UNUSED) +{ + return -1; +} + +#define lstat xlstat + +#endif + +#ifndef HAVE_READLINK + +/* Dummy version of readlink for systems that don't have it. */ + +static ssize_t +xreadlink (const char *path ATTRIBUTE_UNUSED, char *buf ATTRIBUTE_UNUSED, + size_t bufsz ATTRIBUTE_UNUSED) +{ + return -1; +} + +#define readlink xreadlink + +#endif + +#ifndef HAVE_DL_ITERATE_PHDR + +/* Dummy version of dl_iterate_phdr for systems that don't have it. */ + +#define dl_phdr_info x_dl_phdr_info +#define dl_iterate_phdr x_dl_iterate_phdr + +struct dl_phdr_info +{ + uintptr_t dlpi_addr; + const char *dlpi_name; +}; + +static int +dl_iterate_phdr (int (*callback) (struct dl_phdr_info *, + size_t, void *) ATTRIBUTE_UNUSED, + void *data ATTRIBUTE_UNUSED) +{ + return 0; +} + +#endif /* ! defined (HAVE_DL_ITERATE_PHDR) */ + +/* The configure script must tell us whether we are 32-bit or 64-bit + ELF. We could make this code test and support either possibility, + but there is no point. This code only works for the currently + running executable, which means that we know the ELF mode at + configure time. */ + +#if BACKTRACE_ELF_SIZE != 32 && BACKTRACE_ELF_SIZE != 64 +#error "Unknown BACKTRACE_ELF_SIZE" +#endif + +/* might #include which might define our constants + with slightly different values. Undefine them to be safe. */ + +#undef EI_NIDENT +#undef EI_MAG0 +#undef EI_MAG1 +#undef EI_MAG2 +#undef EI_MAG3 +#undef EI_CLASS +#undef EI_DATA +#undef EI_VERSION +#undef ELF_MAG0 +#undef ELF_MAG1 +#undef ELF_MAG2 +#undef ELF_MAG3 +#undef ELFCLASS32 +#undef ELFCLASS64 +#undef ELFDATA2LSB +#undef ELFDATA2MSB +#undef EV_CURRENT +#undef ET_DYN +#undef EM_PPC64 +#undef EF_PPC64_ABI +#undef SHN_LORESERVE +#undef SHN_XINDEX +#undef SHN_UNDEF +#undef SHT_PROGBITS +#undef SHT_SYMTAB +#undef SHT_STRTAB +#undef SHT_DYNSYM +#undef SHF_COMPRESSED +#undef STT_OBJECT +#undef STT_FUNC +#undef NT_GNU_BUILD_ID +#undef ELFCOMPRESS_ZLIB +#undef ELFCOMPRESS_ZSTD + +/* Basic types. */ + +typedef uint16_t b_elf_half; /* Elf_Half. */ +typedef uint32_t b_elf_word; /* Elf_Word. */ +typedef int32_t b_elf_sword; /* Elf_Sword. */ + +#if BACKTRACE_ELF_SIZE == 32 + +typedef uint32_t b_elf_addr; /* Elf_Addr. */ +typedef uint32_t b_elf_off; /* Elf_Off. */ + +typedef uint32_t b_elf_wxword; /* 32-bit Elf_Word, 64-bit ELF_Xword. */ + +#else + +typedef uint64_t b_elf_addr; /* Elf_Addr. */ +typedef uint64_t b_elf_off; /* Elf_Off. */ +typedef uint64_t b_elf_xword; /* Elf_Xword. */ +typedef int64_t b_elf_sxword; /* Elf_Sxword. */ + +typedef uint64_t b_elf_wxword; /* 32-bit Elf_Word, 64-bit ELF_Xword. */ + +#endif + +/* Data structures and associated constants. */ + +#define EI_NIDENT 16 + +typedef struct { + unsigned char e_ident[EI_NIDENT]; /* ELF "magic number" */ + b_elf_half e_type; /* Identifies object file type */ + b_elf_half e_machine; /* Specifies required architecture */ + b_elf_word e_version; /* Identifies object file version */ + b_elf_addr e_entry; /* Entry point virtual address */ + b_elf_off e_phoff; /* Program header table file offset */ + b_elf_off e_shoff; /* Section header table file offset */ + b_elf_word e_flags; /* Processor-specific flags */ + b_elf_half e_ehsize; /* ELF header size in bytes */ + b_elf_half e_phentsize; /* Program header table entry size */ + b_elf_half e_phnum; /* Program header table entry count */ + b_elf_half e_shentsize; /* Section header table entry size */ + b_elf_half e_shnum; /* Section header table entry count */ + b_elf_half e_shstrndx; /* Section header string table index */ +} b_elf_ehdr; /* Elf_Ehdr. */ + +#define EI_MAG0 0 +#define EI_MAG1 1 +#define EI_MAG2 2 +#define EI_MAG3 3 +#define EI_CLASS 4 +#define EI_DATA 5 +#define EI_VERSION 6 + +#define ELFMAG0 0x7f +#define ELFMAG1 'E' +#define ELFMAG2 'L' +#define ELFMAG3 'F' + +#define ELFCLASS32 1 +#define ELFCLASS64 2 + +#define ELFDATA2LSB 1 +#define ELFDATA2MSB 2 + +#define EV_CURRENT 1 + +#define ET_DYN 3 + +#define EM_PPC64 21 +#define EF_PPC64_ABI 3 + +typedef struct { + b_elf_word sh_name; /* Section name, index in string tbl */ + b_elf_word sh_type; /* Type of section */ + b_elf_wxword sh_flags; /* Miscellaneous section attributes */ + b_elf_addr sh_addr; /* Section virtual addr at execution */ + b_elf_off sh_offset; /* Section file offset */ + b_elf_wxword sh_size; /* Size of section in bytes */ + b_elf_word sh_link; /* Index of another section */ + b_elf_word sh_info; /* Additional section information */ + b_elf_wxword sh_addralign; /* Section alignment */ + b_elf_wxword sh_entsize; /* Entry size if section holds table */ +} b_elf_shdr; /* Elf_Shdr. */ + +#define SHN_UNDEF 0x0000 /* Undefined section */ +#define SHN_LORESERVE 0xFF00 /* Begin range of reserved indices */ +#define SHN_XINDEX 0xFFFF /* Section index is held elsewhere */ + +#define SHT_PROGBITS 1 +#define SHT_SYMTAB 2 +#define SHT_STRTAB 3 +#define SHT_DYNSYM 11 + +#define SHF_COMPRESSED 0x800 + +#if BACKTRACE_ELF_SIZE == 32 + +typedef struct +{ + b_elf_word st_name; /* Symbol name, index in string tbl */ + b_elf_addr st_value; /* Symbol value */ + b_elf_word st_size; /* Symbol size */ + unsigned char st_info; /* Symbol binding and type */ + unsigned char st_other; /* Visibility and other data */ + b_elf_half st_shndx; /* Symbol section index */ +} b_elf_sym; /* Elf_Sym. */ + +#else /* BACKTRACE_ELF_SIZE != 32 */ + +typedef struct +{ + b_elf_word st_name; /* Symbol name, index in string tbl */ + unsigned char st_info; /* Symbol binding and type */ + unsigned char st_other; /* Visibility and other data */ + b_elf_half st_shndx; /* Symbol section index */ + b_elf_addr st_value; /* Symbol value */ + b_elf_xword st_size; /* Symbol size */ +} b_elf_sym; /* Elf_Sym. */ + +#endif /* BACKTRACE_ELF_SIZE != 32 */ + +#define STT_OBJECT 1 +#define STT_FUNC 2 + +typedef struct +{ + uint32_t namesz; + uint32_t descsz; + uint32_t type; + char name[1]; +} b_elf_note; + +#define NT_GNU_BUILD_ID 3 + +#if BACKTRACE_ELF_SIZE == 32 + +typedef struct +{ + b_elf_word ch_type; /* Compresstion algorithm */ + b_elf_word ch_size; /* Uncompressed size */ + b_elf_word ch_addralign; /* Alignment for uncompressed data */ +} b_elf_chdr; /* Elf_Chdr */ + +#else /* BACKTRACE_ELF_SIZE != 32 */ + +typedef struct +{ + b_elf_word ch_type; /* Compression algorithm */ + b_elf_word ch_reserved; /* Reserved */ + b_elf_xword ch_size; /* Uncompressed size */ + b_elf_xword ch_addralign; /* Alignment for uncompressed data */ +} b_elf_chdr; /* Elf_Chdr */ + +#endif /* BACKTRACE_ELF_SIZE != 32 */ + +#define ELFCOMPRESS_ZLIB 1 +#define ELFCOMPRESS_ZSTD 2 + +/* Names of sections, indexed by enum dwarf_section in internal.h. */ + +static const char * const dwarf_section_names[DEBUG_MAX] = +{ + ".debug_info", + ".debug_line", + ".debug_abbrev", + ".debug_ranges", + ".debug_str", + ".debug_addr", + ".debug_str_offsets", + ".debug_line_str", + ".debug_rnglists" +}; + +/* Information we gather for the sections we care about. */ + +struct debug_section_info +{ + /* Section file offset. */ + off_t offset; + /* Section size. */ + size_t size; + /* Section contents, after read from file. */ + const unsigned char *data; + /* Whether the SHF_COMPRESSED flag is set for the section. */ + int compressed; +}; + +/* Information we keep for an ELF symbol. */ + +struct elf_symbol +{ + /* The name of the symbol. */ + const char *name; + /* The address of the symbol. */ + uintptr_t address; + /* The size of the symbol. */ + size_t size; +}; + +/* Information to pass to elf_syminfo. */ + +struct elf_syminfo_data +{ + /* Symbols for the next module. */ + struct elf_syminfo_data *next; + /* The ELF symbols, sorted by address. */ + struct elf_symbol *symbols; + /* The number of symbols. */ + size_t count; +}; + +/* A view that works for either a file or memory. */ + +struct elf_view +{ + struct backtrace_view view; + int release; /* If non-zero, must call backtrace_release_view. */ +}; + +/* Information about PowerPC64 ELFv1 .opd section. */ + +struct elf_ppc64_opd_data +{ + /* Address of the .opd section. */ + b_elf_addr addr; + /* Section data. */ + const char *data; + /* Size of the .opd section. */ + size_t size; + /* Corresponding section view. */ + struct elf_view view; +}; + +/* Create a view of SIZE bytes from DESCRIPTOR/MEMORY at OFFSET. */ + +static int +elf_get_view (struct backtrace_state *state, int descriptor, + const unsigned char *memory, size_t memory_size, off_t offset, + uint64_t size, backtrace_error_callback error_callback, + void *data, struct elf_view *view) +{ + if (memory == NULL) + { + view->release = 1; + return backtrace_get_view (state, descriptor, offset, size, + error_callback, data, &view->view); + } + else + { + if ((uint64_t) offset + size > (uint64_t) memory_size) + { + error_callback (data, "out of range for in-memory file", 0); + return 0; + } + view->view.data = (const void *) (memory + offset); + view->view.base = NULL; + view->view.len = size; + view->release = 0; + return 1; + } +} + +/* Release a view read by elf_get_view. */ + +static void +elf_release_view (struct backtrace_state *state, struct elf_view *view, + backtrace_error_callback error_callback, void *data) +{ + if (view->release) + backtrace_release_view (state, &view->view, error_callback, data); +} + +/* Compute the CRC-32 of BUF/LEN. This uses the CRC used for + .gnu_debuglink files. */ + +static uint32_t +elf_crc32 (uint32_t crc, const unsigned char *buf, size_t len) +{ + static const uint32_t crc32_table[256] = + { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, + 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, + 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, + 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, + 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856, + 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, + 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, + 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, + 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, + 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, + 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, + 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, + 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, + 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, + 0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, + 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, + 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, + 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010, + 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, + 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, + 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, + 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, + 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, + 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, + 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, + 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c, + 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, + 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, + 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, + 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, + 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, + 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, + 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, + 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278, + 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, + 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, + 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, + 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, + 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, + 0x2d02ef8d + }; + const unsigned char *end; + + crc = ~crc; + for (end = buf + len; buf < end; ++ buf) + crc = crc32_table[(crc ^ *buf) & 0xff] ^ (crc >> 8); + return ~crc; +} + +/* Return the CRC-32 of the entire file open at DESCRIPTOR. */ + +static uint32_t +elf_crc32_file (struct backtrace_state *state, int descriptor, + backtrace_error_callback error_callback, void *data) +{ + struct stat st; + struct backtrace_view file_view; + uint32_t ret; + + if (fstat (descriptor, &st) < 0) + { + error_callback (data, "fstat", errno); + return 0; + } + + if (!backtrace_get_view (state, descriptor, 0, st.st_size, error_callback, + data, &file_view)) + return 0; + + ret = elf_crc32 (0, (const unsigned char *) file_view.data, st.st_size); + + backtrace_release_view (state, &file_view, error_callback, data); + + return ret; +} + +/* A dummy callback function used when we can't find a symbol + table. */ + +static void +elf_nosyms (struct backtrace_state *state ATTRIBUTE_UNUSED, + uintptr_t addr ATTRIBUTE_UNUSED, + backtrace_syminfo_callback callback ATTRIBUTE_UNUSED, + backtrace_error_callback error_callback, void *data) +{ + error_callback (data, "no symbol table in ELF executable", -1); +} + +/* A callback function used when we can't find any debug info. */ + +static int +elf_nodebug (struct backtrace_state *state, uintptr_t pc, + backtrace_full_callback callback, + backtrace_error_callback error_callback, void *data) +{ + if (state->syminfo_fn != NULL && state->syminfo_fn != elf_nosyms) + { + struct backtrace_call_full bdata; + + /* Fetch symbol information so that we can least get the + function name. */ + + bdata.full_callback = callback; + bdata.full_error_callback = error_callback; + bdata.full_data = data; + bdata.ret = 0; + state->syminfo_fn (state, pc, backtrace_syminfo_to_full_callback, + backtrace_syminfo_to_full_error_callback, &bdata); + return bdata.ret; + } + + error_callback (data, "no debug info in ELF executable", -1); + return 0; +} + +/* Compare struct elf_symbol for qsort. */ + +static int +elf_symbol_compare (const void *v1, const void *v2) +{ + const struct elf_symbol *e1 = (const struct elf_symbol *) v1; + const struct elf_symbol *e2 = (const struct elf_symbol *) v2; + + if (e1->address < e2->address) + return -1; + else if (e1->address > e2->address) + return 1; + else + return 0; +} + +/* Compare an ADDR against an elf_symbol for bsearch. We allocate one + extra entry in the array so that this can look safely at the next + entry. */ + +static int +elf_symbol_search (const void *vkey, const void *ventry) +{ + const uintptr_t *key = (const uintptr_t *) vkey; + const struct elf_symbol *entry = (const struct elf_symbol *) ventry; + uintptr_t addr; + + addr = *key; + if (addr < entry->address) + return -1; + else if (addr >= entry->address + entry->size) + return 1; + else + return 0; +} + +/* Initialize the symbol table info for elf_syminfo. */ + +static int +elf_initialize_syminfo (struct backtrace_state *state, + uintptr_t base_address, + const unsigned char *symtab_data, size_t symtab_size, + const unsigned char *strtab, size_t strtab_size, + backtrace_error_callback error_callback, + void *data, struct elf_syminfo_data *sdata, + struct elf_ppc64_opd_data *opd) +{ + size_t sym_count; + const b_elf_sym *sym; + size_t elf_symbol_count; + size_t elf_symbol_size; + struct elf_symbol *elf_symbols; + size_t i; + unsigned int j; + + sym_count = symtab_size / sizeof (b_elf_sym); + + /* We only care about function symbols. Count them. */ + sym = (const b_elf_sym *) symtab_data; + elf_symbol_count = 0; + for (i = 0; i < sym_count; ++i, ++sym) + { + int info; + + info = sym->st_info & 0xf; + if ((info == STT_FUNC || info == STT_OBJECT) + && sym->st_shndx != SHN_UNDEF) + ++elf_symbol_count; + } + + elf_symbol_size = elf_symbol_count * sizeof (struct elf_symbol); + elf_symbols = ((struct elf_symbol *) + backtrace_alloc (state, elf_symbol_size, error_callback, + data)); + if (elf_symbols == NULL) + return 0; + + sym = (const b_elf_sym *) symtab_data; + j = 0; + for (i = 0; i < sym_count; ++i, ++sym) + { + int info; + + info = sym->st_info & 0xf; + if (info != STT_FUNC && info != STT_OBJECT) + continue; + if (sym->st_shndx == SHN_UNDEF) + continue; + if (sym->st_name >= strtab_size) + { + error_callback (data, "symbol string index out of range", 0); + backtrace_free (state, elf_symbols, elf_symbol_size, error_callback, + data); + return 0; + } + elf_symbols[j].name = (const char *) strtab + sym->st_name; + /* Special case PowerPC64 ELFv1 symbols in .opd section, if the symbol + is a function descriptor, read the actual code address from the + descriptor. */ + if (opd + && sym->st_value >= opd->addr + && sym->st_value < opd->addr + opd->size) + elf_symbols[j].address + = *(const b_elf_addr *) (opd->data + (sym->st_value - opd->addr)); + else + elf_symbols[j].address = sym->st_value; + elf_symbols[j].address += base_address; + elf_symbols[j].size = sym->st_size; + ++j; + } + + backtrace_qsort (elf_symbols, elf_symbol_count, sizeof (struct elf_symbol), + elf_symbol_compare); + + sdata->next = NULL; + sdata->symbols = elf_symbols; + sdata->count = elf_symbol_count; + + return 1; +} + +/* Add EDATA to the list in STATE. */ + +static void +elf_add_syminfo_data (struct backtrace_state *state, + struct elf_syminfo_data *edata) +{ + if (!state->threaded) + { + struct elf_syminfo_data **pp; + + for (pp = (struct elf_syminfo_data **) (void *) &state->syminfo_data; + *pp != NULL; + pp = &(*pp)->next) + ; + *pp = edata; + } + else + { + while (1) + { + struct elf_syminfo_data **pp; + + pp = (struct elf_syminfo_data **) (void *) &state->syminfo_data; + + while (1) + { + struct elf_syminfo_data *p; + + p = backtrace_atomic_load_pointer (pp); + + if (p == NULL) + break; + + pp = &p->next; + } + + if (__sync_bool_compare_and_swap (pp, NULL, edata)) + break; + } + } +} + +/* Return the symbol name and value for an ADDR. */ + +static void +elf_syminfo (struct backtrace_state *state, uintptr_t addr, + backtrace_syminfo_callback callback, + backtrace_error_callback error_callback ATTRIBUTE_UNUSED, + void *data) +{ + struct elf_syminfo_data *edata; + struct elf_symbol *sym = NULL; + + if (!state->threaded) + { + for (edata = (struct elf_syminfo_data *) state->syminfo_data; + edata != NULL; + edata = edata->next) + { + sym = ((struct elf_symbol *) + bsearch (&addr, edata->symbols, edata->count, + sizeof (struct elf_symbol), elf_symbol_search)); + if (sym != NULL) + break; + } + } + else + { + struct elf_syminfo_data **pp; + + pp = (struct elf_syminfo_data **) (void *) &state->syminfo_data; + while (1) + { + edata = backtrace_atomic_load_pointer (pp); + if (edata == NULL) + break; + + sym = ((struct elf_symbol *) + bsearch (&addr, edata->symbols, edata->count, + sizeof (struct elf_symbol), elf_symbol_search)); + if (sym != NULL) + break; + + pp = &edata->next; + } + } + + if (sym == NULL) + callback (data, addr, NULL, 0, 0); + else + callback (data, addr, sym->name, sym->address, sym->size); +} + +/* Return whether FILENAME is a symlink. */ + +static int +elf_is_symlink (const char *filename) +{ + struct stat st; + + if (lstat (filename, &st) < 0) + return 0; + return S_ISLNK (st.st_mode); +} + +/* Return the results of reading the symlink FILENAME in a buffer + allocated by backtrace_alloc. Return the length of the buffer in + *LEN. */ + +static char * +elf_readlink (struct backtrace_state *state, const char *filename, + backtrace_error_callback error_callback, void *data, + size_t *plen) +{ + size_t len; + char *buf; + + len = 128; + while (1) + { + ssize_t rl; + + buf = (char*)backtrace_alloc (state, len, error_callback, data); + if (buf == NULL) + return NULL; + rl = readlink (filename, buf, len); + if (rl < 0) + { + backtrace_free (state, buf, len, error_callback, data); + return NULL; + } + if ((size_t) rl < len - 1) + { + buf[rl] = '\0'; + *plen = len; + return buf; + } + backtrace_free (state, buf, len, error_callback, data); + len *= 2; + } +} + +#define SYSTEM_BUILD_ID_DIR "/usr/lib/debug/.build-id/" + +/* Open a separate debug info file, using the build ID to find it. + Returns an open file descriptor, or -1. + + The GDB manual says that the only place gdb looks for a debug file + when the build ID is known is in /usr/lib/debug/.build-id. */ + +static int +elf_open_debugfile_by_buildid (struct backtrace_state *state, + const char *buildid_data, size_t buildid_size, + const char *filename, + backtrace_error_callback error_callback, + void *data) +{ + const char * const prefix = SYSTEM_BUILD_ID_DIR; + const size_t prefix_len = strlen (prefix); + const char * const suffix = ".debug"; + const size_t suffix_len = strlen (suffix); + size_t len; + char *bd_filename; + char *t; + size_t i; + int ret; + int does_not_exist; + + len = prefix_len + buildid_size * 2 + suffix_len + 2; + bd_filename = (char*)backtrace_alloc (state, len, error_callback, data); + if (bd_filename == NULL) + return -1; + + t = bd_filename; + memcpy (t, prefix, prefix_len); + t += prefix_len; + for (i = 0; i < buildid_size; i++) + { + unsigned char b; + unsigned char nib; + + b = (unsigned char) buildid_data[i]; + nib = (b & 0xf0) >> 4; + *t++ = nib < 10 ? '0' + nib : 'a' + nib - 10; + nib = b & 0x0f; + *t++ = nib < 10 ? '0' + nib : 'a' + nib - 10; + if (i == 0) + *t++ = '/'; + } + memcpy (t, suffix, suffix_len); + t[suffix_len] = '\0'; + + ret = backtrace_open (bd_filename, error_callback, data, &does_not_exist); + + backtrace_free (state, bd_filename, len, error_callback, data); + + /* gdb checks that the debuginfo file has the same build ID note. + That seems kind of pointless to me--why would it have the right + name but not the right build ID?--so skipping the check. */ + +#ifdef TRACY_DEBUGINFOD + if (ret == -1) + return GetDebugInfoDescriptor( buildid_data, buildid_size, filename ); + else + return ret; +#else + return ret; +#endif +} + +/* Try to open a file whose name is PREFIX (length PREFIX_LEN) + concatenated with PREFIX2 (length PREFIX2_LEN) concatenated with + DEBUGLINK_NAME. Returns an open file descriptor, or -1. */ + +static int +elf_try_debugfile (struct backtrace_state *state, const char *prefix, + size_t prefix_len, const char *prefix2, size_t prefix2_len, + const char *debuglink_name, + backtrace_error_callback error_callback, void *data) +{ + size_t debuglink_len; + size_t try_len; + char *Try; + int does_not_exist; + int ret; + + debuglink_len = strlen (debuglink_name); + try_len = prefix_len + prefix2_len + debuglink_len + 1; + Try = (char*)backtrace_alloc (state, try_len, error_callback, data); + if (Try == NULL) + return -1; + + memcpy (Try, prefix, prefix_len); + memcpy (Try + prefix_len, prefix2, prefix2_len); + memcpy (Try + prefix_len + prefix2_len, debuglink_name, debuglink_len); + Try[prefix_len + prefix2_len + debuglink_len] = '\0'; + + ret = backtrace_open (Try, error_callback, data, &does_not_exist); + + backtrace_free (state, Try, try_len, error_callback, data); + + return ret; +} + +/* Find a separate debug info file, using the debuglink section data + to find it. Returns an open file descriptor, or -1. */ + +static int +elf_find_debugfile_by_debuglink (struct backtrace_state *state, + const char *filename, + const char *debuglink_name, + backtrace_error_callback error_callback, + void *data) +{ + int ret; + char *alc; + size_t alc_len; + const char *slash; + int ddescriptor; + const char *prefix; + size_t prefix_len; + + /* Resolve symlinks in FILENAME. Since FILENAME is fairly likely to + be /proc/self/exe, symlinks are common. We don't try to resolve + the whole path name, just the base name. */ + ret = -1; + alc = NULL; + alc_len = 0; + while (elf_is_symlink (filename)) + { + char *new_buf; + size_t new_len; + + new_buf = elf_readlink (state, filename, error_callback, data, &new_len); + if (new_buf == NULL) + break; + + if (new_buf[0] == '/') + filename = new_buf; + else + { + slash = strrchr (filename, '/'); + if (slash == NULL) + filename = new_buf; + else + { + size_t clen; + char *c; + + slash++; + clen = slash - filename + strlen (new_buf) + 1; + c = (char*)backtrace_alloc (state, clen, error_callback, data); + if (c == NULL) + goto done; + + memcpy (c, filename, slash - filename); + memcpy (c + (slash - filename), new_buf, strlen (new_buf)); + c[slash - filename + strlen (new_buf)] = '\0'; + backtrace_free (state, new_buf, new_len, error_callback, data); + filename = c; + new_buf = c; + new_len = clen; + } + } + + if (alc != NULL) + backtrace_free (state, alc, alc_len, error_callback, data); + alc = new_buf; + alc_len = new_len; + } + + /* Look for DEBUGLINK_NAME in the same directory as FILENAME. */ + + slash = strrchr (filename, '/'); + if (slash == NULL) + { + prefix = ""; + prefix_len = 0; + } + else + { + slash++; + prefix = filename; + prefix_len = slash - filename; + } + + ddescriptor = elf_try_debugfile (state, prefix, prefix_len, "", 0, + debuglink_name, error_callback, data); + if (ddescriptor >= 0) + { + ret = ddescriptor; + goto done; + } + + /* Look for DEBUGLINK_NAME in a .debug subdirectory of FILENAME. */ + + ddescriptor = elf_try_debugfile (state, prefix, prefix_len, ".debug/", + strlen (".debug/"), debuglink_name, + error_callback, data); + if (ddescriptor >= 0) + { + ret = ddescriptor; + goto done; + } + + /* Look for DEBUGLINK_NAME in /usr/lib/debug. */ + + ddescriptor = elf_try_debugfile (state, "/usr/lib/debug/", + strlen ("/usr/lib/debug/"), prefix, + prefix_len, debuglink_name, + error_callback, data); + if (ddescriptor >= 0) + ret = ddescriptor; + + done: + if (alc != NULL && alc_len > 0) + backtrace_free (state, alc, alc_len, error_callback, data); + return ret; +} + +/* Open a separate debug info file, using the debuglink section data + to find it. Returns an open file descriptor, or -1. */ + +static int +elf_open_debugfile_by_debuglink (struct backtrace_state *state, + const char *filename, + const char *debuglink_name, + uint32_t debuglink_crc, + backtrace_error_callback error_callback, + void *data) +{ + int ddescriptor; + + ddescriptor = elf_find_debugfile_by_debuglink (state, filename, + debuglink_name, + error_callback, data); + if (ddescriptor < 0) + return -1; + + if (debuglink_crc != 0) + { + uint32_t got_crc; + + got_crc = elf_crc32_file (state, ddescriptor, error_callback, data); + if (got_crc != debuglink_crc) + { + backtrace_close (ddescriptor, error_callback, data); + return -1; + } + } + + return ddescriptor; +} + +/* A function useful for setting a breakpoint for an inflation failure + when this code is compiled with -g. */ + +static void +elf_uncompress_failed(void) +{ +} + +/* *PVAL is the current value being read from the stream, and *PBITS + is the number of valid bits. Ensure that *PVAL holds at least 15 + bits by reading additional bits from *PPIN, up to PINEND, as + needed. Updates *PPIN, *PVAL and *PBITS. Returns 1 on success, 0 + on error. */ + +static int +elf_fetch_bits (const unsigned char **ppin, const unsigned char *pinend, + uint64_t *pval, unsigned int *pbits) +{ + unsigned int bits; + const unsigned char *pin; + uint64_t val; + uint32_t next; + + bits = *pbits; + if (bits >= 15) + return 1; + pin = *ppin; + val = *pval; + + if (unlikely (pinend - pin < 4)) + { + elf_uncompress_failed (); + return 0; + } + +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) \ + && defined(__ORDER_BIG_ENDIAN__) \ + && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ \ + || __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + /* We've ensured that PIN is aligned. */ + next = *(const uint32_t *)pin; + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + next = __builtin_bswap32 (next); +#endif +#else + next = pin[0] | (pin[1] << 8) | (pin[2] << 16) | (pin[3] << 24); +#endif + + val |= (uint64_t)next << bits; + bits += 32; + pin += 4; + + /* We will need the next four bytes soon. */ + __builtin_prefetch (pin, 0, 0); + + *ppin = pin; + *pval = val; + *pbits = bits; + return 1; +} + +/* This is like elf_fetch_bits, but it fetchs the bits backward, and ensures at + least 16 bits. This is for zstd. */ + +static int +elf_fetch_bits_backward (const unsigned char **ppin, + const unsigned char *pinend, + uint64_t *pval, unsigned int *pbits) +{ + unsigned int bits; + const unsigned char *pin; + uint64_t val; + uint32_t next; + + bits = *pbits; + if (bits >= 16) + return 1; + pin = *ppin; + val = *pval; + + if (unlikely (pin <= pinend)) + { + if (bits == 0) + { + elf_uncompress_failed (); + return 0; + } + return 1; + } + + pin -= 4; + +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) \ + && defined(__ORDER_BIG_ENDIAN__) \ + && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ \ + || __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + /* We've ensured that PIN is aligned. */ + next = *(const uint32_t *)pin; + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + next = __builtin_bswap32 (next); +#endif +#else + next = pin[0] | (pin[1] << 8) | (pin[2] << 16) | (pin[3] << 24); +#endif + + val <<= 32; + val |= next; + bits += 32; + + if (unlikely (pin < pinend)) + { + val >>= (pinend - pin) * 8; + bits -= (pinend - pin) * 8; + } + + *ppin = pin; + *pval = val; + *pbits = bits; + return 1; +} + +/* Initialize backward fetching when the bitstream starts with a 1 bit in the + last byte in memory (which is the first one that we read). This is used by + zstd decompression. Returns 1 on success, 0 on error. */ + +static int +elf_fetch_backward_init (const unsigned char **ppin, + const unsigned char *pinend, + uint64_t *pval, unsigned int *pbits) +{ + const unsigned char *pin; + unsigned int stream_start; + uint64_t val; + unsigned int bits; + + pin = *ppin; + stream_start = (unsigned int)*pin; + if (unlikely (stream_start == 0)) + { + elf_uncompress_failed (); + return 0; + } + val = 0; + bits = 0; + + /* Align to a 32-bit boundary. */ + while ((((uintptr_t)pin) & 3) != 0) + { + val <<= 8; + val |= (uint64_t)*pin; + bits += 8; + --pin; + } + + val <<= 8; + val |= (uint64_t)*pin; + bits += 8; + + *ppin = pin; + *pval = val; + *pbits = bits; + if (!elf_fetch_bits_backward (ppin, pinend, pval, pbits)) + return 0; + + *pbits -= __builtin_clz (stream_start) - (sizeof (unsigned int) - 1) * 8 + 1; + + if (!elf_fetch_bits_backward (ppin, pinend, pval, pbits)) + return 0; + + return 1; +} + +/* Huffman code tables, like the rest of the zlib format, are defined + by RFC 1951. We store a Huffman code table as a series of tables + stored sequentially in memory. Each entry in a table is 16 bits. + The first, main, table has 256 entries. It is followed by a set of + secondary tables of length 2 to 128 entries. The maximum length of + a code sequence in the deflate format is 15 bits, so that is all we + need. Each secondary table has an index, which is the offset of + the table in the overall memory storage. + + The deflate format says that all codes of a given bit length are + lexicographically consecutive. Perhaps we could have 130 values + that require a 15-bit code, perhaps requiring three secondary + tables of size 128. I don't know if this is actually possible, but + it suggests that the maximum size required for secondary tables is + 3 * 128 + 3 * 64 ... == 768. The zlib enough program reports 660 + as the maximum. We permit 768, since in addition to the 256 for + the primary table, with two bytes per entry, and with the two + tables we need, that gives us a page. + + A single table entry needs to store a value or (for the main table + only) the index and size of a secondary table. Values range from 0 + to 285, inclusive. Secondary table indexes, per above, range from + 0 to 510. For a value we need to store the number of bits we need + to determine that value (one value may appear multiple times in the + table), which is 1 to 8. For a secondary table we need to store + the number of bits used to index into the table, which is 1 to 7. + And of course we need 1 bit to decide whether we have a value or a + secondary table index. So each entry needs 9 bits for value/table + index, 3 bits for size, 1 bit what it is. For simplicity we use 16 + bits per entry. */ + +/* Number of entries we allocate to for one code table. We get a page + for the two code tables we need. */ + +#define ZLIB_HUFFMAN_TABLE_SIZE (1024) + +/* Bit masks and shifts for the values in the table. */ + +#define ZLIB_HUFFMAN_VALUE_MASK 0x01ff +#define ZLIB_HUFFMAN_BITS_SHIFT 9 +#define ZLIB_HUFFMAN_BITS_MASK 0x7 +#define ZLIB_HUFFMAN_SECONDARY_SHIFT 12 + +/* For working memory while inflating we need two code tables, we need + an array of code lengths (max value 15, so we use unsigned char), + and an array of unsigned shorts used while building a table. The + latter two arrays must be large enough to hold the maximum number + of code lengths, which RFC 1951 defines as 286 + 30. */ + +#define ZLIB_TABLE_SIZE \ + (2 * ZLIB_HUFFMAN_TABLE_SIZE * sizeof (uint16_t) \ + + (286 + 30) * sizeof (uint16_t) \ + + (286 + 30) * sizeof (unsigned char)) + +#define ZLIB_TABLE_CODELEN_OFFSET \ + (2 * ZLIB_HUFFMAN_TABLE_SIZE * sizeof (uint16_t) \ + + (286 + 30) * sizeof (uint16_t)) + +#define ZLIB_TABLE_WORK_OFFSET \ + (2 * ZLIB_HUFFMAN_TABLE_SIZE * sizeof (uint16_t)) + +#ifdef BACKTRACE_GENERATE_FIXED_HUFFMAN_TABLE + +/* Used by the main function that generates the fixed table to learn + the table size. */ +static size_t final_next_secondary; + +#endif + +/* Build a Huffman code table from an array of lengths in CODES of + length CODES_LEN. The table is stored into *TABLE. ZDEBUG_TABLE + is the same as for elf_zlib_inflate, used to find some work space. + Returns 1 on success, 0 on error. */ + +static int +elf_zlib_inflate_table (unsigned char *codes, size_t codes_len, + uint16_t *zdebug_table, uint16_t *table) +{ + uint16_t count[16]; + uint16_t start[16]; + uint16_t prev[16]; + uint16_t firstcode[7]; + uint16_t *next; + size_t i; + size_t j; + unsigned int code; + size_t next_secondary; + + /* Count the number of code of each length. Set NEXT[val] to be the + next value after VAL with the same bit length. */ + + next = (uint16_t *) (((unsigned char *) zdebug_table) + + ZLIB_TABLE_WORK_OFFSET); + + memset (&count[0], 0, 16 * sizeof (uint16_t)); + for (i = 0; i < codes_len; ++i) + { + if (unlikely (codes[i] >= 16)) + { + elf_uncompress_failed (); + return 0; + } + + if (count[codes[i]] == 0) + { + start[codes[i]] = i; + prev[codes[i]] = i; + } + else + { + next[prev[codes[i]]] = i; + prev[codes[i]] = i; + } + + ++count[codes[i]]; + } + + /* For each length, fill in the table for the codes of that + length. */ + + memset (table, 0, ZLIB_HUFFMAN_TABLE_SIZE * sizeof (uint16_t)); + + /* Handle the values that do not require a secondary table. */ + + code = 0; + for (j = 1; j <= 8; ++j) + { + unsigned int jcnt; + unsigned int val; + + jcnt = count[j]; + if (jcnt == 0) + continue; + + if (unlikely (jcnt > (1U << j))) + { + elf_uncompress_failed (); + return 0; + } + + /* There are JCNT values that have this length, the values + starting from START[j] continuing through NEXT[VAL]. Those + values are assigned consecutive values starting at CODE. */ + + val = start[j]; + for (i = 0; i < jcnt; ++i) + { + uint16_t tval; + size_t ind; + unsigned int incr; + + /* In the compressed bit stream, the value VAL is encoded as + J bits with the value C. */ + + if (unlikely ((val & ~ZLIB_HUFFMAN_VALUE_MASK) != 0)) + { + elf_uncompress_failed (); + return 0; + } + + tval = val | ((j - 1) << ZLIB_HUFFMAN_BITS_SHIFT); + + /* The table lookup uses 8 bits. If J is less than 8, we + don't know what the other bits will be. We need to fill + in all possibilities in the table. Since the Huffman + code is unambiguous, those entries can't be used for any + other code. */ + + for (ind = code; ind < 0x100; ind += 1 << j) + { + if (unlikely (table[ind] != 0)) + { + elf_uncompress_failed (); + return 0; + } + table[ind] = tval; + } + + /* Advance to the next value with this length. */ + if (i + 1 < jcnt) + val = next[val]; + + /* The Huffman codes are stored in the bitstream with the + most significant bit first, as is required to make them + unambiguous. The effect is that when we read them from + the bitstream we see the bit sequence in reverse order: + the most significant bit of the Huffman code is the least + significant bit of the value we read from the bitstream. + That means that to make our table lookups work, we need + to reverse the bits of CODE. Since reversing bits is + tedious and in general requires using a table, we instead + increment CODE in reverse order. That is, if the number + of bits we are currently using, here named J, is 3, we + count as 000, 100, 010, 110, 001, 101, 011, 111, which is + to say the numbers from 0 to 7 but with the bits + reversed. Going to more bits, aka incrementing J, + effectively just adds more zero bits as the beginning, + and as such does not change the numeric value of CODE. + + To increment CODE of length J in reverse order, find the + most significant zero bit and set it to one while + clearing all higher bits. In other words, add 1 modulo + 2^J, only reversed. */ + + incr = 1U << (j - 1); + while ((code & incr) != 0) + incr >>= 1; + if (incr == 0) + code = 0; + else + { + code &= incr - 1; + code += incr; + } + } + } + + /* Handle the values that require a secondary table. */ + + /* Set FIRSTCODE, the number at which the codes start, for each + length. */ + + for (j = 9; j < 16; j++) + { + unsigned int jcnt; + unsigned int k; + + jcnt = count[j]; + if (jcnt == 0) + continue; + + /* There are JCNT values that have this length, the values + starting from START[j]. Those values are assigned + consecutive values starting at CODE. */ + + firstcode[j - 9] = code; + + /* Reverse add JCNT to CODE modulo 2^J. */ + for (k = 0; k < j; ++k) + { + if ((jcnt & (1U << k)) != 0) + { + unsigned int m; + unsigned int bit; + + bit = 1U << (j - k - 1); + for (m = 0; m < j - k; ++m, bit >>= 1) + { + if ((code & bit) == 0) + { + code += bit; + break; + } + code &= ~bit; + } + jcnt &= ~(1U << k); + } + } + if (unlikely (jcnt != 0)) + { + elf_uncompress_failed (); + return 0; + } + } + + /* For J from 9 to 15, inclusive, we store COUNT[J] consecutive + values starting at START[J] with consecutive codes starting at + FIRSTCODE[J - 9]. In the primary table we need to point to the + secondary table, and the secondary table will be indexed by J - 9 + bits. We count down from 15 so that we install the larger + secondary tables first, as the smaller ones may be embedded in + the larger ones. */ + + next_secondary = 0; /* Index of next secondary table (after primary). */ + for (j = 15; j >= 9; j--) + { + unsigned int jcnt; + unsigned int val; + size_t primary; /* Current primary index. */ + size_t secondary; /* Offset to current secondary table. */ + size_t secondary_bits; /* Bit size of current secondary table. */ + + jcnt = count[j]; + if (jcnt == 0) + continue; + + val = start[j]; + code = firstcode[j - 9]; + primary = 0x100; + secondary = 0; + secondary_bits = 0; + for (i = 0; i < jcnt; ++i) + { + uint16_t tval; + size_t ind; + unsigned int incr; + + if ((code & 0xff) != primary) + { + uint16_t tprimary; + + /* Fill in a new primary table entry. */ + + primary = code & 0xff; + + tprimary = table[primary]; + if (tprimary == 0) + { + /* Start a new secondary table. */ + + if (unlikely ((next_secondary & ZLIB_HUFFMAN_VALUE_MASK) + != next_secondary)) + { + elf_uncompress_failed (); + return 0; + } + + secondary = next_secondary; + secondary_bits = j - 8; + next_secondary += 1 << secondary_bits; + table[primary] = (secondary + + ((j - 8) << ZLIB_HUFFMAN_BITS_SHIFT) + + (1U << ZLIB_HUFFMAN_SECONDARY_SHIFT)); + } + else + { + /* There is an existing entry. It had better be a + secondary table with enough bits. */ + if (unlikely ((tprimary + & (1U << ZLIB_HUFFMAN_SECONDARY_SHIFT)) + == 0)) + { + elf_uncompress_failed (); + return 0; + } + secondary = tprimary & ZLIB_HUFFMAN_VALUE_MASK; + secondary_bits = ((tprimary >> ZLIB_HUFFMAN_BITS_SHIFT) + & ZLIB_HUFFMAN_BITS_MASK); + if (unlikely (secondary_bits < j - 8)) + { + elf_uncompress_failed (); + return 0; + } + } + } + + /* Fill in secondary table entries. */ + + tval = val | ((j - 8) << ZLIB_HUFFMAN_BITS_SHIFT); + + for (ind = code >> 8; + ind < (1U << secondary_bits); + ind += 1U << (j - 8)) + { + if (unlikely (table[secondary + 0x100 + ind] != 0)) + { + elf_uncompress_failed (); + return 0; + } + table[secondary + 0x100 + ind] = tval; + } + + if (i + 1 < jcnt) + val = next[val]; + + incr = 1U << (j - 1); + while ((code & incr) != 0) + incr >>= 1; + if (incr == 0) + code = 0; + else + { + code &= incr - 1; + code += incr; + } + } + } + +#ifdef BACKTRACE_GENERATE_FIXED_HUFFMAN_TABLE + final_next_secondary = next_secondary; +#endif + + return 1; +} + +#ifdef BACKTRACE_GENERATE_FIXED_HUFFMAN_TABLE + +/* Used to generate the fixed Huffman table for block type 1. */ + +#include + +static uint16_t table[ZLIB_TABLE_SIZE]; +static unsigned char codes[288]; + +int +main () +{ + size_t i; + + for (i = 0; i <= 143; ++i) + codes[i] = 8; + for (i = 144; i <= 255; ++i) + codes[i] = 9; + for (i = 256; i <= 279; ++i) + codes[i] = 7; + for (i = 280; i <= 287; ++i) + codes[i] = 8; + if (!elf_zlib_inflate_table (&codes[0], 288, &table[0], &table[0])) + { + fprintf (stderr, "elf_zlib_inflate_table failed\n"); + exit (EXIT_FAILURE); + } + + printf ("static const uint16_t elf_zlib_default_table[%#zx] =\n", + final_next_secondary + 0x100); + printf ("{\n"); + for (i = 0; i < final_next_secondary + 0x100; i += 8) + { + size_t j; + + printf (" "); + for (j = i; j < final_next_secondary + 0x100 && j < i + 8; ++j) + printf (" %#x,", table[j]); + printf ("\n"); + } + printf ("};\n"); + printf ("\n"); + + for (i = 0; i < 32; ++i) + codes[i] = 5; + if (!elf_zlib_inflate_table (&codes[0], 32, &table[0], &table[0])) + { + fprintf (stderr, "elf_zlib_inflate_table failed\n"); + exit (EXIT_FAILURE); + } + + printf ("static const uint16_t elf_zlib_default_dist_table[%#zx] =\n", + final_next_secondary + 0x100); + printf ("{\n"); + for (i = 0; i < final_next_secondary + 0x100; i += 8) + { + size_t j; + + printf (" "); + for (j = i; j < final_next_secondary + 0x100 && j < i + 8; ++j) + printf (" %#x,", table[j]); + printf ("\n"); + } + printf ("};\n"); + + return 0; +} + +#endif + +/* The fixed tables generated by the #ifdef'ed out main function + above. */ + +static const uint16_t elf_zlib_default_table[0x170] = +{ + 0xd00, 0xe50, 0xe10, 0xf18, 0xd10, 0xe70, 0xe30, 0x1230, + 0xd08, 0xe60, 0xe20, 0x1210, 0xe00, 0xe80, 0xe40, 0x1250, + 0xd04, 0xe58, 0xe18, 0x1200, 0xd14, 0xe78, 0xe38, 0x1240, + 0xd0c, 0xe68, 0xe28, 0x1220, 0xe08, 0xe88, 0xe48, 0x1260, + 0xd02, 0xe54, 0xe14, 0xf1c, 0xd12, 0xe74, 0xe34, 0x1238, + 0xd0a, 0xe64, 0xe24, 0x1218, 0xe04, 0xe84, 0xe44, 0x1258, + 0xd06, 0xe5c, 0xe1c, 0x1208, 0xd16, 0xe7c, 0xe3c, 0x1248, + 0xd0e, 0xe6c, 0xe2c, 0x1228, 0xe0c, 0xe8c, 0xe4c, 0x1268, + 0xd01, 0xe52, 0xe12, 0xf1a, 0xd11, 0xe72, 0xe32, 0x1234, + 0xd09, 0xe62, 0xe22, 0x1214, 0xe02, 0xe82, 0xe42, 0x1254, + 0xd05, 0xe5a, 0xe1a, 0x1204, 0xd15, 0xe7a, 0xe3a, 0x1244, + 0xd0d, 0xe6a, 0xe2a, 0x1224, 0xe0a, 0xe8a, 0xe4a, 0x1264, + 0xd03, 0xe56, 0xe16, 0xf1e, 0xd13, 0xe76, 0xe36, 0x123c, + 0xd0b, 0xe66, 0xe26, 0x121c, 0xe06, 0xe86, 0xe46, 0x125c, + 0xd07, 0xe5e, 0xe1e, 0x120c, 0xd17, 0xe7e, 0xe3e, 0x124c, + 0xd0f, 0xe6e, 0xe2e, 0x122c, 0xe0e, 0xe8e, 0xe4e, 0x126c, + 0xd00, 0xe51, 0xe11, 0xf19, 0xd10, 0xe71, 0xe31, 0x1232, + 0xd08, 0xe61, 0xe21, 0x1212, 0xe01, 0xe81, 0xe41, 0x1252, + 0xd04, 0xe59, 0xe19, 0x1202, 0xd14, 0xe79, 0xe39, 0x1242, + 0xd0c, 0xe69, 0xe29, 0x1222, 0xe09, 0xe89, 0xe49, 0x1262, + 0xd02, 0xe55, 0xe15, 0xf1d, 0xd12, 0xe75, 0xe35, 0x123a, + 0xd0a, 0xe65, 0xe25, 0x121a, 0xe05, 0xe85, 0xe45, 0x125a, + 0xd06, 0xe5d, 0xe1d, 0x120a, 0xd16, 0xe7d, 0xe3d, 0x124a, + 0xd0e, 0xe6d, 0xe2d, 0x122a, 0xe0d, 0xe8d, 0xe4d, 0x126a, + 0xd01, 0xe53, 0xe13, 0xf1b, 0xd11, 0xe73, 0xe33, 0x1236, + 0xd09, 0xe63, 0xe23, 0x1216, 0xe03, 0xe83, 0xe43, 0x1256, + 0xd05, 0xe5b, 0xe1b, 0x1206, 0xd15, 0xe7b, 0xe3b, 0x1246, + 0xd0d, 0xe6b, 0xe2b, 0x1226, 0xe0b, 0xe8b, 0xe4b, 0x1266, + 0xd03, 0xe57, 0xe17, 0xf1f, 0xd13, 0xe77, 0xe37, 0x123e, + 0xd0b, 0xe67, 0xe27, 0x121e, 0xe07, 0xe87, 0xe47, 0x125e, + 0xd07, 0xe5f, 0xe1f, 0x120e, 0xd17, 0xe7f, 0xe3f, 0x124e, + 0xd0f, 0xe6f, 0xe2f, 0x122e, 0xe0f, 0xe8f, 0xe4f, 0x126e, + 0x290, 0x291, 0x292, 0x293, 0x294, 0x295, 0x296, 0x297, + 0x298, 0x299, 0x29a, 0x29b, 0x29c, 0x29d, 0x29e, 0x29f, + 0x2a0, 0x2a1, 0x2a2, 0x2a3, 0x2a4, 0x2a5, 0x2a6, 0x2a7, + 0x2a8, 0x2a9, 0x2aa, 0x2ab, 0x2ac, 0x2ad, 0x2ae, 0x2af, + 0x2b0, 0x2b1, 0x2b2, 0x2b3, 0x2b4, 0x2b5, 0x2b6, 0x2b7, + 0x2b8, 0x2b9, 0x2ba, 0x2bb, 0x2bc, 0x2bd, 0x2be, 0x2bf, + 0x2c0, 0x2c1, 0x2c2, 0x2c3, 0x2c4, 0x2c5, 0x2c6, 0x2c7, + 0x2c8, 0x2c9, 0x2ca, 0x2cb, 0x2cc, 0x2cd, 0x2ce, 0x2cf, + 0x2d0, 0x2d1, 0x2d2, 0x2d3, 0x2d4, 0x2d5, 0x2d6, 0x2d7, + 0x2d8, 0x2d9, 0x2da, 0x2db, 0x2dc, 0x2dd, 0x2de, 0x2df, + 0x2e0, 0x2e1, 0x2e2, 0x2e3, 0x2e4, 0x2e5, 0x2e6, 0x2e7, + 0x2e8, 0x2e9, 0x2ea, 0x2eb, 0x2ec, 0x2ed, 0x2ee, 0x2ef, + 0x2f0, 0x2f1, 0x2f2, 0x2f3, 0x2f4, 0x2f5, 0x2f6, 0x2f7, + 0x2f8, 0x2f9, 0x2fa, 0x2fb, 0x2fc, 0x2fd, 0x2fe, 0x2ff, +}; + +static const uint16_t elf_zlib_default_dist_table[0x100] = +{ + 0x800, 0x810, 0x808, 0x818, 0x804, 0x814, 0x80c, 0x81c, + 0x802, 0x812, 0x80a, 0x81a, 0x806, 0x816, 0x80e, 0x81e, + 0x801, 0x811, 0x809, 0x819, 0x805, 0x815, 0x80d, 0x81d, + 0x803, 0x813, 0x80b, 0x81b, 0x807, 0x817, 0x80f, 0x81f, + 0x800, 0x810, 0x808, 0x818, 0x804, 0x814, 0x80c, 0x81c, + 0x802, 0x812, 0x80a, 0x81a, 0x806, 0x816, 0x80e, 0x81e, + 0x801, 0x811, 0x809, 0x819, 0x805, 0x815, 0x80d, 0x81d, + 0x803, 0x813, 0x80b, 0x81b, 0x807, 0x817, 0x80f, 0x81f, + 0x800, 0x810, 0x808, 0x818, 0x804, 0x814, 0x80c, 0x81c, + 0x802, 0x812, 0x80a, 0x81a, 0x806, 0x816, 0x80e, 0x81e, + 0x801, 0x811, 0x809, 0x819, 0x805, 0x815, 0x80d, 0x81d, + 0x803, 0x813, 0x80b, 0x81b, 0x807, 0x817, 0x80f, 0x81f, + 0x800, 0x810, 0x808, 0x818, 0x804, 0x814, 0x80c, 0x81c, + 0x802, 0x812, 0x80a, 0x81a, 0x806, 0x816, 0x80e, 0x81e, + 0x801, 0x811, 0x809, 0x819, 0x805, 0x815, 0x80d, 0x81d, + 0x803, 0x813, 0x80b, 0x81b, 0x807, 0x817, 0x80f, 0x81f, + 0x800, 0x810, 0x808, 0x818, 0x804, 0x814, 0x80c, 0x81c, + 0x802, 0x812, 0x80a, 0x81a, 0x806, 0x816, 0x80e, 0x81e, + 0x801, 0x811, 0x809, 0x819, 0x805, 0x815, 0x80d, 0x81d, + 0x803, 0x813, 0x80b, 0x81b, 0x807, 0x817, 0x80f, 0x81f, + 0x800, 0x810, 0x808, 0x818, 0x804, 0x814, 0x80c, 0x81c, + 0x802, 0x812, 0x80a, 0x81a, 0x806, 0x816, 0x80e, 0x81e, + 0x801, 0x811, 0x809, 0x819, 0x805, 0x815, 0x80d, 0x81d, + 0x803, 0x813, 0x80b, 0x81b, 0x807, 0x817, 0x80f, 0x81f, + 0x800, 0x810, 0x808, 0x818, 0x804, 0x814, 0x80c, 0x81c, + 0x802, 0x812, 0x80a, 0x81a, 0x806, 0x816, 0x80e, 0x81e, + 0x801, 0x811, 0x809, 0x819, 0x805, 0x815, 0x80d, 0x81d, + 0x803, 0x813, 0x80b, 0x81b, 0x807, 0x817, 0x80f, 0x81f, + 0x800, 0x810, 0x808, 0x818, 0x804, 0x814, 0x80c, 0x81c, + 0x802, 0x812, 0x80a, 0x81a, 0x806, 0x816, 0x80e, 0x81e, + 0x801, 0x811, 0x809, 0x819, 0x805, 0x815, 0x80d, 0x81d, + 0x803, 0x813, 0x80b, 0x81b, 0x807, 0x817, 0x80f, 0x81f, +}; + +/* Inflate a zlib stream from PIN/SIN to POUT/SOUT. Return 1 on + success, 0 on some error parsing the stream. */ + +static int +elf_zlib_inflate (const unsigned char *pin, size_t sin, uint16_t *zdebug_table, + unsigned char *pout, size_t sout) +{ + unsigned char *porigout; + const unsigned char *pinend; + unsigned char *poutend; + + /* We can apparently see multiple zlib streams concatenated + together, so keep going as long as there is something to read. + The last 4 bytes are the checksum. */ + porigout = pout; + pinend = pin + sin; + poutend = pout + sout; + while ((pinend - pin) > 4) + { + uint64_t val; + unsigned int bits; + int last; + + /* Read the two byte zlib header. */ + + if (unlikely ((pin[0] & 0xf) != 8)) /* 8 is zlib encoding. */ + { + /* Unknown compression method. */ + elf_uncompress_failed (); + return 0; + } + if (unlikely ((pin[0] >> 4) > 7)) + { + /* Window size too large. Other than this check, we don't + care about the window size. */ + elf_uncompress_failed (); + return 0; + } + if (unlikely ((pin[1] & 0x20) != 0)) + { + /* Stream expects a predefined dictionary, but we have no + dictionary. */ + elf_uncompress_failed (); + return 0; + } + val = (pin[0] << 8) | pin[1]; + if (unlikely (val % 31 != 0)) + { + /* Header check failure. */ + elf_uncompress_failed (); + return 0; + } + pin += 2; + + /* Align PIN to a 32-bit boundary. */ + + val = 0; + bits = 0; + while ((((uintptr_t) pin) & 3) != 0) + { + val |= (uint64_t)*pin << bits; + bits += 8; + ++pin; + } + + /* Read blocks until one is marked last. */ + + last = 0; + + while (!last) + { + unsigned int type; + const uint16_t *tlit; + const uint16_t *tdist; + + if (!elf_fetch_bits (&pin, pinend, &val, &bits)) + return 0; + + last = val & 1; + type = (val >> 1) & 3; + val >>= 3; + bits -= 3; + + if (unlikely (type == 3)) + { + /* Invalid block type. */ + elf_uncompress_failed (); + return 0; + } + + if (type == 0) + { + uint16_t len; + uint16_t lenc; + + /* An uncompressed block. */ + + /* If we've read ahead more than a byte, back up. */ + while (bits >= 8) + { + --pin; + bits -= 8; + } + + val = 0; + bits = 0; + if (unlikely ((pinend - pin) < 4)) + { + /* Missing length. */ + elf_uncompress_failed (); + return 0; + } + len = pin[0] | (pin[1] << 8); + lenc = pin[2] | (pin[3] << 8); + pin += 4; + lenc = ~lenc; + if (unlikely (len != lenc)) + { + /* Corrupt data. */ + elf_uncompress_failed (); + return 0; + } + if (unlikely (len > (unsigned int) (pinend - pin) + || len > (unsigned int) (poutend - pout))) + { + /* Not enough space in buffers. */ + elf_uncompress_failed (); + return 0; + } + memcpy (pout, pin, len); + pout += len; + pin += len; + + /* Align PIN. */ + while ((((uintptr_t) pin) & 3) != 0) + { + val |= (uint64_t)*pin << bits; + bits += 8; + ++pin; + } + + /* Go around to read the next block. */ + continue; + } + + if (type == 1) + { + tlit = elf_zlib_default_table; + tdist = elf_zlib_default_dist_table; + } + else + { + unsigned int nlit; + unsigned int ndist; + unsigned int nclen; + unsigned char codebits[19]; + unsigned char *plenbase; + unsigned char *plen; + unsigned char *plenend; + + /* Read a Huffman encoding table. The various magic + numbers here are from RFC 1951. */ + + if (!elf_fetch_bits (&pin, pinend, &val, &bits)) + return 0; + + nlit = (val & 0x1f) + 257; + val >>= 5; + ndist = (val & 0x1f) + 1; + val >>= 5; + nclen = (val & 0xf) + 4; + val >>= 4; + bits -= 14; + if (unlikely (nlit > 286 || ndist > 30)) + { + /* Values out of range. */ + elf_uncompress_failed (); + return 0; + } + + /* Read and build the table used to compress the + literal, length, and distance codes. */ + + memset(&codebits[0], 0, 19); + + /* There are always at least 4 elements in the + table. */ + + if (!elf_fetch_bits (&pin, pinend, &val, &bits)) + return 0; + + codebits[16] = val & 7; + codebits[17] = (val >> 3) & 7; + codebits[18] = (val >> 6) & 7; + codebits[0] = (val >> 9) & 7; + val >>= 12; + bits -= 12; + + if (nclen == 4) + goto codebitsdone; + + codebits[8] = val & 7; + val >>= 3; + bits -= 3; + + if (nclen == 5) + goto codebitsdone; + + if (!elf_fetch_bits (&pin, pinend, &val, &bits)) + return 0; + + codebits[7] = val & 7; + val >>= 3; + bits -= 3; + + if (nclen == 6) + goto codebitsdone; + + codebits[9] = val & 7; + val >>= 3; + bits -= 3; + + if (nclen == 7) + goto codebitsdone; + + codebits[6] = val & 7; + val >>= 3; + bits -= 3; + + if (nclen == 8) + goto codebitsdone; + + codebits[10] = val & 7; + val >>= 3; + bits -= 3; + + if (nclen == 9) + goto codebitsdone; + + codebits[5] = val & 7; + val >>= 3; + bits -= 3; + + if (nclen == 10) + goto codebitsdone; + + if (!elf_fetch_bits (&pin, pinend, &val, &bits)) + return 0; + + codebits[11] = val & 7; + val >>= 3; + bits -= 3; + + if (nclen == 11) + goto codebitsdone; + + codebits[4] = val & 7; + val >>= 3; + bits -= 3; + + if (nclen == 12) + goto codebitsdone; + + codebits[12] = val & 7; + val >>= 3; + bits -= 3; + + if (nclen == 13) + goto codebitsdone; + + codebits[3] = val & 7; + val >>= 3; + bits -= 3; + + if (nclen == 14) + goto codebitsdone; + + codebits[13] = val & 7; + val >>= 3; + bits -= 3; + + if (nclen == 15) + goto codebitsdone; + + if (!elf_fetch_bits (&pin, pinend, &val, &bits)) + return 0; + + codebits[2] = val & 7; + val >>= 3; + bits -= 3; + + if (nclen == 16) + goto codebitsdone; + + codebits[14] = val & 7; + val >>= 3; + bits -= 3; + + if (nclen == 17) + goto codebitsdone; + + codebits[1] = val & 7; + val >>= 3; + bits -= 3; + + if (nclen == 18) + goto codebitsdone; + + codebits[15] = val & 7; + val >>= 3; + bits -= 3; + + codebitsdone: + + if (!elf_zlib_inflate_table (codebits, 19, zdebug_table, + zdebug_table)) + return 0; + + /* Read the compressed bit lengths of the literal, + length, and distance codes. We have allocated space + at the end of zdebug_table to hold them. */ + + plenbase = (((unsigned char *) zdebug_table) + + ZLIB_TABLE_CODELEN_OFFSET); + plen = plenbase; + plenend = plen + nlit + ndist; + while (plen < plenend) + { + uint16_t t; + unsigned int b; + uint16_t v; + + if (!elf_fetch_bits (&pin, pinend, &val, &bits)) + return 0; + + t = zdebug_table[val & 0xff]; + + /* The compression here uses bit lengths up to 7, so + a secondary table is never necessary. */ + if (unlikely ((t & (1U << ZLIB_HUFFMAN_SECONDARY_SHIFT)) + != 0)) + { + elf_uncompress_failed (); + return 0; + } + + b = (t >> ZLIB_HUFFMAN_BITS_SHIFT) & ZLIB_HUFFMAN_BITS_MASK; + val >>= b + 1; + bits -= b + 1; + + v = t & ZLIB_HUFFMAN_VALUE_MASK; + if (v < 16) + *plen++ = v; + else if (v == 16) + { + unsigned int c; + unsigned int prev; + + /* Copy previous entry 3 to 6 times. */ + + if (unlikely (plen == plenbase)) + { + elf_uncompress_failed (); + return 0; + } + + /* We used up to 7 bits since the last + elf_fetch_bits, so we have at least 8 bits + available here. */ + + c = 3 + (val & 0x3); + val >>= 2; + bits -= 2; + if (unlikely ((unsigned int) (plenend - plen) < c)) + { + elf_uncompress_failed (); + return 0; + } + + prev = plen[-1]; + switch (c) + { + case 6: + *plen++ = prev; + ATTRIBUTE_FALLTHROUGH; + case 5: + *plen++ = prev; + ATTRIBUTE_FALLTHROUGH; + case 4: + *plen++ = prev; + } + *plen++ = prev; + *plen++ = prev; + *plen++ = prev; + } + else if (v == 17) + { + unsigned int c; + + /* Store zero 3 to 10 times. */ + + /* We used up to 7 bits since the last + elf_fetch_bits, so we have at least 8 bits + available here. */ + + c = 3 + (val & 0x7); + val >>= 3; + bits -= 3; + if (unlikely ((unsigned int) (plenend - plen) < c)) + { + elf_uncompress_failed (); + return 0; + } + + switch (c) + { + case 10: + *plen++ = 0; + ATTRIBUTE_FALLTHROUGH; + case 9: + *plen++ = 0; + ATTRIBUTE_FALLTHROUGH; + case 8: + *plen++ = 0; + ATTRIBUTE_FALLTHROUGH; + case 7: + *plen++ = 0; + ATTRIBUTE_FALLTHROUGH; + case 6: + *plen++ = 0; + ATTRIBUTE_FALLTHROUGH; + case 5: + *plen++ = 0; + ATTRIBUTE_FALLTHROUGH; + case 4: + *plen++ = 0; + } + *plen++ = 0; + *plen++ = 0; + *plen++ = 0; + } + else if (v == 18) + { + unsigned int c; + + /* Store zero 11 to 138 times. */ + + /* We used up to 7 bits since the last + elf_fetch_bits, so we have at least 8 bits + available here. */ + + c = 11 + (val & 0x7f); + val >>= 7; + bits -= 7; + if (unlikely ((unsigned int) (plenend - plen) < c)) + { + elf_uncompress_failed (); + return 0; + } + + memset (plen, 0, c); + plen += c; + } + else + { + elf_uncompress_failed (); + return 0; + } + } + + /* Make sure that the stop code can appear. */ + + plen = plenbase; + if (unlikely (plen[256] == 0)) + { + elf_uncompress_failed (); + return 0; + } + + /* Build the decompression tables. */ + + if (!elf_zlib_inflate_table (plen, nlit, zdebug_table, + zdebug_table)) + return 0; + if (!elf_zlib_inflate_table (plen + nlit, ndist, zdebug_table, + (zdebug_table + + ZLIB_HUFFMAN_TABLE_SIZE))) + return 0; + tlit = zdebug_table; + tdist = zdebug_table + ZLIB_HUFFMAN_TABLE_SIZE; + } + + /* Inflate values until the end of the block. This is the + main loop of the inflation code. */ + + while (1) + { + uint16_t t; + unsigned int b; + uint16_t v; + unsigned int lit; + + if (!elf_fetch_bits (&pin, pinend, &val, &bits)) + return 0; + + t = tlit[val & 0xff]; + b = (t >> ZLIB_HUFFMAN_BITS_SHIFT) & ZLIB_HUFFMAN_BITS_MASK; + v = t & ZLIB_HUFFMAN_VALUE_MASK; + + if ((t & (1U << ZLIB_HUFFMAN_SECONDARY_SHIFT)) == 0) + { + lit = v; + val >>= b + 1; + bits -= b + 1; + } + else + { + t = tlit[v + 0x100 + ((val >> 8) & ((1U << b) - 1))]; + b = (t >> ZLIB_HUFFMAN_BITS_SHIFT) & ZLIB_HUFFMAN_BITS_MASK; + lit = t & ZLIB_HUFFMAN_VALUE_MASK; + val >>= b + 8; + bits -= b + 8; + } + + if (lit < 256) + { + if (unlikely (pout == poutend)) + { + elf_uncompress_failed (); + return 0; + } + + *pout++ = lit; + + /* We will need to write the next byte soon. We ask + for high temporal locality because we will write + to the whole cache line soon. */ + __builtin_prefetch (pout, 1, 3); + } + else if (lit == 256) + { + /* The end of the block. */ + break; + } + else + { + unsigned int dist; + unsigned int len; + + /* Convert lit into a length. */ + + if (lit < 265) + len = lit - 257 + 3; + else if (lit == 285) + len = 258; + else if (unlikely (lit > 285)) + { + elf_uncompress_failed (); + return 0; + } + else + { + unsigned int extra; + + if (!elf_fetch_bits (&pin, pinend, &val, &bits)) + return 0; + + /* This is an expression for the table of length + codes in RFC 1951 3.2.5. */ + lit -= 265; + extra = (lit >> 2) + 1; + len = (lit & 3) << extra; + len += 11; + len += ((1U << (extra - 1)) - 1) << 3; + len += val & ((1U << extra) - 1); + val >>= extra; + bits -= extra; + } + + if (!elf_fetch_bits (&pin, pinend, &val, &bits)) + return 0; + + t = tdist[val & 0xff]; + b = (t >> ZLIB_HUFFMAN_BITS_SHIFT) & ZLIB_HUFFMAN_BITS_MASK; + v = t & ZLIB_HUFFMAN_VALUE_MASK; + + if ((t & (1U << ZLIB_HUFFMAN_SECONDARY_SHIFT)) == 0) + { + dist = v; + val >>= b + 1; + bits -= b + 1; + } + else + { + t = tdist[v + 0x100 + ((val >> 8) & ((1U << b) - 1))]; + b = ((t >> ZLIB_HUFFMAN_BITS_SHIFT) + & ZLIB_HUFFMAN_BITS_MASK); + dist = t & ZLIB_HUFFMAN_VALUE_MASK; + val >>= b + 8; + bits -= b + 8; + } + + /* Convert dist to a distance. */ + + if (dist == 0) + { + /* A distance of 1. A common case, meaning + repeat the last character LEN times. */ + + if (unlikely (pout == porigout)) + { + elf_uncompress_failed (); + return 0; + } + + if (unlikely ((unsigned int) (poutend - pout) < len)) + { + elf_uncompress_failed (); + return 0; + } + + memset (pout, pout[-1], len); + pout += len; + } + else if (unlikely (dist > 29)) + { + elf_uncompress_failed (); + return 0; + } + else + { + if (dist < 4) + dist = dist + 1; + else + { + unsigned int extra; + + if (!elf_fetch_bits (&pin, pinend, &val, &bits)) + return 0; + + /* This is an expression for the table of + distance codes in RFC 1951 3.2.5. */ + dist -= 4; + extra = (dist >> 1) + 1; + dist = (dist & 1) << extra; + dist += 5; + dist += ((1U << (extra - 1)) - 1) << 2; + dist += val & ((1U << extra) - 1); + val >>= extra; + bits -= extra; + } + + /* Go back dist bytes, and copy len bytes from + there. */ + + if (unlikely ((unsigned int) (pout - porigout) < dist)) + { + elf_uncompress_failed (); + return 0; + } + + if (unlikely ((unsigned int) (poutend - pout) < len)) + { + elf_uncompress_failed (); + return 0; + } + + if (dist >= len) + { + memcpy (pout, pout - dist, len); + pout += len; + } + else + { + while (len > 0) + { + unsigned int copy; + + copy = len < dist ? len : dist; + memcpy (pout, pout - dist, copy); + len -= copy; + pout += copy; + } + } + } + } + } + } + } + + /* We should have filled the output buffer. */ + if (unlikely (pout != poutend)) + { + elf_uncompress_failed (); + return 0; + } + + return 1; +} + +/* Verify the zlib checksum. The checksum is in the 4 bytes at + CHECKBYTES, and the uncompressed data is at UNCOMPRESSED / + UNCOMPRESSED_SIZE. Returns 1 on success, 0 on failure. */ + +static int +elf_zlib_verify_checksum (const unsigned char *checkbytes, + const unsigned char *uncompressed, + size_t uncompressed_size) +{ + unsigned int i; + unsigned int cksum; + const unsigned char *p; + uint32_t s1; + uint32_t s2; + size_t hsz; + + cksum = 0; + for (i = 0; i < 4; i++) + cksum = (cksum << 8) | checkbytes[i]; + + s1 = 1; + s2 = 0; + + /* Minimize modulo operations. */ + + p = uncompressed; + hsz = uncompressed_size; + while (hsz >= 5552) + { + for (i = 0; i < 5552; i += 16) + { + /* Manually unroll loop 16 times. */ + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + } + hsz -= 5552; + s1 %= 65521; + s2 %= 65521; + } + + while (hsz >= 16) + { + /* Manually unroll loop 16 times. */ + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + s1 = s1 + *p++; + s2 = s2 + s1; + + hsz -= 16; + } + + for (i = 0; i < hsz; ++i) + { + s1 = s1 + *p++; + s2 = s2 + s1; + } + + s1 %= 65521; + s2 %= 65521; + + if (unlikely ((s2 << 16) + s1 != cksum)) + { + elf_uncompress_failed (); + return 0; + } + + return 1; +} + +/* Inflate a zlib stream from PIN/SIN to POUT/SOUT, and verify the + checksum. Return 1 on success, 0 on error. */ + +static int +elf_zlib_inflate_and_verify (const unsigned char *pin, size_t sin, + uint16_t *zdebug_table, unsigned char *pout, + size_t sout) +{ + if (!elf_zlib_inflate (pin, sin, zdebug_table, pout, sout)) + return 0; + if (!elf_zlib_verify_checksum (pin + sin - 4, pout, sout)) + return 0; + return 1; +} + +/* For working memory during zstd compression, we need + - a literal length FSE table: 512 64-bit values == 4096 bytes + - a match length FSE table: 512 64-bit values == 4096 bytes + - a offset FSE table: 256 64-bit values == 2048 bytes + - a Huffman tree: 2048 uint16_t values == 4096 bytes + - scratch space, one of + - to build an FSE table: 512 uint16_t values == 1024 bytes + - to build a Huffman tree: 512 uint16_t + 256 uint32_t == 2048 bytes +*/ + +#define ZSTD_TABLE_SIZE \ + (2 * 512 * sizeof (struct elf_zstd_fse_baseline_entry) \ + + 256 * sizeof (struct elf_zstd_fse_baseline_entry) \ + + 2048 * sizeof (uint16_t) \ + + 512 * sizeof (uint16_t) + 256 * sizeof (uint32_t)) + +#define ZSTD_TABLE_LITERAL_FSE_OFFSET (0) + +#define ZSTD_TABLE_MATCH_FSE_OFFSET \ + (512 * sizeof (struct elf_zstd_fse_baseline_entry)) + +#define ZSTD_TABLE_OFFSET_FSE_OFFSET \ + (ZSTD_TABLE_MATCH_FSE_OFFSET \ + + 512 * sizeof (struct elf_zstd_fse_baseline_entry)) + +#define ZSTD_TABLE_HUFFMAN_OFFSET \ + (ZSTD_TABLE_OFFSET_FSE_OFFSET \ + + 256 * sizeof (struct elf_zstd_fse_baseline_entry)) + +#define ZSTD_TABLE_WORK_OFFSET \ + (ZSTD_TABLE_HUFFMAN_OFFSET + 2048 * sizeof (uint16_t)) + +/* An entry in a zstd FSE table. */ + +struct elf_zstd_fse_entry +{ + /* The value that this FSE entry represents. */ + unsigned char symbol; + /* The number of bits to read to determine the next state. */ + unsigned char bits; + /* Add the bits to this base to get the next state. */ + uint16_t base; +}; + +static int +elf_zstd_build_fse (const int16_t *, int, uint16_t *, int, + struct elf_zstd_fse_entry *); + +/* Read a zstd FSE table and build the decoding table in *TABLE, updating *PPIN + as it reads. ZDEBUG_TABLE is scratch space; it must be enough for 512 + uint16_t values (1024 bytes). MAXIDX is the maximum number of symbols + permitted. *TABLE_BITS is the maximum number of bits for symbols in the + table: the size of *TABLE is at least 1 << *TABLE_BITS. This updates + *TABLE_BITS to the actual number of bits. Returns 1 on success, 0 on + error. */ + +static int +elf_zstd_read_fse (const unsigned char **ppin, const unsigned char *pinend, + uint16_t *zdebug_table, int maxidx, + struct elf_zstd_fse_entry *table, int *table_bits) +{ + const unsigned char *pin; + int16_t *norm; + uint16_t *next; + uint64_t val; + unsigned int bits; + int accuracy_log; + uint32_t remaining; + uint32_t threshold; + int bits_needed; + int idx; + int prev0; + + pin = *ppin; + + norm = (int16_t *) zdebug_table; + next = zdebug_table + 256; + + if (unlikely (pin + 3 >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + + /* Align PIN to a 32-bit boundary. */ + + val = 0; + bits = 0; + while ((((uintptr_t) pin) & 3) != 0) + { + val |= (uint64_t)*pin << bits; + bits += 8; + ++pin; + } + + if (!elf_fetch_bits (&pin, pinend, &val, &bits)) + return 0; + + accuracy_log = (val & 0xf) + 5; + if (accuracy_log > *table_bits) + { + elf_uncompress_failed (); + return 0; + } + *table_bits = accuracy_log; + val >>= 4; + bits -= 4; + + /* This code is mostly copied from the reference implementation. */ + + /* The number of remaining probabilities, plus 1. This sets the number of + bits that need to be read for the next value. */ + remaining = (1 << accuracy_log) + 1; + + /* The current difference between small and large values, which depends on + the number of remaining values. Small values use one less bit. */ + threshold = 1 << accuracy_log; + + /* The number of bits used to compute threshold. */ + bits_needed = accuracy_log + 1; + + /* The next character value. */ + idx = 0; + + /* Whether the last count was 0. */ + prev0 = 0; + + while (remaining > 1 && idx <= maxidx) + { + uint32_t max; + int32_t count; + + if (!elf_fetch_bits (&pin, pinend, &val, &bits)) + return 0; + + if (prev0) + { + int zidx; + + /* Previous count was 0, so there is a 2-bit repeat flag. If the + 2-bit flag is 0b11, it adds 3 and then there is another repeat + flag. */ + zidx = idx; + while ((val & 0xfff) == 0xfff) + { + zidx += 3 * 6; + val >>= 12; + bits -= 12; + if (!elf_fetch_bits (&pin, pinend, &val, &bits)) + return 0; + } + while ((val & 3) == 3) + { + zidx += 3; + val >>= 2; + bits -= 2; + if (!elf_fetch_bits (&pin, pinend, &val, &bits)) + return 0; + } + /* We have at least 13 bits here, don't need to fetch. */ + zidx += val & 3; + val >>= 2; + bits -= 2; + + if (unlikely (zidx > maxidx)) + { + elf_uncompress_failed (); + return 0; + } + + for (; idx < zidx; idx++) + norm[idx] = 0; + + prev0 = 0; + continue; + } + + max = (2 * threshold - 1) - remaining; + if ((val & (threshold - 1)) < max) + { + /* A small value. */ + count = (int32_t) ((uint32_t) val & (threshold - 1)); + val >>= bits_needed - 1; + bits -= bits_needed - 1; + } + else + { + /* A large value. */ + count = (int32_t) ((uint32_t) val & (2 * threshold - 1)); + if (count >= (int32_t) threshold) + count -= (int32_t) max; + val >>= bits_needed; + bits -= bits_needed; + } + + count--; + if (count >= 0) + remaining -= count; + else + remaining--; + if (unlikely (idx >= 256)) + { + elf_uncompress_failed (); + return 0; + } + norm[idx] = (int16_t) count; + ++idx; + + prev0 = count == 0; + + while (remaining < threshold) + { + bits_needed--; + threshold >>= 1; + } + } + + if (unlikely (remaining != 1)) + { + elf_uncompress_failed (); + return 0; + } + + /* If we've read ahead more than a byte, back up. */ + while (bits >= 8) + { + --pin; + bits -= 8; + } + + *ppin = pin; + + for (; idx <= maxidx; idx++) + norm[idx] = 0; + + return elf_zstd_build_fse (norm, idx, next, *table_bits, table); +} + +/* Build the FSE decoding table from a list of probabilities. This reads from + NORM of length IDX, uses NEXT as scratch space, and writes to *TABLE, whose + size is TABLE_BITS. */ + +static int +elf_zstd_build_fse (const int16_t *norm, int idx, uint16_t *next, + int table_bits, struct elf_zstd_fse_entry *table) +{ + int table_size; + int high_threshold; + int i; + int pos; + int step; + int mask; + + table_size = 1 << table_bits; + high_threshold = table_size - 1; + for (i = 0; i < idx; i++) + { + int16_t n; + + n = norm[i]; + if (n >= 0) + next[i] = (uint16_t) n; + else + { + table[high_threshold].symbol = (unsigned char) i; + high_threshold--; + next[i] = 1; + } + } + + pos = 0; + step = (table_size >> 1) + (table_size >> 3) + 3; + mask = table_size - 1; + for (i = 0; i < idx; i++) + { + int n; + int j; + + n = (int) norm[i]; + for (j = 0; j < n; j++) + { + table[pos].symbol = (unsigned char) i; + pos = (pos + step) & mask; + while (unlikely (pos > high_threshold)) + pos = (pos + step) & mask; + } + } + if (unlikely (pos != 0)) + { + elf_uncompress_failed (); + return 0; + } + + for (i = 0; i < table_size; i++) + { + unsigned char sym; + uint16_t next_state; + int high_bit; + int bits; + + sym = table[i].symbol; + next_state = next[sym]; + ++next[sym]; + + if (next_state == 0) + { + elf_uncompress_failed (); + return 0; + } + high_bit = 31 - __builtin_clz (next_state); + + bits = table_bits - high_bit; + table[i].bits = (unsigned char) bits; + table[i].base = (uint16_t) ((next_state << bits) - table_size); + } + + return 1; +} + +/* Encode the baseline and bits into a single 32-bit value. */ + +#define ZSTD_ENCODE_BASELINE_BITS(baseline, basebits) \ + ((uint32_t)(baseline) | ((uint32_t)(basebits) << 24)) + +#define ZSTD_DECODE_BASELINE(baseline_basebits) \ + ((uint32_t)(baseline_basebits) & 0xffffff) + +#define ZSTD_DECODE_BASEBITS(baseline_basebits) \ + ((uint32_t)(baseline_basebits) >> 24) + +/* Given a literal length code, we need to read a number of bits and add that + to a baseline. For states 0 to 15 the baseline is the state and the number + of bits is zero. */ + +#define ZSTD_LITERAL_LENGTH_BASELINE_OFFSET (16) + +static const uint32_t elf_zstd_literal_length_base[] = +{ + ZSTD_ENCODE_BASELINE_BITS(16, 1), + ZSTD_ENCODE_BASELINE_BITS(18, 1), + ZSTD_ENCODE_BASELINE_BITS(20, 1), + ZSTD_ENCODE_BASELINE_BITS(22, 1), + ZSTD_ENCODE_BASELINE_BITS(24, 2), + ZSTD_ENCODE_BASELINE_BITS(28, 2), + ZSTD_ENCODE_BASELINE_BITS(32, 3), + ZSTD_ENCODE_BASELINE_BITS(40, 3), + ZSTD_ENCODE_BASELINE_BITS(48, 4), + ZSTD_ENCODE_BASELINE_BITS(64, 6), + ZSTD_ENCODE_BASELINE_BITS(128, 7), + ZSTD_ENCODE_BASELINE_BITS(256, 8), + ZSTD_ENCODE_BASELINE_BITS(512, 9), + ZSTD_ENCODE_BASELINE_BITS(1024, 10), + ZSTD_ENCODE_BASELINE_BITS(2048, 11), + ZSTD_ENCODE_BASELINE_BITS(4096, 12), + ZSTD_ENCODE_BASELINE_BITS(8192, 13), + ZSTD_ENCODE_BASELINE_BITS(16384, 14), + ZSTD_ENCODE_BASELINE_BITS(32768, 15), + ZSTD_ENCODE_BASELINE_BITS(65536, 16) +}; + +/* The same applies to match length codes. For states 0 to 31 the baseline is + the state + 3 and the number of bits is zero. */ + +#define ZSTD_MATCH_LENGTH_BASELINE_OFFSET (32) + +static const uint32_t elf_zstd_match_length_base[] = +{ + ZSTD_ENCODE_BASELINE_BITS(35, 1), + ZSTD_ENCODE_BASELINE_BITS(37, 1), + ZSTD_ENCODE_BASELINE_BITS(39, 1), + ZSTD_ENCODE_BASELINE_BITS(41, 1), + ZSTD_ENCODE_BASELINE_BITS(43, 2), + ZSTD_ENCODE_BASELINE_BITS(47, 2), + ZSTD_ENCODE_BASELINE_BITS(51, 3), + ZSTD_ENCODE_BASELINE_BITS(59, 3), + ZSTD_ENCODE_BASELINE_BITS(67, 4), + ZSTD_ENCODE_BASELINE_BITS(83, 4), + ZSTD_ENCODE_BASELINE_BITS(99, 5), + ZSTD_ENCODE_BASELINE_BITS(131, 7), + ZSTD_ENCODE_BASELINE_BITS(259, 8), + ZSTD_ENCODE_BASELINE_BITS(515, 9), + ZSTD_ENCODE_BASELINE_BITS(1027, 10), + ZSTD_ENCODE_BASELINE_BITS(2051, 11), + ZSTD_ENCODE_BASELINE_BITS(4099, 12), + ZSTD_ENCODE_BASELINE_BITS(8195, 13), + ZSTD_ENCODE_BASELINE_BITS(16387, 14), + ZSTD_ENCODE_BASELINE_BITS(32771, 15), + ZSTD_ENCODE_BASELINE_BITS(65539, 16) +}; + +/* An entry in an FSE table used for literal/match/length values. For these we + have to map the symbol to a baseline value, and we have to read zero or more + bits and add that value to the baseline value. Rather than look the values + up in a separate table, we grow the FSE table so that we get better memory + caching. */ + +struct elf_zstd_fse_baseline_entry +{ + /* The baseline for the value that this FSE entry represents.. */ + uint32_t baseline; + /* The number of bits to read to add to the baseline. */ + unsigned char basebits; + /* The number of bits to read to determine the next state. */ + unsigned char bits; + /* Add the bits to this base to get the next state. */ + uint16_t base; +}; + +/* Convert the literal length FSE table FSE_TABLE to an FSE baseline table at + BASELINE_TABLE. Note that FSE_TABLE and BASELINE_TABLE will overlap. */ + +static int +elf_zstd_make_literal_baseline_fse ( + const struct elf_zstd_fse_entry *fse_table, + int table_bits, + struct elf_zstd_fse_baseline_entry *baseline_table) +{ + size_t count; + const struct elf_zstd_fse_entry *pfse; + struct elf_zstd_fse_baseline_entry *pbaseline; + + /* Convert backward to avoid overlap. */ + + count = 1U << table_bits; + pfse = fse_table + count; + pbaseline = baseline_table + count; + while (pfse > fse_table) + { + unsigned char symbol; + unsigned char bits; + uint16_t base; + + --pfse; + --pbaseline; + symbol = pfse->symbol; + bits = pfse->bits; + base = pfse->base; + if (symbol < ZSTD_LITERAL_LENGTH_BASELINE_OFFSET) + { + pbaseline->baseline = (uint32_t)symbol; + pbaseline->basebits = 0; + } + else + { + unsigned int idx; + uint32_t basebits; + + if (unlikely (symbol > 35)) + { + elf_uncompress_failed (); + return 0; + } + idx = symbol - ZSTD_LITERAL_LENGTH_BASELINE_OFFSET; + basebits = elf_zstd_literal_length_base[idx]; + pbaseline->baseline = ZSTD_DECODE_BASELINE(basebits); + pbaseline->basebits = ZSTD_DECODE_BASEBITS(basebits); + } + pbaseline->bits = bits; + pbaseline->base = base; + } + + return 1; +} + +/* Convert the offset length FSE table FSE_TABLE to an FSE baseline table at + BASELINE_TABLE. Note that FSE_TABLE and BASELINE_TABLE will overlap. */ + +static int +elf_zstd_make_offset_baseline_fse ( + const struct elf_zstd_fse_entry *fse_table, + int table_bits, + struct elf_zstd_fse_baseline_entry *baseline_table) +{ + size_t count; + const struct elf_zstd_fse_entry *pfse; + struct elf_zstd_fse_baseline_entry *pbaseline; + + /* Convert backward to avoid overlap. */ + + count = 1U << table_bits; + pfse = fse_table + count; + pbaseline = baseline_table + count; + while (pfse > fse_table) + { + unsigned char symbol; + unsigned char bits; + uint16_t base; + + --pfse; + --pbaseline; + symbol = pfse->symbol; + bits = pfse->bits; + base = pfse->base; + if (unlikely (symbol > 31)) + { + elf_uncompress_failed (); + return 0; + } + + /* The simple way to write this is + + pbaseline->baseline = (uint32_t)1 << symbol; + pbaseline->basebits = symbol; + + That will give us an offset value that corresponds to the one + described in the RFC. However, for offset values > 3, we have to + subtract 3. And for offset values 1, 2, 3 we use a repeated offset. + The baseline is always a power of 2, and is never 0, so for these low + values we will see one entry that is baseline 1, basebits 0, and one + entry that is baseline 2, basebits 1. All other entries will have + baseline >= 4 and basebits >= 2. + + So we can check for RFC offset <= 3 by checking for basebits <= 1. + And that means that we can subtract 3 here and not worry about doing + it in the hot loop. */ + + pbaseline->baseline = (uint32_t)1 << symbol; + if (symbol >= 2) + pbaseline->baseline -= 3; + pbaseline->basebits = symbol; + pbaseline->bits = bits; + pbaseline->base = base; + } + + return 1; +} + +/* Convert the match length FSE table FSE_TABLE to an FSE baseline table at + BASELINE_TABLE. Note that FSE_TABLE and BASELINE_TABLE will overlap. */ + +static int +elf_zstd_make_match_baseline_fse ( + const struct elf_zstd_fse_entry *fse_table, + int table_bits, + struct elf_zstd_fse_baseline_entry *baseline_table) +{ + size_t count; + const struct elf_zstd_fse_entry *pfse; + struct elf_zstd_fse_baseline_entry *pbaseline; + + /* Convert backward to avoid overlap. */ + + count = 1U << table_bits; + pfse = fse_table + count; + pbaseline = baseline_table + count; + while (pfse > fse_table) + { + unsigned char symbol; + unsigned char bits; + uint16_t base; + + --pfse; + --pbaseline; + symbol = pfse->symbol; + bits = pfse->bits; + base = pfse->base; + if (symbol < ZSTD_MATCH_LENGTH_BASELINE_OFFSET) + { + pbaseline->baseline = (uint32_t)symbol + 3; + pbaseline->basebits = 0; + } + else + { + unsigned int idx; + uint32_t basebits; + + if (unlikely (symbol > 52)) + { + elf_uncompress_failed (); + return 0; + } + idx = symbol - ZSTD_MATCH_LENGTH_BASELINE_OFFSET; + basebits = elf_zstd_match_length_base[idx]; + pbaseline->baseline = ZSTD_DECODE_BASELINE(basebits); + pbaseline->basebits = ZSTD_DECODE_BASEBITS(basebits); + } + pbaseline->bits = bits; + pbaseline->base = base; + } + + return 1; +} + +#ifdef BACKTRACE_GENERATE_ZSTD_FSE_TABLES + +/* Used to generate the predefined FSE decoding tables for zstd. */ + +#include + +/* These values are straight from RFC 8878. */ + +static int16_t lit[36] = +{ + 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, + -1,-1,-1,-1 +}; + +static int16_t match[53] = +{ + 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1, + -1,-1,-1,-1,-1 +}; + +static int16_t offset[29] = +{ + 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 +}; + +static uint16_t next[256]; + +static void +print_table (const struct elf_zstd_fse_baseline_entry *table, size_t size) +{ + size_t i; + + printf ("{\n"); + for (i = 0; i < size; i += 3) + { + int j; + + printf (" "); + for (j = 0; j < 3 && i + j < size; ++j) + printf (" { %u, %d, %d, %d },", table[i + j].baseline, + table[i + j].basebits, table[i + j].bits, + table[i + j].base); + printf ("\n"); + } + printf ("};\n"); +} + +int +main () +{ + struct elf_zstd_fse_entry lit_table[64]; + struct elf_zstd_fse_baseline_entry lit_baseline[64]; + struct elf_zstd_fse_entry match_table[64]; + struct elf_zstd_fse_baseline_entry match_baseline[64]; + struct elf_zstd_fse_entry offset_table[32]; + struct elf_zstd_fse_baseline_entry offset_baseline[32]; + + if (!elf_zstd_build_fse (lit, sizeof lit / sizeof lit[0], next, + 6, lit_table)) + { + fprintf (stderr, "elf_zstd_build_fse failed\n"); + exit (EXIT_FAILURE); + } + + if (!elf_zstd_make_literal_baseline_fse (lit_table, 6, lit_baseline)) + { + fprintf (stderr, "elf_zstd_make_literal_baseline_fse failed\n"); + exit (EXIT_FAILURE); + } + + printf ("static const struct elf_zstd_fse_baseline_entry " + "elf_zstd_lit_table[64] =\n"); + print_table (lit_baseline, + sizeof lit_baseline / sizeof lit_baseline[0]); + printf ("\n"); + + if (!elf_zstd_build_fse (match, sizeof match / sizeof match[0], next, + 6, match_table)) + { + fprintf (stderr, "elf_zstd_build_fse failed\n"); + exit (EXIT_FAILURE); + } + + if (!elf_zstd_make_match_baseline_fse (match_table, 6, match_baseline)) + { + fprintf (stderr, "elf_zstd_make_match_baseline_fse failed\n"); + exit (EXIT_FAILURE); + } + + printf ("static const struct elf_zstd_fse_baseline_entry " + "elf_zstd_match_table[64] =\n"); + print_table (match_baseline, + sizeof match_baseline / sizeof match_baseline[0]); + printf ("\n"); + + if (!elf_zstd_build_fse (offset, sizeof offset / sizeof offset[0], next, + 5, offset_table)) + { + fprintf (stderr, "elf_zstd_build_fse failed\n"); + exit (EXIT_FAILURE); + } + + if (!elf_zstd_make_offset_baseline_fse (offset_table, 5, offset_baseline)) + { + fprintf (stderr, "elf_zstd_make_offset_baseline_fse failed\n"); + exit (EXIT_FAILURE); + } + + printf ("static const struct elf_zstd_fse_baseline_entry " + "elf_zstd_offset_table[32] =\n"); + print_table (offset_baseline, + sizeof offset_baseline / sizeof offset_baseline[0]); + printf ("\n"); + + return 0; +} + +#endif + +/* The fixed tables generated by the #ifdef'ed out main function + above. */ + +static const struct elf_zstd_fse_baseline_entry elf_zstd_lit_table[64] = +{ + { 0, 0, 4, 0 }, { 0, 0, 4, 16 }, { 1, 0, 5, 32 }, + { 3, 0, 5, 0 }, { 4, 0, 5, 0 }, { 6, 0, 5, 0 }, + { 7, 0, 5, 0 }, { 9, 0, 5, 0 }, { 10, 0, 5, 0 }, + { 12, 0, 5, 0 }, { 14, 0, 6, 0 }, { 16, 1, 5, 0 }, + { 20, 1, 5, 0 }, { 22, 1, 5, 0 }, { 28, 2, 5, 0 }, + { 32, 3, 5, 0 }, { 48, 4, 5, 0 }, { 64, 6, 5, 32 }, + { 128, 7, 5, 0 }, { 256, 8, 6, 0 }, { 1024, 10, 6, 0 }, + { 4096, 12, 6, 0 }, { 0, 0, 4, 32 }, { 1, 0, 4, 0 }, + { 2, 0, 5, 0 }, { 4, 0, 5, 32 }, { 5, 0, 5, 0 }, + { 7, 0, 5, 32 }, { 8, 0, 5, 0 }, { 10, 0, 5, 32 }, + { 11, 0, 5, 0 }, { 13, 0, 6, 0 }, { 16, 1, 5, 32 }, + { 18, 1, 5, 0 }, { 22, 1, 5, 32 }, { 24, 2, 5, 0 }, + { 32, 3, 5, 32 }, { 40, 3, 5, 0 }, { 64, 6, 4, 0 }, + { 64, 6, 4, 16 }, { 128, 7, 5, 32 }, { 512, 9, 6, 0 }, + { 2048, 11, 6, 0 }, { 0, 0, 4, 48 }, { 1, 0, 4, 16 }, + { 2, 0, 5, 32 }, { 3, 0, 5, 32 }, { 5, 0, 5, 32 }, + { 6, 0, 5, 32 }, { 8, 0, 5, 32 }, { 9, 0, 5, 32 }, + { 11, 0, 5, 32 }, { 12, 0, 5, 32 }, { 15, 0, 6, 0 }, + { 18, 1, 5, 32 }, { 20, 1, 5, 32 }, { 24, 2, 5, 32 }, + { 28, 2, 5, 32 }, { 40, 3, 5, 32 }, { 48, 4, 5, 32 }, + { 65536, 16, 6, 0 }, { 32768, 15, 6, 0 }, { 16384, 14, 6, 0 }, + { 8192, 13, 6, 0 }, +}; + +static const struct elf_zstd_fse_baseline_entry elf_zstd_match_table[64] = +{ + { 3, 0, 6, 0 }, { 4, 0, 4, 0 }, { 5, 0, 5, 32 }, + { 6, 0, 5, 0 }, { 8, 0, 5, 0 }, { 9, 0, 5, 0 }, + { 11, 0, 5, 0 }, { 13, 0, 6, 0 }, { 16, 0, 6, 0 }, + { 19, 0, 6, 0 }, { 22, 0, 6, 0 }, { 25, 0, 6, 0 }, + { 28, 0, 6, 0 }, { 31, 0, 6, 0 }, { 34, 0, 6, 0 }, + { 37, 1, 6, 0 }, { 41, 1, 6, 0 }, { 47, 2, 6, 0 }, + { 59, 3, 6, 0 }, { 83, 4, 6, 0 }, { 131, 7, 6, 0 }, + { 515, 9, 6, 0 }, { 4, 0, 4, 16 }, { 5, 0, 4, 0 }, + { 6, 0, 5, 32 }, { 7, 0, 5, 0 }, { 9, 0, 5, 32 }, + { 10, 0, 5, 0 }, { 12, 0, 6, 0 }, { 15, 0, 6, 0 }, + { 18, 0, 6, 0 }, { 21, 0, 6, 0 }, { 24, 0, 6, 0 }, + { 27, 0, 6, 0 }, { 30, 0, 6, 0 }, { 33, 0, 6, 0 }, + { 35, 1, 6, 0 }, { 39, 1, 6, 0 }, { 43, 2, 6, 0 }, + { 51, 3, 6, 0 }, { 67, 4, 6, 0 }, { 99, 5, 6, 0 }, + { 259, 8, 6, 0 }, { 4, 0, 4, 32 }, { 4, 0, 4, 48 }, + { 5, 0, 4, 16 }, { 7, 0, 5, 32 }, { 8, 0, 5, 32 }, + { 10, 0, 5, 32 }, { 11, 0, 5, 32 }, { 14, 0, 6, 0 }, + { 17, 0, 6, 0 }, { 20, 0, 6, 0 }, { 23, 0, 6, 0 }, + { 26, 0, 6, 0 }, { 29, 0, 6, 0 }, { 32, 0, 6, 0 }, + { 65539, 16, 6, 0 }, { 32771, 15, 6, 0 }, { 16387, 14, 6, 0 }, + { 8195, 13, 6, 0 }, { 4099, 12, 6, 0 }, { 2051, 11, 6, 0 }, + { 1027, 10, 6, 0 }, +}; + +static const struct elf_zstd_fse_baseline_entry elf_zstd_offset_table[32] = +{ + { 1, 0, 5, 0 }, { 61, 6, 4, 0 }, { 509, 9, 5, 0 }, + { 32765, 15, 5, 0 }, { 2097149, 21, 5, 0 }, { 5, 3, 5, 0 }, + { 125, 7, 4, 0 }, { 4093, 12, 5, 0 }, { 262141, 18, 5, 0 }, + { 8388605, 23, 5, 0 }, { 29, 5, 5, 0 }, { 253, 8, 4, 0 }, + { 16381, 14, 5, 0 }, { 1048573, 20, 5, 0 }, { 1, 2, 5, 0 }, + { 125, 7, 4, 16 }, { 2045, 11, 5, 0 }, { 131069, 17, 5, 0 }, + { 4194301, 22, 5, 0 }, { 13, 4, 5, 0 }, { 253, 8, 4, 16 }, + { 8189, 13, 5, 0 }, { 524285, 19, 5, 0 }, { 2, 1, 5, 0 }, + { 61, 6, 4, 16 }, { 1021, 10, 5, 0 }, { 65533, 16, 5, 0 }, + { 268435453, 28, 5, 0 }, { 134217725, 27, 5, 0 }, { 67108861, 26, 5, 0 }, + { 33554429, 25, 5, 0 }, { 16777213, 24, 5, 0 }, +}; + +/* Read a zstd Huffman table and build the decoding table in *TABLE, reading + and updating *PPIN. This sets *PTABLE_BITS to the number of bits of the + table, such that the table length is 1 << *TABLE_BITS. ZDEBUG_TABLE is + scratch space; it must be enough for 512 uint16_t values + 256 32-bit values + (2048 bytes). Returns 1 on success, 0 on error. */ + +static int +elf_zstd_read_huff (const unsigned char **ppin, const unsigned char *pinend, + uint16_t *zdebug_table, uint16_t *table, int *ptable_bits) +{ + const unsigned char *pin; + unsigned char hdr; + unsigned char *weights; + size_t count; + uint32_t *weight_mark; + size_t i; + uint32_t weight_mask; + size_t table_bits; + + pin = *ppin; + if (unlikely (pin >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + hdr = *pin; + ++pin; + + weights = (unsigned char *) zdebug_table; + + if (hdr < 128) + { + /* Table is compressed using FSE. */ + + struct elf_zstd_fse_entry *fse_table; + int fse_table_bits; + uint16_t *scratch; + const unsigned char *pfse; + const unsigned char *pback; + uint64_t val; + unsigned int bits; + unsigned int state1, state2; + + /* SCRATCH is used temporarily by elf_zstd_read_fse. It overlaps + WEIGHTS. */ + scratch = zdebug_table; + fse_table = (struct elf_zstd_fse_entry *) (scratch + 512); + fse_table_bits = 6; + + pfse = pin; + if (!elf_zstd_read_fse (&pfse, pinend, scratch, 255, fse_table, + &fse_table_bits)) + return 0; + + if (unlikely (pin + hdr > pinend)) + { + elf_uncompress_failed (); + return 0; + } + + /* We no longer need SCRATCH. Start recording weights. We need up to + 256 bytes of weights and 64 bytes of rank counts, so it won't overlap + FSE_TABLE. */ + + pback = pin + hdr - 1; + + if (!elf_fetch_backward_init (&pback, pfse, &val, &bits)) + return 0; + + bits -= fse_table_bits; + state1 = (val >> bits) & ((1U << fse_table_bits) - 1); + bits -= fse_table_bits; + state2 = (val >> bits) & ((1U << fse_table_bits) - 1); + + /* There are two independent FSE streams, tracked by STATE1 and STATE2. + We decode them alternately. */ + + count = 0; + while (1) + { + struct elf_zstd_fse_entry *pt; + uint64_t v; + + pt = &fse_table[state1]; + + if (unlikely (pin < pinend) && bits < pt->bits) + { + if (unlikely (count >= 254)) + { + elf_uncompress_failed (); + return 0; + } + weights[count] = (unsigned char) pt->symbol; + weights[count + 1] = (unsigned char) fse_table[state2].symbol; + count += 2; + break; + } + + if (unlikely (pt->bits == 0)) + v = 0; + else + { + if (!elf_fetch_bits_backward (&pback, pfse, &val, &bits)) + return 0; + + bits -= pt->bits; + v = (val >> bits) & (((uint64_t)1 << pt->bits) - 1); + } + + state1 = pt->base + v; + + if (unlikely (count >= 255)) + { + elf_uncompress_failed (); + return 0; + } + + weights[count] = pt->symbol; + ++count; + + pt = &fse_table[state2]; + + if (unlikely (pin < pinend && bits < pt->bits)) + { + if (unlikely (count >= 254)) + { + elf_uncompress_failed (); + return 0; + } + weights[count] = (unsigned char) pt->symbol; + weights[count + 1] = (unsigned char) fse_table[state1].symbol; + count += 2; + break; + } + + if (unlikely (pt->bits == 0)) + v = 0; + else + { + if (!elf_fetch_bits_backward (&pback, pfse, &val, &bits)) + return 0; + + bits -= pt->bits; + v = (val >> bits) & (((uint64_t)1 << pt->bits) - 1); + } + + state2 = pt->base + v; + + if (unlikely (count >= 255)) + { + elf_uncompress_failed (); + return 0; + } + + weights[count] = pt->symbol; + ++count; + } + + pin += hdr; + } + else + { + /* Table is not compressed. Each weight is 4 bits. */ + + count = hdr - 127; + if (unlikely (pin + ((count + 1) / 2) >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + for (i = 0; i < count; i += 2) + { + unsigned char b; + + b = *pin; + ++pin; + weights[i] = b >> 4; + weights[i + 1] = b & 0xf; + } + } + + weight_mark = (uint32_t *) (weights + 256); + memset (weight_mark, 0, 13 * sizeof (uint32_t)); + weight_mask = 0; + for (i = 0; i < count; ++i) + { + unsigned char w; + + w = weights[i]; + if (unlikely (w > 12)) + { + elf_uncompress_failed (); + return 0; + } + ++weight_mark[w]; + if (w > 0) + weight_mask += 1U << (w - 1); + } + if (unlikely (weight_mask == 0)) + { + elf_uncompress_failed (); + return 0; + } + + table_bits = 32 - __builtin_clz (weight_mask); + if (unlikely (table_bits > 11)) + { + elf_uncompress_failed (); + return 0; + } + + /* Work out the last weight value, which is omitted because the weights must + sum to a power of two. */ + { + uint32_t left; + uint32_t high_bit; + + left = ((uint32_t)1 << table_bits) - weight_mask; + if (left == 0) + { + elf_uncompress_failed (); + return 0; + } + high_bit = 31 - __builtin_clz (left); + if (((uint32_t)1 << high_bit) != left) + { + elf_uncompress_failed (); + return 0; + } + + if (unlikely (count >= 256)) + { + elf_uncompress_failed (); + return 0; + } + + weights[count] = high_bit + 1; + ++count; + ++weight_mark[high_bit + 1]; + } + + if (weight_mark[1] < 2 || (weight_mark[1] & 1) != 0) + { + elf_uncompress_failed (); + return 0; + } + + /* Change WEIGHT_MARK from a count of weights to the index of the first + symbol for that weight. We shift the indexes to also store how many we + have seen so far, below. */ + { + uint32_t next; + + next = 0; + for (i = 0; i < table_bits; ++i) + { + uint32_t cur; + + cur = next; + next += weight_mark[i + 1] << i; + weight_mark[i + 1] = cur; + } + } + + for (i = 0; i < count; ++i) + { + unsigned char weight; + uint32_t length; + uint16_t tval; + size_t start; + uint32_t j; + + weight = weights[i]; + if (weight == 0) + continue; + + length = 1U << (weight - 1); + tval = (i << 8) | (table_bits + 1 - weight); + start = weight_mark[weight]; + for (j = 0; j < length; ++j) + table[start + j] = tval; + weight_mark[weight] += length; + } + + *ppin = pin; + *ptable_bits = (int)table_bits; + + return 1; +} + +/* Read and decompress the literals and store them ending at POUTEND. This + works because we are going to use all the literals in the output, so they + must fit into the output buffer. HUFFMAN_TABLE, and PHUFFMAN_TABLE_BITS + store the Huffman table across calls. SCRATCH is used to read a Huffman + table. Store the start of the decompressed literals in *PPLIT. Update + *PPIN. Return 1 on success, 0 on error. */ + +static int +elf_zstd_read_literals (const unsigned char **ppin, + const unsigned char *pinend, + unsigned char *pout, + unsigned char *poutend, + uint16_t *scratch, + uint16_t *huffman_table, + int *phuffman_table_bits, + unsigned char **pplit) +{ + const unsigned char *pin; + unsigned char *plit; + unsigned char hdr; + uint32_t regenerated_size; + uint32_t compressed_size; + int streams; + uint32_t total_streams_size; + unsigned int huffman_table_bits; + uint64_t huffman_mask; + + pin = *ppin; + if (unlikely (pin >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + hdr = *pin; + ++pin; + + if ((hdr & 3) == 0 || (hdr & 3) == 1) + { + int raw; + + /* Raw_Literals_Block or RLE_Literals_Block */ + + raw = (hdr & 3) == 0; + + switch ((hdr >> 2) & 3) + { + case 0: case 2: + regenerated_size = hdr >> 3; + break; + case 1: + if (unlikely (pin >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + regenerated_size = (hdr >> 4) + ((uint32_t)(*pin) << 4); + ++pin; + break; + case 3: + if (unlikely (pin + 1 >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + regenerated_size = ((hdr >> 4) + + ((uint32_t)*pin << 4) + + ((uint32_t)pin[1] << 12)); + pin += 2; + break; + default: + elf_uncompress_failed (); + return 0; + } + + if (unlikely ((size_t)(poutend - pout) < regenerated_size)) + { + elf_uncompress_failed (); + return 0; + } + + plit = poutend - regenerated_size; + + if (raw) + { + if (unlikely (pin + regenerated_size >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + memcpy (plit, pin, regenerated_size); + pin += regenerated_size; + } + else + { + if (pin >= pinend) + { + elf_uncompress_failed (); + return 0; + } + memset (plit, *pin, regenerated_size); + ++pin; + } + + *ppin = pin; + *pplit = plit; + + return 1; + } + + /* Compressed_Literals_Block or Treeless_Literals_Block */ + + switch ((hdr >> 2) & 3) + { + case 0: case 1: + if (unlikely (pin + 1 >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + regenerated_size = (hdr >> 4) | ((uint32_t)(*pin & 0x3f) << 4); + compressed_size = (uint32_t)*pin >> 6 | ((uint32_t)pin[1] << 2); + pin += 2; + streams = ((hdr >> 2) & 3) == 0 ? 1 : 4; + break; + case 2: + if (unlikely (pin + 2 >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + regenerated_size = (((uint32_t)hdr >> 4) + | ((uint32_t)*pin << 4) + | (((uint32_t)pin[1] & 3) << 12)); + compressed_size = (((uint32_t)pin[1] >> 2) + | ((uint32_t)pin[2] << 6)); + pin += 3; + streams = 4; + break; + case 3: + if (unlikely (pin + 3 >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + regenerated_size = (((uint32_t)hdr >> 4) + | ((uint32_t)*pin << 4) + | (((uint32_t)pin[1] & 0x3f) << 12)); + compressed_size = (((uint32_t)pin[1] >> 6) + | ((uint32_t)pin[2] << 2) + | ((uint32_t)pin[3] << 10)); + pin += 4; + streams = 4; + break; + default: + elf_uncompress_failed (); + return 0; + } + + if (unlikely (pin + compressed_size > pinend)) + { + elf_uncompress_failed (); + return 0; + } + + pinend = pin + compressed_size; + *ppin = pinend; + + if (unlikely ((size_t)(poutend - pout) < regenerated_size)) + { + elf_uncompress_failed (); + return 0; + } + + plit = poutend - regenerated_size; + + *pplit = plit; + + total_streams_size = compressed_size; + if ((hdr & 3) == 2) + { + const unsigned char *ptable; + + /* Compressed_Literals_Block. Read Huffman tree. */ + + ptable = pin; + if (!elf_zstd_read_huff (&ptable, pinend, scratch, huffman_table, + phuffman_table_bits)) + return 0; + + if (unlikely (total_streams_size < (size_t)(ptable - pin))) + { + elf_uncompress_failed (); + return 0; + } + + total_streams_size -= ptable - pin; + pin = ptable; + } + else + { + /* Treeless_Literals_Block. Reuse previous Huffman tree. */ + if (unlikely (*phuffman_table_bits == 0)) + { + elf_uncompress_failed (); + return 0; + } + } + + /* Decompress COMPRESSED_SIZE bytes of data at PIN using the huffman table, + storing REGENERATED_SIZE bytes of decompressed data at PLIT. */ + + huffman_table_bits = (unsigned int)*phuffman_table_bits; + huffman_mask = ((uint64_t)1 << huffman_table_bits) - 1; + + if (streams == 1) + { + const unsigned char *pback; + const unsigned char *pbackend; + uint64_t val; + unsigned int bits; + uint32_t i; + + pback = pin + total_streams_size - 1; + pbackend = pin; + if (!elf_fetch_backward_init (&pback, pbackend, &val, &bits)) + return 0; + + /* This is one of the inner loops of the decompression algorithm, so we + put some effort into optimization. We can't get more than 64 bytes + from a single call to elf_fetch_bits_backward, and we can't subtract + more than 11 bits at a time. */ + + if (regenerated_size >= 64) + { + unsigned char *plitstart; + unsigned char *plitstop; + + plitstart = plit; + plitstop = plit + regenerated_size - 64; + while (plit < plitstop) + { + uint16_t t; + + if (!elf_fetch_bits_backward (&pback, pbackend, &val, &bits)) + return 0; + + if (bits < 16) + break; + + while (bits >= 33) + { + t = huffman_table[(val >> (bits - huffman_table_bits)) + & huffman_mask]; + *plit = t >> 8; + ++plit; + bits -= t & 0xff; + + t = huffman_table[(val >> (bits - huffman_table_bits)) + & huffman_mask]; + *plit = t >> 8; + ++plit; + bits -= t & 0xff; + + t = huffman_table[(val >> (bits - huffman_table_bits)) + & huffman_mask]; + *plit = t >> 8; + ++plit; + bits -= t & 0xff; + } + + while (bits > 11) + { + t = huffman_table[(val >> (bits - huffman_table_bits)) + & huffman_mask]; + *plit = t >> 8; + ++plit; + bits -= t & 0xff; + } + } + + regenerated_size -= plit - plitstart; + } + + for (i = 0; i < regenerated_size; ++i) + { + uint16_t t; + + if (!elf_fetch_bits_backward (&pback, pbackend, &val, &bits)) + return 0; + + if (unlikely (bits < huffman_table_bits)) + { + t = huffman_table[(val << (huffman_table_bits - bits)) + & huffman_mask]; + if (unlikely (bits < (t & 0xff))) + { + elf_uncompress_failed (); + return 0; + } + } + else + t = huffman_table[(val >> (bits - huffman_table_bits)) + & huffman_mask]; + + *plit = t >> 8; + ++plit; + bits -= t & 0xff; + } + + return 1; + } + + { + uint32_t stream_size1, stream_size2, stream_size3, stream_size4; + uint32_t tot; + const unsigned char *pback1, *pback2, *pback3, *pback4; + const unsigned char *pbackend1, *pbackend2, *pbackend3, *pbackend4; + uint64_t val1, val2, val3, val4; + unsigned int bits1, bits2, bits3, bits4; + unsigned char *plit1, *plit2, *plit3, *plit4; + uint32_t regenerated_stream_size; + uint32_t regenerated_stream_size4; + uint16_t t1, t2, t3, t4; + uint32_t i; + uint32_t limit; + + /* Read jump table. */ + if (unlikely (pin + 5 >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + stream_size1 = (uint32_t)*pin | ((uint32_t)pin[1] << 8); + pin += 2; + stream_size2 = (uint32_t)*pin | ((uint32_t)pin[1] << 8); + pin += 2; + stream_size3 = (uint32_t)*pin | ((uint32_t)pin[1] << 8); + pin += 2; + tot = stream_size1 + stream_size2 + stream_size3; + if (unlikely (tot > total_streams_size - 6)) + { + elf_uncompress_failed (); + return 0; + } + stream_size4 = total_streams_size - 6 - tot; + + pback1 = pin + stream_size1 - 1; + pbackend1 = pin; + + pback2 = pback1 + stream_size2; + pbackend2 = pback1 + 1; + + pback3 = pback2 + stream_size3; + pbackend3 = pback2 + 1; + + pback4 = pback3 + stream_size4; + pbackend4 = pback3 + 1; + + if (!elf_fetch_backward_init (&pback1, pbackend1, &val1, &bits1)) + return 0; + if (!elf_fetch_backward_init (&pback2, pbackend2, &val2, &bits2)) + return 0; + if (!elf_fetch_backward_init (&pback3, pbackend3, &val3, &bits3)) + return 0; + if (!elf_fetch_backward_init (&pback4, pbackend4, &val4, &bits4)) + return 0; + + regenerated_stream_size = (regenerated_size + 3) / 4; + + plit1 = plit; + plit2 = plit1 + regenerated_stream_size; + plit3 = plit2 + regenerated_stream_size; + plit4 = plit3 + regenerated_stream_size; + + regenerated_stream_size4 = regenerated_size - regenerated_stream_size * 3; + + /* We can't get more than 64 literal bytes from a single call to + elf_fetch_bits_backward. The fourth stream can be up to 3 bytes less, + so use as the limit. */ + + limit = regenerated_stream_size4 <= 64 ? 0 : regenerated_stream_size4 - 64; + i = 0; + while (i < limit) + { + if (!elf_fetch_bits_backward (&pback1, pbackend1, &val1, &bits1)) + return 0; + if (!elf_fetch_bits_backward (&pback2, pbackend2, &val2, &bits2)) + return 0; + if (!elf_fetch_bits_backward (&pback3, pbackend3, &val3, &bits3)) + return 0; + if (!elf_fetch_bits_backward (&pback4, pbackend4, &val4, &bits4)) + return 0; + + /* We can't subtract more than 11 bits at a time. */ + + do + { + t1 = huffman_table[(val1 >> (bits1 - huffman_table_bits)) + & huffman_mask]; + t2 = huffman_table[(val2 >> (bits2 - huffman_table_bits)) + & huffman_mask]; + t3 = huffman_table[(val3 >> (bits3 - huffman_table_bits)) + & huffman_mask]; + t4 = huffman_table[(val4 >> (bits4 - huffman_table_bits)) + & huffman_mask]; + + *plit1 = t1 >> 8; + ++plit1; + bits1 -= t1 & 0xff; + + *plit2 = t2 >> 8; + ++plit2; + bits2 -= t2 & 0xff; + + *plit3 = t3 >> 8; + ++plit3; + bits3 -= t3 & 0xff; + + *plit4 = t4 >> 8; + ++plit4; + bits4 -= t4 & 0xff; + + ++i; + } + while (bits1 > 11 && bits2 > 11 && bits3 > 11 && bits4 > 11); + } + + while (i < regenerated_stream_size) + { + int use4; + + use4 = i < regenerated_stream_size4; + + if (!elf_fetch_bits_backward (&pback1, pbackend1, &val1, &bits1)) + return 0; + if (!elf_fetch_bits_backward (&pback2, pbackend2, &val2, &bits2)) + return 0; + if (!elf_fetch_bits_backward (&pback3, pbackend3, &val3, &bits3)) + return 0; + if (use4) + { + if (!elf_fetch_bits_backward (&pback4, pbackend4, &val4, &bits4)) + return 0; + } + + if (unlikely (bits1 < huffman_table_bits)) + { + t1 = huffman_table[(val1 << (huffman_table_bits - bits1)) + & huffman_mask]; + if (unlikely (bits1 < (t1 & 0xff))) + { + elf_uncompress_failed (); + return 0; + } + } + else + t1 = huffman_table[(val1 >> (bits1 - huffman_table_bits)) + & huffman_mask]; + + if (unlikely (bits2 < huffman_table_bits)) + { + t2 = huffman_table[(val2 << (huffman_table_bits - bits2)) + & huffman_mask]; + if (unlikely (bits2 < (t2 & 0xff))) + { + elf_uncompress_failed (); + return 0; + } + } + else + t2 = huffman_table[(val2 >> (bits2 - huffman_table_bits)) + & huffman_mask]; + + if (unlikely (bits3 < huffman_table_bits)) + { + t3 = huffman_table[(val3 << (huffman_table_bits - bits3)) + & huffman_mask]; + if (unlikely (bits3 < (t3 & 0xff))) + { + elf_uncompress_failed (); + return 0; + } + } + else + t3 = huffman_table[(val3 >> (bits3 - huffman_table_bits)) + & huffman_mask]; + + if (use4) + { + if (unlikely (bits4 < huffman_table_bits)) + { + t4 = huffman_table[(val4 << (huffman_table_bits - bits4)) + & huffman_mask]; + if (unlikely (bits4 < (t4 & 0xff))) + { + elf_uncompress_failed (); + return 0; + } + } + else + t4 = huffman_table[(val4 >> (bits4 - huffman_table_bits)) + & huffman_mask]; + + *plit4 = t4 >> 8; + ++plit4; + bits4 -= t4 & 0xff; + } + + *plit1 = t1 >> 8; + ++plit1; + bits1 -= t1 & 0xff; + + *plit2 = t2 >> 8; + ++plit2; + bits2 -= t2 & 0xff; + + *plit3 = t3 >> 8; + ++plit3; + bits3 -= t3 & 0xff; + + ++i; + } + } + + return 1; +} + +/* The information used to decompress a sequence code, which can be a literal + length, an offset, or a match length. */ + +struct elf_zstd_seq_decode +{ + const struct elf_zstd_fse_baseline_entry *table; + int table_bits; +}; + +/* Unpack a sequence code compression mode. */ + +static int +elf_zstd_unpack_seq_decode (int mode, + const unsigned char **ppin, + const unsigned char *pinend, + const struct elf_zstd_fse_baseline_entry *predef, + int predef_bits, + uint16_t *scratch, + int maxidx, + struct elf_zstd_fse_baseline_entry *table, + int table_bits, + int (*conv)(const struct elf_zstd_fse_entry *, + int, + struct elf_zstd_fse_baseline_entry *), + struct elf_zstd_seq_decode *decode) +{ + switch (mode) + { + case 0: + decode->table = predef; + decode->table_bits = predef_bits; + break; + + case 1: + { + struct elf_zstd_fse_entry entry; + + if (unlikely (*ppin >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + entry.symbol = **ppin; + ++*ppin; + entry.bits = 0; + entry.base = 0; + decode->table_bits = 0; + if (!conv (&entry, 0, table)) + return 0; + } + break; + + case 2: + { + struct elf_zstd_fse_entry *fse_table; + + /* We use the same space for the simple FSE table and the baseline + table. */ + fse_table = (struct elf_zstd_fse_entry *)table; + decode->table_bits = table_bits; + if (!elf_zstd_read_fse (ppin, pinend, scratch, maxidx, fse_table, + &decode->table_bits)) + return 0; + if (!conv (fse_table, decode->table_bits, table)) + return 0; + decode->table = table; + } + break; + + case 3: + if (unlikely (decode->table_bits == -1)) + { + elf_uncompress_failed (); + return 0; + } + break; + + default: + elf_uncompress_failed (); + return 0; + } + + return 1; +} + +/* Decompress a zstd stream from PIN/SIN to POUT/SOUT. Code based on RFC 8878. + Return 1 on success, 0 on error. */ + +static int +elf_zstd_decompress (const unsigned char *pin, size_t sin, + unsigned char *zdebug_table, unsigned char *pout, + size_t sout) +{ + const unsigned char *pinend; + unsigned char *poutstart; + unsigned char *poutend; + struct elf_zstd_seq_decode literal_decode; + struct elf_zstd_fse_baseline_entry *literal_fse_table; + struct elf_zstd_seq_decode match_decode; + struct elf_zstd_fse_baseline_entry *match_fse_table; + struct elf_zstd_seq_decode offset_decode; + struct elf_zstd_fse_baseline_entry *offset_fse_table; + uint16_t *huffman_table; + int huffman_table_bits; + uint32_t repeated_offset1; + uint32_t repeated_offset2; + uint32_t repeated_offset3; + uint16_t *scratch; + unsigned char hdr; + int has_checksum; + uint64_t content_size; + int last_block; + + pinend = pin + sin; + poutstart = pout; + poutend = pout + sout; + + literal_decode.table = NULL; + literal_decode.table_bits = -1; + literal_fse_table = ((struct elf_zstd_fse_baseline_entry *) + (zdebug_table + ZSTD_TABLE_LITERAL_FSE_OFFSET)); + + match_decode.table = NULL; + match_decode.table_bits = -1; + match_fse_table = ((struct elf_zstd_fse_baseline_entry *) + (zdebug_table + ZSTD_TABLE_MATCH_FSE_OFFSET)); + + offset_decode.table = NULL; + offset_decode.table_bits = -1; + offset_fse_table = ((struct elf_zstd_fse_baseline_entry *) + (zdebug_table + ZSTD_TABLE_OFFSET_FSE_OFFSET)); + huffman_table = ((uint16_t *) + (zdebug_table + ZSTD_TABLE_HUFFMAN_OFFSET)); + huffman_table_bits = 0; + scratch = ((uint16_t *) + (zdebug_table + ZSTD_TABLE_WORK_OFFSET)); + + repeated_offset1 = 1; + repeated_offset2 = 4; + repeated_offset3 = 8; + + if (unlikely (sin < 4)) + { + elf_uncompress_failed (); + return 0; + } + + /* These values are the zstd magic number. */ + if (unlikely (pin[0] != 0x28 + || pin[1] != 0xb5 + || pin[2] != 0x2f + || pin[3] != 0xfd)) + { + elf_uncompress_failed (); + return 0; + } + + pin += 4; + + if (unlikely (pin >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + + hdr = *pin++; + + /* We expect a single frame. */ + if (unlikely ((hdr & (1 << 5)) == 0)) + { + elf_uncompress_failed (); + return 0; + } + /* Reserved bit must be zero. */ + if (unlikely ((hdr & (1 << 3)) != 0)) + { + elf_uncompress_failed (); + return 0; + } + /* We do not expect a dictionary. */ + if (unlikely ((hdr & 3) != 0)) + { + elf_uncompress_failed (); + return 0; + } + has_checksum = (hdr & (1 << 2)) != 0; + switch (hdr >> 6) + { + case 0: + if (unlikely (pin >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + content_size = (uint64_t) *pin++; + break; + case 1: + if (unlikely (pin + 1 >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + content_size = (((uint64_t) pin[0]) | (((uint64_t) pin[1]) << 8)) + 256; + pin += 2; + break; + case 2: + if (unlikely (pin + 3 >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + content_size = ((uint64_t) pin[0] + | (((uint64_t) pin[1]) << 8) + | (((uint64_t) pin[2]) << 16) + | (((uint64_t) pin[3]) << 24)); + pin += 4; + break; + case 3: + if (unlikely (pin + 7 >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + content_size = ((uint64_t) pin[0] + | (((uint64_t) pin[1]) << 8) + | (((uint64_t) pin[2]) << 16) + | (((uint64_t) pin[3]) << 24) + | (((uint64_t) pin[4]) << 32) + | (((uint64_t) pin[5]) << 40) + | (((uint64_t) pin[6]) << 48) + | (((uint64_t) pin[7]) << 56)); + pin += 8; + break; + default: + elf_uncompress_failed (); + return 0; + } + + if (unlikely (content_size != (size_t) content_size + || (size_t) content_size != sout)) + { + elf_uncompress_failed (); + return 0; + } + + last_block = 0; + while (!last_block) + { + uint32_t block_hdr; + int block_type; + uint32_t block_size; + + if (unlikely (pin + 2 >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + block_hdr = ((uint32_t) pin[0] + | (((uint32_t) pin[1]) << 8) + | (((uint32_t) pin[2]) << 16)); + pin += 3; + + last_block = block_hdr & 1; + block_type = (block_hdr >> 1) & 3; + block_size = block_hdr >> 3; + + switch (block_type) + { + case 0: + /* Raw_Block */ + if (unlikely ((size_t) block_size > (size_t) (pinend - pin))) + { + elf_uncompress_failed (); + return 0; + } + if (unlikely ((size_t) block_size > (size_t) (poutend - pout))) + { + elf_uncompress_failed (); + return 0; + } + memcpy (pout, pin, block_size); + pout += block_size; + pin += block_size; + break; + + case 1: + /* RLE_Block */ + if (unlikely (pin >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + if (unlikely ((size_t) block_size > (size_t) (poutend - pout))) + { + elf_uncompress_failed (); + return 0; + } + memset (pout, *pin, block_size); + pout += block_size; + pin++; + break; + + case 2: + { + const unsigned char *pblockend; + unsigned char *plitstack; + unsigned char *plit; + uint32_t literal_count; + unsigned char seq_hdr; + size_t seq_count; + size_t seq; + const unsigned char *pback; + uint64_t val; + unsigned int bits; + unsigned int literal_state; + unsigned int offset_state; + unsigned int match_state; + + /* Compressed_Block */ + if (unlikely ((size_t) block_size > (size_t) (pinend - pin))) + { + elf_uncompress_failed (); + return 0; + } + + pblockend = pin + block_size; + + /* Read the literals into the end of the output space, and leave + PLIT pointing at them. */ + + if (!elf_zstd_read_literals (&pin, pblockend, pout, poutend, + scratch, huffman_table, + &huffman_table_bits, + &plitstack)) + return 0; + plit = plitstack; + literal_count = poutend - plit; + + seq_hdr = *pin; + pin++; + if (seq_hdr < 128) + seq_count = seq_hdr; + else if (seq_hdr < 255) + { + if (unlikely (pin >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + seq_count = ((seq_hdr - 128) << 8) + *pin; + pin++; + } + else + { + if (unlikely (pin + 1 >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + seq_count = *pin + (pin[1] << 8) + 0x7f00; + pin += 2; + } + + if (seq_count > 0) + { + int (*pfn)(const struct elf_zstd_fse_entry *, + int, struct elf_zstd_fse_baseline_entry *); + + if (unlikely (pin >= pinend)) + { + elf_uncompress_failed (); + return 0; + } + seq_hdr = *pin; + ++pin; + + pfn = elf_zstd_make_literal_baseline_fse; + if (!elf_zstd_unpack_seq_decode ((seq_hdr >> 6) & 3, + &pin, pinend, + &elf_zstd_lit_table[0], 6, + scratch, 35, + literal_fse_table, 9, pfn, + &literal_decode)) + return 0; + + pfn = elf_zstd_make_offset_baseline_fse; + if (!elf_zstd_unpack_seq_decode ((seq_hdr >> 4) & 3, + &pin, pinend, + &elf_zstd_offset_table[0], 5, + scratch, 31, + offset_fse_table, 8, pfn, + &offset_decode)) + return 0; + + pfn = elf_zstd_make_match_baseline_fse; + if (!elf_zstd_unpack_seq_decode ((seq_hdr >> 2) & 3, + &pin, pinend, + &elf_zstd_match_table[0], 6, + scratch, 52, + match_fse_table, 9, pfn, + &match_decode)) + return 0; + } + + pback = pblockend - 1; + if (!elf_fetch_backward_init (&pback, pin, &val, &bits)) + return 0; + + bits -= literal_decode.table_bits; + literal_state = ((val >> bits) + & ((1U << literal_decode.table_bits) - 1)); + + if (!elf_fetch_bits_backward (&pback, pin, &val, &bits)) + return 0; + bits -= offset_decode.table_bits; + offset_state = ((val >> bits) + & ((1U << offset_decode.table_bits) - 1)); + + if (!elf_fetch_bits_backward (&pback, pin, &val, &bits)) + return 0; + bits -= match_decode.table_bits; + match_state = ((val >> bits) + & ((1U << match_decode.table_bits) - 1)); + + seq = 0; + while (1) + { + const struct elf_zstd_fse_baseline_entry *pt; + uint32_t offset_basebits; + uint32_t offset_baseline; + uint32_t offset_bits; + uint32_t offset_base; + uint32_t offset; + uint32_t match_baseline; + uint32_t match_bits; + uint32_t match_base; + uint32_t match; + uint32_t literal_baseline; + uint32_t literal_bits; + uint32_t literal_base; + uint32_t literal; + uint32_t need; + uint32_t add; + + pt = &offset_decode.table[offset_state]; + offset_basebits = pt->basebits; + offset_baseline = pt->baseline; + offset_bits = pt->bits; + offset_base = pt->base; + + /* This case can be more than 16 bits, which is all that + elf_fetch_bits_backward promises. */ + need = offset_basebits; + add = 0; + if (unlikely (need > 16)) + { + if (!elf_fetch_bits_backward (&pback, pin, &val, &bits)) + return 0; + bits -= 16; + add = (val >> bits) & ((1U << 16) - 1); + need -= 16; + add <<= need; + } + if (need > 0) + { + if (!elf_fetch_bits_backward (&pback, pin, &val, &bits)) + return 0; + bits -= need; + add += (val >> bits) & ((1U << need) - 1); + } + + offset = offset_baseline + add; + + pt = &match_decode.table[match_state]; + need = pt->basebits; + match_baseline = pt->baseline; + match_bits = pt->bits; + match_base = pt->base; + + add = 0; + if (need > 0) + { + if (!elf_fetch_bits_backward (&pback, pin, &val, &bits)) + return 0; + bits -= need; + add = (val >> bits) & ((1U << need) - 1); + } + + match = match_baseline + add; + + pt = &literal_decode.table[literal_state]; + need = pt->basebits; + literal_baseline = pt->baseline; + literal_bits = pt->bits; + literal_base = pt->base; + + add = 0; + if (need > 0) + { + if (!elf_fetch_bits_backward (&pback, pin, &val, &bits)) + return 0; + bits -= need; + add = (val >> bits) & ((1U << need) - 1); + } + + literal = literal_baseline + add; + + /* See the comment in elf_zstd_make_offset_baseline_fse. */ + if (offset_basebits > 1) + { + repeated_offset3 = repeated_offset2; + repeated_offset2 = repeated_offset1; + repeated_offset1 = offset; + } + else + { + if (unlikely (literal == 0)) + ++offset; + switch (offset) + { + case 1: + offset = repeated_offset1; + break; + case 2: + offset = repeated_offset2; + repeated_offset2 = repeated_offset1; + repeated_offset1 = offset; + break; + case 3: + offset = repeated_offset3; + repeated_offset3 = repeated_offset2; + repeated_offset2 = repeated_offset1; + repeated_offset1 = offset; + break; + case 4: + offset = repeated_offset1 - 1; + repeated_offset3 = repeated_offset2; + repeated_offset2 = repeated_offset1; + repeated_offset1 = offset; + break; + } + } + + ++seq; + if (seq < seq_count) + { + uint32_t v; + + /* Update the three states. */ + + if (!elf_fetch_bits_backward (&pback, pin, &val, &bits)) + return 0; + + need = literal_bits; + bits -= need; + v = (val >> bits) & (((uint32_t)1 << need) - 1); + + literal_state = literal_base + v; + + if (!elf_fetch_bits_backward (&pback, pin, &val, &bits)) + return 0; + + need = match_bits; + bits -= need; + v = (val >> bits) & (((uint32_t)1 << need) - 1); + + match_state = match_base + v; + + if (!elf_fetch_bits_backward (&pback, pin, &val, &bits)) + return 0; + + need = offset_bits; + bits -= need; + v = (val >> bits) & (((uint32_t)1 << need) - 1); + + offset_state = offset_base + v; + } + + /* The next sequence is now in LITERAL, OFFSET, MATCH. */ + + /* Copy LITERAL bytes from the literals. */ + + if (unlikely ((size_t)(poutend - pout) < literal)) + { + elf_uncompress_failed (); + return 0; + } + + if (unlikely (literal_count < literal)) + { + elf_uncompress_failed (); + return 0; + } + + literal_count -= literal; + + /* Often LITERAL is small, so handle small cases quickly. */ + switch (literal) + { + case 8: + *pout++ = *plit++; + /* FALLTHROUGH */ + case 7: + *pout++ = *plit++; + /* FALLTHROUGH */ + case 6: + *pout++ = *plit++; + /* FALLTHROUGH */ + case 5: + *pout++ = *plit++; + /* FALLTHROUGH */ + case 4: + *pout++ = *plit++; + /* FALLTHROUGH */ + case 3: + *pout++ = *plit++; + /* FALLTHROUGH */ + case 2: + *pout++ = *plit++; + /* FALLTHROUGH */ + case 1: + *pout++ = *plit++; + break; + + case 0: + break; + + default: + if (unlikely ((size_t)(plit - pout) < literal)) + { + uint32_t move; + + move = plit - pout; + while (literal > move) + { + memcpy (pout, plit, move); + pout += move; + plit += move; + literal -= move; + } + } + + memcpy (pout, plit, literal); + pout += literal; + plit += literal; + } + + if (match > 0) + { + /* Copy MATCH bytes from the decoded output at OFFSET. */ + + if (unlikely ((size_t)(poutend - pout) < match)) + { + elf_uncompress_failed (); + return 0; + } + + if (unlikely ((size_t)(pout - poutstart) < offset)) + { + elf_uncompress_failed (); + return 0; + } + + if (offset >= match) + { + memcpy (pout, pout - offset, match); + pout += match; + } + else + { + while (match > 0) + { + uint32_t copy; + + copy = match < offset ? match : offset; + memcpy (pout, pout - offset, copy); + match -= copy; + pout += copy; + } + } + } + + if (unlikely (seq >= seq_count)) + { + /* Copy remaining literals. */ + if (literal_count > 0 && plit != pout) + { + if (unlikely ((size_t)(poutend - pout) + < literal_count)) + { + elf_uncompress_failed (); + return 0; + } + + if ((size_t)(plit - pout) < literal_count) + { + uint32_t move; + + move = plit - pout; + while (literal_count > move) + { + memcpy (pout, plit, move); + pout += move; + plit += move; + literal_count -= move; + } + } + + memcpy (pout, plit, literal_count); + } + + pout += literal_count; + + break; + } + } + + pin = pblockend; + } + break; + + case 3: + default: + elf_uncompress_failed (); + return 0; + } + } + + if (has_checksum) + { + if (unlikely (pin + 4 > pinend)) + { + elf_uncompress_failed (); + return 0; + } + + /* We don't currently verify the checksum. Currently running GNU ld with + --compress-debug-sections=zstd does not seem to generate a + checksum. */ + + pin += 4; + } + + if (pin != pinend) + { + elf_uncompress_failed (); + return 0; + } + + return 1; +} + +#define ZDEBUG_TABLE_SIZE \ + (ZLIB_TABLE_SIZE > ZSTD_TABLE_SIZE ? ZLIB_TABLE_SIZE : ZSTD_TABLE_SIZE) + +/* Uncompress the old compressed debug format, the one emitted by + --compress-debug-sections=zlib-gnu. The compressed data is in + COMPRESSED / COMPRESSED_SIZE, and the function writes to + *UNCOMPRESSED / *UNCOMPRESSED_SIZE. ZDEBUG_TABLE is work space to + hold Huffman tables. Returns 0 on error, 1 on successful + decompression or if something goes wrong. In general we try to + carry on, by returning 1, even if we can't decompress. */ + +static int +elf_uncompress_zdebug (struct backtrace_state *state, + const unsigned char *compressed, size_t compressed_size, + uint16_t *zdebug_table, + backtrace_error_callback error_callback, void *data, + unsigned char **uncompressed, size_t *uncompressed_size) +{ + size_t sz; + size_t i; + unsigned char *po; + + *uncompressed = NULL; + *uncompressed_size = 0; + + /* The format starts with the four bytes ZLIB, followed by the 8 + byte length of the uncompressed data in big-endian order, + followed by a zlib stream. */ + + if (compressed_size < 12 || memcmp (compressed, "ZLIB", 4) != 0) + return 1; + + sz = 0; + for (i = 0; i < 8; i++) + sz = (sz << 8) | compressed[i + 4]; + + if (*uncompressed != NULL && *uncompressed_size >= sz) + po = *uncompressed; + else + { + po = (unsigned char *) backtrace_alloc (state, sz, error_callback, data); + if (po == NULL) + return 0; + } + + if (!elf_zlib_inflate_and_verify (compressed + 12, compressed_size - 12, + zdebug_table, po, sz)) + return 1; + + *uncompressed = po; + *uncompressed_size = sz; + + return 1; +} + +/* Uncompress the new compressed debug format, the official standard + ELF approach emitted by --compress-debug-sections=zlib-gabi. The + compressed data is in COMPRESSED / COMPRESSED_SIZE, and the + function writes to *UNCOMPRESSED / *UNCOMPRESSED_SIZE. + ZDEBUG_TABLE is work space as for elf_uncompress_zdebug. Returns 0 + on error, 1 on successful decompression or if something goes wrong. + In general we try to carry on, by returning 1, even if we can't + decompress. */ + +static int +elf_uncompress_chdr (struct backtrace_state *state, + const unsigned char *compressed, size_t compressed_size, + uint16_t *zdebug_table, + backtrace_error_callback error_callback, void *data, + unsigned char **uncompressed, size_t *uncompressed_size) +{ + b_elf_chdr chdr; + char *alc; + size_t alc_len; + unsigned char *po; + + *uncompressed = NULL; + *uncompressed_size = 0; + + /* The format starts with an ELF compression header. */ + if (compressed_size < sizeof (b_elf_chdr)) + return 1; + + /* The lld linker can misalign a compressed section, so we can't safely read + the fields directly as we can for other ELF sections. See + https://github.com/ianlancetaylor/libbacktrace/pull/120. */ + memcpy (&chdr, compressed, sizeof (b_elf_chdr)); + + alc = NULL; + alc_len = 0; + if (*uncompressed != NULL && *uncompressed_size >= chdr.ch_size) + po = *uncompressed; + else + { + alc_len = chdr.ch_size; + alc = (char*)backtrace_alloc (state, alc_len, error_callback, data); + if (alc == NULL) + return 0; + po = (unsigned char *) alc; + } + + switch (chdr.ch_type) + { + case ELFCOMPRESS_ZLIB: + if (!elf_zlib_inflate_and_verify (compressed + sizeof (b_elf_chdr), + compressed_size - sizeof (b_elf_chdr), + zdebug_table, po, chdr.ch_size)) + goto skip; + break; + + case ELFCOMPRESS_ZSTD: + if (!elf_zstd_decompress (compressed + sizeof (b_elf_chdr), + compressed_size - sizeof (b_elf_chdr), + (unsigned char *)zdebug_table, po, + chdr.ch_size)) + goto skip; + break; + + default: + /* Unsupported compression algorithm. */ + goto skip; + } + + *uncompressed = po; + *uncompressed_size = chdr.ch_size; + + return 1; + + skip: + if (alc != NULL && alc_len > 0) + backtrace_free (state, alc, alc_len, error_callback, data); + return 1; +} + +/* This function is a hook for testing the zlib support. It is only + used by tests. */ + +int +backtrace_uncompress_zdebug (struct backtrace_state *state, + const unsigned char *compressed, + size_t compressed_size, + backtrace_error_callback error_callback, + void *data, unsigned char **uncompressed, + size_t *uncompressed_size) +{ + uint16_t *zdebug_table; + int ret; + + zdebug_table = ((uint16_t *) backtrace_alloc (state, ZDEBUG_TABLE_SIZE, + error_callback, data)); + if (zdebug_table == NULL) + return 0; + ret = elf_uncompress_zdebug (state, compressed, compressed_size, + zdebug_table, error_callback, data, + uncompressed, uncompressed_size); + backtrace_free (state, zdebug_table, ZDEBUG_TABLE_SIZE, + error_callback, data); + return ret; +} + +/* This function is a hook for testing the zstd support. It is only used by + tests. */ + +int +backtrace_uncompress_zstd (struct backtrace_state *state, + const unsigned char *compressed, + size_t compressed_size, + backtrace_error_callback error_callback, + void *data, unsigned char *uncompressed, + size_t uncompressed_size) +{ + unsigned char *zdebug_table; + int ret; + + zdebug_table = ((unsigned char *) backtrace_alloc (state, ZDEBUG_TABLE_SIZE, + error_callback, data)); + if (zdebug_table == NULL) + return 0; + ret = elf_zstd_decompress (compressed, compressed_size, + zdebug_table, uncompressed, uncompressed_size); + backtrace_free (state, zdebug_table, ZDEBUG_TABLE_SIZE, + error_callback, data); + return ret; +} + +/* Number of LZMA states. */ +#define LZMA_STATES (12) + +/* Number of LZMA position states. The pb value of the property byte + is the number of bits to include in these states, and the maximum + value of pb is 4. */ +#define LZMA_POS_STATES (16) + +/* Number of LZMA distance states. These are used match distances + with a short match length: up to 4 bytes. */ +#define LZMA_DIST_STATES (4) + +/* Number of LZMA distance slots. LZMA uses six bits to encode larger + match lengths, so 1 << 6 possible probabilities. */ +#define LZMA_DIST_SLOTS (64) + +/* LZMA distances 0 to 3 are encoded directly, larger values use a + probability model. */ +#define LZMA_DIST_MODEL_START (4) + +/* The LZMA probability model ends at 14. */ +#define LZMA_DIST_MODEL_END (14) + +/* LZMA distance slots for distances less than 127. */ +#define LZMA_FULL_DISTANCES (128) + +/* LZMA uses four alignment bits. */ +#define LZMA_ALIGN_SIZE (16) + +/* LZMA match length is encoded with 4, 5, or 10 bits, some of which + are already known. */ +#define LZMA_LEN_LOW_SYMBOLS (8) +#define LZMA_LEN_MID_SYMBOLS (8) +#define LZMA_LEN_HIGH_SYMBOLS (256) + +/* LZMA literal encoding. */ +#define LZMA_LITERAL_CODERS_MAX (16) +#define LZMA_LITERAL_CODER_SIZE (0x300) + +/* LZMA is based on a large set of probabilities, each managed + independently. Each probability is an 11 bit number that we store + in a uint16_t. We use a single large array of probabilities. */ + +/* Lengths of entries in the LZMA probabilities array. The names used + here are copied from the Linux kernel implementation. */ + +#define LZMA_PROB_IS_MATCH_LEN (LZMA_STATES * LZMA_POS_STATES) +#define LZMA_PROB_IS_REP_LEN LZMA_STATES +#define LZMA_PROB_IS_REP0_LEN LZMA_STATES +#define LZMA_PROB_IS_REP1_LEN LZMA_STATES +#define LZMA_PROB_IS_REP2_LEN LZMA_STATES +#define LZMA_PROB_IS_REP0_LONG_LEN (LZMA_STATES * LZMA_POS_STATES) +#define LZMA_PROB_DIST_SLOT_LEN (LZMA_DIST_STATES * LZMA_DIST_SLOTS) +#define LZMA_PROB_DIST_SPECIAL_LEN (LZMA_FULL_DISTANCES - LZMA_DIST_MODEL_END) +#define LZMA_PROB_DIST_ALIGN_LEN LZMA_ALIGN_SIZE +#define LZMA_PROB_MATCH_LEN_CHOICE_LEN 1 +#define LZMA_PROB_MATCH_LEN_CHOICE2_LEN 1 +#define LZMA_PROB_MATCH_LEN_LOW_LEN (LZMA_POS_STATES * LZMA_LEN_LOW_SYMBOLS) +#define LZMA_PROB_MATCH_LEN_MID_LEN (LZMA_POS_STATES * LZMA_LEN_MID_SYMBOLS) +#define LZMA_PROB_MATCH_LEN_HIGH_LEN LZMA_LEN_HIGH_SYMBOLS +#define LZMA_PROB_REP_LEN_CHOICE_LEN 1 +#define LZMA_PROB_REP_LEN_CHOICE2_LEN 1 +#define LZMA_PROB_REP_LEN_LOW_LEN (LZMA_POS_STATES * LZMA_LEN_LOW_SYMBOLS) +#define LZMA_PROB_REP_LEN_MID_LEN (LZMA_POS_STATES * LZMA_LEN_MID_SYMBOLS) +#define LZMA_PROB_REP_LEN_HIGH_LEN LZMA_LEN_HIGH_SYMBOLS +#define LZMA_PROB_LITERAL_LEN \ + (LZMA_LITERAL_CODERS_MAX * LZMA_LITERAL_CODER_SIZE) + +/* Offsets into the LZMA probabilities array. This is mechanically + generated from the above lengths. */ + +#define LZMA_PROB_IS_MATCH_OFFSET 0 +#define LZMA_PROB_IS_REP_OFFSET \ + (LZMA_PROB_IS_MATCH_OFFSET + LZMA_PROB_IS_MATCH_LEN) +#define LZMA_PROB_IS_REP0_OFFSET \ + (LZMA_PROB_IS_REP_OFFSET + LZMA_PROB_IS_REP_LEN) +#define LZMA_PROB_IS_REP1_OFFSET \ + (LZMA_PROB_IS_REP0_OFFSET + LZMA_PROB_IS_REP0_LEN) +#define LZMA_PROB_IS_REP2_OFFSET \ + (LZMA_PROB_IS_REP1_OFFSET + LZMA_PROB_IS_REP1_LEN) +#define LZMA_PROB_IS_REP0_LONG_OFFSET \ + (LZMA_PROB_IS_REP2_OFFSET + LZMA_PROB_IS_REP2_LEN) +#define LZMA_PROB_DIST_SLOT_OFFSET \ + (LZMA_PROB_IS_REP0_LONG_OFFSET + LZMA_PROB_IS_REP0_LONG_LEN) +#define LZMA_PROB_DIST_SPECIAL_OFFSET \ + (LZMA_PROB_DIST_SLOT_OFFSET + LZMA_PROB_DIST_SLOT_LEN) +#define LZMA_PROB_DIST_ALIGN_OFFSET \ + (LZMA_PROB_DIST_SPECIAL_OFFSET + LZMA_PROB_DIST_SPECIAL_LEN) +#define LZMA_PROB_MATCH_LEN_CHOICE_OFFSET \ + (LZMA_PROB_DIST_ALIGN_OFFSET + LZMA_PROB_DIST_ALIGN_LEN) +#define LZMA_PROB_MATCH_LEN_CHOICE2_OFFSET \ + (LZMA_PROB_MATCH_LEN_CHOICE_OFFSET + LZMA_PROB_MATCH_LEN_CHOICE_LEN) +#define LZMA_PROB_MATCH_LEN_LOW_OFFSET \ + (LZMA_PROB_MATCH_LEN_CHOICE2_OFFSET + LZMA_PROB_MATCH_LEN_CHOICE2_LEN) +#define LZMA_PROB_MATCH_LEN_MID_OFFSET \ + (LZMA_PROB_MATCH_LEN_LOW_OFFSET + LZMA_PROB_MATCH_LEN_LOW_LEN) +#define LZMA_PROB_MATCH_LEN_HIGH_OFFSET \ + (LZMA_PROB_MATCH_LEN_MID_OFFSET + LZMA_PROB_MATCH_LEN_MID_LEN) +#define LZMA_PROB_REP_LEN_CHOICE_OFFSET \ + (LZMA_PROB_MATCH_LEN_HIGH_OFFSET + LZMA_PROB_MATCH_LEN_HIGH_LEN) +#define LZMA_PROB_REP_LEN_CHOICE2_OFFSET \ + (LZMA_PROB_REP_LEN_CHOICE_OFFSET + LZMA_PROB_REP_LEN_CHOICE_LEN) +#define LZMA_PROB_REP_LEN_LOW_OFFSET \ + (LZMA_PROB_REP_LEN_CHOICE2_OFFSET + LZMA_PROB_REP_LEN_CHOICE2_LEN) +#define LZMA_PROB_REP_LEN_MID_OFFSET \ + (LZMA_PROB_REP_LEN_LOW_OFFSET + LZMA_PROB_REP_LEN_LOW_LEN) +#define LZMA_PROB_REP_LEN_HIGH_OFFSET \ + (LZMA_PROB_REP_LEN_MID_OFFSET + LZMA_PROB_REP_LEN_MID_LEN) +#define LZMA_PROB_LITERAL_OFFSET \ + (LZMA_PROB_REP_LEN_HIGH_OFFSET + LZMA_PROB_REP_LEN_HIGH_LEN) + +#define LZMA_PROB_TOTAL_COUNT \ + (LZMA_PROB_LITERAL_OFFSET + LZMA_PROB_LITERAL_LEN) + +/* Check that the number of LZMA probabilities is the same as the + Linux kernel implementation. */ + +#if LZMA_PROB_TOTAL_COUNT != 1846 + (1 << 4) * 0x300 + #error Wrong number of LZMA probabilities +#endif + +/* Expressions for the offset in the LZMA probabilities array of a + specific probability. */ + +#define LZMA_IS_MATCH(state, pos) \ + (LZMA_PROB_IS_MATCH_OFFSET + (state) * LZMA_POS_STATES + (pos)) +#define LZMA_IS_REP(state) \ + (LZMA_PROB_IS_REP_OFFSET + (state)) +#define LZMA_IS_REP0(state) \ + (LZMA_PROB_IS_REP0_OFFSET + (state)) +#define LZMA_IS_REP1(state) \ + (LZMA_PROB_IS_REP1_OFFSET + (state)) +#define LZMA_IS_REP2(state) \ + (LZMA_PROB_IS_REP2_OFFSET + (state)) +#define LZMA_IS_REP0_LONG(state, pos) \ + (LZMA_PROB_IS_REP0_LONG_OFFSET + (state) * LZMA_POS_STATES + (pos)) +#define LZMA_DIST_SLOT(dist, slot) \ + (LZMA_PROB_DIST_SLOT_OFFSET + (dist) * LZMA_DIST_SLOTS + (slot)) +#define LZMA_DIST_SPECIAL(dist) \ + (LZMA_PROB_DIST_SPECIAL_OFFSET + (dist)) +#define LZMA_DIST_ALIGN(dist) \ + (LZMA_PROB_DIST_ALIGN_OFFSET + (dist)) +#define LZMA_MATCH_LEN_CHOICE \ + LZMA_PROB_MATCH_LEN_CHOICE_OFFSET +#define LZMA_MATCH_LEN_CHOICE2 \ + LZMA_PROB_MATCH_LEN_CHOICE2_OFFSET +#define LZMA_MATCH_LEN_LOW(pos, sym) \ + (LZMA_PROB_MATCH_LEN_LOW_OFFSET + (pos) * LZMA_LEN_LOW_SYMBOLS + (sym)) +#define LZMA_MATCH_LEN_MID(pos, sym) \ + (LZMA_PROB_MATCH_LEN_MID_OFFSET + (pos) * LZMA_LEN_MID_SYMBOLS + (sym)) +#define LZMA_MATCH_LEN_HIGH(sym) \ + (LZMA_PROB_MATCH_LEN_HIGH_OFFSET + (sym)) +#define LZMA_REP_LEN_CHOICE \ + LZMA_PROB_REP_LEN_CHOICE_OFFSET +#define LZMA_REP_LEN_CHOICE2 \ + LZMA_PROB_REP_LEN_CHOICE2_OFFSET +#define LZMA_REP_LEN_LOW(pos, sym) \ + (LZMA_PROB_REP_LEN_LOW_OFFSET + (pos) * LZMA_LEN_LOW_SYMBOLS + (sym)) +#define LZMA_REP_LEN_MID(pos, sym) \ + (LZMA_PROB_REP_LEN_MID_OFFSET + (pos) * LZMA_LEN_MID_SYMBOLS + (sym)) +#define LZMA_REP_LEN_HIGH(sym) \ + (LZMA_PROB_REP_LEN_HIGH_OFFSET + (sym)) +#define LZMA_LITERAL(code, size) \ + (LZMA_PROB_LITERAL_OFFSET + (code) * LZMA_LITERAL_CODER_SIZE + (size)) + +/* Read an LZMA varint from BUF, reading and updating *POFFSET, + setting *VAL. Returns 0 on error, 1 on success. */ + +static int +elf_lzma_varint (const unsigned char *compressed, size_t compressed_size, + size_t *poffset, uint64_t *val) +{ + size_t off; + int i; + uint64_t v; + unsigned char b; + + off = *poffset; + i = 0; + v = 0; + while (1) + { + if (unlikely (off >= compressed_size)) + { + elf_uncompress_failed (); + return 0; + } + b = compressed[off]; + v |= (b & 0x7f) << (i * 7); + ++off; + if ((b & 0x80) == 0) + { + *poffset = off; + *val = v; + return 1; + } + ++i; + if (unlikely (i >= 9)) + { + elf_uncompress_failed (); + return 0; + } + } +} + +/* Normalize the LZMA range decoder, pulling in an extra input byte if + needed. */ + +static void +elf_lzma_range_normalize (const unsigned char *compressed, + size_t compressed_size, size_t *poffset, + uint32_t *prange, uint32_t *pcode) +{ + if (*prange < (1U << 24)) + { + if (unlikely (*poffset >= compressed_size)) + { + /* We assume this will be caught elsewhere. */ + elf_uncompress_failed (); + return; + } + *prange <<= 8; + *pcode <<= 8; + *pcode += compressed[*poffset]; + ++*poffset; + } +} + +/* Read and return a single bit from the LZMA stream, reading and + updating *PROB. Each bit comes from the range coder. */ + +static int +elf_lzma_bit (const unsigned char *compressed, size_t compressed_size, + uint16_t *prob, size_t *poffset, uint32_t *prange, + uint32_t *pcode) +{ + uint32_t bound; + + elf_lzma_range_normalize (compressed, compressed_size, poffset, + prange, pcode); + bound = (*prange >> 11) * (uint32_t) *prob; + if (*pcode < bound) + { + *prange = bound; + *prob += ((1U << 11) - *prob) >> 5; + return 0; + } + else + { + *prange -= bound; + *pcode -= bound; + *prob -= *prob >> 5; + return 1; + } +} + +/* Read an integer of size BITS from the LZMA stream, most significant + bit first. The bits are predicted using PROBS. */ + +static uint32_t +elf_lzma_integer (const unsigned char *compressed, size_t compressed_size, + uint16_t *probs, uint32_t bits, size_t *poffset, + uint32_t *prange, uint32_t *pcode) +{ + uint32_t sym; + uint32_t i; + + sym = 1; + for (i = 0; i < bits; i++) + { + int bit; + + bit = elf_lzma_bit (compressed, compressed_size, probs + sym, poffset, + prange, pcode); + sym <<= 1; + sym += bit; + } + return sym - (1 << bits); +} + +/* Read an integer of size BITS from the LZMA stream, least + significant bit first. The bits are predicted using PROBS. */ + +static uint32_t +elf_lzma_reverse_integer (const unsigned char *compressed, + size_t compressed_size, uint16_t *probs, + uint32_t bits, size_t *poffset, uint32_t *prange, + uint32_t *pcode) +{ + uint32_t sym; + uint32_t val; + uint32_t i; + + sym = 1; + val = 0; + for (i = 0; i < bits; i++) + { + int bit; + + bit = elf_lzma_bit (compressed, compressed_size, probs + sym, poffset, + prange, pcode); + sym <<= 1; + sym += bit; + val += bit << i; + } + return val; +} + +/* Read a length from the LZMA stream. IS_REP picks either LZMA_MATCH + or LZMA_REP probabilities. */ + +static uint32_t +elf_lzma_len (const unsigned char *compressed, size_t compressed_size, + uint16_t *probs, int is_rep, unsigned int pos_state, + size_t *poffset, uint32_t *prange, uint32_t *pcode) +{ + uint16_t *probs_choice; + uint16_t *probs_sym; + uint32_t bits; + uint32_t len; + + probs_choice = probs + (is_rep + ? LZMA_REP_LEN_CHOICE + : LZMA_MATCH_LEN_CHOICE); + if (elf_lzma_bit (compressed, compressed_size, probs_choice, poffset, + prange, pcode)) + { + probs_choice = probs + (is_rep + ? LZMA_REP_LEN_CHOICE2 + : LZMA_MATCH_LEN_CHOICE2); + if (elf_lzma_bit (compressed, compressed_size, probs_choice, + poffset, prange, pcode)) + { + probs_sym = probs + (is_rep + ? LZMA_REP_LEN_HIGH (0) + : LZMA_MATCH_LEN_HIGH (0)); + bits = 8; + len = 2 + 8 + 8; + } + else + { + probs_sym = probs + (is_rep + ? LZMA_REP_LEN_MID (pos_state, 0) + : LZMA_MATCH_LEN_MID (pos_state, 0)); + bits = 3; + len = 2 + 8; + } + } + else + { + probs_sym = probs + (is_rep + ? LZMA_REP_LEN_LOW (pos_state, 0) + : LZMA_MATCH_LEN_LOW (pos_state, 0)); + bits = 3; + len = 2; + } + + len += elf_lzma_integer (compressed, compressed_size, probs_sym, bits, + poffset, prange, pcode); + return len; +} + +/* Uncompress one LZMA block from a minidebug file. The compressed + data is at COMPRESSED + *POFFSET. Update *POFFSET. Store the data + into the memory at UNCOMPRESSED, size UNCOMPRESSED_SIZE. CHECK is + the stream flag from the xz header. Return 1 on successful + decompression. */ + +static int +elf_uncompress_lzma_block (const unsigned char *compressed, + size_t compressed_size, unsigned char check, + uint16_t *probs, unsigned char *uncompressed, + size_t uncompressed_size, size_t *poffset) +{ + size_t off; + size_t block_header_offset; + size_t block_header_size; + unsigned char block_flags; + uint64_t header_compressed_size; + uint64_t header_uncompressed_size; + unsigned char lzma2_properties; + size_t crc_offset; + uint32_t computed_crc; + uint32_t stream_crc; + size_t uncompressed_offset; + size_t dict_start_offset; + unsigned int lc; + unsigned int lp; + unsigned int pb; + uint32_t range; + uint32_t code; + uint32_t lstate; + uint32_t dist[4]; + + off = *poffset; + block_header_offset = off; + + /* Block header size is a single byte. */ + if (unlikely (off >= compressed_size)) + { + elf_uncompress_failed (); + return 0; + } + block_header_size = (compressed[off] + 1) * 4; + if (unlikely (off + block_header_size > compressed_size)) + { + elf_uncompress_failed (); + return 0; + } + + /* Block flags. */ + block_flags = compressed[off + 1]; + if (unlikely ((block_flags & 0x3c) != 0)) + { + elf_uncompress_failed (); + return 0; + } + + off += 2; + + /* Optional compressed size. */ + header_compressed_size = 0; + if ((block_flags & 0x40) != 0) + { + *poffset = off; + if (!elf_lzma_varint (compressed, compressed_size, poffset, + &header_compressed_size)) + return 0; + off = *poffset; + } + + /* Optional uncompressed size. */ + header_uncompressed_size = 0; + if ((block_flags & 0x80) != 0) + { + *poffset = off; + if (!elf_lzma_varint (compressed, compressed_size, poffset, + &header_uncompressed_size)) + return 0; + off = *poffset; + } + + /* The recipe for creating a minidebug file is to run the xz program + with no arguments, so we expect exactly one filter: lzma2. */ + + if (unlikely ((block_flags & 0x3) != 0)) + { + elf_uncompress_failed (); + return 0; + } + + if (unlikely (off + 2 >= block_header_offset + block_header_size)) + { + elf_uncompress_failed (); + return 0; + } + + /* The filter ID for LZMA2 is 0x21. */ + if (unlikely (compressed[off] != 0x21)) + { + elf_uncompress_failed (); + return 0; + } + ++off; + + /* The size of the filter properties for LZMA2 is 1. */ + if (unlikely (compressed[off] != 1)) + { + elf_uncompress_failed (); + return 0; + } + ++off; + + lzma2_properties = compressed[off]; + ++off; + + if (unlikely (lzma2_properties > 40)) + { + elf_uncompress_failed (); + return 0; + } + + /* The properties describe the dictionary size, but we don't care + what that is. */ + + /* Skip to just before CRC, verifying zero bytes in between. */ + crc_offset = block_header_offset + block_header_size - 4; + if (unlikely (crc_offset + 4 > compressed_size)) + { + elf_uncompress_failed (); + return 0; + } + for (; off < crc_offset; off++) + { + if (compressed[off] != 0) + { + elf_uncompress_failed (); + return 0; + } + } + + /* Block header CRC. */ + computed_crc = elf_crc32 (0, compressed + block_header_offset, + block_header_size - 4); + stream_crc = (compressed[off] + | (compressed[off + 1] << 8) + | (compressed[off + 2] << 16) + | (compressed[off + 3] << 24)); + if (unlikely (computed_crc != stream_crc)) + { + elf_uncompress_failed (); + return 0; + } + off += 4; + + /* Read a sequence of LZMA2 packets. */ + + uncompressed_offset = 0; + dict_start_offset = 0; + lc = 0; + lp = 0; + pb = 0; + lstate = 0; + while (off < compressed_size) + { + unsigned char control; + + range = 0xffffffff; + code = 0; + + control = compressed[off]; + ++off; + if (unlikely (control == 0)) + { + /* End of packets. */ + break; + } + + if (control == 1 || control >= 0xe0) + { + /* Reset dictionary to empty. */ + dict_start_offset = uncompressed_offset; + } + + if (control < 0x80) + { + size_t chunk_size; + + /* The only valid values here are 1 or 2. A 1 means to + reset the dictionary (done above). Then we see an + uncompressed chunk. */ + + if (unlikely (control > 2)) + { + elf_uncompress_failed (); + return 0; + } + + /* An uncompressed chunk is a two byte size followed by + data. */ + + if (unlikely (off + 2 > compressed_size)) + { + elf_uncompress_failed (); + return 0; + } + + chunk_size = compressed[off] << 8; + chunk_size += compressed[off + 1]; + ++chunk_size; + + off += 2; + + if (unlikely (off + chunk_size > compressed_size)) + { + elf_uncompress_failed (); + return 0; + } + if (unlikely (uncompressed_offset + chunk_size > uncompressed_size)) + { + elf_uncompress_failed (); + return 0; + } + + memcpy (uncompressed + uncompressed_offset, compressed + off, + chunk_size); + uncompressed_offset += chunk_size; + off += chunk_size; + } + else + { + size_t uncompressed_chunk_start; + size_t uncompressed_chunk_size; + size_t compressed_chunk_size; + size_t limit; + + /* An LZMA chunk. This starts with an uncompressed size and + a compressed size. */ + + if (unlikely (off + 4 >= compressed_size)) + { + elf_uncompress_failed (); + return 0; + } + + uncompressed_chunk_start = uncompressed_offset; + + uncompressed_chunk_size = (control & 0x1f) << 16; + uncompressed_chunk_size += compressed[off] << 8; + uncompressed_chunk_size += compressed[off + 1]; + ++uncompressed_chunk_size; + + compressed_chunk_size = compressed[off + 2] << 8; + compressed_chunk_size += compressed[off + 3]; + ++compressed_chunk_size; + + off += 4; + + /* Bit 7 (0x80) is set. + Bits 6 and 5 (0x40 and 0x20) are as follows: + 0: don't reset anything + 1: reset state + 2: reset state, read properties + 3: reset state, read properties, reset dictionary (done above) */ + + if (control >= 0xc0) + { + unsigned char props; + + /* Bit 6 is set, read properties. */ + + if (unlikely (off >= compressed_size)) + { + elf_uncompress_failed (); + return 0; + } + props = compressed[off]; + ++off; + if (unlikely (props > (4 * 5 + 4) * 9 + 8)) + { + elf_uncompress_failed (); + return 0; + } + pb = 0; + while (props >= 9 * 5) + { + props -= 9 * 5; + ++pb; + } + lp = 0; + while (props > 9) + { + props -= 9; + ++lp; + } + lc = props; + if (unlikely (lc + lp > 4)) + { + elf_uncompress_failed (); + return 0; + } + } + + if (control >= 0xa0) + { + size_t i; + + /* Bit 5 or 6 is set, reset LZMA state. */ + + lstate = 0; + memset (&dist, 0, sizeof dist); + for (i = 0; i < LZMA_PROB_TOTAL_COUNT; i++) + probs[i] = 1 << 10; + range = 0xffffffff; + code = 0; + } + + /* Read the range code. */ + + if (unlikely (off + 5 > compressed_size)) + { + elf_uncompress_failed (); + return 0; + } + + /* The byte at compressed[off] is ignored for some + reason. */ + + code = ((compressed[off + 1] << 24) + + (compressed[off + 2] << 16) + + (compressed[off + 3] << 8) + + compressed[off + 4]); + off += 5; + + /* This is the main LZMA decode loop. */ + + limit = off + compressed_chunk_size; + *poffset = off; + while (*poffset < limit) + { + unsigned int pos_state; + + if (unlikely (uncompressed_offset + == (uncompressed_chunk_start + + uncompressed_chunk_size))) + { + /* We've decompressed all the expected bytes. */ + break; + } + + pos_state = ((uncompressed_offset - dict_start_offset) + & ((1 << pb) - 1)); + + if (elf_lzma_bit (compressed, compressed_size, + probs + LZMA_IS_MATCH (lstate, pos_state), + poffset, &range, &code)) + { + uint32_t len; + + if (elf_lzma_bit (compressed, compressed_size, + probs + LZMA_IS_REP (lstate), + poffset, &range, &code)) + { + int short_rep; + uint32_t next_dist; + + /* Repeated match. */ + + short_rep = 0; + if (elf_lzma_bit (compressed, compressed_size, + probs + LZMA_IS_REP0 (lstate), + poffset, &range, &code)) + { + if (elf_lzma_bit (compressed, compressed_size, + probs + LZMA_IS_REP1 (lstate), + poffset, &range, &code)) + { + if (elf_lzma_bit (compressed, compressed_size, + probs + LZMA_IS_REP2 (lstate), + poffset, &range, &code)) + { + next_dist = dist[3]; + dist[3] = dist[2]; + } + else + { + next_dist = dist[2]; + } + dist[2] = dist[1]; + } + else + { + next_dist = dist[1]; + } + + dist[1] = dist[0]; + dist[0] = next_dist; + } + else + { + if (!elf_lzma_bit (compressed, compressed_size, + (probs + + LZMA_IS_REP0_LONG (lstate, + pos_state)), + poffset, &range, &code)) + short_rep = 1; + } + + if (lstate < 7) + lstate = short_rep ? 9 : 8; + else + lstate = 11; + + if (short_rep) + len = 1; + else + len = elf_lzma_len (compressed, compressed_size, + probs, 1, pos_state, poffset, + &range, &code); + } + else + { + uint32_t dist_state; + uint32_t dist_slot; + uint16_t *probs_dist; + + /* Match. */ + + if (lstate < 7) + lstate = 7; + else + lstate = 10; + dist[3] = dist[2]; + dist[2] = dist[1]; + dist[1] = dist[0]; + len = elf_lzma_len (compressed, compressed_size, + probs, 0, pos_state, poffset, + &range, &code); + + if (len < 4 + 2) + dist_state = len - 2; + else + dist_state = 3; + probs_dist = probs + LZMA_DIST_SLOT (dist_state, 0); + dist_slot = elf_lzma_integer (compressed, + compressed_size, + probs_dist, 6, + poffset, &range, + &code); + if (dist_slot < LZMA_DIST_MODEL_START) + dist[0] = dist_slot; + else + { + uint32_t limit; + + limit = (dist_slot >> 1) - 1; + dist[0] = 2 + (dist_slot & 1); + if (dist_slot < LZMA_DIST_MODEL_END) + { + dist[0] <<= limit; + probs_dist = (probs + + LZMA_DIST_SPECIAL(dist[0] + - dist_slot + - 1)); + dist[0] += + elf_lzma_reverse_integer (compressed, + compressed_size, + probs_dist, + limit, poffset, + &range, &code); + } + else + { + uint32_t dist0; + uint32_t i; + + dist0 = dist[0]; + for (i = 0; i < limit - 4; i++) + { + uint32_t mask; + + elf_lzma_range_normalize (compressed, + compressed_size, + poffset, + &range, &code); + range >>= 1; + code -= range; + mask = -(code >> 31); + code += range & mask; + dist0 <<= 1; + dist0 += mask + 1; + } + dist0 <<= 4; + probs_dist = probs + LZMA_DIST_ALIGN (0); + dist0 += + elf_lzma_reverse_integer (compressed, + compressed_size, + probs_dist, 4, + poffset, + &range, &code); + dist[0] = dist0; + } + } + } + + if (unlikely (uncompressed_offset + - dict_start_offset < dist[0] + 1)) + { + elf_uncompress_failed (); + return 0; + } + if (unlikely (uncompressed_offset + len > uncompressed_size)) + { + elf_uncompress_failed (); + return 0; + } + + if (dist[0] == 0) + { + /* A common case, meaning repeat the last + character LEN times. */ + memset (uncompressed + uncompressed_offset, + uncompressed[uncompressed_offset - 1], + len); + uncompressed_offset += len; + } + else if (dist[0] + 1 >= len) + { + memcpy (uncompressed + uncompressed_offset, + uncompressed + uncompressed_offset - dist[0] - 1, + len); + uncompressed_offset += len; + } + else + { + while (len > 0) + { + uint32_t copy; + + copy = len < dist[0] + 1 ? len : dist[0] + 1; + memcpy (uncompressed + uncompressed_offset, + (uncompressed + uncompressed_offset + - dist[0] - 1), + copy); + len -= copy; + uncompressed_offset += copy; + } + } + } + else + { + unsigned char prev; + unsigned char low; + size_t high; + uint16_t *lit_probs; + unsigned int sym; + + /* Literal value. */ + + if (uncompressed_offset > 0) + prev = uncompressed[uncompressed_offset - 1]; + else + prev = 0; + low = prev >> (8 - lc); + high = (((uncompressed_offset - dict_start_offset) + & ((1 << lp) - 1)) + << lc); + lit_probs = probs + LZMA_LITERAL (low + high, 0); + if (lstate < 7) + sym = elf_lzma_integer (compressed, compressed_size, + lit_probs, 8, poffset, &range, + &code); + else + { + unsigned int match; + unsigned int bit; + unsigned int match_bit; + unsigned int idx; + + sym = 1; + if (uncompressed_offset >= dist[0] + 1) + match = uncompressed[uncompressed_offset - dist[0] - 1]; + else + match = 0; + match <<= 1; + bit = 0x100; + do + { + match_bit = match & bit; + match <<= 1; + idx = bit + match_bit + sym; + sym <<= 1; + if (elf_lzma_bit (compressed, compressed_size, + lit_probs + idx, poffset, + &range, &code)) + { + ++sym; + bit &= match_bit; + } + else + { + bit &= ~ match_bit; + } + } + while (sym < 0x100); + } + + if (unlikely (uncompressed_offset >= uncompressed_size)) + { + elf_uncompress_failed (); + return 0; + } + + uncompressed[uncompressed_offset] = (unsigned char) sym; + ++uncompressed_offset; + if (lstate <= 3) + lstate = 0; + else if (lstate <= 9) + lstate -= 3; + else + lstate -= 6; + } + } + + elf_lzma_range_normalize (compressed, compressed_size, poffset, + &range, &code); + + off = *poffset; + } + } + + /* We have reached the end of the block. Pad to four byte + boundary. */ + off = (off + 3) &~ (size_t) 3; + if (unlikely (off > compressed_size)) + { + elf_uncompress_failed (); + return 0; + } + + switch (check) + { + case 0: + /* No check. */ + break; + + case 1: + /* CRC32 */ + if (unlikely (off + 4 > compressed_size)) + { + elf_uncompress_failed (); + return 0; + } + computed_crc = elf_crc32 (0, uncompressed, uncompressed_offset); + stream_crc = (compressed[off] + | (compressed[off + 1] << 8) + | (compressed[off + 2] << 16) + | (compressed[off + 3] << 24)); + if (computed_crc != stream_crc) + { + elf_uncompress_failed (); + return 0; + } + off += 4; + break; + + case 4: + /* CRC64. We don't bother computing a CRC64 checksum. */ + if (unlikely (off + 8 > compressed_size)) + { + elf_uncompress_failed (); + return 0; + } + off += 8; + break; + + case 10: + /* SHA. We don't bother computing a SHA checksum. */ + if (unlikely (off + 32 > compressed_size)) + { + elf_uncompress_failed (); + return 0; + } + off += 32; + break; + + default: + elf_uncompress_failed (); + return 0; + } + + *poffset = off; + + return 1; +} + +/* Uncompress LZMA data found in a minidebug file. The minidebug + format is described at + https://sourceware.org/gdb/current/onlinedocs/gdb/MiniDebugInfo.html. + Returns 0 on error, 1 on successful decompression. For this + function we return 0 on failure to decompress, as the calling code + will carry on in that case. */ + +static int +elf_uncompress_lzma (struct backtrace_state *state, + const unsigned char *compressed, size_t compressed_size, + backtrace_error_callback error_callback, void *data, + unsigned char **uncompressed, size_t *uncompressed_size) +{ + size_t header_size; + size_t footer_size; + unsigned char check; + uint32_t computed_crc; + uint32_t stream_crc; + size_t offset; + size_t index_size; + size_t footer_offset; + size_t index_offset; + uint64_t index_compressed_size; + uint64_t index_uncompressed_size; + unsigned char *mem; + uint16_t *probs; + size_t compressed_block_size; + + /* The format starts with a stream header and ends with a stream + footer. */ + header_size = 12; + footer_size = 12; + if (unlikely (compressed_size < header_size + footer_size)) + { + elf_uncompress_failed (); + return 0; + } + + /* The stream header starts with a magic string. */ + if (unlikely (memcmp (compressed, "\375" "7zXZ\0", 6) != 0)) + { + elf_uncompress_failed (); + return 0; + } + + /* Next come stream flags. The first byte is zero, the second byte + is the check. */ + if (unlikely (compressed[6] != 0)) + { + elf_uncompress_failed (); + return 0; + } + check = compressed[7]; + if (unlikely ((check & 0xf8) != 0)) + { + elf_uncompress_failed (); + return 0; + } + + /* Next comes a CRC of the stream flags. */ + computed_crc = elf_crc32 (0, compressed + 6, 2); + stream_crc = (compressed[8] + | (compressed[9] << 8) + | (compressed[10] << 16) + | (compressed[11] << 24)); + if (unlikely (computed_crc != stream_crc)) + { + elf_uncompress_failed (); + return 0; + } + + /* Now that we've parsed the header, parse the footer, so that we + can get the uncompressed size. */ + + /* The footer ends with two magic bytes. */ + + offset = compressed_size; + if (unlikely (memcmp (compressed + offset - 2, "YZ", 2) != 0)) + { + elf_uncompress_failed (); + return 0; + } + offset -= 2; + + /* Before that are the stream flags, which should be the same as the + flags in the header. */ + if (unlikely (compressed[offset - 2] != 0 + || compressed[offset - 1] != check)) + { + elf_uncompress_failed (); + return 0; + } + offset -= 2; + + /* Before that is the size of the index field, which precedes the + footer. */ + index_size = (compressed[offset - 4] + | (compressed[offset - 3] << 8) + | (compressed[offset - 2] << 16) + | (compressed[offset - 1] << 24)); + index_size = (index_size + 1) * 4; + offset -= 4; + + /* Before that is a footer CRC. */ + computed_crc = elf_crc32 (0, compressed + offset, 6); + stream_crc = (compressed[offset - 4] + | (compressed[offset - 3] << 8) + | (compressed[offset - 2] << 16) + | (compressed[offset - 1] << 24)); + if (unlikely (computed_crc != stream_crc)) + { + elf_uncompress_failed (); + return 0; + } + offset -= 4; + + /* The index comes just before the footer. */ + if (unlikely (offset < index_size + header_size)) + { + elf_uncompress_failed (); + return 0; + } + + footer_offset = offset; + offset -= index_size; + index_offset = offset; + + /* The index starts with a zero byte. */ + if (unlikely (compressed[offset] != 0)) + { + elf_uncompress_failed (); + return 0; + } + ++offset; + + /* Next is the number of blocks. We expect zero blocks for an empty + stream, and otherwise a single block. */ + if (unlikely (compressed[offset] == 0)) + { + *uncompressed = NULL; + *uncompressed_size = 0; + return 1; + } + if (unlikely (compressed[offset] != 1)) + { + elf_uncompress_failed (); + return 0; + } + ++offset; + + /* Next is the compressed size and the uncompressed size. */ + if (!elf_lzma_varint (compressed, compressed_size, &offset, + &index_compressed_size)) + return 0; + if (!elf_lzma_varint (compressed, compressed_size, &offset, + &index_uncompressed_size)) + return 0; + + /* Pad to a four byte boundary. */ + offset = (offset + 3) &~ (size_t) 3; + + /* Next is a CRC of the index. */ + computed_crc = elf_crc32 (0, compressed + index_offset, + offset - index_offset); + stream_crc = (compressed[offset] + | (compressed[offset + 1] << 8) + | (compressed[offset + 2] << 16) + | (compressed[offset + 3] << 24)); + if (unlikely (computed_crc != stream_crc)) + { + elf_uncompress_failed (); + return 0; + } + offset += 4; + + /* We should now be back at the footer. */ + if (unlikely (offset != footer_offset)) + { + elf_uncompress_failed (); + return 0; + } + + /* Allocate space to hold the uncompressed data. If we succeed in + uncompressing the LZMA data, we never free this memory. */ + mem = (unsigned char *) backtrace_alloc (state, index_uncompressed_size, + error_callback, data); + if (unlikely (mem == NULL)) + return 0; + *uncompressed = mem; + *uncompressed_size = index_uncompressed_size; + + /* Allocate space for probabilities. */ + probs = ((uint16_t *) + backtrace_alloc (state, + LZMA_PROB_TOTAL_COUNT * sizeof (uint16_t), + error_callback, data)); + if (unlikely (probs == NULL)) + { + backtrace_free (state, mem, index_uncompressed_size, error_callback, + data); + return 0; + } + + /* Uncompress the block, which follows the header. */ + offset = 12; + if (!elf_uncompress_lzma_block (compressed, compressed_size, check, probs, + mem, index_uncompressed_size, &offset)) + { + backtrace_free (state, mem, index_uncompressed_size, error_callback, + data); + return 0; + } + + compressed_block_size = offset - 12; + if (unlikely (compressed_block_size + != ((index_compressed_size + 3) &~ (size_t) 3))) + { + elf_uncompress_failed (); + backtrace_free (state, mem, index_uncompressed_size, error_callback, + data); + return 0; + } + + offset = (offset + 3) &~ (size_t) 3; + if (unlikely (offset != index_offset)) + { + elf_uncompress_failed (); + backtrace_free (state, mem, index_uncompressed_size, error_callback, + data); + return 0; + } + + return 1; +} + +/* This function is a hook for testing the LZMA support. It is only + used by tests. */ + +int +backtrace_uncompress_lzma (struct backtrace_state *state, + const unsigned char *compressed, + size_t compressed_size, + backtrace_error_callback error_callback, + void *data, unsigned char **uncompressed, + size_t *uncompressed_size) +{ + return elf_uncompress_lzma (state, compressed, compressed_size, + error_callback, data, uncompressed, + uncompressed_size); +} + +/* Add the backtrace data for one ELF file. Returns 1 on success, + 0 on failure (in both cases descriptor is closed) or -1 if exe + is non-zero and the ELF file is ET_DYN, which tells the caller that + elf_add will need to be called on the descriptor again after + base_address is determined. */ + +static int +elf_add (struct backtrace_state *state, const char *filename, int descriptor, + const unsigned char *memory, size_t memory_size, + uintptr_t base_address, struct elf_ppc64_opd_data *caller_opd, + backtrace_error_callback error_callback, void *data, + fileline *fileline_fn, int *found_sym, int *found_dwarf, + struct dwarf_data **fileline_entry, int exe, int debuginfo, + const char *with_buildid_data, uint32_t with_buildid_size) +{ + struct elf_view ehdr_view; + b_elf_ehdr ehdr; + off_t shoff; + unsigned int shnum; + unsigned int shstrndx; + struct elf_view shdrs_view; + int shdrs_view_valid; + const b_elf_shdr *shdrs; + const b_elf_shdr *shstrhdr; + size_t shstr_size; + off_t shstr_off; + struct elf_view names_view; + int names_view_valid; + const char *names; + unsigned int symtab_shndx; + unsigned int dynsym_shndx; + unsigned int i; + struct debug_section_info sections[DEBUG_MAX]; + struct debug_section_info zsections[DEBUG_MAX]; + struct elf_view symtab_view; + int symtab_view_valid; + struct elf_view strtab_view; + int strtab_view_valid; + struct elf_view buildid_view; + int buildid_view_valid; + const char *buildid_data; + uint32_t buildid_size; + struct elf_view debuglink_view; + int debuglink_view_valid; + const char *debuglink_name; + uint32_t debuglink_crc; + struct elf_view debugaltlink_view; + int debugaltlink_view_valid; + const char *debugaltlink_name; + const char *debugaltlink_buildid_data; + uint32_t debugaltlink_buildid_size; + struct elf_view gnu_debugdata_view; + int gnu_debugdata_view_valid; + size_t gnu_debugdata_size; + unsigned char *gnu_debugdata_uncompressed; + size_t gnu_debugdata_uncompressed_size; + off_t min_offset; + off_t max_offset; + off_t debug_size; + struct elf_view debug_view; + int debug_view_valid; + unsigned int using_debug_view; + uint16_t *zdebug_table; + struct elf_view split_debug_view[DEBUG_MAX]; + unsigned char split_debug_view_valid[DEBUG_MAX]; + struct elf_ppc64_opd_data opd_data, *opd; + int opd_view_valid; + struct dwarf_sections dwarf_sections; + struct dwarf_data *fileline_altlink = NULL; + + if (!debuginfo) + { + *found_sym = 0; + *found_dwarf = 0; + } + + shdrs_view_valid = 0; + names_view_valid = 0; + symtab_view_valid = 0; + strtab_view_valid = 0; + buildid_view_valid = 0; + buildid_data = NULL; + buildid_size = 0; + debuglink_view_valid = 0; + debuglink_name = NULL; + debuglink_crc = 0; + debugaltlink_view_valid = 0; + debugaltlink_name = NULL; + debugaltlink_buildid_data = NULL; + debugaltlink_buildid_size = 0; + gnu_debugdata_view_valid = 0; + gnu_debugdata_size = 0; + debug_view_valid = 0; + memset (&split_debug_view_valid[0], 0, sizeof split_debug_view_valid); + opd = NULL; + opd_view_valid = 0; + + if (!elf_get_view (state, descriptor, memory, memory_size, 0, sizeof ehdr, + error_callback, data, &ehdr_view)) + goto fail; + + memcpy (&ehdr, ehdr_view.view.data, sizeof ehdr); + + elf_release_view (state, &ehdr_view, error_callback, data); + + if (ehdr.e_ident[EI_MAG0] != ELFMAG0 + || ehdr.e_ident[EI_MAG1] != ELFMAG1 + || ehdr.e_ident[EI_MAG2] != ELFMAG2 + || ehdr.e_ident[EI_MAG3] != ELFMAG3) + { + error_callback (data, "executable file is not ELF", 0); + goto fail; + } + if (ehdr.e_ident[EI_VERSION] != EV_CURRENT) + { + error_callback (data, "executable file is unrecognized ELF version", 0); + goto fail; + } + +#if BACKTRACE_ELF_SIZE == 32 +#define BACKTRACE_ELFCLASS ELFCLASS32 +#else +#define BACKTRACE_ELFCLASS ELFCLASS64 +#endif + + if (ehdr.e_ident[EI_CLASS] != BACKTRACE_ELFCLASS) + { + error_callback (data, "executable file is unexpected ELF class", 0); + goto fail; + } + + if (ehdr.e_ident[EI_DATA] != ELFDATA2LSB + && ehdr.e_ident[EI_DATA] != ELFDATA2MSB) + { + error_callback (data, "executable file has unknown endianness", 0); + goto fail; + } + + /* If the executable is ET_DYN, it is either a PIE, or we are running + directly a shared library with .interp. We need to wait for + dl_iterate_phdr in that case to determine the actual base_address. */ + if (exe && ehdr.e_type == ET_DYN) + return -1; + + shoff = ehdr.e_shoff; + shnum = ehdr.e_shnum; + shstrndx = ehdr.e_shstrndx; + + if ((shnum == 0 || shstrndx == SHN_XINDEX) + && shoff != 0) + { + struct elf_view shdr_view; + const b_elf_shdr *shdr; + + if (!elf_get_view (state, descriptor, memory, memory_size, shoff, + sizeof shdr, error_callback, data, &shdr_view)) + goto fail; + + shdr = (const b_elf_shdr *) shdr_view.view.data; + + if (shnum == 0) + shnum = shdr->sh_size; + + if (shstrndx == SHN_XINDEX) + { + shstrndx = shdr->sh_link; + + /* Versions of the GNU binutils between 2.12 and 2.18 did + not handle objects with more than SHN_LORESERVE sections + correctly. All large section indexes were offset by + 0x100. There is more information at + http://sourceware.org/bugzilla/show_bug.cgi?id-5900 . + Fortunately these object files are easy to detect, as the + GNU binutils always put the section header string table + near the end of the list of sections. Thus if the + section header string table index is larger than the + number of sections, then we know we have to subtract + 0x100 to get the real section index. */ + if (shstrndx >= shnum && shstrndx >= SHN_LORESERVE + 0x100) + shstrndx -= 0x100; + } + + elf_release_view (state, &shdr_view, error_callback, data); + } + + if (shnum == 0 || shstrndx == 0) + goto fail; + + /* To translate PC to file/line when using DWARF, we need to find + the .debug_info and .debug_line sections. */ + + /* Read the section headers, skipping the first one. */ + + if (!elf_get_view (state, descriptor, memory, memory_size, + shoff + sizeof (b_elf_shdr), + (shnum - 1) * sizeof (b_elf_shdr), + error_callback, data, &shdrs_view)) + goto fail; + shdrs_view_valid = 1; + shdrs = (const b_elf_shdr *) shdrs_view.view.data; + + /* Read the section names. */ + + shstrhdr = &shdrs[shstrndx - 1]; + shstr_size = shstrhdr->sh_size; + shstr_off = shstrhdr->sh_offset; + + if (!elf_get_view (state, descriptor, memory, memory_size, shstr_off, + shstrhdr->sh_size, error_callback, data, &names_view)) + goto fail; + names_view_valid = 1; + names = (const char *) names_view.view.data; + + symtab_shndx = 0; + dynsym_shndx = 0; + + memset (sections, 0, sizeof sections); + memset (zsections, 0, sizeof zsections); + + /* Look for the symbol table. */ + for (i = 1; i < shnum; ++i) + { + const b_elf_shdr *shdr; + unsigned int sh_name; + const char *name; + int j; + + shdr = &shdrs[i - 1]; + + if (shdr->sh_type == SHT_SYMTAB) + symtab_shndx = i; + else if (shdr->sh_type == SHT_DYNSYM) + dynsym_shndx = i; + + sh_name = shdr->sh_name; + if (sh_name >= shstr_size) + { + error_callback (data, "ELF section name out of range", 0); + goto fail; + } + + name = names + sh_name; + + for (j = 0; j < (int) DEBUG_MAX; ++j) + { + if (strcmp (name, dwarf_section_names[j]) == 0) + { + sections[j].offset = shdr->sh_offset; + sections[j].size = shdr->sh_size; + sections[j].compressed = (shdr->sh_flags & SHF_COMPRESSED) != 0; + break; + } + } + + if (name[0] == '.' && name[1] == 'z') + { + for (j = 0; j < (int) DEBUG_MAX; ++j) + { + if (strcmp (name + 2, dwarf_section_names[j] + 1) == 0) + { + zsections[j].offset = shdr->sh_offset; + zsections[j].size = shdr->sh_size; + break; + } + } + } + + /* Read the build ID if present. This could check for any + SHT_NOTE section with the right note name and type, but gdb + looks for a specific section name. */ + if ((!debuginfo || with_buildid_data != NULL) + && !buildid_view_valid + && strcmp (name, ".note.gnu.build-id") == 0) + { + const b_elf_note *note; + + if (!elf_get_view (state, descriptor, memory, memory_size, + shdr->sh_offset, shdr->sh_size, error_callback, + data, &buildid_view)) + goto fail; + + buildid_view_valid = 1; + note = (const b_elf_note *) buildid_view.view.data; + if (note->type == NT_GNU_BUILD_ID + && note->namesz == 4 + && strncmp (note->name, "GNU", 4) == 0 + && shdr->sh_size <= 12 + ((note->namesz + 3) & ~ 3) + note->descsz) + { + buildid_data = ¬e->name[0] + ((note->namesz + 3) & ~ 3); + buildid_size = note->descsz; + } + + if (with_buildid_size != 0) + { + if (buildid_size != with_buildid_size) + goto fail; + + if (memcmp (buildid_data, with_buildid_data, buildid_size) != 0) + goto fail; + } + } + + /* Read the debuglink file if present. */ + if (!debuginfo + && !debuglink_view_valid + && strcmp (name, ".gnu_debuglink") == 0) + { + const char *debuglink_data; + size_t crc_offset; + + if (!elf_get_view (state, descriptor, memory, memory_size, + shdr->sh_offset, shdr->sh_size, error_callback, + data, &debuglink_view)) + goto fail; + + debuglink_view_valid = 1; + debuglink_data = (const char *) debuglink_view.view.data; + crc_offset = strnlen (debuglink_data, shdr->sh_size); + crc_offset = (crc_offset + 3) & ~3; + if (crc_offset + 4 <= shdr->sh_size) + { + debuglink_name = debuglink_data; + debuglink_crc = *(const uint32_t*)(debuglink_data + crc_offset); + } + } + + if (!debugaltlink_view_valid + && strcmp (name, ".gnu_debugaltlink") == 0) + { + const char *debugaltlink_data; + size_t debugaltlink_name_len; + + if (!elf_get_view (state, descriptor, memory, memory_size, + shdr->sh_offset, shdr->sh_size, error_callback, + data, &debugaltlink_view)) + goto fail; + + debugaltlink_view_valid = 1; + debugaltlink_data = (const char *) debugaltlink_view.view.data; + debugaltlink_name = debugaltlink_data; + debugaltlink_name_len = strnlen (debugaltlink_data, shdr->sh_size); + if (debugaltlink_name_len < shdr->sh_size) + { + /* Include terminating zero. */ + debugaltlink_name_len += 1; + + debugaltlink_buildid_data + = debugaltlink_data + debugaltlink_name_len; + debugaltlink_buildid_size = shdr->sh_size - debugaltlink_name_len; + } + } + + if (!gnu_debugdata_view_valid + && strcmp (name, ".gnu_debugdata") == 0) + { + if (!elf_get_view (state, descriptor, memory, memory_size, + shdr->sh_offset, shdr->sh_size, error_callback, + data, &gnu_debugdata_view)) + goto fail; + + gnu_debugdata_size = shdr->sh_size; + gnu_debugdata_view_valid = 1; + } + + /* Read the .opd section on PowerPC64 ELFv1. */ + if (ehdr.e_machine == EM_PPC64 + && (ehdr.e_flags & EF_PPC64_ABI) < 2 + && shdr->sh_type == SHT_PROGBITS + && strcmp (name, ".opd") == 0) + { + if (!elf_get_view (state, descriptor, memory, memory_size, + shdr->sh_offset, shdr->sh_size, error_callback, + data, &opd_data.view)) + goto fail; + + opd = &opd_data; + opd->addr = shdr->sh_addr; + opd->data = (const char *) opd_data.view.view.data; + opd->size = shdr->sh_size; + opd_view_valid = 1; + } + } + + /* A debuginfo file may not have a useful .opd section, but we can use the + one from the original executable. */ + if (opd == NULL) + opd = caller_opd; + + if (symtab_shndx == 0) + symtab_shndx = dynsym_shndx; + if (symtab_shndx != 0) + { + const b_elf_shdr *symtab_shdr; + unsigned int strtab_shndx; + const b_elf_shdr *strtab_shdr; + struct elf_syminfo_data *sdata; + + symtab_shdr = &shdrs[symtab_shndx - 1]; + strtab_shndx = symtab_shdr->sh_link; + if (strtab_shndx >= shnum) + { + error_callback (data, + "ELF symbol table strtab link out of range", 0); + goto fail; + } + strtab_shdr = &shdrs[strtab_shndx - 1]; + + if (!elf_get_view (state, descriptor, memory, memory_size, + symtab_shdr->sh_offset, symtab_shdr->sh_size, + error_callback, data, &symtab_view)) + goto fail; + symtab_view_valid = 1; + + if (!elf_get_view (state, descriptor, memory, memory_size, + strtab_shdr->sh_offset, strtab_shdr->sh_size, + error_callback, data, &strtab_view)) + goto fail; + strtab_view_valid = 1; + + sdata = ((struct elf_syminfo_data *) + backtrace_alloc (state, sizeof *sdata, error_callback, data)); + if (sdata == NULL) + goto fail; + + if (!elf_initialize_syminfo (state, base_address, + (const unsigned char*)symtab_view.view.data, symtab_shdr->sh_size, + (const unsigned char*)strtab_view.view.data, strtab_shdr->sh_size, + error_callback, data, sdata, opd)) + { + backtrace_free (state, sdata, sizeof *sdata, error_callback, data); + goto fail; + } + + /* We no longer need the symbol table, but we hold on to the + string table permanently. */ + elf_release_view (state, &symtab_view, error_callback, data); + symtab_view_valid = 0; + strtab_view_valid = 0; + + *found_sym = 1; + + elf_add_syminfo_data (state, sdata); + } + + elf_release_view (state, &shdrs_view, error_callback, data); + shdrs_view_valid = 0; + elf_release_view (state, &names_view, error_callback, data); + names_view_valid = 0; + + /* If the debug info is in a separate file, read that one instead. */ + + if (buildid_data != NULL) + { + int d; + + d = elf_open_debugfile_by_buildid (state, buildid_data, buildid_size, + filename, error_callback, data); + if (d >= 0) + { + int ret; + + elf_release_view (state, &buildid_view, error_callback, data); + if (debuglink_view_valid) + elf_release_view (state, &debuglink_view, error_callback, data); + if (debugaltlink_view_valid) + elf_release_view (state, &debugaltlink_view, error_callback, data); + ret = elf_add (state, "", d, NULL, 0, base_address, opd, + error_callback, data, fileline_fn, found_sym, + found_dwarf, NULL, 0, 1, NULL, 0); + if (ret < 0) + backtrace_close (d, error_callback, data); + else if (descriptor >= 0) + backtrace_close (descriptor, error_callback, data); + return ret; + } + } + + if (buildid_view_valid) + { + elf_release_view (state, &buildid_view, error_callback, data); + buildid_view_valid = 0; + } + + if (debuglink_name != NULL) + { + int d; + + d = elf_open_debugfile_by_debuglink (state, filename, debuglink_name, + debuglink_crc, error_callback, + data); + if (d >= 0) + { + int ret; + + elf_release_view (state, &debuglink_view, error_callback, data); + if (debugaltlink_view_valid) + elf_release_view (state, &debugaltlink_view, error_callback, data); + ret = elf_add (state, "", d, NULL, 0, base_address, opd, + error_callback, data, fileline_fn, found_sym, + found_dwarf, NULL, 0, 1, NULL, 0); + if (ret < 0) + backtrace_close (d, error_callback, data); + else if (descriptor >= 0) + backtrace_close(descriptor, error_callback, data); + return ret; + } + } + + if (debuglink_view_valid) + { + elf_release_view (state, &debuglink_view, error_callback, data); + debuglink_view_valid = 0; + } + + if (debugaltlink_name != NULL) + { + int d; + + d = elf_open_debugfile_by_debuglink (state, filename, debugaltlink_name, + 0, error_callback, data); + if (d >= 0) + { + int ret; + + ret = elf_add (state, filename, d, NULL, 0, base_address, opd, + error_callback, data, fileline_fn, found_sym, + found_dwarf, &fileline_altlink, 0, 1, + debugaltlink_buildid_data, debugaltlink_buildid_size); + elf_release_view (state, &debugaltlink_view, error_callback, data); + debugaltlink_view_valid = 0; + if (ret < 0) + { + backtrace_close (d, error_callback, data); + return ret; + } + } + } + + if (debugaltlink_view_valid) + { + elf_release_view (state, &debugaltlink_view, error_callback, data); + debugaltlink_view_valid = 0; + } + + if (gnu_debugdata_view_valid) + { + int ret; + + ret = elf_uncompress_lzma (state, + ((const unsigned char *) + gnu_debugdata_view.view.data), + gnu_debugdata_size, error_callback, data, + &gnu_debugdata_uncompressed, + &gnu_debugdata_uncompressed_size); + + elf_release_view (state, &gnu_debugdata_view, error_callback, data); + gnu_debugdata_view_valid = 0; + + if (ret) + { + ret = elf_add (state, filename, -1, gnu_debugdata_uncompressed, + gnu_debugdata_uncompressed_size, base_address, opd, + error_callback, data, fileline_fn, found_sym, + found_dwarf, NULL, 0, 0, NULL, 0); + if (ret >= 0 && descriptor >= 0) + backtrace_close(descriptor, error_callback, data); + return ret; + } + } + + if (opd_view_valid) + { + elf_release_view (state, &opd->view, error_callback, data); + opd_view_valid = 0; + opd = NULL; + } + + /* Read all the debug sections in a single view, since they are + probably adjacent in the file. If any of sections are + uncompressed, we never release this view. */ + + min_offset = 0; + max_offset = 0; + debug_size = 0; + for (i = 0; i < (int) DEBUG_MAX; ++i) + { + off_t end; + + if (sections[i].size != 0) + { + if (min_offset == 0 || sections[i].offset < min_offset) + min_offset = sections[i].offset; + end = sections[i].offset + sections[i].size; + if (end > max_offset) + max_offset = end; + debug_size += sections[i].size; + } + if (zsections[i].size != 0) + { + if (min_offset == 0 || zsections[i].offset < min_offset) + min_offset = zsections[i].offset; + end = zsections[i].offset + zsections[i].size; + if (end > max_offset) + max_offset = end; + debug_size += zsections[i].size; + } + } + if (min_offset == 0 || max_offset == 0) + { + if (descriptor >= 0) + { + if (!backtrace_close (descriptor, error_callback, data)) + goto fail; + } + return 1; + } + + /* If the total debug section size is large, assume that there are + gaps between the sections, and read them individually. */ + + if (max_offset - min_offset < 0x20000000 + || max_offset - min_offset < debug_size + 0x10000) + { + if (!elf_get_view (state, descriptor, memory, memory_size, min_offset, + max_offset - min_offset, error_callback, data, + &debug_view)) + goto fail; + debug_view_valid = 1; + } + else + { + memset (&split_debug_view[0], 0, sizeof split_debug_view); + for (i = 0; i < (int) DEBUG_MAX; ++i) + { + struct debug_section_info *dsec; + + if (sections[i].size != 0) + dsec = §ions[i]; + else if (zsections[i].size != 0) + dsec = &zsections[i]; + else + continue; + + if (!elf_get_view (state, descriptor, memory, memory_size, + dsec->offset, dsec->size, error_callback, data, + &split_debug_view[i])) + goto fail; + split_debug_view_valid[i] = 1; + + if (sections[i].size != 0) + sections[i].data = ((const unsigned char *) + split_debug_view[i].view.data); + else + zsections[i].data = ((const unsigned char *) + split_debug_view[i].view.data); + } + } + + /* We've read all we need from the executable. */ + if (descriptor >= 0) + { + if (!backtrace_close (descriptor, error_callback, data)) + goto fail; + descriptor = -1; + } + + using_debug_view = 0; + if (debug_view_valid) + { + for (i = 0; i < (int) DEBUG_MAX; ++i) + { + if (sections[i].size == 0) + sections[i].data = NULL; + else + { + sections[i].data = ((const unsigned char *) debug_view.view.data + + (sections[i].offset - min_offset)); + ++using_debug_view; + } + + if (zsections[i].size == 0) + zsections[i].data = NULL; + else + zsections[i].data = ((const unsigned char *) debug_view.view.data + + (zsections[i].offset - min_offset)); + } + } + + /* Uncompress the old format (--compress-debug-sections=zlib-gnu). */ + + zdebug_table = NULL; + for (i = 0; i < (int) DEBUG_MAX; ++i) + { + if (sections[i].size == 0 && zsections[i].size > 0) + { + unsigned char *uncompressed_data; + size_t uncompressed_size; + + if (zdebug_table == NULL) + { + zdebug_table = ((uint16_t *) + backtrace_alloc (state, ZLIB_TABLE_SIZE, + error_callback, data)); + if (zdebug_table == NULL) + goto fail; + } + + uncompressed_data = NULL; + uncompressed_size = 0; + if (!elf_uncompress_zdebug (state, zsections[i].data, + zsections[i].size, zdebug_table, + error_callback, data, + &uncompressed_data, &uncompressed_size)) + goto fail; + sections[i].data = uncompressed_data; + sections[i].size = uncompressed_size; + sections[i].compressed = 0; + + if (split_debug_view_valid[i]) + { + elf_release_view (state, &split_debug_view[i], + error_callback, data); + split_debug_view_valid[i] = 0; + } + } + } + + if (zdebug_table != NULL) + { + backtrace_free (state, zdebug_table, ZLIB_TABLE_SIZE, + error_callback, data); + zdebug_table = NULL; + } + + /* Uncompress the official ELF format + (--compress-debug-sections=zlib-gabi, --compress-debug-sections=zstd). */ + for (i = 0; i < (int) DEBUG_MAX; ++i) + { + unsigned char *uncompressed_data; + size_t uncompressed_size; + + if (sections[i].size == 0 || !sections[i].compressed) + continue; + + if (zdebug_table == NULL) + { + zdebug_table = ((uint16_t *) + backtrace_alloc (state, ZDEBUG_TABLE_SIZE, + error_callback, data)); + if (zdebug_table == NULL) + goto fail; + } + + uncompressed_data = NULL; + uncompressed_size = 0; + if (!elf_uncompress_chdr (state, sections[i].data, sections[i].size, + zdebug_table, error_callback, data, + &uncompressed_data, &uncompressed_size)) + goto fail; + sections[i].data = uncompressed_data; + sections[i].size = uncompressed_size; + sections[i].compressed = 0; + + if (debug_view_valid) + --using_debug_view; + else if (split_debug_view_valid[i]) + { + elf_release_view (state, &split_debug_view[i], error_callback, data); + split_debug_view_valid[i] = 0; + } + } + + if (zdebug_table != NULL) + backtrace_free (state, zdebug_table, ZDEBUG_TABLE_SIZE, + error_callback, data); + + if (debug_view_valid && using_debug_view == 0) + { + elf_release_view (state, &debug_view, error_callback, data); + debug_view_valid = 0; + } + + for (i = 0; i < (int) DEBUG_MAX; ++i) + { + dwarf_sections.data[i] = sections[i].data; + dwarf_sections.size[i] = sections[i].size; + } + + if (!backtrace_dwarf_add (state, base_address, &dwarf_sections, + ehdr.e_ident[EI_DATA] == ELFDATA2MSB, + fileline_altlink, + error_callback, data, fileline_fn, + fileline_entry)) + goto fail; + + *found_dwarf = 1; + + return 1; + + fail: + if (shdrs_view_valid) + elf_release_view (state, &shdrs_view, error_callback, data); + if (names_view_valid) + elf_release_view (state, &names_view, error_callback, data); + if (symtab_view_valid) + elf_release_view (state, &symtab_view, error_callback, data); + if (strtab_view_valid) + elf_release_view (state, &strtab_view, error_callback, data); + if (debuglink_view_valid) + elf_release_view (state, &debuglink_view, error_callback, data); + if (debugaltlink_view_valid) + elf_release_view (state, &debugaltlink_view, error_callback, data); + if (gnu_debugdata_view_valid) + elf_release_view (state, &gnu_debugdata_view, error_callback, data); + if (buildid_view_valid) + elf_release_view (state, &buildid_view, error_callback, data); + if (debug_view_valid) + elf_release_view (state, &debug_view, error_callback, data); + for (i = 0; i < (int) DEBUG_MAX; ++i) + { + if (split_debug_view_valid[i]) + elf_release_view (state, &split_debug_view[i], error_callback, data); + } + if (opd_view_valid) + elf_release_view (state, &opd->view, error_callback, data); + if (descriptor >= 0) + backtrace_close (descriptor, error_callback, data); + return 0; +} + +/* Data passed to phdr_callback. */ + +struct phdr_data +{ + struct backtrace_state *state; + backtrace_error_callback error_callback; + void *data; + fileline *fileline_fn; + int *found_sym; + int *found_dwarf; + const char *exe_filename; + int exe_descriptor; +}; + +/* Callback passed to dl_iterate_phdr. Load debug info from shared + libraries. */ + +struct PhdrIterate +{ + char* dlpi_name; + ElfW(Addr) dlpi_addr; + ElfW(Addr) dlpi_end_addr; +}; +FastVector s_phdrData(16); + +struct ElfAddrRange +{ + ElfW(Addr) dlpi_addr; + ElfW(Addr) dlpi_end_addr; +}; +FastVector s_sortedKnownElfRanges(16); + +static int address_in_known_elf_ranges(uintptr_t pc) +{ + auto it = std::lower_bound( s_sortedKnownElfRanges.begin(), s_sortedKnownElfRanges.end(), pc, + []( const ElfAddrRange& lhs, const uintptr_t rhs ) { return uintptr_t(lhs.dlpi_addr) > rhs; } ); + if( it != s_sortedKnownElfRanges.end() && pc <= it->dlpi_end_addr ) + { + return true; + } + return false; +} + +static int +phdr_callback_mock (struct dl_phdr_info *info, size_t size ATTRIBUTE_UNUSED, + void *pdata) +{ + if( address_in_known_elf_ranges(info->dlpi_addr) ) + { + return 0; + } + + auto ptr = s_phdrData.push_next(); + if (info->dlpi_name) + { + size_t sz = strlen (info->dlpi_name) + 1; + ptr->dlpi_name = (char*)tracy_malloc (sz); + memcpy (ptr->dlpi_name, info->dlpi_name, sz); + } + else ptr->dlpi_name = nullptr; + ptr->dlpi_addr = info->dlpi_addr; + + // calculate the end address as well, so we can quickly determine if a PC is within the range of this image + ptr->dlpi_end_addr = uintptr_t(info->dlpi_addr) + (info->dlpi_phnum ? uintptr_t( + info->dlpi_phdr[info->dlpi_phnum - 1].p_vaddr + + info->dlpi_phdr[info->dlpi_phnum - 1].p_memsz) : 0); + + return 0; +} + +static int +#ifdef __i386__ +__attribute__ ((__force_align_arg_pointer__)) +#endif +phdr_callback (struct PhdrIterate *info, void *pdata) +{ + struct phdr_data *pd = (struct phdr_data *) pdata; + const char *filename; + int descriptor; + int does_not_exist; + fileline elf_fileline_fn; + int found_dwarf; + + /* There is not much we can do if we don't have the module name, + unless executable is ET_DYN, where we expect the very first + phdr_callback to be for the PIE. */ + if (info->dlpi_name == NULL || info->dlpi_name[0] == '\0') + { + if (pd->exe_descriptor == -1) + return 0; + filename = pd->exe_filename; + descriptor = pd->exe_descriptor; + pd->exe_descriptor = -1; + } + else + { + if (pd->exe_descriptor != -1) + { + backtrace_close (pd->exe_descriptor, pd->error_callback, pd->data); + pd->exe_descriptor = -1; + } + + filename = info->dlpi_name; + descriptor = backtrace_open (info->dlpi_name, pd->error_callback, + pd->data, &does_not_exist); + if (descriptor < 0) + return 0; + } + + if (elf_add (pd->state, filename, descriptor, NULL, 0, info->dlpi_addr, NULL, + pd->error_callback, pd->data, &elf_fileline_fn, pd->found_sym, + &found_dwarf, NULL, 0, 0, NULL, 0)) + { + if (found_dwarf) + { + *pd->found_dwarf = 1; + *pd->fileline_fn = elf_fileline_fn; + } + } + + return 0; +} + +static int elf_iterate_phdr_and_add_new_files(phdr_data *pd) +{ + assert(s_phdrData.empty()); + // dl_iterate_phdr, will only add entries for elf files loaded in a previously unseen range + dl_iterate_phdr(phdr_callback_mock, nullptr); + + if(s_phdrData.size() == 0) + { + return 0; + } + + uint32_t headersAdded = 0; + for (auto &v : s_phdrData) + { + phdr_callback(&v, (void *)pd); + + auto newEntry = s_sortedKnownElfRanges.push_next(); + newEntry->dlpi_addr = v.dlpi_addr; + newEntry->dlpi_end_addr = v.dlpi_end_addr; + + tracy_free(v.dlpi_name); + + headersAdded++; + } + + s_phdrData.clear(); + + std::sort( s_sortedKnownElfRanges.begin(), s_sortedKnownElfRanges.end(), + []( const ElfAddrRange& lhs, const ElfAddrRange& rhs ) { return lhs.dlpi_addr > rhs.dlpi_addr; } ); + + return headersAdded; +} + +#ifdef TRACY_LIBBACKTRACE_ELF_DYNLOAD_SUPPORT +/* Request an elf entry update if the pc passed in is not in any of the known elf ranges. +This could mean that new images were dlopened and we need to add those new elf entries */ +static int elf_refresh_address_ranges_if_needed(struct backtrace_state *state, uintptr_t pc) +{ + if ( address_in_known_elf_ranges(pc) ) + { + return 0; + } + + struct phdr_data pd; + int found_sym = 0; + int found_dwarf = 0; + fileline fileline_fn = nullptr; + pd.state = state; + pd.error_callback = nullptr; + pd.data = nullptr; + pd.fileline_fn = &fileline_fn; + pd.found_sym = &found_sym; + pd.found_dwarf = &found_dwarf; + pd.exe_filename = nullptr; + pd.exe_descriptor = -1; + + return elf_iterate_phdr_and_add_new_files(&pd); +} +#endif //#ifdef TRACY_LIBBACKTRACE_ELF_DYNLOAD_SUPPORT + +/* Initialize the backtrace data we need from an ELF executable. At + the ELF level, all we need to do is find the debug info + sections. */ + +int +backtrace_initialize (struct backtrace_state *state, const char *filename, + int descriptor, backtrace_error_callback error_callback, + void *data, fileline *fileline_fn) +{ + int ret; + int found_sym; + int found_dwarf; + fileline elf_fileline_fn = elf_nodebug; + struct phdr_data pd; + + ret = elf_add (state, filename, descriptor, NULL, 0, 0, NULL, error_callback, + data, &elf_fileline_fn, &found_sym, &found_dwarf, NULL, 1, 0, + NULL, 0); + if (!ret) + return 0; + + pd.state = state; + pd.error_callback = error_callback; + pd.data = data; + pd.fileline_fn = &elf_fileline_fn; + pd.found_sym = &found_sym; + pd.found_dwarf = &found_dwarf; + pd.exe_filename = filename; + pd.exe_descriptor = ret < 0 ? descriptor : -1; + + elf_iterate_phdr_and_add_new_files(&pd); + + if (!state->threaded) + { + if (found_sym) + state->syminfo_fn = elf_syminfo; + else if (state->syminfo_fn == NULL) + state->syminfo_fn = elf_nosyms; + } + else + { + if (found_sym) + backtrace_atomic_store_pointer (&state->syminfo_fn, &elf_syminfo); + else + (void) __sync_bool_compare_and_swap (&state->syminfo_fn, NULL, + elf_nosyms); + } + + if (!state->threaded) + *fileline_fn = state->fileline_fn; + else + *fileline_fn = backtrace_atomic_load_pointer (&state->fileline_fn); + + if (*fileline_fn == NULL || *fileline_fn == elf_nodebug) + *fileline_fn = elf_fileline_fn; + + // install an address range refresh callback so we can cope with dynamically loaded elf files +#ifdef TRACY_LIBBACKTRACE_ELF_DYNLOAD_SUPPORT + state->request_known_address_ranges_refresh_fn = elf_refresh_address_ranges_if_needed; +#else + state->request_known_address_ranges_refresh_fn = NULL; +#endif + + return 1; +} + +} diff --git a/third_party/tracy/public/libbacktrace/mmapio.cpp b/third_party/tracy/public/libbacktrace/mmapio.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0e8f599bb82abec8bcfe1adfbac11b3ab974d635 --- /dev/null +++ b/third_party/tracy/public/libbacktrace/mmapio.cpp @@ -0,0 +1,115 @@ +/* mmapio.c -- File views using mmap. + Copyright (C) 2012-2021 Free Software Foundation, Inc. + Written by Ian Lance Taylor, Google. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + (1) Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + (2) Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + (3) The name of the author may not be used to + endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. */ + +#include "config.h" + +#include +#include +#include +#include + +#include "backtrace.hpp" +#include "internal.hpp" + +#ifndef HAVE_DECL_GETPAGESIZE +extern int getpagesize (void); +#endif + +#ifndef MAP_FAILED +#define MAP_FAILED ((void *)-1) +#endif + +namespace tracy +{ + +/* This file implements file views and memory allocation when mmap is + available. */ + +/* Create a view of SIZE bytes from DESCRIPTOR at OFFSET. */ + +int +backtrace_get_view (struct backtrace_state *state ATTRIBUTE_UNUSED, + int descriptor, off_t offset, uint64_t size, + backtrace_error_callback error_callback, + void *data, struct backtrace_view *view) +{ + size_t pagesize; + unsigned int inpage; + off_t pageoff; + void *map; + + if ((uint64_t) (size_t) size != size) + { + error_callback (data, "file size too large", 0); + return 0; + } + + pagesize = getpagesize (); + inpage = offset % pagesize; + pageoff = offset - inpage; + + size += inpage; + size = (size + (pagesize - 1)) & ~ (pagesize - 1); + + map = mmap (NULL, size, PROT_READ, MAP_PRIVATE, descriptor, pageoff); + if (map == MAP_FAILED) + { + error_callback (data, "mmap", errno); + return 0; + } + + view->data = (char *) map + inpage; + view->base = map; + view->len = size; + + return 1; +} + +/* Release a view read by backtrace_get_view. */ + +void +backtrace_release_view (struct backtrace_state *state ATTRIBUTE_UNUSED, + struct backtrace_view *view, + backtrace_error_callback error_callback, + void *data) +{ + union { + const void *cv; + void *v; + } cc; + + cc.cv = view->base; + if (munmap (cc.v, view->len) < 0) + error_callback (data, "munmap", errno); +} + +} diff --git a/third_party/tracy/public/libbacktrace/sort.cpp b/third_party/tracy/public/libbacktrace/sort.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6daee0a64fd2300c01494e3429cb77a462c8ebbd --- /dev/null +++ b/third_party/tracy/public/libbacktrace/sort.cpp @@ -0,0 +1,113 @@ +/* sort.c -- Sort without allocating memory + Copyright (C) 2012-2021 Free Software Foundation, Inc. + Written by Ian Lance Taylor, Google. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + (1) Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + (2) Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + (3) The name of the author may not be used to + endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. */ + +#include "config.h" + +#include +#include + +#include "backtrace.hpp" +#include "internal.hpp" + +namespace tracy +{ + +/* The GNU glibc version of qsort allocates memory, which we must not + do if we are invoked by a signal handler. So provide our own + sort. */ + +static void +swap (char *a, char *b, size_t size) +{ + size_t i; + + for (i = 0; i < size; i++, a++, b++) + { + char t; + + t = *a; + *a = *b; + *b = t; + } +} + +void +backtrace_qsort (void *basearg, size_t count, size_t size, + int (*compar) (const void *, const void *)) +{ + char *base = (char *) basearg; + size_t i; + size_t mid; + + tail_recurse: + if (count < 2) + return; + + /* The symbol table and DWARF tables, which is all we use this + routine for, tend to be roughly sorted. Pick the middle element + in the array as our pivot point, so that we are more likely to + cut the array in half for each recursion step. */ + swap (base, base + (count / 2) * size, size); + + mid = 0; + for (i = 1; i < count; i++) + { + if ((*compar) (base, base + i * size) > 0) + { + ++mid; + if (i != mid) + swap (base + mid * size, base + i * size, size); + } + } + + if (mid > 0) + swap (base, base + mid * size, size); + + /* Recurse with the smaller array, loop with the larger one. That + ensures that our maximum stack depth is log count. */ + if (2 * mid < count) + { + backtrace_qsort (base, mid, size, compar); + base += (mid + 1) * size; + count -= mid + 1; + goto tail_recurse; + } + else + { + backtrace_qsort (base + (mid + 1) * size, count - (mid + 1), + size, compar); + count = mid; + goto tail_recurse; + } +} + +} diff --git a/third_party/tracy/public/libbacktrace/state.cpp b/third_party/tracy/public/libbacktrace/state.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ea3c137c5d57189dc81dbd3f0e9d233a08885cdc --- /dev/null +++ b/third_party/tracy/public/libbacktrace/state.cpp @@ -0,0 +1,76 @@ +/* state.c -- Create the backtrace state. + Copyright (C) 2012-2021 Free Software Foundation, Inc. + Written by Ian Lance Taylor, Google. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + (1) Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + (2) Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + (3) The name of the author may not be used to + endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. */ + +#include "config.h" + +#include +#include + +#include "backtrace.hpp" +#include "internal.hpp" + +namespace tracy +{ + +/* Create the backtrace state. This will then be passed to all the + other routines. */ + +struct backtrace_state * +backtrace_create_state (const char *filename, int threaded, + backtrace_error_callback error_callback, + void *data) +{ + struct backtrace_state init_state; + struct backtrace_state *state; + +#ifndef HAVE_SYNC_FUNCTIONS + if (threaded) + { + error_callback (data, "backtrace library does not support threads", 0); + return NULL; + } +#endif + + memset (&init_state, 0, sizeof init_state); + init_state.filename = filename; + init_state.threaded = threaded; + + state = ((struct backtrace_state *) + backtrace_alloc (&init_state, sizeof *state, error_callback, data)); + if (state == NULL) + return NULL; + *state = init_state; + + return state; +} + +} diff --git a/third_party/tracy/server/TracyCharUtil.hpp b/third_party/tracy/server/TracyCharUtil.hpp new file mode 100644 index 0000000000000000000000000000000000000000..f96f9ec56ac3feb7eeb1f9080c7de56cdf1c0206 --- /dev/null +++ b/third_party/tracy/server/TracyCharUtil.hpp @@ -0,0 +1,76 @@ +#ifndef __TRACY__CHARUTIL_HPP__ +#define __TRACY__CHARUTIL_HPP__ + +#include +#include +#include + +#define XXH_INLINE_ALL +#include "tracy_xxhash.h" + +namespace tracy +{ +namespace charutil +{ + +static inline size_t hash( const char* str ) +{ + const auto sz = strlen( str ); + return XXH3_64bits( str, sz ); +} + +static inline size_t hash( const char* str, size_t sz ) +{ + return XXH3_64bits( str, sz ); +} + +struct Hasher +{ + size_t operator()( const char* key ) const + { + return hash( key ); + } +}; + +struct Comparator +{ + bool operator()( const char* lhs, const char* rhs ) const + { + return strcmp( lhs, rhs ) == 0; + } +}; + +struct LessComparator +{ + bool operator()( const char* lhs, const char* rhs ) const + { + return strcmp( lhs, rhs ) < 0; + } +}; + +struct StringKey +{ + const char* ptr; + size_t sz; + + struct Hasher + { + size_t operator()( const StringKey& key ) const + { + return hash( key.ptr, key.sz ); + } + }; + + struct Comparator + { + bool operator()( const StringKey& lhs, const StringKey& rhs ) const + { + return lhs.sz == rhs.sz && memcmp( lhs.ptr, rhs.ptr, lhs.sz ) == 0; + } + }; +}; + +} +} + +#endif diff --git a/third_party/tracy/server/TracyEvent.hpp b/third_party/tracy/server/TracyEvent.hpp new file mode 100644 index 0000000000000000000000000000000000000000..8557a78553c6d7c0e72d2e7fd655d1e159ab57c5 --- /dev/null +++ b/third_party/tracy/server/TracyEvent.hpp @@ -0,0 +1,850 @@ +#ifndef __TRACYEVENT_HPP__ +#define __TRACYEVENT_HPP__ + +#include +#include +#include +#include +#include + +#include "TracyCharUtil.hpp" +#include "TracyShortPtr.hpp" +#include "TracySortedVector.hpp" +#include "TracyVector.hpp" +#include "tracy_robin_hood.h" +#include "../public/common/TracyForceInline.hpp" +#include "../public/common/TracyQueue.hpp" + +namespace tracy +{ + +#pragma pack( push, 1 ) + +struct StringRef +{ + enum Type { Ptr, Idx }; + + tracy_force_inline StringRef() : str( 0 ), __data( 0 ) {} + tracy_force_inline StringRef( Type t, uint64_t data ) + : str( data ) + , __data( 0 ) + { + isidx = t == Idx; + active = 1; + } + + uint64_t str; + + union + { + struct + { + uint8_t isidx : 1; + uint8_t active : 1; + }; + uint8_t __data; + }; +}; + +struct StringRefHasher +{ + size_t operator()( const StringRef& key ) const + { + return charutil::hash( (const char*)&key, sizeof( StringRef ) ); + } +}; + +struct StringRefComparator +{ + bool operator()( const StringRef& lhs, const StringRef& rhs ) const + { + return memcmp( &lhs, &rhs, sizeof( StringRef ) ) == 0; + } +}; + +class StringIdx +{ +public: + tracy_force_inline StringIdx() { memset( m_idx, 0, sizeof( m_idx ) ); } + tracy_force_inline StringIdx( uint32_t idx ) + { + SetIdx( idx ); + } + + tracy_force_inline void SetIdx( uint32_t idx ) + { + idx++; + memcpy( m_idx, &idx, 3 ); + } + + tracy_force_inline uint32_t Idx() const + { + uint32_t idx = 0; + memcpy( &idx, m_idx, 3 ); + assert( idx != 0 ); + return idx - 1; + } + + tracy_force_inline bool Active() const + { + uint32_t zero = 0; + return memcmp( m_idx, &zero, 3 ) != 0; + } + +private: + uint8_t m_idx[3]; +}; + +struct StringIdxHasher +{ + size_t operator()( const StringIdx& key ) const + { + return charutil::hash( (const char*)&key, sizeof( StringIdx ) ); + } +}; + +struct StringIdxComparator +{ + bool operator()( const StringIdx& lhs, const StringIdx& rhs ) const + { + return memcmp( &lhs, &rhs, sizeof( StringIdx ) ) == 0; + } +}; + +class Int24 +{ +public: + tracy_force_inline Int24() { memset( m_val, 0, sizeof( m_val ) ); } + tracy_force_inline Int24( uint32_t val ) + { + SetVal( val ); + } + + tracy_force_inline void SetVal( uint32_t val ) + { + memcpy( m_val, &val, 2 ); + val >>= 16; + memcpy( m_val+2, &val, 1 ); + } + + tracy_force_inline uint32_t Val() const + { + uint8_t hi; + memcpy( &hi, m_val+2, 1 ); + uint16_t lo; + memcpy( &lo, m_val, 2 ); + return ( uint32_t( hi ) << 16 ) | lo; + } + +private: + uint8_t m_val[3]; +}; + +class Int48 +{ +public: + tracy_force_inline Int48() {} + tracy_force_inline Int48( int64_t val ) + { + SetVal( val ); + } + + tracy_force_inline void Clear() + { + memset( m_val, 0, 6 ); + } + + tracy_force_inline void SetVal( int64_t val ) + { + memcpy( m_val, &val, 4 ); + val >>= 32; + memcpy( m_val+4, &val, 2 ); + } + + tracy_force_inline int64_t Val() const + { + int16_t hi; + memcpy( &hi, m_val+4, 2 ); + uint32_t lo; + memcpy( &lo, m_val, 4 ); + return ( int64_t( uint64_t( hi ) << 32 ) ) | lo; + } + + tracy_force_inline bool IsNonNegative() const + { + return ( m_val[5] >> 7 ) == 0; + } + +private: + uint8_t m_val[6]; +}; + +struct Int48Sort { bool operator()( const Int48& lhs, const Int48& rhs ) { return lhs.Val() < rhs.Val(); }; }; + + +struct SourceLocationBase +{ + StringRef name; + StringRef function; + StringRef file; + uint32_t line; + uint32_t color; +}; + +struct SourceLocation : public SourceLocationBase +{ + mutable uint32_t namehash; +}; + +enum { SourceLocationSize = sizeof( SourceLocation ) }; + + +struct ZoneEvent +{ + tracy_force_inline ZoneEvent() {}; + + tracy_force_inline int64_t Start() const { return int64_t( _start_srcloc ) >> 16; } + tracy_force_inline void SetStart( int64_t start ) { assert( start < (int64_t)( 1ull << 47 ) ); memcpy( ((char*)&_start_srcloc)+2, &start, 4 ); memcpy( ((char*)&_start_srcloc)+6, ((char*)&start)+4, 2 ); } + tracy_force_inline int64_t End() const { return int64_t( _end_child1 ) >> 16; } + tracy_force_inline void SetEnd( int64_t end ) { assert( end < (int64_t)( 1ull << 47 ) ); memcpy( ((char*)&_end_child1)+2, &end, 4 ); memcpy( ((char*)&_end_child1)+6, ((char*)&end)+4, 2 ); } + tracy_force_inline bool IsEndValid() const { return ( _end_child1 >> 63 ) == 0; } + tracy_force_inline int16_t SrcLoc() const { return int16_t( _start_srcloc & 0xFFFF ); } + tracy_force_inline void SetSrcLoc( int16_t srcloc ) { memcpy( &_start_srcloc, &srcloc, 2 ); } + tracy_force_inline int32_t Child() const { int32_t child; memcpy( &child, &_child2, 4 ); return child; } + tracy_force_inline void SetChild( int32_t child ) { memcpy( &_child2, &child, 4 ); } + tracy_force_inline bool HasChildren() const { uint8_t tmp; memcpy( &tmp, ((char*)&_end_child1)+1, 1 ); return ( tmp >> 7 ) == 0; } + + tracy_force_inline void SetStartSrcLoc( int64_t start, int16_t srcloc ) { assert( start < (int64_t)( 1ull << 47 ) ); start <<= 16; start |= uint16_t( srcloc ); memcpy( &_start_srcloc, &start, 8 ); } + + uint64_t _start_srcloc; + uint16_t _child2; + uint64_t _end_child1; + uint32_t extra; +}; + +enum { ZoneEventSize = sizeof( ZoneEvent ) }; +static_assert( std::is_standard_layout::value, "ZoneEvent is not standard layout" ); + + +struct ZoneExtra +{ + Int24 callstack; + StringIdx text; + StringIdx name; + Int24 color; +}; + +enum { ZoneExtraSize = sizeof( ZoneExtra ) }; + + +// This union exploits the fact that the current implementations of x64 and arm64 do not provide +// full 64 bit address space. The high bits must be bit-extended, so 0x80... is an invalid pointer. +// This allows using the highest bit as a selector between a native pointer and a table index here. +union CallstackFrameId +{ + struct + { + uint64_t idx : 62; + uint64_t sel : 1; + uint64_t custom : 1; + }; + uint64_t data; +}; + +enum { CallstackFrameIdSize = sizeof( CallstackFrameId ) }; + +static tracy_force_inline bool operator==( const CallstackFrameId& lhs, const CallstackFrameId& rhs ) { return lhs.data == rhs.data; } + + +struct SampleData +{ + Int48 time; + Int24 callstack; +}; + +enum { SampleDataSize = sizeof( SampleData ) }; + +struct SampleDataSort { bool operator()( const SampleData& lhs, const SampleData& rhs ) { return lhs.time.Val() < rhs.time.Val(); }; }; + + +struct SampleDataRange +{ + Int48 time; + uint16_t thread; + CallstackFrameId ip; +}; + +enum { SampleDataRangeSize = sizeof( SampleDataRange ) }; + + +struct HwSampleData +{ + SortedVector cycles; + SortedVector retired; + SortedVector cacheRef; + SortedVector cacheMiss; + SortedVector branchRetired; + SortedVector branchMiss; + + bool is_sorted() const + { + return + cycles.is_sorted() && + retired.is_sorted() && + cacheRef.is_sorted() && + cacheMiss.is_sorted() && + branchRetired.is_sorted() && + branchMiss.is_sorted(); + } + + void sort() + { + if( !cycles.is_sorted() ) cycles.sort(); + if( !retired.is_sorted() ) retired.sort(); + if( !cacheRef.is_sorted() ) cacheRef.sort(); + if( !cacheMiss.is_sorted() ) cacheMiss.sort(); + if( !branchRetired.is_sorted() ) branchRetired.sort(); + if( !branchMiss.is_sorted() ) branchMiss.sort(); + } +}; + +enum { HwSampleDataSize = sizeof( HwSampleData ) }; + + +struct LockEvent +{ + enum class Type : uint8_t + { + Wait, + Obtain, + Release, + WaitShared, + ObtainShared, + ReleaseShared + }; + + tracy_force_inline int64_t Time() const { return int64_t( _time_srcloc ) >> 16; } + tracy_force_inline void SetTime( int64_t time ) { assert( time < (int64_t)( 1ull << 47 ) ); memcpy( ((char*)&_time_srcloc)+2, &time, 4 ); memcpy( ((char*)&_time_srcloc)+6, ((char*)&time)+4, 2 ); } + tracy_force_inline int16_t SrcLoc() const { return int16_t( _time_srcloc & 0xFFFF ); } + tracy_force_inline void SetSrcLoc( int16_t srcloc ) { memcpy( &_time_srcloc, &srcloc, 2 ); } + + uint64_t _time_srcloc; + uint8_t thread; + Type type; +}; + +struct LockEventShared : public LockEvent +{ + uint64_t waitShared; + uint64_t sharedList; +}; + +struct LockEventPtr +{ + short_ptr ptr; + uint8_t lockingThread; + uint8_t lockCount; + uint64_t waitList; +}; + +enum { LockEventSize = sizeof( LockEvent ) }; +enum { LockEventSharedSize = sizeof( LockEventShared ) }; +enum { LockEventPtrSize = sizeof( LockEventPtr ) }; + +enum { MaxLockThreads = sizeof( LockEventPtr::waitList ) * 8 }; +static_assert( std::numeric_limits::max() >= MaxLockThreads, "Not enough space for lock count." ); + + +enum class LockType : uint8_t; + +struct LockMap +{ + struct TimeRange + { + int64_t start = std::numeric_limits::max(); + int64_t end = std::numeric_limits::min(); + }; + + StringIdx customName; + int16_t srcloc; + Vector timeline; + unordered_flat_map threadMap; + std::vector threadList; + LockType type; + int64_t timeAnnounce; + int64_t timeTerminate; + bool valid; + bool isContended; + uint64_t lockingThread; + + TimeRange range[64]; +}; + +struct LockHighlight +{ + int64_t id; + int64_t begin; + int64_t end; + uint8_t thread; + bool blocked; +}; + + +struct GpuEvent +{ + tracy_force_inline int64_t CpuStart() const { return int64_t( _cpuStart_srcloc ) >> 16; } + tracy_force_inline void SetCpuStart( int64_t cpuStart ) { assert( cpuStart < (int64_t)( 1ull << 47 ) ); memcpy( ((char*)&_cpuStart_srcloc)+2, &cpuStart, 4 ); memcpy( ((char*)&_cpuStart_srcloc)+6, ((char*)&cpuStart)+4, 2 ); } + tracy_force_inline int64_t CpuEnd() const { return int64_t( _cpuEnd_thread ) >> 16; } + tracy_force_inline void SetCpuEnd( int64_t cpuEnd ) { assert( cpuEnd < (int64_t)( 1ull << 47 ) ); memcpy( ((char*)&_cpuEnd_thread)+2, &cpuEnd, 4 ); memcpy( ((char*)&_cpuEnd_thread)+6, ((char*)&cpuEnd)+4, 2 ); } + tracy_force_inline int64_t GpuStart() const { return int64_t( _gpuStart_child1 ) >> 16; } + tracy_force_inline void SetGpuStart( int64_t gpuStart ) { /*assert( gpuStart < (int64_t)( 1ull << 47 ) );*/ memcpy( ((char*)&_gpuStart_child1)+2, &gpuStart, 4 ); memcpy( ((char*)&_gpuStart_child1)+6, ((char*)&gpuStart)+4, 2 ); } + tracy_force_inline int64_t GpuEnd() const { return int64_t( _gpuEnd_child2 ) >> 16; } + tracy_force_inline void SetGpuEnd( int64_t gpuEnd ) { assert( gpuEnd < (int64_t)( 1ull << 47 ) ); memcpy( ((char*)&_gpuEnd_child2)+2, &gpuEnd, 4 ); memcpy( ((char*)&_gpuEnd_child2)+6, ((char*)&gpuEnd)+4, 2 ); } + tracy_force_inline int16_t SrcLoc() const { return int16_t( _cpuStart_srcloc & 0xFFFF ); } + tracy_force_inline void SetSrcLoc( int16_t srcloc ) { memcpy( &_cpuStart_srcloc, &srcloc, 2 ); } + tracy_force_inline uint16_t Thread() const { return uint16_t( _cpuEnd_thread & 0xFFFF ); } + tracy_force_inline void SetThread( uint16_t thread ) { memcpy( &_cpuEnd_thread, &thread, 2 ); } + tracy_force_inline int32_t Child() const { return int32_t( uint32_t( _gpuStart_child1 & 0xFFFF ) | ( uint32_t( _gpuEnd_child2 & 0xFFFF ) << 16 ) ); } + tracy_force_inline void SetChild( int32_t child ) { memcpy( &_gpuStart_child1, &child, 2 ); memcpy( &_gpuEnd_child2, ((char*)&child)+2, 2 ); } + + uint64_t _cpuStart_srcloc; + uint64_t _cpuEnd_thread; + uint64_t _gpuStart_child1; + uint64_t _gpuEnd_child2; + Int24 callstack; +}; + +enum { GpuEventSize = sizeof( GpuEvent ) }; +static_assert( std::is_standard_layout::value, "GpuEvent is not standard layout" ); + + +struct MemEvent +{ + tracy_force_inline uint64_t Ptr() const { return uint64_t( int64_t( _ptr_csalloc1 ) >> 8 ); } + tracy_force_inline void SetPtr( uint64_t ptr ) { memcpy( ((char*)&_ptr_csalloc1)+1, &ptr, 4 ); memcpy( ((char*)&_ptr_csalloc1)+5, ((char*)&ptr)+4, 2 ); memcpy( ((char*)&_ptr_csalloc1)+7, ((char*)&ptr)+6, 1 ); } + tracy_force_inline uint64_t Size() const { return _size_csalloc2 >> 16; } + tracy_force_inline void SetSize( uint64_t size ) { assert( size < ( 1ull << 47 ) ); memcpy( ((char*)&_size_csalloc2)+2, &size, 4 ); memcpy( ((char*)&_size_csalloc2)+6, ((char*)&size)+4, 2 ); } + tracy_force_inline uint32_t CsAlloc() const { return uint8_t( _ptr_csalloc1 ) | ( uint16_t( _size_csalloc2 ) << 8 ); } + tracy_force_inline void SetCsAlloc( uint32_t csAlloc ) { memcpy( &_ptr_csalloc1, &csAlloc, 1 ); memcpy( &_size_csalloc2, ((char*)&csAlloc)+1, 2 ); } + tracy_force_inline int64_t TimeAlloc() const { return int64_t( _time_thread_alloc ) >> 16; } + tracy_force_inline void SetTimeAlloc( int64_t time ) { assert( time < (int64_t)( 1ull << 47 ) ); memcpy( ((char*)&_time_thread_alloc)+2, &time, 4 ); memcpy( ((char*)&_time_thread_alloc)+6, ((char*)&time)+4, 2 ); } + tracy_force_inline int64_t TimeFree() const { return int64_t( _time_thread_free ) >> 16; } + tracy_force_inline void SetTimeFree( int64_t time ) { assert( time < (int64_t)( 1ull << 47 ) ); memcpy( ((char*)&_time_thread_free)+2, &time, 4 ); memcpy( ((char*)&_time_thread_free)+6, ((char*)&time)+4, 2 ); } + tracy_force_inline uint16_t ThreadAlloc() const { return uint16_t( _time_thread_alloc ); } + tracy_force_inline void SetThreadAlloc( uint16_t thread ) { memcpy( &_time_thread_alloc, &thread, 2 ); } + tracy_force_inline uint16_t ThreadFree() const { return uint16_t( _time_thread_free ); } + tracy_force_inline void SetThreadFree( uint16_t thread ) { memcpy( &_time_thread_free, &thread, 2 ); } + + tracy_force_inline void SetTimeThreadAlloc( int64_t time, uint16_t thread ) { time <<= 16; time |= thread; memcpy( &_time_thread_alloc, &time, 8 ); } + tracy_force_inline void SetTimeThreadFree( int64_t time, uint16_t thread ) { uint64_t t; memcpy( &t, &time, 8 ); t <<= 16; t |= thread; memcpy( &_time_thread_free, &t, 8 ); } + + uint64_t _ptr_csalloc1; + uint64_t _size_csalloc2; + Int24 csFree; + uint64_t _time_thread_alloc; + uint64_t _time_thread_free; +}; + +enum { MemEventSize = sizeof( MemEvent ) }; +static_assert( std::is_standard_layout::value, "MemEvent is not standard layout" ); + + +struct CallstackFrameBasic +{ + StringIdx name; + StringIdx file; + uint32_t line; +}; + +struct CallstackFrame : public CallstackFrameBasic +{ + uint64_t symAddr; +}; + +struct SymbolData : public CallstackFrameBasic +{ + StringIdx imageName; + StringIdx callFile; + uint32_t callLine; + uint8_t isInline; + Int24 size; +}; + +enum { CallstackFrameBasicSize = sizeof( CallstackFrameBasic ) }; +enum { CallstackFrameSize = sizeof( CallstackFrame ) }; +enum { SymbolDataSize = sizeof( SymbolData ) }; + + +struct SymbolLocation +{ + uint64_t addr; + uint32_t len; +}; + +enum { SymbolLocationSize = sizeof( SymbolLocation ) }; + + +struct CallstackFrameData +{ + short_ptr data; + uint8_t size; + StringIdx imageName; +}; + +enum { CallstackFrameDataSize = sizeof( CallstackFrameData ) }; + + +struct MemCallstackFrameTree +{ + MemCallstackFrameTree( CallstackFrameId id ) : frame( id ), alloc( 0 ), count( 0 ) {} + + CallstackFrameId frame; + uint64_t alloc; + uint32_t count; + unordered_flat_map children; + unordered_flat_set callstacks; +}; + +enum { MemCallstackFrameTreeSize = sizeof( MemCallstackFrameTree ) }; + + +struct CallstackFrameTree +{ + CallstackFrameTree( CallstackFrameId id ) : frame( id ), count( 0 ) {} + + CallstackFrameId frame; + uint32_t count; + unordered_flat_map children; +}; + +enum { CallstackFrameTreeSize = sizeof( CallstackFrameTree ) }; + + +struct CrashEvent +{ + uint64_t thread = 0; + int64_t time = 0; + uint64_t message = 0; + uint32_t callstack = 0; +}; + +enum { CrashEventSize = sizeof( CrashEvent ) }; + + +struct ContextSwitchData +{ + enum : int8_t { Fiber = 99 }; + enum : int8_t { NoState = 100 }; + enum : int8_t { Wakeup = -2 }; + + tracy_force_inline int64_t Start() const { return int64_t( _start_cpu ) >> 16; } + tracy_force_inline void SetStart( int64_t start ) { assert( start < (int64_t)( 1ull << 47 ) ); memcpy( ((char*)&_start_cpu)+2, &start, 4 ); memcpy( ((char*)&_start_cpu)+6, ((char*)&start)+4, 2 ); } + tracy_force_inline int64_t End() const { return int64_t( _end_reason_state ) >> 16; } + tracy_force_inline void SetEnd( int64_t end ) { assert( end < (int64_t)( 1ull << 47 ) ); memcpy( ((char*)&_end_reason_state)+2, &end, 4 ); memcpy( ((char*)&_end_reason_state)+6, ((char*)&end)+4, 2 ); } + tracy_force_inline bool IsEndValid() const { return ( _end_reason_state >> 63 ) == 0; } + tracy_force_inline uint8_t Cpu() const { return uint8_t( _start_cpu & 0xFF ); } + tracy_force_inline void SetCpu( uint8_t cpu ) { memcpy( &_start_cpu, &cpu, 1 ); } + tracy_force_inline int8_t Reason() const { return int8_t( (_end_reason_state >> 8) & 0xFF ); } + tracy_force_inline void SetReason( int8_t reason ) { memcpy( ((char*)&_end_reason_state)+1, &reason, 1 ); } + tracy_force_inline int8_t State() const { return int8_t( _end_reason_state & 0xFF ); } + tracy_force_inline void SetState( int8_t state ) { memcpy( &_end_reason_state, &state, 1 ); } + tracy_force_inline int64_t WakeupVal() const { return _wakeup.Val(); } + tracy_force_inline void SetWakeup( int64_t wakeup ) { assert( wakeup < (int64_t)( 1ull << 47 ) ); _wakeup.SetVal( wakeup ); } + tracy_force_inline uint16_t Thread() const { return _thread; } + tracy_force_inline void SetThread( uint16_t thread ) { _thread = thread; } + + tracy_force_inline void SetStartCpu( int64_t start, uint8_t cpu ) { assert( start < (int64_t)( 1ull << 47 ) ); _start_cpu = ( uint64_t( start ) << 16 ) | cpu; } + tracy_force_inline void SetEndReasonState( int64_t end, int8_t reason, int8_t state ) { assert( end < (int64_t)( 1ull << 47 ) ); _end_reason_state = ( uint64_t( end ) << 16 ) | ( uint64_t( reason ) << 8 ) | uint8_t( state ); } + + uint64_t _start_cpu; + uint64_t _end_reason_state; + Int48 _wakeup; + uint16_t _thread; +}; + +enum { ContextSwitchDataSize = sizeof( ContextSwitchData ) }; + + +struct ContextSwitchCpu +{ + tracy_force_inline int64_t Start() const { return int64_t( _start_thread ) >> 16; } + tracy_force_inline void SetStart( int64_t start ) { assert( start < (int64_t)( 1ull << 47 ) ); memcpy( ((char*)&_start_thread)+2, &start, 4 ); memcpy( ((char*)&_start_thread)+6, ((char*)&start)+4, 2 ); } + tracy_force_inline int64_t End() const { int64_t v; memcpy( &v, ((char*)&_end)-2, 8 ); return v >> 16; } + tracy_force_inline void SetEnd( int64_t end ) { assert( end < (int64_t)( 1ull << 47 ) ); _end.SetVal( end ); } + tracy_force_inline bool IsEndValid() const { return _end.IsNonNegative(); } + tracy_force_inline uint16_t Thread() const { return uint16_t( _start_thread ); } + tracy_force_inline void SetThread( uint16_t thread ) { memcpy( &_start_thread, &thread, 2 ); } + + tracy_force_inline void SetStartThread( int64_t start, uint16_t thread ) { assert( start < (int64_t)( 1ull << 47 ) ); _start_thread = ( uint64_t( start ) << 16 ) | thread; } + + uint64_t _start_thread; + Int48 _end; +}; + +enum { ContextSwitchCpuSize = sizeof( ContextSwitchCpu ) }; + + +struct ContextSwitchUsage +{ + ContextSwitchUsage() {} + ContextSwitchUsage( int64_t time, uint8_t other, uint8_t own ) { SetTime( time ); SetOther( other ); SetOwn( own ); } + + tracy_force_inline int64_t Time() const { return int64_t( _time_other_own ) >> 16; } + tracy_force_inline void SetTime( int64_t time ) { assert( time < (int64_t)( 1ull << 47 ) ); memcpy( ((char*)&_time_other_own)+2, &time, 4 ); memcpy( ((char*)&_time_other_own)+6, ((char*)&time)+4, 2 ); } + tracy_force_inline uint8_t Other() const { return uint8_t( _time_other_own ); } + tracy_force_inline void SetOther( uint8_t other ) { memcpy( &_time_other_own, &other, 1 ); } + tracy_force_inline uint8_t Own() const { uint8_t v; memcpy( &v, ((char*)&_time_other_own)+1, 1 );return v; } + tracy_force_inline void SetOwn( uint8_t own ) { memcpy( ((char*)&_time_other_own)+1, &own, 1 ); } + + uint64_t _time_other_own; +}; + +enum { ContextSwitchUsageSize = sizeof( ContextSwitchUsage ) }; + + +struct MessageData +{ + int64_t time; + StringRef ref; + uint16_t thread; + uint32_t color; + Int24 callstack; +}; + +enum { MessageDataSize = sizeof( MessageData ) }; + + +struct PlotItem +{ + Int48 time; + double val; +}; + +enum { PlotItemSize = sizeof( PlotItem ) }; + + +struct FrameEvent +{ + int64_t start; + int64_t end; + int32_t frameImage; +}; + +enum { FrameEventSize = sizeof( FrameEvent ) }; + + +struct FrameImage +{ + short_ptr ptr; + uint32_t csz; + uint16_t w, h; + uint32_t frameRef; + uint8_t flip; +}; + +enum { FrameImageSize = sizeof( FrameImage ) }; + + +struct GhostZone +{ + Int48 start, end; + Int24 frame; + int32_t child; +}; + +enum { GhostZoneSize = sizeof( GhostZone ) }; + + +struct ChildSample +{ + Int48 time; + uint64_t addr; +}; + +enum { ChildSampleSize = sizeof( ChildSample ) }; + +#pragma pack( pop ) + + +struct ThreadData +{ + uint64_t id; + uint64_t count; + Vector> timeline; + Vector> stack; + Vector> messages; + uint32_t nextZoneId; + Vector zoneIdStack; +#ifndef TRACY_NO_STATISTICS + Vector childTimeStack; + Vector ghostZones; + uint64_t ghostIdx; + SortedVector postponedSamples; +#endif + Vector samples; + SampleData pendingSample; + Vector ctxSwitchSamples; + uint64_t kernelSampleCnt; + uint8_t isFiber; + ThreadData* fiber; + uint8_t* stackCount; + int32_t groupHint; + + tracy_force_inline void IncStackCount( int16_t srcloc ) { stackCount[uint16_t(srcloc)]++; } + tracy_force_inline bool DecStackCount( int16_t srcloc ) { return --stackCount[uint16_t(srcloc)] != 0; } +}; + +struct GpuCtxThreadData +{ + Vector> timeline; + Vector> stack; +}; + +struct GpuCtxData +{ + int64_t timeDiff; + uint64_t thread; + uint64_t count; + float period; + GpuContextType type; + bool hasPeriod; + bool hasCalibration; + int64_t calibratedGpuTime; + int64_t calibratedCpuTime; + double calibrationMod; + int64_t lastGpuTime; + uint64_t overflow; + uint32_t overflowMul; + StringIdx name; + unordered_flat_map threadData; + short_ptr query[64*1024]; +}; + +enum { GpuCtxDataSize = sizeof( GpuCtxData ) }; + + +enum class PlotType : uint8_t +{ + User, + Memory, + SysTime, + Power +}; + +// Keep this in sync with enum in TracyC.h +enum class PlotValueFormatting : uint8_t +{ + Number, + Memory, + Percentage, + Watt +}; + +struct PlotData +{ + struct PlotItemSort { bool operator()( const PlotItem& lhs, const PlotItem& rhs ) { return lhs.time.Val() < rhs.time.Val(); }; }; + + uint64_t name; + double min; + double max; + double sum; + SortedVector data; + PlotType type; + PlotValueFormatting format; + uint8_t showSteps; + uint8_t fill; + uint32_t color; + + double rMin, rMax, num; +}; + +struct MemData +{ + Vector data; + Vector frees; + unordered_flat_map active; + uint64_t high = std::numeric_limits::min(); + uint64_t low = std::numeric_limits::max(); + uint64_t usage = 0; + PlotData* plot = nullptr; + bool reconstruct = false; + uint64_t name = 0; +}; + +struct FrameData +{ + uint64_t name; + Vector frames; + uint8_t continuous; + + int64_t min = std::numeric_limits::max(); + int64_t max = std::numeric_limits::min(); + int64_t total = 0; + double sumSq = 0; +}; + +struct StringLocation +{ + const char* ptr; + uint32_t idx; +}; + +struct SourceLocationHasher +{ + size_t operator()( const SourceLocation* ptr ) const + { + return charutil::hash( (const char*)ptr, sizeof( SourceLocationBase ) ); + } +}; + +struct SourceLocationComparator +{ + bool operator()( const SourceLocation* lhs, const SourceLocation* rhs ) const + { + return memcmp( lhs, rhs, sizeof( SourceLocationBase ) ) == 0; + } +}; + + +struct ContextSwitch +{ + Vector v; + int64_t runningTime = 0; +}; + +struct CpuData +{ + Vector cs; +}; + +struct CpuThreadData +{ + int64_t runningTime = 0; + uint32_t runningRegions = 0; + uint32_t migrations = 0; +}; + +enum { CpuThreadDataSize = sizeof( CpuThreadData ) }; + + +struct Parameter +{ + uint32_t idx; + StringRef name; + bool isBool; + int32_t val; +}; + + +struct SymbolStats +{ + uint32_t incl, excl; + unordered_flat_map parents; + unordered_flat_map baseParents; +}; + +enum { SymbolStatsSize = sizeof( SymbolStats ) }; + +} + +#endif diff --git a/third_party/tracy/server/TracyFileHeader.hpp b/third_party/tracy/server/TracyFileHeader.hpp new file mode 100644 index 0000000000000000000000000000000000000000..650de534a8d950c731ce51349181e3850eb85ef1 --- /dev/null +++ b/third_party/tracy/server/TracyFileHeader.hpp @@ -0,0 +1,22 @@ +#ifndef __TRACYFILEHEADER_HPP__ +#define __TRACYFILEHEADER_HPP__ + +#include + +#include "../public/common/TracyForceInline.hpp" + +namespace tracy +{ + +static const uint8_t TracyHeader[4] = { 't', 'r', 253, 'P' }; +static const uint8_t Lz4Header[4] = { 't', 'l', 'Z', 4 }; +static const uint8_t ZstdHeader[4] = { 't', 'Z', 's', 't' }; + +static constexpr tracy_force_inline int FileVersion( uint8_t h5, uint8_t h6, uint8_t h7 ) +{ + return ( h5 << 16 ) | ( h6 << 8 ) | h7; +} + +} + +#endif diff --git a/third_party/tracy/server/TracyFileMeta.hpp b/third_party/tracy/server/TracyFileMeta.hpp new file mode 100644 index 0000000000000000000000000000000000000000..6d493dc3257547312b070167d41dc657d857131e --- /dev/null +++ b/third_party/tracy/server/TracyFileMeta.hpp @@ -0,0 +1,18 @@ +#ifndef __TRACYFILEMETA_HPP__ +#define __TRACYFILEMETA_HPP__ + +#include +#include + +#include "../public/common/tracy_lz4.hpp" +#include "../zstd/zstd.h" + +namespace tracy +{ + +constexpr size_t FileBufSize = 64 * 1024; +constexpr size_t FileBoundSize = std::max( LZ4_COMPRESSBOUND( FileBufSize ), ZSTD_COMPRESSBOUND( FileBufSize ) ); + +} + +#endif diff --git a/third_party/tracy/server/TracyFileRead.hpp b/third_party/tracy/server/TracyFileRead.hpp new file mode 100644 index 0000000000000000000000000000000000000000..3c71054de627fe26ea9a6f8e3308eed71f83c7ed --- /dev/null +++ b/third_party/tracy/server/TracyFileRead.hpp @@ -0,0 +1,601 @@ +#ifndef __TRACYFILEREAD_HPP__ +#define __TRACYFILEREAD_HPP__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifdef _MSC_VER +# define stat64 _stat64 +#endif +#if defined __APPLE__ || defined __FreeBSD__ +# define stat64 stat +#endif + +#include "TracyFileHeader.hpp" +#include "TracyFileMeta.hpp" +#include "TracyMmap.hpp" +#include "../public/common/TracyYield.hpp" +#include "../public/common/tracy_lz4.hpp" +#include "../public/common/TracyForceInline.hpp" +#include "../zstd/zstd.h" + +namespace tracy +{ + +struct NotTracyDump : public std::exception {}; +struct FileReadError : public std::exception {}; + +class ReadStream +{ +public: + ReadStream( uint8_t type ) + : m_stream( nullptr ) + , m_streamZstd( nullptr ) + , m_buf( new char[FileBufSize] ) + , m_second( new char[FileBufSize] ) + { + switch( type ) + { + case 0: + m_stream = LZ4_createStreamDecode(); + break; + case 1: + m_streamZstd = ZSTD_createDStream(); + break; + default: + assert( false ); + break; + } + } + + ~ReadStream() + { + delete[] m_buf; + delete[] m_second; + + if( m_stream ) LZ4_freeStreamDecode( m_stream ); + if( m_streamZstd ) ZSTD_freeDStream( m_streamZstd ); + } + + void Decompress( const char* src, uint32_t size ) + { + std::swap( m_buf, m_second ); + if( m_stream ) + { + m_size = (size_t)LZ4_decompress_safe_continue( m_stream, src, m_buf, size, FileBufSize ); + } + else + { + ZSTD_outBuffer out = { m_buf, FileBufSize, 0 }; + ZSTD_inBuffer in = { src, size, 0 }; + ZSTD_decompressStream( m_streamZstd, &out, &in ); + m_size = out.pos; + } + } + + const char* GetBuffer() const { return m_buf; } + size_t GetSize() const { return m_size; } + +private: + LZ4_streamDecode_t* m_stream; + ZSTD_DStream* m_streamZstd; + + char* m_buf; + char* m_second; + + size_t m_size; +}; + +class FileRead +{ + struct StreamHandle + { + StreamHandle( uint8_t type ) : stream( type ), outputReady( false ) {} + + ReadStream stream; + const char* src; + uint32_t size; + + bool inputReady = false; + bool exit = false; + alignas(64) std::atomic outputReady; + + std::mutex signalLock; + std::condition_variable signal; + + std::thread thread; + }; + +public: + static FileRead* Open( const char* fn ) + { + auto f = fopen( fn, "rb" ); + return f ? new FileRead( f, fn ) : nullptr; + } + + ~FileRead() + { + for( auto& v : m_streams ) + { + std::lock_guard lock( v->signalLock ); + v->exit = true; + v->signal.notify_one(); + } + for( auto& v : m_streams ) v->thread.join(); + m_streams.clear(); + if( m_data ) munmap( m_data, m_dataSize ); + } + + tracy_force_inline void Read( void* ptr, size_t size ) + { + if( size <= FileBufSize - m_offset ) + { + ReadSmall( ptr, size ); + } + else + { + ReadBig( ptr, size ); + } + } + + tracy_force_inline void Skip( size_t size ) + { + if( size <= FileBufSize - m_offset ) + { + m_offset += size; + } + else + { + SkipBig( size ); + } + } + + template + tracy_force_inline void Read( T& v ) + { + if( sizeof( T ) <= FileBufSize - m_offset ) + { + memcpy( &v, m_buf + m_offset, sizeof( T ) ); + m_offset += sizeof( T ); + } + else + { + T tmp; + ReadBig( &tmp, sizeof( T ) ); + memcpy( &v, &tmp, sizeof( T ) ); + } + } + + template + tracy_force_inline void Read2( T& v0, U& v1 ) + { + if( sizeof( T ) + sizeof( U ) <= FileBufSize - m_offset ) + { + memcpy( &v0, m_buf + m_offset, sizeof( T ) ); + memcpy( &v1, m_buf + m_offset + sizeof( T ), sizeof( U ) ); + m_offset += sizeof( T ) + sizeof( U ); + } + else + { + char tmp[sizeof( T ) + sizeof( U )]; + ReadBig( tmp, sizeof( T ) + sizeof( U ) ); + memcpy( &v0, tmp, sizeof( T ) ); + memcpy( &v1, tmp + sizeof( T ), sizeof( U ) ); + } + } + + template + tracy_force_inline void Read3( T& v0, U& v1, V& v2 ) + { + if( sizeof( T ) + sizeof( U ) + sizeof( V ) <= FileBufSize - m_offset ) + { + memcpy( &v0, m_buf + m_offset, sizeof( T ) ); + memcpy( &v1, m_buf + m_offset + sizeof( T ), sizeof( U ) ); + memcpy( &v2, m_buf + m_offset + sizeof( T ) + sizeof( U ), sizeof( V ) ); + m_offset += sizeof( T ) + sizeof( U ) + sizeof( V ); + } + else + { + char tmp[sizeof( T ) + sizeof( U ) + sizeof( V )]; + ReadBig( tmp, sizeof( T ) + sizeof( U ) + sizeof( V ) ); + memcpy( &v0, tmp, sizeof( T ) ); + memcpy( &v1, tmp + sizeof( T ), sizeof( U ) ); + memcpy( &v2, tmp + sizeof( T ) + sizeof( U ), sizeof( V ) ); + } + } + + template + tracy_force_inline void Read4( T& v0, U& v1, V& v2, W& v3 ) + { + if( sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) <= FileBufSize - m_offset ) + { + memcpy( &v0, m_buf + m_offset, sizeof( T ) ); + memcpy( &v1, m_buf + m_offset + sizeof( T ), sizeof( U ) ); + memcpy( &v2, m_buf + m_offset + sizeof( T ) + sizeof( U ), sizeof( V ) ); + memcpy( &v3, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ), sizeof( W ) ); + m_offset += sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ); + } + else + { + char tmp[sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W )]; + ReadBig( tmp, sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) ); + memcpy( &v0, tmp, sizeof( T ) ); + memcpy( &v1, tmp + sizeof( T ), sizeof( U ) ); + memcpy( &v2, tmp + sizeof( T ) + sizeof( U ), sizeof( V ) ); + memcpy( &v3, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ), sizeof( W ) ); + } + } + + template + tracy_force_inline void Read5( T& v0, U& v1, V& v2, W& v3, X& v4 ) + { + if( sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) <= FileBufSize - m_offset ) + { + memcpy( &v0, m_buf + m_offset, sizeof( T ) ); + memcpy( &v1, m_buf + m_offset + sizeof( T ), sizeof( U ) ); + memcpy( &v2, m_buf + m_offset + sizeof( T ) + sizeof( U ), sizeof( V ) ); + memcpy( &v3, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ), sizeof( W ) ); + memcpy( &v4, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ), sizeof( X ) ); + m_offset += sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ); + } + else + { + char tmp[sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X )]; + ReadBig( tmp, sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) ); + memcpy( &v0, tmp, sizeof( T ) ); + memcpy( &v1, tmp + sizeof( T ), sizeof( U ) ); + memcpy( &v2, tmp + sizeof( T ) + sizeof( U ), sizeof( V ) ); + memcpy( &v3, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ), sizeof( W ) ); + memcpy( &v4, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ), sizeof( X ) ); + } + } + + template + tracy_force_inline void Read6( T& v0, U& v1, V& v2, W& v3, X& v4, Y& v5 ) + { + if( sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) <= FileBufSize - m_offset ) + { + memcpy( &v0, m_buf + m_offset, sizeof( T ) ); + memcpy( &v1, m_buf + m_offset + sizeof( T ), sizeof( U ) ); + memcpy( &v2, m_buf + m_offset + sizeof( T ) + sizeof( U ), sizeof( V ) ); + memcpy( &v3, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ), sizeof( W ) ); + memcpy( &v4, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ), sizeof( X ) ); + memcpy( &v5, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ), sizeof( Y ) ); + m_offset += sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ); + } + else + { + char tmp[sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y )]; + ReadBig( tmp, sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) ); + memcpy( &v0, tmp, sizeof( T ) ); + memcpy( &v1, tmp + sizeof( T ), sizeof( U ) ); + memcpy( &v2, tmp + sizeof( T ) + sizeof( U ), sizeof( V ) ); + memcpy( &v3, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ), sizeof( W ) ); + memcpy( &v4, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ), sizeof( X ) ); + memcpy( &v5, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ), sizeof( Y ) ); + } + } + + template + tracy_force_inline void Read7( T& v0, U& v1, V& v2, W& v3, X& v4, Y& v5, Z& v6 ) + { + if( sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) <= FileBufSize - m_offset ) + { + memcpy( &v0, m_buf + m_offset, sizeof( T ) ); + memcpy( &v1, m_buf + m_offset + sizeof( T ), sizeof( U ) ); + memcpy( &v2, m_buf + m_offset + sizeof( T ) + sizeof( U ), sizeof( V ) ); + memcpy( &v3, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ), sizeof( W ) ); + memcpy( &v4, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ), sizeof( X ) ); + memcpy( &v5, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ), sizeof( Y ) ); + memcpy( &v6, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ), sizeof( Z ) ); + m_offset += sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ); + } + else + { + char tmp[sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z )]; + ReadBig( tmp, sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) ); + memcpy( &v0, tmp, sizeof( T ) ); + memcpy( &v1, tmp + sizeof( T ), sizeof( U ) ); + memcpy( &v2, tmp + sizeof( T ) + sizeof( U ), sizeof( V ) ); + memcpy( &v3, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ), sizeof( W ) ); + memcpy( &v4, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ), sizeof( X ) ); + memcpy( &v5, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ), sizeof( Y ) ); + memcpy( &v6, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ), sizeof( Z ) ); + } + } + + template + tracy_force_inline void Read8( T& v0, U& v1, V& v2, W& v3, X& v4, Y& v5, Z& v6, A& v7 ) + { + if( sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) + sizeof( A ) <= FileBufSize - m_offset ) + { + memcpy( &v0, m_buf + m_offset, sizeof( T ) ); + memcpy( &v1, m_buf + m_offset + sizeof( T ), sizeof( U ) ); + memcpy( &v2, m_buf + m_offset + sizeof( T ) + sizeof( U ), sizeof( V ) ); + memcpy( &v3, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ), sizeof( W ) ); + memcpy( &v4, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ), sizeof( X ) ); + memcpy( &v5, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ), sizeof( Y ) ); + memcpy( &v6, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ), sizeof( Z ) ); + memcpy( &v7, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ), sizeof( A ) ); + m_offset += sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) + sizeof( A ); + } + else + { + char tmp[sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) + sizeof( A )]; + ReadBig( tmp, sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) + sizeof( A ) ); + memcpy( &v0, tmp, sizeof( T ) ); + memcpy( &v1, tmp + sizeof( T ), sizeof( U ) ); + memcpy( &v2, tmp + sizeof( T ) + sizeof( U ), sizeof( V ) ); + memcpy( &v3, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ), sizeof( W ) ); + memcpy( &v4, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ), sizeof( X ) ); + memcpy( &v5, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ), sizeof( Y ) ); + memcpy( &v6, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ), sizeof( Z ) ); + memcpy( &v7, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ), sizeof( A ) ); + } + } + + template + tracy_force_inline void Read9( T& v0, U& v1, V& v2, W& v3, X& v4, Y& v5, Z& v6, A& v7, B& v8 ) + { + if( sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) + sizeof( A ) + sizeof( B ) <= FileBufSize - m_offset ) + { + memcpy( &v0, m_buf + m_offset, sizeof( T ) ); + memcpy( &v1, m_buf + m_offset + sizeof( T ), sizeof( U ) ); + memcpy( &v2, m_buf + m_offset + sizeof( T ) + sizeof( U ), sizeof( V ) ); + memcpy( &v3, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ), sizeof( W ) ); + memcpy( &v4, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ), sizeof( X ) ); + memcpy( &v5, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ), sizeof( Y ) ); + memcpy( &v6, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ), sizeof( Z ) ); + memcpy( &v7, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ), sizeof( A ) ); + memcpy( &v8, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) + sizeof( A ), sizeof( B ) ); + m_offset += sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) + sizeof( A ) + sizeof( B ); + } + else + { + char tmp[sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) + sizeof( A ) + sizeof( B )]; + ReadBig( tmp, sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) + sizeof( A ) + sizeof( B ) ); + memcpy( &v0, tmp, sizeof( T ) ); + memcpy( &v1, tmp + sizeof( T ), sizeof( U ) ); + memcpy( &v2, tmp + sizeof( T ) + sizeof( U ), sizeof( V ) ); + memcpy( &v3, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ), sizeof( W ) ); + memcpy( &v4, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ), sizeof( X ) ); + memcpy( &v5, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ), sizeof( Y ) ); + memcpy( &v6, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ), sizeof( Z ) ); + memcpy( &v7, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ), sizeof( A ) ); + memcpy( &v8, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) + sizeof( A ), sizeof( B ) ); + } + } + + template + tracy_force_inline void Read10( T& v0, U& v1, V& v2, W& v3, X& v4, Y& v5, Z& v6, A& v7, B& v8, C& v9 ) + { + if( sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) + sizeof( A ) + sizeof( B ) + sizeof( C ) <= FileBufSize - m_offset ) + { + memcpy( &v0, m_buf + m_offset, sizeof( T ) ); + memcpy( &v1, m_buf + m_offset + sizeof( T ), sizeof( U ) ); + memcpy( &v2, m_buf + m_offset + sizeof( T ) + sizeof( U ), sizeof( V ) ); + memcpy( &v3, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ), sizeof( W ) ); + memcpy( &v4, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ), sizeof( X ) ); + memcpy( &v5, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ), sizeof( Y ) ); + memcpy( &v6, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ), sizeof( Z ) ); + memcpy( &v7, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ), sizeof( A ) ); + memcpy( &v8, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) + sizeof( A ), sizeof( B ) ); + memcpy( &v9, m_buf + m_offset + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) + sizeof( A ) + sizeof( B ), sizeof( C ) ); + m_offset += sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) + sizeof( A ) + sizeof( B ) + sizeof( C ); + } + else + { + char tmp[sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) + sizeof( A ) + sizeof( B ) + sizeof( C )]; + ReadBig( tmp, sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) + sizeof( A ) + sizeof( B ) + sizeof( C ) ); + memcpy( &v0, tmp, sizeof( T ) ); + memcpy( &v1, tmp + sizeof( T ), sizeof( U ) ); + memcpy( &v2, tmp + sizeof( T ) + sizeof( U ), sizeof( V ) ); + memcpy( &v3, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ), sizeof( W ) ); + memcpy( &v4, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ), sizeof( X ) ); + memcpy( &v5, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ), sizeof( Y ) ); + memcpy( &v6, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ), sizeof( Z ) ); + memcpy( &v7, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ), sizeof( A ) ); + memcpy( &v8, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) + sizeof( A ), sizeof( B ) ); + memcpy( &v9, tmp + sizeof( T ) + sizeof( U ) + sizeof( V ) + sizeof( W ) + sizeof( X ) + sizeof( Y ) + sizeof( Z ) + sizeof( A ) + sizeof( B ), sizeof( C ) ); + } + } + + const std::string& GetFilename() const { return m_filename; } + +private: + FileRead( FILE* f, const char* fn ) + : m_data( nullptr ) + , m_offset( 0 ) + , m_streamId( 0 ) + , m_filename( fn ) + { + char hdr[4]; + if( fread( hdr, 1, sizeof( hdr ), f ) != sizeof( hdr ) ) + { + fclose( f ); + throw NotTracyDump(); + } + + uint8_t streams = 1; + uint8_t type; + m_dataOffset = sizeof( hdr ); + + if( memcmp( hdr, TracyHeader, sizeof( hdr ) ) == 0 ) + { + if( fread( &type, 1, 1, f ) != 1 || type > 1 ) + { + fclose( f ); + throw NotTracyDump(); + } + if( fread( &streams, 1, 1, f ) != 1 ) + { + fclose( f ); + throw NotTracyDump(); + } + m_dataOffset += 2; + } + else if( memcmp( hdr, Lz4Header, sizeof( hdr ) ) == 0 ) + { + type = 0; + } + else if( memcmp( hdr, ZstdHeader, sizeof( hdr ) ) == 0 ) + { + type = 1; + } + else + { + fclose( f ); + throw NotTracyDump(); + } + + struct stat64 buf; + if( stat64( fn, &buf ) == 0 ) + { + m_dataSize = buf.st_size; + } + else + { + fclose( f ); + throw FileReadError(); + } + + m_data = (char*)mmap( nullptr, m_dataSize, PROT_READ, MAP_SHARED, fileno( f ), 0 ); + fclose( f ); + if( !m_data ) + { + throw FileReadError(); + } + + for( int i=0; i<(int)streams; i++ ) + { + if( m_dataOffset == m_dataSize ) break; + + const auto sz = ReadBlockSize(); + auto uptr = std::make_unique( type ); + uptr->src = m_data + m_dataOffset; + uptr->size = sz; + uptr->inputReady = true; + uptr->thread = std::thread( [ptr = uptr.get()] { Worker( ptr ); } ); + m_streams.emplace_back( std::move( uptr ) ); + m_dataOffset += sz; + } + + GetNextDataBlock(); + } + + tracy_force_inline uint32_t ReadBlockSize() + { + uint32_t sz; + memcpy( &sz, m_data + m_dataOffset, sizeof( sz ) ); + m_dataOffset += sizeof( sz ); + return sz; + } + + static void Worker( StreamHandle* hnd ) + { + for(;;) + { + std::unique_lock lock( hnd->signalLock ); + hnd->signal.wait( lock, [&] { return hnd->inputReady || hnd->exit; } ); + if( hnd->exit ) return; + lock.unlock(); + + hnd->stream.Decompress( hnd->src, hnd->size ); + hnd->inputReady = false; + hnd->outputReady.store( true, std::memory_order_release ); + } + } + + tracy_force_inline void ReadSmall( void* ptr, size_t size ) + { + memcpy( ptr, m_buf + m_offset, size ); + m_offset += size; + } + + void ReadBig( void* ptr, size_t size ) + { + assert( size > 0 ); + auto dst = (char*)ptr; + do + { + size_t sz; + if( m_offset == FileBufSize ) + { + sz = std::min( size, FileBufSize ); + GetNextDataBlock(); + memcpy( dst, m_buf, sz ); + m_offset = sz; + } + else + { + sz = std::min( size, FileBufSize - m_offset ); + memcpy( dst, m_buf + m_offset, sz ); + m_offset += sz; + } + + dst += sz; + size -= sz; + } + while( size > 0 ); + } + + void SkipBig( size_t size ) + { + while( size > 0 ) + { + if( m_offset == FileBufSize ) GetNextDataBlock(); + const auto sz = std::min( size, FileBufSize - m_offset ); + m_offset += sz; + size -= sz; + } + } + + void GetNextDataBlock() + { + auto& hnd = *m_streams[m_streamId]; + while( hnd.outputReady.load( std::memory_order_acquire ) == false ) { YieldThread(); } + hnd.outputReady.store( false, std::memory_order_relaxed ); + m_buf = hnd.stream.GetBuffer(); + m_offset = 0; + + if( m_dataOffset < m_dataSize ) + { + const auto sz = ReadBlockSize(); + std::unique_lock lock( hnd.signalLock ); + hnd.src = m_data + m_dataOffset; + hnd.size = sz; + hnd.inputReady = true; + hnd.signal.notify_one(); + lock.unlock(); + m_dataOffset += sz; + } + + m_streamId = ( m_streamId + 1 ) % m_streams.size(); + } + + char* m_data; + const char* m_buf; + uint64_t m_dataSize; + uint64_t m_dataOffset; + size_t m_offset; + int m_streamId; + + std::string m_filename; + + std::vector> m_streams; +}; + +} + +#endif diff --git a/third_party/tracy/server/TracyFileWrite.hpp b/third_party/tracy/server/TracyFileWrite.hpp new file mode 100644 index 0000000000000000000000000000000000000000..8af7175f7d3ce2302b134ebc22ee3c4247c00a39 --- /dev/null +++ b/third_party/tracy/server/TracyFileWrite.hpp @@ -0,0 +1,303 @@ +#ifndef __TRACYFILEWRITE_HPP__ +#define __TRACYFILEWRITE_HPP__ + +#ifdef _MSC_VER +# pragma warning( disable: 4267 ) // conversion from don't care to whatever, possible loss of data +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "TracyFileHeader.hpp" +#include "TracyFileMeta.hpp" +#include "../public/common/tracy_lz4.hpp" +#include "../public/common/tracy_lz4hc.hpp" +#include "../public/common/TracyForceInline.hpp" +#include "../zstd/zstd.h" + +namespace tracy +{ + +enum class FileCompression +{ + Fast, + Slow, + Extreme, + Zstd +}; + +class WriteStream +{ +public: + WriteStream( FileCompression comp, int level ) + : m_stream( nullptr ) + , m_streamHC( nullptr ) + , m_streamZstd( nullptr ) + , m_buf( new char[FileBufSize] ) + , m_second( new char[FileBufSize] ) + , m_compressed( new char[FileBoundSize] ) + { + switch( comp ) + { + case FileCompression::Fast: + m_stream = LZ4_createStream(); + break; + case FileCompression::Slow: + m_streamHC = LZ4_createStreamHC(); + break; + case FileCompression::Extreme: + m_streamHC = LZ4_createStreamHC(); + LZ4_resetStreamHC( m_streamHC, LZ4HC_CLEVEL_MAX ); + break; + case FileCompression::Zstd: + m_streamZstd = ZSTD_createCStream(); + ZSTD_CCtx_setParameter( m_streamZstd, ZSTD_c_compressionLevel, level ); + ZSTD_CCtx_setParameter( m_streamZstd, ZSTD_c_contentSizeFlag, 0 ); + break; + default: + assert( false ); + break; + } + } + + ~WriteStream() + { + delete[] m_buf; + delete[] m_second; + delete[] m_compressed; + + if( m_stream ) LZ4_freeStream( m_stream ); + if( m_streamHC ) LZ4_freeStreamHC( m_streamHC ); + if( m_streamZstd ) ZSTD_freeCStream( m_streamZstd ); + } + + char* GetInputBuffer() { return m_buf; } + const char* GetCompressedData() const { return m_compressed; } + uint32_t GetSize() const { return m_size; } + + void Compress( uint32_t sz ) + { + if( m_stream ) + { + m_size = LZ4_compress_fast_continue( m_stream, m_buf, m_compressed, sz, FileBoundSize, 1 ); + } + else if( m_streamZstd ) + { + ZSTD_outBuffer out = { m_compressed, FileBoundSize, 0 }; + ZSTD_inBuffer in = { m_buf, sz, 0 }; + const auto ret = ZSTD_compressStream2( m_streamZstd, &out, &in, ZSTD_e_flush ); + assert( ret == 0 ); + m_size = out.pos; + } + else + { + m_size = LZ4_compress_HC_continue( m_streamHC, m_buf, m_compressed, sz, FileBoundSize ); + } + + std::swap( m_buf, m_second ); + } + +private: + LZ4_stream_t* m_stream; + LZ4_streamHC_t* m_streamHC; + ZSTD_CStream* m_streamZstd; + + char* m_buf; + char* m_second; + char* m_compressed; + uint32_t m_size; +}; + +class FileWrite +{ + struct StreamHandle + { + StreamHandle( FileCompression comp, int level ) : stream( comp, level ) {} + + WriteStream stream; + uint32_t size; + + bool inputReady = false; + bool outputReady = false; + bool exit = false; + + std::mutex signalLock; + std::condition_variable signal; + + std::thread thread; + }; + +public: + static FileWrite* Open( const char* fn, FileCompression comp = FileCompression::Fast, int level = 1, int streams = -1 ) + { + auto f = fopen( fn, "wb" ); + if( !f ) return nullptr; + if( streams <= 0 ) streams = std::max( 1, std::thread::hardware_concurrency() ); + if( streams > 255 ) streams = 255; + return new FileWrite( f, comp, level, streams ); + } + + ~FileWrite() + { + Finish(); + fclose( m_file ); + } + + void Finish() + { + if( m_offset > 0 ) WriteBlock(); + while( m_streamPending > 0 ) ProcessPending(); + for( auto& v : m_streams ) + { + std::lock_guard lock( v->signalLock ); + v->exit = true; + v->signal.notify_one(); + } + for( auto& v : m_streams ) v->thread.join(); + m_streams.clear(); + } + + tracy_force_inline void Write( const void* ptr, size_t size ) + { + if( m_offset + size <= FileBufSize ) + { + WriteSmall( ptr, size ); + } + else + { + WriteBig( ptr, size ); + } + } + + std::pair GetCompressionStatistics() const { return std::make_pair( m_srcBytes, m_dstBytes ); } + +private: + FileWrite( FILE* f, FileCompression comp, int level, int streams ) + : m_offset( 0 ) + , m_file( f ) + , m_srcBytes( 0 ) + , m_dstBytes( 0 ) + { + assert( streams > 0 ); + assert( streams < 256 ); + + fwrite( TracyHeader, 1, sizeof( TracyHeader ), m_file ); + uint8_t u8 = comp == FileCompression::Zstd ? 1 : 0; + fwrite( &u8, 1, 1, m_file ); + u8 = streams; + fwrite( &u8, 1, 1, m_file ); + + m_streams.reserve( streams ); + for( int i=0; i( comp, level ); + uptr->thread = std::thread( [ptr = uptr.get()]{ Worker( ptr ); } ); + m_streams.emplace_back( std::move( uptr ) ); + } + + m_buf = m_streams[m_streamId]->stream.GetInputBuffer(); + } + + tracy_force_inline void WriteSmall( const void* ptr, size_t size ) + { + memcpy( m_buf + m_offset, ptr, size ); + m_offset += size; + } + + void WriteBig( const void* ptr, size_t size ) + { + auto src = (const char*)ptr; + while( size > 0 ) + { + const auto sz = std::min( size, FileBufSize - m_offset ); + memcpy( m_buf + m_offset, src, sz ); + m_offset += sz; + src += sz; + size -= sz; + + if( m_offset == FileBufSize ) + { + WriteBlock(); + } + } + } + + void WriteBlock() + { + m_srcBytes += m_offset; + + auto& hnd = *m_streams[m_streamId]; + assert( hnd.stream.GetInputBuffer() == m_buf ); + + std::unique_lock lock( hnd.signalLock ); + hnd.inputReady = true; + hnd.size = m_offset; + hnd.signal.notify_one(); + lock.unlock(); + + m_streamPending++; + m_streamId = ( m_streamId + 1 ) % m_streams.size(); + if( m_streamPending == m_streams.size() ) ProcessPending(); + + m_offset = 0; + m_buf = m_streams[m_streamId]->stream.GetInputBuffer(); + } + + void ProcessPending() + { + assert( m_streamPending > 0 ); + int id = ( m_streamId + m_streams.size() - m_streamPending ) % m_streams.size(); + m_streamPending--; + auto& hnd = *m_streams[id]; + + std::unique_lock lock( hnd.signalLock ); + hnd.signal.wait( lock, [&hnd]{ return hnd.outputReady; } ); + lock.unlock(); + + hnd.outputReady = false; + const uint32_t size = hnd.stream.GetSize(); + m_dstBytes += size; + fwrite( &size, 1, sizeof( size ), m_file ); + fwrite( hnd.stream.GetCompressedData(), 1, size, m_file ); + } + + static void Worker( StreamHandle* hnd ) + { + std::unique_lock lock( hnd->signalLock ); + for(;;) + { + hnd->signal.wait( lock, [&hnd]{ return hnd->inputReady || hnd->exit; } ); + if( hnd->exit ) return; + lock.unlock(); + + hnd->stream.Compress( hnd->size ); + hnd->inputReady = false; + + lock.lock(); + hnd->outputReady = true; + hnd->signal.notify_one(); + } + } + + char* m_buf; + size_t m_offset; + + int m_streamId = 0; + int m_streamPending = 0; + std::vector> m_streams; + FILE* m_file; + + size_t m_srcBytes; + size_t m_dstBytes; +}; + +} + +#endif diff --git a/third_party/tracy/server/TracyMemory.hpp b/third_party/tracy/server/TracyMemory.hpp new file mode 100644 index 0000000000000000000000000000000000000000..01e468d706b7d9ea45922e2a5045e97e23f990a5 --- /dev/null +++ b/third_party/tracy/server/TracyMemory.hpp @@ -0,0 +1,14 @@ +#ifndef __TRACYMEMORY_HPP__ +#define __TRACYMEMORY_HPP__ + +#include +#include + +namespace tracy +{ + +extern std::atomic memUsage; + +} + +#endif diff --git a/third_party/tracy/server/TracyMmap.cpp b/third_party/tracy/server/TracyMmap.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bec61a0e0ab8b8b794753386e73a7deb3d553cd7 --- /dev/null +++ b/third_party/tracy/server/TracyMmap.cpp @@ -0,0 +1,38 @@ +#include "TracyMmap.hpp" + +#if defined _WIN32 +# include +# include + +void* mmap( void* addr, size_t length, int prot, int flags, int fd, off_t offset ) +{ + HANDLE hnd; + void* map = nullptr; + + switch( prot ) + { + case PROT_READ: + if( hnd = CreateFileMapping( HANDLE( _get_osfhandle( fd ) ), nullptr, PAGE_READONLY, 0, 0, nullptr ) ) + { + map = MapViewOfFile( hnd, FILE_MAP_READ, 0, 0, length ); + CloseHandle( hnd ); + } + break; + case PROT_WRITE: + if( hnd = CreateFileMapping( HANDLE( _get_osfhandle( fd ) ), nullptr, PAGE_READWRITE, 0, 0, nullptr ) ) + { + map = MapViewOfFile( hnd, FILE_MAP_WRITE, 0, 0, length ); + CloseHandle( hnd ); + } + break; + } + + return map ? (char*)map + offset : (void*)-1; +} + +int munmap( void* addr, size_t length ) +{ + return UnmapViewOfFile( addr ) != 0 ? 0 : -1; +} + +#endif diff --git a/third_party/tracy/server/TracyMmap.hpp b/third_party/tracy/server/TracyMmap.hpp new file mode 100644 index 0000000000000000000000000000000000000000..d7f99b9930123671f7232a94e989c32c641bfa54 --- /dev/null +++ b/third_party/tracy/server/TracyMmap.hpp @@ -0,0 +1,19 @@ +#ifndef __TRACYMMAP_HPP__ +#define __TRACYMMAP_HPP__ + +#if !defined _WIN32 +# include +#else +# include +# include + +# define PROT_READ 1 +# define PROT_WRITE 2 +# define MAP_SHARED 0 + +void* mmap( void* addr, size_t length, int prot, int flags, int fd, off_t offset ); +int munmap( void* addr, size_t length ); + +#endif + +#endif diff --git a/third_party/tracy/server/TracyPopcnt.hpp b/third_party/tracy/server/TracyPopcnt.hpp new file mode 100644 index 0000000000000000000000000000000000000000..945324fd91d617e0c618e4ecc3d487060fc2b506 --- /dev/null +++ b/third_party/tracy/server/TracyPopcnt.hpp @@ -0,0 +1,40 @@ +#ifndef __TRACYPOPCNT_HPP__ +#define __TRACYPOPCNT_HPP__ + +#include +#include + +#if defined _WIN64 +# include +# define TracyCountBits __popcnt64 +# define TracyLzcnt __lzcnt64 +#elif defined __GNUC__ || defined __clang__ +static inline uint64_t TracyCountBits( uint64_t i ) +{ + return uint64_t( __builtin_popcountll( i ) ); +} +static inline uint64_t TracyLzcnt( uint64_t i ) +{ + return uint64_t( __builtin_clzll( i ) ); +} +#else +static inline uint64_t TracyCountBits( uint64_t i ) +{ + i = i - ( (i >> 1) & 0x5555555555555555 ); + i = ( i & 0x3333333333333333 ) + ( (i >> 2) & 0x3333333333333333 ); + i = ( (i + (i >> 4) ) & 0x0F0F0F0F0F0F0F0F ); + return ( i * (0x0101010101010101) ) >> 56; +} +static inline uint64_t TracyLzcnt( uint64_t i ) +{ + i |= i >> 1; + i |= i >> 2; + i |= i >> 4; + i |= i >> 8; + i |= i >> 16; + i |= i >> 32; + return 64 - TracyCountBits( i ); +} +#endif + +#endif diff --git a/third_party/tracy/server/TracyPrint.cpp b/third_party/tracy/server/TracyPrint.cpp new file mode 100644 index 0000000000000000000000000000000000000000..62bd8e450ec2eb7d556ff6f095dfbb76a33f4e66 --- /dev/null +++ b/third_party/tracy/server/TracyPrint.cpp @@ -0,0 +1,458 @@ +#ifdef _MSC_VER +# pragma warning( disable: 4244 ) // conversion from don't care to whatever, possible loss of data +#endif +#ifdef __MINGW32__ +# define __STDC_FORMAT_MACROS +#endif + +#include +#include +#include +#include +#include +#include // llabs() +#include + +#include "TracyPrint.hpp" + +namespace tracy +{ + +static const char* IntTable100 = + "00010203040506070809" + "10111213141516171819" + "20212223242526272829" + "30313233343536373839" + "40414243444546474849" + "50515253545556575859" + "60616263646566676869" + "70717273747576777879" + "80818283848586878889" + "90919293949596979899"; + +static inline void PrintTinyInt( char*& buf, uint64_t v ) +{ + assert( v < 100 ); + if( v >= 10 ) + { + *buf++ = '0' + v/10; + } + *buf++ = '0' + v%10; +} + +static inline void PrintTinyInt0( char*& buf, uint64_t v ) +{ + assert( v < 100 ); + if( v >= 10 ) + { + *buf++ = '0' + v/10; + } + else + { + *buf++ = '0'; + } + *buf++ = '0' + v%10; +} + +static inline void PrintSmallInt( char*& buf, uint64_t v ) +{ + assert( v < 1000 ); + if( v >= 100 ) + { + memcpy( buf, IntTable100 + v/10*2, 2 ); + buf += 2; + } + else if( v >= 10 ) + { + *buf++ = '0' + v/10; + } + *buf++ = '0' + v%10; +} + +static inline void PrintSmallInt0( char*& buf, uint64_t v ) +{ + assert( v < 1000 ); + if( v >= 100 ) + { + memcpy( buf, IntTable100 + v/10*2, 2 ); + buf += 2; + } + else if( v >= 10 ) + { + *buf++ = '0'; + *buf++ = '0' + v/10; + } + else + { + memcpy( buf, "00", 2 ); + buf += 2; + } + *buf++ = '0' + v%10; +} + +static inline void PrintFrac00( char*& buf, uint64_t v ) +{ + *buf++ = '.'; + v += 5; + if( v/10%10 == 0 ) + { + *buf++ = '0' + v/100; + } + else + { + memcpy( buf, IntTable100 + v/10*2, 2 ); + buf += 2; + } +} + +static inline void PrintFrac0( char*& buf, uint64_t v ) +{ + *buf++ = '.'; + *buf++ = '0' + (v+50)/100; +} + +static inline void PrintSmallIntFrac( char*& buf, uint64_t v ) +{ + uint64_t in = v / 1000; + uint64_t fr = v % 1000; + if( fr >= 995 ) + { + if( in < 999 ) + { + PrintSmallInt( buf, in+1 ); + } + else + { + memcpy( buf, "1000", 4 ); + buf += 4; + } + } + else + { + PrintSmallInt( buf, in ); + if( fr > 5 ) + { + PrintFrac00( buf, fr ); + } + } +} + +static inline void PrintSecondsFrac( char*& buf, uint64_t v ) +{ + uint64_t in = v / 1000; + uint64_t fr = v % 1000; + if( fr >= 950 ) + { + PrintTinyInt0( buf, in+1 ); + } + else + { + PrintTinyInt0( buf, in ); + if( fr > 50 ) + { + PrintFrac0( buf, fr ); + } + } +} + +const char* TimeToString( int64_t _ns ) +{ + enum { Pool = 8 }; + static char bufpool[Pool][64]; + static int bufsel = 0; + char* buf = bufpool[bufsel]; + char* bufstart = buf; + bufsel = ( bufsel + 1 ) % Pool; + + uint64_t ns; + if( _ns < 0 ) + { + *buf = '-'; + buf++; + ns = -_ns; + } + else + { + ns = _ns; + } + + if( ns < 1000 ) + { + PrintSmallInt( buf, ns ); + memcpy( buf, " ns", 4 ); + } + else if( ns < 1000ll * 1000 ) + { + PrintSmallIntFrac( buf, ns ); + memcpy( buf, " \xce\xbcs", 5 ); + } + else if( ns < 1000ll * 1000 * 1000 ) + { + PrintSmallIntFrac( buf, ns / 1000 ); + memcpy( buf, " ms", 4 ); + } + else if( ns < 1000ll * 1000 * 1000 * 60 ) + { + PrintSmallIntFrac( buf, ns / ( 1000ll * 1000 ) ); + memcpy( buf, " s", 3 ); + } + else if( ns < 1000ll * 1000 * 1000 * 60 * 60 ) + { + const auto m = int64_t( ns / ( 1000ll * 1000 * 1000 * 60 ) ); + const auto s = int64_t( ns - m * ( 1000ll * 1000 * 1000 * 60 ) ) / ( 1000ll * 1000 ); + PrintTinyInt( buf, m ); + *buf++ = ':'; + PrintSecondsFrac( buf, s ); + *buf++ = '\0'; + } + else if( ns < 1000ll * 1000 * 1000 * 60 * 60 * 24 ) + { + const auto h = int64_t( ns / ( 1000ll * 1000 * 1000 * 60 * 60 ) ); + const auto m = int64_t( ns / ( 1000ll * 1000 * 1000 * 60 ) - h * 60 ); + const auto s = int64_t( ns / ( 1000ll * 1000 * 1000 ) - h * ( 60 * 60 ) - m * 60 ); + PrintTinyInt( buf, h ); + *buf++ = ':'; + PrintTinyInt0( buf, m ); + *buf++ = ':'; + PrintTinyInt0( buf, s ); + *buf++ = '\0'; + } + else + { + const auto d = int64_t( ns / ( 1000ll * 1000 * 1000 * 60 * 60 * 24 ) ); + const auto h = int64_t( ns / ( 1000ll * 1000 * 1000 * 60 * 60 ) - d * 24 ); + const auto m = int64_t( ns / ( 1000ll * 1000 * 1000 * 60 ) - d * ( 60 * 24 ) - h * 60 ); + const auto s = int64_t( ns / ( 1000ll * 1000 * 1000 ) - d * ( 60 * 60 * 24 ) - h * ( 60 * 60 ) - m * 60 ); + assert( d < 100 ); + PrintTinyInt( buf, d ); + *buf++ = 'd'; + PrintTinyInt0( buf, h ); + *buf++ = ':'; + PrintTinyInt0( buf, m ); + *buf++ = ':'; + PrintTinyInt0( buf, s ); + *buf++ = '\0'; + } + return bufstart; +} + +const char* TimeToStringExact( int64_t _ns ) +{ + enum { Pool = 8 }; + static char bufpool[Pool][64]; + static int bufsel = 0; + char* buf = bufpool[bufsel]; + char* bufstart = buf; + bufsel = ( bufsel + 1 ) % Pool; + + uint64_t ns; + if( _ns < 0 ) + { + *buf = '-'; + buf++; + ns = -_ns; + } + else + { + ns = _ns; + } + + const char* numStart = buf; + + if( ns >= 1000ll * 1000 * 1000 * 60 * 60 * 24 ) + { + const auto d = int64_t( ns / ( 1000ll * 1000 * 1000 * 60 * 60 * 24 ) ); + const auto h = int64_t( ns / ( 1000ll * 1000 * 1000 * 60 * 60 ) - d * 24 ); + const auto m = int64_t( ns / ( 1000ll * 1000 * 1000 * 60 ) - d * ( 60 * 24 ) - h * 60 ); + const auto s = int64_t( ns / ( 1000ll * 1000 * 1000 ) - d * ( 60 * 60 * 24 ) - h * ( 60 * 60 ) - m * 60 ); + if( d < 100 ) + { + PrintTinyInt( buf, d ); + *buf++ = 'd'; + } + else + { + memcpy( buf, "100+d", 5 ); + buf += 5; + } + PrintTinyInt0( buf, h ); + *buf++ = ':'; + PrintTinyInt0( buf, m ); + *buf++ = ':'; + PrintTinyInt0( buf, s ); + ns %= 1000ll * 1000 * 1000; + } + else if( ns >= 1000ll * 1000 * 1000 * 60 * 60 ) + { + const auto h = int64_t( ns / ( 1000ll * 1000 * 1000 * 60 * 60 ) ); + const auto m = int64_t( ns / ( 1000ll * 1000 * 1000 * 60 ) - h * 60 ); + const auto s = int64_t( ns / ( 1000ll * 1000 * 1000 ) - h * ( 60 * 60 ) - m * 60 ); + PrintTinyInt( buf, h ); + *buf++ = ':'; + PrintTinyInt0( buf, m ); + *buf++ = ':'; + PrintTinyInt0( buf, s ); + ns %= 1000ll * 1000 * 1000; + } + else if( ns >= 1000ll * 1000 * 1000 * 60 ) + { + const auto m = int64_t( ns / ( 1000ll * 1000 * 1000 * 60 ) ); + const auto s = int64_t( ns / ( 1000ll * 1000 * 1000 ) - m * 60 ); + PrintTinyInt( buf, m ); + *buf++ = ':'; + PrintTinyInt0( buf, s ); + ns %= 1000ll * 1000 * 1000; + } + else if( ns >= 1000ll * 1000 * 1000 ) + { + PrintTinyInt( buf, int64_t( ns / ( 1000ll * 1000 * 1000 ) ) ); + *buf++ = 's'; + ns %= 1000ll * 1000 * 1000; + } + + if( ns > 0 ) + { + if( buf != numStart ) *buf++ = ' '; + if( ns >= 1000ll * 1000 ) + { + PrintSmallInt0( buf, int64_t( ns / ( 1000ll * 1000 ) ) ); + *buf++ = ','; + ns %= 1000ll * 1000; + } + else + { + memcpy( buf, "000,", 4 ); + buf += 4; + } + if( ns >= 1000ll ) + { + PrintSmallInt0( buf, int64_t( ns / 1000ll ) ); + *buf++ = ','; + ns %= 1000ll; + } + else + { + memcpy( buf, "000,", 4 ); + buf += 4; + } + PrintSmallInt0( buf, ns ); + *buf++ = 'n'; + *buf++ = 's'; + } + else + { + memcpy( buf, "000,000,000ns", 13 ); + buf += 13; + } + + *buf++ = '\0'; + + return bufstart; +} + +const char* MemSizeToString( int64_t val ) +{ + enum { Pool = 8 }; + static char bufpool[Pool][64]; + static int bufsel = 0; + char* buf = bufpool[bufsel]; + bufsel = ( bufsel + 1 ) % Pool; + + const auto aval = llabs( val ); + + if( aval < 10000ll ) + { + sprintf( buf, "%" PRIi64 " bytes", val ); + return buf; + } + + enum class Unit + { + Kilobyte, + Megabyte, + Gigabyte, + Terabyte + }; + Unit unit; + + char* ptr; + if( aval < 10000ll * 1024 ) + { + ptr = PrintFloat( buf, buf+64, val / 1024., 2 ); + unit = Unit::Kilobyte; + } + else if( aval < 10000ll * 1024 * 1024 ) + { + ptr = PrintFloat( buf, buf+64, val / ( 1024. * 1024 ), 2 ); + unit = Unit::Megabyte; + } + else if( aval < 10000ll * 1024 * 1024 * 1024 ) + { + ptr = PrintFloat( buf, buf+64, val / ( 1024. * 1024 * 1024 ), 2 ); + unit = Unit::Gigabyte; + } + else + { + ptr = PrintFloat( buf, buf+64, val / ( 1024. * 1024 * 1024 * 1024 ), 2 ); + unit = Unit::Terabyte; + } + + ptr--; + while( ptr >= buf && *ptr == '0' ) ptr--; + if( *ptr != '.' ) ptr++; + + *ptr++ = ' '; + switch( unit ) + { + case Unit::Kilobyte: + *ptr++ = 'K'; + break; + case Unit::Megabyte: + *ptr++ = 'M'; + break; + case Unit::Gigabyte: + *ptr++ = 'G'; + break; + case Unit::Terabyte: + *ptr++ = 'T'; + break; + default: + assert( false ); + break; + } + *ptr++ = 'B'; + *ptr++ = '\0'; + + return buf; +} + +const char* LocationToString( const char* fn, uint32_t line ) +{ + if( line == 0 ) return fn; + + enum { Pool = 8 }; + static char bufpool[Pool][4096]; + static int bufsel = 0; + char* buf = bufpool[bufsel]; + bufsel = ( bufsel + 1 ) % Pool; + + sprintf( buf, "%s:%i", fn, line ); + return buf; +} + +namespace detail +{ + +char* RealToStringGetBuffer() +{ + enum { Pool = 8 }; + static char bufpool[Pool][64]; + static int bufsel = 0; + char* buf = bufpool[bufsel]; + bufsel = ( bufsel + 1 ) % Pool; + return buf; +} + +} + +} diff --git a/third_party/tracy/server/TracySlab.hpp b/third_party/tracy/server/TracySlab.hpp new file mode 100644 index 0000000000000000000000000000000000000000..f099286d91f69fcffcfcfecb86b32685a5109d19 --- /dev/null +++ b/third_party/tracy/server/TracySlab.hpp @@ -0,0 +1,159 @@ +#ifndef __TRACYSLAB_HPP__ +#define __TRACYSLAB_HPP__ + +#include +#include +#include + +#include "TracyMemory.hpp" +#include "../public/common/TracyForceInline.hpp" + +namespace tracy +{ + +template +class Slab +{ +public: + Slab() + : m_ptr( new char[BlockSize] ) + , m_offset( 0 ) + , m_buffer( { m_ptr } ) + , m_usage( BlockSize ) + { + memUsage.fetch_add( BlockSize, std::memory_order_relaxed ); + } + + ~Slab() + { + memUsage.fetch_sub( m_usage, std::memory_order_relaxed ); + for( auto& v : m_buffer ) + { + delete[] v; + } + } + + tracy_force_inline void* AllocRaw( size_t size ) + { + assert( size <= BlockSize ); + const auto offset = m_offset; + if( offset + size > BlockSize ) + { + return DoAlloc( size ); + } + else + { + void* ret = m_ptr + offset; + m_offset += size; + return ret; + } + } + + template + tracy_force_inline T* AllocInit() + { + const auto size = sizeof( T ); + auto ret = AllocRaw( size ); + new( ret ) T; + return (T*)ret; + } + + template + tracy_force_inline T* AllocInit( size_t sz ) + { + const auto size = sizeof( T ) * sz; + auto ret = AllocRaw( size ); + T* ptr = (T*)ret; + for( size_t i=0; i + tracy_force_inline T* Alloc() + { + return (T*)AllocRaw( sizeof( T ) ); + } + + template + tracy_force_inline T* Alloc( size_t size ) + { + return (T*)AllocRaw( sizeof( T ) * size ); + } + + tracy_force_inline void Unalloc( size_t size ) + { + assert( size <= m_offset ); + m_offset -= size; + } + + tracy_force_inline void* AllocBig( size_t size ) + { + const auto offset = m_offset; + if( offset + size <= BlockSize ) + { + void* ret = m_ptr + offset; + m_offset += size; + return ret; + } + else if( size <= BlockSize && BlockSize - offset <= 1024 ) + { + return DoAlloc( size ); + } + else + { + memUsage.fetch_add( size, std::memory_order_relaxed ); + m_usage += size; + auto ret = new char[size]; + m_buffer.emplace_back( ret ); + return ret; + } + } + + void Reset() + { + if( m_buffer.size() > 1 ) + { + memUsage.fetch_sub( m_usage - BlockSize, std::memory_order_relaxed ); + m_usage = BlockSize; + for( int i=1; i m_buffer; + size_t m_usage; +}; + +} + +#endif diff --git a/third_party/tracy/server/TracySort.hpp b/third_party/tracy/server/TracySort.hpp new file mode 100644 index 0000000000000000000000000000000000000000..fce0c0a7a3ee0e9815cc0edf6cb48004b2048136 --- /dev/null +++ b/third_party/tracy/server/TracySort.hpp @@ -0,0 +1,19 @@ +#ifndef __TRACYSORT_HPP__ +#define __TRACYSORT_HPP__ + +#ifndef NO_PARALLEL_SORT +# if !defined __APPLE__ && !defined __EMSCRIPTEN__ && ( ( defined _MSC_VER && _MSVC_LANG >= 201703L ) || __cplusplus >= 201703L ) +# if __has_include() +# include +# include +# else +# define NO_PARALLEL_SORT +# endif +# else +# define NO_PARALLEL_SORT +# endif +#endif + +#include "tracy_pdqsort.h" + +#endif diff --git a/third_party/tracy/server/TracySortedVector.hpp b/third_party/tracy/server/TracySortedVector.hpp new file mode 100644 index 0000000000000000000000000000000000000000..f157198fc14b42716093cb64ef131516dd6fff71 --- /dev/null +++ b/third_party/tracy/server/TracySortedVector.hpp @@ -0,0 +1,127 @@ +#ifndef __TRACYSORTEDVECTOR_HPP__ +#define __TRACYSORTEDVECTOR_HPP__ + +#include "TracySort.hpp" +#include "TracyVector.hpp" + +namespace tracy +{ + +#pragma pack( push, 1 ) +template> +class SortedVector +{ +public: + using iterator = T*; + using const_iterator = const T*; + + tracy_force_inline SortedVector() + : sortedEnd( 0 ) + {} + + SortedVector( const SortedVector& ) = delete; + tracy_force_inline SortedVector( SortedVector&& src ) noexcept + : v( std::move( src.v ) ) + , sortedEnd( src.sortedEnd ) + { + } + + tracy_force_inline SortedVector( const T& value ) + : v( value ) + , sortedEnd( 0 ) + { + } + + SortedVector& operator=( const SortedVector& ) = delete; + tracy_force_inline SortedVector& operator=( SortedVector&& src ) noexcept + { + v = std::move( src.v ); + sortedEnd = src.sortedEnd; + return *this; + } + + tracy_force_inline void swap( SortedVector& other ) + { + v.swap( other.v ); + std::swap( sortedEnd, other.sortedEnd ); + } + + tracy_force_inline bool empty() const { return v.empty(); } + tracy_force_inline size_t size() const { return v.size(); } + tracy_force_inline bool is_sorted() const { return sortedEnd == 0; } + + tracy_force_inline T* data() { return v.data(); } + tracy_force_inline const T* data() const { return v.data(); }; + + tracy_force_inline T* begin() { return v.begin(); } + tracy_force_inline const T* begin() const { return v.begin(); } + tracy_force_inline T* end() { return v.end(); } + tracy_force_inline const T* end() const { return v.end(); } + + tracy_force_inline T& front() { return v.front(); } + tracy_force_inline const T& front() const { return v.front(); } + + tracy_force_inline T& back() { return v.back(); } + tracy_force_inline const T& back() const { return v.back(); } + + tracy_force_inline T& operator[]( size_t idx ) { return v[idx]; } + tracy_force_inline const T& operator[]( size_t idx ) const { return v[idx]; } + + tracy_force_inline void push_back( const T& val ) { push_back( val, CompareDefault() ); } + + template + tracy_force_inline void push_back( const T& val, Compare comp ) + { + if( sortedEnd == 0 && !v.empty() && !comp( v.back(), val ) ) + { + sortedEnd = (uint32_t)v.size(); + } + v.push_back( val ); + } + + tracy_force_inline void reserve( size_t cap ) { v.reserve( cap ); } + template + tracy_force_inline void reserve_exact( uint32_t sz, Slab& slab ) { v.reserve_exact( sz, slab ); } + + tracy_force_inline void clear() { v.clear(); sortedEnd = 0; } + + tracy_force_inline T* erase( T* begin, T* end ) + { + assert( is_sorted() ); + return v.erase( begin, end ); + } + + tracy_force_inline void sort() { sort( CompareDefault() ); } + tracy_force_inline void ensure_sorted() { if( !is_sorted() ) sort(); } + + template + void sort( Compare comp ) + { + assert( !is_sorted() ); + const auto sb = v.begin(); + const auto se = sb + sortedEnd; + const auto sl = se - 1; + const auto ue = v.end(); +#ifdef NO_PARALLEL_SORT + pdqsort_branchless( se, ue, comp ); +#else + std::sort( std::execution::par_unseq, se, ue, comp ); +#endif + const auto ss = std::lower_bound( sb, se, *se, comp ); + const auto uu = std::lower_bound( se, ue, *sl, comp ); + std::inplace_merge( ss, se, uu, comp ); + sortedEnd = 0; + } + +private: + Vector v; + uint32_t sortedEnd; +}; + +#pragma pack( pop ) + +enum { SortedVectorSize = sizeof( SortedVector ) }; + +} + +#endif diff --git a/third_party/tracy/server/TracyStringDiscovery.hpp b/third_party/tracy/server/TracyStringDiscovery.hpp new file mode 100644 index 0000000000000000000000000000000000000000..449b6f093bcc3da23542e59dcb8873f2065abde4 --- /dev/null +++ b/third_party/tracy/server/TracyStringDiscovery.hpp @@ -0,0 +1,88 @@ +#ifndef __TRACYSTRINGDISCOVERY_HPP__ +#define __TRACYSTRINGDISCOVERY_HPP__ + +#include "../public/common/TracyForceInline.hpp" +#include "tracy_robin_hood.h" +#include "TracyCharUtil.hpp" +#include "TracyEvent.hpp" +#include "TracyVector.hpp" + +namespace tracy +{ + +template +class StringDiscovery +{ +public: + tracy_force_inline Vector& Data() { return m_data; } + tracy_force_inline const Vector& Data() const { return m_data; } + + tracy_force_inline bool IsPending() const { return !m_pending.empty(); } + + // Merge( destination, postponed ) + template + tracy_force_inline void StringDiscovered( uint64_t name, const StringLocation& sl, U& stringMap, std::function Merge ) + { + auto pit = m_pending.find( name ); + assert( pit != m_pending.end() ); + + auto it = m_rev.find( sl.ptr ); + if( it == m_rev.end() ) + { + m_map.emplace( name, pit->second ); + m_rev.emplace( sl.ptr, pit->second ); + m_data.push_back( pit->second ); + stringMap.emplace( name, sl.ptr ); + } + else + { + auto item = it->second; + m_map.emplace( name, item ); + Merge( item, pit->second ); + } + + m_pending.erase( pit ); + } + + tracy_force_inline T Retrieve( uint64_t name, const std::function& Create, const std::function& Query ) + { + auto it = m_map.find( name ); + if( it == m_map.end() ) + { + auto pit = m_pending.find( name ); + if( pit == m_pending.end() ) + { + T item = Create( name ); + if( item ) + { + m_pending.emplace( name, item ); + Query( name ); + } + return item; + } + else + { + return pit->second; + } + } + else + { + return it->second; + } + } + + tracy_force_inline void AddExternal( const T& val ) + { + m_data.push_back( val ); + } + +private: + Vector m_data; + unordered_flat_map m_pending; + unordered_flat_map m_map; + unordered_flat_map m_rev; +}; + +} + +#endif diff --git a/third_party/tracy/server/TracySysUtil.cpp b/third_party/tracy/server/TracySysUtil.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0f3bb36c0d3473733af5e7ea751e34f8190aab45 --- /dev/null +++ b/third_party/tracy/server/TracySysUtil.cpp @@ -0,0 +1,43 @@ + + +#include "TracySysUtil.hpp" + +#ifdef _WIN32 +# include +#elif defined __linux__ +# include +#elif defined __APPLE__ || defined BSD +# include +# include +#endif + +namespace tracy +{ + +size_t GetPhysicalMemorySize() +{ +#ifdef _WIN32 + MEMORYSTATUSEX statex; + statex.dwLength = sizeof( statex ); + GlobalMemoryStatusEx( &statex ); + return statex.ullTotalPhys; +#elif defined __linux__ + struct sysinfo sysInfo; + sysinfo( &sysInfo ); + return sysInfo.totalram; +#elif defined __APPLE__ + size_t memSize; + size_t sz = sizeof( memSize ); + sysctlbyname( "hw.memsize", &memSize, &sz, nullptr, 0 ); + return memSize; +#elif defined BSD + size_t memSize; + size_t sz = sizeof( memSize ); + sysctlbyname( "hw.physmem", &memSize, &sz, nullptr, 0 ); + return memSize; +#else + return 0; +#endif +} + +} diff --git a/third_party/tracy/server/TracySysUtil.hpp b/third_party/tracy/server/TracySysUtil.hpp new file mode 100644 index 0000000000000000000000000000000000000000..523ca577b580c34a8894c4103614e3d317d0e4eb --- /dev/null +++ b/third_party/tracy/server/TracySysUtil.hpp @@ -0,0 +1,13 @@ +#ifndef __TRACYSYSUTIL_HPP__ +#define __TRACYSYSUTIL_HPP__ + +#include + +namespace tracy +{ + +size_t GetPhysicalMemorySize(); + +} + +#endif diff --git a/third_party/tracy/server/TracyTaskDispatch.cpp b/third_party/tracy/server/TracyTaskDispatch.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ef51d244ace0cfef1e2600b33ba8b7b06730d140 --- /dev/null +++ b/third_party/tracy/server/TracyTaskDispatch.cpp @@ -0,0 +1,88 @@ +#include +#include + +#include "../public/common/TracySystem.hpp" +#include "TracyTaskDispatch.hpp" + +namespace tracy +{ + +TaskDispatch::TaskDispatch( size_t workers, const char* name ) + : m_exit( false ) + , m_jobs( 0 ) +{ + m_workers.reserve( workers ); + for( size_t i=0; i& f ) +{ + std::lock_guard lock( m_queueLock ); + m_queue.emplace_back( f ); + m_cvWork.notify_one(); +} + +void TaskDispatch::Queue( std::function&& f ) +{ + std::lock_guard lock( m_queueLock ); + m_queue.emplace_back( std::move( f ) ); + m_cvWork.notify_one(); +} + +void TaskDispatch::Sync() +{ + std::unique_lock lock( m_queueLock ); + while( !m_queue.empty() ) + { + auto f = m_queue.back(); + m_queue.pop_back(); + lock.unlock(); + f(); + lock.lock(); + } + m_cvJobs.wait( lock, [this]{ return m_jobs == 0; } ); +} + +void TaskDispatch::Worker() +{ + for(;;) + { + std::unique_lock lock( m_queueLock ); + m_cvWork.wait( lock, [this]{ return !m_queue.empty() || m_exit.load( std::memory_order_acquire ); } ); + if( m_exit.load( std::memory_order_acquire ) ) return; + auto f = m_queue.back(); + m_queue.pop_back(); + m_jobs++; + lock.unlock(); + f(); + lock.lock(); + m_jobs--; + if( m_jobs == 0 && m_queue.empty() ) m_cvJobs.notify_one(); + lock.unlock(); + } +} + +void TaskDispatch::SetName( const char* name, size_t num ) +{ + char tmp[128]; + snprintf( tmp, sizeof( tmp ), "%s #%zu", name, num ); + SetThreadName( tmp ); +} + +} diff --git a/third_party/tracy/server/TracyTaskDispatch.hpp b/third_party/tracy/server/TracyTaskDispatch.hpp new file mode 100644 index 0000000000000000000000000000000000000000..3a7e74ef7c9dbeb1d14b3f6c08e5c2333611745c --- /dev/null +++ b/third_party/tracy/server/TracyTaskDispatch.hpp @@ -0,0 +1,40 @@ +#ifndef __TRACYTASKDISPATCH_HPP__ +#define __TRACYTASKDISPATCH_HPP__ + +#include +#include +#include +#include +#include +#include + +namespace tracy +{ + +class TaskDispatch +{ +public: + TaskDispatch( size_t workers, const char* name ); + ~TaskDispatch(); + + void Queue( const std::function& f ); + void Queue( std::function&& f ); + + void Sync(); + +private: + void Worker(); + void SetName( const char* name, size_t num ); + + std::vector> m_queue; + std::mutex m_queueLock; + std::condition_variable m_cvWork, m_cvJobs; + std::atomic m_exit; + size_t m_jobs; + + std::vector m_workers; +}; + +} + +#endif diff --git a/third_party/tracy/server/TracyTextureCompression.cpp b/third_party/tracy/server/TracyTextureCompression.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c9cedab90a89131d0c6debceffdf6aa6ae935165 --- /dev/null +++ b/third_party/tracy/server/TracyTextureCompression.cpp @@ -0,0 +1,211 @@ +#include "../zstd/zstd.h" + +#include "TracyEvent.hpp" +#include "TracyTextureCompression.hpp" + +namespace tracy +{ + +TextureCompression::TextureCompression() + : m_buf( nullptr ) + , m_bufSize( 0 ) + , m_cctx( ZSTD_createCCtx() ) + , m_dctx( ZSTD_createDCtx() ) + , m_dict( nullptr ) +{ +} + +TextureCompression::~TextureCompression() +{ + delete[] m_buf; + ZSTD_freeCCtx( m_cctx ); + ZSTD_freeDCtx( m_dctx ); + ZSTD_freeDDict( m_dict ); +} + +uint32_t TextureCompression::Pack( struct ZSTD_CCtx_s* ctx, char*& buf, size_t& bufsz, const char* image, uint32_t inBytes ) +{ + const auto maxout = ZSTD_COMPRESSBOUND( inBytes ); + if( bufsz < maxout ) + { + bufsz = maxout; + delete[] buf; + buf = new char[maxout]; + } + assert( ctx ); + auto ret = (uint32_t)ZSTD_compressCCtx( ctx, buf, maxout, image, inBytes, 3 ); +#ifndef TRACY_NO_STATISTICS + m_inputBytes.fetch_add( inBytes, std::memory_order_relaxed ); + m_outputBytes.fetch_add( ret, std::memory_order_relaxed ); +#endif + return ret; +} + +uint32_t TextureCompression::Pack( struct ZSTD_CCtx_s* ctx, const struct ZSTD_CDict_s* dict, char*& buf, size_t& bufsz, const char* image, uint32_t inBytes ) +{ + const auto maxout = ZSTD_COMPRESSBOUND( inBytes ); + if( bufsz < maxout ) + { + bufsz = maxout; + delete[] buf; + buf = new char[maxout]; + } + assert( ctx ); + auto ret = (uint32_t)ZSTD_compress_usingCDict( ctx, buf, maxout, image, inBytes, dict ); +#ifndef TRACY_NO_STATISTICS + m_inputBytes.fetch_add( inBytes, std::memory_order_relaxed ); + m_outputBytes.fetch_add( ret, std::memory_order_relaxed ); +#endif + return ret; +} + +const char* TextureCompression::Unpack( const FrameImage& image ) +{ + const auto outsz = size_t( image.w ) * size_t( image.h ) / 2; + if( m_bufSize < outsz ) + { + m_bufSize = outsz; + delete[] m_buf; + m_buf = new char[outsz]; + } + assert( m_dctx ); + if( m_dict ) + { + ZSTD_decompress_usingDDict( m_dctx, m_buf, outsz, image.ptr, image.csz, m_dict ); + } + else + { + ZSTD_decompressDCtx( m_dctx, m_buf, outsz, image.ptr, image.csz ); + } + return m_buf; +} + +static constexpr uint8_t Dxtc4To3Table[256] = { + 85, 84, 86, 86, 81, 80, 82, 82, 89, 88, 90, 90, 89, 88, 90, 90, + 69, 68, 70, 70, 65, 64, 66, 66, 73, 72, 74, 74, 73, 72, 74, 74, + 101, 100, 102, 102, 97, 96, 98, 98, 105, 104, 106, 106, 105, 104, 106, 106, + 101, 100, 102, 102, 97, 96, 98, 98, 105, 104, 106, 106, 105, 104, 106, 106, + 21, 20, 22, 22, 17, 16, 18, 18, 25, 24, 26, 26, 25, 24, 26, 26, + 5, 4, 6, 6, 1, 0, 2, 2, 9, 8, 10, 10, 9, 8, 10, 10, + 37, 36, 38, 38, 33, 32, 34, 34, 41, 40, 42, 42, 41, 40, 42, 42, + 37, 36, 38, 38, 33, 32, 34, 34, 41, 40, 42, 42, 41, 40, 42, 42, + 149, 148, 150, 150, 145, 144, 146, 146, 153, 152, 154, 154, 153, 152, 154, 154, + 133, 132, 134, 134, 129, 128, 130, 130, 137, 136, 138, 138, 137, 136, 138, 138, + 165, 164, 166, 166, 161, 160, 162, 162, 169, 168, 170, 170, 169, 168, 170, 170, + 165, 164, 166, 166, 161, 160, 162, 162, 169, 168, 170, 170, 169, 168, 170, 170, + 149, 148, 150, 150, 145, 144, 146, 146, 153, 152, 154, 154, 153, 152, 154, 154, + 133, 132, 134, 134, 129, 128, 130, 130, 137, 136, 138, 138, 137, 136, 138, 138, + 165, 164, 166, 166, 161, 160, 162, 162, 169, 168, 170, 170, 169, 168, 170, 170, + 165, 164, 166, 166, 161, 160, 162, 162, 169, 168, 170, 170, 169, 168, 170, 170 +}; + +static tracy_force_inline int max3( int a, int b, int c ) +{ + if( a > b ) + { + return a > c ? a : c; + } + else + { + return b > c ? b : c; + } +} + +static constexpr int TrTbl1[] = { 12, 12, 12, 12, 6, 6, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }; +static constexpr int TrTbl2[] = { 12, 12, 6, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 }; +static constexpr int TrTbl3[] = { 48, 48, 48, 32, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24 }; + +void TextureCompression::Rdo( char* data, size_t blocks ) +{ + assert( blocks > 0 ); + do + { + uint64_t blk; + memcpy( &blk, data, 8 ); + + uint32_t idx = blk >> 32; + if( idx == 0x55555555 ) + { + data += 8; + continue; + } + + uint16_t c0 = blk & 0xFFFF; + uint16_t c1 = ( blk >> 16 ) & 0xFFFF; + + const int r0b = c0 & 0xF800; + const int g0b = c0 & 0x07E0; + const int b0b = c0 & 0x001F; + + const int r1b = c1 & 0xF800; + const int g1b = c1 & 0x07E0; + const int b1b = c1 & 0x001F; + + const int r0 = ( r0b >> 8 ) | ( r0b >> 13 ); + const int g0 = ( g0b >> 3 ) | ( g0b >> 9 ); + const int b0 = ( b0b << 3 ) | ( b0b >> 2 ); + + const int r1 = ( r1b >> 8 ) | ( r1b >> 13 ); + const int g1 = ( g1b >> 3 ) | ( g1b >> 9 ); + const int b1 = ( b1b << 3 ) | ( b1b >> 2 ); + + const int dr = abs( r0 - r1 ); + const int dg = abs( g0 - g1 ); + const int db = abs( b0 - b1 ); + + const int maxChan1 = max3( r0-1, g0, b0-2 ); + const int maxDelta1 = max3( dr-1, dg, db-2 ); + const int tr1 = TrTbl1[maxChan1 / 4]; + if( maxDelta1 <= tr1 ) + { + uint64_t blk = + ( ( ( r0b + r1b ) >> 1 ) & 0xF800 ) | + ( ( ( g0b + g1b ) >> 1 ) & 0x07E0 ) | + ( ( ( b0b + b1b ) >> 1 ) ); + memcpy( data, &blk, 8 ); + } + else + { + const int maxChan23 = max3( r0-2, g0, b0-5 ); + const int maxDelta23 = max3( dr-2, dg, db-5 ); + const int tr2 = TrTbl2[maxChan23 / 16]; + if( maxDelta23 <= tr2 ) + { + idx &= 0x55555555; + memcpy( data+4, &idx, 4 ); + } + else + { + const int tr3 = TrTbl3[maxChan23 / 16]; + if( maxDelta23 <= tr3 ) + { + uint64_t c = c1 | ( uint64_t( c0 ) << 16 ); + for( int k=0; k<4; k++ ) c |= uint64_t( Dxtc4To3Table[(idx >> (k*8)) & 0xFF] ) << ( 32 + k*8 ); + memcpy( data, &c, 8 ); + } + } + } + + data += 8; + } + while( --blocks ); +} + +void TextureCompression::FixOrder( char* data, size_t blocks ) +{ + assert( blocks > 0 ); + do + { + uint32_t tmp; + memcpy( &tmp, data+4, 4 ); + tmp = ~tmp; + uint32_t t0 = tmp & 0x55555555; + uint32_t t1 = tmp & 0xAAAAAAAA; + tmp = ( ( t0 << 1 ) | ( t1 >> 1 ) ) ^ t1; + memcpy( data+4, &tmp, 4 ); + data += 8; + } + while( --blocks ); +} + +} diff --git a/third_party/tracy/server/TracyTextureCompression.hpp b/third_party/tracy/server/TracyTextureCompression.hpp new file mode 100644 index 0000000000000000000000000000000000000000..d5db71f475b2c0939e653fd378227e720a6725d0 --- /dev/null +++ b/third_party/tracy/server/TracyTextureCompression.hpp @@ -0,0 +1,64 @@ +#ifndef __TRACY__TEXTURECOMPRESSION_HPP__ +#define __TRACY__TEXTURECOMPRESSION_HPP__ + +#include + +#include +#include +#include + +#include "TracySlab.hpp" + +struct ZSTD_CCtx_s; +struct ZSTD_DCtx_s; +struct ZSTD_CDict_s; +struct ZSTD_DDict_s; + +namespace tracy +{ + +struct FrameImage; + +class TextureCompression +{ +public: + TextureCompression(); + ~TextureCompression(); + + void SetDict( struct ZSTD_DDict_s* dict ) { m_dict = dict; } + + uint32_t Pack( struct ZSTD_CCtx_s* ctx, char*& buf, size_t& bufsz, const char* image, uint32_t inBytes ); + uint32_t Pack( struct ZSTD_CCtx_s* ctx, const struct ZSTD_CDict_s* dict, char*& buf, size_t& bufsz, const char* image, uint32_t inBytes ); + + template + const char* Pack( const char* image, uint32_t inBytes, uint32_t& csz, Slab& slab ) + { + const auto outsz = Pack( m_cctx, m_buf, m_bufSize, image, inBytes ); + auto ptr = (char*)slab.AllocBig( outsz ); + memcpy( ptr, m_buf, outsz ); + csz = outsz; + return ptr; + } + + const char* Unpack( const FrameImage& image ); + + void Rdo( char* data, size_t blocks ); + void FixOrder( char* data, size_t blocks ); + + uint64_t GetInputBytesCount() const { return m_inputBytes.load( std::memory_order_relaxed ); } + uint64_t GetOutputBytesCount() const { return m_outputBytes.load( std::memory_order_relaxed ); } + +private: + char* m_buf; + size_t m_bufSize; + struct ZSTD_CCtx_s* m_cctx; + struct ZSTD_DCtx_s* m_dctx; + struct ZSTD_DDict_s* m_dict; + + std::atomic m_inputBytes { 0 }; + std::atomic m_outputBytes { 0 }; +}; + +} + +#endif diff --git a/third_party/tracy/server/TracyThreadCompress.cpp b/third_party/tracy/server/TracyThreadCompress.cpp new file mode 100644 index 0000000000000000000000000000000000000000..566d6741aea49f8dbde6fa198f60b59fc7dd6439 --- /dev/null +++ b/third_party/tracy/server/TracyThreadCompress.cpp @@ -0,0 +1,72 @@ +#include + +#include "TracyFileRead.hpp" +#include "TracyFileWrite.hpp" +#include "TracyThreadCompress.hpp" + +namespace tracy +{ + +ThreadCompress::ThreadCompress() + : m_threadLast( std::numeric_limits::max(), 0 ) +{ +} + +void ThreadCompress::InitZero() +{ + assert( m_threadExpand.empty() ); + m_threadExpand.push_back( 0 ); +} + +void ThreadCompress::Load( FileRead& f ) +{ + assert( m_threadExpand.empty() ); + assert( m_threadMap.empty() ); + + uint64_t sz; + f.Read( sz ); + if( sz != 0 ) + { + m_threadExpand.reserve_and_use( sz ); + f.Read( m_threadExpand.data(), sizeof( uint64_t ) * sz ); + m_threadMap.reserve( sz ); + for( size_t i=0; isecond; + return it->second; + } + else + { + return CompressThreadNew( thread ); + } +} + +uint16_t ThreadCompress::CompressThreadNew( uint64_t thread ) +{ + auto sz = m_threadExpand.size(); + m_threadExpand.push_back( thread ); + m_threadMap.emplace( thread, sz ); + m_threadLast.first = thread; + m_threadLast.second = sz; + return sz; +} + +} diff --git a/third_party/tracy/server/TracyVarArray.hpp b/third_party/tracy/server/TracyVarArray.hpp new file mode 100644 index 0000000000000000000000000000000000000000..83515794dddbd4b1b682efb1d167d0cc4e43a71d --- /dev/null +++ b/third_party/tracy/server/TracyVarArray.hpp @@ -0,0 +1,97 @@ +#ifndef __TRACYVARARRAY_HPP__ +#define __TRACYVARARRAY_HPP__ + +#include +#include + +#define XXH_INLINE_ALL +#include "tracy_xxhash.h" + +#include "../public/common/TracyForceInline.hpp" +#include "TracyCharUtil.hpp" +#include "TracyEvent.hpp" +#include "TracyMemory.hpp" +#include "TracyShortPtr.hpp" + +namespace tracy +{ + +#pragma pack( push, 1 ) +template +class VarArray +{ +public: + VarArray( uint16_t size, const T* data ) + : m_size( size ) + , m_ptr( data ) + { + CalcHash(); + } + + VarArray( const VarArray& ) = delete; + VarArray( VarArray&& ) = delete; + + VarArray& operator=( const VarArray& ) = delete; + VarArray& operator=( VarArray&& ) = delete; + + tracy_force_inline uint32_t get_hash() const { return m_hash; } + + tracy_force_inline bool empty() const { return m_size == 0; } + tracy_force_inline uint16_t size() const { return m_size; } + + tracy_force_inline const T* data() const { return m_ptr; }; + + tracy_force_inline const T* begin() const { return m_ptr; } + tracy_force_inline const T* end() const { return m_ptr + m_size; } + + tracy_force_inline const T& front() const { assert( m_size > 0 ); return m_ptr[0]; } + tracy_force_inline const T& back() const { assert( m_size > 0 ); return m_ptr[m_size - 1]; } + + tracy_force_inline const T& operator[]( size_t idx ) const { return m_ptr[idx]; } + +private: + tracy_force_inline void CalcHash(); + + uint16_t m_size; + uint32_t m_hash; + const short_ptr m_ptr; +}; +#pragma pack( pop ) + +enum { VarArraySize = sizeof( VarArray ) }; + + +template +inline void VarArray::CalcHash() +{ + m_hash = uint32_t( XXH3_64bits( m_ptr.get(), m_size * sizeof( T ) ) ); +} + +template +static inline bool Compare( const VarArray& lhs, const VarArray& rhs ) +{ + if( lhs.size() != rhs.size() || lhs.get_hash() != rhs.get_hash() ) return false; + return memcmp( lhs.data(), rhs.data(), lhs.size() * sizeof( T ) ) == 0; +} + +template +struct VarArrayHasher +{ + size_t operator()( const VarArray* arr ) const + { + return arr->get_hash(); + } +}; + +template +struct VarArrayComparator +{ + bool operator()( const VarArray* lhs, const VarArray* rhs ) const + { + return Compare( *lhs, *rhs ); + } +}; + +} + +#endif diff --git a/third_party/tracy/server/TracyVector.hpp b/third_party/tracy/server/TracyVector.hpp new file mode 100644 index 0000000000000000000000000000000000000000..c44a4417f2db3b2fb3cebcef0da1c12860f91c43 --- /dev/null +++ b/third_party/tracy/server/TracyVector.hpp @@ -0,0 +1,357 @@ +#ifndef __TRACYVECTOR_HPP__ +#define __TRACYVECTOR_HPP__ + +#include +#include +#include +#include +#include +#include + +#include "../public/common/TracyForceInline.hpp" +#include "TracyMemory.hpp" +#include "TracyPopcnt.hpp" +#include "TracyShortPtr.hpp" +#include "TracySlab.hpp" + +//#define TRACY_VECTOR_DEBUG + +namespace tracy +{ + +#pragma pack( push, 1 ) +template +class Vector +{ + constexpr uint8_t MaxCapacity() { return 0x7F; } + +public: + using iterator = T*; + using const_iterator = const T*; + + tracy_force_inline Vector() + { + memset( (char*)this, 0, sizeof( Vector ) ); + } + + Vector( const Vector& ) = delete; + tracy_force_inline Vector( Vector&& src ) noexcept + { + memcpy( (char*)this, &src, sizeof( Vector ) ); + memset( (char*)&src, 0, sizeof( Vector ) ); + } + + tracy_force_inline Vector( const T& value ) + : m_ptr( (T*)malloc( sizeof( T ) ) ) + , m_size( 1 ) + , m_capacity( 0 ) + , m_magic( 0 ) + { + memUsage.fetch_add( sizeof( T ), std::memory_order_relaxed ); + new(m_ptr) T( value ); + } + + tracy_force_inline ~Vector() + { + if( m_capacity != MaxCapacity() && m_ptr ) + { + memUsage.fetch_sub( Capacity() * sizeof( T ), std::memory_order_relaxed ); + free( m_ptr ); + } + } + + Vector& operator=( const Vector& ) = delete; + tracy_force_inline Vector& operator=( Vector&& src ) noexcept + { + if( m_capacity != MaxCapacity() && m_ptr ) + { + memUsage.fetch_sub( Capacity() * sizeof( T ), std::memory_order_relaxed ); + free( m_ptr ); + } + memcpy( (char*)this, &src, sizeof( Vector ) ); + memset( (char*)&src, 0, sizeof( Vector ) ); + return *this; + } + + tracy_force_inline void swap( Vector& other ) + { + uint8_t tmp[sizeof( Vector )]; + memcpy( (char*)tmp, &other, sizeof( Vector ) ); + memcpy( (char*)&other, this, sizeof( Vector ) ); + memcpy( (char*)this, tmp, sizeof( Vector ) ); + } + + tracy_force_inline bool empty() const { return m_size == 0; } + tracy_force_inline size_t size() const { return m_size; } + + tracy_force_inline void set_size( size_t sz ) { assert( m_capacity != MaxCapacity() ); m_size = sz; } + + tracy_force_inline T* data() { return m_ptr; } + tracy_force_inline const T* data() const { return m_ptr; }; + + tracy_force_inline T* begin() { return m_ptr; } + tracy_force_inline const T* begin() const { return m_ptr; } + tracy_force_inline T* end() { return m_ptr + m_size; } + tracy_force_inline const T* end() const { return m_ptr + m_size; } + + tracy_force_inline T& front() { assert( m_size > 0 ); return m_ptr[0]; } + tracy_force_inline const T& front() const { assert( m_size > 0 ); return m_ptr[0]; } + + tracy_force_inline T& back() { assert( m_size > 0 ); return m_ptr[m_size - 1]; } + tracy_force_inline const T& back() const { assert( m_size > 0 ); return m_ptr[m_size - 1]; } + + tracy_force_inline T& operator[]( size_t idx ) { return m_ptr[idx]; } + tracy_force_inline const T& operator[]( size_t idx ) const { return m_ptr[idx]; } + + tracy_force_inline void push_back( const T& v ) + { + assert( m_capacity != MaxCapacity() ); + if( m_size == Capacity() ) AllocMore(); + new(m_ptr+m_size) T( v ); + m_size++; + } + + tracy_force_inline void push_back_non_empty( const T& v ) + { + assert( m_capacity != MaxCapacity() ); + assert( m_ptr ); + if( m_size == CapacityNoNullptrCheck() ) AllocMore(); + new(m_ptr+m_size) T( v ); + m_size++; + } + + tracy_force_inline void push_back_no_space_check( const T& v ) + { + assert( m_capacity != MaxCapacity() ); + assert( m_size < Capacity() ); + new(m_ptr+m_size) T( v ); + m_size++; + } + + tracy_force_inline void push_back( T&& v ) + { + assert( m_capacity != MaxCapacity() ); + if( m_size == Capacity() ) AllocMore(); + new(m_ptr+m_size) T( std::move( v ) ); + m_size++; + } + + tracy_force_inline T& push_next() + { + assert( m_capacity != MaxCapacity() ); + if( m_size == Capacity() ) AllocMore(); + new(m_ptr+m_size) T(); + return m_ptr[m_size++]; + } + + tracy_force_inline T& push_next_non_empty() + { + assert( m_capacity != MaxCapacity() ); + assert( m_ptr ); + if( m_size == CapacityNoNullptrCheck() ) AllocMore(); + new(m_ptr+m_size) T(); + return m_ptr[m_size++]; + } + + tracy_force_inline T& push_next_no_space_check() + { + assert( m_capacity != MaxCapacity() ); + assert( m_size < Capacity() ); + new(m_ptr+m_size) T(); + return m_ptr[m_size++]; + } + + T* insert( T* it, const T& v ) + { + assert( m_capacity != MaxCapacity() ); + assert( it >= m_ptr && it <= m_ptr + m_size ); + const auto dist = it - m_ptr; + if( m_size == Capacity() ) AllocMore(); + if( dist != m_size ) memmove( m_ptr + dist + 1, m_ptr + dist, ( m_size - dist ) * sizeof( T ) ); + m_size++; + new(m_ptr+dist) T( v ); + m_ptr[dist] = v; + return m_ptr + dist; + } + + T* insert( T* it, T&& v ) + { + assert( m_capacity != MaxCapacity() ); + assert( it >= m_ptr && it <= m_ptr + m_size ); + const auto dist = it - m_ptr; + if( m_size == Capacity() ) AllocMore(); + if( dist != m_size ) memmove( m_ptr + dist + 1, m_ptr + dist, ( m_size - dist ) * sizeof( T ) ); + m_size++; + new(m_ptr+dist) T( std::move( v ) ); + return m_ptr + dist; + } + + void insert( T* it, T* begin, T* end ) + { + assert( m_capacity != MaxCapacity() ); + assert( it >= m_ptr && it <= m_ptr + m_size ); + const auto sz = end - begin; + const auto dist = it - m_ptr; + while( m_size + sz > Capacity() ) AllocMore(); + if( dist != m_size ) memmove( m_ptr + dist + sz, m_ptr + dist, ( m_size - dist ) * sizeof( T ) ); + m_size += sz; + memcpy( m_ptr + dist, begin, sz * sizeof( T ) ); + } + + T* erase( T* it ) + { + assert( m_capacity != MaxCapacity() ); + assert( it >= m_ptr && it <= m_ptr + m_size ); + m_size--; + memmove( it, it+1, ( m_size - ( it - m_ptr ) ) * sizeof( T ) ); + return it; + } + + T* erase( T* begin, T* end ) + { + assert( m_capacity != MaxCapacity() ); + assert( begin >= m_ptr && begin <= m_ptr + m_size ); + assert( end >= m_ptr && end <= m_ptr + m_size ); + assert( begin <= end ); + + const auto dist = end - begin; + if( dist > 0 ) + { + memmove( begin, end, ( m_size - ( end - m_ptr ) ) * sizeof( T ) ); + m_size -= dist; + } + return begin; + } + + tracy_force_inline void pop_back() + { + assert( m_capacity != MaxCapacity() ); + assert( m_size > 0 ); + m_size--; + } + + tracy_force_inline T& back_and_pop() + { + assert( m_capacity != MaxCapacity() ); + assert( m_size > 0 ); + m_size--; + return m_ptr[m_size]; + } + + tracy_force_inline void reserve( size_t cap ) + { + if( cap == 0 || cap <= Capacity() ) return; + reserve_non_zero( cap ); + } + + void reserve_non_zero( size_t cap ) + { + assert( m_capacity != MaxCapacity() ); + cap--; + cap |= cap >> 1; + cap |= cap >> 2; + cap |= cap >> 4; + cap |= cap >> 8; + cap |= cap >> 16; + cap = TracyCountBits( cap ); + memUsage.fetch_add( ( ( 1 << cap ) - Capacity() ) * sizeof( T ), std::memory_order_relaxed ); + m_capacity = cap; + Realloc(); + } + + tracy_force_inline void reserve_and_use( size_t sz ) + { + assert( m_capacity != MaxCapacity() ); + reserve( sz ); + m_size = sz; + } + + template + tracy_force_inline void reserve_exact( uint32_t sz, Slab& slab ) + { + assert( !m_ptr ); + m_capacity = MaxCapacity(); + m_size = sz; + m_ptr = (T*)slab.AllocBig( sizeof( T ) * sz ); + } + + tracy_force_inline void clear() + { + assert( m_capacity != MaxCapacity() ); + m_size = 0; + } + + tracy_force_inline bool is_magic() const { return m_magic; } + tracy_force_inline void set_magic() { assert( !m_magic ); m_magic = 1; } + +private: + tracy_no_inline void AllocMore() + { + assert( m_capacity != MaxCapacity() ); + + if( m_ptr == nullptr ) + { + memUsage.fetch_add( sizeof( T ), std::memory_order_relaxed ); + m_ptr = (T*)malloc( sizeof( T ) ); + m_capacity = 0; + } + else + { + memUsage.fetch_add( Capacity() * sizeof( T ), std::memory_order_relaxed ); + m_capacity++; + Realloc(); + } + } + + void Realloc() + { + T* ptr = (T*)malloc( sizeof( T ) * CapacityNoNullptrCheck() ); + if( m_size != 0 ) + { + if( std::is_trivially_copyable() ) + { + memcpy( (char*)ptr, m_ptr, m_size * sizeof( T ) ); + } + else + { + for( uint32_t i=0; i m_ptr; +#endif + uint32_t m_size; + uint8_t m_capacity : 7; + uint8_t m_magic : 1; +}; + + +template struct VectorAdapterDirect { const T& operator()( const T& it ) const { return it; } }; +template struct VectorAdapterPointer { const T& operator()( const short_ptr& it ) const { return *it; } }; + +#pragma pack( pop ) + +enum { VectorSize = sizeof( Vector ) }; + +} + +#endif diff --git a/third_party/tracy/server/TracyWorker.cpp b/third_party/tracy/server/TracyWorker.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5c2972914c57292e993976eff63438eb00baea9f --- /dev/null +++ b/third_party/tracy/server/TracyWorker.cpp @@ -0,0 +1,8516 @@ +#ifdef _MSC_VER +# pragma warning( disable: 4244 4267 ) // conversion from don't care to whatever, possible loss of data +#endif + +#ifdef _WIN32 +# include +#elif defined __FreeBSD__ +# include +#else +# include +#endif + +#include +#include +#include +#include + +#ifdef __MINGW32__ +# define __STDC_FORMAT_MACROS +#endif +#include +#include + +#include + +#define ZDICT_STATIC_LINKING_ONLY +#include "../zstd/zdict.h" + +#include "../public/common/TracyProtocol.hpp" +#include "../public/common/TracySystem.hpp" +#include "../public/common/TracyYield.hpp" +#include "../public/common/TracyStackFrames.hpp" +#include "../public/common/TracyVersion.hpp" +#include "TracyFileRead.hpp" +#include "TracyFileWrite.hpp" +#include "TracyPrint.hpp" +#include "TracySort.hpp" +#include "TracyTaskDispatch.hpp" +#include "TracyWorker.hpp" + +namespace tracy +{ + +static tracy_force_inline uint32_t UnpackFileLine( uint64_t packed, uint32_t& line ) +{ + line = packed & 0xFFFFFFFF; + return packed >> 32; +} + +static bool SourceFileValid( const char* fn, uint64_t olderThan ) +{ + struct stat buf; + if( stat( fn, &buf ) == 0 && ( buf.st_mode & S_IFREG ) != 0 ) + { + return (uint64_t)buf.st_mtime < olderThan; + } + return false; +} + + +static const uint8_t FileHeader[8] { 't', 'r', 'a', 'c', 'y', Version::Major, Version::Minor, Version::Patch }; +enum { FileHeaderMagic = 5 }; +static const int CurrentVersion = FileVersion( Version::Major, Version::Minor, Version::Patch ); +static const int MinSupportedVersion = FileVersion( 0, 9, 0 ); + + +static void UpdateLockCountLockable( LockMap& lockmap, size_t pos ) +{ + auto& timeline = lockmap.timeline; + bool isContended = lockmap.isContended; + uint8_t lockingThread; + uint8_t lockCount; + uint64_t waitList; + + if( pos == 0 ) + { + lockingThread = 0; + lockCount = 0; + waitList = 0; + } + else + { + const auto& tl = timeline[pos-1]; + lockingThread = tl.lockingThread; + lockCount = tl.lockCount; + waitList = tl.waitList; + } + const auto end = timeline.size(); + + while( pos != end ) + { + auto& tl = timeline[pos]; + const auto tbit = uint64_t( 1 ) << tl.ptr->thread; + switch( (LockEvent::Type)tl.ptr->type ) + { + case LockEvent::Type::Wait: + waitList |= tbit; + break; + case LockEvent::Type::Obtain: + assert( lockCount < std::numeric_limits::max() ); + assert( ( waitList & tbit ) != 0 ); + waitList &= ~tbit; + lockingThread = tl.ptr->thread; + lockCount++; + break; + case LockEvent::Type::Release: + assert( lockCount > 0 ); + lockCount--; + break; + default: + break; + } + tl.lockingThread = lockingThread; + tl.waitList = waitList; + tl.lockCount = lockCount; + if( !isContended ) isContended = lockCount != 0 && waitList != 0; + pos++; + } + + lockmap.isContended = isContended; +} + +static void UpdateLockCountSharedLockable( LockMap& lockmap, size_t pos ) +{ + auto& timeline = lockmap.timeline; + bool isContended = lockmap.isContended; + uint8_t lockingThread; + uint8_t lockCount; + uint64_t waitShared; + uint64_t waitList; + uint64_t sharedList; + + if( pos == 0 ) + { + lockingThread = 0; + lockCount = 0; + waitShared = 0; + waitList = 0; + sharedList = 0; + } + else + { + const auto& tl = timeline[pos-1]; + const auto tlp = (const LockEventShared*)(const LockEvent*)tl.ptr; + lockingThread = tl.lockingThread; + lockCount = tl.lockCount; + waitShared = tlp->waitShared; + waitList = tl.waitList; + sharedList = tlp->sharedList; + } + const auto end = timeline.size(); + + // ObtainShared and ReleaseShared should assert on lockCount == 0, but + // due to the async retrieval of data from threads that's not possible. + while( pos != end ) + { + auto& tl = timeline[pos]; + const auto tlp = (LockEventShared*)(LockEvent*)tl.ptr; + const auto tbit = uint64_t( 1 ) << tlp->thread; + switch( (LockEvent::Type)tlp->type ) + { + case LockEvent::Type::Wait: + waitList |= tbit; + break; + case LockEvent::Type::WaitShared: + waitShared |= tbit; + break; + case LockEvent::Type::Obtain: + assert( lockCount < std::numeric_limits::max() ); + assert( ( waitList & tbit ) != 0 ); + waitList &= ~tbit; + lockingThread = tlp->thread; + lockCount++; + break; + case LockEvent::Type::Release: + assert( lockCount > 0 ); + lockCount--; + break; + case LockEvent::Type::ObtainShared: + assert( ( waitShared & tbit ) != 0 ); + assert( ( sharedList & tbit ) == 0 ); + waitShared &= ~tbit; + sharedList |= tbit; + break; + case LockEvent::Type::ReleaseShared: + assert( ( sharedList & tbit ) != 0 ); + sharedList &= ~tbit; + break; + default: + break; + } + tl.lockingThread = lockingThread; + tlp->waitShared = waitShared; + tl.waitList = waitList; + tlp->sharedList = sharedList; + tl.lockCount = lockCount; + if( !isContended ) isContended = ( lockCount != 0 && ( waitList != 0 || waitShared != 0 ) ) || ( sharedList != 0 && waitList != 0 ); + pos++; + } + + lockmap.isContended = isContended; +} + +static inline void UpdateLockCount( LockMap& lockmap, size_t pos ) +{ + if( lockmap.type == LockType::Lockable ) + { + UpdateLockCountLockable( lockmap, pos ); + } + else + { + UpdateLockCountSharedLockable( lockmap, pos ); + } +} + +static tracy_force_inline void WriteTimeOffset( FileWrite& f, int64_t& refTime, int64_t time ) +{ + int64_t timeOffset = time - refTime; + refTime += timeOffset; + f.Write( &timeOffset, sizeof( timeOffset ) ); +} + +static tracy_force_inline int64_t ReadTimeOffset( FileRead& f, int64_t& refTime ) +{ + int64_t timeOffset; + f.Read( timeOffset ); + refTime += timeOffset; + return refTime; +} + +static tracy_force_inline void UpdateLockRange( LockMap& lockmap, const LockEvent& ev, int64_t lt ) +{ + auto& range = lockmap.range[ev.thread]; + if( range.start > lt ) range.start = lt; + if( range.end < lt ) range.end = lt; +} + +template +static uint64_t ReadHwSampleVec( FileRead& f, SortedVector& vec, Slab& slab ) +{ + uint64_t sz; + f.Read( sz ); + if( sz != 0 ) + { + int64_t refTime = 0; + vec.reserve_exact( sz, slab ); + for( uint64_t i=0; i::max() ); + m_data.memory = m_slab.AllocInit(); + m_data.memNameMap.emplace( 0, m_data.memory ); + + memset( (char*)m_gpuCtxMap, 0, sizeof( m_gpuCtxMap ) ); + +#ifndef TRACY_NO_STATISTICS + m_data.sourceLocationZonesReady = true; + m_data.gpuSourceLocationZonesReady = true; + m_data.callstackSamplesReady = true; + m_data.ghostZonesReady = true; + m_data.ctxUsageReady = true; + m_data.symbolSamplesReady = true; +#endif + + m_thread = std::thread( [this] { SetThreadName( "Tracy Worker" ); Exec(); } ); + m_threadNet = std::thread( [this] { SetThreadName( "Tracy Network" ); Network(); } ); +} + +Worker::Worker( const char* name, const char* program, const std::vector& timeline, const std::vector& messages, const std::vector& plots, const std::unordered_map& threadNames ) + : m_hasData( true ) + , m_delay( 0 ) + , m_resolution( 0 ) + , m_captureName( name ) + , m_captureProgram( program ) + , m_captureTime( 0 ) + , m_executableTime( 0 ) + , m_pid( 0 ) + , m_samplingPeriod( 0 ) + , m_stream( nullptr ) + , m_buffer( nullptr ) + , m_onDemand( false ) + , m_inconsistentSamples( false ) + , m_memoryLimit( -1 ) + , m_traceVersion( CurrentVersion ) +{ + m_data.sourceLocationExpand.push_back( 0 ); + m_data.localThreadCompress.InitZero(); + m_data.callstackPayload.push_back( nullptr ); + m_data.zoneExtra.push_back( ZoneExtra {} ); + m_data.symbolLocInline.push_back( std::numeric_limits::max() ); + m_data.memory = m_slab.AllocInit(); + m_data.memNameMap.emplace( 0, m_data.memory ); + + m_data.lastTime = 0; + if( !timeline.empty() ) + { + m_data.lastTime = timeline.back().timestamp; + } + if( !messages.empty() ) + { + if( m_data.lastTime < (int64_t)messages.back().timestamp ) m_data.lastTime = messages.back().timestamp; + } + if( !plots.empty() ) + { + for( auto& v : plots ) + { + if( m_data.lastTime < v.data.back().first ) m_data.lastTime = v.data.back().first; + } + } + + for( auto& v : timeline ) + { + if( !v.isEnd ) + { + SourceLocation srcloc {{ + StringRef(), + StringRef( StringRef::Idx, StoreString( v.name.c_str(), v.name.size() ).idx ), + StringRef( StringRef::Idx, StoreString( v.locFile.c_str(), v.locFile.size() ).idx ), + v.locLine, + 0 + }}; + int key; + auto it = m_data.sourceLocationPayloadMap.find( &srcloc ); + if( it == m_data.sourceLocationPayloadMap.end() ) + { + auto slptr = m_slab.Alloc(); + memcpy( slptr, &srcloc, sizeof( srcloc ) ); + uint32_t idx = m_data.sourceLocationPayload.size(); + m_data.sourceLocationPayloadMap.emplace( slptr, idx ); + m_data.sourceLocationPayload.push_back( slptr ); + key = -int16_t( idx + 1 ); +#ifndef TRACY_NO_STATISTICS + auto res = m_data.sourceLocationZones.emplace( key, SourceLocationZones() ); + m_data.srclocZonesLast.first = key; + m_data.srclocZonesLast.second = &res.first->second; + +#else + auto res = m_data.sourceLocationZonesCnt.emplace( key, 0 ); + m_data.srclocCntLast.first = key; + m_data.srclocCntLast.second = &res.first->second; +#endif + } + else + { + key = -int16_t( it->second + 1 ); + } + + auto zone = AllocZoneEvent(); + zone->SetStartSrcLoc( v.timestamp, key ); + zone->SetEnd( -1 ); + zone->SetChild( -1 ); + + if( !v.text.empty() ) + { + auto& extra = RequestZoneExtra( *zone ); + extra.text = StringIdx( StoreString( v.text.c_str(), v.text.size() ).idx ); + } + + if( m_threadCtx != v.tid ) + { + m_threadCtx = v.tid; + m_threadCtxData = NoticeThread( v.tid ); + } + NewZone( zone ); + } + else + { + auto td = NoticeThread( v.tid ); + if( td->zoneIdStack.empty() ) continue; + td->zoneIdStack.pop_back(); + auto& stack = td->stack; + auto zone = stack.back_and_pop(); + td->DecStackCount( zone->SrcLoc() ); + zone->SetEnd( v.timestamp ); + +#ifndef TRACY_NO_STATISTICS + ZoneThreadData ztd; + ztd.SetZone( zone ); + ztd.SetThread( CompressThread( v.tid ) ); + auto slz = GetSourceLocationZones( zone->SrcLoc() ); + slz->zones.push_back( ztd ); +#else + CountZoneStatistics( zone ); +#endif + } + } + + std::unordered_map frameNames; + + for( auto& v : messages ) + { + // There is no specific chrome-tracing type for frame events. We use messages that contain the word "frame" + std::string lower( v.message ); + std::transform( lower.begin(), lower.end(), lower.begin(), []( char c ) { return char( std::tolower( c ) ); } ); + if( lower.find( "frame" ) != std::string::npos ) + { + // Reserve 0 as the default FrameSet, since it replaces the name with "Frame" and we want to keep our custom names. + auto result = frameNames.emplace( v.message, frameNames.size() + 1 ); + auto fd = m_data.frames.Retrieve( result.first->second, [&] ( uint64_t name ) { + auto fd = m_slab.AllocInit(); + fd->name = name; + fd->continuous = 1; + return fd; + }, [&] ( uint64_t name ) { + HandleFrameName( name, v.message.c_str(), v.message.length() ); + }); + + int64_t time = v.timestamp; + fd->frames.push_back( FrameEvent{ time, -1, -1 } ); + if ( m_data.lastTime < time ) m_data.lastTime = time; + } + else + { + auto msg = m_slab.Alloc(); + msg->time = v.timestamp; + msg->ref = StringRef( StringRef::Type::Idx, StoreString( v.message.c_str(), v.message.size() ).idx ); + msg->thread = CompressThread( v.tid ); + msg->color = 0xFFFFFFFF; + msg->callstack.SetVal( 0 ); + + if( m_threadCtx != v.tid ) + { + m_threadCtx = v.tid; + m_threadCtxData = nullptr; + } + InsertMessageData( msg ); + } + } + + for( auto& v : plots ) + { + uint64_t nptr = (uint64_t)&v.name; + auto it = m_data.strings.find( nptr ); + if( it == m_data.strings.end() ) + { + const auto sl = StoreString( v.name.c_str(), v.name.size() ); + m_data.strings.emplace( nptr, sl.ptr ); + } + + auto plot = m_slab.AllocInit(); + plot->name = nptr; + plot->type = PlotType::User; + plot->format = v.format; + plot->showSteps = false; + plot->fill = true; + plot->color = 0; + + double sum = 0; + double min = v.data.begin()->second; + double max = v.data.begin()->second; + plot->data.reserve_exact( v.data.size(), m_slab ); + size_t idx = 0; + for( auto& p : v.data ) + { + plot->data[idx].time.SetVal( p.first ); + plot->data[idx].val = p.second; + idx++; + if( min > p.second ) min = p.second; + else if( max < p.second ) max = p.second; + sum += p.second; + } + plot->min = min; + plot->max = max; + plot->sum = sum; + + m_data.plots.Data().push_back( plot ); + } + + for( auto& t : m_threadMap ) + { + auto name = threadNames.find(t.first); + if( name != threadNames.end() ) + { + char buf[128]; + int len; + if( t.first <= std::numeric_limits::max() ) + { + len = snprintf( buf, sizeof( buf ), "(%" PRIu64 ") %s", t.first, name->second.c_str() ); + } + else + { + len = snprintf( buf, sizeof( buf ), "(PID %" PRIu64 " TID %" PRIu64 ") %s", t.first >> 32, t.first & 0xFFFFFFFF, name->second.c_str() ); + } + AddThreadString( t.first, buf, len ); + } + else + { + char buf[64]; + int len; + if( t.first <= std::numeric_limits::max() ) + { + len = sprintf( buf, "%" PRIu64, t.first ); + } + else + { + len = sprintf( buf, "PID %" PRIu64 " TID %" PRIu64, t.first >> 32, t.first & 0xFFFFFFFF ); + } + AddThreadString( t.first, buf, len ); + } + } + + // Add a default frame if we didn't have any framesets + if( frameNames.empty() ) + { + m_data.framesBase = m_data.frames.Retrieve( 0, [this] ( uint64_t name ) { + auto fd = m_slab.AllocInit(); + fd->name = name; + fd->continuous = 1; + return fd; + }, [this] ( uint64_t name ) { + assert( name == 0 ); + char tmp[6] = "Frame"; + HandleFrameName( name, tmp, 5 ); + } ); + + m_data.framesBase->frames.push_back( FrameEvent{ 0, -1, -1 } ); + m_data.framesBase->frames.push_back( FrameEvent{ 0, -1, -1 } ); + } +} + +Worker::Worker( FileRead& f, EventType::Type eventMask, bool bgTasks, bool allowStringModification ) + : m_hasData( true ) + , m_stream( nullptr ) + , m_buffer( nullptr ) + , m_inconsistentSamples( false ) + , m_memoryLimit( -1 ) + , m_allowStringModification( allowStringModification ) +{ + auto loadStart = std::chrono::high_resolution_clock::now(); + + int fileVer = 0; + + uint8_t hdr[8]; + f.Read( hdr, sizeof( hdr ) ); + if( memcmp( FileHeader, hdr, FileHeaderMagic ) == 0 ) + { + fileVer = FileVersion( hdr[FileHeaderMagic], hdr[FileHeaderMagic+1], hdr[FileHeaderMagic+2] ); + if( fileVer > CurrentVersion ) + { + throw UnsupportedVersion( fileVer ); + } + if( fileVer < MinSupportedVersion ) + { + throw LegacyVersion( fileVer ); + } + + f.Read( m_delay ); + } + else + { + throw LegacyVersion( FileVersion( 0, 2, 0 ) ); + } + m_traceVersion = fileVer; + + s_loadProgress.total.store( 11, std::memory_order_relaxed ); + s_loadProgress.subTotal.store( 0, std::memory_order_relaxed ); + s_loadProgress.progress.store( LoadProgress::Initialization, std::memory_order_relaxed ); + f.Read8( m_resolution, m_timerMul, m_data.lastTime, m_data.frameOffset, m_pid, m_samplingPeriod, m_data.cpuArch, m_data.cpuId ); + f.Read( m_data.cpuManufacturer, 12 ); + m_data.cpuManufacturer[12] = '\0'; + + if( fileVer >= FileVersion( 0, 9, 2 ) ) + { + uint8_t flag; + f.Read( flag ); + m_onDemand = flag; + } + else + { + m_onDemand = m_data.frameOffset != 0; + } + + uint64_t sz; + { + f.Read( sz ); + assert( sz < 1024 ); + char tmp[1024]; + f.Read( tmp, sz ); + m_captureName = std::string( tmp, tmp+sz ); + if( m_captureName.empty() ) m_captureName = f.GetFilename(); + } + { + f.Read( sz ); + assert( sz < 1024 ); + char tmp[1024]; + f.Read( tmp, sz ); + m_captureProgram = std::string( tmp, tmp+sz ); + f.Read( m_captureTime ); + } + + f.Read( m_executableTime ); + + { + f.Read( sz ); + assert( sz < 1024 ); + char tmp[1024]; + f.Read( tmp, sz ); + m_hostInfo = std::string( tmp, tmp+sz ); + } + + f.Read( sz ); + m_data.cpuTopology.reserve( sz ); + for( uint64_t i=0; i> {} ).first; + package.second.reserve( psz ); + for( uint64_t j=0; j {} ).first; + core.second.reserve( csz ); + for( uint64_t k=0; k(); + uint64_t fsz; + f.Read3( ptr->name, ptr->continuous, fsz ); + ptr->frames.reserve_exact( fsz, m_slab ); + int64_t refTime = 0; + if( ptr->continuous ) + { + for( uint64_t j=0; jframes[j].start = ReadTimeOffset( f, refTime ); + ptr->frames[j].end = -1; + f.Read( &ptr->frames[j].frameImage, sizeof( int32_t ) ); + } + } + else + { + for( uint64_t j=0; jframes[j].start = ReadTimeOffset( f, refTime ); + ptr->frames[j].end = ReadTimeOffset( f, refTime ); + f.Read( &ptr->frames[j].frameImage, sizeof( int32_t ) ); + } + } + for( uint64_t j=0; j 0 ) + { + ptr->min = std::min( ptr->min, timeSpan ); + ptr->max = std::max( ptr->max, timeSpan ); + ptr->total += timeSpan; + ptr->sumSq += double( timeSpan ) * timeSpan; + } + } + m_data.frames.Data()[i] = ptr; + } + m_data.framesBase = m_data.frames.Data()[0]; + assert( m_data.framesBase->name == 0 ); + + unordered_flat_map pointerMap; + + f.Read( sz ); + m_data.stringMap.reserve( sz ); + + if( !m_allowStringModification ) + { + m_data.stringData.reserve_exact( sz, m_slab ); + } + + for( uint64_t i=0; i( ssz+1 ); + f.Read( dst, ssz ); + dst[ssz] = '\0'; + m_data.stringMap.emplace( charutil::StringKey { dst, size_t( ssz ) }, i ); + + if( m_allowStringModification ) + { + m_data.stringData.push_back( dst ); + } + else + { + m_data.stringData[i] = ( dst ); + } + + pointerMap.emplace( ptr, dst ); + } + + f.Read( sz ); + for( uint64_t i=0; isecond ); + } + } + + f.Read( sz ); + for( uint64_t i=0; isecond ); + } + } + + f.Read( sz ); + for( uint64_t i=0; isecond, it2->second ) ); + } + } + + m_data.localThreadCompress.Load( f ); + m_data.externalThreadCompress.Load( f ); + + f.Read( sz ); + if( sz > std::numeric_limits::max() ) + { + s_loadProgress.total.store( 0, std::memory_order_relaxed ); + char buf[256]; + sprintf( buf, "Too many static source locations (%s)", RealToString( sz ) ); + throw LoadFailure( buf ); + } + for( uint64_t i=0; i std::numeric_limits::max() ) + { + s_loadProgress.total.store( 0, std::memory_order_relaxed ); + char buf[256]; + sprintf( buf, "Too many dynamic source locations (%s)", RealToString( sz ) ); + throw LoadFailure( buf ); + } + m_data.sourceLocationPayload.reserve_exact( sz, m_slab ); + for( uint64_t i=0; i(); + f.Read( srcloc, sizeof( SourceLocationBase ) ); + srcloc->namehash = 0; + m_data.sourceLocationPayload[i] = srcloc; + m_data.sourceLocationPayloadMap.emplace( srcloc, int16_t( i ) ); + } + +#ifndef TRACY_NO_STATISTICS + m_data.sourceLocationZones.reserve( sle + sz ); + + f.Read( sz ); + for( uint64_t i=0; isecond.zones.reserve( cnt ); + } + + f.Read( sz ); + for( uint64_t i=0; isecond.zones.reserve( cnt ); + } +#else + f.Read( sz ); + for( uint64_t i=0; i(); + auto& lockmap = *lockmapPtr; + uint32_t id; + uint64_t tsz; + f.Read8( id, lockmap.customName, lockmap.srcloc, lockmap.type, lockmap.valid, lockmap.timeAnnounce, lockmap.timeTerminate, tsz ); + lockmap.isContended = false; + lockmap.threadMap.reserve( tsz ); + lockmap.threadList.reserve( tsz ); + for( uint64_t i=0; i(); + const auto lt = ReadTimeOffset( f, refTime ); + lev->SetTime( lt ); + int16_t srcloc; + f.Read( srcloc ); + lev->SetSrcLoc( srcloc ); + f.Read( &lev->thread, sizeof( LockEvent::thread ) + sizeof( LockEvent::type ) ); + *ptr++ = { lev }; + UpdateLockRange( lockmap, *lev, lt ); + } + } + else + { + for( uint64_t i=0; i(); + const auto lt = ReadTimeOffset( f, refTime ); + lev->SetTime( lt ); + int16_t srcloc; + f.Read( srcloc ); + lev->SetSrcLoc( srcloc ); + f.Read( &lev->thread, sizeof( LockEventShared::thread ) + sizeof( LockEventShared::type ) ); + *ptr++ = { lev }; + UpdateLockRange( lockmap, *lev, lt ); + } + } + UpdateLockCount( lockmap, 0 ); + m_data.lockMap.emplace( id, lockmapPtr ); + } + } + else + { + for( uint64_t i=0; i msgMap; + f.Read( sz ); + if( eventMask & EventType::Messages ) + { + m_data.messages.reserve_exact( sz, m_slab ); + int64_t refTime = 0; + for( uint64_t i=0; i(); + msgdata->time = ReadTimeOffset( f, refTime ); + f.Read3( msgdata->ref, msgdata->color, msgdata->callstack ); + m_data.messages[i] = msgdata; + msgMap.emplace( ptr, msgdata ); + } + } + else + { + f.Skip( sz * ( sizeof( uint64_t ) + sizeof( MessageData::time ) + sizeof( MessageData::ref ) + sizeof( MessageData::color ) + sizeof( MessageData::callstack ) ) ); + } + + f.Read( sz ); + assert( sz != 0 ); + m_data.zoneExtra.reserve_exact( sz, m_slab ); + f.Read( m_data.zoneExtra.data(), sz * sizeof( ZoneExtra ) ); + + s_loadProgress.progress.store( LoadProgress::Zones, std::memory_order_relaxed ); + f.Read( sz ); + s_loadProgress.subTotal.store( sz, std::memory_order_relaxed ); + s_loadProgress.subProgress.store( 0, std::memory_order_relaxed ); + f.Read( sz ); + m_data.zoneChildren.reserve_exact( sz, m_slab ); + memset( (char*)m_data.zoneChildren.data(), 0, sizeof( Vector> ) * sz ); + int32_t childIdx = 0; + f.Read( sz ); + m_data.threads.reserve_exact( sz, m_slab ); + for( uint64_t i=0; i(); + uint64_t tid; + if( fileVer >= FileVersion( 0, 11, 1 ) ) + { + f.Read5( tid, td->count, td->kernelSampleCnt, td->isFiber, td->groupHint ); + } + else + { + f.Read4( tid, td->count, td->kernelSampleCnt, td->isFiber ); + td->groupHint = 0; + } + td->id = tid; + m_data.zonesCnt += td->count; + uint32_t tsz; + f.Read( tsz ); + if( tsz != 0 ) + { + ReadTimeline( f, td->timeline, tsz, 0, childIdx ); + } + uint64_t msz; + f.Read( msz ); + if( eventMask & EventType::Messages ) + { + const auto ctid = CompressThread( tid ); + td->messages.reserve_exact( msz, m_slab ); + for( uint64_t j=0; jmessages[j] = md; + md->thread = ctid; + } + } + else + { + f.Skip( msz * sizeof( uint64_t ) ); + } + uint64_t ssz; + f.Read( ssz ); + if( ssz != 0 ) + { + if( eventMask & EventType::Samples ) + { + int64_t refTime = 0; + td->ctxSwitchSamples.reserve_exact( ssz, m_slab ); + auto ptr = td->ctxSwitchSamples.data(); + for( uint64_t j=0; jtime.SetVal( ReadTimeOffset( f, refTime ) ); + f.Read( &ptr->callstack, sizeof( ptr->callstack ) ); + ptr++; + } + } + else + { + f.Skip( ssz * ( 8 + 3 ) ); + } + } + f.Read( ssz ); + if( ssz != 0 ) + { + if( eventMask & EventType::Samples ) + { + m_data.samplesCnt += ssz; + int64_t refTime = 0; + td->samples.reserve_exact( ssz, m_slab ); + auto ptr = td->samples.data(); + for( uint64_t j=0; jtime.SetVal( ReadTimeOffset( f, refTime ) ); + f.Read( &ptr->callstack, sizeof( ptr->callstack ) ); + ptr++; + } + } + else + { + f.Skip( ssz * ( 8 + 3 ) ); + } + } + m_data.threads[i] = td; + m_threadMap.emplace( tid, td ); + } + + s_loadProgress.progress.store( LoadProgress::GpuZones, std::memory_order_relaxed ); + f.Read( sz ); + s_loadProgress.subTotal.store( sz, std::memory_order_relaxed ); + s_loadProgress.subProgress.store( 0, std::memory_order_relaxed ); + f.Read( sz ); + m_data.gpuChildren.reserve_exact( sz, m_slab ); + memset( (char*)m_data.gpuChildren.data(), 0, sizeof( Vector> ) * sz ); + childIdx = 0; + f.Read( sz ); + m_data.gpuData.reserve_exact( sz, m_slab ); + for( uint64_t i=0; i(); + uint8_t calibration; + f.Read7( ctx->thread, calibration, ctx->count, ctx->period, ctx->type, ctx->name, ctx->overflow ); + ctx->hasCalibration = calibration; + ctx->hasPeriod = ctx->period != 1.f; + m_data.gpuCnt += ctx->count; + uint64_t tdsz; + f.Read( tdsz ); + for( uint64_t j=0; jthreadData.emplace( tid, GpuCtxThreadData {} ).first; + ReadTimeline( f, td->second.timeline, tsz, refTime, refGpuTime, childIdx ); + } + } + m_data.gpuData[i] = ctx; + } + + s_loadProgress.progress.store( LoadProgress::Plots, std::memory_order_relaxed ); + f.Read( sz ); + if( eventMask & EventType::Plots ) + { + m_data.plots.Data().reserve( sz ); + s_loadProgress.subTotal.store( sz, std::memory_order_relaxed ); + for( uint64_t i=0; i(); + uint64_t psz; + f.Read10( pd->type, pd->format, pd->showSteps, pd->fill, pd->color, pd->name, pd->min, pd->max, pd->sum, psz ); + pd->data.reserve_exact( psz, m_slab ); + auto ptr = pd->data.data(); + int64_t refTime = 0; + for( uint64_t j=0; jval ); + refTime += t; + ptr->time = refTime; + ptr++; + } + m_data.plots.Data().push_back_no_space_check( pd ); + } + } + else + { + for( uint64_t i=0; i() ); + if( memname == 0 ) m_data.memory = mit.first->second; + auto& memdata = *mit.first->second; + memdata.data.reserve_exact( sz, m_slab ); + uint64_t activeSz, freesSz; + f.Read2( activeSz, freesSz ); + memdata.active.reserve( activeSz ); + memdata.frees.reserve_exact( freesSz, m_slab ); + auto mem = memdata.data.data(); + s_loadProgress.subTotal.store( sz, std::memory_order_relaxed ); + size_t fidx = 0; + int64_t refTime = 0; + auto& frees = memdata.frees; + auto& active = memdata.active; + + for( uint64_t i=0; icsFree, timeAlloc, timeFree, threadAlloc, threadFree ); + mem->SetPtr( ptr ); + mem->SetSize( size ); + mem->SetCsAlloc( csAlloc.Val() ); + refTime += timeAlloc; + mem->SetTimeThreadAlloc( refTime, threadAlloc ); + if( timeFree >= 0 ) + { + mem->SetTimeThreadFree( timeFree + refTime, threadFree ); + frees[fidx++] = i; + } + else + { + mem->SetTimeThreadFree( timeFree, threadFree ); + active.emplace( ptr, i ); + } + mem++; + } + memload += sz; + f.Read4( memdata.high, memdata.low, memdata.usage, memdata.name ); + + if( sz != 0 ) + { + memdata.reconstruct = true; + } + } + else + { + f.Skip( 2 * sizeof( uint64_t ) ); + f.Skip( sz * ( sizeof( uint64_t ) + sizeof( uint64_t ) + sizeof( Int24 ) + sizeof( Int24 ) + sizeof( int64_t ) * 2 + sizeof( uint16_t ) * 2 ) ); + f.Skip( sizeof( MemData::high ) + sizeof( MemData::low ) + sizeof( MemData::usage ) + sizeof( MemData::name ) ); + } + } + + s_loadProgress.subTotal.store( 0, std::memory_order_relaxed ); + s_loadProgress.progress.store( LoadProgress::CallStacks, std::memory_order_relaxed ); + f.Read( sz ); + m_data.callstackPayload.reserve_exact( sz+1, m_slab ); + m_data.callstackPayload[0] = nullptr; + for( uint64_t i=0; i ) + csz * sizeof( CallstackFrameId ); + auto mem = (char*)m_slab.AllocRaw( memsize ); + + auto data = (CallstackFrameId*)mem; + f.Read( data, csz * sizeof( CallstackFrameId ) ); + + auto arr = (VarArray*)( mem + csz * sizeof( CallstackFrameId ) ); + new(arr) VarArray( csz, data ); + + m_data.callstackPayload[i+1] = arr; + } + + f.Read( sz ); + m_data.callstackFrameMap.reserve( sz ); + for( uint64_t i=0; i(); + f.Read3( id, frameData->size, frameData->imageName ); + + frameData->data = m_slab.Alloc( frameData->size ); + f.Read( frameData->data, sizeof( CallstackFrame ) * frameData->size ); + + m_data.callstackFrameMap.emplace( id, frameData ); + } + + f.Read( sz ); + if( sz > 0 ) + { + m_data.appInfo.reserve_exact( sz, m_slab ); + f.Read( m_data.appInfo.data(), sizeof( m_data.appInfo[0] ) * sz ); + } + + s_loadProgress.subTotal.store( 0, std::memory_order_relaxed ); + s_loadProgress.progress.store( LoadProgress::FrameImages, std::memory_order_relaxed ); + + if( eventMask & EventType::FrameImages ) + { + ZSTD_CDict* cdict = nullptr; + uint32_t dsz; + f.Read( dsz ); + auto dict = new char[dsz]; + f.Read( dict, dsz ); + cdict = ZSTD_createCDict( dict, dsz, 3 ); + m_texcomp.SetDict( ZSTD_createDDict( dict, dsz ) ); + delete[] dict; + + f.Read( sz ); + m_data.frameImage.reserve_exact( sz, m_slab ); + s_loadProgress.subTotal.store( sz, std::memory_order_relaxed ); + if( sz != 0 ) + { + struct JobData + { + enum State : int { InProgress, Available, DataReady }; + FrameImage* fi; + char* buf = nullptr; + size_t bufsz = 0; + char* outbuf = nullptr; + size_t outsz = 0; + ZSTD_CCtx* ctx = ZSTD_createCCtx(); + alignas(64) std::atomic state = Available; + }; + +#ifdef __EMSCRIPTEN__ + const int jobs = 1; +#else + // Leave one thread for file reader, second thread for dispatch (this thread) + // Minimum 2 threads to have at least two buffers (one in use, second one filling up) + const auto jobs = std::max( std::thread::hardware_concurrency() - 2, 2 ); +#endif + auto td = std::make_unique( jobs, "FrImg Zstd" ); + auto data = std::make_unique( jobs ); + + for( uint64_t i=0; i(); + f.Read3( fi->w, fi->h, fi->flip ); + const auto sz = size_t( fi->w * fi->h / 2 ); + + int idx = -1; + for(;;) + { + for( int j=0; jcsz ); + memcpy( tmp, data[j].outbuf, data[j].fi->csz ); + data[j].fi->ptr = tmp; + } + idx = j; + break; + } + } + if( idx >= 0 ) break; + YieldThread(); + } + + if( data[idx].bufsz < sz ) + { + data[idx].bufsz = sz; + delete[] data[idx].buf; + data[idx].buf = new char[sz]; + } + f.Read( data[idx].buf, sz ); + data[idx].fi = fi; + + data[idx].state.store( JobData::InProgress, std::memory_order_release ); + td->Queue( [this, &data, idx, fi, cdict] { + if( cdict ) + { + fi->csz = m_texcomp.Pack( data[idx].ctx, cdict, data[idx].outbuf, data[idx].outsz, data[idx].buf, fi->w * fi->h / 2 ); + } + else + { + fi->csz = m_texcomp.Pack( data[idx].ctx, data[idx].outbuf, data[idx].outsz, data[idx].buf, fi->w * fi->h / 2 ); + } + data[idx].state.store( JobData::DataReady, std::memory_order_release ); + } ); + + m_data.frameImage[i] = fi; + } + td->Sync(); + td.reset(); + for( int i=0; icsz ); + memcpy( tmp, data[i].outbuf, data[i].fi->csz ); + data[i].fi->ptr = tmp; + } + ZSTD_freeCCtx( data[i].ctx ); + delete[] data[i].buf; + delete[] data[i].outbuf; + } + + const auto& frames = GetFramesBase()->frames; + const auto fsz = uint32_t( frames.size() ); + for( uint32_t i=0; iframeRef = i; + } + } + } + + ZSTD_freeCDict( cdict ); + } + else + { + uint32_t dsz; + f.Read( dsz ); + f.Skip( dsz ); + f.Read( sz ); + s_loadProgress.subTotal.store( sz, std::memory_order_relaxed ); + for( uint64_t i=0; iframes ) + { + v.frameImage = -1; + } + } + + s_loadProgress.subTotal.store( 0, std::memory_order_relaxed ); + s_loadProgress.progress.store( LoadProgress::ContextSwitches, std::memory_order_relaxed ); + + if( eventMask & EventType::ContextSwitches ) + { + f.Read( sz ); + s_loadProgress.subTotal.store( sz, std::memory_order_relaxed ); + m_data.ctxSwitch.reserve( sz ); + for( uint64_t i=0; i(); + data->v.reserve_exact( csz, m_slab ); + int64_t runningTime = 0; + int64_t refTime = 0; + auto ptr = data->v.data(); + for( uint64_t j=0; jSetWakeup( refTime ); + refTime += deltaStart; + ptr->SetStartCpu( refTime, cpu ); + if( diff > 0 ) runningTime += diff; + refTime += diff; + ptr->SetEndReasonState( refTime, reason, state ); + ptr->SetThread( CompressThread( thread ) ); + ptr++; + } + data->runningTime = runningTime; + m_data.ctxSwitch.emplace( thread, data ); + } + } + else + { + f.Read( sz ); + s_loadProgress.subTotal.store( sz, std::memory_order_relaxed ); + for( uint64_t i=0; iSetStartThread( refTime, thread ); + refTime += deltaEnd; + ptr->SetEnd( refTime ); + ptr++; + } + cnt += sz; + } + s_loadProgress.subProgress.store( cnt, std::memory_order_relaxed ); + } + } + else + { + for( int i=0; i<256; i++ ) + { + f.Read( sz ); + f.Skip( sz * ( sizeof( int64_t ) * 2 + sizeof( uint16_t ) ) ); + } + } + + f.Read( sz ); + for( uint64_t i=0; isecond; + ReadHwSampleVec( f, data.cycles, m_slab ); + ReadHwSampleVec( f, data.retired, m_slab ); + ReadHwSampleVec( f, data.cacheRef, m_slab ); + ReadHwSampleVec( f, data.cacheMiss, m_slab ); + if( ReadHwSampleVec( f, data.branchRetired, m_slab ) != 0 ) m_data.hasBranchRetirement = true; + ReadHwSampleVec( f, data.branchMiss, m_slab ); + } + + f.Read( sz ); + if( eventMask & EventType::SourceCache ) + { + m_data.sourceFileCache.reserve( sz ); + for( uint64_t i=0; i( len+1 ); + f.Read( key, len ); + key[len] = '\0'; + f.Read( len ); + auto data = (char*)m_slab.AllocBig( len ); + f.Read( data, len ); + m_data.sourceFileCache.emplace( key, MemoryBlock { data, len } ); + } + } + else + { + for( uint64_t i=0; i( std::chrono::high_resolution_clock::now() - loadStart ).count(); + + if( !bgTasks ) + { + m_backgroundDone.store( true, std::memory_order_relaxed ); + } + else + { + m_backgroundDone.store( false, std::memory_order_relaxed ); +#ifndef TRACY_NO_STATISTICS + m_threadBackground = std::thread( [this, eventMask] { + std::vector jobs; + + if( !m_data.ctxSwitch.empty() && m_data.cpuDataCount != 0 ) + { + jobs.emplace_back( std::thread( [this] { ReconstructContextSwitchUsage(); } ) ); + } + + for( auto& mem : m_data.memNameMap ) + { + if( mem.second->reconstruct ) jobs.emplace_back( std::thread( [this, mem = mem.second] { ReconstructMemAllocPlot( *mem ); } ) ); + } + + std::function>&, uint16_t)> ProcessTimeline; + ProcessTimeline = [this, &ProcessTimeline] ( uint8_t* countMap, Vector>& _vec, uint16_t thread ) + { + if( m_shutdown.load( std::memory_order_relaxed ) ) return; + assert( _vec.is_magic() ); + auto& vec = *(Vector*)( &_vec ); + for( auto& zone : vec ) + { + if( zone.IsEndValid() ) ReconstructZoneStatistics( countMap, zone, thread ); + if( zone.HasChildren() ) + { + countMap[uint16_t(zone.SrcLoc())]++; + ProcessTimeline( countMap, GetZoneChildrenMutable( zone.Child() ), thread ); + countMap[uint16_t(zone.SrcLoc())]--; + } + } + }; + + jobs.emplace_back( std::thread( [this, ProcessTimeline] { + for( auto& t : m_data.threads ) + { + if( m_shutdown.load( std::memory_order_relaxed ) ) return; + if( !t->timeline.empty() ) + { + uint8_t countMap[64*1024]; + // Don't touch thread compression cache in a thread. + ProcessTimeline( countMap, t->timeline, m_data.localThreadCompress.DecompressMustRaw( t->id ) ); + } + } + std::lock_guard lock( m_data.lock ); + m_data.sourceLocationZonesReady = true; + } ) ); + + std::function>&, uint16_t)> ProcessTimelineGpu; + ProcessTimelineGpu = [this, &ProcessTimelineGpu] ( Vector>& _vec, uint16_t thread ) + { + if( m_shutdown.load( std::memory_order_relaxed ) ) return; + assert( _vec.is_magic() ); + auto& vec = *(Vector*)( &_vec ); + for( auto& zone : vec ) + { + if( zone.GpuEnd() >= 0 ) ReconstructZoneStatistics( zone, thread ); + if( zone.Child() >= 0 ) + { + ProcessTimelineGpu( GetGpuChildrenMutable( zone.Child() ), thread ); + } + } + }; + + jobs.emplace_back( std::thread( [this, ProcessTimelineGpu] { + for( auto& t : m_data.gpuData ) + { + for( auto& td : t->threadData ) + { + if( m_shutdown.load( std::memory_order_relaxed ) ) return; + if( !td.second.timeline.empty() ) + { + ProcessTimelineGpu( td.second.timeline, td.first ); + } + } + } + std::lock_guard lock( m_data.lock ); + m_data.gpuSourceLocationZonesReady = true; + } ) ); + + if( eventMask & EventType::Samples ) + { + jobs.emplace_back( std::thread( [this] { + unordered_flat_map counts; + uint32_t total = 0; + for( auto& t : m_data.threads ) total += t->samples.size(); + if( total != 0 ) + { + for( auto& t : m_data.threads ) + { + if( m_shutdown.load( std::memory_order_relaxed ) ) return; + auto cit = t->ctxSwitchSamples.begin(); + for( auto& sd : t->samples ) + { + bool isCtxSwitch = false; + if( cit != t->ctxSwitchSamples.end() ) + { + const auto sdt = sd.time.Val(); + cit = std::lower_bound( cit, t->ctxSwitchSamples.end(), sdt, []( const auto& l, const auto& r ) { return (uint64_t)l.time.Val() < (uint64_t)r; } ); + isCtxSwitch = cit != t->ctxSwitchSamples.end() && cit->time.Val() == sdt; + } + if( !isCtxSwitch ) + { + const auto cs = sd.callstack.Val(); + auto it = counts.find( cs ); + if( it == counts.end() ) + { + counts.emplace( cs, 1 ); + } + else + { + it->second++; + } + + const auto& callstack = GetCallstack( cs ); + auto& ip = callstack[0]; + auto frame = GetCallstackFrame( ip ); + if( frame ) + { + const auto symAddr = frame->data[0].symAddr; + auto it = m_data.instructionPointersMap.find( symAddr ); + if( it == m_data.instructionPointersMap.end() ) + { + m_data.instructionPointersMap.emplace( symAddr, unordered_flat_map { { ip, 1 } } ); + } + else + { + auto fit = it->second.find( ip ); + if( fit == it->second.end() ) + { + it->second.emplace( ip, 1 ); + } + else + { + fit->second++; + } + } + } + } + } + } + for( auto& v : counts ) UpdateSampleStatistics( v.first, v.second, false ); + } + std::lock_guard lock( m_data.lock ); + m_data.callstackSamplesReady = true; + } ) ); + + jobs.emplace_back( std::thread( [this] { + uint32_t gcnt = 0; + for( auto& t : m_data.threads ) + { + if( m_shutdown.load( std::memory_order_relaxed ) ) return; + if( !t->samples.empty() ) + { + if( t->samples[0].time.Val() != 0 ) + { + for( auto& sd : t->samples ) + { + gcnt += AddGhostZone( GetCallstack( sd.callstack.Val() ), &t->ghostZones, sd.time.Val() ); + } + } + else + { + for( auto& sd : t->samples ) + { + const auto st = sd.time.Val(); + if( st != 0 ) gcnt += AddGhostZone( GetCallstack( sd.callstack.Val() ), &t->ghostZones, st ); + } + } + } + } + std::lock_guard lock( m_data.lock ); + m_data.ghostZonesReady = true; + m_data.ghostCnt = gcnt; + } ) ); + + jobs.emplace_back( std::thread( [this] { + for( auto& t : m_data.threads ) + { + uint16_t tid = CompressThread( t->id ); + for( auto& v : t->samples ) + { + const auto& time = v.time; + const auto cs = v.callstack.Val(); + const auto& callstack = GetCallstack( cs ); + auto& ip = callstack[0]; + auto frame = GetCallstackFrame( ip ); + if( frame ) + { + const auto symAddr = frame->data[0].symAddr; + auto it = m_data.symbolSamples.find( symAddr ); + if( it == m_data.symbolSamples.end() ) + { + m_data.symbolSamples.emplace( symAddr, Vector( SampleDataRange { time, tid, ip } ) ); + } + else + { + it->second.push_back_non_empty( SampleDataRange { time, tid, ip } ); + } + } + auto childAddr = GetCanonicalPointer( callstack[0] ); + for( uint16_t i=1; i( ChildSample { time, childAddr } ) ); + } + else + { + it->second.push_back_non_empty( ChildSample { time, childAddr } ); + } + childAddr = addr; + } + } + } + for( auto& v : m_data.symbolSamples ) + { + pdqsort_branchless( v.second.begin(), v.second.end(), []( const auto& lhs, const auto& rhs ) { return lhs.time.Val() < rhs.time.Val(); } ); + } + for( auto& v : m_data.childSamples ) + { + pdqsort_branchless( v.second.begin(), v.second.end(), []( const auto& lhs, const auto& rhs ) { return lhs.time.Val() < rhs.time.Val(); } ); + } + std::lock_guard lock( m_data.lock ); + m_data.symbolSamplesReady = true; + } ) ); + } + + for( auto& job : jobs ) job.join(); + m_backgroundDone.store( true, std::memory_order_relaxed ); + } ); +#else + m_backgroundDone.store( true, std::memory_order_relaxed ); +#endif + } +} + +Worker::~Worker() +{ + Shutdown(); + + if( m_threadNet.joinable() ) m_threadNet.join(); + if( m_thread.joinable() ) m_thread.join(); + if( m_threadBackground.joinable() ) m_threadBackground.join(); + + delete[] m_buffer; + LZ4_freeStreamDecode( (LZ4_streamDecode_t*)m_stream ); + + delete[] m_frameImageBuffer; + delete[] m_tmpBuf; + + for( auto& v : m_data.threads ) + { + v->timeline.~Vector(); + v->stack.~Vector(); + v->messages.~Vector(); + v->zoneIdStack.~Vector(); + v->samples.~Vector(); +#ifndef TRACY_NO_STATISTICS + v->childTimeStack.~Vector(); + v->ghostZones.~Vector(); +#endif + } + for( auto& v : m_data.gpuData ) + { + for( auto& vt : v->threadData ) + { + vt.second.timeline.~Vector(); + vt.second.stack.~Vector(); + } + } + for( auto& v : m_data.plots.Data() ) + { + v->~PlotData(); + } + for( auto& v : m_data.frames.Data() ) + { + v->~FrameData(); + } + for( auto& v : m_data.lockMap ) + { + v.second->~LockMap(); + } + for( auto& v : m_data.zoneChildren ) + { + v.~Vector(); + } + for( auto& v : m_data.memNameMap ) + { + v.second->~MemData(); + } + for( auto& v : m_data.ctxSwitch ) + { + v.second->v.~Vector(); + } + for( auto& v : m_data.gpuChildren ) + { + v.~Vector(); + } +#ifndef TRACY_NO_STATISTICS + for( auto& v : m_data.ghostChildren ) + { + v.~Vector(); + } +#endif +} + +uint64_t Worker::GetLockCount() const +{ + uint64_t cnt = 0; + for( auto& l : m_data.lockMap ) + { + cnt += l.second->timeline.size(); + } + return cnt; +} + +uint64_t Worker::GetPlotCount() const +{ + uint64_t cnt = 0; + for( auto& p : m_data.plots.Data() ) + { + if( p->type == PlotType::User ) + { + cnt += p->data.size(); + } + } + return cnt; +} + +uint64_t Worker::GetTracyPlotCount() const +{ + uint64_t cnt = 0; + for( auto& p : m_data.plots.Data() ) + { + if( p->type != PlotType::User ) + { + cnt += p->data.size(); + } + } + return cnt; +} + +uint64_t Worker::GetContextSwitchCount() const +{ + uint64_t cnt = 0; + for( auto& v : m_data.ctxSwitch ) + { + cnt += v.second->v.size(); + } + return cnt; +} + +uint64_t Worker::GetContextSwitchPerCpuCount() const +{ + uint64_t cnt = 0; + for( int i=0; ictxSwitchSamples.size(); + } + return cnt; +} +#endif + +uint64_t Worker::GetPidFromTid( uint64_t tid ) const +{ + auto it = m_data.tidToPid.find( tid ); + if( it == m_data.tidToPid.end() ) return 0; + return it->second; +} + +const ContextSwitch* const Worker::GetContextSwitchDataImpl( uint64_t thread ) +{ + auto it = m_data.ctxSwitch.find( thread ); + if( it != m_data.ctxSwitch.end() ) + { + m_data.ctxSwitchLast.first = thread; + m_data.ctxSwitchLast.second = it->second; + return it->second; + } + else + { + return nullptr; + } +} + +size_t Worker::GetFullFrameCount( const FrameData& fd ) const +{ + const auto sz = fd.frames.size(); + assert( sz != 0 ); + + if( fd.continuous ) + { + if( IsConnected() ) + { + return sz - 1; + } + else + { + return sz; + } + } + else + { + const auto& last = fd.frames.back(); + if( last.end >= 0 ) + { + return sz; + } + else + { + return sz - 1; + } + } +} + +bool Worker::AreFramesUsed() const +{ + if( m_data.frames.Data().size() > 1 ) return true; + return m_data.framesBase->frames.size() > 2; +} + +int64_t Worker::GetFirstTime() const +{ + if( !m_onDemand ) return 0; + assert( m_data.framesBase->frames.size() >= 2 ); + return m_data.framesBase->frames[2].start; +} + +int64_t Worker::GetFrameTime( const FrameData& fd, size_t idx ) const +{ + if( fd.continuous ) + { + if( idx < fd.frames.size() - 1 ) + { + return fd.frames[idx+1].start - fd.frames[idx].start; + } + else + { + assert( m_data.lastTime != 0 ); + return m_data.lastTime - fd.frames.back().start; + } + } + else + { + const auto& frame = fd.frames[idx]; + if( frame.end >= 0 ) + { + return frame.end - frame.start; + } + else + { + return m_data.lastTime - fd.frames.back().start; + } + } +} + +int64_t Worker::GetFrameBegin( const FrameData& fd, size_t idx ) const +{ + assert( idx < fd.frames.size() ); + return fd.frames[idx].start; +} + +int64_t Worker::GetFrameEnd( const FrameData& fd, size_t idx ) const +{ + if( fd.continuous ) + { + if( idx < fd.frames.size() - 1 ) + { + return fd.frames[idx+1].start; + } + else + { + return m_data.lastTime; + } + } + else + { + if( idx < fd.frames.size() && fd.frames[idx].end >= 0 ) + { + return fd.frames[idx].end; + } + else + { + return m_data.lastTime; + } + } +} + +const FrameImage* Worker::GetFrameImage( const FrameData& fd, size_t idx ) const +{ + assert( idx < fd.frames.size() ); + const auto& v = fd.frames[idx].frameImage; + if( v < 0 ) return nullptr; + return m_data.frameImage[v]; +} + +std::pair Worker::GetFrameRange( const FrameData& fd, int64_t from, int64_t to ) +{ + auto zitbegin = std::lower_bound( fd.frames.begin(), fd.frames.end(), from, [] ( const auto& lhs, const auto& rhs ) { return lhs.start < rhs; } ); + if( zitbegin == fd.frames.end() ) zitbegin--; + + const auto zitend = std::lower_bound( zitbegin, fd.frames.end(), to, [] ( const auto& lhs, const auto& rhs ) { return lhs.start < rhs; } ); + + int zbegin = std::distance( fd.frames.begin(), zitbegin ); + if( zbegin > 0 && zitbegin->start != from ) --zbegin; + const int zend = std::distance( fd.frames.begin(), zitend ); + + return std::make_pair( zbegin, zend ); +} + +const CallstackFrameData* Worker::GetCallstackFrame( const CallstackFrameId& ptr ) const +{ + assert( ptr.custom == 0 ); + auto it = m_data.callstackFrameMap.find( ptr ); + if( it == m_data.callstackFrameMap.end() ) + { + return nullptr; + } + else + { + return it->second; + } +} + +#ifndef TRACY_NO_STATISTICS +const CallstackFrameData* Worker::GetParentCallstackFrame( const CallstackFrameId& ptr ) const +{ + assert( ptr.custom == 1 ); + auto it = m_data.parentCallstackFrameMap.find( ptr ); + if( it == m_data.parentCallstackFrameMap.end() ) + { + return nullptr; + } + else + { + return it->second; + } +} + +const Vector* Worker::GetSamplesForSymbol( uint64_t symAddr ) const +{ + assert( m_data.symbolSamplesReady ); + auto it = m_data.symbolSamples.find( symAddr ); + if( it == m_data.symbolSamples.end() ) return nullptr; + return &it->second; +} + +const Vector* Worker::GetChildSamples( uint64_t addr ) const +{ + assert( m_data.symbolSamplesReady ); + auto it = m_data.childSamples.find( addr ); + if( it == m_data.childSamples.end() ) return nullptr; + return &it->second; +} +#endif + +const SymbolData* Worker::GetSymbolData( uint64_t sym ) const +{ + auto it = m_data.symbolMap.find( sym ); + if( it == m_data.symbolMap.end() ) + { + return nullptr; + } + else + { + return &it->second; + } +} + +bool Worker::HasSymbolCode( uint64_t sym ) const +{ + return m_data.symbolCode.find( sym ) != m_data.symbolCode.end(); +} + +const char* Worker::GetSymbolCode( uint64_t sym, uint32_t& len ) const +{ + auto it = m_data.symbolCode.find( sym ); + if( it == m_data.symbolCode.end() ) return nullptr; + len = it->second.len; + return it->second.data; +} + +uint64_t Worker::GetSymbolForAddress( uint64_t address ) +{ + DoPostponedSymbols(); + auto it = std::lower_bound( m_data.symbolLoc.begin(), m_data.symbolLoc.end(), address, [] ( const auto& l, const auto& r ) { return l.addr + l.len < r; } ); + if( it == m_data.symbolLoc.end() || address < it->addr ) return 0; + return it->addr; +} + +uint64_t Worker::GetSymbolForAddress( uint64_t address, uint32_t& offset ) +{ + DoPostponedSymbols(); + auto it = std::lower_bound( m_data.symbolLoc.begin(), m_data.symbolLoc.end(), address, [] ( const auto& l, const auto& r ) { return l.addr + l.len < r; } ); + if( it == m_data.symbolLoc.end() || address < it->addr ) return 0; + offset = address - it->addr; + return it->addr; +} + +uint64_t Worker::GetInlineSymbolForAddress( uint64_t address ) const +{ + auto it = m_data.codeSymbolMap.find( address ); + if( it == m_data.codeSymbolMap.end() ) return 0; + return it->second; +} + +StringIdx Worker::GetLocationForAddress( uint64_t address, uint32_t& line ) const +{ + auto frame = GetCallstackFrame( PackPointer( address ) ); + if( !frame ) + { + line = 0; + return StringIdx(); + } + else + { + auto subFrame = frame->data[0]; + line = subFrame.line; + return subFrame.file; + } +} + +const uint64_t* Worker::GetInlineSymbolList( uint64_t sym, uint32_t len ) +{ + DoPostponedInlineSymbols(); + auto it = std::lower_bound( m_data.symbolLocInline.begin(), m_data.symbolLocInline.end(), sym ); + if( it == m_data.symbolLocInline.end() ) return nullptr; + if( *it >= sym + len ) return nullptr; + return it; +} + +int64_t Worker::GetZoneEndImpl( const ZoneEvent& ev ) +{ + assert( !ev.IsEndValid() ); + auto ptr = &ev; + for(;;) + { + if( !ptr->HasChildren() ) return ptr->Start(); + auto& children = GetZoneChildren( ptr->Child() ); + if( children.is_magic() ) + { + auto& c = *(Vector*)&children; + ptr = &c.back(); + } + else + { + ptr = children.back(); + } + if( ptr->IsEndValid() ) return ptr->End(); + } +} + +int64_t Worker::GetZoneEndImpl( const GpuEvent& ev ) +{ + assert( ev.GpuEnd() < 0 ); + auto ptr = &ev; + for(;;) + { + if( ptr->Child() < 0 ) return ptr->GpuStart() >= 0 ? ptr->GpuStart() : m_data.lastTime; + auto& children = GetGpuChildren( ptr->Child() ); + if( children.is_magic() ) + { + auto& c = *(Vector*)&children; + ptr = &c.back(); + } + else + { + ptr = children.back(); + } + if( ptr->GpuEnd() >= 0 ) return ptr->GpuEnd(); + } +} + +uint32_t Worker::FindStringIdx( const char* str ) const +{ + if( !str ) return 0; + charutil::StringKey key = { str, strlen( str ) }; + auto sit = m_data.stringMap.find( key ); + if( sit == m_data.stringMap.end() ) + { + return 0; + } + else + { + return sit->second; + } +} + +const char* Worker::GetString( uint64_t ptr ) const +{ + const auto it = m_data.strings.find( ptr ); + if( it == m_data.strings.end() || it->second == nullptr ) + { + return "???"; + } + else + { + return it->second; + } +} + +const char* Worker::GetString( const StringRef& ref ) const +{ + if( ref.isidx ) + { + assert( ref.active ); + return m_data.stringData[ref.str]; + } + else + { + if( ref.active ) + { + return GetString( ref.str ); + } + else + { + return "???"; + } + } +} + +const char* Worker::GetString( const StringIdx& idx ) const +{ + assert( idx.Active() ); + return m_data.stringData[idx.Idx()]; +} + +static const char* BadExternalThreadNames[] = { + "ntdll.dll", + "???", + nullptr +}; + +const char* Worker::GetThreadName( uint64_t id ) const +{ + const auto it = m_data.threadNames.find( id ); + if( it == m_data.threadNames.end() ) + { + const auto eit = m_data.externalNames.find( id ); + if( eit == m_data.externalNames.end() ) + { + return "???"; + } + else + { + return eit->second.second; + } + } + else + { + // Client should send additional information about thread name, to make this check unnecessary + const auto txt = it->second; + if( txt[0] >= '0' && txt[0] <= '9' && (uint64_t)atoi( txt ) == id ) + { + const auto eit = m_data.externalNames.find( id ); + if( eit != m_data.externalNames.end() ) + { + const char* ext = eit->second.second; + const char** ptr = BadExternalThreadNames; + while( *ptr ) + { + if( strcmp( *ptr, ext ) == 0 ) return txt; + ptr++; + } + return ext; + } + } + return txt; + } +} + +bool Worker::IsThreadLocal( uint64_t id ) +{ + auto td = RetrieveThread( id ); + return td && ( td->count > 0 || !td->samples.empty() ); +} + +bool Worker::IsThreadFiber( uint64_t id ) +{ + auto td = RetrieveThread( id ); + return td && ( td->isFiber ); +} + +const SourceLocation& Worker::GetSourceLocation( int16_t srcloc ) const +{ + if( srcloc < 0 ) + { + return *m_data.sourceLocationPayload[-srcloc-1]; + } + else + { + const auto it = m_data.sourceLocation.find( m_data.sourceLocationExpand[srcloc] ); + assert( it != m_data.sourceLocation.end() ); + return it->second; + } +} + +std::pair Worker::GetExternalName( uint64_t id ) const +{ + const auto it = m_data.externalNames.find( id ); + if( it == m_data.externalNames.end() ) + { + return std::make_pair( "???", "???" ); + } + else + { + return it->second; + } +} + +const char* Worker::GetZoneName( const SourceLocation& srcloc ) const +{ + if( srcloc.name.active ) + { + return GetString( srcloc.name ); + } + else + { + return GetString( srcloc.function ); + } +} + +const char* Worker::GetZoneName( const ZoneEvent& ev ) const +{ + auto& srcloc = GetSourceLocation( ev.SrcLoc() ); + return GetZoneName( ev, srcloc ); +} + +const char* Worker::GetZoneName( const ZoneEvent& ev, const SourceLocation& srcloc ) const +{ + if( HasZoneExtra( ev ) && GetZoneExtra( ev ).name.Active() ) + { + return GetString( GetZoneExtra( ev ).name ); + } + else if( srcloc.name.active ) + { + return GetString( srcloc.name ); + } + else + { + return GetString( srcloc.function ); + } +} + +const char* Worker::GetZoneName( const GpuEvent& ev ) const +{ + auto& srcloc = GetSourceLocation( ev.SrcLoc() ); + return GetZoneName( srcloc ); +} + +static bool strstr_nocase( const char* l, const char* r ) +{ + const auto lsz = strlen( l ); + const auto rsz = strlen( r ); + auto ll = (char*)alloca( lsz + 1 ); + auto rl = (char*)alloca( rsz + 1 ); + for( size_t i=0; i Worker::GetMatchingSourceLocation( const char* query, bool ignoreCase ) const +{ + std::vector match; + + const auto sz = m_data.sourceLocationExpand.size(); + for( size_t i=1; isecond; + const auto str = GetString( srcloc.name.active ? srcloc.name : srcloc.function ); + bool found = false; + if( ignoreCase ) + { + found = strstr_nocase( str, query ); + } + else + { + found = strstr( str, query ) != nullptr; + } + if( found ) + { + match.push_back( (int16_t)i ); + } + } + + for( auto& srcloc : m_data.sourceLocationPayload ) + { + const auto str = GetString( srcloc->name.active ? srcloc->name : srcloc->function ); + bool found = false; + if( ignoreCase ) + { + found = strstr_nocase( str, query ); + } + else + { + found = strstr( str, query ) != nullptr; + } + if( found ) + { + auto it = m_data.sourceLocationPayloadMap.find( (const SourceLocation*)srcloc ); + assert( it != m_data.sourceLocationPayloadMap.end() ); + match.push_back( -int16_t( it->second + 1 ) ); + } + } + + return match; +} + +#ifndef TRACY_NO_STATISTICS +Worker::SourceLocationZones& Worker::GetZonesForSourceLocation( int16_t srcloc ) +{ + assert( AreSourceLocationZonesReady() ); + static SourceLocationZones empty; + auto it = m_data.sourceLocationZones.find( srcloc ); + return it != m_data.sourceLocationZones.end() ? it->second : empty; +} + +const Worker::SourceLocationZones& Worker::GetZonesForSourceLocation( int16_t srcloc ) const +{ + assert( AreSourceLocationZonesReady() ); + static const SourceLocationZones empty; + auto it = m_data.sourceLocationZones.find( srcloc ); + return it != m_data.sourceLocationZones.end() ? it->second : empty; +} + +const SymbolStats* Worker::GetSymbolStats( uint64_t symAddr ) const +{ + assert( AreCallstackSamplesReady() ); + auto it = m_data.symbolStats.find( symAddr ); + if( it == m_data.symbolStats.end() ) + { + return nullptr; + } + else + { + return &it->second; + } +} + +const unordered_flat_map* Worker::GetSymbolInstructionPointers( uint64_t symAddr ) const +{ + assert( AreCallstackSamplesReady() ); + auto it = m_data.instructionPointersMap.find( symAddr ); + if( it == m_data.instructionPointersMap.end() ) + { + return nullptr; + } + else + { + return &it->second; + } +} +#endif + +void Worker::Network() +{ + auto ShouldExit = [this] { return m_shutdown.load( std::memory_order_relaxed ); }; + auto lz4buf = std::unique_ptr( new char[LZ4Size] ); + + for(;;) + { + { + std::unique_lock lock( m_netWriteLock ); + m_netWriteCv.wait( lock, [this] { return m_netWriteCnt > 0 || m_shutdown.load( std::memory_order_relaxed ); } ); + if( m_shutdown.load( std::memory_order_relaxed ) ) goto close; + m_netWriteCnt--; + } + + auto buf = m_buffer + m_bufferOffset; + lz4sz_t lz4sz; + if( !m_sock.Read( &lz4sz, sizeof( lz4sz ), 10, ShouldExit ) ) goto close; + if( !m_sock.Read( lz4buf.get(), lz4sz, 10, ShouldExit ) ) goto close; + auto bb = m_bytes.load( std::memory_order_relaxed ); + m_bytes.store( bb + sizeof( lz4sz ) + lz4sz, std::memory_order_relaxed ); + + auto sz = LZ4_decompress_safe_continue( (LZ4_streamDecode_t*)m_stream, lz4buf.get(), buf, lz4sz, TargetFrameSize ); + assert( sz >= 0 ); + bb = m_decBytes.load( std::memory_order_relaxed ); + m_decBytes.store( bb + sz, std::memory_order_relaxed ); + + { + std::lock_guard lock( m_netReadLock ); + m_netRead.push_back( NetBuffer { m_bufferOffset, sz } ); + m_netReadCv.notify_one(); + } + + m_bufferOffset += sz; + if( m_bufferOffset > TargetFrameSize * 2 ) m_bufferOffset = 0; + } + +close: + std::lock_guard lock( m_netReadLock ); + m_netRead.push_back( NetBuffer { -1 } ); + m_netReadCv.notify_one(); +} + +void Worker::Exec() +{ + auto ShouldExit = [this] { return m_shutdown.load( std::memory_order_relaxed ); }; + + for(;;) + { + if( m_shutdown.load( std::memory_order_relaxed ) ) { m_netWriteCv.notify_one(); return; }; + if( m_sock.Connect( m_addr.c_str(), m_port ) ) break; + std::this_thread::sleep_for( std::chrono::milliseconds( 10 ) ); + } + + std::chrono::time_point t0; + + m_sock.Send( HandshakeShibboleth, HandshakeShibbolethSize ); + uint32_t protocolVersion = ProtocolVersion; + m_sock.Send( &protocolVersion, sizeof( protocolVersion ) ); + HandshakeStatus handshake; + if( !m_sock.Read( &handshake, sizeof( handshake ), 10, ShouldExit ) ) + { + m_handshake.store( HandshakeDropped, std::memory_order_relaxed ); + goto close; + } + m_handshake.store( handshake, std::memory_order_relaxed ); + switch( handshake ) + { + case HandshakeWelcome: + break; + case HandshakeProtocolMismatch: + case HandshakeNotAvailable: + default: + goto close; + } + + m_data.framesBase = m_data.frames.Retrieve( 0, [this] ( uint64_t name ) { + auto fd = m_slab.AllocInit(); + fd->name = name; + fd->continuous = 1; + return fd; + }, [this] ( uint64_t name ) { + assert( name == 0 ); + char tmp[6] = "Frame"; + HandleFrameName( name, tmp, 5 ); + } ); + + { + WelcomeMessage welcome; + if( !m_sock.Read( &welcome, sizeof( welcome ), 10, ShouldExit ) ) + { + m_handshake.store( HandshakeDropped, std::memory_order_relaxed ); + goto close; + } + m_timerMul = welcome.timerMul; + m_data.baseTime = welcome.initBegin; + const auto initEnd = TscTime( welcome.initEnd ); + m_data.framesBase->frames.push_back( FrameEvent{ 0, -1, -1 } ); + m_data.framesBase->frames.push_back( FrameEvent{ initEnd, -1, -1 } ); + m_data.lastTime = initEnd; + m_delay = TscPeriod( welcome.delay ); + m_resolution = TscPeriod( welcome.resolution ); + m_pid = welcome.pid; + m_samplingPeriod = welcome.samplingPeriod; + m_onDemand = welcome.flags & WelcomeFlag::OnDemand; + m_captureProgram = welcome.programName; + m_captureTime = welcome.epoch; + m_executableTime = welcome.exectime; + m_ignoreMemFreeFaults = ( welcome.flags & WelcomeFlag::OnDemand ) || ( welcome.flags & WelcomeFlag::IsApple ); + m_ignoreFrameEndFaults = welcome.flags & WelcomeFlag::OnDemand; + m_data.cpuArch = (CpuArchitecture)welcome.cpuArch; + m_codeTransfer = welcome.flags & WelcomeFlag::CodeTransfer; + m_combineSamples = welcome.flags & WelcomeFlag::CombineSamples; + m_identifySamples = welcome.flags & WelcomeFlag::IdentifySamples; + m_data.cpuId = welcome.cpuId; + memcpy( m_data.cpuManufacturer, welcome.cpuManufacturer, 12 ); + m_data.cpuManufacturer[12] = '\0'; + + char dtmp[64]; + time_t date = welcome.epoch; + auto lt = localtime( &date ); + strftime( dtmp, 64, "%F %T", lt ); + char tmp[1024]; + sprintf( tmp, "%s @ %s", welcome.programName, dtmp ); + m_captureName = tmp; + + m_hostInfo = welcome.hostInfo; + + if( m_onDemand ) + { + OnDemandPayloadMessage onDemand; + if( !m_sock.Read( &onDemand, sizeof( onDemand ), 10, ShouldExit ) ) + { + m_handshake.store( HandshakeDropped, std::memory_order_relaxed ); + goto close; + } + m_data.frameOffset = onDemand.frames; + m_data.framesBase->frames.push_back( FrameEvent{ TscTime( onDemand.currentTime ), -1, -1 } ); + } + } + + m_serverQuerySpaceBase = m_serverQuerySpaceLeft = std::min( ( m_sock.GetSendBufSize() / ServerQueryPacketSize ), 8*1024 ) - 4; // leave space for terminate request + m_hasData.store( true, std::memory_order_release ); + + LZ4_setStreamDecode( (LZ4_streamDecode_t*)m_stream, nullptr, 0 ); + m_connected.store( true, std::memory_order_relaxed ); + { + std::lock_guard lock( m_netWriteLock ); + m_netWriteCnt = 2; + m_netWriteCv.notify_one(); + } + + t0 = std::chrono::high_resolution_clock::now(); + + for(;;) + { + if( m_shutdown.load( std::memory_order_relaxed ) || ( m_memoryLimit > 0 && memUsage.load( std::memory_order_relaxed ) > m_memoryLimit ) ) + { + QueryTerminate(); + goto close; + } + + NetBuffer netbuf; + { + std::unique_lock lock( m_netReadLock ); + m_netReadCv.wait( lock, [this] { return !m_netRead.empty(); } ); + netbuf = m_netRead.front(); + m_netRead.erase( m_netRead.begin() ); + } + if( netbuf.bufferOffset < 0 ) goto close; + + const char* ptr = m_buffer + netbuf.bufferOffset; + const char* end = ptr + netbuf.size; + + { + std::lock_guard lock( m_data.lock ); + while( ptr < end ) + { + auto ev = (const QueueItem*)ptr; + if( !DispatchProcess( *ev, ptr ) ) + { + if( m_failure != Failure::None ) HandleFailure( ptr, end ); + QueryTerminate(); + goto close; + } + } + + { + std::lock_guard lock( m_netWriteLock ); + m_netWriteCnt++; + m_netWriteCv.notify_one(); + } + + if( m_serverQuerySpaceLeft > 0 && !m_serverQueryQueuePrio.empty() ) + { + const auto toSend = std::min( m_serverQuerySpaceLeft, m_serverQueryQueuePrio.size() ); + m_sock.Send( m_serverQueryQueuePrio.data(), toSend * ServerQueryPacketSize ); + m_serverQuerySpaceLeft -= toSend; + if( toSend == m_serverQueryQueuePrio.size() ) + { + m_serverQueryQueuePrio.clear(); + } + else + { + m_serverQueryQueuePrio.erase( m_serverQueryQueuePrio.begin(), m_serverQueryQueuePrio.begin() + toSend ); + } + } + if( m_serverQuerySpaceLeft > 0 && !m_serverQueryQueue.empty() ) + { + const auto toSend = std::min( m_serverQuerySpaceLeft, m_serverQueryQueue.size() ); + m_sock.Send( m_serverQueryQueue.data(), toSend * ServerQueryPacketSize ); + m_serverQuerySpaceLeft -= toSend; + if( toSend == m_serverQueryQueue.size() ) + { + m_serverQueryQueue.clear(); + } + else + { + m_serverQueryQueue.erase( m_serverQueryQueue.begin(), m_serverQueryQueue.begin() + toSend ); + } + } + } + + auto t1 = std::chrono::high_resolution_clock::now(); + auto td = std::chrono::duration_cast( t1 - t0 ).count(); + enum { MbpsUpdateTime = 200 }; + if( td > MbpsUpdateTime ) + { + UpdateMbps( td ); + t0 = t1; + } + + if( m_terminate ) + { + if( m_pendingStrings != 0 || m_pendingThreads != 0 || m_pendingSourceLocation != 0 || m_pendingCallstackFrames != 0 || + m_data.plots.IsPending() || m_pendingCallstackId != 0 || m_pendingExternalNames != 0 || + m_pendingCallstackSubframes != 0 || m_pendingFrameImageData.image != nullptr || !m_pendingSymbols.empty() || + m_pendingSymbolCode != 0 || !m_serverQueryQueue.empty() || !m_serverQueryQueuePrio.empty() || + m_pendingSourceLocationPayload != 0 || m_pendingSingleString.ptr != nullptr || m_pendingSecondString.ptr != nullptr || + !m_sourceCodeQuery.empty() || m_pendingFibers != 0 ) + { + continue; + } + if( !m_crashed && !m_disconnect ) + { + bool done = true; + for( auto& v : m_data.threads ) + { + if( !v->stack.empty() ) + { + done = false; + break; + } + } + if( !done ) continue; + } + QueryTerminate(); + UpdateMbps( 0 ); + break; + } + } + +close: + Shutdown(); + m_netWriteCv.notify_one(); + m_sock.Close(); + m_connected.store( false, std::memory_order_relaxed ); +} + +void Worker::UpdateMbps( int64_t td ) +{ + const auto bytes = m_bytes.exchange( 0, std::memory_order_relaxed ); + const auto decBytes = m_decBytes.exchange( 0, std::memory_order_relaxed ); + std::lock_guard lock( m_mbpsData.lock ); + if( td != 0 ) + { + m_mbpsData.mbps.erase( m_mbpsData.mbps.begin() ); + m_mbpsData.mbps.emplace_back( bytes / ( td * 125.f ) ); + } + m_mbpsData.compRatio = decBytes == 0 ? 1 : float( bytes ) / decBytes; + m_mbpsData.queue = m_serverQueryQueue.size() + m_serverQueryQueuePrio.size(); + m_mbpsData.transferred += bytes; +} + +bool Worker::IsFailureThreadStringRetrieved() +{ + const auto name = GetThreadName( m_failureData.thread ); + return strcmp( name, "???" ) != 0; +} + +bool Worker::IsCallstackRetrieved( uint32_t callstack ) +{ + auto& cs = GetCallstack( callstack ); + for( auto& v : cs ) + { + auto frameData = GetCallstackFrame( v ); + if( !frameData ) return false; + } + return true; +} + +bool Worker::IsSourceLocationRetrieved( int16_t srcloc ) +{ + auto& sl = GetSourceLocation( srcloc ); + auto func = GetString( sl.function ); + auto file = GetString( sl.file ); + return strcmp( func, "???" ) != 0 && strcmp( file, "???" ) != 0; +} + +bool Worker::HasAllFailureData() +{ + if( m_failureData.thread != 0 && !IsFailureThreadStringRetrieved() ) return false; + if( m_failureData.srcloc != 0 && !IsSourceLocationRetrieved( m_failureData.srcloc ) ) return false; + if( m_failureData.callstack != 0 && !IsCallstackRetrieved( m_failureData.callstack ) ) return false; + return true; +} + +void Worker::HandleFailure( const char* ptr, const char* end ) +{ + if( HasAllFailureData() ) return; + for(;;) + { + while( ptr < end ) + { + auto ev = (const QueueItem*)ptr; + DispatchFailure( *ev, ptr ); + } + if( HasAllFailureData() ) return; + + { + std::lock_guard lock( m_netWriteLock ); + m_netWriteCnt++; + m_netWriteCv.notify_one(); + } + + if( m_serverQuerySpaceLeft > 0 && !m_serverQueryQueuePrio.empty() ) + { + const auto toSend = std::min( m_serverQuerySpaceLeft, m_serverQueryQueuePrio.size() ); + m_sock.Send( m_serverQueryQueuePrio.data(), toSend * ServerQueryPacketSize ); + m_serverQuerySpaceLeft -= toSend; + if( toSend == m_serverQueryQueuePrio.size() ) + { + m_serverQueryQueuePrio.clear(); + } + else + { + m_serverQueryQueuePrio.erase( m_serverQueryQueuePrio.begin(), m_serverQueryQueuePrio.begin() + toSend ); + } + } + if( m_serverQuerySpaceLeft > 0 && !m_serverQueryQueue.empty() ) + { + const auto toSend = std::min( m_serverQuerySpaceLeft, m_serverQueryQueue.size() ); + m_sock.Send( m_serverQueryQueue.data(), toSend * ServerQueryPacketSize ); + m_serverQuerySpaceLeft -= toSend; + if( toSend == m_serverQueryQueue.size() ) + { + m_serverQueryQueue.clear(); + } + else + { + m_serverQueryQueue.erase( m_serverQueryQueue.begin(), m_serverQueryQueue.begin() + toSend ); + } + } + + if( m_shutdown.load( std::memory_order_relaxed ) ) return; + + NetBuffer netbuf; + { + std::unique_lock lock( m_netReadLock ); + m_netReadCv.wait( lock, [this] { return !m_netRead.empty(); } ); + netbuf = m_netRead.front(); + m_netRead.erase( m_netRead.begin() ); + } + if( netbuf.bufferOffset < 0 ) return; + + ptr = m_buffer + netbuf.bufferOffset; + end = ptr + netbuf.size; + } +} + +void Worker::DispatchFailure( const QueueItem& ev, const char*& ptr ) +{ + if( ev.hdr.idx >= (int)QueueType::StringData ) + { + ptr += sizeof( QueueHeader ) + sizeof( QueueStringTransfer ); + if( ev.hdr.type == QueueType::FrameImageData || + ev.hdr.type == QueueType::SymbolCode || + ev.hdr.type == QueueType::SourceCode ) + { + if( ev.hdr.type == QueueType::SymbolCode || ev.hdr.type == QueueType::SourceCode ) + { + m_serverQuerySpaceLeft++; + } + uint32_t sz; + memcpy( &sz, ptr, sizeof( sz ) ); + ptr += sizeof( sz ) + sz; + } + else + { + uint16_t sz; + memcpy( &sz, ptr, sizeof( sz ) ); + ptr += sizeof( sz ); + switch( ev.hdr.type ) + { + case QueueType::StringData: + AddString( ev.stringTransfer.ptr, ptr, sz ); + m_serverQuerySpaceLeft++; + break; + case QueueType::ThreadName: + AddThreadString( ev.stringTransfer.ptr, ptr, sz ); + m_serverQuerySpaceLeft++; + break; + case QueueType::FiberName: + AddFiberName( ev.stringTransfer.ptr, ptr, sz ); + m_serverQuerySpaceLeft++; + break; + case QueueType::PlotName: + case QueueType::FrameName: + case QueueType::ExternalName: + m_serverQuerySpaceLeft++; + break; + default: + break; + } + ptr += sz; + } + } + else + { + uint16_t sz; + switch( ev.hdr.type ) + { + case QueueType::SingleStringData: + ptr += sizeof( QueueHeader ); + memcpy( &sz, ptr, sizeof( sz ) ); + ptr += sizeof( sz ); + AddSingleStringFailure( ptr, sz ); + ptr += sz; + break; + case QueueType::SecondStringData: + ptr += sizeof( QueueHeader ); + memcpy( &sz, ptr, sizeof( sz ) ); + ptr += sizeof( sz ); + AddSecondString( ptr, sz ); + ptr += sz; + break; + default: + ptr += QueueDataSize[ev.hdr.idx]; + switch( ev.hdr.type ) + { + case QueueType::SourceLocation: + AddSourceLocation( ev.srcloc ); + m_serverQuerySpaceLeft++; + break; + case QueueType::CallstackFrameSize: + ProcessCallstackFrameSize( ev.callstackFrameSize ); + m_serverQuerySpaceLeft++; + break; + case QueueType::CallstackFrame: + ProcessCallstackFrame( ev.callstackFrame, false ); + break; + case QueueType::SymbolInformation: + case QueueType::AckServerQueryNoop: + case QueueType::AckSourceCodeNotAvailable: + case QueueType::AckSymbolCodeNotAvailable: + m_serverQuerySpaceLeft++; + break; + default: + break; + } + } + } +} + +void Worker::Query( ServerQuery type, uint64_t data, uint32_t extra ) +{ + ServerQueryPacket query { type, data, extra }; + if( m_serverQuerySpaceLeft > 0 && m_serverQueryQueuePrio.empty() && m_serverQueryQueue.empty() ) + { + m_serverQuerySpaceLeft--; + m_sock.Send( &query, ServerQueryPacketSize ); + } + else if( IsQueryPrio( type ) ) + { + m_serverQueryQueuePrio.push_back( query ); + } + else + { + m_serverQueryQueue.push_back( query ); + } +} + +void Worker::QueryTerminate() +{ + ServerQueryPacket query { ServerQueryTerminate, 0, 0 }; + m_sock.Send( &query, ServerQueryPacketSize ); +} + +void Worker::QuerySourceFile( const char* fn, const char* image ) +{ + if( image ) QueryDataTransfer( image, strlen( image ) + 1 ); + QueryDataTransfer( fn, strlen( fn ) + 1 ); + m_sourceCodeQuery.emplace( m_nextSourceCodeQuery, fn ); + Query( ServerQuerySourceCode, m_nextSourceCodeQuery ); + m_nextSourceCodeQuery++; +} + +void Worker::QueryDataTransfer( const void* ptr, size_t size ) +{ + Query( ServerQueryDataTransfer, size ); + auto data = (const char*)ptr; + while( size > 0 ) + { + uint64_t d8; + uint32_t d4; + if( size >= 12 ) + { + memcpy( &d8, data, 8 ); + memcpy( &d4, data+8, 4 ); + data += 12; + size -= 12; + } + else if( size > 8 ) + { + memcpy( &d8, data, 8 ); + memset( &d4, 0, 4 ); + memcpy( &d4, data+8, size-8 ); + size = 0; + } + else + { + memset( &d8, 0, 8 ); + memset( &d4, 0, 4 ); + memcpy( &d8, data, size ); + size = 0; + } + Query( ServerQueryDataTransferPart, d8, d4 ); + } +} + +bool Worker::DispatchProcess( const QueueItem& ev, const char*& ptr ) +{ + if( ev.hdr.idx >= (int)QueueType::StringData ) + { + ptr += sizeof( QueueHeader ) + sizeof( QueueStringTransfer ); + if( ev.hdr.type == QueueType::FrameImageData || + ev.hdr.type == QueueType::SymbolCode || + ev.hdr.type == QueueType::SourceCode ) + { + uint32_t sz; + memcpy( &sz, ptr, sizeof( sz ) ); + ptr += sizeof( sz ); + switch( ev.hdr.type ) + { + case QueueType::FrameImageData: + AddFrameImageData( ptr, sz ); + break; + case QueueType::SymbolCode: + AddSymbolCode( ev.stringTransfer.ptr, ptr, sz ); + m_serverQuerySpaceLeft++; + break; + case QueueType::SourceCode: + AddSourceCode( (uint32_t)ev.stringTransfer.ptr, ptr, sz ); + m_serverQuerySpaceLeft++; + break; + default: + assert( false ); + break; + } + ptr += sz; + } + else + { + uint16_t sz; + memcpy( &sz, ptr, sizeof( sz ) ); + ptr += sizeof( sz ); + switch( ev.hdr.type ) + { + case QueueType::StringData: + AddString( ev.stringTransfer.ptr, ptr, sz ); + m_serverQuerySpaceLeft++; + break; + case QueueType::ThreadName: + AddThreadString( ev.stringTransfer.ptr, ptr, sz ); + m_serverQuerySpaceLeft++; + break; + case QueueType::FiberName: + AddFiberName( ev.stringTransfer.ptr, ptr, sz ); + m_serverQuerySpaceLeft++; + break; + case QueueType::PlotName: + HandlePlotName( ev.stringTransfer.ptr, ptr, sz ); + m_serverQuerySpaceLeft++; + break; + case QueueType::SourceLocationPayload: + AddSourceLocationPayload( ptr, sz ); + break; + case QueueType::CallstackPayload: + AddCallstackPayload( ptr, sz ); + break; + case QueueType::FrameName: + HandleFrameName( ev.stringTransfer.ptr, ptr, sz ); + m_serverQuerySpaceLeft++; + break; + case QueueType::CallstackAllocPayload: + AddCallstackAllocPayload( ptr ); + break; + case QueueType::ExternalName: + AddExternalName( ev.stringTransfer.ptr, ptr, sz ); + m_serverQuerySpaceLeft++; + break; + case QueueType::ExternalThreadName: + AddExternalThreadName( ev.stringTransfer.ptr, ptr, sz ); + break; + default: + assert( false ); + break; + } + ptr += sz; + } + return true; + } + else + { + uint16_t sz; + switch( ev.hdr.type ) + { + case QueueType::SingleStringData: + ptr += sizeof( QueueHeader ); + memcpy( &sz, ptr, sizeof( sz ) ); + ptr += sizeof( sz ); + AddSingleString( ptr, sz ); + ptr += sz; + return true; + case QueueType::SecondStringData: + ptr += sizeof( QueueHeader ); + memcpy( &sz, ptr, sizeof( sz ) ); + ptr += sizeof( sz ); + AddSecondString( ptr, sz ); + ptr += sz; + return true; + default: + ptr += QueueDataSize[ev.hdr.idx]; + return Process( ev ); + } + } +} + +void Worker::CheckSourceLocation( uint64_t ptr ) +{ + if( m_data.checkSrclocLast != ptr ) + { + m_data.checkSrclocLast = ptr; + if( m_data.sourceLocation.find( ptr ) == m_data.sourceLocation.end() ) + { + NewSourceLocation( ptr ); + } + } +} + +void Worker::NewSourceLocation( uint64_t ptr ) +{ + static const SourceLocation emptySourceLocation = {}; + + if( m_data.sourceLocation.size() > std::numeric_limits::max() ) + { + SourceLocationOverflowFailure(); + return; + } + + m_data.sourceLocation.emplace( ptr, emptySourceLocation ); + m_pendingSourceLocation++; + m_sourceLocationQueue.push_back( ptr ); + + Query( ServerQuerySourceLocation, ptr ); +} + +int16_t Worker::ShrinkSourceLocationReal( uint64_t srcloc ) +{ + auto it = m_sourceLocationShrink.find( srcloc ); + if( it != m_sourceLocationShrink.end() ) + { + m_data.shrinkSrclocLast.first = srcloc; + m_data.shrinkSrclocLast.second = it->second; + return it->second; + } + else + { + return NewShrinkedSourceLocation( srcloc ); + } +} + +int16_t Worker::NewShrinkedSourceLocation( uint64_t srcloc ) +{ + assert( m_data.sourceLocationExpand.size() < std::numeric_limits::max() ); + const auto sz = int16_t( m_data.sourceLocationExpand.size() ); + m_data.sourceLocationExpand.push_back( srcloc ); +#ifndef TRACY_NO_STATISTICS + auto res = m_data.sourceLocationZones.emplace( sz, SourceLocationZones() ); + m_data.srclocZonesLast.first = sz; + m_data.srclocZonesLast.second = &res.first->second; +#else + auto res = m_data.sourceLocationZonesCnt.emplace( sz, 0 ); + m_data.srclocCntLast.first = sz; + m_data.srclocCntLast.second = &res.first->second; +#endif + m_sourceLocationShrink.emplace( srcloc, sz ); + m_data.shrinkSrclocLast.first = srcloc; + m_data.shrinkSrclocLast.second = sz; + return sz; +} + +void Worker::InsertMessageData( MessageData* msg ) +{ + if( m_data.messages.empty() ) + { + m_data.messages.push_back( msg ); + } + else if( m_data.messages.back()->time < msg->time ) + { + m_data.messages.push_back_non_empty( msg ); + } + else + { + auto mit = std::lower_bound( m_data.messages.begin(), m_data.messages.end(), msg->time, [] ( const auto& lhs, const auto& rhs ) { return lhs->time < rhs; } ); + m_data.messages.insert( mit, msg ); + } + + auto td = GetCurrentThreadData(); + auto vec = &td->messages; + if( vec->empty() ) + { + vec->push_back( msg ); + } + else if( vec->back()->time < msg->time ) + { + vec->push_back_non_empty( msg ); + } + else + { + auto tmit = std::lower_bound( vec->begin(), vec->end(), msg->time, [] ( const auto& lhs, const auto& rhs ) { return lhs->time < rhs; } ); + vec->insert( tmit, msg ); + } +} + +ThreadData* Worker::NoticeThreadReal( uint64_t thread ) +{ + auto it = m_threadMap.find( thread ); + if( it != m_threadMap.end() ) + { + m_data.threadDataLast.first = thread; + m_data.threadDataLast.second = it->second; + return it->second; + } + else + { + CheckThreadString( thread ); + return NewThread( thread, false, 0 ); + } +} + +ThreadData* Worker::RetrieveThreadReal( uint64_t thread ) +{ + auto it = m_threadMap.find( thread ); + if( it != m_threadMap.end() ) + { + m_data.threadDataLast.first = thread; + m_data.threadDataLast.second = it->second; + return it->second; + } + else + { + return nullptr; + } +} + +ThreadData* Worker::GetCurrentThreadData() +{ + auto td = m_threadCtxData; + if( !td ) td = m_threadCtxData = NoticeThread( m_threadCtx ); + if( td->fiber ) td = td->fiber; + return td; +} + +#ifndef TRACY_NO_STATISTICS +Worker::SourceLocationZones* Worker::GetSourceLocationZonesReal( uint16_t srcloc ) +{ + auto it = m_data.sourceLocationZones.find( srcloc ); + assert( it != m_data.sourceLocationZones.end() ); + m_data.srclocZonesLast.first = srcloc; + m_data.srclocZonesLast.second = &it->second; + return &it->second; +} + +Worker::GpuSourceLocationZones* Worker::GetGpuSourceLocationZonesReal( uint16_t srcloc ) +{ + auto it = m_data.gpuSourceLocationZones.find( srcloc ); + if( it == m_data.gpuSourceLocationZones.end() ) + { + it = m_data.gpuSourceLocationZones.emplace( srcloc, GpuSourceLocationZones() ).first; + } + m_data.gpuZonesLast.first = srcloc; + m_data.gpuZonesLast.second = &it->second; + return &it->second; +} +#else +uint64_t* Worker::GetSourceLocationZonesCntReal( uint16_t srcloc ) +{ + auto it = m_data.sourceLocationZonesCnt.find( srcloc ); + assert( it != m_data.sourceLocationZonesCnt.end() ); + m_data.srclocCntLast.first = srcloc; + m_data.srclocCntLast.second = &it->second; + return &it->second; +} + +uint64_t* Worker::GetGpuSourceLocationZonesCntReal( uint16_t srcloc ) +{ + auto it = m_data.gpuSourceLocationZonesCnt.find( srcloc ); + if( it == m_data.gpuSourceLocationZonesCnt.end() ) + { + it = m_data.gpuSourceLocationZonesCnt.emplace( srcloc, 0 ).first; + } + m_data.gpuCntLast.first = srcloc; + m_data.gpuCntLast.second = &it->second; + return &it->second; +} +#endif + +const ThreadData* Worker::GetThreadData( uint64_t tid ) const +{ + auto it = m_threadMap.find( tid ); + if( it == m_threadMap.end() ) return nullptr; + return it->second; +} + +const MemData& Worker::GetMemoryNamed( uint64_t name ) const +{ + auto it = m_data.memNameMap.find( name ); + assert( it != m_data.memNameMap.end() ); + return *it->second; +} + +ThreadData* Worker::NewThread( uint64_t thread, bool fiber, int32_t groupHint ) +{ + auto td = m_slab.AllocInit(); + td->id = thread; + td->count = 0; + td->nextZoneId = 0; +#ifndef TRACY_NO_STATISTICS + td->ghostIdx = 0; +#endif + td->kernelSampleCnt = 0; + td->pendingSample.time.Clear(); + td->isFiber = fiber; + td->fiber = nullptr; + td->stackCount = (uint8_t*)m_slab.AllocBig( sizeof( uint8_t ) * 64*1024 ); + memset( td->stackCount, 0, sizeof( uint8_t ) * 64*1024 ); + td->groupHint = groupHint; + m_data.threads.push_back( td ); + m_threadMap.emplace( thread, td ); + m_data.threadDataLast.first = thread; + m_data.threadDataLast.second = td; + return td; +} + +void Worker::NewZone( ZoneEvent* zone ) +{ + m_data.zonesCnt++; + + auto td = GetCurrentThreadData(); + td->count++; + td->IncStackCount( zone->SrcLoc() ); + const auto ssz = td->stack.size(); + if( ssz == 0 ) + { + td->stack.push_back( zone ); + td->timeline.push_back( zone ); + } + else + { + auto& back = td->stack.data()[ssz-1]; + if( !back->HasChildren() ) + { + back->SetChild( int32_t( m_data.zoneChildren.size() ) ); + if( m_data.zoneVectorCache.empty() ) + { + m_data.zoneChildren.push_back( Vector>( zone ) ); + } + else + { + Vector> vze = std::move( m_data.zoneVectorCache.back_and_pop() ); + assert( !vze.empty() ); + vze.clear(); + vze.push_back_non_empty( zone ); + m_data.zoneChildren.push_back( std::move( vze ) ); + } + } + else + { + const auto backChild = back->Child(); + assert( !m_data.zoneChildren[backChild].empty() ); + m_data.zoneChildren[backChild].push_back_non_empty( zone ); + } + td->stack.push_back_non_empty( zone ); + } + + td->zoneIdStack.push_back( td->nextZoneId ); + td->nextZoneId = 0; + +#ifndef TRACY_NO_STATISTICS + td->childTimeStack.push_back( 0 ); +#endif +} + +void Worker::InsertLockEvent( LockMap& lockmap, LockEvent* lev, uint64_t thread, int64_t time ) +{ + if( m_data.lastTime < time ) m_data.lastTime = time; + + NoticeThread( thread ); + + auto it = lockmap.threadMap.find( thread ); + if( it == lockmap.threadMap.end() ) + { + assert( lockmap.threadList.size() < MaxLockThreads ); + it = lockmap.threadMap.emplace( thread, lockmap.threadList.size() ).first; + lockmap.threadList.emplace_back( thread ); + } + lev->thread = it->second; + assert( lev->thread == it->second ); + auto& timeline = lockmap.timeline; + if( timeline.empty() ) + { + timeline.push_back( { lev } ); + UpdateLockCount( lockmap, timeline.size() - 1 ); + } + else + { + assert( timeline.back().ptr->Time() <= time ); + timeline.push_back_non_empty( { lev } ); + UpdateLockCount( lockmap, timeline.size() - 1 ); + } + + auto& range = lockmap.range[it->second]; + if( range.start > time ) range.start = time; + if( range.end < time ) range.end = time; +} + +bool Worker::CheckString( uint64_t ptr ) +{ + if( ptr == 0 ) return true; + if( m_data.strings.find( ptr ) != m_data.strings.end() ) return true; + + m_data.strings.emplace( ptr, "???" ); + m_pendingStrings++; + + Query( ServerQueryString, ptr ); + return false; +} + +void Worker::CheckThreadString( uint64_t id ) +{ + if( m_data.threadNames.find( id ) != m_data.threadNames.end() ) return; + + m_data.threadNames.emplace( id, "???" ); + m_pendingThreads++; + + if( m_sock.IsValid() ) Query( ServerQueryThreadString, id ); +} + +void Worker::CheckFiberName( uint64_t id, uint64_t tid ) +{ + if( m_data.threadNames.find( tid ) != m_data.threadNames.end() ) return; + + m_data.threadNames.emplace( tid, "???" ); + m_pendingFibers++; + + if( m_sock.IsValid() ) Query( ServerQueryFiberName, id ); +} + +void Worker::CheckExternalName( uint64_t id ) +{ + if( m_data.externalNames.find( id ) != m_data.externalNames.end() ) return; + + m_data.externalNames.emplace( id, std::make_pair( "???", "???" ) ); + m_pendingExternalNames += 2; + + Query( ServerQueryExternalName, id ); +} + +void Worker::AddSourceLocation( const QueueSourceLocation& srcloc ) +{ + assert( m_pendingSourceLocation > 0 ); + m_pendingSourceLocation--; + + const auto ptr = m_sourceLocationQueue.front(); + m_sourceLocationQueue.erase( m_sourceLocationQueue.begin() ); + + auto it = m_data.sourceLocation.find( ptr ); + assert( it != m_data.sourceLocation.end() ); + CheckString( srcloc.name ); + if( CheckString( srcloc.file ) ) + { + StringRef ref( StringRef::Ptr, srcloc.file ); + if( srcloc.file != 0 && m_checkedFileStrings.find( ref ) == m_checkedFileStrings.end() && m_pendingFileStrings.find( ref ) == m_pendingFileStrings.end() ) + { + CacheSource( ref ); + } + } + else + { + StringRef ref( StringRef::Ptr, srcloc.file ); + assert( m_checkedFileStrings.find( ref ) == m_checkedFileStrings.end() ); + if( m_pendingFileStrings.find( ref ) == m_pendingFileStrings.end() ) + { + m_pendingFileStrings.emplace( ref ); + } + } + CheckString( srcloc.function ); + const uint32_t color = ( srcloc.b << 16 ) | ( srcloc.g << 8 ) | srcloc.r; + it->second = SourceLocation {{ srcloc.name == 0 ? StringRef() : StringRef( StringRef::Ptr, srcloc.name ), StringRef( StringRef::Ptr, srcloc.function ), StringRef( StringRef::Ptr, srcloc.file ), srcloc.line, color }}; +} + +void Worker::AddSourceLocationPayload( const char* data, size_t sz ) +{ + const auto start = data; + + assert( m_pendingSourceLocationPayload == 0 ); + + uint32_t color, line; + memcpy( &color, data, 4 ); + memcpy( &line, data + 4, 4 ); + data += 8; + auto end = data + strlen( data ); + + const auto func = StoreString( data, end - data ); + end++; + + data = end; + end = data + strlen( data ); + const auto source = StoreString( data, end - data ); + end++; + + const auto nsz = sz - ( end - start ); + + color = ( ( color & 0x00FF0000 ) >> 16 ) | + ( ( color & 0x0000FF00 ) ) | + ( ( color & 0x000000FF ) << 16 ); + + SourceLocation srcloc {{ nsz == 0 ? StringRef() : StringRef( StringRef::Idx, StoreString( end, nsz ).idx ), StringRef( StringRef::Idx, func.idx ), StringRef( StringRef::Idx, source.idx ), line, color }}; + auto it = m_data.sourceLocationPayloadMap.find( &srcloc ); + if( it == m_data.sourceLocationPayloadMap.end() ) + { + auto slptr = m_slab.Alloc(); + memcpy( slptr, &srcloc, sizeof( srcloc ) ); + uint32_t idx = m_data.sourceLocationPayload.size(); + if( idx+1 > std::numeric_limits::max() ) + { + SourceLocationOverflowFailure(); + return; + } + m_data.sourceLocationPayloadMap.emplace( slptr, idx ); + m_pendingSourceLocationPayload = -int16_t( idx + 1 ); + m_data.sourceLocationPayload.push_back( slptr ); + if( m_checkedFileStrings.find( srcloc.file ) == m_checkedFileStrings.end() ) + { + CacheSource( srcloc.file ); + } + const auto key = -int16_t( idx + 1 ); +#ifndef TRACY_NO_STATISTICS + auto res = m_data.sourceLocationZones.emplace( key, SourceLocationZones() ); + m_data.srclocZonesLast.first = key; + m_data.srclocZonesLast.second = &res.first->second; +#else + auto res = m_data.sourceLocationZonesCnt.emplace( key, 0 ); + m_data.srclocCntLast.first = key; + m_data.srclocCntLast.second = &res.first->second; +#endif + } + else + { + m_pendingSourceLocationPayload = -int16_t( it->second + 1 ); + } +} + +void Worker::AddString( uint64_t ptr, const char* str, size_t sz ) +{ + assert( m_pendingStrings > 0 ); + m_pendingStrings--; + auto it = m_data.strings.find( ptr ); + assert( it != m_data.strings.end() && strcmp( it->second, "???" ) == 0 ); + const auto sl = StoreString( str, sz ); + it->second = sl.ptr; + + StringRef ref( StringRef::Ptr, ptr ); + auto sit = m_pendingFileStrings.find( ref ); + if( sit != m_pendingFileStrings.end() ) + { + m_pendingFileStrings.erase( sit ); + CacheSource( ref ); + } +} + +void Worker::AddThreadString( uint64_t id, const char* str, size_t sz ) +{ + assert( m_pendingThreads > 0 ); + m_pendingThreads--; + auto it = m_data.threadNames.find( id ); + assert( it != m_data.threadNames.end() && strcmp( it->second, "???" ) == 0 ); + const auto sl = StoreString( str, sz ); + it->second = sl.ptr; +} + +void Worker::AddFiberName( uint64_t id, const char* str, size_t sz ) +{ + assert( m_pendingFibers > 0 ); + m_pendingFibers--; + auto it = m_data.fiberToThreadMap.find( id ); + assert( it != m_data.fiberToThreadMap.end() ); + auto tit = m_data.threadNames.find( it->second ); + assert( tit != m_data.threadNames.end() && strcmp( tit->second, "???" ) == 0 ); + const auto sl = StoreString( str, sz ); + tit->second = sl.ptr; +} + +void Worker::AddSingleString( const char* str, size_t sz ) +{ + assert( m_pendingSingleString.ptr == nullptr ); + m_pendingSingleString = StoreString( str, sz ); +} + +void Worker::AddSingleStringFailure( const char* str, size_t sz ) +{ + // During failure dispatch processing of most events is ignored, but string data + // is still send. Just ignore anything that was already in the staging area. + m_pendingSingleString = StoreString( str, sz ); +} + +void Worker::AddSecondString( const char* str, size_t sz ) +{ + assert( m_pendingSecondString.ptr == nullptr ); + m_pendingSecondString = StoreString( str, sz ); +} + +void Worker::AddExternalName( uint64_t ptr, const char* str, size_t sz ) +{ + assert( m_pendingExternalNames > 0 ); + m_pendingExternalNames--; + auto it = m_data.externalNames.find( ptr ); + assert( it != m_data.externalNames.end() && strcmp( it->second.first, "???" ) == 0 ); + const auto sl = StoreString( str, sz ); + it->second.first = sl.ptr; +} + +void Worker::AddExternalThreadName( uint64_t ptr, const char* str, size_t sz ) +{ + assert( m_pendingExternalNames > 0 ); + m_pendingExternalNames--; + auto it = m_data.externalNames.find( ptr ); + assert( it != m_data.externalNames.end() && strcmp( it->second.second, "???" ) == 0 ); + const auto sl = StoreString( str, sz ); + it->second.second = sl.ptr; +} + +void Worker::AddFrameImageData( const char* data, size_t sz ) +{ + assert( m_pendingFrameImageData.image == nullptr ); + assert( sz % 8 == 0 ); + // Input data buffer cannot be changed, as it is used as LZ4 dictionary. + if( m_frameImageBufferSize < sz ) + { + m_frameImageBufferSize = sz; + delete[] m_frameImageBuffer; + m_frameImageBuffer = new char[sz]; + } + auto src = (uint8_t*)data; + auto dst = (uint8_t*)m_frameImageBuffer; + memcpy( dst, src, sz ); + m_texcomp.FixOrder( (char*)dst, sz/8 ); + m_texcomp.Rdo( (char*)dst, sz/8 ); + m_pendingFrameImageData.image = m_texcomp.Pack( m_frameImageBuffer, sz, m_pendingFrameImageData.csz, m_slab ); +} + +void Worker::AddSymbolCode( uint64_t ptr, const char* data, size_t sz ) +{ + assert( m_pendingSymbolCode > 0 ); + m_pendingSymbolCode--; + + auto code = (char*)m_slab.AllocBig( sz ); + memcpy( code, data, sz ); + m_data.symbolCode.emplace( ptr, MemoryBlock{ code, uint32_t( sz ) } ); + m_data.symbolCodeSize += sz; + + if( m_data.cpuArch == CpuArchUnknown ) return; + csh handle; + cs_err rval = CS_ERR_ARCH; + switch( m_data.cpuArch ) + { + case CpuArchX86: + rval = cs_open( CS_ARCH_X86, CS_MODE_32, &handle ); + break; + case CpuArchX64: + rval = cs_open( CS_ARCH_X86, CS_MODE_64, &handle ); + break; + case CpuArchArm32: + rval = cs_open( CS_ARCH_ARM, CS_MODE_ARM, &handle ); + break; + case CpuArchArm64: + rval = cs_open( CS_ARCH_ARM64, CS_MODE_ARM, &handle ); + break; + default: + assert( false ); + break; + } + if( rval != CS_ERR_OK ) return; + cs_option( handle, CS_OPT_DETAIL, CS_OPT_ON ); + cs_insn* insn; + size_t cnt = cs_disasm( handle, (const uint8_t*)code, sz, ptr, 0, &insn ); + if( cnt > 0 ) + { + for( size_t i=0; isecond; + m_sourceCodeQuery.erase( it ); + if( m_data.sourceFileCache.find( file ) != m_data.sourceFileCache.end() ) return; + auto src = (char*)m_slab.AllocBig( sz ); + memcpy( src, data, sz ); + m_data.sourceFileCache.emplace( file, MemoryBlock{ src, uint32_t( sz ) } ); +} + +CallstackFrameId Worker::PackPointer( uint64_t ptr ) const +{ + assert( ( ( ptr & 0x3000000000000000 ) << 2 ) == ( ptr & 0xC000000000000000 ) ); + CallstackFrameId id; + id.idx = ptr; + id.sel = 0; + id.custom = 0; + return id; +} + +uint64_t Worker::GetCanonicalPointer( const CallstackFrameId& id ) const +{ + assert( id.sel == 0 ); + return ( id.idx & 0x3FFFFFFFFFFFFFFF ) | ( ( id.idx & 0x3000000000000000 ) << 2 ); +} + +void Worker::AddCallstackPayload( const char* _data, size_t _sz ) +{ + assert( m_pendingCallstackId == 0 ); + + const auto sz = _sz / sizeof( uint64_t ); + const auto memsize = sizeof( VarArray ) + sz * sizeof( CallstackFrameId ); + auto mem = (char*)m_slab.AllocRaw( memsize ); + + auto data = (CallstackFrameId*)mem; + auto dst = data; + auto src = (uint64_t*)_data; + for( size_t i=0; i*)( mem + sz * sizeof( CallstackFrameId ) ); + new(arr) VarArray( sz, data ); + + uint32_t idx; + auto it = m_data.callstackMap.find( arr ); + if( it == m_data.callstackMap.end() ) + { + idx = m_data.callstackPayload.size(); + m_data.callstackMap.emplace( arr, idx ); + m_data.callstackPayload.push_back( arr ); + + for( auto& frame : *arr ) + { + auto fit = m_data.callstackFrameMap.find( frame ); + if( fit == m_data.callstackFrameMap.end() ) + { + m_pendingCallstackFrames++; + Query( ServerQueryCallstackFrame, GetCanonicalPointer( frame ) ); + } + } + } + else + { + idx = it->second; + m_slab.Unalloc( memsize ); + } + + m_pendingCallstackId = idx; +} + +void Worker::AddCallstackAllocPayload( const char* data ) +{ + CallstackFrameId stack[64]; + uint8_t sz; + memcpy( &sz, data, 1 ); data++; + assert( sz <= 64 ); + for( uint8_t i=0; i(); + memcpy( frame, &cf, sizeof( CallstackFrame ) ); + auto frameData = m_slab.AllocInit(); + frameData->data = frame; + frameData->size = 1; + id.idx = m_callstackAllocNextIdx++; + id.sel = 1; + id.custom = 0; + m_data.callstackFrameMap.emplace( id, frameData ); + m_data.revFrameMap.emplace( frameData, id ); + } + else + { + id = it->second; + } + stack[i] = id; + } + + VarArray* arr; + size_t memsize; + if( m_pendingCallstackId != 0 ) + { + const auto nativeCs = m_data.callstackPayload[m_pendingCallstackId]; + const auto nsz = nativeCs->size(); + const auto tsz = sz + nsz; + + memsize = sizeof( VarArray ) + tsz * sizeof( CallstackFrameId ); + auto mem = (char*)m_slab.AllocRaw( memsize ); + memcpy( mem, stack, sizeof( CallstackFrameId ) * sz ); + memcpy( mem + sizeof( CallstackFrameId ) * sz, nativeCs->data(), sizeof( CallstackFrameId ) * nsz ); + + arr = (VarArray*)( mem + tsz * sizeof( CallstackFrameId ) ); + new(arr) VarArray( tsz, (CallstackFrameId*)mem ); + } + else + { + memsize = sizeof( VarArray ) + sz * sizeof( CallstackFrameId ); + auto mem = (char*)m_slab.AllocRaw( memsize ); + memcpy( mem, stack, sizeof( CallstackFrameId ) * sz ); + + arr = (VarArray*)( mem + sz * sizeof( CallstackFrameId ) ); + new(arr) VarArray( sz, (CallstackFrameId*)mem ); + } + + uint32_t idx; + auto it = m_data.callstackMap.find( arr ); + if( it == m_data.callstackMap.end() ) + { + idx = m_data.callstackPayload.size(); + m_data.callstackMap.emplace( arr, idx ); + m_data.callstackPayload.push_back( arr ); + + for( auto& frame : *arr ) + { + auto fit = m_data.callstackFrameMap.find( frame ); + if( fit == m_data.callstackFrameMap.end() ) + { + m_pendingCallstackFrames++; + Query( ServerQueryCallstackFrame, GetCanonicalPointer( frame ) ); + } + } + } + else + { + idx = it->second; + m_slab.Unalloc( memsize ); + } + + m_pendingCallstackId = idx; +} + +uint32_t Worker::MergeCallstacks( uint32_t first, uint32_t second ) +{ + const auto& cs1 = GetCallstack( first ); + const auto& cs2 = GetCallstack( second ); + + const auto sz1 = cs1.size(); + const auto sz2 = cs2.size(); + const auto tsz = sz1 + sz2; + + size_t memsize = sizeof( VarArray ) + tsz * sizeof( CallstackFrameId ); + auto mem = (char*)m_slab.AllocRaw( memsize ); + memcpy( mem, cs1.data(), sizeof( CallstackFrameId ) * sz1 ); + memcpy( mem + sizeof( CallstackFrameId ) * sz1, cs2.data(), sizeof( CallstackFrameId ) * sz2 ); + + VarArray* arr = (VarArray*)( mem + tsz * sizeof( CallstackFrameId ) ); + new(arr) VarArray( tsz, (CallstackFrameId*)mem ); + + uint32_t idx; + auto it = m_data.callstackMap.find( arr ); + if( it == m_data.callstackMap.end() ) + { + idx = m_data.callstackPayload.size(); + m_data.callstackMap.emplace( arr, idx ); + m_data.callstackPayload.push_back( arr ); + } + else + { + idx = it->second; + m_slab.Unalloc( memsize ); + } + return idx; +} + +void Worker::InsertPlot( PlotData* plot, int64_t time, double val ) +{ + if( plot->data.empty() ) + { + plot->min = val; + plot->max = val; + plot->sum = val; + plot->data.push_back( { Int48( time ), val } ); + } + else + { + if( plot->min > val ) plot->min = val; + else if( plot->max < val ) plot->max = val; + plot->sum += val; + plot->data.push_back( { Int48( time ), val } ); + } +} + +void Worker::HandlePlotName( uint64_t name, const char* str, size_t sz ) +{ + const auto sl = StoreString( str, sz ); + m_data.plots.StringDiscovered( name, sl, m_data.strings, [this] ( PlotData* dst, PlotData* src ) { + for( auto& v : src->data ) + { + InsertPlot( dst, v.time.Val(), v.val ); + } + } ); +} + +void Worker::HandleFrameName( uint64_t name, const char* str, size_t sz ) +{ + const auto sl = StoreString( str, sz ); + m_data.frames.StringDiscovered( name, sl, m_data.strings, [] ( FrameData* dst, FrameData* src ) { + auto sz = dst->frames.size(); + dst->frames.insert( dst->frames.end(), src->frames.begin(), src->frames.end() ); + std::inplace_merge( dst->frames.begin(), dst->frames.begin() + sz, dst->frames.end(), [] ( const auto& lhs, const auto& rhs ) { return lhs.start < rhs.start; } ); + } ); +} + +void Worker::DoPostponedSymbols() +{ + if( m_data.newSymbolsIndex >= 0 ) + { +#ifdef NO_PARALLEL_SORT + pdqsort_branchless( m_data.symbolLoc.begin() + m_data.newSymbolsIndex, m_data.symbolLoc.end(), [] ( const auto& l, const auto& r ) { return l.addr < r.addr; } ); +#else + std::sort( std::execution::par_unseq, m_data.symbolLoc.begin() + m_data.newSymbolsIndex, m_data.symbolLoc.end(), [] ( const auto& l, const auto& r ) { return l.addr < r.addr; } ); +#endif + const auto ms = std::lower_bound( m_data.symbolLoc.begin(), m_data.symbolLoc.begin() + m_data.newSymbolsIndex, m_data.symbolLoc[m_data.newSymbolsIndex], [] ( const auto& l, const auto& r ) { return l.addr < r.addr; } ); + std::inplace_merge( ms, m_data.symbolLoc.begin() + m_data.newSymbolsIndex, m_data.symbolLoc.end(), [] ( const auto& l, const auto& r ) { return l.addr < r.addr; } ); + m_data.newSymbolsIndex = -1; + } +} + +void Worker::DoPostponedInlineSymbols() +{ + if( m_data.newInlineSymbolsIndex >= 0 ) + { +#ifdef NO_PARALLEL_SORT + pdqsort_branchless( m_data.symbolLocInline.begin() + m_data.newInlineSymbolsIndex, m_data.symbolLocInline.end() ); +#else + std::sort( std::execution::par_unseq, m_data.symbolLocInline.begin() + m_data.newInlineSymbolsIndex, m_data.symbolLocInline.end() ); +#endif + const auto ms = std::lower_bound( m_data.symbolLocInline.begin(), m_data.symbolLocInline.begin() + m_data.newInlineSymbolsIndex, m_data.symbolLocInline[m_data.newInlineSymbolsIndex] ); + std::inplace_merge( ms, m_data.symbolLocInline.begin() + m_data.newInlineSymbolsIndex, m_data.symbolLocInline.end() ); + m_data.newInlineSymbolsIndex = -1; + } +} + +void Worker::DoPostponedWorkAll() +{ + DoPostponedWork(); + DoPostponedSymbols(); + DoPostponedInlineSymbols(); + + for( auto& plot : m_data.plots.Data() ) + { + if( !plot->data.is_sorted() ) plot->data.sort(); + } +} + +void Worker::DoPostponedWork() +{ +#ifndef TRACY_NO_STATISTICS + if( m_data.newFramesWereReceived ) + { + HandlePostponedSamples(); + HandlePostponedGhostZones(); + m_data.newFramesWereReceived = false; + } + + if( m_identifySamples && m_data.newContextSwitchesReceived ) + { + for( auto& td : m_data.threads ) + { + if( !td->postponedSamples.empty() ) + { + auto ctx = GetContextSwitchData( td->id ); + if( ctx ) + { + td->postponedSamples.ensure_sorted(); + auto sit = td->postponedSamples.begin(); + auto cit = std::lower_bound( ctx->v.begin(), ctx->v.end(), sit->time.Val(), [] ( const auto& l, const auto& r ) { return (uint64_t)l.End() < (uint64_t)r; } ); + if( cit != ctx->v.end() ) + { + do + { + if( sit->time.Val() == cit->Start() ) + { + td->ctxSwitchSamples.push_back( *sit ); + } + else + { + ProcessCallstackSampleImplStats( *sit, *td ); + } + if( ++sit == td->postponedSamples.end() ) break; + cit = std::lower_bound( cit, ctx->v.end(), sit->time.Val(), [] ( const auto& l, const auto& r ) { return (uint64_t)l.End() < (uint64_t)r; } ); + } + while( cit != ctx->v.end() ); + if( sit == td->postponedSamples.end() ) + { + td->postponedSamples.clear(); + } + else + { + td->postponedSamples.erase( td->postponedSamples.begin(), sit ); + } + } + } + } + } + m_data.newContextSwitchesReceived = false; + } +#endif +} + +#ifndef TRACY_NO_STATISTICS +void Worker::HandlePostponedSamples() +{ + assert( m_data.newFramesWereReceived ); + if( m_data.postponedSamples.empty() ) return; + auto it = m_data.postponedSamples.begin(); + do + { + UpdateSampleStatisticsPostponed( it ); + } + while( it != m_data.postponedSamples.end() ); +} + +void Worker::GetStackWithInlines( Vector& ret, const VarArray& cs ) +{ + ret.clear(); + int idx = cs.size() - 1; + do + { + auto& entry = cs[idx]; + const auto frame = GetCallstackFrame( entry ); + if( frame ) + { + uint8_t i = frame->size; + do + { + i--; + ret.push_back( InlineStackData { frame->data[i].symAddr, entry, i } ); + } + while( i != 0 ); + } + else + { + ret.push_back( InlineStackData{ GetCanonicalPointer( entry ), entry, 0 } ); + } + } + while( idx-- > 0 ); +} + +int Worker::AddGhostZone( const VarArray& cs, Vector* vec, uint64_t t ) +{ + GetStackWithInlines( m_inlineStack, cs ); + + if( !vec->empty() && vec->back().end.Val() > (int64_t)t ) + { + const auto refBackTime = vec->back().end.Val(); + auto tmp = vec; + for(;;) + { + auto& back = tmp->back(); + if( back.end.Val() != refBackTime ) break; + back.end.SetVal( t ); + if( back.child < 0 ) break; + tmp = &m_data.ghostChildren[back.child]; + } + } + const int64_t refBackTime = vec->empty() ? 0 : vec->back().end.Val(); + int gcnt = 0; + size_t idx = 0; + while( !vec->empty() && idx < m_inlineStack.size() ) + { + auto& back = vec->back(); + const auto& backKey = m_data.ghostFrames[back.frame.Val()]; + const auto backFrame = GetCallstackFrame( backKey.frame ); + if( !backFrame ) break; + const auto& inlineFrame = backFrame->data[backKey.inlineFrame]; + if( inlineFrame.symAddr != m_inlineStack[idx].symAddr ) break; + if( back.end.Val() != refBackTime ) break; + back.end.SetVal( t + m_samplingPeriod ); + if( ++idx == m_inlineStack.size() ) break; + if( back.child < 0 ) + { + back.child = m_data.ghostChildren.size(); + vec = &m_data.ghostChildren.push_next(); + } + else + { + vec = &m_data.ghostChildren[back.child]; + } + } + while( idx < m_inlineStack.size() ) + { + gcnt++; + uint32_t fid; + GhostKey key { m_inlineStack[idx].frame, m_inlineStack[idx].inlineFrame }; + auto it = m_data.ghostFramesMap.find( key ); + if( it == m_data.ghostFramesMap.end() ) + { + fid = uint32_t( m_data.ghostFrames.size() ); + m_data.ghostFrames.push_back( key ); + m_data.ghostFramesMap.emplace( key, fid ); + } + else + { + fid = it->second; + } + auto& zone = vec->push_next(); + zone.start.SetVal( t ); + zone.end.SetVal( t + m_samplingPeriod ); + zone.frame.SetVal( fid ); + if( ++idx == m_inlineStack.size() ) + { + zone.child = -1; + } + else + { + zone.child = m_data.ghostChildren.size(); + vec = &m_data.ghostChildren.push_next(); + } + } + return gcnt; +} + +void Worker::HandlePostponedGhostZones() +{ + assert( m_data.newFramesWereReceived ); + if( !m_data.ghostZonesPostponed ) return; + bool postponed = false; + for( auto& td : m_data.threads ) + { + while( td->ghostIdx != td->samples.size() ) + { + const auto& sample = td->samples[td->ghostIdx]; + const auto& cs = GetCallstack( sample.callstack.Val() ); + const auto cssz = cs.size(); + + uint16_t i; + for( i=0; ighostIdx++; + m_data.ghostCnt += AddGhostZone( cs, &td->ghostZones, sample.time.Val() ); + } + } + m_data.ghostZonesPostponed = postponed; +} +#endif + +uint32_t Worker::GetSingleStringIdx() +{ + assert( m_pendingSingleString.ptr != nullptr ); + const auto idx = m_pendingSingleString.idx; + m_pendingSingleString.ptr = nullptr; + return idx; +} + +uint32_t Worker::GetSecondStringIdx() +{ + assert( m_pendingSecondString.ptr != nullptr ); + const auto idx = m_pendingSecondString.idx; + m_pendingSecondString.ptr = nullptr; + return idx; +} + +StringLocation Worker::StoreString( const char* str, size_t sz ) +{ + StringLocation ret; + charutil::StringKey key = { str, sz }; + auto sit = m_data.stringMap.find( key ); + if( sit == m_data.stringMap.end() ) + { + auto ptr = m_slab.Alloc( sz+1 ); + memcpy( ptr, str, sz ); + ptr[sz] = '\0'; + ret.ptr = ptr; + ret.idx = m_data.stringData.size(); + m_data.stringMap.emplace( charutil::StringKey { ptr, sz }, m_data.stringData.size() ); + m_data.stringData.push_back( ptr ); + } + else + { + ret.ptr = sit->first.ptr; + ret.idx = sit->second; + } + return ret; +} + +bool Worker::Process( const QueueItem& ev ) +{ + switch( ev.hdr.type ) + { + case QueueType::ThreadContext: + ProcessThreadContext( ev.threadCtx ); + break; + case QueueType::ZoneBegin: + ProcessZoneBegin( ev.zoneBegin ); + break; + case QueueType::ZoneBeginCallstack: + ProcessZoneBeginCallstack( ev.zoneBegin ); + break; + case QueueType::ZoneBeginAllocSrcLoc: + ProcessZoneBeginAllocSrcLoc( ev.zoneBeginLean ); + break; + case QueueType::ZoneBeginAllocSrcLocCallstack: + ProcessZoneBeginAllocSrcLocCallstack( ev.zoneBeginLean ); + break; + case QueueType::ZoneEnd: + ProcessZoneEnd( ev.zoneEnd ); + break; + case QueueType::ZoneValidation: + ProcessZoneValidation( ev.zoneValidation ); + break; + case QueueType::FrameMarkMsg: + ProcessFrameMark( ev.frameMark ); + break; + case QueueType::FrameMarkMsgStart: + ProcessFrameMarkStart( ev.frameMark ); + break; + case QueueType::FrameMarkMsgEnd: + ProcessFrameMarkEnd( ev.frameMark ); + break; + case QueueType::FrameVsync: + ProcessFrameVsync( ev.frameVsync ); + break; + case QueueType::FrameImage: + ProcessFrameImage( ev.frameImage ); + break; + case QueueType::SourceLocation: + AddSourceLocation( ev.srcloc ); + m_serverQuerySpaceLeft++; + break; + case QueueType::ZoneText: + ProcessZoneText(); + break; + case QueueType::ZoneName: + ProcessZoneName(); + break; + case QueueType::ZoneColor: + ProcessZoneColor( ev.zoneColor ); + break; + case QueueType::ZoneValue: + ProcessZoneValue( ev.zoneValue ); + break; + case QueueType::LockAnnounce: + ProcessLockAnnounce( ev.lockAnnounce ); + break; + case QueueType::LockTerminate: + ProcessLockTerminate( ev.lockTerminate ); + break; + case QueueType::LockWait: + ProcessLockWait( ev.lockWait ); + break; + case QueueType::LockObtain: + ProcessLockObtain( ev.lockObtain ); + break; + case QueueType::LockRelease: + ProcessLockRelease( ev.lockRelease ); + break; + case QueueType::LockSharedWait: + ProcessLockSharedWait( ev.lockWait ); + break; + case QueueType::LockSharedObtain: + ProcessLockSharedObtain( ev.lockObtain ); + break; + case QueueType::LockSharedRelease: + ProcessLockSharedRelease( ev.lockReleaseShared ); + break; + case QueueType::LockMark: + ProcessLockMark( ev.lockMark ); + break; + case QueueType::LockName: + ProcessLockName( ev.lockName ); + break; + case QueueType::PlotDataInt: + ProcessPlotDataInt( ev.plotDataInt ); + break; + case QueueType::PlotDataFloat: + ProcessPlotDataFloat( ev.plotDataFloat ); + break; + case QueueType::PlotDataDouble: + ProcessPlotDataDouble( ev.plotDataDouble ); + break; + case QueueType::PlotConfig: + ProcessPlotConfig( ev.plotConfig ); + break; + case QueueType::Message: + ProcessMessage( ev.message ); + break; + case QueueType::MessageLiteral: + ProcessMessageLiteral( ev.messageLiteral ); + break; + case QueueType::MessageColor: + ProcessMessageColor( ev.messageColor ); + break; + case QueueType::MessageLiteralColor: + ProcessMessageLiteralColor( ev.messageColorLiteral ); + break; + case QueueType::MessageCallstack: + ProcessMessageCallstack( ev.message ); + break; + case QueueType::MessageLiteralCallstack: + ProcessMessageLiteralCallstack( ev.messageLiteral ); + break; + case QueueType::MessageColorCallstack: + ProcessMessageColorCallstack( ev.messageColor ); + break; + case QueueType::MessageLiteralColorCallstack: + ProcessMessageLiteralColorCallstack( ev.messageColorLiteral ); + break; + case QueueType::MessageAppInfo: + ProcessMessageAppInfo( ev.message ); + break; + case QueueType::GpuNewContext: + ProcessGpuNewContext( ev.gpuNewContext ); + break; + case QueueType::GpuZoneBegin: + ProcessGpuZoneBegin( ev.gpuZoneBegin, false ); + break; + case QueueType::GpuZoneBeginCallstack: + ProcessGpuZoneBeginCallstack( ev.gpuZoneBegin, false ); + break; + case QueueType::GpuZoneBeginAllocSrcLoc: + ProcessGpuZoneBeginAllocSrcLoc( ev.gpuZoneBeginLean, false ); + break; + case QueueType::GpuZoneBeginAllocSrcLocCallstack: + ProcessGpuZoneBeginAllocSrcLocCallstack( ev.gpuZoneBeginLean, false ); + break; + case QueueType::GpuZoneEnd: + ProcessGpuZoneEnd( ev.gpuZoneEnd, false ); + break; + case QueueType::GpuZoneBeginSerial: + ProcessGpuZoneBegin( ev.gpuZoneBegin, true ); + break; + case QueueType::GpuZoneBeginCallstackSerial: + ProcessGpuZoneBeginCallstack( ev.gpuZoneBegin, true ); + break; + case QueueType::GpuZoneBeginAllocSrcLocSerial: + ProcessGpuZoneBeginAllocSrcLoc( ev.gpuZoneBeginLean, true ); + break; + case QueueType::GpuZoneBeginAllocSrcLocCallstackSerial: + ProcessGpuZoneBeginAllocSrcLocCallstack( ev.gpuZoneBeginLean, true ); + break; + case QueueType::GpuZoneEndSerial: + ProcessGpuZoneEnd( ev.gpuZoneEnd, true ); + break; + case QueueType::GpuTime: + ProcessGpuTime( ev.gpuTime ); + break; + case QueueType::GpuCalibration: + ProcessGpuCalibration( ev.gpuCalibration ); + break; + case QueueType::GpuTimeSync: + ProcessGpuTimeSync( ev.gpuTimeSync ); + break; + case QueueType::GpuContextName: + ProcessGpuContextName( ev.gpuContextName ); + break; + case QueueType::MemAlloc: + ProcessMemAlloc( ev.memAlloc ); + break; + case QueueType::MemAllocNamed: + ProcessMemAllocNamed( ev.memAlloc ); + break; + case QueueType::MemFree: + ProcessMemFree( ev.memFree ); + break; + case QueueType::MemFreeNamed: + ProcessMemFreeNamed( ev.memFree ); + break; + case QueueType::MemAllocCallstack: + ProcessMemAllocCallstack( ev.memAlloc ); + break; + case QueueType::MemAllocCallstackNamed: + ProcessMemAllocCallstackNamed( ev.memAlloc ); + break; + case QueueType::MemFreeCallstack: + ProcessMemFreeCallstack( ev.memFree ); + break; + case QueueType::MemFreeCallstackNamed: + ProcessMemFreeCallstackNamed( ev.memFree ); + break; + case QueueType::CallstackSerial: + ProcessCallstackSerial(); + break; + case QueueType::Callstack: + case QueueType::CallstackAlloc: + ProcessCallstack(); + break; + case QueueType::CallstackSample: + ProcessCallstackSample( ev.callstackSample ); + break; + case QueueType::CallstackSampleContextSwitch: + ProcessCallstackSampleContextSwitch( ev.callstackSample ); + break; + case QueueType::CallstackFrameSize: + ProcessCallstackFrameSize( ev.callstackFrameSize ); + m_serverQuerySpaceLeft++; + break; + case QueueType::CallstackFrame: + ProcessCallstackFrame( ev.callstackFrame, true ); + break; + case QueueType::SymbolInformation: + ProcessSymbolInformation( ev.symbolInformation ); + m_serverQuerySpaceLeft++; + break; + case QueueType::Terminate: + m_terminate = true; + break; + case QueueType::KeepAlive: + break; + case QueueType::Crash: + m_crashed = true; + break; + case QueueType::CrashReport: + ProcessCrashReport( ev.crashReport ); + break; + case QueueType::SysTimeReport: + ProcessSysTime( ev.sysTime ); + break; + case QueueType::SysPowerReport: + ProcessSysPower( ev.sysPower ); + break; + case QueueType::ContextSwitch: + ProcessContextSwitch( ev.contextSwitch ); + break; + case QueueType::ThreadWakeup: + ProcessThreadWakeup( ev.threadWakeup ); + break; + case QueueType::TidToPid: + ProcessTidToPid( ev.tidToPid ); + break; + case QueueType::HwSampleCpuCycle: + ProcessHwSampleCpuCycle( ev.hwSample ); + break; + case QueueType::HwSampleInstructionRetired: + ProcessHwSampleInstructionRetired( ev.hwSample ); + break; + case QueueType::HwSampleCacheReference: + ProcessHwSampleCacheReference( ev.hwSample ); + break; + case QueueType::HwSampleCacheMiss: + ProcessHwSampleCacheMiss( ev.hwSample ); + break; + case QueueType::HwSampleBranchRetired: + ProcessHwSampleBranchRetired( ev.hwSample ); + break; + case QueueType::HwSampleBranchMiss: + ProcessHwSampleBranchMiss( ev.hwSample ); + break; + case QueueType::ParamSetup: + ProcessParamSetup( ev.paramSetup ); + break; + case QueueType::AckServerQueryNoop: + m_serverQuerySpaceLeft++; + break; + case QueueType::AckSourceCodeNotAvailable: + ProcessSourceCodeNotAvailable( ev.sourceCodeNotAvailable ); + m_serverQuerySpaceLeft++; + break; + case QueueType::AckSymbolCodeNotAvailable: + m_pendingSymbolCode--; + m_serverQuerySpaceLeft++; + break; + case QueueType::CpuTopology: + ProcessCpuTopology( ev.cpuTopology ); + break; + case QueueType::MemNamePayload: + ProcessMemNamePayload( ev.memName ); + break; + case QueueType::ThreadGroupHint: + ProcessThreadGroupHint( ev.threadGroupHint ); + break; + case QueueType::FiberEnter: + ProcessFiberEnter( ev.fiberEnter ); + break; + case QueueType::FiberLeave: + ProcessFiberLeave( ev.fiberLeave ); + break; + default: + assert( false ); + break; + } + + return m_failure == Failure::None; +} + +void Worker::ProcessThreadContext( const QueueThreadContext& ev ) +{ + m_refTimeThread = 0; + if( m_threadCtx != ev.thread ) + { + m_threadCtx = ev.thread; + m_threadCtxData = RetrieveThread( ev.thread ); + } +} + +static tracy_force_inline int64_t RefTime( int64_t& reference, int64_t delta ) +{ + const auto refTime = reference + delta; + reference = refTime; + return refTime; +} + +void Worker::ProcessZoneBeginImpl( ZoneEvent* zone, const QueueZoneBegin& ev ) +{ + CheckSourceLocation( ev.srcloc ); + + const auto start = TscTime( RefTime( m_refTimeThread, ev.time ) ); + zone->SetStartSrcLoc( start, ShrinkSourceLocation( ev.srcloc ) ); + zone->SetEnd( -1 ); + zone->SetChild( -1 ); + + if( m_data.lastTime < start ) m_data.lastTime = start; + + NewZone( zone ); +} + +void Worker::ProcessZoneBeginAllocSrcLocImpl( ZoneEvent* zone, const QueueZoneBeginLean& ev ) +{ + assert( m_pendingSourceLocationPayload != 0 ); + + const auto start = TscTime( RefTime( m_refTimeThread, ev.time ) ); + zone->SetStartSrcLoc( start, m_pendingSourceLocationPayload ); + zone->SetEnd( -1 ); + zone->SetChild( -1 ); + + if( m_data.lastTime < start ) m_data.lastTime = start; + + NewZone( zone ); + + m_pendingSourceLocationPayload = 0; +} + +ZoneEvent* Worker::AllocZoneEvent() +{ + ZoneEvent* ret; +#ifndef TRACY_NO_STATISTICS + ret = m_slab.Alloc(); +#else + if( m_zoneEventPool.empty() ) + { + ret = m_slab.Alloc(); + } + else + { + ret = m_zoneEventPool.back_and_pop(); + } +#endif + ret->extra = 0; + return ret; +} + +void Worker::ProcessZoneBegin( const QueueZoneBegin& ev ) +{ + auto zone = AllocZoneEvent(); + ProcessZoneBeginImpl( zone, ev ); +} + +void Worker::ProcessZoneBeginCallstack( const QueueZoneBegin& ev ) +{ + auto zone = AllocZoneEvent(); + ProcessZoneBeginImpl( zone, ev ); + auto td = GetCurrentThreadData(); + auto it = m_nextCallstack.find( td->id ); + assert( it != m_nextCallstack.end() ); + auto& extra = RequestZoneExtra( *zone ); + extra.callstack.SetVal( it->second ); + it->second = 0; +} + +void Worker::ProcessZoneBeginAllocSrcLoc( const QueueZoneBeginLean& ev ) +{ + auto zone = AllocZoneEvent(); + ProcessZoneBeginAllocSrcLocImpl( zone, ev ); +} + +void Worker::ProcessZoneBeginAllocSrcLocCallstack( const QueueZoneBeginLean& ev ) +{ + auto zone = AllocZoneEvent(); + ProcessZoneBeginAllocSrcLocImpl( zone, ev ); + auto td = GetCurrentThreadData(); + auto it = m_nextCallstack.find( td->id ); + assert( it != m_nextCallstack.end() ); + auto& extra = RequestZoneExtra( *zone ); + extra.callstack.SetVal( it->second ); + it->second = 0; +} + +void Worker::ProcessZoneEnd( const QueueZoneEnd& ev ) +{ + auto td = GetCurrentThreadData(); + if( td->zoneIdStack.empty() ) + { + ZoneDoubleEndFailure( td->id, td->timeline.empty() ? nullptr : td->timeline.back() ); + return; + } + auto zoneId = td->zoneIdStack.back_and_pop(); + if( zoneId != td->nextZoneId ) + { + ZoneStackFailure( td->id, td->stack.back() ); + return; + } + td->nextZoneId = 0; + + auto& stack = td->stack; + assert( !stack.empty() ); + auto zone = stack.back_and_pop(); + assert( zone->End() == -1 ); + const auto isReentry = td->DecStackCount( zone->SrcLoc() ); + const auto timeEnd = TscTime( RefTime( m_refTimeThread, ev.time ) ); + zone->SetEnd( timeEnd ); + assert( timeEnd >= zone->Start() ); + + if( m_data.lastTime < timeEnd ) m_data.lastTime = timeEnd; + + if( zone->HasChildren() ) + { + auto& childVec = m_data.zoneChildren[zone->Child()]; + const auto sz = childVec.size(); + if( sz <= 8 * 1024 ) + { + Vector> fitVec; +#ifndef TRACY_NO_STATISTICS + fitVec.reserve_exact( sz, m_slab ); + memcpy( fitVec.data(), childVec.data(), sz * sizeof( short_ptr ) ); +#else + fitVec.set_magic(); + auto& fv = *((Vector*)&fitVec); + fv.reserve_exact( sz, m_slab ); + auto dst = fv.data(); + for( auto& ze : childVec ) + { + ZoneEvent* src = ze; + memcpy( dst++, src, sizeof( ZoneEvent ) ); + m_zoneEventPool.push_back( src ); + } +#endif + fitVec.swap( childVec ); + m_data.zoneVectorCache.push_back( std::move( fitVec ) ); + } + } + +#ifndef TRACY_NO_STATISTICS + assert( !td->childTimeStack.empty() ); + const auto timeSpan = timeEnd - zone->Start(); + if( timeSpan > 0 ) + { + const auto ctid = CompressThread( td->id ); + ZoneThreadData ztd; + ztd.SetZone( zone ); + ztd.SetThread( ctid ); + + auto slz = GetSourceLocationZones( zone->SrcLoc() ); + slz->zones.push_back( ztd ); + if( slz->min > timeSpan ) slz->min = timeSpan; + if( slz->max < timeSpan ) slz->max = timeSpan; + slz->total += timeSpan; + slz->sumSq += double( timeSpan ) * timeSpan; + const auto selfSpan = timeSpan - td->childTimeStack.back_and_pop(); + if( slz->selfMin > selfSpan ) slz->selfMin = selfSpan; + if( slz->selfMax < selfSpan ) slz->selfMax = selfSpan; + slz->selfTotal += selfSpan; + + if( !isReentry ) + { + slz->nonReentrantCount++; + if( slz->nonReentrantMin > timeSpan ) slz->nonReentrantMin = timeSpan; + if( slz->nonReentrantMax < timeSpan ) slz->nonReentrantMax = timeSpan; + slz->nonReentrantTotal += timeSpan; + } + if( !td->childTimeStack.empty() ) + { + td->childTimeStack.back() += timeSpan; + } + + auto it = slz->threadCnt.find( ctid ); + if( it == slz->threadCnt.end() ) + { + slz->threadCnt.emplace( ctid, 1 ); + } + else + { + it->second++; + } + } + else + { + td->childTimeStack.pop_back(); + } +#else + CountZoneStatistics( zone ); +#endif +} + +void Worker::ZoneStackFailure( uint64_t thread, const ZoneEvent* ev ) +{ + m_failure = Failure::ZoneStack; + m_failureData.thread = thread; + m_failureData.srcloc = ev->SrcLoc(); +} + +void Worker::ZoneDoubleEndFailure( uint64_t thread, const ZoneEvent* ev ) +{ + m_failure = Failure::ZoneDoubleEnd; + m_failureData.thread = thread; + m_failureData.srcloc = ev ? ev->SrcLoc() : 0; +} + +void Worker::ZoneTextFailure( uint64_t thread, const char* text ) +{ + m_failure = Failure::ZoneText; + m_failureData.thread = thread; + m_failureData.message = text; +} + +void Worker::ZoneValueFailure( uint64_t thread, uint64_t value ) +{ + char buf[128]; + if( (int64_t)value < 0 ) + { + sprintf( buf, "Zone value was: %" PRIu64 " (unsigned), %" PRIi64 " (signed)", value, (int64_t)value ); + } + else + { + sprintf( buf, "Zone value was: %" PRIu64, value ); + } + + m_failure = Failure::ZoneValue; + m_failureData.thread = thread; + m_failureData.message = buf; +} + +void Worker::ZoneColorFailure( uint64_t thread ) +{ + m_failure = Failure::ZoneColor; + m_failureData.thread = thread; +} + +void Worker::ZoneNameFailure( uint64_t thread ) +{ + m_failure = Failure::ZoneName; + m_failureData.thread = thread; +} + +void Worker::MemFreeFailure( uint64_t thread ) +{ + m_failure = Failure::MemFree; + m_failureData.thread = thread; + m_failureData.callstack = m_serialNextCallstack; +} + +void Worker::MemAllocTwiceFailure( uint64_t thread ) +{ + m_failure = Failure::MemAllocTwice; + m_failureData.thread = thread; + m_failureData.callstack = m_serialNextCallstack; +} + +void Worker::FrameEndFailure() +{ + m_failure = Failure::FrameEnd; +} + +void Worker::FrameImageIndexFailure() +{ + m_failure = Failure::FrameImageIndex; +} + +void Worker::FrameImageTwiceFailure() +{ + m_failure = Failure::FrameImageTwice; +} + +void Worker::FiberLeaveFailure() +{ + m_failure = Failure::FiberLeave; +} + +void Worker::SourceLocationOverflowFailure() +{ + m_failure = Failure::SourceLocationOverflow; +} + +void Worker::ProcessZoneValidation( const QueueZoneValidation& ev ) +{ + auto td = GetCurrentThreadData(); + td->nextZoneId = ev.id; +} + +void Worker::ProcessFrameMark( const QueueFrameMark& ev ) +{ + auto fd = m_data.frames.Retrieve( ev.name, [this] ( uint64_t name ) { + auto fd = m_slab.AllocInit(); + fd->name = name; + fd->continuous = 1; + return fd; + }, [this] ( uint64_t name ) { + Query( ServerQueryFrameName, name ); + } ); + + int32_t frameImage = -1; + if( ev.name == 0 ) + { + auto fis = m_frameImageStaging.find( fd->frames.size() ); + if( fis != m_frameImageStaging.end() ) + { + frameImage = fis->second; + m_frameImageStaging.erase( fis ); + } + } + + assert( fd->continuous == 1 ); + const auto time = TscTime( ev.time ); + assert( fd->frames.empty() || fd->frames.back().start <= time ); + fd->frames.push_back( FrameEvent{ time, -1, frameImage } ); + if( m_data.lastTime < time ) m_data.lastTime = time; + +#ifndef TRACY_NO_STATISTICS + const auto timeSpan = GetFrameTime( *fd, fd->frames.size() - 1 ); + if( timeSpan > 0 ) + { + fd->min = std::min( fd->min, timeSpan ); + fd->max = std::max( fd->max, timeSpan ); + fd->total += timeSpan; + fd->sumSq += double( timeSpan ) * timeSpan; + } +#endif +} + +void Worker::ProcessFrameMarkStart( const QueueFrameMark& ev ) +{ + auto fd = m_data.frames.Retrieve( ev.name, [this] ( uint64_t name ) { + auto fd = m_slab.AllocInit(); + fd->name = name; + fd->continuous = 0; + return fd; + }, [this] ( uint64_t name ) { + Query( ServerQueryFrameName, name ); + } ); + + assert( fd->continuous == 0 ); + const auto time = TscTime( ev.time ); + assert( fd->frames.empty() || ( fd->frames.back().end <= time && fd->frames.back().end != -1 ) ); + fd->frames.push_back( FrameEvent{ time, -1, -1 } ); + if( m_data.lastTime < time ) m_data.lastTime = time; +} + +void Worker::ProcessFrameMarkEnd( const QueueFrameMark& ev ) +{ + auto fd = m_data.frames.Retrieve( ev.name, [this] ( uint64_t name ) -> FrameData* { + return nullptr; + }, [this] ( uint64_t name ) { + Query( ServerQueryFrameName, name ); + } ); + + if( !fd ) + { + if( !m_ignoreFrameEndFaults ) FrameEndFailure(); + return; + } + + assert( fd->continuous == 0 ); + if( fd->frames.empty() ) + { + if( !m_ignoreFrameEndFaults ) FrameEndFailure(); + return; + } + const auto time = TscTime( ev.time ); + assert( fd->frames.back().end == -1 ); + fd->frames.back().end = time; + if( m_data.lastTime < time ) m_data.lastTime = time; + +#ifndef TRACY_NO_STATISTICS + const auto timeSpan = GetFrameTime( *fd, fd->frames.size() - 1 ); + if( timeSpan > 0 ) + { + fd->min = std::min( fd->min, timeSpan ); + fd->max = std::max( fd->max, timeSpan ); + fd->total += timeSpan; + fd->sumSq += double( timeSpan ) * timeSpan; + } +#endif +} + +void Worker::ProcessFrameVsync( const QueueFrameVsync& ev ) +{ + auto it = m_vsyncFrameMap.find( ev.id ); + if( it == m_vsyncFrameMap.end() ) + { + auto fd = m_slab.AllocInit(); + // Hackfix workaround to maintain backwards compatibility. + // Frame name pointers won't be in kernel space. Exploit that to store custom IDs. + fd->name = uint64_t( m_vsyncFrameMap.size() ) | 0x8000000000000000; + fd->continuous = 1; + m_data.frames.AddExternal( fd ); + it = m_vsyncFrameMap.emplace( ev.id, fd ).first; + } + auto fd = it->second; + assert( fd->continuous == 1 ); + const auto time = TscTime( ev.time ); + assert( fd->frames.empty() || fd->frames.back().start <= time ); + fd->frames.push_back( FrameEvent{ time, -1, -1 } ); + if( m_data.lastTime < time ) m_data.lastTime = time; + +#ifndef TRACY_NO_STATISTICS + const auto timeSpan = GetFrameTime( *fd, fd->frames.size() - 1 ); + if( timeSpan > 0 ) + { + fd->min = std::min( fd->min, timeSpan ); + fd->max = std::max( fd->max, timeSpan ); + fd->total += timeSpan; + fd->sumSq += double( timeSpan ) * timeSpan; + } +#endif +} + +void Worker::ProcessFrameImage( const QueueFrameImage& ev ) +{ + assert( m_pendingFrameImageData.image != nullptr ); + + auto& frames = m_data.framesBase->frames; + const auto fidx = int64_t( ev.frame ) - int64_t( m_data.frameOffset ) + 1; + if( m_onDemand && fidx <= 1 ) + { + m_pendingFrameImageData.image = nullptr; + return; + } + else if( fidx <= 0 ) + { + FrameImageIndexFailure(); + return; + } + + auto fi = m_slab.Alloc(); + fi->ptr = m_pendingFrameImageData.image; + fi->csz = m_pendingFrameImageData.csz; + fi->w = ev.w; + fi->h = ev.h; + fi->frameRef = uint32_t( fidx ); + fi->flip = ev.flip; + + const auto idx = m_data.frameImage.size(); + m_data.frameImage.push_back( fi ); + m_pendingFrameImageData.image = nullptr; + + if( fidx >= (int64_t)frames.size() ) + { + if( m_frameImageStaging.find( fidx ) != m_frameImageStaging.end() ) + { + FrameImageTwiceFailure(); + return; + } + m_frameImageStaging.emplace( fidx, idx ); + } + else if( frames[fidx].frameImage >= 0 ) + { + FrameImageTwiceFailure(); + } + else + { + frames[fidx].frameImage = idx; + } +} + +void Worker::ProcessZoneText() +{ + auto td = RetrieveThread( m_threadCtx ); + if( !td ) + { + ZoneTextFailure( m_threadCtx, m_pendingSingleString.ptr ); + return; + } + if( td->fiber ) td = td->fiber; + if( td->stack.empty() || td->nextZoneId != td->zoneIdStack.back() ) + { + ZoneTextFailure( td->id, m_pendingSingleString.ptr ); + return; + } + + const auto ptr = m_pendingSingleString.ptr; + const auto idx = GetSingleStringIdx(); + + td->nextZoneId = 0; + auto& stack = td->stack; + auto zone = stack.back(); + auto& extra = RequestZoneExtra( *zone ); + if( !extra.text.Active() ) + { + extra.text = StringIdx( idx ); + } + else + { + const auto str0 = GetString( extra.text ); + const auto str1 = ptr; + const auto len0 = strlen( str0 ); + const auto len1 = strlen( str1 ); + const auto bsz = len0+len1+1; + if( m_tmpBufSize < bsz ) + { + delete[] m_tmpBuf; + m_tmpBuf = new char[bsz]; + m_tmpBufSize = bsz; + } + char* buf = m_tmpBuf; + memcpy( buf, str0, len0 ); + buf[len0] = '\n'; + memcpy( buf+len0+1, str1, len1 ); + extra.text = StringIdx( StoreString( buf, bsz ).idx ); + } +} + +void Worker::ProcessZoneName() +{ + auto td = RetrieveThread( m_threadCtx ); + if( !td ) + { + ZoneNameFailure( m_threadCtx ); + return; + } + if( td->fiber ) td = td->fiber; + if( td->stack.empty() || td->nextZoneId != td->zoneIdStack.back() ) + { + ZoneNameFailure( td->id ); + return; + } + + td->nextZoneId = 0; + auto& stack = td->stack; + auto zone = stack.back(); + auto& extra = RequestZoneExtra( *zone ); + extra.name = StringIdx( GetSingleStringIdx() ); +} + +void Worker::ProcessZoneColor( const QueueZoneColor& ev ) +{ + auto td = RetrieveThread( m_threadCtx ); + if( !td ) + { + ZoneColorFailure( m_threadCtx ); + return; + } + if( td->fiber ) td = td->fiber; + if( td->stack.empty() || td->nextZoneId != td->zoneIdStack.back() ) + { + ZoneColorFailure( td->id ); + return; + } + + td->nextZoneId = 0; + auto& stack = td->stack; + auto zone = stack.back(); + auto& extra = RequestZoneExtra( *zone ); + const uint32_t color = ( ev.b << 16 ) | ( ev.g << 8 ) | ev.r; + extra.color = color; +} + +void Worker::ProcessZoneValue( const QueueZoneValue& ev ) +{ + char tmp[32]; + const auto tsz = sprintf( tmp, "%" PRIu64, ev.value ); + + auto td = RetrieveThread( m_threadCtx ); + if( !td ) + { + ZoneValueFailure( m_threadCtx, ev.value ); + return; + } + if( td->fiber ) td = td->fiber; + if( td->stack.empty() || td->nextZoneId != td->zoneIdStack.back() ) + { + ZoneValueFailure( td->id, ev.value ); + return; + } + + td->nextZoneId = 0; + auto& stack = td->stack; + auto zone = stack.back(); + auto& extra = RequestZoneExtra( *zone ); + if( !extra.text.Active() ) + { + extra.text = StringIdx( StoreString( tmp, tsz ).idx ); + } + else + { + const auto str0 = GetString( extra.text ); + const auto len0 = strlen( str0 ); + const auto bsz = len0+tsz+1; + if( m_tmpBufSize < bsz ) + { + delete[] m_tmpBuf; + m_tmpBuf = new char[bsz]; + m_tmpBufSize = bsz; + } + char* buf = m_tmpBuf; + memcpy( buf, str0, len0 ); + buf[len0] = '\n'; + memcpy( buf+len0+1, tmp, tsz ); + extra.text = StringIdx( StoreString( buf, bsz ).idx ); + } +} + +void Worker::ProcessLockAnnounce( const QueueLockAnnounce& ev ) +{ + auto it = m_data.lockMap.find( ev.id ); + assert( it == m_data.lockMap.end() ); + auto lm = m_slab.AllocInit(); + lm->srcloc = ShrinkSourceLocation( ev.lckloc ); + lm->type = ev.type; + lm->timeAnnounce = TscTime( ev.time ); + lm->timeTerminate = 0; + lm->valid = true; + lm->isContended = false; + lm->lockingThread = 0; + m_data.lockMap.emplace( ev.id, lm ); + CheckSourceLocation( ev.lckloc ); +} + +void Worker::ProcessLockTerminate( const QueueLockTerminate& ev ) +{ + auto it = m_data.lockMap.find( ev.id ); + assert( it != m_data.lockMap.end() ); + it->second->timeTerminate = TscTime( ev.time ); +} + +void Worker::ProcessLockWait( const QueueLockWait& ev ) +{ + auto it = m_data.lockMap.find( ev.id ); + assert( it != m_data.lockMap.end() ); + auto& lock = *it->second; + + auto lev = lock.type == LockType::Lockable ? m_slab.Alloc() : m_slab.Alloc(); + const auto time = TscTime( RefTime( m_refTimeSerial, ev.time ) ); + lev->SetTime( time ); + lev->SetSrcLoc( 0 ); + lev->type = LockEvent::Type::Wait; + + InsertLockEvent( lock, lev, ev.thread, time ); +} + +void Worker::ProcessLockObtain( const QueueLockObtain& ev ) +{ + auto it = m_data.lockMap.find( ev.id ); + assert( it != m_data.lockMap.end() ); + auto& lock = *it->second; + + auto lev = lock.type == LockType::Lockable ? m_slab.Alloc() : m_slab.Alloc(); + const auto time = TscTime( RefTime( m_refTimeSerial, ev.time ) ); + lev->SetTime( time ); + lev->SetSrcLoc( 0 ); + lev->type = LockEvent::Type::Obtain; + + InsertLockEvent( lock, lev, ev.thread, time ); + lock.lockingThread = ev.thread; +} + +void Worker::ProcessLockRelease( const QueueLockRelease& ev ) +{ + auto it = m_data.lockMap.find( ev.id ); + assert( it != m_data.lockMap.end() ); + auto& lock = *it->second; + + auto lev = lock.type == LockType::Lockable ? m_slab.Alloc() : m_slab.Alloc(); + const auto time = TscTime( RefTime( m_refTimeSerial, ev.time ) ); + lev->SetTime( time ); + lev->SetSrcLoc( 0 ); + lev->type = LockEvent::Type::Release; + + InsertLockEvent( lock, lev, lock.lockingThread, time ); +} + +void Worker::ProcessLockSharedWait( const QueueLockWait& ev ) +{ + auto it = m_data.lockMap.find( ev.id ); + assert( it != m_data.lockMap.end() ); + auto& lock = *it->second; + + assert( lock.type == LockType::SharedLockable ); + auto lev = m_slab.Alloc(); + const auto time = TscTime( RefTime( m_refTimeSerial, ev.time ) ); + lev->SetTime( time ); + lev->SetSrcLoc( 0 ); + lev->type = LockEvent::Type::WaitShared; + + InsertLockEvent( lock, lev, ev.thread, time ); +} + +void Worker::ProcessLockSharedObtain( const QueueLockObtain& ev ) +{ + auto it = m_data.lockMap.find( ev.id ); + assert( it != m_data.lockMap.end() ); + auto& lock = *it->second; + + assert( lock.type == LockType::SharedLockable ); + auto lev = m_slab.Alloc(); + const auto time = TscTime( RefTime( m_refTimeSerial, ev.time ) ); + lev->SetTime( time ); + lev->SetSrcLoc( 0 ); + lev->type = LockEvent::Type::ObtainShared; + + InsertLockEvent( lock, lev, ev.thread, time ); +} + +void Worker::ProcessLockSharedRelease( const QueueLockReleaseShared& ev ) +{ + auto it = m_data.lockMap.find( ev.id ); + assert( it != m_data.lockMap.end() ); + auto& lock = *it->second; + + assert( lock.type == LockType::SharedLockable ); + auto lev = m_slab.Alloc(); + const auto time = TscTime( RefTime( m_refTimeSerial, ev.time ) ); + lev->SetTime( time ); + lev->SetSrcLoc( 0 ); + lev->type = LockEvent::Type::ReleaseShared; + + InsertLockEvent( lock, lev, ev.thread, time ); +} + +void Worker::ProcessLockMark( const QueueLockMark& ev ) +{ + CheckSourceLocation( ev.srcloc ); + auto lit = m_data.lockMap.find( ev.id ); + assert( lit != m_data.lockMap.end() ); + auto& lockmap = *lit->second; + auto tid = lockmap.threadMap.find( ev.thread ); + assert( tid != lockmap.threadMap.end() ); + const auto thread = tid->second; + auto it = lockmap.timeline.end(); + for(;;) + { + --it; + if( it->ptr->thread == thread ) + { + switch( it->ptr->type ) + { + case LockEvent::Type::Obtain: + case LockEvent::Type::ObtainShared: + case LockEvent::Type::Wait: + case LockEvent::Type::WaitShared: + it->ptr->SetSrcLoc( ShrinkSourceLocation( ev.srcloc ) ); + return; + default: + break; + } + } + } +} + +void Worker::ProcessLockName( const QueueLockName& ev ) +{ + auto lit = m_data.lockMap.find( ev.id ); + assert( lit != m_data.lockMap.end() ); + lit->second->customName = StringIdx( GetSingleStringIdx() ); +} + +void Worker::ProcessPlotDataInt( const QueuePlotDataInt& ev ) +{ + ProcessPlotDataImpl( ev.name, ev.time, (double)ev.val ); +} + +void Worker::ProcessPlotDataFloat( const QueuePlotDataFloat& ev ) +{ + if( !isfinite( ev.val ) ) return; + ProcessPlotDataImpl( ev.name, ev.time, (double)ev.val ); +} + +void Worker::ProcessPlotDataDouble( const QueuePlotDataDouble& ev ) +{ + if( !isfinite( ev.val ) ) return; + ProcessPlotDataImpl( ev.name, ev.time, ev.val ); +} + +void Worker::ProcessPlotDataImpl( uint64_t name, int64_t evTime, double val ) +{ + PlotData* plot = m_data.plots.Retrieve( name, [this] ( uint64_t name ) { + auto plot = m_slab.AllocInit(); + plot->name = name; + plot->type = PlotType::User; + plot->format = PlotValueFormatting::Number; + plot->showSteps = false; + plot->fill = true; + plot->color = 0; + return plot; + }, [this]( uint64_t name ) { + Query( ServerQueryPlotName, name ); + } ); + + const auto time = TscTime( RefTime( m_refTimeThread, evTime ) ); + if( m_data.lastTime < time ) m_data.lastTime = time; + InsertPlot( plot, time, val ); +} + +void Worker::ProcessPlotConfig( const QueuePlotConfig& ev ) +{ + PlotData* plot = m_data.plots.Retrieve( ev.name, [this] ( uint64_t name ) { + auto plot = m_slab.AllocInit(); + plot->name = name; + plot->type = PlotType::User; + return plot; + }, [this]( uint64_t name ) { + Query( ServerQueryPlotName, name ); + } ); + + plot->format = (PlotValueFormatting)ev.type; + plot->showSteps = ev.step; + plot->fill = ev.fill; + plot->color = ev.color & 0xFFFFFF; +} + +void Worker::ProcessMessage( const QueueMessage& ev ) +{ + auto td = GetCurrentThreadData(); + auto msg = m_slab.Alloc(); + const auto time = TscTime( ev.time ); + msg->time = time; + msg->ref = StringRef( StringRef::Type::Idx, GetSingleStringIdx() ); + msg->thread = CompressThread( td->id ); + msg->color = 0xFFFFFFFF; + msg->callstack.SetVal( 0 ); + if( m_data.lastTime < time ) m_data.lastTime = time; + InsertMessageData( msg ); +} + +void Worker::ProcessMessageLiteral( const QueueMessageLiteral& ev ) +{ + auto td = GetCurrentThreadData(); + CheckString( ev.text ); + auto msg = m_slab.Alloc(); + const auto time = TscTime( ev.time ); + msg->time = time; + msg->ref = StringRef( StringRef::Type::Ptr, ev.text ); + msg->thread = CompressThread( td->id ); + msg->color = 0xFFFFFFFF; + msg->callstack.SetVal( 0 ); + if( m_data.lastTime < time ) m_data.lastTime = time; + InsertMessageData( msg ); +} + +void Worker::ProcessMessageColor( const QueueMessageColor& ev ) +{ + auto td = GetCurrentThreadData(); + auto msg = m_slab.Alloc(); + const auto time = TscTime( ev.time ); + msg->time = time; + msg->ref = StringRef( StringRef::Type::Idx, GetSingleStringIdx() ); + msg->thread = CompressThread( td->id ); + msg->color = 0xFF000000 | ( ev.b << 16 ) | ( ev.g << 8 ) | ev.r; + msg->callstack.SetVal( 0 ); + if( m_data.lastTime < time ) m_data.lastTime = time; + InsertMessageData( msg ); +} + +void Worker::ProcessMessageLiteralColor( const QueueMessageColorLiteral& ev ) +{ + auto td = GetCurrentThreadData(); + CheckString( ev.text ); + auto msg = m_slab.Alloc(); + const auto time = TscTime( ev.time ); + msg->time = time; + msg->ref = StringRef( StringRef::Type::Ptr, ev.text ); + msg->thread = CompressThread( td->id ); + msg->color = 0xFF000000 | ( ev.b << 16 ) | ( ev.g << 8 ) | ev.r; + msg->callstack.SetVal( 0 ); + if( m_data.lastTime < time ) m_data.lastTime = time; + InsertMessageData( msg ); +} + +void Worker::ProcessMessageCallstack( const QueueMessage& ev ) +{ + auto td = GetCurrentThreadData(); + ProcessMessage( ev ); + auto it = m_nextCallstack.find( td->id ); + assert( it != m_nextCallstack.end() ); + td->messages.back()->callstack.SetVal( it->second ); + it->second = 0; +} + +void Worker::ProcessMessageLiteralCallstack( const QueueMessageLiteral& ev ) +{ + auto td = GetCurrentThreadData(); + ProcessMessageLiteral( ev ); + auto it = m_nextCallstack.find( td->id ); + assert( it != m_nextCallstack.end() ); + td->messages.back()->callstack.SetVal( it->second ); + it->second = 0; +} + +void Worker::ProcessMessageColorCallstack( const QueueMessageColor& ev ) +{ + auto td = GetCurrentThreadData(); + ProcessMessageColor( ev ); + auto it = m_nextCallstack.find( td->id ); + assert( it != m_nextCallstack.end() ); + td->messages.back()->callstack.SetVal( it->second ); + it->second = 0; +} + +void Worker::ProcessMessageLiteralColorCallstack( const QueueMessageColorLiteral& ev ) +{ + auto td = GetCurrentThreadData(); + ProcessMessageLiteralColor( ev ); + auto it = m_nextCallstack.find( td->id ); + assert( it != m_nextCallstack.end() ); + td->messages.back()->callstack.SetVal( it->second ); + it->second = 0; +} + +void Worker::ProcessMessageAppInfo( const QueueMessage& ev ) +{ + m_data.appInfo.push_back( StringRef( StringRef::Type::Idx, GetSingleStringIdx() ) ); + const auto time = TscTime( ev.time ); + if( m_data.lastTime < time ) m_data.lastTime = time; +} + +void Worker::ProcessGpuNewContext( const QueueGpuNewContext& ev ) +{ + assert( !m_gpuCtxMap[ev.context] ); + assert( ev.type != GpuContextType::Invalid ); + + int64_t gpuTime; + if( ev.period == 1.f ) + { + gpuTime = ev.gpuTime; + } + else + { + gpuTime = int64_t( double( ev.period ) * ev.gpuTime ); // precision loss + } + + const auto cpuTime = TscTime( ev.cpuTime ); + auto gpu = m_slab.AllocInit(); + memset( (char*)gpu->query, 0, sizeof( gpu->query ) ); + gpu->timeDiff = cpuTime - gpuTime; + gpu->thread = ev.thread; + gpu->period = ev.period; + gpu->count = 0; + gpu->type = ev.type; + gpu->hasPeriod = ev.period != 1.f; + gpu->hasCalibration = ev.flags & GpuContextCalibration; + gpu->calibratedGpuTime = gpuTime; + gpu->calibratedCpuTime = cpuTime; + gpu->calibrationMod = 1.; + gpu->lastGpuTime = 0; + gpu->overflow = 0; + gpu->overflowMul = 0; + m_data.gpuData.push_back( gpu ); + m_gpuCtxMap[ev.context] = gpu; +} + +void Worker::ProcessGpuZoneBeginImpl( GpuEvent* zone, const QueueGpuZoneBegin& ev, bool serial ) +{ + CheckSourceLocation( ev.srcloc ); + zone->SetSrcLoc( ShrinkSourceLocation( ev.srcloc ) ); + ProcessGpuZoneBeginImplCommon( zone, ev, serial ); +} + +void Worker::ProcessGpuZoneBeginAllocSrcLocImpl( GpuEvent* zone, const QueueGpuZoneBeginLean& ev, bool serial ) +{ + assert( m_pendingSourceLocationPayload != 0 ); + zone->SetSrcLoc( m_pendingSourceLocationPayload ); + ProcessGpuZoneBeginImplCommon( zone, ev, serial ); + m_pendingSourceLocationPayload = 0; +} + +void Worker::ProcessGpuZoneBeginImplCommon( GpuEvent* zone, const QueueGpuZoneBeginLean& ev, bool serial ) +{ + m_data.gpuCnt++; + + auto ctx = m_gpuCtxMap[ev.context].get(); + assert( ctx ); + + int64_t cpuTime; + if( serial ) + { + cpuTime = RefTime( m_refTimeSerial, ev.cpuTime ); + } + else + { + cpuTime = RefTime( m_refTimeThread, ev.cpuTime ); + } + const auto time = TscTime( cpuTime ); + zone->SetCpuStart( time ); + zone->SetCpuEnd( -1 ); + zone->SetGpuStart( -1 ); + zone->SetGpuEnd( -1 ); + zone->callstack.SetVal( 0 ); + zone->SetChild( -1 ); + + uint64_t ztid; + if( ctx->thread == 0 ) + { + // Vulkan, OpenCL and Direct3D 12 contexts are not bound to any single thread. + zone->SetThread( CompressThread( ev.thread ) ); + ztid = ev.thread; + } + else + { + // OpenGL and Direct3D11 doesn't need per-zone thread id. It still can be sent, + // because it may be needed for callstack collection purposes. + zone->SetThread( 0 ); + ztid = 0; + } + + if( m_data.lastTime < time ) m_data.lastTime = time; + + auto td = ctx->threadData.find( ztid ); + if( td == ctx->threadData.end() ) + { + td = ctx->threadData.emplace( ztid, GpuCtxThreadData {} ).first; + } + auto timeline = &td->second.timeline; + auto& stack = td->second.stack; + if( !stack.empty() ) + { + auto back = stack.back(); + if( back->Child() < 0 ) + { + back->SetChild( int32_t( m_data.gpuChildren.size() ) ); + m_data.gpuChildren.push_back( Vector>() ); + } + timeline = &m_data.gpuChildren[back->Child()]; + } + + timeline->push_back( zone ); + stack.push_back( zone ); + + assert( !ctx->query[ev.queryId] ); + ctx->query[ev.queryId] = zone; +} + +void Worker::ProcessGpuZoneBegin( const QueueGpuZoneBegin& ev, bool serial ) +{ + auto zone = m_slab.Alloc(); + ProcessGpuZoneBeginImpl( zone, ev, serial ); +} + +void Worker::ProcessGpuZoneBeginCallstack( const QueueGpuZoneBegin& ev, bool serial ) +{ + auto zone = m_slab.Alloc(); + ProcessGpuZoneBeginImpl( zone, ev, serial ); + if( serial ) + { + assert( m_serialNextCallstack != 0 ); + zone->callstack.SetVal( m_serialNextCallstack ); + m_serialNextCallstack = 0; + } + else + { + auto td = GetCurrentThreadData(); + auto it = m_nextCallstack.find( td->id ); + assert( it != m_nextCallstack.end() ); + zone->callstack.SetVal( it->second ); + it->second = 0; + } +} + +void Worker::ProcessGpuZoneBeginAllocSrcLoc( const QueueGpuZoneBeginLean& ev, bool serial ) +{ + auto zone = m_slab.Alloc(); + ProcessGpuZoneBeginAllocSrcLocImpl( zone, ev, serial ); +} + +void Worker::ProcessGpuZoneBeginAllocSrcLocCallstack( const QueueGpuZoneBeginLean& ev, bool serial ) +{ + auto zone = m_slab.Alloc(); + ProcessGpuZoneBeginAllocSrcLocImpl( zone, ev, serial ); + if( serial ) + { + assert( m_serialNextCallstack != 0 ); + zone->callstack.SetVal( m_serialNextCallstack ); + m_serialNextCallstack = 0; + } + else + { + auto td = GetCurrentThreadData(); + auto it = m_nextCallstack.find( td->id ); + assert( it != m_nextCallstack.end() ); + zone->callstack.SetVal( it->second ); + it->second = 0; + } +} + +void Worker::ProcessGpuZoneEnd( const QueueGpuZoneEnd& ev, bool serial ) +{ + auto ctx = m_gpuCtxMap[ev.context]; + assert( ctx ); + + auto td = ctx->threadData.find( ev.thread ); + assert( td != ctx->threadData.end() ); + + assert( !td->second.stack.empty() ); + auto zone = td->second.stack.back_and_pop(); + + assert( !ctx->query[ev.queryId] ); + ctx->query[ev.queryId] = zone; + + int64_t cpuTime; + if( serial ) + { + cpuTime = RefTime( m_refTimeSerial, ev.cpuTime ); + } + else + { + cpuTime = RefTime( m_refTimeThread, ev.cpuTime ); + } + const auto time = TscTime( cpuTime ); + zone->SetCpuEnd( time ); + if( m_data.lastTime < time ) m_data.lastTime = time; +} + +void Worker::ProcessGpuTime( const QueueGpuTime& ev ) +{ + auto ctx = m_gpuCtxMap[ev.context]; + assert( ctx ); + + int64_t tgpu = RefTime( m_refTimeGpu, ev.gpuTime ); + if( tgpu < ctx->lastGpuTime - ( 1u << 31 ) ) + { + if( ctx->overflow == 0 ) + { + ctx->overflow = uint64_t( 1 ) << ( 64 - TracyLzcnt( ctx->lastGpuTime ) ); + } + ctx->overflowMul++; + } + ctx->lastGpuTime = tgpu; + if( ctx->overflow != 0 ) + { + tgpu += ctx->overflow * ctx->overflowMul; + } + + int64_t gpuTime; + if( !ctx->hasPeriod ) + { + if( !ctx->hasCalibration ) + { + gpuTime = tgpu + ctx->timeDiff; + } + else + { + gpuTime = int64_t( ( tgpu - ctx->calibratedGpuTime ) * ctx->calibrationMod + ctx->calibratedCpuTime ); + } + } + else + { + if( !ctx->hasCalibration ) + { + gpuTime = int64_t( double( ctx->period ) * tgpu ) + ctx->timeDiff; // precision loss + } + else + { + gpuTime = int64_t( ( double( ctx->period ) * tgpu - ctx->calibratedGpuTime ) * ctx->calibrationMod + ctx->calibratedCpuTime ); + } + } + + auto zone = ctx->query[ev.queryId]; + assert( zone ); + ctx->query[ev.queryId] = nullptr; + + if( zone->GpuStart() < 0 ) + { + zone->SetGpuStart( gpuTime ); + ctx->count++; + } + else + { + zone->SetGpuEnd( gpuTime ); +#ifndef TRACY_NO_STATISTICS + const auto gpuStart = zone->GpuStart(); + const auto timeSpan = gpuTime - gpuStart; + if( timeSpan > 0 ) + { + GpuZoneThreadData ztd; + ztd.SetZone( zone ); + ztd.SetThread( zone->Thread() ); + auto slz = GetGpuSourceLocationZones( zone->SrcLoc() ); + slz->zones.push_back( ztd ); + if( slz->min > timeSpan ) slz->min = timeSpan; + if( slz->max < timeSpan ) slz->max = timeSpan; + slz->total += timeSpan; + slz->sumSq += double( timeSpan ) * timeSpan; + } +#else + CountZoneStatistics( zone ); +#endif + } + if( m_data.lastTime < gpuTime ) m_data.lastTime = gpuTime; +} + +void Worker::ProcessGpuCalibration( const QueueGpuCalibration& ev ) +{ + auto ctx = m_gpuCtxMap[ev.context]; + assert( ctx ); + assert( ctx->hasCalibration ); + + int64_t gpuTime; + if( !ctx->hasPeriod ) + { + gpuTime = ev.gpuTime; + } + else + { + gpuTime = int64_t( double( ctx->period ) * ev.gpuTime ); // precision loss + } + + const auto cpuDelta = ev.cpuDelta; + const auto gpuDelta = gpuTime - ctx->calibratedGpuTime; + ctx->calibrationMod = double( cpuDelta ) / gpuDelta; + ctx->calibratedGpuTime = gpuTime; + ctx->calibratedCpuTime = TscTime( ev.cpuTime ); +} + +void Worker::ProcessGpuTimeSync( const QueueGpuTimeSync& ev ) +{ + auto ctx = m_gpuCtxMap[ev.context]; + assert( ctx ); + + int64_t gpuTime; + if( ctx->period == 1.f ) + { + gpuTime = ev.gpuTime; + } + else + { + gpuTime = int64_t( double( ctx->period ) * ev.gpuTime ); // precision loss + } + + const auto cpuTime = TscTime( ev.cpuTime ); + + ctx->timeDiff = cpuTime - gpuTime; + ctx->lastGpuTime = 0; + ctx->overflow = 0; + ctx->overflowMul = 0; +} + +void Worker::ProcessGpuContextName( const QueueGpuContextName& ev ) +{ + auto ctx = m_gpuCtxMap[ev.context]; + assert( ctx ); + const auto idx = GetSingleStringIdx(); + ctx->name = StringIdx( idx ); +} + +MemEvent* Worker::ProcessMemAllocImpl( MemData& memdata, const QueueMemAlloc& ev ) +{ + if( memdata.active.find( ev.ptr ) != memdata.active.end() ) + { + MemAllocTwiceFailure( ev.thread ); + return nullptr; + } + + const auto time = TscTime( RefTime( m_refTimeSerial, ev.time ) ); + if( m_data.lastTime < time ) m_data.lastTime = time; + NoticeThread( ev.thread ); + + assert( memdata.data.empty() || memdata.data.back().TimeAlloc() <= time ); + + memdata.active.emplace( ev.ptr, memdata.data.size() ); + + const auto ptr = ev.ptr; + uint32_t lo; + uint16_t hi; + memcpy( &lo, ev.size, 4 ); + memcpy( &hi, ev.size+4, 2 ); + const uint64_t size = lo | ( uint64_t( hi ) << 32 ); + + auto& mem = memdata.data.push_next(); + mem.SetPtr( ptr ); + mem.SetSize( size ); + mem.SetTimeThreadAlloc( time, CompressThread( ev.thread ) ); + mem.SetTimeThreadFree( -1, 0 ); + mem.SetCsAlloc( 0 ); + mem.csFree.SetVal( 0 ); + + const auto low = memdata.low; + const auto high = memdata.high; + const auto ptrend = ptr + size; + + memdata.low = std::min( low, ptr ); + memdata.high = std::max( high, ptrend ); + memdata.usage += size; + + MemAllocChanged( memdata, time ); + return &mem; +} + +MemEvent* Worker::ProcessMemFreeImpl( MemData& memdata, const QueueMemFree& ev ) +{ + const auto refTime = RefTime( m_refTimeSerial, ev.time ); + + auto it = memdata.active.find( ev.ptr ); + if( it == memdata.active.end() ) + { + if( ev.ptr == 0 ) return nullptr; + + if( !m_ignoreMemFreeFaults ) + { + CheckThreadString( ev.thread ); + MemFreeFailure( ev.thread ); + } + return nullptr; + } + + const auto time = TscTime( refTime ); + if( m_data.lastTime < time ) m_data.lastTime = time; + NoticeThread( ev.thread ); + + memdata.frees.push_back( it->second ); + auto& mem = memdata.data[it->second]; + mem.SetTimeThreadFree( time, CompressThread( ev.thread ) ); + memdata.usage -= mem.Size(); + memdata.active.erase( it ); + + MemAllocChanged( memdata, time ); + return &mem; +} + +MemEvent* Worker::ProcessMemAlloc( const QueueMemAlloc& ev ) +{ + assert( m_memNamePayload == 0 ); + return ProcessMemAllocImpl( *m_data.memory, ev ); +} + +MemEvent* Worker::ProcessMemAllocNamed( const QueueMemAlloc& ev ) +{ + assert( m_memNamePayload != 0 ); + auto memname = m_memNamePayload; + m_memNamePayload = 0; + auto it = m_data.memNameMap.find( memname ); + if( it == m_data.memNameMap.end() ) + { + CheckString( memname ); + it = m_data.memNameMap.emplace( memname, m_slab.AllocInit() ).first; + it->second->name = memname; + } + return ProcessMemAllocImpl( *it->second, ev ); +} + +MemEvent* Worker::ProcessMemFree( const QueueMemFree& ev ) +{ + assert( m_memNamePayload == 0 ); + return ProcessMemFreeImpl( *m_data.memory, ev ); +} + +MemEvent* Worker::ProcessMemFreeNamed( const QueueMemFree& ev ) +{ + assert( m_memNamePayload != 0 ); + auto memname = m_memNamePayload; + m_memNamePayload = 0; + auto it = m_data.memNameMap.find( memname ); + if( it == m_data.memNameMap.end() ) + { + CheckString( memname ); + it = m_data.memNameMap.emplace( memname, m_slab.AllocInit() ).first; + it->second->name = memname; + } + return ProcessMemFreeImpl( *it->second, ev ); +} + +void Worker::ProcessMemAllocCallstack( const QueueMemAlloc& ev ) +{ + auto mem = ProcessMemAlloc( ev ); + assert( m_serialNextCallstack != 0 ); + if( mem ) mem->SetCsAlloc( m_serialNextCallstack ); + m_serialNextCallstack = 0; +} + +void Worker::ProcessMemAllocCallstackNamed( const QueueMemAlloc& ev ) +{ + assert( m_memNamePayload != 0 ); + auto memname = m_memNamePayload; + m_memNamePayload = 0; + auto it = m_data.memNameMap.find( memname ); + if( it == m_data.memNameMap.end() ) + { + CheckString( memname ); + it = m_data.memNameMap.emplace( memname, m_slab.AllocInit() ).first; + it->second->name = memname; + } + auto mem = ProcessMemAllocImpl( *it->second, ev ); + assert( m_serialNextCallstack != 0 ); + if( mem ) mem->SetCsAlloc( m_serialNextCallstack ); + m_serialNextCallstack = 0; +} + +void Worker::ProcessMemFreeCallstack( const QueueMemFree& ev ) +{ + auto mem = ProcessMemFree( ev ); + assert( m_serialNextCallstack != 0 ); + if( mem ) mem->csFree.SetVal( m_serialNextCallstack ); + m_serialNextCallstack = 0; +} + +void Worker::ProcessMemFreeCallstackNamed( const QueueMemFree& ev ) +{ + assert( m_memNamePayload != 0 ); + auto memname = m_memNamePayload; + m_memNamePayload = 0; + auto it = m_data.memNameMap.find( memname ); + if( it == m_data.memNameMap.end() ) + { + CheckString( memname ); + it = m_data.memNameMap.emplace( memname, m_slab.AllocInit() ).first; + it->second->name = memname; + } + auto mem = ProcessMemFreeImpl( *it->second, ev ); + assert( m_serialNextCallstack != 0 ); + if( mem ) mem->csFree.SetVal( m_serialNextCallstack ); + m_serialNextCallstack = 0; +} + +void Worker::ProcessCallstackSerial() +{ + assert( m_pendingCallstackId != 0 ); + assert( m_serialNextCallstack == 0 ); + m_serialNextCallstack = m_pendingCallstackId; + m_pendingCallstackId = 0; +} + +void Worker::ProcessCallstack() +{ + assert( m_pendingCallstackId != 0 ); + auto td = GetCurrentThreadData(); + auto it = m_nextCallstack.find( td->id ); + if( it == m_nextCallstack.end() ) it = m_nextCallstack.emplace( td->id, 0 ).first; + assert( it->second == 0 ); + it->second = m_pendingCallstackId; + m_pendingCallstackId = 0; +} + +void Worker::ProcessCallstackSampleInsertSample( const SampleData& sd, ThreadData& td ) +{ + const auto t = sd.time.Val(); + if( td.samples.empty() ) + { + td.samples.push_back( sd ); + } + else if( t != 0 && td.samples.back().time.Val() >= t ) + { + m_inconsistentSamples = true; + auto it = std::lower_bound( td.samples.begin(), td.samples.end(), t, []( const auto& lhs, const auto& rhs ) { return lhs.time.Val() < rhs; } ); + assert( it != td.samples.end() ); + if( it->time.Val() != t ) + { + td.samples.push_back_non_empty( sd ); + } + else + { + const auto mcs = MergeCallstacks( it->callstack.Val(), sd.callstack.Val() ); + it->callstack.SetVal( mcs ); + + // This is a fixup of an already processed sample. Fixing stats is non-trivial, so just exit here. + return; + } + } + else + { + td.samples.push_back_non_empty( sd ); + } + + const auto callstack = sd.callstack.Val(); + const auto& cs = GetCallstack( callstack ); + const auto& ip = cs[0]; + if( GetCanonicalPointer( ip ) >> 63 != 0 ) td.kernelSampleCnt++; + m_data.samplesCnt++; +} + +void Worker::ProcessCallstackSampleImpl( const SampleData& sd, ThreadData& td ) +{ + ProcessCallstackSampleInsertSample( sd, td ); + +#ifndef TRACY_NO_STATISTICS + const auto t = sd.time.Val(); + if( t == 0 || !m_identifySamples ) + { + ProcessCallstackSampleImplStats( sd, td ); + } + else + { + bool postpone = false; + auto ctx = GetContextSwitchData( td.id ); + if( !ctx ) + { + postpone = true; + } + else + { + auto it = std::lower_bound( ctx->v.begin(), ctx->v.end(), sd.time.Val(), [] ( const auto& l, const auto& r ) { return (uint64_t)l.End() < (uint64_t)r; } ); + if( it == ctx->v.end() ) + { + postpone = true; + } + else if( sd.time.Val() == it->Start() ) + { + td.ctxSwitchSamples.push_back( sd ); + } + else + { + ProcessCallstackSampleImplStats( sd, td ); + } + } + if( postpone ) + { + td.postponedSamples.push_back( sd ); + } + } +#endif +} + +#ifndef TRACY_NO_STATISTICS +void Worker::ProcessCallstackSampleImplStats( const SampleData& sd, ThreadData& td ) +{ + const auto t = sd.time.Val(); + const auto callstack = sd.callstack.Val(); + const auto& cs = GetCallstack( callstack ); + const auto& ip = cs[0]; + + uint16_t tid = CompressThread( td.id ); + + auto frame = GetCallstackFrame( ip ); + if( frame ) + { + const auto symAddr = frame->data[0].symAddr; + auto it = m_data.instructionPointersMap.find( symAddr ); + if( it == m_data.instructionPointersMap.end() ) + { + m_data.instructionPointersMap.emplace( symAddr, unordered_flat_map { { ip, 1 } } ); + } + else + { + auto fit = it->second.find( ip ); + if( fit == it->second.end() ) + { + it->second.emplace( ip, 1 ); + } + else + { + fit->second++; + } + } + auto sit = m_data.symbolSamples.find( symAddr ); + if( sit == m_data.symbolSamples.end() ) + { + m_data.symbolSamples.emplace( symAddr, Vector( SampleDataRange { sd.time, tid, ip } ) ); + } + else + { + if( sit->second.back().time.Val() <= sd.time.Val() ) + { + sit->second.push_back_non_empty( SampleDataRange { sd.time, tid, ip } ); + } + else + { + auto iit = std::upper_bound( sit->second.begin(), sit->second.end(), sd.time.Val(), [] ( const auto& lhs, const auto& rhs ) { return lhs < rhs.time.Val(); } ); + sit->second.insert( iit, SampleDataRange { sd.time, tid, ip } ); + } + } + } + else + { + auto it = m_data.pendingInstructionPointers.find( ip ); + if( it == m_data.pendingInstructionPointers.end() ) + { + m_data.pendingInstructionPointers.emplace( ip, 1 ); + } + else + { + it->second++; + } + auto sit = m_data.pendingSymbolSamples.find( ip ); + if( sit == m_data.pendingSymbolSamples.end() ) + { + m_data.pendingSymbolSamples.emplace( ip, Vector( SampleDataRange { sd.time, tid, ip } ) ); + } + else + { + sit->second.push_back_non_empty( SampleDataRange { sd.time, tid, ip } ); + } + } + + auto childAddr = GetCanonicalPointer( cs[0] ); + for( uint16_t i=1; i( ChildSample { sd.time, childAddr } ) ); + } + else + { + it->second.push_back_non_empty( ChildSample { sd.time, childAddr } ); + } + childAddr = addr; + } + + const auto framesKnown = UpdateSampleStatistics( callstack, 1, true ); + + if( t != 0 ) + { + //assert( td.samples.size() > td.ghostIdx ); + if( framesKnown && td.ghostIdx + 1 == td.samples.size() ) + { + td.ghostIdx++; + m_data.ghostCnt += AddGhostZone( cs, &td.ghostZones, t ); + } + else + { + m_data.ghostZonesPostponed = true; + } + } +} +#endif + +void Worker::ProcessCallstackSample( const QueueCallstackSample& ev ) +{ + assert( m_pendingCallstackId != 0 ); + const auto callstack = m_pendingCallstackId; + m_pendingCallstackId = 0; + + const auto refTime = RefTime( m_refTimeCtx, ev.time ); + const auto t = refTime == 0 ? 0 : TscTime( refTime ); + if( m_data.lastTime < t ) m_data.lastTime = t; + + auto& td = *NoticeThread( ev.thread ); + + SampleData sd; + sd.time.SetVal( t ); + sd.callstack.SetVal( callstack ); + + if( m_combineSamples && t != 0 ) + { + const auto pendingTime = td.pendingSample.time.Val(); + if( pendingTime == 0 ) + { + td.pendingSample = sd; + } + else + { + if( pendingTime == t ) + { + const auto mcs = MergeCallstacks( td.pendingSample.callstack.Val(), callstack ); + sd.callstack.SetVal( mcs ); + ProcessCallstackSampleImpl( sd, td ); + td.pendingSample.time.Clear(); + } + else + { + ProcessCallstackSampleImpl( td.pendingSample, td ); + td.pendingSample = sd; + } + } + } + else + { + ProcessCallstackSampleImpl( sd, td ); + } +} + +void Worker::ProcessCallstackSampleContextSwitch( const QueueCallstackSample& ev ) +{ + assert( m_pendingCallstackId != 0 ); + const auto callstack = m_pendingCallstackId; + m_pendingCallstackId = 0; + + const auto refTime = RefTime( m_refTimeCtx, ev.time ); + const auto t = refTime == 0 ? 0 : TscTime( refTime ); + if( m_data.lastTime < t ) m_data.lastTime = t; + + auto& td = *NoticeThread( ev.thread ); + + SampleData sd; + sd.time.SetVal( t ); + sd.callstack.SetVal( callstack ); + + ProcessCallstackSampleInsertSample( sd, td ); + + td.ctxSwitchSamples.push_back( sd ); +} + +void Worker::ProcessCallstackFrameSize( const QueueCallstackFrameSize& ev ) +{ + assert( !m_callstackFrameStaging ); + assert( m_pendingCallstackSubframes == 0 ); + assert( m_pendingCallstackFrames > 0 ); + m_pendingCallstackFrames--; + m_pendingCallstackSubframes = ev.size; +#ifndef TRACY_NO_STATISTICS + m_data.newFramesWereReceived = true; +#endif + + const auto idx = GetSingleStringIdx(); + + // Frames may be duplicated due to recursion + auto fmit = m_data.callstackFrameMap.find( PackPointer( ev.ptr ) ); + if( fmit == m_data.callstackFrameMap.end() ) + { + m_callstackFrameStaging = m_slab.Alloc(); + m_callstackFrameStaging->size = ev.size; + m_callstackFrameStaging->data = m_slab.Alloc( ev.size ); + m_callstackFrameStaging->imageName = StringIdx( idx ); + + m_callstackFrameStagingPtr = ev.ptr; + } +} + +void Worker::ProcessCallstackFrame( const QueueCallstackFrame& ev, bool querySymbols ) +{ + assert( m_pendingCallstackSubframes > 0 ); + + const auto nitidx = GetSingleStringIdx(); + const auto fitidx = GetSecondStringIdx(); + + if( m_callstackFrameStaging ) + { + const auto idx = m_callstackFrameStaging->size - m_pendingCallstackSubframes; + const auto file = StringIdx( fitidx ); + + if( m_pendingCallstackSubframes > 1 && idx == 0 ) + { + auto fstr = GetString( file ); + auto flen = strlen( fstr ); + if( flen >= s_tracySkipSubframesMinLen ) + { + auto ptr = s_tracySkipSubframes; + do + { + if( flen >= ptr->len && memcmp( fstr + flen - ptr->len, ptr->str, ptr->len ) == 0 ) + { + m_pendingCallstackSubframes--; + m_callstackFrameStaging->size--; + return; + } + ptr++; + } + while( ptr->str ); + } + } + + const auto name = StringIdx( nitidx ); + m_callstackFrameStaging->data[idx].name = name; + m_callstackFrameStaging->data[idx].file = file; + m_callstackFrameStaging->data[idx].line = ev.line; + m_callstackFrameStaging->data[idx].symAddr = ev.symAddr; + + if( querySymbols && ev.symAddr != 0 && m_data.symbolMap.find( ev.symAddr ) == m_data.symbolMap.end() && m_pendingSymbols.find( ev.symAddr ) == m_pendingSymbols.end() ) + { + m_pendingSymbols.emplace( ev.symAddr, SymbolPending { name, m_callstackFrameStaging->imageName, file, ev.line, ev.symLen, idx < m_callstackFrameStaging->size - 1 } ); + Query( ServerQuerySymbol, ev.symAddr ); + } + + StringRef ref( StringRef::Idx, fitidx ); + auto cit = m_checkedFileStrings.find( ref ); + if( cit == m_checkedFileStrings.end() ) CacheSource( ref, m_callstackFrameStaging->imageName ); + + const auto frameId = PackPointer( m_callstackFrameStagingPtr ); +#ifndef TRACY_NO_STATISTICS + auto it = m_data.pendingInstructionPointers.find( frameId ); + if( it != m_data.pendingInstructionPointers.end() ) + { + if( ev.symAddr != 0 ) + { + auto sit = m_data.instructionPointersMap.find( ev.symAddr ); + if( sit == m_data.instructionPointersMap.end() ) + { + m_data.instructionPointersMap.emplace( ev.symAddr, unordered_flat_map { { it->first, it->second } } ); + } + else + { + assert( sit->second.find( it->first ) == sit->second.end() ); + sit->second.emplace( it->first, it->second ); + } + } + m_data.pendingInstructionPointers.erase( it ); + } + auto pit = m_data.pendingSymbolSamples.find( frameId ); + if( pit != m_data.pendingSymbolSamples.end() ) + { + if( ev.symAddr != 0 ) + { + auto sit = m_data.symbolSamples.find( ev.symAddr ); + if( sit == m_data.symbolSamples.end() ) + { + pdqsort_branchless( pit->second.begin(), pit->second.end(), [] ( const auto& lhs, const auto& rhs ) { return lhs.time.Val() < rhs.time.Val(); } ); + m_data.symbolSamples.emplace( ev.symAddr, std::move( pit->second ) ); + } + else + { + for( auto& v : pit->second ) + { + if( sit->second.back().time.Val() <= v.time.Val() ) + { + sit->second.push_back_non_empty( v ); + } + else + { + auto iit = std::upper_bound( sit->second.begin(), sit->second.end(), v.time.Val(), [] ( const auto& lhs, const auto& rhs ) { return lhs < rhs.time.Val(); } ); + sit->second.insert( iit, v ); + } + } + } + } + m_data.pendingSymbolSamples.erase( pit ); + } +#endif + + if( --m_pendingCallstackSubframes == 0 ) + { + assert( m_data.callstackFrameMap.find( frameId ) == m_data.callstackFrameMap.end() ); + m_data.callstackFrameMap.emplace( frameId, m_callstackFrameStaging ); + m_data.codeSymbolMap.emplace( m_callstackFrameStagingPtr, m_callstackFrameStaging->data[0].symAddr ); + m_callstackFrameStaging = nullptr; + } + } + else + { + m_pendingCallstackSubframes--; + } +} + +void Worker::ProcessSymbolInformation( const QueueSymbolInformation& ev ) +{ + auto it = m_pendingSymbols.find( ev.symAddr ); + assert( it != m_pendingSymbols.end() ); + + const auto idx = GetSingleStringIdx(); + + SymbolData sd; + sd.name = it->second.name; + sd.file = StringIdx( idx ); + sd.line = ev.line; + sd.imageName = it->second.imageName; + sd.callFile = it->second.file; + sd.callLine = it->second.line; + sd.isInline = it->second.isInline; + sd.size.SetVal( it->second.size ); + m_data.symbolMap.emplace( ev.symAddr, sd ); + + if( m_codeTransfer && it->second.size > 0 && it->second.size <= 128*1024 ) + { + m_pendingSymbolCode++; + Query( ServerQuerySymbolCode, ev.symAddr, it->second.size ); + } + + if( !it->second.isInline ) + { + if( m_data.newSymbolsIndex < 0 ) m_data.newSymbolsIndex = int64_t( m_data.symbolLoc.size() ); + m_data.symbolLoc.push_back( SymbolLocation { ev.symAddr, it->second.size } ); + } + else + { + if( m_data.newInlineSymbolsIndex < 0 ) m_data.newInlineSymbolsIndex = int64_t( m_data.symbolLocInline.size() ); + m_data.symbolLocInline.push_back( ev.symAddr ); + } + + StringRef ref( StringRef::Idx, idx ); + auto cit = m_checkedFileStrings.find( ref ); + if( cit == m_checkedFileStrings.end() ) CacheSource( ref, it->second.imageName ); + + m_pendingSymbols.erase( it ); +} + +void Worker::ProcessCrashReport( const QueueCrashReport& ev ) +{ + CheckString( ev.text ); + + auto td = GetCurrentThreadData(); + m_data.crashEvent.thread = td->id; + m_data.crashEvent.time = TscTime( ev.time ); + m_data.crashEvent.message = ev.text; + + auto it = m_nextCallstack.find( td->id ); + if( it != m_nextCallstack.end() && it->second != 0 ) + { + m_data.crashEvent.callstack = it->second; + it->second = 0; + } + else + { + m_data.crashEvent.callstack = 0; + } +} + +void Worker::ProcessSysTime( const QueueSysTime& ev ) +{ + const auto time = TscTime( ev.time ); + if( m_data.lastTime < time ) m_data.lastTime = time; + const auto val = ev.sysTime; + if( !m_sysTimePlot ) + { + m_sysTimePlot = m_slab.AllocInit(); + m_sysTimePlot->name = 0; + m_sysTimePlot->type = PlotType::SysTime; + m_sysTimePlot->format = PlotValueFormatting::Percentage; + m_sysTimePlot->showSteps = false; + m_sysTimePlot->fill = true; + m_sysTimePlot->color = 0; + m_sysTimePlot->min = val; + m_sysTimePlot->max = val; + m_sysTimePlot->sum = val; + m_sysTimePlot->data.push_back( { time, val } ); + m_data.plots.Data().push_back( m_sysTimePlot ); + } + else + { + assert( !m_sysTimePlot->data.empty() ); + assert( m_sysTimePlot->data.back().time.Val() <= time ); + if( m_sysTimePlot->min > val ) m_sysTimePlot->min = val; + else if( m_sysTimePlot->max < val ) m_sysTimePlot->max = val; + m_sysTimePlot->sum += val; + m_sysTimePlot->data.push_back( { time, val } ); + } +} + +void Worker::ProcessSysPower( const QueueSysPower& ev ) +{ + const auto time = TscTime( ev.time ); + auto it = m_powerData.find( ev.name ); + if( it == m_powerData.end() ) + { + CheckString( ev.name ); + PlotData* plot = m_slab.AllocInit(); + plot->name = ev.name; + plot->type = PlotType::Power; + plot->format = PlotValueFormatting::Watt; + plot->showSteps = false; + plot->fill = true; + plot->color = 0; + m_data.plots.Data().push_back( plot ); + m_powerData.emplace( ev.name, PowerData { time, plot } ); + } + else + { + const auto dt = time - it->second.lastTime; + it->second.lastTime = time; + if( m_data.lastTime < time ) m_data.lastTime = time; + // ev.delta is Microjoule, dt is nanoseconds + // power is Watt = J / s + const auto power = ev.delta * 1000. / dt; + InsertPlot( it->second.plot, time, power ); + } +} + +void Worker::ProcessContextSwitch( const QueueContextSwitch& ev ) +{ +#ifndef TRACY_NO_STATISTICS + m_data.newContextSwitchesReceived = true; +#endif + + const auto time = TscTime( RefTime( m_refTimeCtx, ev.time ) ); + if( m_data.lastTime < time ) m_data.lastTime = time; + + if( ev.cpu >= m_data.cpuDataCount ) m_data.cpuDataCount = ev.cpu + 1; + auto& cs = m_data.cpuData[ev.cpu].cs; + if( ev.oldThread != 0 ) + { + auto it = m_data.ctxSwitch.find( ev.oldThread ); + if( it != m_data.ctxSwitch.end() ) + { + auto& data = it->second->v; + assert( !data.empty() ); + auto& item = data.back(); + assert( item.Start() <= time ); + assert( item.End() == -1 ); + item.SetEnd( time ); + item.SetReason( ev.reason ); + item.SetState( ev.state ); + + const auto dt = time - item.Start(); + it->second->runningTime += dt; + + auto tdit = m_data.cpuThreadData.find( ev.oldThread ); + if( tdit == m_data.cpuThreadData.end() ) + { + tdit = m_data.cpuThreadData.emplace( ev.oldThread, CpuThreadData {} ).first; + } + tdit->second.runningRegions++; + tdit->second.runningTime += dt; + } + if( !cs.empty() ) + { + auto& cx = cs.back(); + assert( m_data.externalThreadCompress.DecompressThread( cx.Thread() ) == ev.oldThread ); + cx.SetEnd( time ); + } + } + if( ev.newThread != 0 ) + { + auto it = m_data.ctxSwitch.find( ev.newThread ); + if( it == m_data.ctxSwitch.end() ) + { + auto ctx = m_slab.AllocInit(); + it = m_data.ctxSwitch.emplace( ev.newThread, ctx ).first; + } + auto& data = it->second->v; + ContextSwitchData* item = nullptr; + bool migration = false; + if( !data.empty() && data.back().Reason() == ContextSwitchData::Wakeup ) + { + item = &data.back(); + if( data.size() > 1 ) + { + migration = data[data.size()-2].Cpu() != ev.cpu; + } + } + else + { + assert( data.empty() || (uint64_t)data.back().End() <= (uint64_t)time ); + if( !data.empty() ) + { + migration = data.back().Cpu() != ev.cpu; + } + item = &data.push_next(); + item->SetWakeup( time ); + } + item->SetStart( time ); + item->SetEnd( -1 ); + item->SetCpu( ev.cpu ); + item->SetReason( -1 ); + item->SetState( -1 ); + item->SetThread( 0 ); + + auto& cx = cs.push_next(); + cx.SetStart( time ); + cx.SetEnd( -1 ); + cx.SetThread( m_data.externalThreadCompress.CompressThread( ev.newThread ) ); + + CheckExternalName( ev.newThread ); + + if( migration ) + { + auto tdit = m_data.cpuThreadData.find( ev.newThread ); + if( tdit == m_data.cpuThreadData.end() ) + { + tdit = m_data.cpuThreadData.emplace( ev.newThread, CpuThreadData {} ).first; + } + tdit->second.migrations++; + } + } +} + +void Worker::ProcessThreadWakeup( const QueueThreadWakeup& ev ) +{ + const auto time = TscTime( RefTime( m_refTimeCtx, ev.time ) ); + if( m_data.lastTime < time ) m_data.lastTime = time; + + auto it = m_data.ctxSwitch.find( ev.thread ); + if( it == m_data.ctxSwitch.end() ) + { + auto ctx = m_slab.AllocInit(); + it = m_data.ctxSwitch.emplace( ev.thread, ctx ).first; + } + auto& data = it->second->v; + if( !data.empty() && !data.back().IsEndValid() ) return; // wakeup of a running thread + auto& item = data.push_next(); + item.SetWakeup( time ); + item.SetStart( time ); + item.SetEnd( -1 ); + item.SetCpu( 0 ); + item.SetReason( ContextSwitchData::Wakeup ); + item.SetState( -1 ); + item.SetThread( 0 ); +} + +void Worker::ProcessTidToPid( const QueueTidToPid& ev ) +{ + if( m_data.tidToPid.find( ev.tid ) == m_data.tidToPid.end() ) m_data.tidToPid.emplace( ev.tid, ev.pid ); +} + +void Worker::ProcessHwSampleCpuCycle( const QueueHwSample& ev ) +{ + const auto time = ev.time == 0 ? 0 : TscTime( ev.time ); + if( m_data.lastTime < time ) m_data.lastTime = time; + auto it = m_data.hwSamples.find( ev.ip ); + if( it == m_data.hwSamples.end() ) it = m_data.hwSamples.emplace( ev.ip, HwSampleData {} ).first; + it->second.cycles.push_back( time ); +} + +void Worker::ProcessHwSampleInstructionRetired( const QueueHwSample& ev ) +{ + const auto time = ev.time == 0 ? 0 : TscTime( ev.time ); + if( m_data.lastTime < time ) m_data.lastTime = time; + auto it = m_data.hwSamples.find( ev.ip ); + if( it == m_data.hwSamples.end() ) it = m_data.hwSamples.emplace( ev.ip, HwSampleData {} ).first; + it->second.retired.push_back( time ); +} + +void Worker::ProcessHwSampleCacheReference( const QueueHwSample& ev ) +{ + const auto time = ev.time == 0 ? 0 : TscTime( ev.time ); + if( m_data.lastTime < time ) m_data.lastTime = time; + auto it = m_data.hwSamples.find( ev.ip ); + if( it == m_data.hwSamples.end() ) it = m_data.hwSamples.emplace( ev.ip, HwSampleData {} ).first; + it->second.cacheRef.push_back( time ); +} + +void Worker::ProcessHwSampleCacheMiss( const QueueHwSample& ev ) +{ + const auto time = ev.time == 0 ? 0 : TscTime( ev.time ); + if( m_data.lastTime < time ) m_data.lastTime = time; + auto it = m_data.hwSamples.find( ev.ip ); + if( it == m_data.hwSamples.end() ) it = m_data.hwSamples.emplace( ev.ip, HwSampleData {} ).first; + it->second.cacheMiss.push_back( time ); +} + +void Worker::ProcessHwSampleBranchRetired( const QueueHwSample& ev ) +{ + const auto time = ev.time == 0 ? 0 : TscTime( ev.time ); + if( m_data.lastTime < time ) m_data.lastTime = time; + auto it = m_data.hwSamples.find( ev.ip ); + if( it == m_data.hwSamples.end() ) it = m_data.hwSamples.emplace( ev.ip, HwSampleData {} ).first; + it->second.branchRetired.push_back( time ); + m_data.hasBranchRetirement = true; +} + +void Worker::ProcessHwSampleBranchMiss( const QueueHwSample& ev ) +{ + const auto time = ev.time == 0 ? 0 : TscTime( ev.time ); + if( m_data.lastTime < time ) m_data.lastTime = time; + auto it = m_data.hwSamples.find( ev.ip ); + if( it == m_data.hwSamples.end() ) it = m_data.hwSamples.emplace( ev.ip, HwSampleData {} ).first; + it->second.branchMiss.push_back( time ); +} + +void Worker::ProcessParamSetup( const QueueParamSetup& ev ) +{ + CheckString( ev.name ); + m_params.push_back( Parameter { ev.idx, StringRef( StringRef::Ptr, ev.name ), bool( ev.isBool ), ev.val } ); +} + +void Worker::ProcessSourceCodeNotAvailable( const QueueSourceCodeNotAvailable& ev ) +{ + auto it = m_sourceCodeQuery.find( ev.id ); + assert( it != m_sourceCodeQuery.end() ); + m_sourceCodeQuery.erase( it ); +} + +void Worker::ProcessCpuTopology( const QueueCpuTopology& ev ) +{ + auto package = m_data.cpuTopology.find( ev.package ); + if( package == m_data.cpuTopology.end() ) package = m_data.cpuTopology.emplace( ev.package, unordered_flat_map> {} ).first; + auto core = package->second.find( ev.core ); + if( core == package->second.end() ) core = package->second.emplace( ev.core, std::vector {} ).first; + core->second.emplace_back( ev.thread ); + + assert( m_data.cpuTopologyMap.find( ev.thread ) == m_data.cpuTopologyMap.end() ); + m_data.cpuTopologyMap.emplace( ev.thread, CpuThreadTopology { ev.package, ev.core } ); +} + +void Worker::ProcessMemNamePayload( const QueueMemNamePayload& ev ) +{ + assert( m_memNamePayload == 0 ); + m_memNamePayload = ev.name; +} + +void Worker::ProcessThreadGroupHint( const QueueThreadGroupHint& ev ) +{ + auto td = RetrieveThread( ev.thread ); + assert( td ); + td->groupHint = ev.groupHint; + m_pendingThreadHints.emplace_back( ev.thread ); +} + +void Worker::ProcessFiberEnter( const QueueFiberEnter& ev ) +{ + const auto t = TscTime( RefTime( m_refTimeThread, ev.time ) ); + if( m_data.lastTime < t ) m_data.lastTime = t; + + uint64_t tid; + auto it = m_data.fiberToThreadMap.find( ev.fiber ); + if( it == m_data.fiberToThreadMap.end() ) + { + tid = ( uint64_t(1) << 32 ) | m_data.fiberToThreadMap.size(); + m_data.fiberToThreadMap.emplace( ev.fiber, tid ); + NewThread( tid, true, ev.groupHint ); + CheckFiberName( ev.fiber, tid ); + } + else + { + tid = it->second; + } + + auto td = NoticeThread( ev.thread ); + if( td->fiber ) + { + auto cit = m_data.ctxSwitch.find( td->fiber->id ); + assert( cit != m_data.ctxSwitch.end() ); + auto& data = cit->second->v; + assert( !data.empty() ); + auto& item = data.back(); + item.SetEnd( t ); + } + td->fiber = RetrieveThread( tid ); + assert( td->fiber ); + + auto cit = m_data.ctxSwitch.find( tid ); + if( cit == m_data.ctxSwitch.end() ) + { + auto ctx = m_slab.AllocInit(); + cit = m_data.ctxSwitch.emplace( tid, ctx ).first; + } + auto& data = cit->second->v; + auto& item = data.push_next(); + item.SetStartCpu( t, 0 ); + item.SetWakeup( t ); + item.SetEndReasonState( -1, ContextSwitchData::Fiber, -1 ); + item.SetThread( CompressThread( ev.thread ) ); +} + +void Worker::ProcessFiberLeave( const QueueFiberLeave& ev ) +{ + const auto t = TscTime( RefTime( m_refTimeThread, ev.time ) ); + if( m_data.lastTime < t ) m_data.lastTime = t; + + auto td = RetrieveThread( ev.thread ); + if( !td->fiber ) + { + FiberLeaveFailure(); + return; + } + + auto cit = m_data.ctxSwitch.find( td->fiber->id ); + assert( cit != m_data.ctxSwitch.end() ); + auto& data = cit->second->v; + assert( !data.empty() ); + auto& item = data.back(); + item.SetEnd( t ); + + const auto dt = t - item.Start(); + cit->second->runningTime += dt; + + td->fiber = nullptr; +} + +void Worker::MemAllocChanged( MemData& memdata, int64_t time ) +{ + const auto val = (double)memdata.usage; + if( !memdata.plot ) + { + CreateMemAllocPlot( memdata ); + memdata.plot->min = val; + memdata.plot->max = val; + memdata.plot->sum = val; + memdata.plot->data.push_back( { time, val } ); + } + else + { + assert( !memdata.plot->data.empty() ); + assert( memdata.plot->data.back().time.Val() <= time ); + if( memdata.plot->min > val ) memdata.plot->min = val; + else if( memdata.plot->max < val ) memdata.plot->max = val; + memdata.plot->sum += val; + memdata.plot->data.push_back( { time, val } ); + } +} + +void Worker::CreateMemAllocPlot( MemData& memdata ) +{ + assert( !memdata.plot ); + memdata.plot = m_slab.AllocInit(); + memdata.plot->name = memdata.name; + memdata.plot->type = PlotType::Memory; + memdata.plot->format = PlotValueFormatting::Memory; + memdata.plot->showSteps = true; + memdata.plot->fill = true; + memdata.plot->color = 0; + memdata.plot->data.push_back( { GetFirstTime(), 0. } ); + m_data.plots.Data().push_back( memdata.plot ); +} + +void Worker::ReconstructMemAllocPlot( MemData& mem ) +{ +#ifdef NO_PARALLEL_SORT + pdqsort_branchless( mem.frees.begin(), mem.frees.end(), [&mem] ( const auto& lhs, const auto& rhs ) { return mem.data[lhs].TimeFree() < mem.data[rhs].TimeFree(); } ); +#else + std::sort( std::execution::par_unseq, mem.frees.begin(), mem.frees.end(), [&mem] ( const auto& lhs, const auto& rhs ) { return mem.data[lhs].TimeFree() < mem.data[rhs].TimeFree(); } ); +#endif + + const auto psz = mem.data.size() + mem.frees.size() + 1; + + PlotData* plot; + { + std::lock_guard lock( m_data.lock ); + plot = m_slab.AllocInit(); + plot->data.reserve_exact( psz, m_slab ); + } + + plot->name = mem.name; + plot->type = PlotType::Memory; + plot->format = PlotValueFormatting::Memory; + plot->showSteps = true; + plot->fill = true; + plot->color = 0; + + auto aptr = mem.data.begin(); + auto aend = mem.data.end(); + auto fptr = mem.frees.begin(); + auto fend = mem.frees.end(); + + double sum = 0; + double max = 0; + double usage = 0; + + auto ptr = plot->data.data(); + ptr->time = GetFirstTime(); + ptr->val = 0; + ptr++; + + if( aptr != aend && fptr != fend ) + { + auto atime = aptr->TimeAlloc(); + auto ftime = mem.data[*fptr].TimeFree(); + + for(;;) + { + if( atime < ftime ) + { + usage += int64_t( aptr->Size() ); + assert( usage >= 0 ); + if( max < usage ) max = usage; + sum += usage; + ptr->time = atime; + ptr->val = usage; + ptr++; + aptr++; + if( aptr == aend ) break; + atime = aptr->TimeAlloc(); + } + else + { + usage -= int64_t( mem.data[*fptr].Size() ); + assert( usage >= 0 ); + if( max < usage ) max = usage; + sum += usage; + ptr->time = ftime; + ptr->val = usage; + ptr++; + fptr++; + if( fptr == fend ) break; + ftime = mem.data[*fptr].TimeFree(); + } + } + } + + while( aptr != aend ) + { + assert( aptr->TimeFree() < 0 ); + int64_t time = aptr->TimeAlloc(); + usage += int64_t( aptr->Size() ); + assert( usage >= 0 ); + if( max < usage ) max = usage; + sum += usage; + ptr->time = time; + ptr->val = usage; + ptr++; + aptr++; + } + while( fptr != fend ) + { + const auto& memData = mem.data[*fptr]; + int64_t time = memData.TimeFree(); + usage -= int64_t( memData.Size() ); + assert( usage >= 0 ); + assert( max >= usage ); + sum += usage; + ptr->time = time; + ptr->val = usage; + ptr++; + fptr++; + } + + plot->min = 0; + plot->max = max; + plot->sum = sum; + + std::lock_guard lock( m_data.lock ); + m_data.plots.Data().insert( m_data.plots.Data().begin(), plot ); + mem.plot = plot; +} + +#ifndef TRACY_NO_STATISTICS +void Worker::ReconstructContextSwitchUsage() +{ + assert( m_data.cpuDataCount != 0 ); + const auto cpucnt = m_data.cpuDataCount; + + auto& vec = m_data.ctxUsage; + vec.push_back( ContextSwitchUsage( 0, 0, 0 ) ); + + struct Cpu + { + bool startDone; + Vector::iterator it; + Vector::iterator end; + }; + std::vector cpus; + cpus.reserve( cpucnt ); + for( int i=0; i::max(); + bool atEnd = true; + for( int i=0; iStart() : cpus[i].it->End(); + if( ct < nextTime ) nextTime = ct; + } + } + if( atEnd ) break; + for( int i=0; iStart() : cpus[i].it->End(); + if( nextTime != ct ) break; + const auto tid = DecompressThreadExternal( cpus[i].it->Thread() ); + const auto isOwn = IsThreadLocal( tid ) || GetPidFromTid( tid ) == m_pid; + if( !cpus[i].startDone ) + { + if( isOwn ) + { + own++; + assert( own <= cpucnt ); + } + else + { + other++; + assert( other <= cpucnt ); + } + if( !cpus[i].it->IsEndValid() ) + { + cpus[i].it++; + assert( cpus[i].it == cpus[i].end ); + } + else + { + cpus[i].startDone = true; + } + } + else + { + if( isOwn ) + { + assert( own > 0 ); + own--; + } + else + { + assert( other > 0 ); + other--; + } + cpus[i].startDone = false; + cpus[i].it++; + } + } + } + const auto& back = vec.back(); + if( back.Other() != other || back.Own() != own ) + { + vec.push_back( ContextSwitchUsage( nextTime, other, own ) ); + } + } + + std::lock_guard lock( m_data.lock ); + m_data.ctxUsageReady = true; +} + +bool Worker::UpdateSampleStatistics( uint32_t callstack, uint32_t count, bool canPostpone ) +{ + const auto& cs = GetCallstack( callstack ); + const auto cssz = cs.size(); + + auto frames = (const CallstackFrameData**)alloca( cssz * sizeof( CallstackFrameData* ) ); + for( uint16_t i=0; isecond += count; + } + } + return false; + } + else + { + frames[i] = frame; + } + } + + if( canPostpone ) + { + auto it = m_data.postponedSamples.find( callstack ); + if( it != m_data.postponedSamples.end() ) + { + count += it->second; + m_data.postponedSamples.erase( it ); + } + } + + UpdateSampleStatisticsImpl( frames, cssz, count, cs ); + return true; +} + +void Worker::UpdateSampleStatisticsPostponed( decltype(Worker::DataBlock::postponedSamples.begin())& it ) +{ + const auto& cs = GetCallstack( it->first ); + const auto cssz = cs.size(); + + auto frames = (const CallstackFrameData**)alloca( cssz * sizeof( CallstackFrameData* ) ); + for( uint16_t i=0; isecond, cs ); + it = m_data.postponedSamples.erase( it ); +} + +void Worker::UpdateSampleStatisticsImpl( const CallstackFrameData** frames, uint16_t framesCount, uint32_t count, const VarArray& cs ) +{ + const auto fexcl = frames[0]; + const auto fxsz = fexcl->size; + const auto& frame0 = fexcl->data[0]; + auto sym0 = m_data.symbolStats.find( frame0.symAddr ); + if( sym0 == m_data.symbolStats.end() ) sym0 = m_data.symbolStats.emplace( frame0.symAddr, SymbolStats { 0, 0 } ).first; + sym0->second.excl += count; + for( uint8_t f=1; fdata[f]; + auto sym = m_data.symbolStats.find( frame.symAddr ); + if( sym == m_data.symbolStats.end() ) sym = m_data.symbolStats.emplace( frame.symAddr, SymbolStats { 0, 0 } ).first; + sym->second.incl += count; + } + for( uint16_t c=1; csize; + for( uint8_t f=0; fdata[f]; + auto sym = m_data.symbolStats.find( frame.symAddr ); + if( sym == m_data.symbolStats.end() ) sym = m_data.symbolStats.emplace( frame.symAddr, SymbolStats { 0, 0 } ).first; + sym->second.incl += count; + } + } + + CallstackFrameId parentFrameId; + if( fxsz != 1 ) + { + auto cfdata = (CallstackFrame*)alloca( uint8_t( fxsz-1 ) * sizeof( CallstackFrame ) ); + for( int i=0; idata[i+1]; + } + CallstackFrameData cfd; + cfd.data = cfdata; + cfd.size = fxsz-1; + cfd.imageName = fexcl->imageName; + + auto it = m_data.revParentFrameMap.find( &cfd ); + if( it == m_data.revParentFrameMap.end() ) + { + auto frame = m_slab.Alloc( fxsz-1 ); + memcpy( frame, cfdata, ( fxsz-1 ) * sizeof( CallstackFrame ) ); + auto frameData = m_slab.AllocInit(); + frameData->data = frame; + frameData->size = fxsz - 1; + frameData->imageName = fexcl->imageName; + parentFrameId.idx = m_callstackParentNextIdx++; + parentFrameId.sel = 0; + parentFrameId.custom = 1; + m_data.parentCallstackFrameMap.emplace( parentFrameId, frameData ); + m_data.revParentFrameMap.emplace( frameData, parentFrameId ); + } + else + { + parentFrameId = it->second; + } + } + + uint32_t parentIdx; + { + const auto sz = framesCount - ( fxsz == 1 ); + const auto memsize = sizeof( VarArray ) + sz * sizeof( CallstackFrameId ); + auto mem = (char*)m_slab.AllocRaw( memsize ); + + auto data = (CallstackFrameId*)mem; + auto dst = data; + if( fxsz == 1 ) + { + for( int i=0; i*)( mem + sz * sizeof( CallstackFrameId ) ); + new(arr) VarArray( sz, data ); + + auto it = m_data.parentCallstackMap.find( arr ); + if( it == m_data.parentCallstackMap.end() ) + { + parentIdx = m_data.parentCallstackPayload.size(); + m_data.parentCallstackMap.emplace( arr, parentIdx ); + m_data.parentCallstackPayload.push_back( arr ); + } + else + { + parentIdx = it->second; + m_slab.Unalloc( memsize ); + } + } + + sym0 = m_data.symbolStats.find( frame0.symAddr ); + auto sit = sym0->second.parents.find( parentIdx ); + if( sit == sym0->second.parents.end() ) + { + sym0->second.parents.emplace( parentIdx, count ); + } + else + { + sit->second += count; + } + + uint32_t baseParentIdx; + { + const auto sz = framesCount - 1; + const auto memsize = sizeof( VarArray ) + sz * sizeof( CallstackFrameId ); + auto mem = (char*)m_slab.AllocRaw( memsize ); + + auto data = (CallstackFrameId*)mem; + auto dst = data; + for( int i=0; i*)( mem + sz * sizeof( CallstackFrameId ) ); + new(arr) VarArray( sz, data ); + + auto it = m_data.parentCallstackMap.find( arr ); + if( it == m_data.parentCallstackMap.end() ) + { + baseParentIdx = m_data.parentCallstackPayload.size(); + m_data.parentCallstackMap.emplace( arr, baseParentIdx ); + m_data.parentCallstackPayload.push_back( arr ); + } + else + { + baseParentIdx = it->second; + m_slab.Unalloc( memsize ); + } + } + + auto bit = sym0->second.baseParents.find( baseParentIdx ); + if( bit == sym0->second.baseParents.end() ) + { + sym0->second.baseParents.emplace( baseParentIdx, count ); + } + else + { + bit->second += count; + } +} +#endif + +int64_t Worker::ReadTimeline( FileRead& f, ZoneEvent* zone, int64_t refTime, int32_t& childIdx ) +{ + uint32_t sz; + f.Read( sz ); + return ReadTimelineHaveSize( f, zone, refTime, childIdx, sz ); +} + +int64_t Worker::ReadTimelineHaveSize( FileRead& f, ZoneEvent* zone, int64_t refTime, int32_t& childIdx, uint32_t sz ) +{ + if( sz == 0 ) + { + zone->SetChild( -1 ); + return refTime; + } + else + { + const auto idx = childIdx; + childIdx++; + zone->SetChild( idx ); + return ReadTimeline( f, m_data.zoneChildren[idx], sz, refTime, childIdx ); + } +} + +void Worker::ReadTimeline( FileRead& f, GpuEvent* zone, int64_t& refTime, int64_t& refGpuTime, int32_t& childIdx ) +{ + uint64_t sz; + f.Read( sz ); + ReadTimelineHaveSize( f, zone, refTime, refGpuTime, childIdx, sz ); +} + +void Worker::ReadTimelineHaveSize( FileRead& f, GpuEvent* zone, int64_t& refTime, int64_t& refGpuTime, int32_t& childIdx, uint64_t sz ) +{ + if( sz == 0 ) + { + zone->SetChild( -1 ); + } + else + { + const auto idx = childIdx; + childIdx++; + zone->SetChild( idx ); + ReadTimeline( f, m_data.gpuChildren[idx], sz, refTime, refGpuTime, childIdx ); + } +} + +#ifndef TRACY_NO_STATISTICS +void Worker::ReconstructZoneStatistics( uint8_t* countMap, ZoneEvent& zone, uint16_t thread ) +{ + assert( zone.IsEndValid() ); + auto timeSpan = zone.End() - zone.Start(); + if( timeSpan > 0 ) + { + auto it = m_data.sourceLocationZones.find( zone.SrcLoc() ); + assert( it != m_data.sourceLocationZones.end() ); + + ZoneThreadData ztd; + ztd.SetZone( &zone ); + ztd.SetThread( thread ); + + auto& slz = it->second; + slz.zones.push_back( ztd ); + if( slz.min > timeSpan ) slz.min = timeSpan; + if( slz.max < timeSpan ) slz.max = timeSpan; + slz.total += timeSpan; + slz.sumSq += double( timeSpan ) * timeSpan; + + if( countMap[uint16_t(zone.SrcLoc())] == 0 ) + { + slz.nonReentrantCount++; + if( slz.nonReentrantMin > timeSpan ) slz.nonReentrantMin = timeSpan; + if( slz.nonReentrantMax < timeSpan ) slz.nonReentrantMax = timeSpan; + slz.nonReentrantTotal += timeSpan; + } + + if( zone.HasChildren() ) + { + auto& children = GetZoneChildren( zone.Child() ); + assert( children.is_magic() ); + auto& c = *(Vector*)( &children ); + for( auto& v : c ) + { + const auto childSpan = std::max( int64_t( 0 ), v.End() - v.Start() ); + timeSpan -= childSpan; + } + } + + if( slz.selfMin > timeSpan ) slz.selfMin = timeSpan; + if( slz.selfMax < timeSpan ) slz.selfMax = timeSpan; + slz.selfTotal += timeSpan; + + auto tit = slz.threadCnt.find( thread ); + if( tit == slz.threadCnt.end() ) + { + slz.threadCnt.emplace( thread, 1 ); + } + else + { + tit->second++; + } + } +} + +void Worker::ReconstructZoneStatistics( GpuEvent& zone, uint16_t thread ) +{ + assert( zone.GpuEnd() >= 0 ); + auto timeSpan = zone.GpuEnd() - zone.GpuStart(); + if( timeSpan > 0 ) + { + auto it = m_data.gpuSourceLocationZones.find( zone.SrcLoc() ); + if( it == m_data.gpuSourceLocationZones.end() ) + { + it = m_data.gpuSourceLocationZones.emplace( zone.SrcLoc(), GpuSourceLocationZones {} ).first; + } + GpuZoneThreadData ztd; + ztd.SetZone( &zone ); + ztd.SetThread( thread ); + auto& slz = it->second; + slz.zones.push_back( ztd ); + if( slz.min > timeSpan ) slz.min = timeSpan; + if( slz.max < timeSpan ) slz.max = timeSpan; + slz.total += timeSpan; + slz.sumSq += double( timeSpan ) * timeSpan; + } +} +#else +void Worker::CountZoneStatistics( ZoneEvent* zone ) +{ + auto cnt = GetSourceLocationZonesCnt( zone->SrcLoc() ); + (*cnt)++; +} + +void Worker::CountZoneStatistics( GpuEvent* zone ) +{ + auto cnt = GetGpuSourceLocationZonesCnt( zone->SrcLoc() ); + (*cnt)++; +} +#endif + +int64_t Worker::ReadTimeline( FileRead& f, Vector>& _vec, uint32_t size, int64_t refTime, int32_t& childIdx ) +{ + assert( size != 0 ); + const auto lp = s_loadProgress.subProgress.load( std::memory_order_relaxed ); + s_loadProgress.subProgress.store( lp + size, std::memory_order_relaxed ); + auto& vec = *(Vector*)( &_vec ); + vec.set_magic(); + vec.reserve_exact( size, m_slab ); + auto zone = vec.begin(); + auto end = vec.end() - 1; + + int16_t srcloc; + int64_t tstart, tend; + uint32_t childSz, extra; + f.Read4( srcloc, tstart, extra, childSz ); + + while( zone != end ) + { + refTime += tstart; + zone->SetStartSrcLoc( refTime, srcloc ); + zone->extra = extra; + refTime = ReadTimelineHaveSize( f, zone, refTime, childIdx, childSz ); + f.Read5( tend, srcloc, tstart, extra, childSz ); + refTime += tend; + zone->SetEnd( refTime ); +#ifdef TRACY_NO_STATISTICS + CountZoneStatistics( zone ); +#endif + zone++; + } + + refTime += tstart; + zone->SetStartSrcLoc( refTime, srcloc ); + zone->extra = extra; + refTime = ReadTimelineHaveSize( f, zone, refTime, childIdx, childSz ); + f.Read( tend ); + refTime += tend; + zone->SetEnd( refTime ); +#ifdef TRACY_NO_STATISTICS + CountZoneStatistics( zone ); +#endif + + return refTime; +} + +void Worker::ReadTimeline( FileRead& f, Vector>& _vec, uint64_t size, int64_t& refTime, int64_t& refGpuTime, int32_t& childIdx ) +{ + assert( size != 0 ); + const auto lp = s_loadProgress.subProgress.load( std::memory_order_relaxed ); + s_loadProgress.subProgress.store( lp + size, std::memory_order_relaxed ); + auto& vec = *(Vector*)( &_vec ); + vec.set_magic(); + vec.reserve_exact( size, m_slab ); + auto zone = vec.begin(); + auto end = vec.end(); + do + { + int64_t tcpu, tgpu; + int16_t srcloc; + uint16_t thread; + uint64_t childSz; + f.Read6( tcpu, tgpu, srcloc, zone->callstack, thread, childSz ); + zone->SetSrcLoc( srcloc ); + zone->SetThread( thread ); + refTime += tcpu; + refGpuTime += tgpu; + zone->SetCpuStart( refTime ); + zone->SetGpuStart( refGpuTime ); + + ReadTimelineHaveSize( f, zone, refTime, refGpuTime, childIdx, childSz ); + + f.Read2( tcpu, tgpu ); + refTime += tcpu; + refGpuTime += tgpu; + zone->SetCpuEnd( refTime ); + zone->SetGpuEnd( refGpuTime ); + } + while( ++zone != end ); +} + +void Worker::Disconnect() +{ + //Query( ServerQueryDisconnect, 0 ); + Shutdown(); + m_disconnect = true; +} + +static void WriteHwSampleVec( FileWrite& f, SortedVector& vec ) +{ + uint64_t sz = vec.size(); + f.Write( &sz, sizeof( sz ) ); + if( sz != 0 ) + { + if( !vec.is_sorted() ) vec.sort(); + int64_t refTime = 0; + for( auto& v : vec ) + { + WriteTimeOffset( f, refTime, v.Val() ); + } + } +} + +void Worker::Write( FileWrite& f, bool fiDict ) +{ + DoPostponedWorkAll(); + + f.Write( FileHeader, sizeof( FileHeader ) ); + + f.Write( &m_delay, sizeof( m_delay ) ); + f.Write( &m_resolution, sizeof( m_resolution ) ); + f.Write( &m_timerMul, sizeof( m_timerMul ) ); + f.Write( &m_data.lastTime, sizeof( m_data.lastTime ) ); + f.Write( &m_data.frameOffset, sizeof( m_data.frameOffset ) ); + f.Write( &m_pid, sizeof( m_pid ) ); + f.Write( &m_samplingPeriod, sizeof( m_samplingPeriod ) ); + f.Write( &m_data.cpuArch, sizeof( m_data.cpuArch ) ); + f.Write( &m_data.cpuId, sizeof( m_data.cpuId ) ); + f.Write( m_data.cpuManufacturer, 12 ); + + uint8_t flag = m_onDemand; + f.Write( &flag, sizeof( flag ) ); + + uint64_t sz = m_captureName.size(); + f.Write( &sz, sizeof( sz ) ); + f.Write( m_captureName.c_str(), sz ); + + sz = m_captureProgram.size(); + f.Write( &sz, sizeof( sz ) ); + f.Write( m_captureProgram.c_str(), sz ); + + f.Write( &m_captureTime, sizeof( m_captureTime ) ); + f.Write( &m_executableTime, sizeof( m_executableTime ) ); + + sz = m_hostInfo.size(); + f.Write( &sz, sizeof( sz ) ); + f.Write( m_hostInfo.c_str(), sz ); + + sz = m_data.cpuTopology.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& package : m_data.cpuTopology ) + { + sz = package.second.size(); + f.Write( &package.first, sizeof( package.first ) ); + f.Write( &sz, sizeof( sz ) ); + for( auto& core : package.second ) + { + sz = core.second.size(); + f.Write( &core.first, sizeof( core.first ) ); + f.Write( &sz, sizeof( sz ) ); + for( auto& thread : core.second ) + { + f.Write( &thread, sizeof( thread ) ); + } + } + } + + f.Write( &m_data.crashEvent, sizeof( m_data.crashEvent ) ); + + sz = m_data.frames.Data().size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& fd : m_data.frames.Data() ) + { + int64_t refTime = 0; + f.Write( &fd->name, sizeof( fd->name ) ); + f.Write( &fd->continuous, sizeof( fd->continuous ) ); + sz = fd->frames.size(); + f.Write( &sz, sizeof( sz ) ); + if( fd->continuous ) + { + for( auto& fe : fd->frames ) + { + WriteTimeOffset( f, refTime, fe.start ); + f.Write( &fe.frameImage, sizeof( fe.frameImage ) ); + } + } + else + { + for( auto& fe : fd->frames ) + { + WriteTimeOffset( f, refTime, fe.start ); + WriteTimeOffset( f, refTime, fe.end ); + f.Write( &fe.frameImage, sizeof( fe.frameImage ) ); + } + } + } + + sz = m_data.stringData.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.stringData ) + { + uint64_t ptr = (uint64_t)v; + f.Write( &ptr, sizeof( ptr ) ); + sz = strlen( v ); + f.Write( &sz, sizeof( sz ) ); + f.Write( v, sz ); + } + + sz = m_data.strings.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.strings ) + { + f.Write( &v.first, sizeof( v.first ) ); + uint64_t ptr = (uint64_t)v.second; + f.Write( &ptr, sizeof( ptr ) ); + } + + sz = m_data.threadNames.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.threadNames ) + { + f.Write( &v.first, sizeof( v.first ) ); + uint64_t ptr = (uint64_t)v.second; + f.Write( &ptr, sizeof( ptr ) ); + } + + sz = m_data.externalNames.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.externalNames ) + { + f.Write( &v.first, sizeof( v.first ) ); + uint64_t ptr = (uint64_t)v.second.first; + f.Write( &ptr, sizeof( ptr ) ); + ptr = (uint64_t)v.second.second; + f.Write( &ptr, sizeof( ptr ) ); + } + + m_data.localThreadCompress.Save( f ); + m_data.externalThreadCompress.Save( f ); + + sz = m_data.sourceLocation.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.sourceLocation ) + { + f.Write( &v.first, sizeof( v.first ) ); + f.Write( &v.second, sizeof( SourceLocationBase ) ); + } + + sz = m_data.sourceLocationExpand.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.sourceLocationExpand ) + { + f.Write( &v, sizeof( v ) ); + } + + sz = m_data.sourceLocationPayload.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.sourceLocationPayload ) + { + f.Write( v, sizeof( SourceLocationBase ) ); + } + +#ifndef TRACY_NO_STATISTICS + sz = m_data.sourceLocationZones.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.sourceLocationZones ) + { + int16_t id = v.first; + uint64_t cnt = v.second.zones.size(); + f.Write( &id, sizeof( id ) ); + f.Write( &cnt, sizeof( cnt ) ); + } + + sz = m_data.gpuSourceLocationZones.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.gpuSourceLocationZones ) + { + int16_t id = v.first; + uint64_t cnt = v.second.zones.size(); + f.Write( &id, sizeof( id ) ); + f.Write( &cnt, sizeof( cnt ) ); + } +#else + sz = m_data.sourceLocationZonesCnt.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.sourceLocationZonesCnt ) + { + int16_t id = v.first; + uint64_t cnt = v.second; + f.Write( &id, sizeof( id ) ); + f.Write( &cnt, sizeof( cnt ) ); + } + + sz = m_data.gpuSourceLocationZonesCnt.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.gpuSourceLocationZonesCnt ) + { + int16_t id = v.first; + uint64_t cnt = v.second; + f.Write( &id, sizeof( id ) ); + f.Write( &cnt, sizeof( cnt ) ); + } +#endif + + sz = m_data.lockMap.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.lockMap ) + { + f.Write( &v.first, sizeof( v.first ) ); + f.Write( &v.second->customName, sizeof( v.second->customName ) ); + f.Write( &v.second->srcloc, sizeof( v.second->srcloc ) ); + f.Write( &v.second->type, sizeof( v.second->type ) ); + f.Write( &v.second->valid, sizeof( v.second->valid ) ); + f.Write( &v.second->timeAnnounce, sizeof( v.second->timeAnnounce ) ); + f.Write( &v.second->timeTerminate, sizeof( v.second->timeTerminate ) ); + sz = v.second->threadList.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& t : v.second->threadList ) + { + f.Write( &t, sizeof( t ) ); + } + int64_t refTime = v.second->timeAnnounce; + sz = v.second->timeline.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& lev : v.second->timeline ) + { + WriteTimeOffset( f, refTime, lev.ptr->Time() ); + const int16_t srcloc = lev.ptr->SrcLoc(); + f.Write( &srcloc, sizeof( srcloc ) ); + f.Write( &lev.ptr->thread, sizeof( lev.ptr->thread ) ); + f.Write( &lev.ptr->type, sizeof( lev.ptr->type ) ); + } + } + + { + int64_t refTime = 0; + sz = m_data.messages.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.messages ) + { + const auto ptr = (uint64_t)(MessageData*)v; + f.Write( &ptr, sizeof( ptr ) ); + WriteTimeOffset( f, refTime, v->time ); + f.Write( &v->ref, sizeof( v->ref ) ); + f.Write( &v->color, sizeof( v->color ) ); + f.Write( &v->callstack, sizeof( v->callstack ) ); + } + } + + sz = m_data.zoneExtra.size(); + f.Write( &sz, sizeof( sz ) ); + f.Write( m_data.zoneExtra.data(), sz * sizeof( ZoneExtra ) ); + + sz = 0; + for( auto& v : m_data.threads ) sz += v->count; + f.Write( &sz, sizeof( sz ) ); + sz = m_data.zoneChildren.size(); + f.Write( &sz, sizeof( sz ) ); + sz = m_data.threads.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& thread : m_data.threads ) + { + int64_t refTime = 0; + f.Write( &thread->id, sizeof( thread->id ) ); + f.Write( &thread->count, sizeof( thread->count ) ); + f.Write( &thread->kernelSampleCnt, sizeof( thread->kernelSampleCnt ) ); + f.Write( &thread->isFiber, sizeof( thread->isFiber ) ); + f.Write( &thread->groupHint, sizeof( thread->groupHint ) ); + WriteTimeline( f, thread->timeline, refTime ); + sz = thread->messages.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : thread->messages ) + { + auto ptr = uint64_t( (MessageData*)v ); + f.Write( &ptr, sizeof( ptr ) ); + } + sz = thread->ctxSwitchSamples.size(); + f.Write( &sz, sizeof( sz ) ); + refTime = 0; + for( auto& v : thread->ctxSwitchSamples ) + { + WriteTimeOffset( f, refTime, v.time.Val() ); + f.Write( &v.callstack, sizeof( v.callstack ) ); + } + if( m_inconsistentSamples ) + { +#ifdef NO_PARALLEL_SORT + pdqsort_branchless( thread->samples.begin(), thread->samples.end(), [] ( const auto& lhs, const auto& rhs ) { return lhs.time.Val() < rhs.time.Val(); } ); +#else + std::sort( std::execution::par_unseq, thread->samples.begin(), thread->samples.end(), [] ( const auto& lhs, const auto& rhs ) { return lhs.time.Val() < rhs.time.Val(); } ); +#endif + } + sz = thread->samples.size(); + f.Write( &sz, sizeof( sz ) ); + refTime = 0; + for( auto& v : thread->samples ) + { + WriteTimeOffset( f, refTime, v.time.Val() ); + f.Write( &v.callstack, sizeof( v.callstack ) ); + } + } + + sz = 0; + for( auto& v : m_data.gpuData ) sz += v->count; + f.Write( &sz, sizeof( sz ) ); + sz = m_data.gpuChildren.size(); + f.Write( &sz, sizeof( sz ) ); + sz = m_data.gpuData.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& ctx : m_data.gpuData ) + { + f.Write( &ctx->thread, sizeof( ctx->thread ) ); + uint8_t calibration = ctx->hasCalibration; + f.Write( &calibration, sizeof( calibration ) ); + f.Write( &ctx->count, sizeof( ctx->count ) ); + f.Write( &ctx->period, sizeof( ctx->period ) ); + f.Write( &ctx->type, sizeof( ctx->type ) ); + f.Write( &ctx->name, sizeof( ctx->name ) ); + f.Write( &ctx->overflow, sizeof( ctx->overflow ) ); + sz = ctx->threadData.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& td : ctx->threadData ) + { + int64_t refTime = 0; + int64_t refGpuTime = 0; + uint64_t tid = td.first; + f.Write( &tid, sizeof( tid ) ); + WriteTimeline( f, td.second.timeline, refTime, refGpuTime ); + } + } + + sz = m_data.plots.Data().size(); + for( auto& plot : m_data.plots.Data() ) { if( plot->type == PlotType::Memory ) sz--; } + f.Write( &sz, sizeof( sz ) ); + for( auto& plot : m_data.plots.Data() ) + { + if( plot->type == PlotType::Memory ) continue; + f.Write( &plot->type, sizeof( plot->type ) ); + f.Write( &plot->format, sizeof( plot->format ) ); + f.Write( &plot->showSteps, sizeof( plot->showSteps ) ); + f.Write( &plot->fill, sizeof( plot->fill ) ); + f.Write( &plot->color, sizeof( plot->color ) ); + f.Write( &plot->name, sizeof( plot->name ) ); + f.Write( &plot->min, sizeof( plot->min ) ); + f.Write( &plot->max, sizeof( plot->max ) ); + f.Write( &plot->sum, sizeof( plot->sum ) ); + int64_t refTime = 0; + sz = plot->data.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : plot->data ) + { + WriteTimeOffset( f, refTime, v.time.Val() ); + f.Write( &v.val, sizeof( v.val ) ); + } + } + + sz = m_data.memNameMap.size(); + f.Write( &sz, sizeof( sz ) ); + sz = 0; + for( auto& memory : m_data.memNameMap ) + { + sz += memory.second->data.size(); + } + f.Write( &sz, sizeof( sz ) ); + for( auto& memory : m_data.memNameMap ) + { + uint64_t name = memory.first; + f.Write( &name, sizeof( name ) ); + + auto& memdata = *memory.second; + int64_t refTime = 0; + sz = memdata.data.size(); + f.Write( &sz, sizeof( sz ) ); + sz = memdata.active.size(); + f.Write( &sz, sizeof( sz ) ); + sz = memdata.frees.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& mem : memdata.data ) + { + const auto ptr = mem.Ptr(); + const auto size = mem.Size(); + const Int24 csAlloc = mem.CsAlloc(); + f.Write( &ptr, sizeof( ptr ) ); + f.Write( &size, sizeof( size ) ); + f.Write( &csAlloc, sizeof( csAlloc ) ); + f.Write( &mem.csFree, sizeof( mem.csFree ) ); + + int64_t timeAlloc = mem.TimeAlloc(); + uint16_t threadAlloc = mem.ThreadAlloc(); + int64_t timeFree = mem.TimeFree(); + uint16_t threadFree = mem.ThreadFree(); + WriteTimeOffset( f, refTime, timeAlloc ); + int64_t freeOffset = timeFree < 0 ? timeFree : timeFree - timeAlloc; + f.Write( &freeOffset, sizeof( freeOffset ) ); + f.Write( &threadAlloc, sizeof( threadAlloc ) ); + f.Write( &threadFree, sizeof( threadFree ) ); + } + f.Write( &memdata.high, sizeof( memdata.high ) ); + f.Write( &memdata.low, sizeof( memdata.low ) ); + f.Write( &memdata.usage, sizeof( memdata.usage ) ); + f.Write( &memdata.name, sizeof( memdata.name ) ); + } + + sz = m_data.callstackPayload.size() - 1; + f.Write( &sz, sizeof( sz ) ); + for( size_t i=1; i<=sz; i++ ) + { + auto& cs = m_data.callstackPayload[i]; + uint16_t csz = cs->size(); + f.Write( &csz, sizeof( csz ) ); + f.Write( cs->data(), sizeof( CallstackFrameId ) * csz ); + } + + sz = m_data.callstackFrameMap.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& frame : m_data.callstackFrameMap ) + { + f.Write( &frame.first, sizeof( CallstackFrameId ) ); + f.Write( &frame.second->size, sizeof( frame.second->size ) ); + f.Write( &frame.second->imageName, sizeof( frame.second->imageName ) ); + f.Write( frame.second->data, sizeof( CallstackFrame ) * frame.second->size ); + } + + sz = m_data.appInfo.size(); + f.Write( &sz, sizeof( sz ) ); + if( sz != 0 ) f.Write( m_data.appInfo.data(), sizeof( m_data.appInfo[0] ) * sz ); + + { + sz = m_data.frameImage.size(); + if( fiDict ) + { + enum : uint32_t { DictSize = 4*1024*1024 }; + enum : uint32_t { SamplesLimit = 1U << 31 }; + uint32_t sNum = 0; + uint32_t sSize = 0; + for( auto& fi : m_data.frameImage ) + { + const auto fisz = fi->w * fi->h / 2; + if( sSize + fisz > SamplesLimit ) break; + sSize += fisz; + sNum++; + } + + uint32_t offset = 0; + auto sdata = new char[sSize]; + auto ssize = new size_t[sSize]; + for( uint32_t i=0; iw * fi->h / 2; + const auto image = m_texcomp.Unpack( *fi ); + memcpy( sdata+offset, image, fisz ); + ssize[i] = fisz; + offset += fisz; + } + assert( offset == sSize ); + + ZDICT_fastCover_params_t params = {}; + params.d = 6; + params.k = 50; + params.f = 30; + params.nbThreads = std::thread::hardware_concurrency(); + params.zParams.compressionLevel = 3; + + auto dict = new char[DictSize]; + const auto dictret = ZDICT_optimizeTrainFromBuffer_fastCover( dict, DictSize, sdata, ssize, sNum, ¶ms ); + if( dictret <= DictSize ) + { + const auto finalDictSize = uint32_t( dictret ); + auto zdict = ZSTD_createCDict( dict, finalDictSize, 3 ); + + f.Write( &finalDictSize, sizeof( finalDictSize ) ); + f.Write( dict, finalDictSize ); + + ZSTD_freeCDict( zdict ); + } + else + { + uint32_t zero = 0; + f.Write( &zero, sizeof( zero ) ); + } + + delete[] dict; + delete[] ssize; + delete[] sdata; + } + else + { + uint32_t zero = 0; + f.Write( &zero, sizeof( zero ) ); + } + f.Write( &sz, sizeof( sz ) ); + for( auto& fi : m_data.frameImage ) + { + f.Write( &fi->w, sizeof( fi->w ) ); + f.Write( &fi->h, sizeof( fi->h ) ); + f.Write( &fi->flip, sizeof( fi->flip ) ); + const auto image = m_texcomp.Unpack( *fi ); + f.Write( image, fi->w * fi->h / 2 ); + } + } + + // Only save context switches relevant to active threads. + std::vector::const_iterator> ctxValid; + ctxValid.reserve( m_data.ctxSwitch.size() ); + for( auto it = m_data.ctxSwitch.begin(); it != m_data.ctxSwitch.end(); ++it ) + { + auto td = RetrieveThread( it->first ); + if( td && ( td->count > 0 || !td->samples.empty() ) ) + { + ctxValid.emplace_back( it ); + } + } + sz = ctxValid.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& ctx : ctxValid ) + { + f.Write( &ctx->first, sizeof( ctx->first ) ); + sz = ctx->second->v.size(); + f.Write( &sz, sizeof( sz ) ); + int64_t refTime = 0; + for( auto& cs : ctx->second->v ) + { + WriteTimeOffset( f, refTime, cs.WakeupVal() ); + WriteTimeOffset( f, refTime, cs.Start() ); + WriteTimeOffset( f, refTime, cs.End() ); + uint8_t cpu = cs.Cpu(); + int8_t reason = cs.Reason(); + int8_t state = cs.State(); + uint64_t thread = DecompressThread( cs.Thread() ); + f.Write( &cpu, sizeof( cpu ) ); + f.Write( &reason, sizeof( reason ) ); + f.Write( &state, sizeof( state ) ); + f.Write( &thread, sizeof( thread ) ); + } + } + + sz = GetContextSwitchPerCpuCount(); + f.Write( &sz, sizeof( sz ) ); + for( int i=0; i<256; i++ ) + { + sz = m_data.cpuData[i].cs.size(); + f.Write( &sz, sizeof( sz ) ); + int64_t refTime = 0; + for( auto& cx : m_data.cpuData[i].cs ) + { + WriteTimeOffset( f, refTime, cx.Start() ); + WriteTimeOffset( f, refTime, cx.End() ); + uint16_t thread = cx.Thread(); + f.Write( &thread, sizeof( thread ) ); + } + } + + sz = m_data.tidToPid.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.tidToPid ) + { + f.Write( &v.first, sizeof( v.first ) ); + f.Write( &v.second, sizeof( v.second ) ); + } + + sz = m_data.cpuThreadData.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.cpuThreadData ) + { + f.Write( &v.first, sizeof( v.first ) ); + f.Write( &v.second, sizeof( v.second ) ); + } + + sz = m_data.symbolLoc.size(); + f.Write( &sz, sizeof( sz ) ); + sz = m_data.symbolLocInline.size(); + f.Write( &sz, sizeof( sz ) ); + sz = m_data.symbolMap.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.symbolMap ) + { + f.Write( &v.first, sizeof( v.first ) ); + f.Write( &v.second, sizeof( v.second ) ); + } + + sz = m_data.symbolCode.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.symbolCode ) + { + f.Write( &v.first, sizeof( v.first ) ); + f.Write( &v.second.len, sizeof( v.second.len ) ); + f.Write( v.second.data, v.second.len ); + } + + sz = m_data.codeSymbolMap.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.codeSymbolMap ) + { + f.Write( &v.first, sizeof( v.first ) ); + f.Write( &v.second, sizeof( v.second ) ); + } + + sz = m_data.hwSamples.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.hwSamples ) + { + f.Write( &v.first, sizeof( v.first ) ); + WriteHwSampleVec( f, v.second.cycles ); + WriteHwSampleVec( f, v.second.retired ); + WriteHwSampleVec( f, v.second.cacheRef ); + WriteHwSampleVec( f, v.second.cacheMiss ); + WriteHwSampleVec( f, v.second.branchRetired ); + WriteHwSampleVec( f, v.second.branchMiss ); + } + + sz = m_data.sourceFileCache.size(); + f.Write( &sz, sizeof( sz ) ); + for( auto& v : m_data.sourceFileCache ) + { + uint32_t s32 = strlen( v.first ); + f.Write( &s32, sizeof( s32 ) ); + f.Write( v.first, s32 ); + f.Write( &v.second.len, sizeof( v.second.len ) ); + f.Write( v.second.data, v.second.len ); + } +} + +void Worker::WriteTimeline( FileWrite& f, const Vector>& vec, int64_t& refTime ) +{ + uint32_t sz = uint32_t( vec.size() ); + f.Write( &sz, sizeof( sz ) ); + if( vec.is_magic() ) + { + WriteTimelineImpl>( f, *(Vector*)( &vec ), refTime ); + } + else + { + WriteTimelineImpl>( f, vec, refTime ); + } +} + +template +void Worker::WriteTimelineImpl( FileWrite& f, const V& vec, int64_t& refTime ) +{ + Adapter a; + for( auto& val : vec ) + { + auto& v = a(val); + int16_t srcloc = v.SrcLoc(); + f.Write( &srcloc, sizeof( srcloc ) ); + int64_t start = v.Start(); + WriteTimeOffset( f, refTime, start ); + f.Write( &v.extra, sizeof( v.extra ) ); + if( !v.HasChildren() ) + { + const uint32_t sz = 0; + f.Write( &sz, sizeof( sz ) ); + } + else + { + WriteTimeline( f, GetZoneChildren( v.Child() ), refTime ); + } + WriteTimeOffset( f, refTime, v.End() ); + } +} + +void Worker::WriteTimeline( FileWrite& f, const Vector>& vec, int64_t& refTime, int64_t& refGpuTime ) +{ + uint64_t sz = vec.size(); + f.Write( &sz, sizeof( sz ) ); + if( vec.is_magic() ) + { + WriteTimelineImpl>( f, *(Vector*)( &vec ), refTime, refGpuTime ); + } + else + { + WriteTimelineImpl>( f, vec, refTime, refGpuTime ); + } +} + +template +void Worker::WriteTimelineImpl( FileWrite& f, const V& vec, int64_t& refTime, int64_t& refGpuTime ) +{ + Adapter a; + for( auto& val : vec ) + { + auto& v = a(val); + WriteTimeOffset( f, refTime, v.CpuStart() ); + WriteTimeOffset( f, refGpuTime, v.GpuStart() ); + const int16_t srcloc = v.SrcLoc(); + f.Write( &srcloc, sizeof( srcloc ) ); + f.Write( &v.callstack, sizeof( v.callstack ) ); + const uint16_t thread = v.Thread(); + f.Write( &thread, sizeof( thread ) ); + + if( v.Child() < 0 ) + { + const uint64_t sz = 0; + f.Write( &sz, sizeof( sz ) ); + } + else + { + WriteTimeline( f, GetGpuChildren( v.Child() ), refTime, refGpuTime ); + } + + WriteTimeOffset( f, refTime, v.CpuEnd() ); + WriteTimeOffset( f, refGpuTime, v.GpuEnd() ); + } +} + +static const char* s_failureReasons[] = { + "", + "Invalid order of zone begin and end events.", + "Zone is ended twice.", + "Zone text transfer destination doesn't match active zone.", + "Zone value transfer destination doesn't match active zone.", + "Zone color transfer destination doesn't match active zone.", + "Zone name transfer destination doesn't match active zone.", + "Memory free event without a matching allocation.", + "Memory allocation event was reported for an address that is already tracked and not freed.", + "Discontinuous frame begin/end mismatch.", + "Frame image offset is invalid.", + "Multiple frame images were sent for a single frame.", + "Fiber execution stopped on a thread which is not executing a fiber.", + "Too many source locations. You cannot have more than 32K static or dynamic source locations.", +}; + +static_assert( sizeof( s_failureReasons ) / sizeof( *s_failureReasons ) == (int)Worker::Failure::NUM_FAILURES, "Missing failure reason description." ); + +const char* Worker::GetFailureString( Worker::Failure failure ) +{ + return s_failureReasons[(int)failure]; +} + +void Worker::SetParameter( size_t paramIdx, int32_t val ) +{ + assert( paramIdx < m_params.size() ); + m_params[paramIdx].val = val; + const auto idx = uint64_t( m_params[paramIdx].idx ); + const auto v = uint64_t( uint32_t( val ) ); + Query( ServerQueryParameter, ( idx << 32 ) | v ); +} + +const Worker::CpuThreadTopology* Worker::GetThreadTopology( uint32_t cpuThread ) const +{ + auto it = m_data.cpuTopologyMap.find( cpuThread ); + if( it == m_data.cpuTopologyMap.end() ) return nullptr; + return &it->second; +} + +ZoneExtra& Worker::AllocZoneExtra( ZoneEvent& ev ) +{ + assert( ev.extra == 0 ); + ev.extra = uint32_t( m_data.zoneExtra.size() ); + auto& extra = m_data.zoneExtra.push_next(); + memset( (char*)&extra, 0, sizeof( extra ) ); + return extra; +} + +ZoneExtra& Worker::RequestZoneExtra( ZoneEvent& ev ) +{ + if( !HasZoneExtra( ev ) ) + { + return AllocZoneExtra( ev ); + } + else + { + return GetZoneExtraMutable( ev ); + } +} + +void Worker::CacheSource( const StringRef& str, const StringIdx& image ) +{ + assert( str.active ); + assert( m_checkedFileStrings.find( str ) == m_checkedFileStrings.end() ); + m_checkedFileStrings.emplace( str ); + auto file = GetString( str ); + // Possible duplication of pointer and index strings + if( m_data.sourceFileCache.find( file ) != m_data.sourceFileCache.end() ) return; + const auto execTime = GetExecutableTime(); + if( SourceFileValid( file, execTime != 0 ? execTime : GetCaptureTime() ) ) + { + CacheSourceFromFile( file ); + } + else if( execTime != 0 ) + { + QuerySourceFile( file, image.Active() ? GetString( image ) : nullptr ); + } +} + +void Worker::CacheSourceFromFile( const char* fn ) +{ + FILE* f = fopen( fn, "rb" ); + fseek( f, 0, SEEK_END ); + const auto sz = ftell( f ); + fseek( f, 0, SEEK_SET ); + auto src = (char*)m_slab.AllocBig( sz ); + fread( src, 1, sz, f ); + fclose( f ); + m_data.sourceFileCache.emplace( fn, MemoryBlock{ src, uint32_t( sz ) } ); +} + +uint64_t Worker::GetSourceFileCacheSize() const +{ + uint64_t cnt = 0; + for( auto& v : m_data.sourceFileCache ) + { + cnt += v.second.len; + } + return cnt; +} + +Worker::MemoryBlock Worker::GetSourceFileFromCache( const char* file ) const +{ + auto it = m_data.sourceFileCache.find( file ); + if( it == m_data.sourceFileCache.end() ) return MemoryBlock {}; + return it->second; +} + +HwSampleData* Worker::GetHwSampleData( uint64_t addr ) +{ + auto it = m_data.hwSamples.find( addr ); + if( it == m_data.hwSamples.end() ) return nullptr; + return &it->second; +} + +uint64_t Worker::GetHwSampleCount() const +{ + uint64_t cnt = 0; + for( auto& v : m_data.hwSamples ) + { + cnt += v.second.cycles.size(); + cnt += v.second.retired.size(); + cnt += v.second.cacheRef.size(); + cnt += v.second.cacheMiss.size(); + cnt += v.second.branchRetired.size(); + cnt += v.second.branchMiss.size(); + } + return cnt; +} + +void Worker::CacheSourceFiles() +{ + const auto execTime = GetExecutableTime(); + + for( auto& sl : m_data.sourceLocationPayload ) + { + const char* file = GetString( sl->file ); + if( m_data.sourceFileCache.find( file ) == m_data.sourceFileCache.end() ) + { + if( SourceFileValid( file, execTime != 0 ? execTime : GetCaptureTime() ) ) CacheSourceFromFile( file ); + } + } + + for( auto& sl : m_data.sourceLocation ) + { + const char* file = GetString( sl.second.file ); + if( m_data.sourceFileCache.find( file ) == m_data.sourceFileCache.end() ) + { + if( SourceFileValid( file, execTime != 0 ? execTime : GetCaptureTime() ) ) CacheSourceFromFile( file ); + } + } + + for( auto& sym : m_data.symbolMap ) + { + const char* file = GetString( sym.second.file ); + if( m_data.sourceFileCache.find( file ) == m_data.sourceFileCache.end() ) + { + if( SourceFileValid( file, execTime != 0 ? execTime : GetCaptureTime() ) ) CacheSourceFromFile( file ); + } + } +} + +} diff --git a/third_party/tracy/server/TracyWorker.hpp b/third_party/tracy/server/TracyWorker.hpp new file mode 100644 index 0000000000000000000000000000000000000000..0a4ca8292c24a6ca0f60bc2f5909b2a6e56b6ace --- /dev/null +++ b/third_party/tracy/server/TracyWorker.hpp @@ -0,0 +1,1109 @@ +#ifndef __TRACYWORKER_HPP__ +#define __TRACYWORKER_HPP__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../public/common/TracyForceInline.hpp" +#include "../public/common/TracyQueue.hpp" +#include "../public/common/TracyProtocol.hpp" +#include "../public/common/TracySocket.hpp" +#include "tracy_robin_hood.h" +#include "TracyEvent.hpp" +#include "TracyShortPtr.hpp" +#include "TracySlab.hpp" +#include "TracyStringDiscovery.hpp" +#include "TracyTextureCompression.hpp" +#include "TracyThreadCompress.hpp" +#include "TracyVarArray.hpp" + + +namespace tracy +{ + +class FileRead; +class FileWrite; + +namespace EventType +{ + enum Type : uint32_t + { + Locks = 1 << 0, + Messages = 1 << 1, + Plots = 1 << 2, + Memory = 1 << 3, + FrameImages = 1 << 4, + ContextSwitches = 1 << 5, + Samples = 1 << 6, + SymbolCode = 1 << 7, + SourceCache = 1 << 8, + + None = 0, + All = std::numeric_limits::max() + }; +} + +struct UnsupportedVersion : public std::exception +{ + UnsupportedVersion( int version ) : version( version ) {} + int version; +}; + +struct LegacyVersion : public std::exception +{ + LegacyVersion( int version ) : version ( version ) {} + int version; +}; + +struct LoadFailure : public std::exception +{ + LoadFailure( const char* msg ) : msg( msg ) {} + std::string msg; +}; + +struct LoadProgress +{ + enum Stage + { + Initialization, + Locks, + Messages, + Zones, + GpuZones, + Plots, + Memory, + CallStacks, + FrameImages, + ContextSwitches, + ContextSwitchesPerCpu + }; + + LoadProgress() : total( 0 ), progress( 0 ), subTotal( 0 ), subProgress( 0 ) {} + + std::atomic total; + std::atomic progress; + std::atomic subTotal; + std::atomic subProgress; +}; + +class Worker +{ +public: + struct ImportEventTimeline + { + uint64_t tid; + uint64_t timestamp; + std::string name; + std::string text; + bool isEnd; + std::string locFile; + uint32_t locLine; + }; + + struct ImportEventMessages + { + uint64_t tid; + uint64_t timestamp; + std::string message; + }; + + struct ImportEventPlots + { + std::string name; + PlotValueFormatting format; + std::vector> data; + }; + + struct ZoneThreadData + { + tracy_force_inline ZoneEvent* Zone() const { return (ZoneEvent*)( _zone_thread >> 16 ); } + tracy_force_inline void SetZone( ZoneEvent* zone ) { assert( ( uint64_t( zone ) & 0xFFFF000000000000 ) == 0 ); memcpy( ((char*)&_zone_thread)+2, &zone, 4 ); memcpy( ((char*)&_zone_thread)+6, ((char*)&zone)+4, 2 ); } + tracy_force_inline uint16_t Thread() const { return uint16_t( _zone_thread & 0xFFFF ); } + tracy_force_inline void SetThread( uint16_t thread ) { memcpy( &_zone_thread, &thread, 2 ); } + + uint64_t _zone_thread; + }; + enum { ZoneThreadDataSize = sizeof( ZoneThreadData ) }; + + struct GpuZoneThreadData + { + tracy_force_inline GpuEvent* Zone() const { return (GpuEvent*)( _zone_thread >> 16 ); } + tracy_force_inline void SetZone( GpuEvent* zone ) { assert( ( uint64_t( zone ) & 0xFFFF000000000000 ) == 0 ); memcpy( ((char*)&_zone_thread)+2, &zone, 4 ); memcpy( ((char*)&_zone_thread)+6, ((char*)&zone)+4, 2 ); } + tracy_force_inline uint16_t Thread() const { return uint16_t( _zone_thread & 0xFFFF ); } + tracy_force_inline void SetThread( uint16_t thread ) { memcpy( &_zone_thread, &thread, 2 ); } + + uint64_t _zone_thread; + }; + enum { GpuZoneThreadDataSize = sizeof( GpuZoneThreadData ) }; + + struct CpuThreadTopology + { + uint32_t package; + uint32_t core; + }; + + struct MemoryBlock + { + const char* data; + uint32_t len; + }; + + struct InlineStackData + { + uint64_t symAddr; + CallstackFrameId frame; + uint8_t inlineFrame; + }; + + struct PowerData + { + int64_t lastTime; + PlotData* plot; + }; + +#pragma pack( push, 1 ) + struct GhostKey + { + CallstackFrameId frame; + uint8_t inlineFrame; + }; +#pragma pack( pop ) + + struct GhostKeyHasher + { + size_t operator()( const GhostKey& key ) const + { + return charutil::hash( (const char*)&key, sizeof( GhostKey ) ); + } + }; + + struct GhostKeyComparator + { + bool operator()( const GhostKey& lhs, const GhostKey& rhs ) const + { + return memcmp( &lhs, &rhs, sizeof( GhostKey ) ) == 0; + } + }; + +private: + struct SourceLocationZones + { + struct ZtdSort { bool operator()( const ZoneThreadData& lhs, const ZoneThreadData& rhs ) { return lhs.Zone()->Start() < rhs.Zone()->Start(); } }; + + SortedVector zones; + int64_t min = std::numeric_limits::max(); + int64_t max = std::numeric_limits::min(); + int64_t total = 0; + double sumSq = 0; + int64_t selfMin = std::numeric_limits::max(); + int64_t selfMax = std::numeric_limits::min(); + int64_t selfTotal = 0; + size_t nonReentrantCount = 0; + int64_t nonReentrantMin = std::numeric_limits::max(); + int64_t nonReentrantMax = std::numeric_limits::min(); + int64_t nonReentrantTotal = 0; + unordered_flat_map threadCnt; + }; + + struct GpuSourceLocationZones + { + struct GpuZtdSort { bool operator()( const GpuZoneThreadData& lhs, const GpuZoneThreadData& rhs ) { return lhs.Zone()->GpuStart() < rhs.Zone()->GpuStart(); } }; + + SortedVector zones; + int64_t min = std::numeric_limits::max(); + int64_t max = std::numeric_limits::min(); + int64_t total = 0; + double sumSq = 0; + }; + + struct CallstackFrameIdHash + { + size_t operator()( const CallstackFrameId& id ) const { return id.data; } + }; + + struct CallstackFrameIdCompare + { + bool operator()( const CallstackFrameId& lhs, const CallstackFrameId& rhs ) const { return lhs.data == rhs.data; } + }; + + struct RevFrameHash + { + size_t operator()( const CallstackFrameData* data ) const + { + size_t hash = data->size; + for( uint8_t i=0; isize; i++ ) + { + const auto& v = data->data[i]; + hash = ( ( hash << 5 ) + hash ) ^ size_t( v.line ); + hash = ( ( hash << 5 ) + hash ) ^ size_t( v.file.Idx() ); + hash = ( ( hash << 5 ) + hash ) ^ size_t( v.name.Idx() ); + } + return hash; + } + }; + + struct RevFrameComp + { + bool operator()( const CallstackFrameData* lhs, const CallstackFrameData* rhs ) const + { + if( lhs->size != rhs->size ) return false; + for( uint8_t i=0; isize; i++ ) + { + if( memcmp( lhs->data + i, rhs->data + i, sizeof( CallstackFrameBasic ) ) != 0 ) return false; + } + return true; + } + }; + + struct SymbolPending + { + StringIdx name; + StringIdx imageName; + StringIdx file; + uint32_t line; + uint32_t size; + bool isInline; + }; + + struct DataBlock + { + std::mutex lock; + StringDiscovery frames; + FrameData* framesBase; + Vector gpuData; + Vector> messages; + StringDiscovery plots; + Vector threads; + Vector zoneExtra; + MemData* memory; + unordered_flat_map memNameMap; + uint64_t zonesCnt = 0; + uint64_t gpuCnt = 0; + uint64_t samplesCnt = 0; + uint64_t ghostCnt = 0; + int64_t baseTime = 0; + int64_t lastTime = 0; + uint64_t frameOffset = 0; + CpuArchitecture cpuArch = CpuArchUnknown; + uint32_t cpuId = 0; + char cpuManufacturer[13]; + + unordered_flat_map strings; + Vector stringData; + unordered_flat_map stringMap; + unordered_flat_map threadNames; + unordered_flat_map> externalNames; + + unordered_flat_map sourceLocation; + Vector> sourceLocationPayload; + unordered_flat_map sourceLocationPayloadMap; + Vector sourceLocationExpand; +#ifndef TRACY_NO_STATISTICS + unordered_flat_map sourceLocationZones; + bool sourceLocationZonesReady = false; + unordered_flat_map gpuSourceLocationZones; + bool gpuSourceLocationZonesReady = false; +#else + unordered_flat_map sourceLocationZonesCnt; + unordered_flat_map gpuSourceLocationZonesCnt; +#endif + + unordered_flat_map*, uint32_t, VarArrayHasher, VarArrayComparator> callstackMap; + Vector>> callstackPayload; + unordered_flat_map callstackFrameMap; + unordered_flat_map revFrameMap; + unordered_flat_map symbolMap; + unordered_flat_map symbolStats; + Vector symbolLoc; + Vector symbolLocInline; + int64_t newSymbolsIndex = -1; + int64_t newInlineSymbolsIndex = -1; + unordered_flat_map codeSymbolMap; + +#ifndef TRACY_NO_STATISTICS + unordered_flat_map*, uint32_t, VarArrayHasher, VarArrayComparator> parentCallstackMap; + Vector>> parentCallstackPayload; + unordered_flat_map parentCallstackFrameMap; + unordered_flat_map revParentFrameMap; + unordered_flat_map postponedSamples; + unordered_flat_map pendingInstructionPointers; + unordered_flat_map> instructionPointersMap; + unordered_flat_map> symbolSamples; + unordered_flat_map, CallstackFrameIdHash, CallstackFrameIdCompare> pendingSymbolSamples; + unordered_flat_map> childSamples; + bool newFramesWereReceived = false; + bool callstackSamplesReady = false; + bool newContextSwitchesReceived = false; + bool ghostZonesReady = false; + bool ghostZonesPostponed = false; + bool symbolSamplesReady = false; +#endif + + unordered_flat_map lockMap; + + ThreadCompress localThreadCompress; + ThreadCompress externalThreadCompress; + + Vector>> zoneChildren; + Vector>> gpuChildren; +#ifndef TRACY_NO_STATISTICS + Vector> ghostChildren; + Vector ghostFrames; + unordered_flat_map ghostFramesMap; +#endif + + Vector>> zoneVectorCache; + + Vector> frameImage; + Vector appInfo; + + CrashEvent crashEvent; + + unordered_flat_map ctxSwitch; + + CpuData cpuData[256]; + int cpuDataCount = 0; + unordered_flat_map tidToPid; + unordered_flat_map cpuThreadData; + + std::pair threadDataLast = std::make_pair( std::numeric_limits::max(), nullptr ); + std::pair ctxSwitchLast = std::make_pair( std::numeric_limits::max(), nullptr ); + uint64_t checkSrclocLast = 0; + std::pair shrinkSrclocLast = std::make_pair( std::numeric_limits::max(), 0 ); +#ifndef TRACY_NO_STATISTICS + std::pair srclocZonesLast = std::make_pair( 0, nullptr ); + std::pair gpuZonesLast = std::make_pair( 0, nullptr ); +#else + std::pair srclocCntLast = std::make_pair( 0, nullptr ); + std::pair gpuCntLast = std::make_pair( 0, nullptr ); +#endif + +#ifndef TRACY_NO_STATISTICS + Vector ctxUsage; + bool ctxUsageReady = false; +#endif + + unordered_flat_map>> cpuTopology; + unordered_flat_map cpuTopologyMap; + + unordered_flat_map symbolCode; + uint64_t symbolCodeSize = 0; + + unordered_flat_map sourceFileCache; + + unordered_flat_map hwSamples; + bool hasBranchRetirement = false; + + unordered_flat_map fiberToThreadMap; + }; + + struct MbpsBlock + { + MbpsBlock() : mbps( 64 ), compRatio( 1.0 ), queue( 0 ), transferred( 0 ) {} + + std::shared_mutex lock; + std::vector mbps; + float compRatio; + size_t queue; + uint64_t transferred; + }; + + struct FailureData + { + uint64_t thread; + int16_t srcloc; + uint32_t callstack; + std::string message; + }; + + struct FrameImagePending + { + const char* image; + uint32_t csz; + }; + +public: + enum class Failure + { + None, + ZoneStack, + ZoneDoubleEnd, + ZoneText, + ZoneValue, + ZoneColor, + ZoneName, + MemFree, + MemAllocTwice, + FrameEnd, + FrameImageIndex, + FrameImageTwice, + FiberLeave, + SourceLocationOverflow, + + NUM_FAILURES + }; + + Worker( const char* addr, uint16_t port, int64_t memoryLimit ); + Worker( const char* name, const char* program, const std::vector& timeline, const std::vector& messages, const std::vector& plots, const std::unordered_map& threadNames ); + Worker( FileRead& f, EventType::Type eventMask = EventType::All, bool bgTasks = true, bool allowStringModification = false); + ~Worker(); + + const std::string& GetAddr() const { return m_addr; } + uint16_t GetPort() const { return m_port; } + const std::string& GetCaptureName() const { return m_captureName; } + const std::string& GetCaptureProgram() const { return m_captureProgram; } + uint64_t GetCaptureTime() const { return m_captureTime; } + uint64_t GetExecutableTime() const { return m_executableTime; } + const std::string& GetHostInfo() const { return m_hostInfo; } + int64_t GetDelay() const { return m_delay; } + int64_t GetResolution() const { return m_resolution; } + uint64_t GetPid() const { return m_pid; }; + CpuArchitecture GetCpuArch() const { return m_data.cpuArch; } + uint32_t GetCpuId() const { return m_data.cpuId; } + const char* GetCpuManufacturer() const { return m_data.cpuManufacturer; } + + std::mutex& GetDataLock() { return m_data.lock; } + size_t GetFrameCount( const FrameData& fd ) const { return fd.frames.size(); } + size_t GetFullFrameCount( const FrameData& fd ) const; + bool AreFramesUsed() const; + int64_t GetFirstTime() const; + int64_t GetLastTime() const { return m_data.lastTime; } + uint64_t GetZoneCount() const { return m_data.zonesCnt; } + uint64_t GetZoneExtraCount() const { return m_data.zoneExtra.size() - 1; } + uint64_t GetGpuZoneCount() const { return m_data.gpuCnt; } + uint64_t GetLockCount() const; + uint64_t GetPlotCount() const; + uint64_t GetTracyPlotCount() const; + uint64_t GetContextSwitchCount() const; + uint64_t GetContextSwitchPerCpuCount() const; + bool HasContextSwitches() const { return !m_data.ctxSwitch.empty(); } + uint64_t GetSrcLocCount() const { return m_data.sourceLocationPayload.size() + m_data.sourceLocation.size(); } + uint64_t GetCallstackPayloadCount() const { return m_data.callstackPayload.size() - 1; } +#ifndef TRACY_NO_STATISTICS + uint64_t GetCallstackParentPayloadCount() const { return m_data.parentCallstackPayload.size(); } + uint64_t GetCallstackParentFrameCount() const { return m_callstackParentNextIdx; } +#endif + uint64_t GetCallstackFrameCount() const { return m_data.callstackFrameMap.size(); } + uint64_t GetCallstackSampleCount() const { return m_data.samplesCnt; } + uint64_t GetSymbolsCount() const { return m_data.symbolMap.size(); } + uint64_t GetSymbolCodeCount() const { return m_data.symbolCode.size(); } + uint64_t GetSymbolCodeSize() const { return m_data.symbolCodeSize; } + uint64_t GetGhostZonesCount() const { return m_data.ghostCnt; } + uint32_t GetFrameImageCount() const { return (uint32_t)m_data.frameImage.size(); } + uint64_t GetStringsCount() const { return m_data.strings.size() + m_data.stringData.size(); } + uint64_t GetHwSampleCountAddress() const { return m_data.hwSamples.size(); } + uint64_t GetHwSampleCount() const; + bool HasHwBranchRetirement() const { return m_data.hasBranchRetirement; } +#ifndef TRACY_NO_STATISTICS + uint64_t GetChildSamplesCountSyms() const { return m_data.childSamples.size(); } + uint64_t GetChildSamplesCountFull() const; + uint64_t GetContextSwitchSampleCount() const; +#endif + uint64_t GetFrameOffset() const { return m_data.frameOffset; } + const FrameData* GetFramesBase() const { return m_data.framesBase; } + const Vector& GetFrames() const { return m_data.frames.Data(); } + const ContextSwitch* const GetContextSwitchData( uint64_t thread ) + { + if( m_data.ctxSwitchLast.first == thread ) return m_data.ctxSwitchLast.second; + return GetContextSwitchDataImpl( thread ); + } + const CpuData* GetCpuData() const { return m_data.cpuData; } + int GetCpuDataCpuCount() const { return m_data.cpuDataCount; } + uint64_t GetPidFromTid( uint64_t tid ) const; + const unordered_flat_map& GetCpuThreadData() const { return m_data.cpuThreadData; } + const unordered_flat_map& GetSourceFileCache() const { return m_data.sourceFileCache; } + uint64_t GetSourceFileCacheCount() const { return m_data.sourceFileCache.size(); } + uint64_t GetSourceFileCacheSize() const; + MemoryBlock GetSourceFileFromCache( const char* file ) const; + HwSampleData* GetHwSampleData( uint64_t addr ); + + int64_t GetFrameTime( const FrameData& fd, size_t idx ) const; + int64_t GetFrameBegin( const FrameData& fd, size_t idx ) const; + int64_t GetFrameEnd( const FrameData& fd, size_t idx ) const; + const FrameImage* GetFrameImage( const FrameData& fd, size_t idx ) const; + std::pair GetFrameRange( const FrameData& fd, int64_t from, int64_t to ); + + const unordered_flat_map& GetLockMap() const { return m_data.lockMap; } + const Vector>& GetMessages() const { return m_data.messages; } + const Vector& GetGpuData() const { return m_data.gpuData; } + const Vector& GetPlots() const { return m_data.plots.Data(); } + const Vector& GetThreadData() const { return m_data.threads; } + const ThreadData* GetThreadData( uint64_t tid ) const; + const MemData& GetMemoryNamed( uint64_t name ) const; + const unordered_flat_map& GetMemNameMap() const { return m_data.memNameMap; } + const Vector>& GetFrameImages() const { return m_data.frameImage; } + const Vector& GetAppInfo() const { return m_data.appInfo; } + + const VarArray& GetCallstack( uint32_t idx ) const { return *m_data.callstackPayload[idx]; } + const CallstackFrameData* GetCallstackFrame( const CallstackFrameId& ptr ) const; + CallstackFrameId PackPointer( uint64_t ptr ) const; + uint64_t GetCanonicalPointer( const CallstackFrameId& id ) const; + const SymbolData* GetSymbolData( uint64_t sym ) const; + bool HasSymbolCode( uint64_t sym ) const; + const char* GetSymbolCode( uint64_t sym, uint32_t& len ) const; + uint64_t GetSymbolForAddress( uint64_t address ); + uint64_t GetSymbolForAddress( uint64_t address, uint32_t& offset ); + uint64_t GetInlineSymbolForAddress( uint64_t address ) const; + bool HasInlineSymbolAddresses() const { return !m_data.codeSymbolMap.empty(); } + StringIdx GetLocationForAddress( uint64_t address, uint32_t& line ) const; + const uint64_t* GetInlineSymbolList( uint64_t sym, uint32_t len ); + + unordered_flat_map& GetCallstackFrameMap() { return m_data.callstackFrameMap; } + +#ifndef TRACY_NO_STATISTICS + const VarArray& GetParentCallstack( uint32_t idx ) const { return *m_data.parentCallstackPayload[idx]; } + const CallstackFrameData* GetParentCallstackFrame( const CallstackFrameId& ptr ) const; + const Vector* GetSamplesForSymbol( uint64_t symAddr ) const; + const Vector* GetChildSamples( uint64_t addr ) const; +#endif + + const CrashEvent& GetCrashEvent() const { return m_data.crashEvent; } + + // Some zones may have incomplete timing data (only start time is available, end hasn't arrived yet). + // GetZoneEnd() will try to infer the end time by looking at child zones (parent zone can't end + // before its children have ended). + // GetZoneEndDirect() will only return zone's direct timing data, without looking at children. + tracy_force_inline int64_t GetZoneEnd( const ZoneEvent& ev ) { return ev.IsEndValid() ? ev.End() : GetZoneEndImpl( ev ); } + tracy_force_inline int64_t GetZoneEnd( const GpuEvent& ev ) { return ev.GpuEnd() >= 0 ? ev.GpuEnd() : GetZoneEndImpl( ev ); } + static tracy_force_inline int64_t GetZoneEndDirect( const ZoneEvent& ev ) { return ev.IsEndValid() ? ev.End() : ev.Start(); } + static tracy_force_inline int64_t GetZoneEndDirect( const GpuEvent& ev ) { return ev.GpuEnd() >= 0 ? ev.GpuEnd() : ev.GpuStart(); } + + uint32_t FindStringIdx( const char* str ) const; + const char* GetString( uint64_t ptr ) const; + const char* GetString( const StringRef& ref ) const; + const char* GetString( const StringIdx& idx ) const; + const char* GetThreadName( uint64_t id ) const; + bool IsThreadLocal( uint64_t id ); + bool IsThreadFiber( uint64_t id ); + const SourceLocation& GetSourceLocation( int16_t srcloc ) const; + std::pair GetExternalName( uint64_t id ) const; + + const char* GetZoneName( const SourceLocation& srcloc ) const; + const char* GetZoneName( const ZoneEvent& ev ) const; + const char* GetZoneName( const ZoneEvent& ev, const SourceLocation& srcloc ) const; + const char* GetZoneName( const GpuEvent& ev ) const; + + tracy_force_inline const Vector>& GetZoneChildren( int32_t idx ) const { return m_data.zoneChildren[idx]; } + tracy_force_inline const Vector>& GetGpuChildren( int32_t idx ) const { return m_data.gpuChildren[idx]; } +#ifndef TRACY_NO_STATISTICS + tracy_force_inline const Vector& GetGhostChildren( int32_t idx ) const { return m_data.ghostChildren[idx]; } + tracy_force_inline const GhostKey& GetGhostFrame( const Int24& frame ) const { return m_data.ghostFrames[frame.Val()]; } +#endif + + tracy_force_inline const bool HasZoneExtra( const ZoneEvent& ev ) const { return ev.extra != 0; } + tracy_force_inline const ZoneExtra& GetZoneExtra( const ZoneEvent& ev ) const { return m_data.zoneExtra[ev.extra]; } + + std::vector GetMatchingSourceLocation( const char* query, bool ignoreCase ) const; + + const unordered_flat_map& GetSymbolMap() const { return m_data.symbolMap; } + +#ifndef TRACY_NO_STATISTICS + SourceLocationZones& GetZonesForSourceLocation( int16_t srcloc ); + const SourceLocationZones& GetZonesForSourceLocation( int16_t srcloc ) const; + const unordered_flat_map& GetSourceLocationZones() const { return m_data.sourceLocationZones; } + const unordered_flat_map& GetGpuSourceLocationZones() const { return m_data.gpuSourceLocationZones; } + bool AreSourceLocationZonesReady() const { return m_data.sourceLocationZonesReady; } + bool AreGpuSourceLocationZonesReady() const { return m_data.gpuSourceLocationZonesReady; } + bool IsCpuUsageReady() const { return m_data.ctxUsageReady; } + const Vector& GetCpuUsage() const { return m_data.ctxUsage; } + + const unordered_flat_map& GetSymbolStats() const { return m_data.symbolStats; } + const SymbolStats* GetSymbolStats( uint64_t symAddr ) const; + const unordered_flat_map* GetSymbolInstructionPointers( uint64_t symAddr ) const; + bool AreCallstackSamplesReady() const { return m_data.callstackSamplesReady; } + bool AreGhostZonesReady() const { return m_data.ghostZonesReady; } + bool AreSymbolSamplesReady() const { return m_data.symbolSamplesReady; } +#endif + + tracy_force_inline uint16_t CompressThread( uint64_t thread ) { return m_data.localThreadCompress.CompressThread( thread ); } + tracy_force_inline uint64_t DecompressThread( uint16_t thread ) const { return m_data.localThreadCompress.DecompressThread( thread ); } + tracy_force_inline uint64_t DecompressThreadExternal( uint16_t thread ) const { return m_data.externalThreadCompress.DecompressThread( thread ); } + + std::shared_mutex& GetMbpsDataLock() { return m_mbpsData.lock; } + const std::vector& GetMbpsData() const { return m_mbpsData.mbps; } + float GetCompRatio() const { return m_mbpsData.compRatio; } + size_t GetSendQueueSize() const { return m_mbpsData.queue; } + size_t GetSendInFlight() const { return m_serverQuerySpaceBase - m_serverQuerySpaceLeft; } + uint64_t GetDataTransferred() const { return m_mbpsData.transferred; } + + bool HasData() const { return m_hasData.load( std::memory_order_acquire ); } + bool IsConnected() const { return m_connected.load( std::memory_order_relaxed ); } + bool IsDataStatic() const { return !m_thread.joinable(); } + bool IsBackgroundDone() const { return m_backgroundDone.load( std::memory_order_relaxed ); } + bool IsOnDemand() const { return m_onDemand; } + void Shutdown() { m_shutdown.store( true, std::memory_order_relaxed ); } + void Disconnect(); + bool WasDisconnectIssued() const { return m_disconnect; } + int64_t GetMemoryLimit() const { return m_memoryLimit; } + + void Write( FileWrite& f, bool fiDict ); + int GetTraceVersion() const { return m_traceVersion; } + uint8_t GetHandshakeStatus() const { return m_handshake.load( std::memory_order_relaxed ); } + int64_t GetSamplingPeriod() const { return m_samplingPeriod; } + bool AreSamplesInconsistent() const { return m_inconsistentSamples; } + + static const LoadProgress& GetLoadProgress() { return s_loadProgress; } + int64_t GetLoadTime() const { return m_loadTime; } + + void ClearFailure() { m_failure = Failure::None; } + Failure GetFailureType() const { return m_failure; } + const FailureData& GetFailureData() const { return m_failureData; } + static const char* GetFailureString( Failure failure ); + + const char* UnpackFrameImage( const FrameImage& image ) { return m_texcomp.Unpack( image ); } + + const Vector& GetParameters() const { return m_params; } + void SetParameter( size_t paramIdx, int32_t val ); + + const decltype(DataBlock::cpuTopology)& GetCpuTopology() const { return m_data.cpuTopology; } + const CpuThreadTopology* GetThreadTopology( uint32_t cpuThread ) const; + + std::pair GetTextureCompressionBytes() const { return std::make_pair( m_texcomp.GetInputBytesCount(), m_texcomp.GetOutputBytesCount() ); } + + void DoPostponedSymbols(); + void DoPostponedInlineSymbols(); + void DoPostponedWork(); + void DoPostponedWorkAll(); + + void CacheSourceFiles(); + + StringLocation StoreString(const char* str, size_t sz); + + std::vector& GetPendingThreadHints() { return m_pendingThreadHints; } + void ClearPendingThreadHints() { m_pendingThreadHints.clear(); } + +private: + void Network(); + void Exec(); + void Query( ServerQuery type, uint64_t data, uint32_t extra = 0 ); + void QueryTerminate(); + void QuerySourceFile( const char* fn, const char* image ); + void QueryDataTransfer( const void* ptr, size_t size ); + + tracy_force_inline bool DispatchProcess( const QueueItem& ev, const char*& ptr ); + tracy_force_inline bool Process( const QueueItem& ev ); + tracy_force_inline void ProcessThreadContext( const QueueThreadContext& ev ); + tracy_force_inline void ProcessZoneBegin( const QueueZoneBegin& ev ); + tracy_force_inline void ProcessZoneBeginCallstack( const QueueZoneBegin& ev ); + tracy_force_inline void ProcessZoneBeginAllocSrcLoc( const QueueZoneBeginLean& ev ); + tracy_force_inline void ProcessZoneBeginAllocSrcLocCallstack( const QueueZoneBeginLean& ev ); + tracy_force_inline void ProcessZoneEnd( const QueueZoneEnd& ev ); + tracy_force_inline void ProcessZoneValidation( const QueueZoneValidation& ev ); + tracy_force_inline void ProcessFrameMark( const QueueFrameMark& ev ); + tracy_force_inline void ProcessFrameMarkStart( const QueueFrameMark& ev ); + tracy_force_inline void ProcessFrameMarkEnd( const QueueFrameMark& ev ); + tracy_force_inline void ProcessFrameVsync( const QueueFrameVsync& ev ); + tracy_force_inline void ProcessFrameImage( const QueueFrameImage& ev ); + tracy_force_inline void ProcessZoneText(); + tracy_force_inline void ProcessZoneName(); + tracy_force_inline void ProcessZoneColor( const QueueZoneColor& ev ); + tracy_force_inline void ProcessZoneValue( const QueueZoneValue& ev ); + tracy_force_inline void ProcessLockAnnounce( const QueueLockAnnounce& ev ); + tracy_force_inline void ProcessLockTerminate( const QueueLockTerminate& ev ); + tracy_force_inline void ProcessLockWait( const QueueLockWait& ev ); + tracy_force_inline void ProcessLockObtain( const QueueLockObtain& ev ); + tracy_force_inline void ProcessLockRelease( const QueueLockRelease& ev ); + tracy_force_inline void ProcessLockSharedWait( const QueueLockWait& ev ); + tracy_force_inline void ProcessLockSharedObtain( const QueueLockObtain& ev ); + tracy_force_inline void ProcessLockSharedRelease( const QueueLockReleaseShared& ev ); + tracy_force_inline void ProcessLockMark( const QueueLockMark& ev ); + tracy_force_inline void ProcessLockName( const QueueLockName& ev ); + tracy_force_inline void ProcessPlotDataInt( const QueuePlotDataInt& ev ); + tracy_force_inline void ProcessPlotDataFloat( const QueuePlotDataFloat& ev ); + tracy_force_inline void ProcessPlotDataDouble( const QueuePlotDataDouble& ev ); + tracy_force_inline void ProcessPlotConfig( const QueuePlotConfig& ev ); + tracy_force_inline void ProcessMessage( const QueueMessage& ev ); + tracy_force_inline void ProcessMessageLiteral( const QueueMessageLiteral& ev ); + tracy_force_inline void ProcessMessageColor( const QueueMessageColor& ev ); + tracy_force_inline void ProcessMessageLiteralColor( const QueueMessageColorLiteral& ev ); + tracy_force_inline void ProcessMessageCallstack( const QueueMessage& ev ); + tracy_force_inline void ProcessMessageLiteralCallstack( const QueueMessageLiteral& ev ); + tracy_force_inline void ProcessMessageColorCallstack( const QueueMessageColor& ev ); + tracy_force_inline void ProcessMessageLiteralColorCallstack( const QueueMessageColorLiteral& ev ); + tracy_force_inline void ProcessMessageAppInfo( const QueueMessage& ev ); + tracy_force_inline void ProcessGpuNewContext( const QueueGpuNewContext& ev ); + tracy_force_inline void ProcessGpuZoneBegin( const QueueGpuZoneBegin& ev, bool serial ); + tracy_force_inline void ProcessGpuZoneBeginCallstack( const QueueGpuZoneBegin& ev, bool serial ); + tracy_force_inline void ProcessGpuZoneBeginAllocSrcLoc( const QueueGpuZoneBeginLean& ev, bool serial ); + tracy_force_inline void ProcessGpuZoneBeginAllocSrcLocCallstack( const QueueGpuZoneBeginLean& ev, bool serial ); + tracy_force_inline void ProcessGpuZoneEnd( const QueueGpuZoneEnd& ev, bool serial ); + tracy_force_inline void ProcessGpuTime( const QueueGpuTime& ev ); + tracy_force_inline void ProcessGpuCalibration( const QueueGpuCalibration& ev ); + tracy_force_inline void ProcessGpuTimeSync( const QueueGpuTimeSync& ev ); + tracy_force_inline void ProcessGpuContextName( const QueueGpuContextName& ev ); + tracy_force_inline MemEvent* ProcessMemAlloc( const QueueMemAlloc& ev ); + tracy_force_inline MemEvent* ProcessMemAllocNamed( const QueueMemAlloc& ev ); + tracy_force_inline MemEvent* ProcessMemFree( const QueueMemFree& ev ); + tracy_force_inline MemEvent* ProcessMemFreeNamed( const QueueMemFree& ev ); + tracy_force_inline void ProcessMemAllocCallstack( const QueueMemAlloc& ev ); + tracy_force_inline void ProcessMemAllocCallstackNamed( const QueueMemAlloc& ev ); + tracy_force_inline void ProcessMemFreeCallstack( const QueueMemFree& ev ); + tracy_force_inline void ProcessMemFreeCallstackNamed( const QueueMemFree& ev ); + tracy_force_inline void ProcessCallstackSerial(); + tracy_force_inline void ProcessCallstack(); + tracy_force_inline void ProcessCallstackSample( const QueueCallstackSample& ev ); + tracy_force_inline void ProcessCallstackSampleContextSwitch( const QueueCallstackSample& ev ); + tracy_force_inline void ProcessCallstackFrameSize( const QueueCallstackFrameSize& ev ); + tracy_force_inline void ProcessCallstackFrame( const QueueCallstackFrame& ev, bool querySymbols ); + tracy_force_inline void ProcessSymbolInformation( const QueueSymbolInformation& ev ); + tracy_force_inline void ProcessCrashReport( const QueueCrashReport& ev ); + tracy_force_inline void ProcessSysTime( const QueueSysTime& ev ); + tracy_force_inline void ProcessSysPower( const QueueSysPower& ev ); + tracy_force_inline void ProcessContextSwitch( const QueueContextSwitch& ev ); + tracy_force_inline void ProcessThreadWakeup( const QueueThreadWakeup& ev ); + tracy_force_inline void ProcessTidToPid( const QueueTidToPid& ev ); + tracy_force_inline void ProcessHwSampleCpuCycle( const QueueHwSample& ev ); + tracy_force_inline void ProcessHwSampleInstructionRetired( const QueueHwSample& ev ); + tracy_force_inline void ProcessHwSampleCacheReference( const QueueHwSample& ev ); + tracy_force_inline void ProcessHwSampleCacheMiss( const QueueHwSample& ev ); + tracy_force_inline void ProcessHwSampleBranchRetired( const QueueHwSample& ev ); + tracy_force_inline void ProcessHwSampleBranchMiss( const QueueHwSample& ev ); + tracy_force_inline void ProcessParamSetup( const QueueParamSetup& ev ); + tracy_force_inline void ProcessSourceCodeNotAvailable( const QueueSourceCodeNotAvailable& ev ); + tracy_force_inline void ProcessCpuTopology( const QueueCpuTopology& ev ); + tracy_force_inline void ProcessMemNamePayload( const QueueMemNamePayload& ev ); + tracy_force_inline void ProcessThreadGroupHint( const QueueThreadGroupHint& ev ); + tracy_force_inline void ProcessFiberEnter( const QueueFiberEnter& ev ); + tracy_force_inline void ProcessFiberLeave( const QueueFiberLeave& ev ); + + tracy_force_inline ZoneEvent* AllocZoneEvent(); + tracy_force_inline void ProcessZoneBeginImpl( ZoneEvent* zone, const QueueZoneBegin& ev ); + tracy_force_inline void ProcessZoneBeginAllocSrcLocImpl( ZoneEvent* zone, const QueueZoneBeginLean& ev ); + tracy_force_inline void ProcessGpuZoneBeginImpl( GpuEvent* zone, const QueueGpuZoneBegin& ev, bool serial ); + tracy_force_inline void ProcessGpuZoneBeginAllocSrcLocImpl( GpuEvent* zone, const QueueGpuZoneBeginLean& ev, bool serial ); + tracy_force_inline void ProcessGpuZoneBeginImplCommon( GpuEvent* zone, const QueueGpuZoneBeginLean& ev, bool serial ); + tracy_force_inline void ProcessPlotDataImpl( uint64_t name, int64_t evTime, double val ); + tracy_force_inline MemEvent* ProcessMemAllocImpl( MemData& memdata, const QueueMemAlloc& ev ); + tracy_force_inline MemEvent* ProcessMemFreeImpl( MemData& memdata, const QueueMemFree& ev ); + tracy_force_inline void ProcessCallstackSampleImpl( const SampleData& sd, ThreadData& td ); + tracy_force_inline void ProcessCallstackSampleInsertSample( const SampleData& sd, ThreadData& td ); +#ifndef TRACY_NO_STATISTICS + tracy_force_inline void ProcessCallstackSampleImplStats( const SampleData& sd, ThreadData& td ); +#endif + + void ZoneStackFailure( uint64_t thread, const ZoneEvent* ev ); + void ZoneDoubleEndFailure( uint64_t thread, const ZoneEvent* ev ); + void ZoneTextFailure( uint64_t thread, const char* text ); + void ZoneValueFailure( uint64_t thread, uint64_t value ); + void ZoneColorFailure( uint64_t thread ); + void ZoneNameFailure( uint64_t thread ); + void MemFreeFailure( uint64_t thread ); + void MemAllocTwiceFailure( uint64_t thread ); + void FrameEndFailure(); + void FrameImageIndexFailure(); + void FrameImageTwiceFailure(); + void FiberLeaveFailure(); + void SourceLocationOverflowFailure(); + + tracy_force_inline void CheckSourceLocation( uint64_t ptr ); + void NewSourceLocation( uint64_t ptr ); + tracy_force_inline int16_t ShrinkSourceLocation( uint64_t srcloc ) + { + if( m_data.shrinkSrclocLast.first == srcloc ) return m_data.shrinkSrclocLast.second; + return ShrinkSourceLocationReal( srcloc ); + } + int16_t ShrinkSourceLocationReal( uint64_t srcloc ); + int16_t NewShrinkedSourceLocation( uint64_t srcloc ); + + tracy_force_inline void MemAllocChanged( MemData& memdata, int64_t time ); + void CreateMemAllocPlot( MemData& memdata ); + void ReconstructMemAllocPlot( MemData& memdata ); + + void InsertMessageData( MessageData* msg ); + + ThreadData* NoticeThreadReal( uint64_t thread ); + ThreadData* NewThread( uint64_t thread, bool fiber, int32_t groupHint ); + tracy_force_inline ThreadData* NoticeThread( uint64_t thread ) + { + if( m_data.threadDataLast.first == thread ) return m_data.threadDataLast.second; + return NoticeThreadReal( thread ); + } + ThreadData* RetrieveThreadReal( uint64_t thread ); + tracy_force_inline ThreadData* RetrieveThread( uint64_t thread ) + { + if( m_data.threadDataLast.first == thread ) return m_data.threadDataLast.second; + return RetrieveThreadReal( thread ); + } + + tracy_force_inline ThreadData* GetCurrentThreadData(); + +#ifndef TRACY_NO_STATISTICS + SourceLocationZones* GetSourceLocationZones( uint16_t srcloc ) + { + if( m_data.srclocZonesLast.first == srcloc ) return m_data.srclocZonesLast.second; + return GetSourceLocationZonesReal( srcloc ); + } + SourceLocationZones* GetSourceLocationZonesReal( uint16_t srcloc ); + + GpuSourceLocationZones* GetGpuSourceLocationZones( uint16_t srcloc ) + { + if( m_data.gpuZonesLast.first == srcloc ) return m_data.gpuZonesLast.second; + return GetGpuSourceLocationZonesReal( srcloc ); + } + GpuSourceLocationZones* GetGpuSourceLocationZonesReal( uint16_t srcloc ); +#else + uint64_t* GetSourceLocationZonesCnt( uint16_t srcloc ) + { + if( m_data.srclocCntLast.first == srcloc ) return m_data.srclocCntLast.second; + return GetSourceLocationZonesCntReal( srcloc ); + } + uint64_t* GetSourceLocationZonesCntReal( uint16_t srcloc ); + + uint64_t* GetGpuSourceLocationZonesCnt( uint16_t srcloc ) + { + if( m_data.gpuCntLast.first == srcloc ) return m_data.gpuCntLast.second; + return GetGpuSourceLocationZonesCntReal( srcloc ); + } + uint64_t* GetGpuSourceLocationZonesCntReal( uint16_t srcloc ); +#endif + + tracy_force_inline void NewZone( ZoneEvent* zone ); + + void InsertLockEvent( LockMap& lockmap, LockEvent* lev, uint64_t thread, int64_t time ); + + bool CheckString( uint64_t ptr ); + void CheckThreadString( uint64_t id ); + void CheckFiberName( uint64_t id, uint64_t tid ); + void CheckExternalName( uint64_t id ); + + void AddSourceLocation( const QueueSourceLocation& srcloc ); + void AddSourceLocationPayload( const char* data, size_t sz ); + + void AddString( uint64_t ptr, const char* str, size_t sz ); + void AddThreadString( uint64_t id, const char* str, size_t sz ); + void AddFiberName( uint64_t id, const char* str, size_t sz ); + void AddSingleString( const char* str, size_t sz ); + void AddSingleStringFailure( const char* str, size_t sz ); + void AddSecondString( const char* str, size_t sz ); + void AddExternalName( uint64_t ptr, const char* str, size_t sz ); + void AddExternalThreadName( uint64_t ptr, const char* str, size_t sz ); + void AddFrameImageData( const char* data, size_t sz ); + void AddSymbolCode( uint64_t ptr, const char* data, size_t sz ); + void AddSourceCode( uint32_t id, const char* data, size_t sz ); + + tracy_force_inline void AddCallstackPayload( const char* data, size_t sz ); + tracy_force_inline void AddCallstackAllocPayload( const char* data ); + uint32_t MergeCallstacks( uint32_t first, uint32_t second ); + + void InsertPlot( PlotData* plot, int64_t time, double val ); + void HandlePlotName( uint64_t name, const char* str, size_t sz ); + void HandleFrameName( uint64_t name, const char* str, size_t sz ); + + void HandlePostponedSamples(); + void HandlePostponedGhostZones(); + + bool IsFailureThreadStringRetrieved(); + bool IsSourceLocationRetrieved( int16_t srcloc ); + bool IsCallstackRetrieved( uint32_t callstack ); + bool HasAllFailureData(); + void HandleFailure( const char* ptr, const char* end ); + void DispatchFailure( const QueueItem& ev, const char*& ptr ); + + uint32_t GetSingleStringIdx(); + uint32_t GetSecondStringIdx(); + const ContextSwitch* const GetContextSwitchDataImpl( uint64_t thread ); + + void CacheSource( const StringRef& str, const StringIdx& image = StringIdx() ); + void CacheSourceFromFile( const char* fn ); + + tracy_force_inline Vector>& GetZoneChildrenMutable( int32_t idx ) { return m_data.zoneChildren[idx]; } + tracy_force_inline Vector>& GetGpuChildrenMutable( int32_t idx ) { return m_data.gpuChildren[idx]; } +#ifndef TRACY_NO_STATISTICS + tracy_force_inline Vector& GetGhostChildrenMutable( int32_t idx ) { return m_data.ghostChildren[idx]; } +#endif + +#ifndef TRACY_NO_STATISTICS + void ReconstructContextSwitchUsage(); + bool UpdateSampleStatistics( uint32_t callstack, uint32_t count, bool canPostpone ); + void UpdateSampleStatisticsPostponed( decltype(Worker::DataBlock::postponedSamples.begin())& it ); + void UpdateSampleStatisticsImpl( const CallstackFrameData** frames, uint16_t framesCount, uint32_t count, const VarArray& cs ); + tracy_force_inline void GetStackWithInlines( Vector& ret, const VarArray& cs ); + tracy_force_inline int AddGhostZone( const VarArray& cs, Vector* vec, uint64_t t ); +#endif + + tracy_force_inline int64_t ReadTimeline( FileRead& f, ZoneEvent* zone, int64_t refTime, int32_t& childIdx ); + tracy_force_inline int64_t ReadTimelineHaveSize( FileRead& f, ZoneEvent* zone, int64_t refTime, int32_t& childIdx, uint32_t sz ); + tracy_force_inline void ReadTimeline( FileRead& f, GpuEvent* zone, int64_t& refTime, int64_t& refGpuTime, int32_t& childIdx ); + tracy_force_inline void ReadTimelineHaveSize( FileRead& f, GpuEvent* zone, int64_t& refTime, int64_t& refGpuTime, int32_t& childIdx, uint64_t sz ); + +#ifndef TRACY_NO_STATISTICS + tracy_force_inline void ReconstructZoneStatistics( uint8_t* countMap, ZoneEvent& zone, uint16_t thread ); + tracy_force_inline void ReconstructZoneStatistics( GpuEvent& zone, uint16_t thread ); +#else + tracy_force_inline void CountZoneStatistics( ZoneEvent* zone ); + tracy_force_inline void CountZoneStatistics( GpuEvent* zone ); +#endif + + tracy_force_inline ZoneExtra& GetZoneExtraMutable( const ZoneEvent& ev ) { return m_data.zoneExtra[ev.extra]; } + tracy_force_inline ZoneExtra& AllocZoneExtra( ZoneEvent& ev ); + tracy_force_inline ZoneExtra& RequestZoneExtra( ZoneEvent& ev ); + + int64_t GetZoneEndImpl( const ZoneEvent& ev ); + int64_t GetZoneEndImpl( const GpuEvent& ev ); + + void UpdateMbps( int64_t td ); + + int64_t ReadTimeline( FileRead& f, Vector>& vec, uint32_t size, int64_t refTime, int32_t& childIdx ); + void ReadTimeline( FileRead& f, Vector>& vec, uint64_t size, int64_t& refTime, int64_t& refGpuTime, int32_t& childIdx ); + + tracy_force_inline void WriteTimeline( FileWrite& f, const Vector>& vec, int64_t& refTime ); + tracy_force_inline void WriteTimeline( FileWrite& f, const Vector>& vec, int64_t& refTime, int64_t& refGpuTime ); + template + void WriteTimelineImpl( FileWrite& f, const V& vec, int64_t& refTime ); + template + void WriteTimelineImpl( FileWrite& f, const V& vec, int64_t& refTime, int64_t& refGpuTime ); + + int64_t TscTime( int64_t tsc ) { return int64_t( ( tsc - m_data.baseTime ) * m_timerMul ); } + int64_t TscTime( uint64_t tsc ) { return int64_t( ( tsc - m_data.baseTime ) * m_timerMul ); } + int64_t TscPeriod( uint64_t tsc ) { return int64_t( tsc * m_timerMul ); } + + Socket m_sock; + std::string m_addr; + uint16_t m_port; + + std::thread m_thread; + std::thread m_threadNet; + std::atomic m_connected { false }; + std::atomic m_hasData; + std::atomic m_shutdown { false }; + + std::atomic m_backgroundDone { true }; + std::thread m_threadBackground; + + int64_t m_delay; + int64_t m_resolution; + double m_timerMul; + std::string m_captureName; + std::string m_captureProgram; + uint64_t m_captureTime; + uint64_t m_executableTime; + std::string m_hostInfo; + uint64_t m_pid; + int64_t m_samplingPeriod; + bool m_terminate = false; + bool m_crashed = false; + bool m_disconnect = false; + void* m_stream; // LZ4_streamDecode_t* + char* m_buffer; + int m_bufferOffset; + bool m_onDemand; + bool m_ignoreMemFreeFaults; + bool m_ignoreFrameEndFaults; + bool m_codeTransfer; + bool m_combineSamples; + bool m_identifySamples = false; + bool m_inconsistentSamples; + bool m_allowStringModification = false; + + short_ptr m_gpuCtxMap[256]; + uint32_t m_pendingCallstackId = 0; + int16_t m_pendingSourceLocationPayload = 0; + Vector m_sourceLocationQueue; + unordered_flat_map m_sourceLocationShrink; + unordered_flat_map m_threadMap; + unordered_flat_map m_vsyncFrameMap; + FrameImagePending m_pendingFrameImageData = {}; + unordered_flat_map m_pendingSymbols; + unordered_flat_set m_pendingFileStrings; + unordered_flat_set m_checkedFileStrings; + StringLocation m_pendingSingleString = {}; + StringLocation m_pendingSecondString = {}; + + uint32_t m_pendingStrings; + uint32_t m_pendingThreads; + uint32_t m_pendingFibers; + uint32_t m_pendingExternalNames; + uint32_t m_pendingSourceLocation; + uint32_t m_pendingCallstackFrames; + uint8_t m_pendingCallstackSubframes; + uint32_t m_pendingSymbolCode; + + CallstackFrameData* m_callstackFrameStaging; + uint64_t m_callstackFrameStagingPtr; + uint64_t m_callstackAllocNextIdx = 0; + uint64_t m_callstackParentNextIdx = 0; + + uint32_t m_serialNextCallstack = 0; + uint64_t m_memNamePayload = 0; + + Slab<64*1024*1024> m_slab; + int64_t m_memoryLimit; + + DataBlock m_data; + MbpsBlock m_mbpsData; + + int m_traceVersion; + std::atomic m_handshake { 0 }; + + static LoadProgress s_loadProgress; + int64_t m_loadTime; + + Failure m_failure = Failure::None; + FailureData m_failureData = {}; + + PlotData* m_sysTimePlot = nullptr; + + Vector m_serverQueryQueue, m_serverQueryQueuePrio; + size_t m_serverQuerySpaceLeft, m_serverQuerySpaceBase; + + unordered_flat_map m_frameImageStaging; + char* m_frameImageBuffer = nullptr; + size_t m_frameImageBufferSize = 0; + TextureCompression m_texcomp; + + uint64_t m_threadCtx = 0; + ThreadData* m_threadCtxData = nullptr; + int64_t m_refTimeThread = 0; + int64_t m_refTimeSerial = 0; + int64_t m_refTimeCtx = 0; + int64_t m_refTimeGpu = 0; + + std::atomic m_bytes { 0 }; + std::atomic m_decBytes { 0 }; + + struct NetBuffer + { + int bufferOffset; + int size; + }; + + std::vector m_netRead; + std::mutex m_netReadLock; + std::condition_variable m_netReadCv; + + int m_netWriteCnt = 0; + std::mutex m_netWriteLock; + std::condition_variable m_netWriteCv; + +#ifdef TRACY_NO_STATISTICS + Vector m_zoneEventPool; +#endif + + Vector m_params; + + char* m_tmpBuf = nullptr; + size_t m_tmpBufSize = 0; + + unordered_flat_map m_nextCallstack; + unordered_flat_map m_sourceCodeQuery; + uint32_t m_nextSourceCodeQuery = 0; + + unordered_flat_map m_powerData; + + Vector m_inlineStack; + + std::vector m_pendingThreadHints; +}; + +} + +#endif diff --git a/third_party/tracy/server/tracy_pdqsort.h b/third_party/tracy/server/tracy_pdqsort.h new file mode 100644 index 0000000000000000000000000000000000000000..5aaa4754d07712541b9b449f4bf90cd9c49bc74c --- /dev/null +++ b/third_party/tracy/server/tracy_pdqsort.h @@ -0,0 +1,524 @@ +/* + pdqsort.h - Pattern-defeating quicksort. + + Copyright (c) 2015 Orson Peters + + This software is provided 'as-is', without any express or implied warranty. In no event will the + authors be held liable for any damages arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, including commercial + applications, and to alter it and redistribute it freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not claim that you wrote the + original software. If you use this software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be misrepresented as + being the original software. + + 3. This notice may not be removed or altered from any source distribution. +*/ + + +#ifndef TRACY_PDQSORT_H +#define TRACY_PDQSORT_H + +#include "../public/common/TracyForceInline.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#define PDQSORT_PREFER_MOVE(x) std::move(x) + +namespace tracy{ + +namespace pdqsort_detail { + enum { + // Partitions below this size are sorted using insertion sort. + insertion_sort_threshold = 24, + + // Partitions above this size use Tukey's ninther to select the pivot. + ninther_threshold = 128, + + // When we detect an already sorted partition, attempt an insertion sort that allows this + // amount of element moves before giving up. + partial_insertion_sort_limit = 8, + + // Must be multiple of 8 due to loop unrolling, and < 256 to fit in unsigned char. + block_size = 64, + + // Cacheline size, assumes power of two. + cacheline_size = 64 + + }; + + template struct is_default_compare : std::false_type { }; + template struct is_default_compare> : std::true_type { }; + template struct is_default_compare> : std::true_type { }; + + // Returns floor(log2(n)), assumes n > 0. + template + tracy_force_inline int log2(T n) { + int log = 0; + while (n >>= 1) ++log; + return log; + } + + // Sorts [begin, end) using insertion sort with the given comparison function. + template + tracy_force_inline void insertion_sort(Iter begin, Iter end, Compare comp) { + typedef typename std::iterator_traits::value_type T; + if (begin == end) return; + + for (Iter cur = begin + 1; cur != end; ++cur) { + Iter sift = cur; + Iter sift_1 = cur - 1; + + // Compare first so we can avoid 2 moves for an element already positioned correctly. + if (comp(*sift, *sift_1)) { + T tmp = PDQSORT_PREFER_MOVE(*sift); + + do { *sift-- = PDQSORT_PREFER_MOVE(*sift_1); } + while (sift != begin && comp(tmp, *--sift_1)); + + *sift = PDQSORT_PREFER_MOVE(tmp); + } + } + } + + // Sorts [begin, end) using insertion sort with the given comparison function. Assumes + // *(begin - 1) is an element smaller than or equal to any element in [begin, end). + template + tracy_force_inline void unguarded_insertion_sort(Iter begin, Iter end, Compare comp) { + typedef typename std::iterator_traits::value_type T; + if (begin == end) return; + + for (Iter cur = begin + 1; cur != end; ++cur) { + Iter sift = cur; + Iter sift_1 = cur - 1; + + // Compare first so we can avoid 2 moves for an element already positioned correctly. + if (comp(*sift, *sift_1)) { + T tmp = PDQSORT_PREFER_MOVE(*sift); + + do { *sift-- = PDQSORT_PREFER_MOVE(*sift_1); } + while (comp(tmp, *--sift_1)); + + *sift = PDQSORT_PREFER_MOVE(tmp); + } + } + } + + // Attempts to use insertion sort on [begin, end). Will return false if more than + // partial_insertion_sort_limit elements were moved, and abort sorting. Otherwise it will + // successfully sort and return true. + template + tracy_force_inline bool partial_insertion_sort(Iter begin, Iter end, Compare comp) { + typedef typename std::iterator_traits::value_type T; + if (begin == end) return true; + + std::size_t limit = 0; + for (Iter cur = begin + 1; cur != end; ++cur) { + Iter sift = cur; + Iter sift_1 = cur - 1; + + // Compare first so we can avoid 2 moves for an element already positioned correctly. + if (comp(*sift, *sift_1)) { + T tmp = PDQSORT_PREFER_MOVE(*sift); + + do { *sift-- = PDQSORT_PREFER_MOVE(*sift_1); } + while (sift != begin && comp(tmp, *--sift_1)); + + *sift = PDQSORT_PREFER_MOVE(tmp); + limit += cur - sift; + } + + if (limit > partial_insertion_sort_limit) return false; + } + + return true; + } + + template + tracy_force_inline void sort2(Iter a, Iter b, Compare comp) { + if (comp(*b, *a)) std::iter_swap(a, b); + } + + // Sorts the elements *a, *b and *c using comparison function comp. + template + tracy_force_inline void sort3(Iter a, Iter b, Iter c, Compare comp) { + sort2(a, b, comp); + sort2(b, c, comp); + sort2(a, b, comp); + } + + template + tracy_force_inline T* align_cacheline(T* p) { +#if defined(UINTPTR_MAX) + std::uintptr_t ip = reinterpret_cast(p); +#else + std::size_t ip = reinterpret_cast(p); +#endif + ip = (ip + cacheline_size - 1) & -cacheline_size; + return reinterpret_cast(ip); + } + + template + tracy_force_inline void swap_offsets(Iter first, Iter last, + unsigned char* offsets_l, unsigned char* offsets_r, + size_t num, bool use_swaps) { + typedef typename std::iterator_traits::value_type T; + if (use_swaps) { + // This case is needed for the descending distribution, where we need + // to have proper swapping for pdqsort to remain O(n). + for (size_t i = 0; i < num; ++i) { + std::iter_swap(first + offsets_l[i], last - offsets_r[i]); + } + } else if (num > 0) { + Iter l = first + offsets_l[0]; Iter r = last - offsets_r[0]; + T tmp(PDQSORT_PREFER_MOVE(*l)); *l = PDQSORT_PREFER_MOVE(*r); + for (size_t i = 1; i < num; ++i) { + l = first + offsets_l[i]; *r = PDQSORT_PREFER_MOVE(*l); + r = last - offsets_r[i]; *l = PDQSORT_PREFER_MOVE(*r); + } + *r = PDQSORT_PREFER_MOVE(tmp); + } + } + + // Partitions [begin, end) around pivot *begin using comparison function comp. Elements equal + // to the pivot are put in the right-hand partition. Returns the position of the pivot after + // partitioning and whether the passed sequence already was correctly partitioned. Assumes the + // pivot is a median of at least 3 elements and that [begin, end) is at least + // insertion_sort_threshold long. Uses branchless partitioning. + template + tracy_force_inline std::pair partition_right_branchless(Iter begin, Iter end, Compare comp) { + typedef typename std::iterator_traits::value_type T; + + // Move pivot into local for speed. + T pivot(PDQSORT_PREFER_MOVE(*begin)); + Iter first = begin; + Iter last = end; + + // Find the first element greater than or equal than the pivot (the median of 3 guarantees + // this exists). + while (comp(*++first, pivot)); + + // Find the first element strictly smaller than the pivot. We have to guard this search if + // there was no element before *first. + if (first - 1 == begin) while (first < last && !comp(*--last, pivot)); + else while ( !comp(*--last, pivot)); + + // If the first pair of elements that should be swapped to partition are the same element, + // the passed in sequence already was correctly partitioned. + bool already_partitioned = first >= last; + if (!already_partitioned) { + std::iter_swap(first, last); + ++first; + + // The following branchless partitioning is derived from "BlockQuicksort: How Branch + // Mispredictions don’t affect Quicksort" by Stefan Edelkamp and Armin Weiss, but + // heavily micro-optimized. + unsigned char offsets_l_storage[block_size + cacheline_size]; + unsigned char offsets_r_storage[block_size + cacheline_size]; + unsigned char* offsets_l = align_cacheline(offsets_l_storage); + unsigned char* offsets_r = align_cacheline(offsets_r_storage); + + Iter offsets_l_base = first; + Iter offsets_r_base = last; + size_t num_l, num_r, start_l, start_r; + num_l = num_r = start_l = start_r = 0; + + while (first < last) { + // Fill up offset blocks with elements that are on the wrong side. + // First we determine how much elements are considered for each offset block. + size_t num_unknown = last - first; + size_t left_split = num_l == 0 ? (num_r == 0 ? num_unknown / 2 : num_unknown) : 0; + size_t right_split = num_r == 0 ? (num_unknown - left_split) : 0; + + // Fill the offset blocks. + if (left_split >= block_size) { + for (size_t i = 0; i < block_size;) { + offsets_l[num_l] = i++; num_l += !comp(*first, pivot); ++first; + offsets_l[num_l] = i++; num_l += !comp(*first, pivot); ++first; + offsets_l[num_l] = i++; num_l += !comp(*first, pivot); ++first; + offsets_l[num_l] = i++; num_l += !comp(*first, pivot); ++first; + offsets_l[num_l] = i++; num_l += !comp(*first, pivot); ++first; + offsets_l[num_l] = i++; num_l += !comp(*first, pivot); ++first; + offsets_l[num_l] = i++; num_l += !comp(*first, pivot); ++first; + offsets_l[num_l] = i++; num_l += !comp(*first, pivot); ++first; + } + } else { + for (size_t i = 0; i < left_split;) { + offsets_l[num_l] = i++; num_l += !comp(*first, pivot); ++first; + } + } + + if (right_split >= block_size) { + for (size_t i = 0; i < block_size;) { + offsets_r[num_r] = ++i; num_r += comp(*--last, pivot); + offsets_r[num_r] = ++i; num_r += comp(*--last, pivot); + offsets_r[num_r] = ++i; num_r += comp(*--last, pivot); + offsets_r[num_r] = ++i; num_r += comp(*--last, pivot); + offsets_r[num_r] = ++i; num_r += comp(*--last, pivot); + offsets_r[num_r] = ++i; num_r += comp(*--last, pivot); + offsets_r[num_r] = ++i; num_r += comp(*--last, pivot); + offsets_r[num_r] = ++i; num_r += comp(*--last, pivot); + } + } else { + for (size_t i = 0; i < right_split;) { + offsets_r[num_r] = ++i; num_r += comp(*--last, pivot); + } + } + + // Swap elements and update block sizes and first/last boundaries. + size_t num = std::min(num_l, num_r); + swap_offsets(offsets_l_base, offsets_r_base, + offsets_l + start_l, offsets_r + start_r, + num, num_l == num_r); + num_l -= num; num_r -= num; + start_l += num; start_r += num; + + if (num_l == 0) { + start_l = 0; + offsets_l_base = first; + } + + if (num_r == 0) { + start_r = 0; + offsets_r_base = last; + } + } + + // We have now fully identified [first, last)'s proper position. Swap the last elements. + if (num_l) { + offsets_l += start_l; + while (num_l--) std::iter_swap(offsets_l_base + offsets_l[num_l], --last); + first = last; + } + if (num_r) { + offsets_r += start_r; + while (num_r--) std::iter_swap(offsets_r_base - offsets_r[num_r], first), ++first; + last = first; + } + } + + // Put the pivot in the right place. + Iter pivot_pos = first - 1; + *begin = PDQSORT_PREFER_MOVE(*pivot_pos); + *pivot_pos = PDQSORT_PREFER_MOVE(pivot); + + return std::make_pair(pivot_pos, already_partitioned); + } + + + + // Partitions [begin, end) around pivot *begin using comparison function comp. Elements equal + // to the pivot are put in the right-hand partition. Returns the position of the pivot after + // partitioning and whether the passed sequence already was correctly partitioned. Assumes the + // pivot is a median of at least 3 elements and that [begin, end) is at least + // insertion_sort_threshold long. + template + tracy_force_inline std::pair partition_right(Iter begin, Iter end, Compare comp) { + typedef typename std::iterator_traits::value_type T; + + // Move pivot into local for speed. + T pivot(PDQSORT_PREFER_MOVE(*begin)); + + Iter first = begin; + Iter last = end; + + // Find the first element greater than or equal than the pivot (the median of 3 guarantees + // this exists). + while (comp(*++first, pivot)); + + // Find the first element strictly smaller than the pivot. We have to guard this search if + // there was no element before *first. + if (first - 1 == begin) while (first < last && !comp(*--last, pivot)); + else while ( !comp(*--last, pivot)); + + // If the first pair of elements that should be swapped to partition are the same element, + // the passed in sequence already was correctly partitioned. + bool already_partitioned = first >= last; + + // Keep swapping pairs of elements that are on the wrong side of the pivot. Previously + // swapped pairs guard the searches, which is why the first iteration is special-cased + // above. + while (first < last) { + std::iter_swap(first, last); + while (comp(*++first, pivot)); + while (!comp(*--last, pivot)); + } + + // Put the pivot in the right place. + Iter pivot_pos = first - 1; + *begin = PDQSORT_PREFER_MOVE(*pivot_pos); + *pivot_pos = PDQSORT_PREFER_MOVE(pivot); + + return std::make_pair(pivot_pos, already_partitioned); + } + + // Similar function to the one above, except elements equal to the pivot are put to the left of + // the pivot and it doesn't check or return if the passed sequence already was partitioned. + // Since this is rarely used (the many equal case), and in that case pdqsort already has O(n) + // performance, no block quicksort is applied here for simplicity. + template + tracy_force_inline Iter partition_left(Iter begin, Iter end, Compare comp) { + typedef typename std::iterator_traits::value_type T; + + T pivot(PDQSORT_PREFER_MOVE(*begin)); + Iter first = begin; + Iter last = end; + + while (comp(pivot, *--last)); + + if (last + 1 == end) while (first < last && !comp(pivot, *++first)); + else while ( !comp(pivot, *++first)); + + while (first < last) { + std::iter_swap(first, last); + while (comp(pivot, *--last)); + while (!comp(pivot, *++first)); + } + + Iter pivot_pos = last; + *begin = PDQSORT_PREFER_MOVE(*pivot_pos); + *pivot_pos = PDQSORT_PREFER_MOVE(pivot); + + return pivot_pos; + } + + + template + inline void pdqsort_loop(Iter begin, Iter end, Compare comp, int bad_allowed, bool leftmost = true) { + typedef typename std::iterator_traits::difference_type diff_t; + + // Use a while loop for tail recursion elimination. + while (true) { + diff_t size = end - begin; + + // Insertion sort is faster for small arrays. + if (size < insertion_sort_threshold) { + if (leftmost) insertion_sort(begin, end, comp); + else unguarded_insertion_sort(begin, end, comp); + return; + } + + // Choose pivot as median of 3 or pseudomedian of 9. + diff_t s2 = size / 2; + if (size > ninther_threshold) { + sort3(begin, begin + s2, end - 1, comp); + sort3(begin + 1, begin + (s2 - 1), end - 2, comp); + sort3(begin + 2, begin + (s2 + 1), end - 3, comp); + sort3(begin + (s2 - 1), begin + s2, begin + (s2 + 1), comp); + std::iter_swap(begin, begin + s2); + } else sort3(begin + s2, begin, end - 1, comp); + + // If *(begin - 1) is the end of the right partition of a previous partition operation + // there is no element in [begin, end) that is smaller than *(begin - 1). Then if our + // pivot compares equal to *(begin - 1) we change strategy, putting equal elements in + // the left partition, greater elements in the right partition. We do not have to + // recurse on the left partition, since it's sorted (all equal). + if (!leftmost && !comp(*(begin - 1), *begin)) { + begin = partition_left(begin, end, comp) + 1; + continue; + } + + // Partition and get results. + std::pair part_result = + Branchless ? partition_right_branchless(begin, end, comp) + : partition_right(begin, end, comp); + Iter pivot_pos = part_result.first; + bool already_partitioned = part_result.second; + + // Check for a highly unbalanced partition. + diff_t l_size = pivot_pos - begin; + diff_t r_size = end - (pivot_pos + 1); + bool highly_unbalanced = l_size < size / 8 || r_size < size / 8; + + // If we got a highly unbalanced partition we shuffle elements to break many patterns. + if (highly_unbalanced) { + // If we had too many bad partitions, switch to heapsort to guarantee O(n log n). + if (--bad_allowed == 0) { + std::make_heap(begin, end, comp); + std::sort_heap(begin, end, comp); + return; + } + + if (l_size >= insertion_sort_threshold) { + std::iter_swap(begin, begin + l_size / 4); + std::iter_swap(pivot_pos - 1, pivot_pos - l_size / 4); + + if (l_size > ninther_threshold) { + std::iter_swap(begin + 1, begin + (l_size / 4 + 1)); + std::iter_swap(begin + 2, begin + (l_size / 4 + 2)); + std::iter_swap(pivot_pos - 2, pivot_pos - (l_size / 4 + 1)); + std::iter_swap(pivot_pos - 3, pivot_pos - (l_size / 4 + 2)); + } + } + + if (r_size >= insertion_sort_threshold) { + std::iter_swap(pivot_pos + 1, pivot_pos + (1 + r_size / 4)); + std::iter_swap(end - 1, end - r_size / 4); + + if (r_size > ninther_threshold) { + std::iter_swap(pivot_pos + 2, pivot_pos + (2 + r_size / 4)); + std::iter_swap(pivot_pos + 3, pivot_pos + (3 + r_size / 4)); + std::iter_swap(end - 2, end - (1 + r_size / 4)); + std::iter_swap(end - 3, end - (2 + r_size / 4)); + } + } + } else { + // If we were decently balanced and we tried to sort an already partitioned + // sequence try to use insertion sort. + if (already_partitioned && partial_insertion_sort(begin, pivot_pos, comp) + && partial_insertion_sort(pivot_pos + 1, end, comp)) return; + } + + // Sort the left partition first using recursion and do tail recursion elimination for + // the right-hand partition. + pdqsort_loop(begin, pivot_pos, comp, bad_allowed, leftmost); + begin = pivot_pos + 1; + leftmost = false; + } + } +} + + +template +inline void pdqsort(Iter begin, Iter end, Compare comp) { + if (begin == end) return; + pdqsort_detail::pdqsort_loop::type>::value && + std::is_arithmetic::value_type>::value>( + begin, end, comp, pdqsort_detail::log2(end - begin)); +} + +template +inline void pdqsort(Iter begin, Iter end) { + typedef typename std::iterator_traits::value_type T; + pdqsort(begin, end, std::less()); +} + +template +inline void pdqsort_branchless(Iter begin, Iter end, Compare comp) { + if (begin == end) return; + pdqsort_detail::pdqsort_loop( + begin, end, comp, pdqsort_detail::log2(end - begin)); +} + +template +tracy_force_inline void pdqsort_branchless(Iter begin, Iter end) { + typedef typename std::iterator_traits::value_type T; + pdqsort_branchless(begin, end, std::less()); +} + +} + +#undef PDQSORT_PREFER_MOVE + +#endif diff --git a/third_party/tracy/server/tracy_xxhash.h b/third_party/tracy/server/tracy_xxhash.h new file mode 100644 index 0000000000000000000000000000000000000000..a18e8c762daaaaa0db8974232a876315ed6ca6f2 --- /dev/null +++ b/third_party/tracy/server/tracy_xxhash.h @@ -0,0 +1,6773 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (C) 2012-2021 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +/*! + * @mainpage xxHash + * + * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed + * limits. + * + * It is proposed in four flavors, in three families: + * 1. @ref XXH32_family + * - Classic 32-bit hash function. Simple, compact, and runs on almost all + * 32-bit and 64-bit systems. + * 2. @ref XXH64_family + * - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most + * 64-bit systems (but _not_ 32-bit systems). + * 3. @ref XXH3_family + * - Modern 64-bit and 128-bit hash function family which features improved + * strength and performance across the board, especially on smaller data. + * It benefits greatly from SIMD and 64-bit without requiring it. + * + * Benchmarks + * --- + * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04. + * The open source benchmark program is compiled with clang v10.0 using -O3 flag. + * + * | Hash Name | ISA ext | Width | Large Data Speed | Small Data Velocity | + * | -------------------- | ------- | ----: | ---------------: | ------------------: | + * | XXH3_64bits() | @b AVX2 | 64 | 59.4 GB/s | 133.1 | + * | MeowHash | AES-NI | 128 | 58.2 GB/s | 52.5 | + * | XXH3_128bits() | @b AVX2 | 128 | 57.9 GB/s | 118.1 | + * | CLHash | PCLMUL | 64 | 37.1 GB/s | 58.1 | + * | XXH3_64bits() | @b SSE2 | 64 | 31.5 GB/s | 133.1 | + * | XXH3_128bits() | @b SSE2 | 128 | 29.6 GB/s | 118.1 | + * | RAM sequential read | | N/A | 28.0 GB/s | N/A | + * | ahash | AES-NI | 64 | 22.5 GB/s | 107.2 | + * | City64 | | 64 | 22.0 GB/s | 76.6 | + * | T1ha2 | | 64 | 22.0 GB/s | 99.0 | + * | City128 | | 128 | 21.7 GB/s | 57.7 | + * | FarmHash | AES-NI | 64 | 21.3 GB/s | 71.9 | + * | XXH64() | | 64 | 19.4 GB/s | 71.0 | + * | SpookyHash | | 64 | 19.3 GB/s | 53.2 | + * | Mum | | 64 | 18.0 GB/s | 67.0 | + * | CRC32C | SSE4.2 | 32 | 13.0 GB/s | 57.9 | + * | XXH32() | | 32 | 9.7 GB/s | 71.9 | + * | City32 | | 32 | 9.1 GB/s | 66.0 | + * | Blake3* | @b AVX2 | 256 | 4.4 GB/s | 8.1 | + * | Murmur3 | | 32 | 3.9 GB/s | 56.1 | + * | SipHash* | | 64 | 3.0 GB/s | 43.2 | + * | Blake3* | @b SSE2 | 256 | 2.4 GB/s | 8.1 | + * | HighwayHash | | 64 | 1.4 GB/s | 6.0 | + * | FNV64 | | 64 | 1.2 GB/s | 62.7 | + * | Blake2* | | 256 | 1.1 GB/s | 5.1 | + * | SHA1* | | 160 | 0.8 GB/s | 5.6 | + * | MD5* | | 128 | 0.6 GB/s | 7.8 | + * @note + * - Hashes which require a specific ISA extension are noted. SSE2 is also noted, + * even though it is mandatory on x64. + * - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic + * by modern standards. + * - Small data velocity is a rough average of algorithm's efficiency for small + * data. For more accurate information, see the wiki. + * - More benchmarks and strength tests are found on the wiki: + * https://github.com/Cyan4973/xxHash/wiki + * + * Usage + * ------ + * All xxHash variants use a similar API. Changing the algorithm is a trivial + * substitution. + * + * @pre + * For functions which take an input and length parameter, the following + * requirements are assumed: + * - The range from [`input`, `input + length`) is valid, readable memory. + * - The only exception is if the `length` is `0`, `input` may be `NULL`. + * - For C++, the objects must have the *TriviallyCopyable* property, as the + * functions access bytes directly as if it was an array of `unsigned char`. + * + * @anchor single_shot_example + * **Single Shot** + * + * These functions are stateless functions which hash a contiguous block of memory, + * immediately returning the result. They are the easiest and usually the fastest + * option. + * + * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits() + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which hashes a null terminated string with XXH32(). + * XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed) + * { + * // NULL pointers are only valid if the length is zero + * size_t length = (string == NULL) ? 0 : strlen(string); + * return XXH32(string, length, seed); + * } + * @endcode + * + * @anchor streaming_example + * **Streaming** + * + * These groups of functions allow incremental hashing of unknown size, even + * more than what would fit in a size_t. + * + * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset() + * + * @code{.c} + * #include + * #include + * #include "xxhash.h" + * // Example for a function which hashes a FILE incrementally with XXH3_64bits(). + * XXH64_hash_t hashFile(FILE* f) + * { + * // Allocate a state struct. Do not just use malloc() or new. + * XXH3_state_t* state = XXH3_createState(); + * assert(state != NULL && "Out of memory!"); + * // Reset the state to start a new hashing session. + * XXH3_64bits_reset(state); + * char buffer[4096]; + * size_t count; + * // Read the file in chunks + * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) { + * // Run update() as many times as necessary to process the data + * XXH3_64bits_update(state, buffer, count); + * } + * // Retrieve the finalized hash. This will not change the state. + * XXH64_hash_t result = XXH3_64bits_digest(state); + * // Free the state. Do not use free(). + * XXH3_freeState(state); + * return result; + * } + * @endcode + * + * @file xxhash.h + * xxHash prototypes and implementation + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* **************************** + * INLINE mode + ******************************/ +/*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Gives access to internal state declaration, required for static allocation. + * + * Incompatible with dynamic linking, due to risks of ABI changes. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #include "xxhash.h" + * @endcode + */ +# define XXH_STATIC_LINKING_ONLY +/* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */ + +/*! + * @brief Gives access to internal definitions. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #define XXH_IMPLEMENTATION + * #include "xxhash.h" + * @endcode + */ +# define XXH_IMPLEMENTATION +/* Do not undef XXH_IMPLEMENTATION for Doxygen */ + +/*! + * @brief Exposes the implementation and marks all functions as `inline`. + * + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length is + * expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * @code{.c} + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * @endcode + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ +# define XXH_INLINE_ALL +# undef XXH_INLINE_ALL +/*! + * @brief Exposes the implementation without marking functions as inline. + */ +# define XXH_PRIVATE_API +# undef XXH_PRIVATE_API +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif + +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif + + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ + /* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ +# undef XXH_versionNumber + /* XXH32 */ +# undef XXH32 +# undef XXH32_createState +# undef XXH32_freeState +# undef XXH32_reset +# undef XXH32_update +# undef XXH32_digest +# undef XXH32_copyState +# undef XXH32_canonicalFromHash +# undef XXH32_hashFromCanonical + /* XXH64 */ +# undef XXH64 +# undef XXH64_createState +# undef XXH64_freeState +# undef XXH64_reset +# undef XXH64_update +# undef XXH64_digest +# undef XXH64_copyState +# undef XXH64_canonicalFromHash +# undef XXH64_hashFromCanonical + /* XXH3_64bits */ +# undef XXH3_64bits +# undef XXH3_64bits_withSecret +# undef XXH3_64bits_withSeed +# undef XXH3_64bits_withSecretandSeed +# undef XXH3_createState +# undef XXH3_freeState +# undef XXH3_copyState +# undef XXH3_64bits_reset +# undef XXH3_64bits_reset_withSeed +# undef XXH3_64bits_reset_withSecret +# undef XXH3_64bits_update +# undef XXH3_64bits_digest +# undef XXH3_generateSecret + /* XXH3_128bits */ +# undef XXH128 +# undef XXH3_128bits +# undef XXH3_128bits_withSeed +# undef XXH3_128bits_withSecret +# undef XXH3_128bits_reset +# undef XXH3_128bits_reset_withSeed +# undef XXH3_128bits_reset_withSecret +# undef XXH3_128bits_reset_withSecretandSeed +# undef XXH3_128bits_update +# undef XXH3_128bits_digest +# undef XXH128_isEqual +# undef XXH128_cmp +# undef XXH128_canonicalFromHash +# undef XXH128_hashFromCanonical + /* Finally, free the namespace itself */ +# undef XXH_NAMESPACE + + /* employ the namespace for XXH_INLINE_ALL */ +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. + */ +# define XXH_IPREF(Id) XXH_NAMESPACE ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + +/*! @brief Marks a global symbol. */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif + + +/* ************************************* +* Compiler specifics +***************************************/ + +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#if defined (__GNUC__) +# define XXH_CONSTF __attribute__((const)) +# define XXH_PUREF __attribute__((pure)) +# define XXH_MALLOCF __attribute__((malloc)) +#else +# define XXH_CONSTF /* disable */ +# define XXH_PUREF +# define XXH_MALLOCF +#endif + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 2 +/*! @brief Version number, encoded as two digits each */ +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) + +/*! + * @brief Obtains the xxHash version. + * + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. + * + * @return @ref XXH_VERSION_NUMBER of the invoked library. + */ +XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void); + + +/* **************************** +* Common basic types +******************************/ +#include /* size_t */ +/*! + * @brief Exit code for the streaming API. + */ +typedef enum { + XXH_OK = 0, /*!< OK */ + XXH_ERROR /*!< Error */ +} XXH_errorcode; + + +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; + +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint32_t XXH32_hash_t; + +#else +# include +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# elif ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +#endif + +/*! + * @} + * + * @defgroup XXH32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that the @ref XXH3_family provides competitive speed for both 32-bit + * and 64-bit systems, and offers true 64/128 bit hash results. + * + * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families + * @see @ref XXH32_impl for implementation details + * @{ + */ + +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s + * + * See @ref single_shot_example "Single Shot Example" for an example. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit hash value. + * + * @see + * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); + +#ifndef XXH_NO_STREAM +/*! + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * @see streaming_example at the top of @ref xxhash.h for an example. + */ + +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. + */ +typedef struct XXH32_state_s XXH32_state_t; + +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * Must be freed with XXH32_freeState(). + * @return An allocated XXH32_state_t on success, `NULL` on failure. + */ +XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * Must be allocated with XXH32_createState(). + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * @return XXH_OK. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); + +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * This function resets and seeds a state. Call it before @ref XXH32_update(). + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated xxHash32 value from that state. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/* + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + */ + +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ +typedef struct { + unsigned char digest[4]; /*!< Hash bytes, big endian */ +} XXH32_canonical_t; + +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); + +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + + +/*! @cond Doxygen ignores this part */ +#ifdef __has_attribute +# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +# define XXH_HAS_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * C23 __STDC_VERSION__ number hasn't been specified yet. For now + * leave as `201711L` (C17 + 1). + * TODO: Update to correct value when its been specified. + */ +#define XXH_C23_VN 201711L +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute) +# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define XXH_HAS_C_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define XXH_HAS_CPP_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute + * introduced in CPP17 and C23. + * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough + */ +#if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_ATTRIBUTE(__fallthrough__) +# define XXH_FALLTHROUGH __attribute__ ((__fallthrough__)) +#else +# define XXH_FALLTHROUGH /* fallthrough */ +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_NOESCAPE for annotated pointers in public API. + * https://clang.llvm.org/docs/AttributeReference.html#noescape + * As of writing this, only supported by clang. + */ +#if XXH_HAS_ATTRIBUTE(noescape) +# define XXH_NOESCAPE __attribute__((noescape)) +#else +# define XXH_NOESCAPE +#endif +/*! @endcond */ + + +/*! + * @} + * @ingroup public + * @{ + */ + +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint64_t XXH64_hash_t; +#else +# include +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif + +/*! + * @} + * + * @defgroup XXH64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. + */ + +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * This function usually runs faster on 64-bit systems, but slower on 32-bit + * systems (see benchmark). + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit hash. + * + * @see + * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ + +/*! + * @brief Allocates an @ref XXH64_state_t. + * + * Must be freed with XXH64_freeState(). + * @return An allocated XXH64_state_t on success, `NULL` on failure. + */ +XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void); + +/*! + * @brief Frees an @ref XXH64_state_t. + * + * Must be allocated with XXH64_createState(). + * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState(). + * @return XXH_OK. + */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH64_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +/*! + * @brief Resets an @ref XXH64_state_t to begin a new hash. + * + * This function resets and seeds a state. Call it before @ref XXH64_update(). + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH64_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH64_state_t. + * + * @note + * Calling XXH64_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated xxHash64 value from that state. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ +/******* Canonical representation *******/ + +/*! + * @brief Canonical (big endian) representation of @ref XXH64_hash_t. + */ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; + +/*! + * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t. + * + * @param dst The @ref XXH64_canonical_t pointer to be stored to. + * @param hash The @ref XXH64_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash); + +/*! + * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t. + * + * @param src The @ref XXH64_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src); + +#ifndef XXH_NO_XXH3 + +/*! + * @} + * ************************************************************************ + * @defgroup XXH3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3 + * at competitive speeds, even without vector support. Further details are + * explained in the implementation. + * + * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD + * implementations for many common platforms: + * - AVX512 + * - AVX2 + * - SSE2 + * - ARM NEON + * - WebAssembly SIMD128 + * - POWER8 VSX + * - s390x ZVector + * This can be controlled via the @ref XXH_VECTOR macro, but it automatically + * selects the best version according to predefined macros. For the x86 family, an + * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generate exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. + */ +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ + +/*! + * @brief 64-bit unseeded variant of XXH3. + * + * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of 0, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see + * XXH32(), XXH64(), XXH3_128bits(): equivalent for the other xxHash algorithms + * @see + * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants + * @see + * XXH3_64bits_reset(), XXH3_64bits_update(), XXH3_64bits_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief 64-bit seeded variant of XXH3 + * + * This variant generates a custom secret on the fly based on default secret + * altered using the `seed` value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * @param input The data to hash + * @param length The length + * @param seed The 64-bit seed to alter the state. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); + +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 + +/*! + * @brief 64-bit variant of XXH3 with a custom "secret". + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that secretSize *must* be large enough (>= XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing "XXH3_generateSecret()" instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + */ + +/*! + * @brief The state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + */ +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH3_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state); + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * This function resets `statePtr` and generate a secret with default parameters. Call it before @ref XXH3_64bits_update(). + * Digest will be equivalent to `XXH3_64bits()`. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * This function resets `statePtr` and generate a secret from `seed`. Call it before @ref XXH3_64bits_update(). + * Digest will be equivalent to `XXH3_64bits_withSeed()`. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the state. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); + +/*! + * XXH3_64bits_reset_withSecret(): + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t. + * + * @note + * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 64-bit hash value from that state. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ + + +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ + +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ +typedef struct { + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ +} XXH128_hash_t; + +/*! + * @brief Unseeded 128-bit variant of XXH3 + * + * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead + * for shorter inputs. + * + * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of 0, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see + * XXH32(), XXH64(), XXH3_64bits(): equivalent for the other xxHash algorithms + * @see + * XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants + * @see + * XXH3_128bits_reset(), XXH3_128bits_update(), XXH3_128bits_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len); +/*! @brief Seeded 128-bit variant of XXH3. @see XXH3_64bits_withSeed(). */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); +/*! @brief Custom secret 128-bit variant of XXH3. @see XXH3_64bits_withSecret(). */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. + */ + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * This function resets `statePtr` and generate a secret with default parameters. Call it before @ref XXH3_128bits_update(). + * Digest will be equivalent to `XXH3_128bits()`. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * This function resets `statePtr` and generate a secret from `seed`. Call it before @ref XXH3_128bits_update(). + * Digest will be equivalent to `XXH3_128bits_withSeed()`. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the state. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); +/*! @brief Custom secret 128-bit variant of XXH3. @see XXH_64bits_reset_withSecret(). */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t. + * + * @note + * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 128-bit hash value from that state. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ + +/*! + * XXH128_isEqual(): + * Return: 1 if `h1` and `h2` are equal, 0 if they are not. + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); + +/*! + * @brief Compares two @ref XXH128_hash_t + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * @return: >0 if *h128_1 > *h128_2 + * =0 if *h128_1 == *h128_2 + * <0 if *h128_1 < *h128_2 + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2); + + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; + + +/*! + * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t. + * + * @param dst The @ref XXH128_canonical_t pointer to be stored to. + * @param hash The @ref XXH128_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + */ +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash); + +/*! + * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t. + * + * @param src The @ref XXH128_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src); + + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ + +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ + + + +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ + +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. + */ + +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t v[4]; /*!< Accumulator lanes */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ +}; /* typedef'd to XXH32_state_t */ + + +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v[4]; /*!< Accumulator lanes */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ +}; /* typedef'd to XXH64_state_t */ + +#ifndef XXH_NO_XXH3 + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ +# include +# define XXH_ALIGN(n) alignas(n) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ +/* In C++ alignas() is a keyword */ +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif + +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif + +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 + +/*! + * @internal + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 + +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. + * + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER + +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) \ + do { \ + XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \ + tmp_xxh3_state_ptr->seed = 0; \ + tmp_xxh3_state_ptr->extSecret = NULL; \ + } while(0) + + +/*! + * simple alias to pre-selected XXH3_128bits variant + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); + + +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ + +/*! + * XXH3_generateSecret(): + * + * Derive a high-entropy secret from any user-defined content, named customSeed. + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection + * than 64-bit seed, as it becomes much more difficult for an external actor to + * guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length @p secretSize into an + * already allocated buffer @p secretBuffer. + * + * The generated secret can then be used with any `*_withSecret()` variant. + * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(), + * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret() + * are part of this list. They all accept a `secret` parameter + * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can + * be employed to ensure proper quality. + * + * @p customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch + * of zeroes. The resulting `secret` will nonetheless provide all required qualities. + * + * @pre + * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN + * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + * + * Example code: + * @code{.c} + * #include + * #include + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Hashes argv[2] using the entropy from argv[1]. + * int main(int argc, char* argv[]) + * { + * char secret[XXH3_SECRET_SIZE_MIN]; + * if (argv != 3) { return 1; } + * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1])); + * XXH64_hash_t h = XXH3_64bits_withSecret( + * argv[2], strlen(argv[2]), + * secret, sizeof(secret) + * ); + * printf("%016llx\n", (unsigned long long) h); + * } + * @endcode + */ +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize); + +/*! + * @brief Generate the same secret as the _withSeed() variants. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * + * Example C++ `std::string` hash class: + * @code{.cpp} + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Slow, seeds each time + * class HashSlow { + * XXH64_hash_t seed; + * public: + * HashSlow(XXH64_hash_t s) : seed{s} {} + * size_t operator()(const std::string& x) const { + * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)}; + * } + * }; + * // Fast, caches the seeded secret for future uses. + * class HashFast { + * unsigned char secret[XXH3_SECRET_SIZE_MIN]; + * public: + * HashFast(XXH64_hash_t s) { + * XXH3_generateSecret_fromSeed(secret, seed); + * } + * size_t operator()(const std::string& x) const { + * return size_t{ + * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret)) + * }; + * } + * }; + * @endcode + * @param secretBuffer A writable buffer of @ref XXH3_SECRET_SIZE_MIN bytes + * @param seed The seed to seed the state. + */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed); + +/*! + * These variants generate hash values using either + * @p seed for "short" keys (< XXH3_MIDSIZE_MAX = 240 bytes) + * or @p secret for "large" keys (>= XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @p secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, via its impact to the seed. + * + * This is not guaranteed when using the secret directly in "small data" scenarios, + * because only portions of the secret are employed for small data. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed); +/*! @copydoc XXH3_64bits_withSecretandSeed() */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#ifndef XXH_NO_STREAM +/*! @copydoc XXH3_64bits_withSecretandSeed() */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +/*! @copydoc XXH3_64bits_withSecretandSeed() */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#endif /* !XXH_NO_STREAM */ + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif + +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ + + +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ + + +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ + +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 + +/* ************************************* +* Tuning parameters +***************************************/ + +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref XXH32_family and you have a strict C90 compiler. + */ +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 + +/*! + * @def XXH_SIZE_OPT + * @brief Controls how much xxHash optimizes for size. + * + * xxHash, when compiled, tends to result in a rather large binary size. This + * is mostly due to heavy usage to forced inlining and constant folding of the + * @ref XXH3_family to increase performance. + * + * However, some developers prefer size over speed. This option can + * significantly reduce the size of the generated code. When using the `-Os` + * or `-Oz` options on GCC or Clang, this is defined to 1 by default, + * otherwise it is defined to 0. + * + * Most of these size optimizations can be controlled manually. + * + * This is a number from 0-2. + * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed + * comes first. + * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more + * conservative and disables hacks that increase code size. It implies the + * options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0, + * and @ref XXH3_NEON_LANES == 8 if they are not already defined. + * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible. + * Performance may cry. For example, the single shot functions just use the + * streaming API. + */ +# define XXH_SIZE_OPT 0 + +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips + * which are platforms known to offer good unaligned memory accesses performance. + * + * It is also disabled by default when @ref XXH_SIZE_OPT >= 1. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ +# define XXH_FORCE_ALIGN_CHECK 0 + +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if + * @ref XXH_SIZE_OPT >= 1, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 + +/*! + * @def XXH3_INLINE_SECRET + * @brief Determines whether to inline the XXH3 withSecret code. + * + * When the secret size is known, the compiler can improve the performance + * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret(). + * + * However, if the secret size is not known, it doesn't have any benefit. This + * happens when xxHash is compiled into a global symbol. Therefore, if + * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0. + * + * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers + * that are *sometimes* force inline on -Og, and it is impossible to automatically + * detect this optimization level. + */ +# define XXH3_INLINE_SECRET 0 + +/*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the finalizer. + * This is generally preferable for performance, + * but depending on exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ +# define XXH32_ENDJMP 0 + +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ + +/*! + * @def XXH_NO_STREAM + * @brief Disables the streaming API. + * + * When xxHash is not inlined and the streaming functions are not used, disabling + * the streaming functions can improve code size significantly, especially with + * the @ref XXH3_family which tends to make constant folded copies of itself. + */ +# define XXH_NO_STREAM +# undef XXH_NO_STREAM /* don't actually */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ + +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for GCC + * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy + * which for some reason does unaligned loads. */ +# if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED)) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +#ifndef XXH_SIZE_OPT + /* default to 1 for -Os or -Oz */ +# if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__) +# define XXH_SIZE_OPT 1 +# else +# define XXH_SIZE_OPT 0 +# endif +#endif + +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ + /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */ +# if XXH_SIZE_OPT >= 1 || \ + defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH3_INLINE_SECRET +# if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \ + || !defined(XXH_INLINE_ALL) +# define XXH3_INLINE_SECRET 0 +# else +# define XXH3_INLINE_SECRET 1 +# endif +#endif + +#ifndef XXH32_ENDJMP +/* generally preferable for performance */ +# define XXH32_ENDJMP 0 +#endif + +/*! + * @defgroup impl Implementation + * @{ + */ + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +#if defined(XXH_NO_STREAM) +/* nothing */ +#elif defined(XXH_NO_STDLIB) + +/* When requesting to disable any mention of stdlib, + * the library loses the ability to invoked malloc / free. + * In practice, it means that functions like `XXH*_createState()` + * will always fail, and return NULL. + * This flag is useful in situations where + * xxhash.h is integrated into some kernel, embedded or limited environment + * without access to dynamic allocation. + */ + +static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; } +static void XXH_free(void* p) { (void)p; } + +#else + +/* + * Modify the local functions below should you wish to use + * different memory routines for malloc() and free() + */ +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than malloc(). + */ +static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); } + +/*! + * @internal + * @brief Modify this function to use a different routine than free(). + */ +static void XXH_free(void* p) { free(p); } + +#endif /* XXH_NO_STDLIB */ + +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than memcpy(). + */ +static void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} + +#include /* ULLONG_MAX */ + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __attribute__((unused)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) +# define XXH_NO_INLINE static __attribute__((noinline)) +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + +#if XXH3_INLINE_SECRET +# define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE +#else +# define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE +#endif + + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif + +#if (XXH_DEBUGLEVEL>=1) +# include /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# if defined(__INTEL_COMPILER) +# define XXH_ASSERT(c) XXH_ASSUME((unsigned char) (c)) +# else +# define XXH_ASSERT(c) XXH_ASSUME(c) +# endif +#endif + +/* note: use after variable declarations */ +#ifndef XXH_STATIC_ASSERT +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0) +# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# else +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) +# endif +# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) +#endif + +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ +#if defined(__GNUC__) || defined(__clang__) +# define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif + +/* Specifically for NEON vectors which use the "w" constraint, on + * Clang. */ +#if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__) +# define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var)) +#else +# define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0) +#endif + +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; + +#ifdef XXH_OLD_NAMES +# warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly" +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +/* *** Memory access *** */ + +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32; + return *((const xxh_unalign32*)ptr); +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* *** Endianness *** */ + +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif + + + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif + + + +/* + * C23 and future versions have standard "unreachable()". + * Once it has been implemented reliably we can add it as an + * additional case: + * + * ``` + * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) + * # include + * # ifdef unreachable + * # define XXH_UNREACHABLE() unreachable() + * # endif + * #endif + * ``` + * + * Note C++23 also has std::unreachable() which can be detected + * as follows: + * ``` + * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L) + * # include + * # define XXH_UNREACHABLE() std::unreachable() + * #endif + * ``` + * NB: `__cpp_lib_unreachable` is defined in the `` header. + * We don't use that as including `` in `extern "C"` blocks + * doesn't work on GCC12 + */ + +#if XXH_HAS_BUILTIN(__builtin_unreachable) +# define XXH_UNREACHABLE() __builtin_unreachable() + +#elif defined(_MSC_VER) +# define XXH_UNREACHABLE() __assume(0) + +#else +# define XXH_UNREACHABLE() +#endif + +#if XXH_HAS_BUILTIN(__builtin_assume) +# define XXH_ASSUME(c) __builtin_assume(c) +#else +# define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); } +#endif + +/*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* *************************** +* Memory reads +*****************************/ + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; + +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} + +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} + +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +} + +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} + + +/* ************************************* +* Misc +***************************************/ +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup XXH32_impl XXH32 implementation + * @ingroup impl + * + * Details on the XXH32 implementation. + * @{ + */ + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif + +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing + * the loop. NEON is only faster on the A53, and with the newer cores, it is less + * than half the speed. + * + * Additionally, this is used on WASM SIMD128 because it JITs to the same + * SIMD instructions and has the same issue. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param hash The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 hash) +{ + hash ^= hash >> 15; + hash *= XXH_PRIME32_2; + hash ^= hash >> 13; + hash *= XXH_PRIME32_3; + hash ^= hash >> 16; + return hash; +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + * @see XXH64_finalize(). + */ +static XXH_PUREF xxh_u32 +XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + hash += (*ptr++) * XXH_PRIME32_5; \ + hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \ +} while (0) + + if (ptr==NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(hash); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 8: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 4: XXH_PROCESS4; + return XXH32_avalanche(hash); + + case 13: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 9: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 14: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 10: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 15: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 11: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 7: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 3: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 2: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 1: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 0: return XXH32_avalanche(hash); + } + XXH_ASSERT(0); + return hash; /* reaching this point is deemed impossible */ + } +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input , len , seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + xxh_u32 h32; + + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=16) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; + } while (input < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + XXH_PRIME32_5; + } + + h32 += (xxh_u32)len; + + return XXH32_finalize(h32, input, len&15, align); +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} + + + +/******* Hash streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + statePtr->v[1] = seed + XXH_PRIME32_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME32_1; + return XXH_OK; +} + + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const xxh_u32* p32 = state->mem32; + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const xxh_u8* const limit = bEnd - 16; + + do { + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v[0], 1) + + XXH_rotl32(state->v[1], 7) + + XXH_rotl32(state->v[2], 12) + + XXH_rotl32(state->v[3], 18); + } else { + h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! + * @ingroup XXH32_family + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * + * The canonical representation uses big endian convention, the same convention + * as human-readable numbers (large digits first). + * + * This way, hash values can be written into a file or buffer, remaining + * comparable across different systems. + * + * The following functions allow transformation of hash values to and from their + * canonical format. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ +/******* Memory access *******/ + +typedef XXH64_hash_t xxh_u64; + +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64; + return *((const xxh_unalign64*)ptr); +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); +} + +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); +} + +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); +} + +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); +} + + +/******* xxh64 *******/ +/*! + * @} + * @defgroup XXH64_impl XXH64 implementation + * @ingroup impl + * + * Details on the XXH64 implementation. + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif + +/*! @copydoc XXH32_round */ +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; + return acc; +} + +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; +} + +/*! @copydoc XXH32_avalanche */ +static xxh_u64 XXH64_avalanche(xxh_u64 hash) +{ + hash ^= hash >> 33; + hash *= XXH_PRIME64_2; + hash ^= hash >> 29; + hash *= XXH_PRIME64_3; + hash ^= hash >> 32; + return hash; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-31 bytes of @p ptr. + * + * There may be up to 31 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 32. + * @param align Whether @p ptr is aligned. + * @return The finalized hash + * @see XXH32_finalize(). + */ +static XXH_PUREF xxh_u64 +XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + if (ptr==NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + hash ^= k1; + hash = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + } + if (len >= 4) { + hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + } + while (len > 0) { + hash ^= (*ptr++) * XXH_PRIME64_5; + hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1; + --len; + } + return XXH64_avalanche(hash); +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH64(). + * + * @param input , len , seed Directly passed from @ref XXH64(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + xxh_u64 h64; + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=32) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 31; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; + v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; + v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; + v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; + } while (input= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, (const xxh_u8*)input, len); + return XXH64_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); + +#endif +} + +/******* Hash Streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH64_family*/ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + statePtr->v[1] = seed + XXH_PRIME64_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME64_1; + return XXH_OK; +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode +XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3)); + p += 32 - state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const xxh_u8* const limit = bEnd - 32; + + do { + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8; + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8; + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8; + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state) +{ + xxh_u64 h64; + + if (state->total_len >= 32) { + h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); + h64 = XXH64_mergeRound(h64, state->v[0]); + h64 = XXH64_mergeRound(h64, state->v[1]); + h64 = XXH64_mergeRound(h64, state->v[2]); + h64 = XXH64_mergeRound(h64, state->v[3]); + } else { + h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; + } + + h64 += (xxh_u64) state->total_len; + + return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#ifndef XXH_NO_XXH3 + +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup XXH3_impl XXH3 implementation + * @ingroup impl + * @{ + */ + +/* === Compiler specifics === */ + +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \ + || (defined (__clang__)) \ + || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \ + || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300)) +/* + * There are a LOT more compilers that recognize __restrict but this + * covers the major ones. + */ +# define XXH_RESTRICT __restrict +#else +# define XXH_RESTRICT /* disable */ +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif + +#ifndef XXH_HAS_INCLUDE +# ifdef __has_include +# define XXH_HAS_INCLUDE(x) __has_include(x) +# else +# define XXH_HAS_INCLUDE(x) 0 +# endif +#endif + +#if defined(__GNUC__) || defined(__clang__) +# if defined(__ARM_FEATURE_SVE) +# include +# endif +# if defined(__ARM_NEON__) || defined(__ARM_NEON) \ + || (defined(_M_ARM) && _M_ARM >= 7) \ + || defined(_M_ARM64) || defined(_M_ARM64EC) \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* WASM SIMD128 via SIMDe */ +# define inline __inline__ /* circumvent a clang bug */ +# include +# undef inline +# elif defined(__AVX2__) +# include +# elif defined(__SSE2__) +# include +# endif +#endif + +#if defined(_MSC_VER) +# include +#endif + +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif + +/* ========================================== + * Vectorization detection + * ========================================== */ + +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * internal macro XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< + * NEON for most ARMv7-A, all AArch64, and WASM SIMD128 + * via the SIMDeverywhere polyfill provided with the + * Emscripten SDK. + */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ + XXH_SVE = 6, /*!< SVE for some ARMv8-A and ARMv9-A */ +}; +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment required for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif + +/* Actual definition */ +#ifndef XXH_DOXYGEN +# define XXH_SCALAR 0 +# define XXH_SSE2 1 +# define XXH_AVX2 2 +# define XXH_AVX512 3 +# define XXH_NEON 4 +# define XXH_VSX 5 +# define XXH_SVE 6 +#endif + +#ifndef XXH_VECTOR /* can be defined on command line */ +# if defined(__ARM_FEATURE_SVE) +# define XXH_VECTOR XXH_SVE +# elif ( \ + defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* wasm simd128 via SIMDe */ \ + ) && ( \ + defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ + ) +# define XXH_VECTOR XXH_NEON +# elif defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif + +/* __ARM_FEATURE_SVE is only supported by GCC & Clang. */ +#if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE) +# ifdef _MSC_VER +# pragma warning(once : 4606) +# else +# warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead." +# endif +# undef XXH_VECTOR +# define XXH_VECTOR XXH_SCALAR +#endif + +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# elif XXH_VECTOR == XXH_SVE /* sve */ +# define XXH_ACC_ALIGN 64 +# endif +#endif + +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#elif XXH_VECTOR == XXH_SVE +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif + +#if defined(__GNUC__) || defined(__clang__) +# define XXH_ALIASING __attribute__((may_alias)) +#else +# define XXH_ALIASING /* nothing */ +#endif + +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + +#if XXH_VECTOR == XXH_NEON + +/* + * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3 + * optimizes out the entire hashLong loop because of the aliasing violation. + * + * However, GCC is also inefficient at load-store optimization with vld1q/vst1q, + * so the only option is to mark it as aliasing. + */ +typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING; + +/*! + * @internal + * @brief `vld1q_u64` but faster and alignment-safe. + * + * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only + * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86). + * + * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it + * prohibits load-store optimizations. Therefore, a direct dereference is used. + * + * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe + * unaligned load. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */ +{ + return *(xxh_aliasing_uint64x2_t const *)ptr; +} +#else +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) +{ + return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr)); +} +#endif + +/*! + * @internal + * @brief `vmlal_u32` on low and high halves of a vector. + * + * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with + * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32` + * with `vmlal_u32`. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11 +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* Inline assembly is the only way */ + __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs)); + return acc; +} +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* This intrinsic works as expected */ + return vmlal_high_u32(acc, lhs, rhs); +} +#else +/* Portable intrinsic versions */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs)); +} +/*! @copydoc XXH_vmlal_low_u32 + * Assume the compiler converts this to vmlal_high_u32 on aarch64 */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs)); +} +#endif + +/*! + * @ingroup tuning + * @brief Controls the NEON to scalar ratio for XXH3 + * + * This can be set to 2, 4, 6, or 8. + * + * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used. + * + * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those + * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU + * bandwidth. + * + * This is even more noticeable on the more advanced cores like the Cortex-A76 which + * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. + * + * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes + * and 2 scalar lanes, which is chosen by default. + * + * This does not apply to Apple processors or 32-bit processors, which run better with + * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes. + * + * This change benefits CPUs with large micro-op buffers without negatively affecting + * most other CPUs: + * + * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | + * |:----------------------|:--------------------|----------:|-----------:|------:| + * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | + * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | + * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | + * | Apple M1 | 4 NEON/8 micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% | + * + * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. + * + * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning + * it effectively becomes worse 4. + * + * @see XXH3_accumulate_512_neon() + */ +# ifndef XXH3_NEON_LANES +# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ + && !defined(__APPLE__) && XXH_SIZE_OPT <= 0 +# define XXH3_NEON_LANES 6 +# else +# define XXH3_NEON_LANES XXH_ACC_NB +# endif +# endif +#endif /* XXH_VECTOR == XXH_NEON */ + +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +/* Annoyingly, these headers _may_ define three macros: `bool`, `vector`, + * and `pixel`. This is a problem for obvious reasons. + * + * These keywords are unnecessary; the spec literally says they are + * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd + * after including the header. + * + * We use pragma push_macro/pop_macro to keep the namespace clean. */ +# pragma push_macro("bool") +# pragma push_macro("vector") +# pragma push_macro("pixel") +/* silence potential macro redefined warnings */ +# undef bool +# undef vector +# undef pixel + +# if defined(__s390x__) +# include +# else +# include +# endif + +/* Restore the original macro values, if applicable. */ +# pragma pop_macro("pixel") +# pragma pop_macro("vector") +# pragma pop_macro("bool") + +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; + +/* + * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue. + */ +typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ + +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +# endif +# endif /* XXH_VSX_BE */ + +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} + +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ + /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ +#endif /* XXH_VECTOR == XXH_VSX */ + +#if XXH_VECTOR == XXH_SVE +#define ACCRND(acc, offset) \ +do { \ + svuint64_t input_vec = svld1_u64(mask, xinput + offset); \ + svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \ + svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \ + svuint64_t swapped = svtbl_u64(input_vec, kSwap); \ + svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \ + svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \ + svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \ + acc = svadd_u64_x(mask, acc, mul); \ +} while (0) +#endif /* XXH_VECTOR == XXH_SVE */ + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if XXH_SIZE_OPT >= 1 +# define XXH_PREFETCH(ptr) (void)(ptr) +# elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ + + +/* ========================================== + * XXH3 default settings + * ========================================== */ + +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif + +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +}; + +static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */ +static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */ + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs , rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. + */ +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. + */ +#elif defined(_M_ARM64) || defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(__umulh) +#endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; + +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; +#endif +} + +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. + * + * The reason for the separate function is to prevent passing too many structs + * around by value. This will hopefully inline the multiply, but we don't force it. + * + * @param lhs , rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() + */ +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) +{ + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; +} + +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} + +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= PRIME_MX1; + h64 = XXH_xorshift64(h64, 32); + return h64; +} + +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= PRIME_MX2; + h64 ^= (h64 >> 35) + len ; + h64 *= PRIME_MX2; + return XXH_xorshift64(h64, 28); +} + + +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ + +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} + +/* + * DISCLAIMER: There are known *seed-dependent* multicollisions here due to + * multiplication by zero, affecting hashes of lengths 17 to 240. + * + * However, they are very unlikely. + * + * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all + * unseeded non-cryptographic hashes, it does not attempt to defend itself + * against specially crafted inputs, only random inputs. + * + * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes + * cancelling out the secret is taken an arbitrary number of times (addressed + * in XXH3_accumulate_512), this collision is very unlikely with random inputs + * and/or proper seeding: + * + * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a + * function that is only called up to 16 times per hash with up to 240 bytes of + * input. + * + * This is not too bad for a non-cryptographic hash function, especially with + * only 64 bit outputs. + * + * The 128-bit variant (which trades some speed for strength) is NOT affected + * by this, although it is always a good idea to use a proper seed if you care + * about strength. + */ +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } +} + +/* For mid range keys, XXH3 uses a Mum-hash variant. */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; +#if XXH_SIZE_OPT >= 1 + /* Smaller and cleaner, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc += XXH3_mix16B(input+16 * i, secret+32*i, seed); + acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed); + } while (i-- != 0); +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); + } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); +#endif + return XXH3_avalanche(acc); + } +} + +#define XXH3_MIDSIZE_MAX 240 + +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + xxh_u64 acc_end; + unsigned int const nbRounds = (unsigned int)len / 16; + unsigned int i; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + /* last bytes */ + acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + XXH_ASSERT(nbRounds >= 8); + acc = XXH3_avalanche(acc); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + /* + * Prevents clang for unrolling the acc loop and interleaving with this one. + */ + XXH_COMPILER_GUARD(acc); + acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + return XXH3_avalanche(acc + acc_end); + } +} + + +/* ======= Long Keys ======= */ + +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif + +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ + +/* + * These macros are to generate an XXH3_accumulate() function. + * The two arguments select the name suffix and target attribute. + * + * The name of this symbol is XXH3_accumulate_() and it calls + * XXH3_accumulate_512_(). + * + * It may be useful to hand implement this function if the compiler fails to + * optimize the inline function. + */ +#define XXH3_ACCUMULATE_TEMPLATE(name) \ +void \ +XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc, \ + const xxh_u8* XXH_RESTRICT input, \ + const xxh_u8* XXH_RESTRICT secret, \ + size_t nbStripes) \ +{ \ + size_t n; \ + for (n = 0; n < nbStripes; n++ ) { \ + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; \ + XXH_PREFETCH(in + XXH_PREFETCH_DIST); \ + XXH3_accumulate_512_##name( \ + acc, \ + in, \ + secret + n*XXH_SECRET_CONSUME_RATE); \ + } \ +} + + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif + + +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) + +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} +XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512) + +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64); + __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos); + + const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); + __m512i* const dest = ( __m512i*) customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i=0; i < nbRounds; ++i) { + dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) + +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2) + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); + + const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); + __m256i* dest = ( __m256i*) customSecret; + +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed); + } +} + +#endif + +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2) + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); +# endif + int i; + + const void* const src16 = XXH3_kSecret; + __m128i* dst16 = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); +# endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + + for (i=0; i < nbRounds; ++i) { + dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_NEON) + +/* forward declarations for the scalar routines */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, size_t lane); + +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, size_t lane); + +/*! + * @internal + * @brief The bulk processing loop for NEON and WASM SIMD128. + * + * The NEON code path is actually partially scalar when running on AArch64. This + * is to optimize the pipelining and can have up to 15% speedup depending on the + * CPU, and it also mitigates some GCC codegen issues. + * + * @see XXH3_NEON_LANES for configuring this and details about this optimization. + * + * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit + * integers instead of the other platforms which mask full 64-bit vectors, + * so the setup is more complicated than just shifting right. + * + * Additionally, there is an optimization for 4 lanes at once noted below. + * + * Since, as stated, the most optimal amount of lanes for Cortexes is 6, + * there needs to be *three* versions of the accumulate operation used + * for the remaining 2 lanes. + * + * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap + * nearly perfectly. + */ + +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); + { /* GCC for darwin arm64 does not like aliasing here */ + xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* xinput = (const uint8_t *) input; + uint8_t const* xsecret = (const uint8_t *) secret; + + size_t i; +#ifdef __wasm_simd128__ + /* + * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret + * is constant propagated, which results in it converting it to this + * inside the loop: + * + * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0) + * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0) + * ... + * + * This requires a full 32-bit address immediate (and therefore a 6 byte + * instruction) as well as an add for each offset. + * + * Putting an asm guard prevents it from folding (at the cost of losing + * the alignment hint), and uses the free offset in `v128.load` instead + * of adding secret_offset each time which overall reduces code size by + * about a kilobyte and improves performance. + */ + XXH_COMPILER_GUARD(xsecret); +#endif + /* Scalar lanes use the normal scalarRound routine */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } + i = 0; + /* 4 NEON lanes at a time. */ + for (; i+1 < XXH3_NEON_LANES / 2; i+=2) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16)); + uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i+1) * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i+1) * 16)); + /* data_swap = swap(data_vec) */ + uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1); + uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1); + uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2); + + /* + * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a + * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to + * get one vector with the low 32 bits of each lane, and one vector + * with the high 32 bits of each lane. + * + * The intrinsic returns a double vector because the original ARMv7-a + * instruction modified both arguments in place. AArch64 and SIMD128 emit + * two instructions from this intrinsic. + * + * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ] + * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ] + */ + uint32x4x2_t unzipped = vuzpq_u32( + vreinterpretq_u32_u64(data_key_1), + vreinterpretq_u32_u64(data_key_2) + ); + /* data_key_lo = data_key & 0xFFFFFFFF */ + uint32x4_t data_key_lo = unzipped.val[0]; + /* data_key_hi = data_key >> 32 */ + uint32x4_t data_key_hi = unzipped.val[1]; + /* + * Then, we can split the vectors horizontally and multiply which, as for most + * widening intrinsics, have a variant that works on both high half vectors + * for free on AArch64. A similar instruction is available on SIMD128. + * + * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi + */ + uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi); + uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi); + /* + * Clang reorders + * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s + * c += a; // add acc.2d, acc.2d, swap.2d + * to + * c += a; // add acc.2d, acc.2d, swap.2d + * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s + * + * While it would make sense in theory since the addition is faster, + * for reasons likely related to umlal being limited to certain NEON + * pipelines, this is worse. A compiler guard fixes this. + */ + XXH_COMPILER_GUARD_CLANG_NEON(sum_1); + XXH_COMPILER_GUARD_CLANG_NEON(sum_2); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64(xacc[i], sum_1); + xacc[i+1] = vaddq_u64(xacc[i+1], sum_2); + } + /* Operate on the remaining NEON lanes 2 at a time. */ + for (; i < XXH3_NEON_LANES / 2; i++) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + /* acc_vec_2 = swap(data_vec) */ + uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* For two lanes, just use VMOVN and VSHRN. */ + /* data_key_lo = data_key & 0xFFFFFFFF; */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* data_key_hi = data_key >> 32; */ + uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32); + /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */ + uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi); + /* Same Clang workaround as before */ + XXH_COMPILER_GUARD_CLANG_NEON(sum); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64 (xacc[i], sum); + } + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_uint64x2_t* xacc = (xxh_aliasing_uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + + size_t i; + /* WASM uses operator overloads and doesn't need these. */ +#ifndef __wasm_simd128__ + /* { prime32_1, prime32_1 } */ + uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1); + /* { 0, prime32_1, 0, prime32_1 } */ + uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32)); +#endif + + /* AArch64 uses both scalar and neon at the same time */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); + uint64x2_t data_vec = veorq_u64(acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* xacc[i] *= XXH_PRIME32_1 */ +#ifdef __wasm_simd128__ + /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */ + xacc[i] = data_key * XXH_PRIME32_1; +#else + /* + * Expanded version with portable NEON intrinsics + * + * lo(x) * lo(y) + (hi(x) * lo(y) << 32) + * + * prod_hi = hi(data_key) * lo(prime) << 32 + * + * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector + * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits + * and avoid the shift. + */ + uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi); + /* Extract low bits for vmlal_u32 */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo); +#endif + } + } +} +#endif + +#if (XXH_VECTOR == XXH_VSX) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* presumed aligned */ + xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + xxh_u8 const* const xinput = (xxh_u8 const*) input; /* no alignment restriction */ + xxh_u8 const* const xsecret = (xxh_u8 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = xacc[i]; + acc_vec += product; + + /* swap high and low halves */ +#ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); +#else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); +#endif + xacc[i] = acc_vec; + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + const xxh_u8* const xsecret = (const xxh_u8*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_SVE) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_sve( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc); + ACCRND(vacc, 0); + svst1_u64(mask, xacc, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } +} + +XXH_FORCE_INLINE void +XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes) +{ + if (nbStripes != 0) { + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc + 0); + do { + /* svprfd(svbool_t, void *, enum svfprop); */ + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(vacc, 0); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } + } +} + +#endif + +/* scalar variants - universal */ + +#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) +/* + * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they + * emit an excess mask and a full 64-bit multiply-add (MADD X-form). + * + * While this might not seem like much, as AArch64 is a 64-bit architecture, only + * big Cortex designs have a full 64-bit multiplier. + * + * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit + * multiplies expand to 2-3 multiplies in microcode. This has a major penalty + * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline. + * + * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does + * not have this penalty and does the mask automatically. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + xxh_u64 ret; + /* note: %x = 64-bit register, %w = 32-bit register */ + __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc)); + return ret; +} +#else +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc; +} +#endif + +/*! + * @internal + * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* xacc = (xxh_u64*) acc; + xxh_u8 const* xinput = (xxh_u8 const*) input; + xxh_u8 const* xsecret = (xxh_u8 const*) secret; + XXH_ASSERT(lane < XXH_ACC_NB); + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + { + xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); + xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ + xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]); + } +} + +/*! + * @internal + * @brief Processes a 64 byte block of data using the scalar path. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + size_t i; + /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */ +#if defined(__GNUC__) && !defined(__clang__) \ + && (defined(__arm__) || defined(__thumb2__)) \ + && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \ + && XXH_SIZE_OPT <= 0 +# pragma GCC unroll 8 +#endif + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar) + +/*! + * @internal + * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + XXH_ASSERT(lane < XXH_ACC_NB); + { + xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); + xxh_u64 acc64 = xacc[lane]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[lane] = acc64; + } +} + +/*! + * @internal + * @brief Scrambles the accumulators after a large chunk has been read + */ +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } +} + +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + /* + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. + */ + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__GNUC__) && defined(__aarch64__) + /* + * UGLY HACK: + * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), it fights for bandwidth with + * the arithmetic instructions. + * + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes the compiler to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * + * See XXH3_NEON_LANES for details on the pipsline. + * + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + XXH_COMPILER_GUARD(kSecretPtr); +#endif + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes the compiler to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } +} + + +typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); + + +#if (XXH_VECTOR == XXH_AVX512) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_accumulate XXH3_accumulate_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + +#elif (XXH_VECTOR == XXH_AVX2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_accumulate XXH3_accumulate_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + +#elif (XXH_VECTOR == XXH_SSE2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_accumulate XXH3_accumulate_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + +#elif (XXH_VECTOR == XXH_NEON) + +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_accumulate XXH3_accumulate_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_VSX) + +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_accumulate XXH3_accumulate_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_SVE) +#define XXH3_accumulate_512 XXH3_accumulate_512_sve +#define XXH3_accumulate XXH3_accumulate_sve +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#else /* scalar */ + +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_accumulate XXH3_accumulate_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#endif + +#if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */ +# undef XXH3_initCustomSecret +# define XXH3_initCustomSecret XXH3_initCustomSecret_scalar +#endif + +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; + + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + f_acc(acc, input + n*block_len, secret, nbStripesPerBlock); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + } + + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + f_acc(acc, input + nb_blocks*block_len, secret, nbStripes); + + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } +} + +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) +{ + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); +} + +static XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; + + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif + } + + return XXH3_avalanche(result64); +} + +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + /* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); +} + +/* + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction cache. + * Note that inside this no_inline function, we do inline the internal loop, + * and provide a statically defined secret size to allow optimization of vector loop. + */ +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ +#if XXH_SIZE_OPT <= 0 + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); +#endif + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + + +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); +} + + +/* === Public entry point === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length) +{ + return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} + +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (length <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize); +} + + +/* === XXH3 streaming === */ +#ifndef XXH_NO_STREAM +/* + * Malloc's a pointer that is always aligned to align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + return NULL; + } +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); + } +} +/*! @ingroup XXH3_family */ +/*! + * @brief Allocate an @ref XXH3_state_t. + * + * Must be freed with XXH3_freeState(). + * @return An allocated XXH3_state_t on success, `NULL` on failure. + */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) +{ + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; +} + +/*! @ingroup XXH3_family */ +/*! + * @brief Frees an @ref XXH3_state_t. + * + * Must be allocated with XXH3_createState(). + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @return XXH_OK. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) +{ + XXH_alignedFree(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state) +{ + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); +} + +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64) +{ + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; +} + +/*! + * @internal + * @brief Processes a large input for XXH3_update() and XXH3_digest_long(). + * + * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block. + * + * @param acc Pointer to the 8 accumulator lanes + * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in the block* + * @param nbStripesPerBlock Number of stripes in a block + * @param input Input pointer + * @param nbStripes Number of stripes to process + * @param secret Secret pointer + * @param secretLimit Offset of the last block in @p secret + * @param f_acc Pointer to an XXH3_accumulate implementation + * @param f_scramble Pointer to an XXH3_scrambleAcc implementation + * @return Pointer past the end of @p input after processing + */ +XXH_FORCE_INLINE const xxh_u8 * +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE; + /* Process full blocks */ + if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) { + /* Process the initial partial block... */ + size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr; + + do { + /* Accumulate and scramble */ + f_acc(acc, input, initialSecret, nbStripesThisIter); + f_scramble(acc, secret + secretLimit); + input += nbStripesThisIter * XXH_STRIPE_LEN; + nbStripes -= nbStripesThisIter; + /* Then continue the loop with the full block size */ + nbStripesThisIter = nbStripesPerBlock; + initialSecret = secret; + } while (nbStripes >= nbStripesPerBlock); + *nbStripesSoFarPtr = 0; + } + /* Process a partial block */ + if (nbStripes > 0) { + f_acc(acc, input, initialSecret, nbStripes); + input += nbStripes * XXH_STRIPE_LEN; + *nbStripesSoFarPtr += nbStripes; + } + /* Return end pointer */ + return input; +} + +#ifndef XXH3_STREAM_USE_STACK +# if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */ +# define XXH3_STREAM_USE_STACK 1 +# endif +#endif +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* XXH_RESTRICT const state, + const xxh_u8* XXH_RESTRICT input, size_t len, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + XXH_ASSERT(state != NULL); + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; + XXH_memcpy(acc, state->acc, sizeof(acc)); +#else + xxh_u64* XXH_RESTRICT const acc = state->acc; +#endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + + /* small input : just fill in tmp buffer */ + if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ + + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + input = XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, nbStripes, + secret, state->secretLimit, + f_acc, f_scramble); + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + + } + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + XXH_memcpy(state->acc, acc, sizeof(acc)); +#endif + } + + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate, XXH3_scrambleAcc); +} + + +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) +{ + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + const xxh_u8* lastStripePtr; + + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + /* Consume remaining stripes then point to remaining data in buffer */ + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate, XXH3_scrambleAcc); + lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN; + } else { /* bufferedSize < XXH_STRIPE_LEN */ + /* Copy to temp buffer */ + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + lastStripePtr = lastStripe; + } + /* Last stripe */ + XXH3_accumulate_512(acc, + lastStripePtr, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ + + +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= PRIME_MX2; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); + + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; + + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } +} + +/* + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} + +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + +#if XXH_SIZE_OPT >= 1 + { + /* Smaller, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed); + } while (i-- != 0); + } +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); +#endif + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { XXH128_hash_t acc; + unsigned i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + /* + * We set as `i` as offset + 32. We do this so that unchanged + * `len` can be used as upper bound. This reaches a sweet spot + * where both x86 and aarch64 get simple agen and good codegen + * for the loop. + */ + for (i = 32; i < 160; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + i - 32, + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + /* + * NB: `i <= len` will duplicate the last 32-bytes if + * len % 32 was zero. This is an unfortunate necessity to keep + * the hash result stable. + */ + for (i=160; i <= len; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + XXH3_MIDSIZE_STARTOFFSET + i - 160, + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + (XXH64_hash_t)0 - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + secretSize + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; + } +} + +/* + * It's important for performance that XXH3_hashLong() is not inlined. + */ +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * It's important for performance to pass @p secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. + * + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate, XXH3_scrambleAcc); +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} + + +/* === Public XXH128 API === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_withSeed(input, len, seed); +} + + +/* === XXH3 128-bit streaming === */ +#ifndef XXH_NO_STREAM +/* + * All initialization and update functions are identical to 64-bit streaming variant. + * The only difference is the finalization routine. + */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + return XXH3_64bits_reset(statePtr); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSeed(statePtr, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_64bits_update(state, input, len); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } + } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ +/* 128-bit utility functions */ + +#include /* memcmp, memcpy */ + +/* return : 1 is equal, 0 if different */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); +} + +/* This prototype is compatible with stdlib's qsort(). + * @return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2) +{ + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} + + +/*====== Canonical representation ======*/ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src) +{ + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} + + + +/* ========================================== + * Secret generators + * ========================================== + */ +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) +{ + XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); + XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize) +{ +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(secretBuffer != NULL); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); +#else + /* production mode, assert() are disabled */ + if (secretBuffer == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; +#endif + + if (customSeedSize == 0) { + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + } +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(customSeed != NULL); +#else + if (customSeed == NULL) return XXH_ERROR; +#endif + + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { size_t pos = 0; + while (pos < secretSize) { + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char*)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + } } + + { size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n=0; n