diff --git a/.github/actions/publish-dotslash/action.yml b/.github/actions/publish-dotslash/action.yml new file mode 100644 index 0000000000000000000000000000000000000000..1ca3a1addc7618ae584666b811d428da0c2e17d2 --- /dev/null +++ b/.github/actions/publish-dotslash/action.yml @@ -0,0 +1,52 @@ +name: Publish DotSlash manifests +description: Run the pinned upstream publisher directly on an Ubuntu runner. + +inputs: + tag: + description: Release tag to publish manifests for. + required: true + configs: + description: Newline-separated paths to DotSlash configuration files. + required: true + +runs: + using: composite + steps: + - name: Checkout DotSlash publisher + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: facebook/dotslash-publish-release + ref: 9c9ec027515c34db9282a09a25a9cab5880b2c52 # v2 + path: .dotslash-publish-release + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + version: "0.12.13" + checksum: "745765a3b6e360ad76743599ae5c42e9278c7edf8bbff9fc76d05bf2623a04dd" + github-token: "" + enable-cache: false + + - name: Publish DotSlash manifests + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + RELEASE_TAG: ${{ inputs.tag }} + DOTSLASH_CONFIGS: ${{ inputs.configs }} + run: | + set -euo pipefail + + # Ubuntu runners already provide Python and gh. Avoid rebuilding the + # upstream Docker image and downloading its apt dependencies per release. + uv venv --python "$(command -v python3)" "${RUNNER_TEMP}/dotslash-publish-release-venv" + python="${RUNNER_TEMP}/dotslash-publish-release-venv/bin/python" + uv pip install --python "$python" \ + --require-hashes --only-binary=:all: -r "${ACTION_PATH}/requirements.txt" + + while IFS= read -r config; do + if [[ -n "$config" ]]; then + "$python" .dotslash-publish-release/process_config.py \ + --config "$config" --tag "$RELEASE_TAG" --upload + fi + done <<< "$DOTSLASH_CONFIGS" diff --git a/.github/actions/publish-dotslash/requirements.txt b/.github/actions/publish-dotslash/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..9a911878b7ea5e54db3a898ec3d7bd2413cc4f58 --- /dev/null +++ b/.github/actions/publish-dotslash/requirements.txt @@ -0,0 +1,111 @@ +# Regenerate: uv pip compile --generate-hashes --no-header --no-annotate requirements.txt +blake3==1.0.8 \ + --hash=sha256:003f61e8c41dd9931edddf1cc6a1bb680fb2ac0ad15493ef4a1df9adc59ce9df \ + --hash=sha256:0171e85d56dec1219abdae5f49a0ed12cb3f86a454c29160a64fd8a8166bba37 \ + --hash=sha256:04b95bcd6c1b61480923adc571ffab377618d170f528f6cf4f6a6fff94e3c309 \ + --hash=sha256:0c492630d384fc4400e9860b16e09e0dbe79e960e95b237b9a1ea3f95ea3125d \ + --hash=sha256:0eab3318ec02f8e16fe549244791ace2ada2c259332f0c77ab22cf94dfff7130 \ + --hash=sha256:119829e13eca177f5626c43f753acc4813eb8ae414c2d70ef7571078177163c7 \ + --hash=sha256:1257be19f2d381c868a34cc822fc7f12f817ddc49681b6d1a2790bfbda1a9865 \ + --hash=sha256:19fc6f2b7edab8acff6895fc6e38c19bd79f4c089e21153020c75dfc7397d52d \ + --hash=sha256:1af0e5a29aa56d4fba904452ae784740997440afd477a15e583c38338e641f41 \ + --hash=sha256:23e181318f5a62e4e6ce7be0875b4f65543abe47fe48cc4f4e9ab8f2bf9e01c5 \ + --hash=sha256:269f255b110840e52b6ce9db02217e39660ebad3e34ddd5bca8b8d378a77e4e1 \ + --hash=sha256:2bacf9757e93ac376c612dc0d45931a591ea0caf5583ab93d99dd32a2905ec43 \ + --hash=sha256:2c33dac2c6112bc23f961a7ca305c7e34702c8177040eb98d0389d13a347b9e1 \ + --hash=sha256:2d78f06f3fb838b34c330e2987090376145cbe5944d8608a0c4779c779618f7b \ + --hash=sha256:3168c457255b5d2a2fc356ba696996fcaff5d38284f968210d54376312107662 \ + --hash=sha256:38209b10482c97e151681ea3e91cc7141f56adbbf4820a7d701a923124b41e6a \ + --hash=sha256:3b5573b052777142b2cecc453d022c3f21aa4aba75011258410bb98f41c1a727 \ + --hash=sha256:3bafdec95dfffa3f6571e529644744e280337df15ddd9728f224ba70c5779b23 \ + --hash=sha256:3cec94ed5676821cf371e9c9d25a41b4f3ebdb5724719b31b2749653b7cc1dfa \ + --hash=sha256:3fb83532f7456ddeb68dae1b36e1f7c52f9cb72852ac01159bbcb1a12b0f8be0 \ + --hash=sha256:411f61cb817eb25547b91944301ccfa624ccdc69d70822866d8ad0b0e9e3ca64 \ + --hash=sha256:421b99cdf1ff2d1bf703bc56c454f4b286fce68454dd8711abbcb5a0df90c19a \ + --hash=sha256:444215a1e5201f8fa4e5c7352e938a7070cd33d66aeb1dd9b1103a64b6920f9e \ + --hash=sha256:44c2815d4616fad7e2d757d121c0a11780f70ffc817547b3059b5c7e224031a7 \ + --hash=sha256:46dc20976bd6c235959ef0246ec73420d1063c3da2839a9c87ca395cf1fd7943 \ + --hash=sha256:497ef8096ec4ac1ffba9a66152cee3992337cebf8ea434331d8fd9ce5423d227 \ + --hash=sha256:4b0543c57eb9d6dac9d4bced63e9f7f7b546886ac04cec8da3c3d9c8f30cbbb7 \ + --hash=sha256:4bacaae75e98dee3b7da6c5ee3b81ee21a3352dd2477d6f1d1dbfd38cdbf158a \ + --hash=sha256:4e02a3c499e35bf51fc15b2738aca1a76410804c877bcd914752cac4f71f052a \ + --hash=sha256:4f54cff7f15d91dc78a63a2dd02a3dccdc932946f271e2adb4130e0b4cf608ba \ + --hash=sha256:504d1399b7fb91dfe5c25722d2807990493185faa1917456455480c36867adb5 \ + --hash=sha256:506a47897a11ebe8f3cdeb52f1365d6a2f83959e98ccb0c830f8f73277d4d358 \ + --hash=sha256:511133bab85ff60ed143424ce484d08c60894ff7323f685d7a6095f43f0c85c3 \ + --hash=sha256:513cc7f0f5a7c035812604c2c852a0c1468311345573de647e310aca4ab165ba \ + --hash=sha256:53304b6df3f8d826850c1d977d9a69edbf3e48476b95f7cb6175ec650856a3c5 \ + --hash=sha256:58ce8d45a5bb5326482de72ea1969a378634236186a970fef63058a5b7b8b435 \ + --hash=sha256:5eb25bca3cee2e0dd746a214784fb36be6a43640c01c55b6b4e26196e72d076c \ + --hash=sha256:63379be58438878eeb76ebe4f0efbeaabf42b79f2cff23b6126b7991588ced67 \ + --hash=sha256:66ca28a673025c40db3eba21a9cac52f559f83637efa675b3f6bd8683f0415f3 \ + --hash=sha256:67d9c42c42eb1c7aedcf901591c743266009fcf48babf6d6f8450f567cb94a84 \ + --hash=sha256:67f1bc11bf59464ef092488c707b13dd4e872db36e25c453dfb6e0c7498df9f1 \ + --hash=sha256:6a6eafc29e4f478d365a87d2f25782a521870c8514bb43734ac85ae9be71caf7 \ + --hash=sha256:6ae7754c7d96e92a70a52e07c732d594cf9924d780f49fffd3a1e9235e0f5ba7 \ + --hash=sha256:725c52c4d393c7bd1a10682df322d480734002a1389b320366c660568708846b \ + --hash=sha256:73e0b4fa25f6e3078526a592fb38fca85ef204fd02eced6731e1cdd9396552d4 \ + --hash=sha256:78731ce7fca46f776ae45fb5271a2a76c4a92c9687dd4337e84b2ae9a174b28f \ + --hash=sha256:7adbbee5dd0c302218eb8acdfd82b7006930eb5798f56f79f9cca89f6f192662 \ + --hash=sha256:7fcfe81b3ae3fb5d2e88be0d3259603ff95f0d5ed69f655c28fdaef31e49a470 \ + --hash=sha256:83347ef28d774e84442c71fc5aa8817498753d47831a533536ead1a7ccd44a41 \ + --hash=sha256:83605dbf43f581d8b7175b7f3bfe5388bad5a7c6ac175c9c11d669da31133f4b \ + --hash=sha256:859cd57bac097a2cd63cb36d64c2f6f16c9edece5590f929e70157478e46dc9e \ + --hash=sha256:88d527c247f9609dc1d45a08fd243e39f0d5300d54c57e048de24d4fa9240ebb \ + --hash=sha256:8956bb9aec47b6c37ccce935a943588f1f5e6e2e85d43bb7cb76a574238f8a9b \ + --hash=sha256:8f2ef8527a7a8afd99b16997d015851ccc0fe2a409082cebb980af2554e5c74c \ + --hash=sha256:9456c829601d72852d8ba0af8dae0610f7def1d59f5942efde1e2ef93e8a8b57 \ + --hash=sha256:94b2d4922c75174083988b8b0d2c78e72be7376ac59ab19c3e5d17a7c9c1cc0f \ + --hash=sha256:97c076d58ee37eb5b2d8d91bb9db59c5a008fd59c71845dc57fe438aeeabaf10 \ + --hash=sha256:98915260c378d4c45088f1e227b053834ed95aeca89336184f192489ad0dd5d4 \ + --hash=sha256:9a8946cb6b1d2b2096daaaa89856f39887bce2b78503fa31b78173e3a86fa281 \ + --hash=sha256:9ab5998e2abd9754819753bc2f1cf3edf82d95402bff46aeef45ed392a5468bf \ + --hash=sha256:9c9fbdacfdeb68f7ca53bb5a7a5a593ec996eaf21155ad5b08d35e6f97e60877 \ + --hash=sha256:9e1d70bf76c02846d0868a3d413eb6c430b76a315e12f1b2e59b5cf56c1f62a3 \ + --hash=sha256:a2336d5b2a801a7256da21150348f41610a6c21dae885a3acb1ebbd7333d88d8 \ + --hash=sha256:a25db3d36b55f5ed6a86470155cc749fc9c5b91c949b8d14f48658f9d960d9ec \ + --hash=sha256:a33b9a1fb6d1d559a8e0d04b041e99419a6bb771311c774f6ff57ed7119c70ed \ + --hash=sha256:a47939f04b89c5c6ff1e51e883e5efab1ea1bf01a02f4d208d216dddd63d0dd8 \ + --hash=sha256:a585357d5d8774aad9ffc12435de457f9e35cde55e0dc8bc43ab590a6929e59f \ + --hash=sha256:a5b9196a4768e5475475bdf2010dc76b2792f95805731a645196d180a7e12570 \ + --hash=sha256:ab4e1dea4fa857944944db78e8f20d99ee2e16b2dea5a14f514fb0607753ac83 \ + --hash=sha256:adccc3a139207e02bb7d7bb0715fe0b87069685aad5f3afff820b2f829467904 \ + --hash=sha256:add39ffa9e5074264f12ab409cdf059bc967817644b455e3fad46139001cc108 \ + --hash=sha256:af18817da1eb5bd65d068fb9c18235edc328a45b09025346fa65fe52b1b0c9b6 \ + --hash=sha256:af394b50c6aa0b1b957a99453d1ee440ef67cd2d1b5669c731647dc723de8a3a \ + --hash=sha256:b201961c6dcc7123e69fff6019298d9e4394c7e585a300e999664bca8070d83c \ + --hash=sha256:b37f63c5952b32b5064096141b394ed17569d318637a06a2cae9bfedd852f0ef \ + --hash=sha256:b4d672c24dc15ec617d212a338a4ca14b449829b6072d09c96c63b6e6b621aed \ + --hash=sha256:bcb04966537777af56c1f399b35525aa70a1225816e121ff95071c33c0f7abca \ + --hash=sha256:c2682ba10fb3b36fe8a6823be50910f1145a8a5158bc555145fadcc4afb984c3 \ + --hash=sha256:c445eff665d21c3b3b44f864f849a2225b1164c08654beb23224a02f087b7ff1 \ + --hash=sha256:c5bfe1a1ada3ed2c715f692ce1a3b221d7f6b4d5e6001d24774ef07de013192c \ + --hash=sha256:c63ece266a43014cf29e772a82857cd8e90315ae3ed53e3c5204851596edd5f2 \ + --hash=sha256:c65e373c8b47174b969ee61a89ee56922f722972eb650192845c8546df8d9db9 \ + --hash=sha256:c7780666dc6be809b49442d6d5ce06fdbe33024a87560b58471103ec17644682 \ + --hash=sha256:c84af132aa09abeadf9a0118c8fb26f4528f3f42c10ef8be0fcf31c478774ec4 \ + --hash=sha256:c9ab5ef61aaa6df2f769548a071fddcacbb0f56102a9c48f40878b10ae1a2657 \ + --hash=sha256:ca8bc7e816370118262da11783e1fdf8992f250789ae08e8182fb7c26e14ded0 \ + --hash=sha256:d17eb6382634b3a5bc0c0e0454d5265b0becaeeadb6801ed25150b39a999d0cc \ + --hash=sha256:d2c3151955efb09ba58cd3e1263521e15e9e3866a40d6bd3556d86fc968e8f95 \ + --hash=sha256:d3fe26f145fcb82931d1820b55c0279f72f8f8e49450dd9d74efbfd409b28423 \ + --hash=sha256:d8da4233984d51471bd4e4366feda1d90d781e712e0a504ea54b1f2b3577557b \ + --hash=sha256:db54946792d2b8c6fa4be73e6e334519f13c1b52e7ff346b3e2ec8ad3eb59401 \ + --hash=sha256:dd03ff08d1b6e4fdda1cd03826f971ae8966ef6f683a8c68aa27fb21904b5aa9 \ + --hash=sha256:ddfc59b0176fb31168f08d5dd536e69b1f4f13b5a0f4b0c3be1003efd47f9308 \ + --hash=sha256:e0fee93d5adcd44378b008c147e84f181f23715307a64f7b3db432394bbfce8b \ + --hash=sha256:e171b169cb7ea618e362a4dddb7a4d4c173bbc08b9ba41ea3086dd1265530d4f \ + --hash=sha256:e2df12f295f95a804338bd300e8fad4a6f54fd49bd4d9c5893855a230b5188a8 \ + --hash=sha256:e4072196547484c95a5a09adbb952e9bb501949f03f9e2a85e7249ef85faaba8 \ + --hash=sha256:e5122a61b3b004bbbd979bdf83a3aaab432da3e2a842d7ddf1c273f2503b4884 \ + --hash=sha256:e5b5da177d62cc4b7edf0cea08fe4dec960c9ac27f916131efa890a01f747b93 \ + --hash=sha256:e6cb2316949c614e64f6b7796b9eca28b1b57406d27debc99398a746e9b426c7 \ + --hash=sha256:e7e12a777f6b798eb8d06f875d6e108e3008bd658d274d8c676dcf98e0f10537 \ + --hash=sha256:e8ae3689f0c7bfa6ce6ae45cab110e4c3442125c4c23b28f1f097856de26e4d1 \ + --hash=sha256:ed972ebd553c0c25363459e9fc71a38c045d8419e365b59acd8cd791eff13981 \ + --hash=sha256:ef153c5860d5bf1cc71aece69b28097d2a392913eb323d6b52555c875d0439fc \ + --hash=sha256:efb1ab89fa47b5acb36f1530c4eb83ae20aaec9ac17808ff014b2650fbc3c7ba \ + --hash=sha256:f13c25c215c284adca0c07cffae1518cafcac3900f1d45fff51c7e6ab98efec1 \ + --hash=sha256:fc9fc4a0a1a13991b23a589fb6fbb8290bd94a72c130e69e27c9769612516424 \ + --hash=sha256:fe1b02ab49bfd969ef50b9f17482a2011c77536654af21807ba5c2674e0bb2a0 \ + --hash=sha256:fe61d52da1d102aabb0e315d3fc613c11c103376aa0a8510aeee159939612f76 diff --git a/.github/actions/run-argument-comment-lint/action.yml b/.github/actions/run-argument-comment-lint/action.yml new file mode 100644 index 0000000000000000000000000000000000000000..31c752e86e3c0f4fae53a1b2e69d3bceb1c21afb --- /dev/null +++ b/.github/actions/run-argument-comment-lint/action.yml @@ -0,0 +1,53 @@ +name: Run argument comment lint +description: Run argument-comment-lint on codex-rs via Bazel. + +inputs: + target: + description: Runner target passed to setup-bazel-ci. + required: true + buildbuddy-api-key: + description: BuildBuddy API key used by Bazel CI. + required: false + default: "" + +runs: + using: composite + steps: + - uses: ./.github/actions/setup-bazel-ci + with: + target: ${{ inputs.target }} + + - name: Install Linux sandbox build dependencies + if: ${{ runner.os == 'Linux' }} + shell: bash + run: | + sudo DEBIAN_FRONTEND=noninteractive apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends pkg-config libcap-dev + + - name: Run argument comment lint on codex-rs via Bazel + if: ${{ runner.os != 'Windows' }} + env: + BUILDBUDDY_API_KEY: ${{ inputs.buildbuddy-api-key }} + shell: bash + run: | + bazel_targets="$(./tools/argument-comment-lint/list-bazel-targets.sh)" + ./.github/scripts/run-bazel-ci.sh \ + -- \ + build \ + --config=argument-comment-lint \ + --keep_going \ + --build_metadata=COMMIT_SHA=${GITHUB_SHA} \ + -- \ + ${bazel_targets} + + - name: Run argument comment lint on codex-rs via Bazel + if: ${{ runner.os == 'Windows' }} + env: + BUILDBUDDY_API_KEY: ${{ inputs.buildbuddy-api-key }} + shell: bash + run: | + ./.github/scripts/run-argument-comment-lint-bazel.sh \ + --config=argument-comment-lint \ + --platforms=//:local_windows \ + --keep_going \ + --build_metadata=COMMIT_SHA=${GITHUB_SHA} diff --git a/.github/actions/setup-akv-pkcs11-codesigning/action.yaml b/.github/actions/setup-akv-pkcs11-codesigning/action.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d79cab42efc1844dca39f87a05ed2a29ad7cbf6 --- /dev/null +++ b/.github/actions/setup-akv-pkcs11-codesigning/action.yaml @@ -0,0 +1,349 @@ +name: Set up AKV PKCS11 code signing +description: Download prebuilt rcodesign and Azure Key Vault PKCS11 provider artifacts, then export macOS signing environment. + +inputs: + setup-mode: + description: signing configures Azure and exports signing env vars; tools-only only downloads signing tools. + required: false + default: signing + rcodesign-blob-uri: + description: Azure Blob URI for the prebuilt Linux/amd64 rcodesign binary. + required: true + rcodesign-sha256: + description: Expected SHA-256 digest for the prebuilt rcodesign binary. + required: true + akv-pkcs11-library-blob-uri: + description: Azure Blob URI for the prebuilt Linux/amd64 AKV PKCS11 provider library. + required: true + akv-pkcs11-library-sha256: + description: Expected SHA-256 digest for the prebuilt AKV PKCS11 provider library. + required: true + azure-client-id: + description: GitHub OIDC client ID for the Azure signer application. + required: true + azure-tenant-id: + description: Azure tenant ID for the signer application. + required: true + azure-subscription-id: + description: Azure subscription ID that owns the signing vault. + required: true + key-vault-name: + description: Azure Key Vault name containing the certificate-backed signing key. + required: true + key-name: + description: Key Vault certificate/key name used as the PKCS11 key label. + required: true + key-version: + description: Optional Key Vault key version to pin while signing. + required: false + default: "" + certificate-sha256: + description: Optional expected SHA-256 fingerprint for the downloaded public certificate. + required: false + default: "" + +outputs: + pkcs11-library: + description: Path to the downloaded AKV PKCS11 provider library. + value: ${{ steps.paths.outputs.pkcs11_library }} + signing-certificate-pem: + description: Path to the downloaded public signing certificate. + value: ${{ steps.paths.outputs.signing_certificate_pem }} + rcodesign: + description: Path to the downloaded rcodesign binary. + value: ${{ steps.paths.outputs.rcodesign }} + +runs: + using: composite + steps: + - name: Validate pinned signing artifacts + shell: bash + env: + SETUP_MODE: ${{ inputs.setup-mode }} + RCODESIGN_BLOB_URI: ${{ inputs.rcodesign-blob-uri }} + RCODESIGN_SHA256: ${{ inputs.rcodesign-sha256 }} + AKV_PKCS11_LIBRARY_BLOB_URI: ${{ inputs.akv-pkcs11-library-blob-uri }} + AKV_PKCS11_LIBRARY_SHA256: ${{ inputs.akv-pkcs11-library-sha256 }} + KEY_VAULT_NAME: ${{ inputs.key-vault-name }} + KEY_NAME: ${{ inputs.key-name }} + run: | + set -euo pipefail + + case "$SETUP_MODE" in + signing|tools-only) + ;; + *) + echo "setup-mode must be 'signing' or 'tools-only', got '$SETUP_MODE'." >&2 + exit 1 + ;; + esac + + for variable_name in RCODESIGN_SHA256 AKV_PKCS11_LIBRARY_SHA256; do + value="${!variable_name}" + if [[ ! "$value" =~ ^[0-9a-f]{64}$ ]]; then + echo "$variable_name must be a lowercase SHA-256 digest." >&2 + exit 1 + fi + done + + for variable_name in RCODESIGN_BLOB_URI AKV_PKCS11_LIBRARY_BLOB_URI; do + value="${!variable_name}" + if [[ ! "$value" =~ ^az://[^/]+/[^/]+/.+ ]]; then + echo "$variable_name must use az:////." >&2 + exit 1 + fi + done + + if [[ "$SETUP_MODE" == "signing" ]]; then + for variable_name in \ + KEY_VAULT_NAME \ + KEY_NAME; do + if [[ -z "${!variable_name}" ]]; then + echo "$variable_name is required for AKV PKCS11 signing." >&2 + exit 1 + fi + done + fi + + - name: Resolve signing tool paths + id: paths + shell: bash + run: | + set -euo pipefail + + if [[ "${RUNNER_OS}" != "Linux" ]]; then + echo "Prebuilt AKV PKCS11 signing tools are only vendored for Linux runners, got ${RUNNER_OS}." >&2 + exit 1 + fi + + if [[ "${RUNNER_ARCH}" != "X64" && "${RUNNER_ARCH}" != "AMD64" ]]; then + echo "Prebuilt AKV PKCS11 signing tools are only vendored for amd64 runners, got ${RUNNER_ARCH}." >&2 + exit 1 + fi + + provider_root="${RUNNER_TEMP}/akv-pkcs11-provider" + rcodesign_root="${RUNNER_TEMP}/rcodesign-root" + signing_certificate_pem="${RUNNER_TEMP}/akv-signing-cert.pem" + library_name="libakv_pkcs_11.so" + + mkdir -p "$provider_root" "$rcodesign_root/bin" + + { + echo "pkcs11_library=$provider_root/$library_name" + echo "pkcs11_manifest=$provider_root/akv-pkcs11-provider.manifest" + echo "rcodesign_root=$rcodesign_root" + echo "rcodesign=$rcodesign_root/bin/rcodesign" + echo "signing_certificate_pem=$signing_certificate_pem" + } >> "$GITHUB_OUTPUT" + + - name: Validate Azure credentials for private signing artifacts + shell: bash + env: + AZURE_CLIENT_ID: ${{ inputs.azure-client-id }} + AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }} + AZURE_SUBSCRIPTION_ID: ${{ inputs.azure-subscription-id }} + run: | + set -euo pipefail + + for variable_name in AZURE_CLIENT_ID AZURE_TENANT_ID AZURE_SUBSCRIPTION_ID; do + if [[ -z "${!variable_name}" ]]; then + echo "$variable_name is required for private AKV PKCS11 signing artifacts." >&2 + exit 1 + fi + done + + - name: Log in to Azure with GitHub OIDC + uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3.0.0 + with: + client-id: ${{ inputs.azure-client-id }} + tenant-id: ${{ inputs.azure-tenant-id }} + subscription-id: ${{ inputs.azure-subscription-id }} + + - name: Install prebuilt signing tools + shell: bash + env: + RCODESIGN_BLOB_URI: ${{ inputs.rcodesign-blob-uri }} + RCODESIGN_SHA256: ${{ inputs.rcodesign-sha256 }} + RCODESIGN: ${{ steps.paths.outputs.rcodesign }} + AKV_PKCS11_LIBRARY_BLOB_URI: ${{ inputs.akv-pkcs11-library-blob-uri }} + AKV_PKCS11_LIBRARY_SHA256: ${{ inputs.akv-pkcs11-library-sha256 }} + PKCS11_LIBRARY: ${{ steps.paths.outputs.pkcs11_library }} + PKCS11_MANIFEST: ${{ steps.paths.outputs.pkcs11_manifest }} + run: | + set -euo pipefail + + download_az_blob_uri() { + local uri="$1" + local destination="$2" + local rest account container blob + + rest="${uri#az://}" + account="${rest%%/*}" + rest="${rest#*/}" + container="${rest%%/*}" + blob="${rest#*/}" + + if [[ -z "$account" || -z "$container" || -z "$blob" || "$blob" == "$rest" ]]; then + echo "Invalid Azure Blob URI. Expected az:////." >&2 + exit 1 + fi + + mkdir -p "$(dirname "$destination")" + rm -f "$destination" + if ! az storage blob download \ + --account-name "$account" \ + --container-name "$container" \ + --name "$blob" \ + --file "$destination" \ + --auth-mode login \ + --only-show-errors \ + >/dev/null 2>&1; then + echo "Failed to download a private signing artifact from Azure Blob Storage." >&2 + exit 1 + fi + } + + verify_sha256() { + local path="$1" + local expected="$2" + local actual + + actual="$(shasum -a 256 "$path" | awk '{ print $1 }')" + if [[ "$actual" != "$expected" ]]; then + echo "SHA-256 verification failed for '$path'." >&2 + exit 1 + fi + } + + echo "Downloading prebuilt rcodesign." + download_az_blob_uri "$RCODESIGN_BLOB_URI" "$RCODESIGN" + verify_sha256 "$RCODESIGN" "$RCODESIGN_SHA256" + chmod 0755 "$RCODESIGN" + + echo "Downloading prebuilt AKV PKCS11 provider." + download_az_blob_uri "$AKV_PKCS11_LIBRARY_BLOB_URI" "$PKCS11_LIBRARY" + verify_sha256 "$PKCS11_LIBRARY" "$AKV_PKCS11_LIBRARY_SHA256" + chmod 0644 "$PKCS11_LIBRARY" + + { + echo "runner_os=$RUNNER_OS" + echo "runner_arch=$RUNNER_ARCH" + echo "library_name=$(basename "$PKCS11_LIBRARY")" + } > "$PKCS11_MANIFEST" + + - name: Verify downloaded signing tools + shell: bash + env: + RCODESIGN: ${{ steps.paths.outputs.rcodesign }} + RCODESIGN_SHA256: ${{ inputs.rcodesign-sha256 }} + PKCS11_LIBRARY: ${{ steps.paths.outputs.pkcs11_library }} + AKV_PKCS11_LIBRARY_SHA256: ${{ inputs.akv-pkcs11-library-sha256 }} + PKCS11_MANIFEST: ${{ steps.paths.outputs.pkcs11_manifest }} + run: | + set -euo pipefail + + verify_sha256() { + local path="$1" + local expected="$2" + local actual + + actual="$(shasum -a 256 "$path" | awk '{ print $1 }')" + if [[ "$actual" != "$expected" ]]; then + echo "SHA-256 verification failed for '$path'." >&2 + exit 1 + fi + } + + if [[ ! -x "$RCODESIGN" ]]; then + echo "rcodesign is missing or not executable at '$RCODESIGN'." >&2 + exit 1 + fi + + if [[ ! -f "$PKCS11_LIBRARY" ]]; then + echo "AKV PKCS11 provider library is missing at '$PKCS11_LIBRARY'." >&2 + exit 1 + fi + + verify_sha256 "$RCODESIGN" "$RCODESIGN_SHA256" + verify_sha256 "$PKCS11_LIBRARY" "$AKV_PKCS11_LIBRARY_SHA256" + + "$RCODESIGN" --version + "$RCODESIGN" notarize --help > /dev/null + + if [[ -f "$PKCS11_MANIFEST" ]]; then + echo "AKV PKCS11 provider artifact manifest is present." + else + echo "AKV PKCS11 provider artifact manifest is absent." >&2 + exit 1 + fi + + - name: Download signing certificate from Key Vault + if: ${{ inputs.setup-mode == 'signing' }} + shell: bash + env: + KEY_VAULT_NAME: ${{ inputs.key-vault-name }} + KEY_NAME: ${{ inputs.key-name }} + KEY_VERSION: ${{ inputs.key-version }} + CERTIFICATE_SHA256: ${{ inputs.certificate-sha256 }} + SIGNING_CERTIFICATE_PEM: ${{ steps.paths.outputs.signing_certificate_pem }} + run: | + set -euo pipefail + + certificate_version_args=() + if [[ -n "$KEY_VERSION" ]]; then + certificate_version_args+=(--version "$KEY_VERSION") + fi + + if ! az keyvault certificate download \ + --vault-name "$KEY_VAULT_NAME" \ + --name "$KEY_NAME" \ + "${certificate_version_args[@]}" \ + --file "$SIGNING_CERTIFICATE_PEM" \ + --encoding PEM \ + --only-show-errors \ + >/dev/null 2>&1; then + echo "Failed to download the public signing certificate from Azure Key Vault." >&2 + exit 1 + fi + + if [[ -n "$CERTIFICATE_SHA256" ]]; then + actual_sha256="$( + openssl x509 -in "$SIGNING_CERTIFICATE_PEM" -noout -fingerprint -sha256 | + awk -F= '{ print toupper($2) }' | + tr -d ':\r\n' + )" + expected_sha256="$(printf '%s' "$CERTIFICATE_SHA256" | tr '[:lower:]' '[:upper:]' | tr -d ':\r\n ')" + if [[ "$actual_sha256" != "$expected_sha256" ]]; then + echo "Downloaded signing certificate SHA-256 did not match the expected fingerprint." >&2 + exit 1 + fi + fi + + - name: Export AKV PKCS11 signing environment + if: ${{ inputs.setup-mode == 'signing' }} + shell: bash + env: + RCODESIGN_ROOT: ${{ steps.paths.outputs.rcodesign_root }} + PKCS11_LIBRARY: ${{ steps.paths.outputs.pkcs11_library }} + SIGNING_CERTIFICATE_PEM: ${{ steps.paths.outputs.signing_certificate_pem }} + KEY_VAULT_NAME: ${{ inputs.key-vault-name }} + KEY_NAME: ${{ inputs.key-name }} + KEY_VERSION: ${{ inputs.key-version }} + run: | + set -euo pipefail + + { + echo "$RCODESIGN_ROOT/bin" + } >> "$GITHUB_PATH" + + { + echo "OAI_CODESIGN_BACKEND=akv-pkcs11" + echo "OAI_AKV_PKCS11_LIBRARY=$PKCS11_LIBRARY" + echo "OAI_AKV_SIGNING_CERTIFICATE_PEM=$SIGNING_CERTIFICATE_PEM" + echo "OAI_AKV_KEY_LABEL=$KEY_NAME" + echo "AZURE_CREDENTIAL_KIND=azurecli" + echo "AZURE_KEYVAULT_NAME=$KEY_VAULT_NAME" + if [[ -n "$KEY_VERSION" ]]; then + echo "AZURE_KEYVAULT_KEY_VERSION=$KEY_VERSION" + fi + } >> "$GITHUB_ENV" diff --git a/.github/actions/setup-ci/action.yml b/.github/actions/setup-ci/action.yml new file mode 100644 index 0000000000000000000000000000000000000000..e0990d7a825b1210fbad6141ead0d3a9590f88ce --- /dev/null +++ b/.github/actions/setup-ci/action.yml @@ -0,0 +1,107 @@ +name: setup-ci +description: Prepare common tools and environment shared by CI jobs. +outputs: + bazel-output-base: + description: Filesystem path used for Bazel's output base. + value: ${{ steps.configure_ci_build_paths.outputs.bazel-output-base }} + cargo-target-dir: + description: Filesystem path used for Cargo's target directory. + value: ${{ steps.configure_ci_build_paths.outputs.cargo-target-dir }} + +runs: + using: composite + steps: + # setup-ci expects either this step or the Unix fallback below to define + # CI_BUILD_ROOT. Windows uses a Dev Drive because Cargo and Bazel + # spend significant time reading and writing build/cache trees. + - name: Configure Dev Drive (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: ./.github/scripts/setup-dev-drive.ps1 + + - name: Configure CI build root (Unix) + if: runner.os != 'Windows' + shell: bash + run: echo "CI_BUILD_ROOT=$HOME/.cache/codex-ci" >> "$GITHUB_ENV" + + - name: Enable unprivileged user namespaces (Linux) + if: runner.os == 'Linux' + shell: bash + run: | + sudo sysctl -w kernel.unprivileged_userns_clone=1 + if sudo sysctl -a 2>/dev/null | grep -q '^kernel.apparmor_restrict_unprivileged_userns'; then + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + fi + + - name: Configure CI build paths + id: configure_ci_build_paths + shell: bash + run: | + set -euo pipefail + # setup-bazel passes output_base explicitly, so keep both it and the + # user root under the shared build root. Keep these directory names tiny + # on every platform so Windows Bazel paths do not overflow argv or + # confuse test MANIFEST handling. + bazel_output_base="$CI_BUILD_ROOT/o" + bazel_output_user_root="$CI_BUILD_ROOT/b" + bazel_repository_cache="$CI_BUILD_ROOT/bazel-repository-cache" + bazel_repo_contents_cache="$CI_BUILD_ROOT/bazel-repo-contents-cache-$GITHUB_RUN_ID-$GITHUB_JOB" + cargo_target_dir="$CI_BUILD_ROOT/cargo-target" + tmp="$CI_BUILD_ROOT/tmp" + + build_dirs=( + "$bazel_output_base" + "$bazel_output_user_root" + "$bazel_repository_cache" + "$bazel_repo_contents_cache" + "$cargo_target_dir" + "$tmp" + ) + mkdir -p "${build_dirs[@]}" + echo "bazel-output-base=$bazel_output_base" >> "$GITHUB_OUTPUT" + echo "cargo-target-dir=$cargo_target_dir" >> "$GITHUB_OUTPUT" + + { + echo "CODEX_REPO_ROOT=$GITHUB_WORKSPACE" + echo "BAZEL_OUTPUT_BASE=$bazel_output_base" + echo "BAZEL_OUTPUT_USER_ROOT=$bazel_output_user_root" + echo "BAZEL_REPOSITORY_CACHE=$bazel_repository_cache" + echo "BAZEL_REPO_CONTENTS_CACHE=$bazel_repo_contents_cache" + echo "CARGO_TARGET_DIR=$cargo_target_dir" + echo "TEMP=$tmp" + echo "TMP=$tmp" + } >> "$GITHUB_ENV" + + - name: Prefer the Git CLI for Cargo git dependencies + shell: bash + run: echo "CARGO_NET_GIT_FETCH_WITH_CLI=true" >> "$GITHUB_ENV" + + - name: Install DotSlash + uses: facebook/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2 + + - name: Install just + uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2.62.49 + with: + tool: just@1.51.0 + + # Some integration tests spawn DotSlash from a stable system path rather + # than inheriting the action-local PATH entry. + - name: Make DotSlash available in PATH (Unix) + if: runner.os != 'Windows' + shell: bash + run: | + if [[ -w /usr/local/bin ]]; then + cp "$(which dotslash)" /usr/local/bin + else + sudo cp "$(which dotslash)" /usr/local/bin + fi + + - name: Make DotSlash available in PATH (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: Copy-Item (Get-Command dotslash).Source -Destination "$env:LOCALAPPDATA\Microsoft\WindowsApps\dotslash.exe" + + - name: Enable Git long paths (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: git config --global core.longpaths true diff --git a/.github/actions/setup-msvc-env/action.yml b/.github/actions/setup-msvc-env/action.yml new file mode 100644 index 0000000000000000000000000000000000000000..ab987a052bdab0d1d6b516eca6dac146c44292d3 --- /dev/null +++ b/.github/actions/setup-msvc-env/action.yml @@ -0,0 +1,20 @@ +name: setup-msvc-env +description: Expose an MSVC developer environment for the requested Windows target. +inputs: + target: + description: Rust target triple that will be built on this Windows runner. + required: true + host-arch: + description: Optional Visual Studio host architecture override. + required: false + default: "" + +runs: + using: composite + steps: + - name: Expose MSVC SDK environment + shell: pwsh + env: + HOST_ARCH: ${{ inputs.host-arch }} + TARGET: ${{ inputs.target }} + run: '& "$env:GITHUB_ACTION_PATH/setup-msvc-env.ps1" -Target "$env:TARGET" -HostArch "$env:HOST_ARCH"' diff --git a/.github/actions/setup-msvc-env/setup-msvc-env.ps1 b/.github/actions/setup-msvc-env/setup-msvc-env.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..e2706d91175bbdd9c9e233bb680f9d82f624625e --- /dev/null +++ b/.github/actions/setup-msvc-env/setup-msvc-env.ps1 @@ -0,0 +1,257 @@ +param( + [Parameter(Mandatory = $true)] + [string]$Target, + + [string]$HostArch = "" +) + +# Cargo can cross-compile the Rust code for Windows ARM64 on a Windows x64 +# runner, but rustup alone does not expose the matching MSVC/UCRT include and +# library paths. Ask Visual Studio for the target-specific developer +# environment, then persist the relevant variables through GITHUB_ENV so the +# later Cargo step sees the same environment as a normal VsDevCmd shell. +switch ($Target) { + "x86_64-pc-windows-msvc" { + $TargetArch = "x64" + $RequiredComponent = "Microsoft.VisualStudio.Component.VC.Tools.x86.x64" + } + "aarch64-pc-windows-msvc" { + $TargetArch = "arm64" + $RequiredComponent = "Microsoft.VisualStudio.Component.VC.Tools.ARM64" + } + default { + throw "Unsupported Windows MSVC target: $Target" + } +} + +# VsDevCmd needs both sides of the cross compile: the architecture of the +# machine running the tools and the architecture of the binaries being linked. +# Infer the host from the runner unless a caller needs to override it. +if (-not $HostArch) { + $HostArch = if ($env:PROCESSOR_ARCHITEW6432 -eq "ARM64" -or $env:PROCESSOR_ARCHITECTURE -eq "ARM64") { + "arm64" + } else { + "x64" + } +} + +$VsWhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" +if (-not (Test-Path $VsWhere)) { + throw "vswhere.exe not found" +} + +# Require the target VC tools component, not merely any Visual Studio install, +# so an x64 archive producer cannot silently link ARM64 tests with the wrong +# SDK/toolchain layout. +$InstallPath = & $VsWhere -latest -products * -requires $RequiredComponent -property installationPath 2>$null +if (-not $InstallPath) { + throw "Could not locate a Visual Studio installation with component $RequiredComponent" +} + +$VsDevCmd = Join-Path $InstallPath "Common7\Tools\VsDevCmd.bat" +if (-not (Test-Path $VsDevCmd)) { + throw "VsDevCmd.bat not found at $VsDevCmd" +} + +$VarsToExport = @( + "INCLUDE", + "LIB", + "LIBPATH", + "PATH", + "UCRTVersion", + "UniversalCRTSdkDir", + "VCINSTALLDIR", + "VCToolsInstallDir", + "WindowsLibPath", + "WindowsSdkBinPath", + "WindowsSdkDir", + "WindowsSDKLibVersion", + "WindowsSDKVersion" +) + +# Run VsDevCmd inside cmd.exe because it is a batch file, then copy just the +# variables Cargo/rustc need into the GitHub Actions environment file. PowerShell +# cannot mutate the parent composite-action environment directly. +$EnvLines = & cmd.exe /c ('"{0}" -no_logo -arch={1} -host_arch={2} >nul && set' -f $VsDevCmd, $TargetArch, $HostArch) +$VcToolsInstallDir = $null +foreach ($Line in $EnvLines) { + if ($Line -notmatch "^(.*?)=(.*)$") { + continue + } + + $Name = $Matches[1] + $Value = $Matches[2] + if ($VarsToExport -contains $Name) { + if ($Name -ieq "Path") { + $Name = "PATH" + } + if ($Name -eq "VCToolsInstallDir") { + $VcToolsInstallDir = $Value + } + "$Name=$Value" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + } +} + +if (-not $VcToolsInstallDir) { + throw "VCToolsInstallDir was not exported by VsDevCmd.bat" +} + +# Prefer Rust's bundled linker when rustup provides one, then Visual Studio's +# LLVM linker, and finally MSVC link.exe. This keeps the cross-compile path close +# to Rust's normal Windows MSVC behavior while still working on runner images +# where one of those linkers is absent. +$Linker = $null +$Rustc = Get-Command rustc -ErrorAction SilentlyContinue +if ($Rustc) { + $Sysroot = (& rustc --print sysroot 2>$null).Trim() + $RustHost = & rustc -vV 2>$null | Select-String "^host: " | ForEach-Object { $_.Line.Substring(6) } + if ($RustHost) { + $RustHost = $RustHost.Trim() + } + if ($Sysroot -and $RustHost) { + $RustLld = Join-Path $Sysroot "lib\rustlib\$RustHost\bin\rust-lld.exe" + if (Test-Path $RustLld) { + $Linker = $RustLld + } + } +} +if (-not $Linker) { + $Linker = Join-Path $InstallPath "VC\Tools\Llvm\x64\bin\lld-link.exe" +} +if (-not (Test-Path $Linker)) { + $Linker = Join-Path $VcToolsInstallDir "bin\Host${HostArch}\${TargetArch}\link.exe" +} +if (-not (Test-Path $Linker)) { + throw "Windows linker not found at $Linker" +} + +# rustc passes `/arm64hazardfree` for ARM64 MSVC links. The lld variants on our +# Windows x64 archive producers reject that flag, including when rustc places it +# inside a response file. Compile a tiny forwarding wrapper that strips only +# that unsupported flag, then delegate every other argument to the real linker. +if ($TargetArch -eq "arm64" -and (Split-Path -Leaf $Linker) -match "lld") { + $WrapperDir = Join-Path $env:RUNNER_TEMP "msvc-lld-wrapper" + New-Item -Path $WrapperDir -ItemType Directory -Force | Out-Null + $WrapperPath = Join-Path $WrapperDir "lld-link-wrapper.exe" + $WrapperSource = @' +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; + +internal static class Program +{ + private static int Main(string[] args) + { + var linker = Environment.GetEnvironmentVariable("MSVC_REAL_LINKER"); + if (string.IsNullOrEmpty(linker)) + { + Console.Error.WriteLine("MSVC_REAL_LINKER is not set"); + return 1; + } + + var startInfo = new ProcessStartInfo(linker) + { + UseShellExecute = false, + }; + var filteredArgs = new List { "-flavor", "link", "/defaultlib:ucrt", "/nodefaultlib:libucrt" }; + foreach (var arg in args) + { + if (!string.Equals(arg, "/arm64hazardfree", StringComparison.OrdinalIgnoreCase)) + { + filteredArgs.Add(QuoteArgument(FilterResponseFile(arg))); + } + } + startInfo.Arguments = string.Join(" ", filteredArgs); + + using var process = Process.Start(startInfo); + if (process is null) + { + Console.Error.WriteLine($"Failed to start linker: {linker}"); + return 1; + } + + process.WaitForExit(); + return process.ExitCode; + } + + private static string FilterResponseFile(string argument) + { + if (argument.Length < 2 || argument[0] != '@') + { + return argument; + } + + var responsePath = argument.Substring(1); + if (!File.Exists(responsePath)) + { + return argument; + } + + var filteredResponsePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".rsp"); + var responseContents = Regex.Replace( + File.ReadAllText(responsePath), + "/arm64hazardfree", + string.Empty, + RegexOptions.IgnoreCase); + File.WriteAllText(filteredResponsePath, responseContents); + return "@" + filteredResponsePath; + } + + private static string QuoteArgument(string argument) + { + if (argument.Length == 0) + { + return "\"\""; + } + if (argument.IndexOfAny(new[] { ' ', '\t', '"' }) < 0) + { + return argument; + } + + var quoted = new StringBuilder("\""); + var backslashes = 0; + foreach (var character in argument) + { + if (character == '\\') + { + backslashes++; + continue; + } + if (character == '"') + { + quoted.Append('\\', (backslashes * 2) + 1); + quoted.Append(character); + backslashes = 0; + continue; + } + + quoted.Append('\\', backslashes); + backslashes = 0; + quoted.Append(character); + } + quoted.Append('\\', backslashes * 2); + quoted.Append('"'); + return quoted.ToString(); + } +} +'@ + $WrapperSourcePath = Join-Path $WrapperDir "lld-link-wrapper.cs" + $WrapperSource | Out-File -FilePath $WrapperSourcePath -Encoding utf8 + $Csc = Join-Path $InstallPath "MSBuild\Current\Bin\Roslyn\csc.exe" + if (-not (Test-Path $Csc)) { + throw "csc.exe not found at $Csc" + } + & $Csc /nologo /target:exe /out:$WrapperPath $WrapperSourcePath + if ($LASTEXITCODE -ne 0) { + throw "Failed to compile lld-link wrapper" + } + "MSVC_REAL_LINKER=$Linker" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + $Linker = $WrapperPath +} + +Write-Output "Using Windows linker: $Linker" +$CargoTarget = $Target.ToUpperInvariant().Replace("-", "_") +"CARGO_TARGET_${CargoTarget}_LINKER=$Linker" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append diff --git a/.github/actions/setup-rusty-v8/action.yml b/.github/actions/setup-rusty-v8/action.yml new file mode 100644 index 0000000000000000000000000000000000000000..7b78d8cbdb07562a56b9ff98632af8f4da8ca028 --- /dev/null +++ b/.github/actions/setup-rusty-v8/action.yml @@ -0,0 +1,64 @@ +name: setup-rusty-v8 +description: Download and verify Codex-built rusty_v8 artifacts for Cargo builds. +inputs: + target: + description: Rust target triple with Codex-built V8 release artifacts. + required: true + +runs: + using: composite + steps: + - name: Configure rusty_v8 artifact overrides and verify checksums + shell: bash + env: + TARGET: ${{ inputs.target }} + run: | + set -euo pipefail + + version="$(python3 "${GITHUB_WORKSPACE}/.github/scripts/rusty_v8_bazel.py" resolved-v8-crate-version)" + release_tag="rusty-v8-v${version}" + base_url="https://github.com/openai/codex/releases/download/${release_tag}" + binding_dir="${RUNNER_TEMP}/rusty_v8" + + profile="ptrcomp_sandbox_release" + + if [[ "$TARGET" == *-pc-windows-msvc ]]; then + archive_name="rusty_v8_${profile}_${TARGET}.lib.gz" + else + archive_name="librusty_v8_${profile}_${TARGET}.a.gz" + fi + binding_name="src_binding_${profile}_${TARGET}.rs" + checksums_name="rusty_v8_${profile}_${TARGET}.sha256" + + archive_path="${binding_dir}/${archive_name}" + binding_path="${binding_dir}/${binding_name}" + checksums_path="${binding_dir}/${checksums_name}" + trusted_checksums="${GITHUB_WORKSPACE}/third_party/v8/rusty_v8_${version//./_}_release_manifests.sha256" + + if command -v sha256sum >/dev/null 2>&1; then + checksum_command=(sha256sum --check -) + else + checksum_command=(shasum -a 256 --check -) + fi + + mkdir -p "${binding_dir}" + curl -fsSL "${base_url}/${checksums_name}" -o "${checksums_path}" + # Check the original manifest bytes even when either checksum file uses CRLF. + expected_manifest_checksum="$(grep -F " ${checksums_name}" "${trusted_checksums}" | cut -d ' ' -f 1)" + actual_manifest_checksum="$(python3 -c 'import hashlib, pathlib, sys; print(hashlib.sha256(pathlib.Path(sys.argv[1]).read_bytes()).hexdigest())' "${checksums_path}")" + if [[ "${actual_manifest_checksum}" != "${expected_manifest_checksum}" ]]; then + echo "Checksum mismatch for ${checksums_name}: expected ${expected_manifest_checksum}, got ${actual_manifest_checksum}" >&2 + exit 1 + fi + curl -fsSL "${base_url}/${archive_name}" -o "${archive_path}" + curl -fsSL "${base_url}/${binding_name}" -o "${binding_path}" + + if [[ "$(wc -l < "${checksums_path}")" -ne 2 ]]; then + echo "Expected exactly two checksums for ${TARGET} in ${checksums_path}" >&2 + exit 1 + fi + + # Existing Windows-built release manifests use CRLF line endings. + (cd "${binding_dir}" && tr -d '\r' < "${checksums_path}" | "${checksum_command[@]}") + echo "RUSTY_V8_ARCHIVE=${archive_path}" >> "${GITHUB_ENV}" + echo "RUSTY_V8_SRC_BINDING_PATH=${binding_path}" >> "${GITHUB_ENV}" diff --git a/.github/actions/windows-code-sign/action.yml b/.github/actions/windows-code-sign/action.yml new file mode 100644 index 0000000000000000000000000000000000000000..24f8632979d48617db446ed92d3d0f962ee76d20 --- /dev/null +++ b/.github/actions/windows-code-sign/action.yml @@ -0,0 +1,73 @@ +name: windows-code-sign +description: Sign Windows binaries with Azure Trusted Signing. +inputs: + target: + description: Target triple for the artifacts to sign. + required: true + binaries: + description: Space-delimited binary basenames to sign. + default: "codex codex-responses-api-proxy codex-windows-sandbox-setup codex-command-runner" + client-id: + description: Azure Trusted Signing client ID. + required: true + tenant-id: + description: Azure tenant ID for Trusted Signing. + required: true + subscription-id: + description: Azure subscription ID for Trusted Signing. + required: true + endpoint: + description: Azure Trusted Signing endpoint. + required: true + account-name: + description: Azure Trusted Signing account name. + required: true + certificate-profile-name: + description: Certificate profile name for signing. + required: true + +runs: + using: composite + steps: + - name: Azure login for Trusted Signing (OIDC) + uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0 + with: + client-id: ${{ inputs.client-id }} + tenant-id: ${{ inputs.tenant-id }} + subscription-id: ${{ inputs.subscription-id }} + + - name: Prepare file list + id: prepare + shell: bash + env: + TARGET: ${{ inputs.target }} + BINARIES: ${{ inputs.binaries }} + run: | + set -euo pipefail + + { + echo "files<> "$GITHUB_OUTPUT" + + - name: Sign Windows binaries with Azure Trusted Signing + uses: azure/trusted-signing-action@1d365fec12862c4aa68fcac418143d73f0cea293 # v0.5.11 + with: + endpoint: ${{ inputs.endpoint }} + trusted-signing-account-name: ${{ inputs.account-name }} + certificate-profile-name: ${{ inputs.certificate-profile-name }} + exclude-environment-credential: true + exclude-workload-identity-credential: true + exclude-managed-identity-credential: true + exclude-shared-token-cache-credential: true + exclude-visual-studio-credential: true + exclude-visual-studio-code-credential: true + exclude-azure-cli-credential: false + exclude-azure-powershell-credential: true + exclude-azure-developer-cli-credential: true + exclude-interactive-browser-credential: true + cache-dependencies: false + files: ${{ steps.prepare.outputs.files }} diff --git a/.github/codex/labels/codex-triage.md b/.github/codex/labels/codex-triage.md new file mode 100644 index 0000000000000000000000000000000000000000..46ed3624167ac88fa79a82edcf9350c670d1caae --- /dev/null +++ b/.github/codex/labels/codex-triage.md @@ -0,0 +1,7 @@ +Troubleshoot whether the reported issue is valid. + +Provide a concise and respectful comment summarizing the findings. + +### {CODEX_ACTION_ISSUE_TITLE} + +{CODEX_ACTION_ISSUE_BODY} diff --git a/.github/scripts/macos-signing/codex-app-server.entitlements.plist b/.github/scripts/macos-signing/codex-app-server.entitlements.plist new file mode 100644 index 0000000000000000000000000000000000000000..26b12f28868c0dca58d8163ebd3112bcd081616b --- /dev/null +++ b/.github/scripts/macos-signing/codex-app-server.entitlements.plist @@ -0,0 +1,10 @@ + + + + + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + + diff --git a/.github/scripts/macos-signing/codex-code-mode-host.entitlements.plist b/.github/scripts/macos-signing/codex-code-mode-host.entitlements.plist new file mode 100644 index 0000000000000000000000000000000000000000..26b12f28868c0dca58d8163ebd3112bcd081616b --- /dev/null +++ b/.github/scripts/macos-signing/codex-code-mode-host.entitlements.plist @@ -0,0 +1,10 @@ + + + + + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + + diff --git a/.github/scripts/macos-signing/codex-responses-api-proxy.entitlements.plist b/.github/scripts/macos-signing/codex-responses-api-proxy.entitlements.plist new file mode 100644 index 0000000000000000000000000000000000000000..d35e43ae588cdfe5570930260829dcd145c4d1e2 --- /dev/null +++ b/.github/scripts/macos-signing/codex-responses-api-proxy.entitlements.plist @@ -0,0 +1,8 @@ + + + + + com.apple.security.cs.allow-jit + + + diff --git a/.github/scripts/macos-signing/codex-voice-host.entitlements.plist b/.github/scripts/macos-signing/codex-voice-host.entitlements.plist new file mode 100644 index 0000000000000000000000000000000000000000..b572d9c04e2698444dfc4945c4e804b7d1f6365d --- /dev/null +++ b/.github/scripts/macos-signing/codex-voice-host.entitlements.plist @@ -0,0 +1,8 @@ + + + + + com.apple.security.device.audio-input + + + diff --git a/.github/scripts/macos-signing/codex.entitlements.plist b/.github/scripts/macos-signing/codex.entitlements.plist new file mode 100644 index 0000000000000000000000000000000000000000..26b12f28868c0dca58d8163ebd3112bcd081616b --- /dev/null +++ b/.github/scripts/macos-signing/codex.entitlements.plist @@ -0,0 +1,10 @@ + + + + + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + + diff --git a/.github/scripts/macos-signing/notarize_macos_binary_with_akv.sh b/.github/scripts/macos-signing/notarize_macos_binary_with_akv.sh new file mode 100644 index 0000000000000000000000000000000000000000..90268aa8bfd94d5d768257875973f1b4c06fbc7c --- /dev/null +++ b/.github/scripts/macos-signing/notarize_macos_binary_with_akv.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +# Notarize a standalone binary and retain its diagnostic report. + +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +Usage: notarize_macos_binary_with_akv.sh --binary PATH [--report-dir PATH] [--max-wait-seconds SECONDS] + +Options: + --binary PATH Signed standalone macOS binary to notarize. + --report-dir PATH Directory for notarization logs. + --max-wait-seconds SECONDS Maximum Apple notarization wait time. +EOF +} + +binary_path="" +report_dir="${RUNNER_TEMP:-/tmp}/macos-binary-notarization-verification" +max_wait_seconds="600" + +while [[ $# -gt 0 ]]; do + case "$1" in + --binary) + binary_path="${2:-}" + shift 2 + ;; + --report-dir) + report_dir="${2:-}" + shift 2 + ;; + --max-wait-seconds) + max_wait_seconds="${2:-}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown notarization argument: $1" >&2 + usage + exit 2 + ;; + esac +done + +if [[ -z "$binary_path" ]]; then + echo "--binary is required." >&2 + usage + exit 2 +fi + +if [[ ! -f "$binary_path" ]]; then + echo "Binary does not exist: $binary_path" >&2 + exit 1 +fi + +if [[ ! "$max_wait_seconds" =~ ^[0-9]+$ ]]; then + echo "--max-wait-seconds must be a non-negative integer." >&2 + exit 2 +fi + +for command_name in python3 zip; do + if ! command -v "$command_name" >/dev/null 2>&1; then + echo "$command_name was not found on PATH." >&2 + exit 1 + fi +done + +missing_environment=0 +for variable_name in \ + APPLE_NOTARIZATION_AKV_KEY_NAME \ + AZURE_KEYVAULT_NAME +do + if [[ -z "${!variable_name:-}" ]]; then + echo "$variable_name must be configured before notarizing a binary." >&2 + missing_environment=1 + fi +done + +if [[ "$missing_environment" -ne 0 ]]; then + exit 2 +fi + +mkdir -p "$report_dir" + +notarization_temp_dir="$(mktemp -d)" +trap 'rm -rf "$notarization_temp_dir" >/dev/null' EXIT + +binary_name="$(basename "$binary_path")" +archive_path="$notarization_temp_dir/${binary_name}.zip" +( + cd "$(dirname "$binary_path")" + zip -q "$archive_path" "$binary_name" +) + +notarization_log="$report_dir/${binary_name}-notarization.log" +python3 "$(dirname "$0")/notarize_with_akv.py" \ + --file "$archive_path" \ + --report-log "$report_dir/${binary_name}-notarization-developer-log.json" \ + --max-wait-seconds "$max_wait_seconds" \ + 2>&1 | tee "$notarization_log" + +{ + echo "binary_name=$binary_name" + echo "max_wait_seconds=$max_wait_seconds" + echo "binary_sha256=$(shasum -a 256 "$binary_path" | awk '{ print $1 }')" + echo "notarization=completed" +} >"$report_dir/${binary_name}-notarization-summary.txt" diff --git a/.github/scripts/macos-signing/notarize_macos_dmg_with_akv.sh b/.github/scripts/macos-signing/notarize_macos_dmg_with_akv.sh new file mode 100644 index 0000000000000000000000000000000000000000..79609dbfa8a8fc3f95d29aa20bc7c7e5282989ed --- /dev/null +++ b/.github/scripts/macos-signing/notarize_macos_dmg_with_akv.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +# Notarize a signed disk image and staple its ticket. + +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +Usage: notarize_macos_dmg_with_akv.sh --dmg PATH [--report-dir PATH] [--max-wait-seconds SECONDS] + +Options: + --dmg PATH Signed DMG to submit to Apple notarization. + --report-dir PATH Directory for notarization logs. + --max-wait-seconds SECONDS Maximum Apple notarization wait time. +EOF +} + +dmg_path="" +report_dir="${RUNNER_TEMP:-/tmp}/macos-notarization-verification" +max_wait_seconds="600" + +while [[ $# -gt 0 ]]; do + case "$1" in + --dmg) + dmg_path="${2:-}" + shift 2 + ;; + --report-dir) + report_dir="${2:-}" + shift 2 + ;; + --max-wait-seconds) + max_wait_seconds="${2:-}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown notarization argument: $1" >&2 + usage + exit 2 + ;; + esac +done + +if [[ -z "$dmg_path" ]]; then + echo "--dmg is required." >&2 + usage + exit 2 +fi + +if [[ ! -f "$dmg_path" ]]; then + echo "DMG does not exist: $dmg_path" >&2 + exit 1 +fi + +if [[ ! "$max_wait_seconds" =~ ^[0-9]+$ ]]; then + echo "--max-wait-seconds must be a non-negative integer." >&2 + exit 2 +fi + +if ! command -v rcodesign > /dev/null 2>&1; then + echo "rcodesign was not found on PATH." >&2 + exit 1 +fi + +missing_environment=0 +for variable_name in \ + APPLE_NOTARIZATION_AKV_KEY_NAME \ + AZURE_KEYVAULT_NAME +do + if [[ -z "${!variable_name:-}" ]]; then + echo "$variable_name must be configured before notarizing a DMG." >&2 + missing_environment=1 + fi +done + +if [[ "$missing_environment" -ne 0 ]]; then + exit 2 +fi + +mkdir -p "$report_dir" + +notarization_log="$report_dir/dmg-notarization.log" +python3 "$(dirname "$0")/notarize_with_akv.py" \ + --file "$dmg_path" \ + --report-log "$report_dir/dmg-notarization-developer-log.json" \ + --max-wait-seconds "$max_wait_seconds" \ + 2>&1 | tee "$notarization_log" + +rcodesign staple "$dmg_path" 2>&1 | tee -a "$notarization_log" + +{ + echo "dmg_path=$dmg_path" + echo "max_wait_seconds=$max_wait_seconds" + echo "dmg_sha256=$(shasum -a 256 "$dmg_path" | awk '{ print $1 }')" + echo "notarization_staple=completed" +} > "$report_dir/dmg-notarization-summary.txt" diff --git a/.github/scripts/macos-signing/notarize_with_akv.py b/.github/scripts/macos-signing/notarize_with_akv.py new file mode 100644 index 0000000000000000000000000000000000000000..1929482cd4ed571a6a397d9c3105dc90162a3650 --- /dev/null +++ b/.github/scripts/macos-signing/notarize_with_akv.py @@ -0,0 +1,519 @@ +#!/usr/bin/env python3 +"""Submit release artifacts for notarization and wait for the result.""" + +import argparse +import base64 +import datetime +import hashlib +import hmac +import json +import os +import re +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +APPLE_NOTARY_URL = "https://appstoreconnect.apple.com/notary/v2/submissions" +AZURE_KEYVAULT_API_VERSION = "7.4" +AWS_REGION = "us-west-2" +JWT_LIFETIME_SECONDS = 15 * 60 +POLL_INTERVAL_SECONDS = 10 +MAX_SINGLE_UPLOAD_BYTES = 5 * 1024 * 1024 * 1024 + + +class NotarizationError(RuntimeError): + """Describe a notarization failure.""" + + +@dataclass(frozen=True) +class NotarizationConfiguration: + """Hold the account and signing-key configuration.""" + + issuer_id: str + apple_key_id: str + vault_name: str + vault_key_name: str + vault_key_version: str + + @classmethod + def from_environment(cls) -> "NotarizationConfiguration": + """Load and validate the notarization configuration.""" + + required = { + "vault_name": "AZURE_KEYVAULT_NAME", + "vault_key_name": "APPLE_NOTARIZATION_AKV_KEY_NAME", + } + values = { + field: os.environ.get(name, "").strip() for field, name in required.items() + } + missing = [name for field, name in required.items() if not values[field]] + if missing: + raise NotarizationError( + "Missing notarization configuration: " + ", ".join(missing) + ) + if not re.fullmatch(r"[A-Za-z][A-Za-z0-9-]{2,23}", values["vault_name"]): + raise NotarizationError("Invalid signing vault name") + if not re.fullmatch(r"[A-Za-z0-9-]+", values["vault_key_name"]): + raise NotarizationError("Invalid notarization key name") + key_version = os.environ.get("APPLE_NOTARIZATION_AKV_KEY_VERSION", "").strip() + if key_version and not re.fullmatch(r"[0-9a-fA-F]{32}", key_version): + raise NotarizationError( + "Notarization key must resolve to a 32-character pinned version" + ) + + command = [ + "az", + "keyvault", + "key", + "show", + "--vault-name", + values["vault_name"], + "--name", + values["vault_key_name"], + ] + if key_version: + command.extend(["--version", key_version]) + command.extend( + [ + "--query", + '{id:key.kid,apple_key_id:tags."apple-key-id",' + 'apple_issuer_id:tags."apple-issuer-id"}', + "--output", + "json", + "--only-show-errors", + ] + ) + + # Resolve the key identifier and version from the same lookup. + try: + result = subprocess.run(command, check=True, capture_output=True, text=True) + except (OSError, subprocess.CalledProcessError) as error: + raise NotarizationError( + "Notarization signing key could not be read" + ) from error + try: + metadata = json.loads(result.stdout) + returned_key_id = metadata["id"] + apple_key_id = metadata["apple_key_id"] + apple_issuer_id = metadata["apple_issuer_id"] + except (json.JSONDecodeError, KeyError, TypeError) as error: + raise NotarizationError("Notarization key metadata is invalid") from error + + key_id_prefix = f"https://{values['vault_name']}.vault.azure.net/keys/{values['vault_key_name']}/" + if not isinstance(returned_key_id, str) or not returned_key_id.startswith( + key_id_prefix + ): + raise NotarizationError("Unexpected notarization signing key") + returned_key_version = returned_key_id[len(key_id_prefix) :] + if not re.fullmatch(r"[0-9a-fA-F]{32}", returned_key_version): + raise NotarizationError( + "Notarization key must resolve to a 32-character pinned version" + ) + if key_version and returned_key_version.lower() != key_version.lower(): + raise NotarizationError("Unexpected notarization key version") + if not isinstance(apple_key_id, str) or not apple_key_id.strip(): + raise NotarizationError("Notarization key must have an apple-key-id tag") + if not isinstance(apple_issuer_id, str) or not apple_issuer_id.strip(): + raise NotarizationError("Notarization key must have an apple-issuer-id tag") + if not re.fullmatch( + r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" + r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}", + apple_issuer_id.strip(), + ): + raise NotarizationError( + "Notarization apple-issuer-id tag must contain a valid UUID" + ) + + values["issuer_id"] = apple_issuer_id.strip() + values["apple_key_id"] = apple_key_id.strip() + values["vault_key_version"] = returned_key_version + return cls(**values) + + @property + def versioned_key_id(self) -> str: + """Return the versioned signing-key identifier.""" + + return ( + f"https://{self.vault_name}.vault.azure.net/keys/" + f"{self.vault_key_name}/{self.vault_key_version}" + ) + + +def base64url_encode(value: bytes) -> str: + """Encode bytes using the unpadded base64url format required by JWTs.""" + + return base64.urlsafe_b64encode(value).decode("ascii").rstrip("=") + + +def base64url_decode(value: str) -> bytes: + """Decode a base64url value and reject malformed input.""" + + try: + return base64.b64decode( + value + "=" * (-len(value) % 4), altchars=b"-_", validate=True + ) + except (ValueError, UnicodeEncodeError) as error: + raise NotarizationError( + "Signing service returned an invalid signature" + ) from error + + +def create_apple_jwt( + configuration: NotarizationConfiguration, *, issued_at: int +) -> str: + """Create an authentication token for notarization requests.""" + + header = {"alg": "ES256", "kid": configuration.apple_key_id, "typ": "JWT"} + # Restrict the token to notarization requests. + claims = { + "iss": configuration.issuer_id, + "iat": issued_at, + "exp": issued_at + JWT_LIFETIME_SECONDS, + "aud": "appstoreconnect-v1", + "scope": ["/notary/v2"], + } + encoded_header = base64url_encode( + json.dumps(header, separators=(",", ":")).encode("utf-8") + ) + encoded_claims = base64url_encode( + json.dumps(claims, separators=(",", ":")).encode("utf-8") + ) + signing_input = f"{encoded_header}.{encoded_claims}" + digest = hashlib.sha256(signing_input.encode("ascii")).digest() + key_url = configuration.versioned_key_id + request_body = json.dumps({"alg": "ES256", "value": base64url_encode(digest)}) + + try: + result = subprocess.run( + [ + "az", + "rest", + "--method", + "post", + "--url", + f"{key_url}/sign?api-version={AZURE_KEYVAULT_API_VERSION}", + "--resource", + "https://vault.azure.net", + "--headers", + "Content-Type=application/json", + "--body", + request_body, + "--output", + "json", + "--only-show-errors", + ], + check=True, + capture_output=True, + text=True, + ) + except (OSError, subprocess.CalledProcessError) as error: + detail = getattr(error, "stderr", "").strip() + message = "Signing service could not sign the authentication token" + if detail: + message = f"{message}: {detail}" + raise NotarizationError(message) from error + + try: + response = json.loads(result.stdout) + returned_key_id = response["kid"] + encoded_signature = response["value"] + except (json.JSONDecodeError, KeyError, TypeError) as error: + raise NotarizationError( + "Signing service returned an invalid response" + ) from error + + # Reject signatures generated with a different key version. + if returned_key_id != key_url: + raise NotarizationError("Signing service returned an unexpected key version") + if not isinstance(encoded_signature, str): + raise NotarizationError("Signing service returned an invalid signature") + signature = base64url_decode(encoded_signature) + # ES256 JWT signatures are 64 raw R || S bytes, not ASN.1/DER-encoded ECDSA signatures. + if len(signature) != 64: + raise NotarizationError("ES256 signature must contain 64 JOSE R || S bytes") + return f"{signing_input}.{base64url_encode(signature)}" + + +def json_request( + url: str, token: str, *, body: dict[str, Any] | None = None +) -> dict[str, Any]: + """Send an authenticated JSON request to Apple's notarization service.""" + + data = None if body is None else json.dumps(body).encode("utf-8") + request = urllib.request.Request( + url, + data=data, + method="GET" if body is None else "POST", + headers={ + "Accept": "application/json", + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + }, + ) + try: + with urllib.request.urlopen(request, timeout=60) as response: + result = json.load(response) + except urllib.error.HTTPError as error: + detail = error.read(1024).decode("utf-8", errors="replace") + raise NotarizationError( + f"Apple Notary API returned HTTP {error.code}: {detail}" + ) from error + except (OSError, json.JSONDecodeError) as error: + raise NotarizationError("Apple Notary API request failed") from error + + if not isinstance(result, dict): + raise NotarizationError("Apple Notary API returned an invalid response") + return result + + +def sha256_file(path: Path) -> str: + """Hash an artifact incrementally so large release files stay off the heap.""" + + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def hmac_sha256(key: bytes, value: str) -> bytes: + """Derive one AWS Signature Version 4 signing-key component.""" + + return hmac.new(key, value.encode("utf-8"), hashlib.sha256).digest() + + +def upload_to_apple_s3( + path: Path, + file_digest: str, + credentials: dict[str, Any], + *, + timestamp: datetime.datetime | None = None, +) -> None: + """Upload an artifact using the provided temporary credentials.""" + + required = ( + "awsAccessKeyId", + "awsSecretAccessKey", + "awsSessionToken", + "bucket", + "object", + ) + if any( + not isinstance(credentials.get(name), str) or not credentials[name] + for name in required + ): + raise NotarizationError( + "Apple returned incomplete temporary upload credentials" + ) + + size = path.stat().st_size + if not 0 < size <= MAX_SINGLE_UPLOAD_BYTES: + raise NotarizationError("Notarization payload must be between 1 byte and 5 GiB") + + now = timestamp or datetime.datetime.now(datetime.timezone.utc) + amz_date = now.strftime("%Y%m%dT%H%M%SZ") + date_stamp = now.strftime("%Y%m%d") + host = f"{credentials['bucket']}.s3.{AWS_REGION}.amazonaws.com" + object_path = "/" + urllib.parse.quote( + credentials["object"].lstrip("/"), safe="/-_.~" + ) + headers = { + "content-type": "application/octet-stream", + "host": host, + "x-amz-content-sha256": file_digest, + "x-amz-date": amz_date, + "x-amz-security-token": credentials["awsSessionToken"], + } + # Build the signature required by the upload endpoint. + signed_headers = ";".join(sorted(headers)) + canonical_headers = "".join(f"{name}:{headers[name]}\n" for name in sorted(headers)) + canonical_request = "\n".join( + ["PUT", object_path, "", canonical_headers, signed_headers, file_digest] + ) + credential_scope = f"{date_stamp}/{AWS_REGION}/s3/aws4_request" + string_to_sign = "\n".join( + [ + "AWS4-HMAC-SHA256", + amz_date, + credential_scope, + hashlib.sha256(canonical_request.encode("utf-8")).hexdigest(), + ] + ) + signing_key = ("AWS4" + credentials["awsSecretAccessKey"]).encode("utf-8") + for component in (date_stamp, AWS_REGION, "s3", "aws4_request"): + signing_key = hmac_sha256(signing_key, component) + signature = hmac.new( + signing_key, string_to_sign.encode("utf-8"), hashlib.sha256 + ).hexdigest() + authorization = ( + "AWS4-HMAC-SHA256 " + f"Credential={credentials['awsAccessKeyId']}/{credential_scope}, " + f"SignedHeaders={signed_headers}, Signature={signature}" + ) + + # Stream the artifact to avoid buffering its full contents. + with path.open("rb") as source: + request = urllib.request.Request( + f"https://{host}{object_path}", + data=source, + method="PUT", + headers={ + **headers, + "Authorization": authorization, + "Content-Length": str(size), + }, + ) + try: + with urllib.request.urlopen(request, timeout=300): + pass + except urllib.error.HTTPError as error: + raise NotarizationError( + f"Apple notarization payload upload failed with HTTP {error.code}" + ) from error + except OSError as error: + raise NotarizationError( + "Apple notarization payload upload failed" + ) from error + + +def write_developer_log( + submission_id: str, token: str, destination: Path | None +) -> None: + """Write the submission's optional diagnostic report.""" + + if destination is None: + return + response = json_request(f"{APPLE_NOTARY_URL}/{submission_id}/logs", token) + try: + log_url = response["data"]["attributes"]["developerLogUrl"] + except (KeyError, TypeError) as error: + raise NotarizationError( + "Apple did not provide a notarization developer log" + ) from error + if not isinstance(log_url, str) or not log_url.startswith("https://"): + raise NotarizationError( + "Apple returned an invalid notarization developer log URL" + ) + try: + # Do not forward unrelated authorization headers to the download URL. + with urllib.request.urlopen(log_url, timeout=60) as response: + destination.write_bytes(response.read()) + except urllib.error.HTTPError as error: + raise NotarizationError( + f"Apple notarization developer log download failed with HTTP {error.code}" + ) from error + except OSError as error: + raise NotarizationError( + "Apple notarization developer log download failed" + ) from error + + +def notarize( + path: Path, + configuration: NotarizationConfiguration, + *, + max_wait_seconds: int, + report_log: Path | None, +) -> str: + """Submit an artifact, upload it to Apple, and wait for a terminal result.""" + + issued_at = int(time.time()) + token = create_apple_jwt(configuration, issued_at=issued_at) + file_digest = sha256_file(path) + # Create the submission before uploading the artifact. + response = json_request( + APPLE_NOTARY_URL, + token, + body={"sha256": file_digest, "submissionName": path.name}, + ) + try: + submission_id = response["data"]["id"] + upload_credentials = response["data"]["attributes"] + except (KeyError, TypeError) as error: + raise NotarizationError( + "Apple returned an invalid notarization submission" + ) from error + if not isinstance(submission_id, str) or not submission_id: + raise NotarizationError("Apple did not return a notarization submission ID") + if not isinstance(upload_credentials, dict): + raise NotarizationError( + "Apple returned invalid notarization upload credentials" + ) + + print(f"Uploading notarization submission {submission_id} for {path.name}") + upload_to_apple_s3(path, file_digest, upload_credentials) + # A monotonic deadline keeps the wait limit stable if the system clock changes. + deadline = time.monotonic() + max_wait_seconds + + # Wait for completion, equivalent to `notarytool submit --wait`. + while True: + # Refresh the token before it expires. + if int(time.time()) >= issued_at + JWT_LIFETIME_SECONDS - 60: + issued_at = int(time.time()) + token = create_apple_jwt(configuration, issued_at=issued_at) + result = json_request(f"{APPLE_NOTARY_URL}/{submission_id}", token) + try: + status = result["data"]["attributes"]["status"] + except (KeyError, TypeError) as error: + raise NotarizationError( + "Apple returned an invalid notarization status" + ) from error + + if status in {"Accepted", "Invalid", "Rejected"}: + # Save diagnostics for both accepted and rejected submissions. + write_developer_log(submission_id, token, report_log) + message = f"Notarization submission {submission_id} completed with status {status}" + if status != "Accepted": + raise NotarizationError(message) + print(message) + return submission_id + if status != "In Progress": + raise NotarizationError(f"Unexpected notarization status: {status!r}") + + remaining = deadline - time.monotonic() + if remaining <= 0: + raise NotarizationError( + f"Notarization submission {submission_id} exceeded {max_wait_seconds} seconds" + ) + print(f"Notarization submission {submission_id} is still in progress") + time.sleep(min(POLL_INTERVAL_SECONDS, remaining)) + + +def main() -> int: + """Run notarization using command-line arguments.""" + + parser = argparse.ArgumentParser() + parser.add_argument("--file", type=Path, required=True) + parser.add_argument("--report-log", type=Path) + parser.add_argument("--max-wait-seconds", type=int, default=600) + arguments = parser.parse_args() + + try: + if not arguments.file.is_file(): + raise NotarizationError( + f"Notarization payload does not exist: {arguments.file}" + ) + if arguments.max_wait_seconds < 0: + raise NotarizationError("--max-wait-seconds must be non-negative") + if arguments.report_log is not None: + arguments.report_log.parent.mkdir(parents=True, exist_ok=True) + notarize( + arguments.file, + NotarizationConfiguration.from_environment(), + max_wait_seconds=arguments.max_wait_seconds, + report_log=arguments.report_log, + ) + except NotarizationError as error: + print(f"Notarization failed: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/macos-signing/provisioned_macos_cli_package.py b/.github/scripts/macos-signing/provisioned_macos_cli_package.py new file mode 100644 index 0000000000000000000000000000000000000000..449dd15499c7dd8f1a85795a4ad47672c031a3fa --- /dev/null +++ b/.github/scripts/macos-signing/provisioned_macos_cli_package.py @@ -0,0 +1,229 @@ +"""Prepare and verify provisioned CLI bundles using an explicitly approved profile.""" + +import argparse +import hashlib +import json +import os +import plistlib +import re +import shutil +import subprocess +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +BUNDLE_ID = "com.openai.codex.cli" +APP = Path("CodexCLI.app") +EXECUTABLE = APP / "Contents/MacOS/codex" +SIGNING = Path(__file__).resolve().parent +HELPERS = ("bin/codex-code-mode-host", "codex-path/rg", "codex-resources/zsh/bin/zsh") +LAUNCHER = """#!/bin/sh +set -eu +entry="$0" +while [ -L "$entry" ]; do + parent=$(CDPATH= cd -P -- "$(dirname -- "$entry")" && pwd) + entry=$(readlink "$entry") + case "$entry" in /*) ;; *) entry="$parent/$entry" ;; esac +done +bin_dir=$(CDPATH= cd -P -- "$(dirname -- "$entry")" && pwd) +exec "$bin_dir/../CodexCLI.app/Contents/MacOS/codex" "$@" +""" + + +@dataclass(frozen=True) +class ProfileConfiguration: + """Independent release expectations; never infer these from the signed package.""" + + profile: Path + profile_sha256: str + certificate_sha256: str + team_id: str + + def __post_init__(self): + for name in ("profile_sha256", "certificate_sha256"): + if not re.fullmatch(r"[0-9a-f]{64}", getattr(self, name)): + raise ValueError(f"{name} must be a lowercase SHA-256 digest") + if not re.fullmatch(r"[A-Z0-9]{10}", self.team_id): + raise ValueError("team_id must be a ten-character Apple Team ID") + + def validate_signing_certificate(self, certificate: Path): + certificate_der = subprocess.check_output( + ["openssl", "x509", "-in", str(certificate), "-outform", "DER"] + ) + if hashlib.sha256(certificate_der).hexdigest() != self.certificate_sha256: + raise ValueError("AKV signing certificate is not authorized by the profile") + + +def load_profile(configuration: ProfileConfiguration): + # Pin the portal-approved CMS as well as checking its signature. -noverify + # skips OpenSSL's non-Apple trust store, not CMS signature verification. + if ( + hashlib.sha256(configuration.profile.read_bytes()).hexdigest() + != configuration.profile_sha256 + ): + raise ValueError("Provisioning profile changed; review and update its pin") + profile = plistlib.loads( + subprocess.check_output( + [ + "openssl", + "cms", + "-verify", + "-inform", + "DER", + "-noverify", + "-in", + str(configuration.profile), + ] + ) + ) + validate_profile(profile, configuration) + return profile + + +def validate_profile(profile, configuration: ProfileConfiguration): + allowed = profile["Entitlements"] + if ( + profile["TeamIdentifier"] != [configuration.team_id] + or profile["ApplicationIdentifierPrefix"] != [configuration.team_id] + or allowed["com.apple.application-identifier"] + != f"{configuration.team_id}.{BUNDLE_ID}" + or allowed["com.apple.developer.team-identifier"] != configuration.team_id + or allowed["keychain-access-groups"] != [f"{configuration.team_id}.*"] + or allowed.get("get-task-allow", False) + or not profile.get("ProvisionsAllDevices") + or "ProvisionedDevices" in profile + ): + raise ValueError("Profile does not authorize the CLI Developer ID identity") + now = datetime.now(timezone.utc).replace(tzinfo=None) + if not profile["CreationDate"] <= now < profile["ExpirationDate"]: + raise ValueError("Provisioning profile is not currently valid") + if [ + hashlib.sha256(cert).hexdigest() for cert in profile["DeveloperCertificates"] + ] != [configuration.certificate_sha256]: + raise ValueError("Profile does not authorize the approved signing certificate") + + +def entitlements(configuration: ProfileConfiguration): + base = plistlib.loads((SIGNING / "codex.entitlements.plist").read_bytes()) + return { + **base, + "com.apple.application-identifier": f"{configuration.team_id}.{BUNDLE_ID}", + "com.apple.developer.team-identifier": configuration.team_id, + "keychain-access-groups": [f"{configuration.team_id}.{BUNDLE_ID}"], + } + + +def prepare(package, reports, configuration: ProfileConfiguration): + load_profile(configuration) + metadata = json.loads((package / "codex-package.json").read_text()) + if ( + metadata["variant"] != "codex" + or metadata["layoutVersion"] != 1 + or metadata["target"] not in ("aarch64-apple-darwin", "x86_64-apple-darwin") + or metadata["entrypoint"] != "bin/codex" + ): + raise ValueError("Expected a canonical macOS CLI package") + for relative in ("bin/codex", *HELPERS): + path = package / relative + if path.is_symlink() or not path.is_file() or not os.access(path, os.X_OK): + raise ValueError(f"Expected a regular executable: {relative}") + # Refuse to overwrite a previously prepared or signed bundle. + (package / EXECUTABLE).parent.mkdir(parents=True, exist_ok=False) + (package / "bin/codex").rename(package / EXECUTABLE) + (package / "bin/codex").write_text(LAUNCHER) + (package / "bin/codex").chmod(0o755) + contents = package / APP / "Contents" + shutil.copyfile(configuration.profile, contents / "embedded.provisionprofile") + (contents / "Info.plist").write_bytes( + plistlib.dumps( + { + "CFBundleIdentifier": BUNDLE_ID, + "CFBundleExecutable": "codex", + "CFBundleName": "Codex CLI", + "CFBundlePackageType": "APPL", + "CFBundleVersion": "1", + } + ) + ) + reports.mkdir(parents=True, exist_ok=True) + # Keep expected entitlements separate from the shared verifier's extracted + # codex-entitlements.plist, otherwise it would overwrite its own expectation. + (reports / "codex-provisioned-entitlements.plist").write_bytes( + plistlib.dumps(entitlements(configuration)) + ) + + +def verify(package, reports, expected_target, configuration: ProfileConfiguration): + """Check profile and certificate pins; the signing driver verifies code.""" + load_profile(configuration) + if ( + package / APP / "Contents/embedded.provisionprofile" + ).read_bytes() != configuration.profile.read_bytes(): + raise ValueError("Embedded profile differs from the reviewed profile") + if (package / "bin/codex").read_text() != LAUNCHER: + raise ValueError("Unexpected CLI launcher") + metadata = json.loads((package / "codex-package.json").read_text()) + if metadata["target"] != expected_target: + raise ValueError( + "Package target differs from the requested verification target" + ) + reports.mkdir(parents=True, exist_ok=True) + (reports / "codex-provisioned-entitlements.plist").write_bytes( + plistlib.dumps(entitlements(configuration)) + ) + # sign_macos_cli_package.py owns shared architecture, signature and entitlement + # checks. Here, require the exact certificate authorized by the profile. + for relative in (EXECUTABLE, *(Path(helper) for helper in HELPERS)): + binary = package / relative + target = package / APP if relative == EXECUTABLE else binary + prefix = reports / f"{binary.name}-cert-" + subprocess.run( + ["codesign", "-d", f"--extract-certificates={prefix}", str(target)], + check=True, + ) + if ( + hashlib.sha256(Path(f"{prefix}0").read_bytes()).hexdigest() + != configuration.certificate_sha256 + ): + raise ValueError(f"Unexpected signing certificate: {relative}") + + +def parse_profile_args(parser, configuration: ProfileConfiguration | None): + """Require independent profile expectations unless the caller supplies them.""" + if configuration is None: + parser.add_argument("--profile", type=Path, required=True) + parser.add_argument("--profile-sha256", required=True) + parser.add_argument("--certificate-sha256", required=True) + parser.add_argument("--team-id", required=True) + args = parser.parse_args() + if configuration is None: + configuration = ProfileConfiguration( + args.profile, args.profile_sha256, args.certificate_sha256, args.team_id + ) + return args, configuration + + +def main(configuration: ProfileConfiguration | None = None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=("prepare", "verify", "validate-profile")) + parser.add_argument("--package", type=Path, default=Path("package")) + parser.add_argument("--reports", type=Path, default=Path("provisioned-reports")) + parser.add_argument("--certificate", type=Path) + parser.add_argument( + "--target", choices=("aarch64-apple-darwin", "x86_64-apple-darwin") + ) + args, configuration = parse_profile_args(parser, configuration) + if args.certificate: + configuration.validate_signing_certificate(args.certificate) + if args.operation == "validate-profile": + load_profile(configuration) + elif args.operation == "prepare": + prepare(args.package, args.reports, configuration) + else: + if not args.target: + parser.error("--target is required for verification") + verify(args.package, args.reports, args.target, configuration) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/macos-signing/sign_macos_cli_package.py b/.github/scripts/macos-signing/sign_macos_cli_package.py new file mode 100644 index 0000000000000000000000000000000000000000..6d555945a7ca4db394d2e377f1a779b79124f64f --- /dev/null +++ b/.github/scripts/macos-signing/sign_macos_cli_package.py @@ -0,0 +1,177 @@ +"""Sign and verify macOS CLI packages using the shared AKV and native tools.""" + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +import provisioned_macos_cli_package as bundle + + +def main(configuration: bundle.ProfileConfiguration | None = None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=("sign", "verify")) + parser.add_argument("package", type=Path) + args, configuration = bundle.parse_profile_args(parser, configuration) + package = args.package.resolve(strict=True) + if not package.is_dir(): + parser.error("package must be an extracted CLI package directory") + repo_root = os.environ.get("CODEX_REPO_ROOT") + if not repo_root: + parser.error("CODEX_REPO_ROOT must be set") + reports = Path(repo_root) / "signing-verification" + reports.mkdir(parents=True, exist_ok=True) + provisioned = os.environ.get("PROVISIONED_MACOS", "false") == "true" + if provisioned: + if args.operation == "sign": + configuration.validate_signing_certificate( + Path(os.environ["OAI_AKV_SIGNING_CERTIFICATE_PEM"]) + ) + bundle.prepare(package, reports, configuration) + else: + bundle.verify(package, reports, os.environ["TARGET"], configuration) + + for relative in ("bin/codex", *bundle.HELPERS): + binary = package / relative + name = binary.name + target = binary + identifier = name + entitlements = bundle.SIGNING / f"{name}.entitlements.plist" + if provisioned and relative == "bin/codex": + binary = package / bundle.EXECUTABLE + target = package / bundle.APP + identifier = bundle.BUNDLE_ID + entitlements = reports / "codex-provisioned-entitlements.plist" + if args.operation == "sign": + command = [ + "bash", + str(bundle.SIGNING / "sign_macos_code.sh"), + "--target", + str(target), + "--identity", + "unused", + "--deep", + "false", + "--options", + "runtime", + "--timestamp", + "true", + ] + if relative.startswith("bin/"): + command.extend( + ["--identifier", identifier, "--entitlements", str(entitlements)] + ) + else: + command.extend(["--identifier", f"com.openai.codex.{name}"]) + subprocess.run(command, check=True) + with (reports / f"{name}-signature.yaml").open("wb") as output: + subprocess.run( + ["rcodesign", "print-signature-info", str(binary)], + stdout=output, + check=True, + ) + if not provisioned: + subprocess.run( + [ + "bash", + str(bundle.SIGNING / "notarize_macos_binary_with_akv.sh"), + "--binary", + str(binary), + "--report-dir", + str(reports / name), + ], + check=True, + ) + else: + target_triple = os.environ["TARGET"] + architectures = { + "aarch64-apple-darwin": "arm64", + "x86_64-apple-darwin": "x86_64", + } + if target_triple not in architectures: + raise ValueError(f"Unexpected macOS target: {target_triple}") + subprocess.run( + ["lipo", str(binary), "-verify_arch", architectures[target_triple]], + check=True, + ) + subprocess.run( + [ + "codesign", + "--verify", + "--strict", + "--verbose=2", + "--test-requirement", + # Require Apple's Developer ID Application certificate chain. + ( + "=anchor apple generic" + " and certificate 1[field.1.2.840.113635.100.6.2.6] exists" + " and certificate leaf[field.1.2.840.113635.100.6.1.13] exists" + ), + str(target), + ], + check=True, + ) + signature = reports / f"{name}-signature.txt" + with signature.open("wb") as output: + subprocess.run( + ["codesign", "-d", "--verbose=4", str(target)], + stderr=output, + check=True, + ) + actual = reports / f"{name}-entitlements.plist" + with actual.open("wb") as output: + subprocess.run( + ["codesign", "-d", "--entitlements", ":-", str(target)], + stdout=output, + check=True, + ) + if relative.startswith("bin/"): + expected = reports / f"{name}-expected.plist" + subprocess.run( + [ + "plutil", + "-convert", + "xml1", + "-o", + str(expected), + str(entitlements), + ], + check=True, + ) + subprocess.run(["plutil", "-convert", "xml1", str(actual)], check=True) + subprocess.run(["diff", "-u", str(expected), str(actual)], check=True) + elif actual.stat().st_size: + raise ValueError( + f"Bundled helper {name} must not have signing entitlements" + ) + + if provisioned and args.operation == "sign": + archive = Path(os.environ["RUNNER_TEMP"]) / "provisioned-cli.zip" + subprocess.run(["zip", "-q", "-r", str(archive), "."], cwd=package, check=True) + command = [ + sys.executable, + str(bundle.SIGNING / "notarize_with_akv.py"), + "--file", + str(archive), + "--report-log", + str(reports / "notarization-developer-log.json"), + "--max-wait-seconds", + "1200", + ] + with ( + (reports / "notarization.log").open("w") as output, + subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True + ) as process, + ): + assert process.stdout is not None + for line in process.stdout: + print(line, end="", flush=True) + output.write(line) + if returncode := process.wait(): + raise subprocess.CalledProcessError(returncode, command) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/macos-signing/sign_macos_code.sh b/.github/scripts/macos-signing/sign_macos_code.sh new file mode 100644 index 0000000000000000000000000000000000000000..9f86741410dbfd5500632b84e9fcd5f1eaf58c8a --- /dev/null +++ b/.github/scripts/macos-signing/sign_macos_code.sh @@ -0,0 +1,245 @@ +#!/usr/bin/env bash + +# Small compatibility wrapper around native codesign and rcodesign. +# +# Existing packaging scripts call this instead of choosing a signing backend +# directly. OAI_CODESIGN_BACKEND=akv-pkcs11 routes signing through rcodesign +# while preserving the option, entitlement, identifier, timestamp, and deep +# signing surface used by the native codesign path. + +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +Usage: sign_macos_code.sh --target PATH --identity IDENTITY [options] + +Options: + --deep true|false + --entitlements PATH + --identifier IDENTIFIER + --identity IDENTITY + --options FLAGS + --target PATH + --timestamp true|false|none +EOF +} + +target="" +identity="" +options="" +entitlements_file="" +identifier="" +deep="false" +timestamp="true" + +while [[ $# -gt 0 ]]; do + case "$1" in + --deep) + deep="${2:-}" + shift 2 + ;; + --entitlements) + entitlements_file="${2:-}" + shift 2 + ;; + --identifier) + identifier="${2:-}" + shift 2 + ;; + --identity) + identity="${2:-}" + shift 2 + ;; + --options) + options="${2:-}" + shift 2 + ;; + --target) + target="${2:-}" + shift 2 + ;; + --timestamp) + timestamp="${2:-}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown signing argument: $1" >&2 + usage + exit 2 + ;; + esac +done + +if [[ -z "$target" ]]; then + echo "--target is required." >&2 + usage + exit 2 +fi + +if [[ ! -e "$target" ]]; then + echo "Signing target does not exist: $target" >&2 + exit 1 +fi + +case "$deep" in + true|false) ;; + *) + echo "--deep must be true or false, got '$deep'." >&2 + exit 2 + ;; +esac + +case "$timestamp" in + true|false|none) ;; + *) + echo "--timestamp must be true, false, or none, got '$timestamp'." >&2 + exit 2 + ;; +esac + +sign_with_codesign() { + if [[ -z "$identity" ]]; then + echo "Native codesign requires --identity." >&2 + exit 2 + fi + + local -a args + args=(--force) + + if [[ "$deep" == "true" ]]; then + args+=(--deep) + fi + + if [[ -n "$options" ]]; then + args+=(--options "$options") + fi + + case "$timestamp" in + true) + args+=(--timestamp) + ;; + false|none) + args+=(--timestamp=none) + ;; + esac + + if [[ -n "$entitlements_file" ]]; then + args+=(--entitlements "$entitlements_file") + fi + + if [[ -n "$identifier" ]]; then + args+=(--identifier "$identifier") + fi + + args+=(--sign "$identity" "$target") + codesign "${args[@]}" +} + +append_rcodesign_flags() { + local raw_options="$1" + local option="" + + if [[ -z "$raw_options" ]]; then + return 0 + fi + + IFS=',' read -ra split_options <<< "$raw_options" + for option in "${split_options[@]}"; do + option="${option//[[:space:]]/}" + [[ -z "$option" ]] && continue + + case "$option" in + host|hard|kill|expires|restrict|library|runtime|linker-signed) + rcodesign_args+=(--code-signature-flags "$option") + ;; + *) + echo "Unsupported rcodesign code signature option: $option" >&2 + exit 2 + ;; + esac + done +} + +rcodesign_options_require_notarization() { + local raw_options="$1" + local option="" + + if [[ -z "$raw_options" || "$timestamp" != "true" ]]; then + return 1 + fi + + IFS=',' read -ra split_options <<< "$raw_options" + for option in "${split_options[@]}"; do + option="${option//[[:space:]]/}" + if [[ "$option" == "runtime" ]]; then + return 0 + fi + done + + return 1 +} + +sign_with_rcodesign() { + : "${OAI_AKV_PKCS11_LIBRARY:?OAI_AKV_PKCS11_LIBRARY is required for AKV PKCS11 signing.}" + : "${OAI_AKV_SIGNING_CERTIFICATE_PEM:?OAI_AKV_SIGNING_CERTIFICATE_PEM is required for AKV PKCS11 signing.}" + : "${OAI_AKV_KEY_LABEL:?OAI_AKV_KEY_LABEL is required for AKV PKCS11 signing.}" + + if ! command -v rcodesign >/dev/null 2>&1; then + echo "rcodesign was not found on PATH." >&2 + exit 1 + fi + + local -a rcodesign_args + rcodesign_args=( + sign + --config-file /dev/null + --pkcs11-library "$OAI_AKV_PKCS11_LIBRARY" + --pkcs11-certificate-file "$OAI_AKV_SIGNING_CERTIFICATE_PEM" + --pkcs11-key-label "$OAI_AKV_KEY_LABEL" + ) + + if [[ "$deep" == "false" ]]; then + rcodesign_args+=(--shallow) + fi + + case "$timestamp" in + true) + ;; + false|none) + rcodesign_args+=(--timestamp-url none) + ;; + esac + + append_rcodesign_flags "$options" + if rcodesign_options_require_notarization "$options"; then + rcodesign_args+=(--for-notarization) + fi + + if [[ -n "$entitlements_file" ]]; then + rcodesign_args+=(--entitlements-xml-file "$entitlements_file") + fi + + if [[ -n "$identifier" ]]; then + rcodesign_args+=(--binary-identifier "$identifier") + fi + + rcodesign_args+=("$target") + rcodesign "${rcodesign_args[@]}" +} + +case "${OAI_CODESIGN_BACKEND:-codesign}" in + codesign|"") + sign_with_codesign + ;; + akv-pkcs11) + sign_with_rcodesign + ;; + *) + echo "Unsupported OAI_CODESIGN_BACKEND: ${OAI_CODESIGN_BACKEND}" >&2 + exit 2 + ;; +esac diff --git a/.github/scripts/macos-signing/test_notarize_with_akv.py b/.github/scripts/macos-signing/test_notarize_with_akv.py new file mode 100644 index 0000000000000000000000000000000000000000..7c3aa50534880e072785d6187da0946087f3a415 --- /dev/null +++ b/.github/scripts/macos-signing/test_notarize_with_akv.py @@ -0,0 +1,257 @@ +import contextlib +import hashlib +import io +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +SIGNING_DIRECTORY = Path(__file__).resolve().parent +sys.path.insert(0, str(SIGNING_DIRECTORY)) +import notarize_with_akv as notary # noqa: E402 + +CONFIGURATION = notary.NotarizationConfiguration( + "01234567-89ab-cdef-0123-456789abcdef", + "APPLEKEY01", + "notary-vault", + "example-signing-key", + "0123456789abcdef" * 2, +) +ENVIRONMENT = { + "AZURE_KEYVAULT_NAME": CONFIGURATION.vault_name, + "APPLE_NOTARIZATION_AKV_KEY_NAME": CONFIGURATION.vault_key_name, +} +CREDENTIALS = { + "awsAccessKeyId": "temporary-access-key", + "awsSecretAccessKey": "temporary-secret-key", + "awsSessionToken": "temporary-session-token", + "bucket": "apple-notary-bucket", + "object": "uploads/release artifact.dmg", +} + + +def response(payload): + return io.BytesIO( + json.dumps(payload).encode() if isinstance(payload, dict) else payload + ) + + +def azure_response(payload): + return subprocess.CompletedProcess([], 0, stdout=json.dumps(payload), stderr="") + + +class NotarizationTest(unittest.TestCase): + def test_validates_the_apple_key_tags_and_pins_its_version(self) -> None: + metadata = { + "id": CONFIGURATION.versioned_key_id, + "apple_key_id": "APPLEKEY01", + "apple_issuer_id": CONFIGURATION.issuer_id, + } + with ( + patch.dict(os.environ, ENVIRONMENT, clear=True), + patch.object( + notary.subprocess, + "run", + return_value=azure_response(metadata), + ) as show, + ): + self.assertEqual( + notary.NotarizationConfiguration.from_environment(), + CONFIGURATION, + ) + show.return_value = azure_response({**metadata, "apple_key_id": None}) + with self.assertRaisesRegex(notary.NotarizationError, "apple-key-id tag"): + notary.NotarizationConfiguration.from_environment() + for issuer_id in (None, "", " "): + with self.subTest(issuer_id=issuer_id): + show.return_value = azure_response( + {**metadata, "apple_issuer_id": issuer_id} + ) + with self.assertRaisesRegex( + notary.NotarizationError, "apple-issuer-id tag" + ): + notary.NotarizationConfiguration.from_environment() + show.return_value = azure_response( + {**metadata, "apple_issuer_id": "invalid-issuer"} + ) + with self.assertRaisesRegex(notary.NotarizationError, "valid UUID"): + notary.NotarizationConfiguration.from_environment() + self.assertIn( + '{id:key.kid,apple_key_id:tags."apple-key-id",' + 'apple_issuer_id:tags."apple-issuer-id"}', + show.call_args.args[0], + ) + + invalid_environment = { + **ENVIRONMENT, + "APPLE_NOTARIZATION_AKV_KEY_NAME": "../bad", + } + with ( + patch.dict(os.environ, invalid_environment, clear=True), + self.assertRaisesRegex(notary.NotarizationError, "key name"), + ): + notary.NotarizationConfiguration.from_environment() + + def test_signs_the_notary_scoped_digest_with_the_pinned_key(self) -> None: + signature = bytes(range(64)) + payload = { + "kid": CONFIGURATION.versioned_key_id, + "value": notary.base64url_encode(signature), + } + with patch.object( + notary.subprocess, "run", return_value=azure_response(payload) + ) as sign: + token = notary.create_apple_jwt(CONFIGURATION, issued_at=1_780_000_000) + header, claims, encoded_signature = token.split(".") + self.assertEqual( + json.loads(notary.base64url_decode(header))["kid"], "APPLEKEY01" + ) + self.assertEqual( + json.loads(notary.base64url_decode(claims))["iss"], CONFIGURATION.issuer_id + ) + self.assertEqual( + json.loads(notary.base64url_decode(claims))["scope"], + ["/notary/v2"], + ) + self.assertEqual(notary.base64url_decode(encoded_signature), signature) + command = sign.call_args.args[0] + body = json.loads(command[command.index("--body") + 1]) + self.assertEqual(body["alg"], "ES256") + self.assertEqual( + notary.base64url_decode(body["value"]), + hashlib.sha256(f"{header}.{claims}".encode()).digest(), + ) + + def test_rejects_wrong_key_versions_and_invalid_signatures(self) -> None: + cases = ( + ( + CONFIGURATION.versioned_key_id + "wrong", + bytes(64), + "unexpected key version", + ), + (CONFIGURATION.versioned_key_id, b"short", "64 JOSE"), + ) + for key_id, signature, message in cases: + payload = {"kid": key_id, "value": notary.base64url_encode(signature)} + with ( + self.subTest(message=message), + patch.object( + notary.subprocess, + "run", + return_value=azure_response(payload), + ), + self.assertRaisesRegex(notary.NotarizationError, message), + ): + notary.create_apple_jwt(CONFIGURATION, issued_at=1) + + def test_uploads_polls_and_retains_sanitized_apple_diagnostics(self) -> None: + signature = azure_response( + { + "kid": CONFIGURATION.versioned_key_id, + "value": notary.base64url_encode(bytes(64)), + } + ) + responses = [ + response({"data": {"id": "submission-1", "attributes": CREDENTIALS}}), + response(b""), + response({"data": {"attributes": {"status": "In Progress"}}}), + response({"data": {"attributes": {"status": "Accepted"}}}), + response( + { + "data": { + "attributes": { + "developerLogUrl": "https://logs.example.com/log" + } + } + } + ), + response(b'{"status":"Accepted"}'), + ] + with tempfile.TemporaryDirectory() as directory: + artifact, log = ( + Path(directory) / "codex.zip", + Path(directory) / "notary-log.json", + ) + artifact.write_bytes(b"signed release binary") + output = io.StringIO() + with ( + patch.object(notary.subprocess, "run", return_value=signature), + patch.object( + notary.urllib.request, "urlopen", side_effect=responses + ) as requests, + patch.object(notary.time, "sleep") as sleep, + contextlib.redirect_stdout(output), + ): + submission = notary.notarize( + artifact, CONFIGURATION, max_wait_seconds=600, report_log=log + ) + self.assertEqual(submission, "submission-1") + self.assertEqual(json.loads(log.read_text()), {"status": "Accepted"}) + sleep.assert_called_once_with(10) + upload = requests.call_args_list[1].args[0] + self.assertEqual(upload.get_method(), "PUT") + self.assertNotIn("temporary-secret-key", upload.get_header("Authorization")) + self.assertNotIn("temporary-secret-key", output.getvalue()) + + +class NotarizationWrapperTest(unittest.TestCase): + def test_binary_and_dmg_wrappers_preserve_notarization_contracts( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + root, tools = Path(directory), Path(directory) / "tools" + tools.mkdir() + call_log = root / "calls.txt" + environment = { + **os.environ, + **ENVIRONMENT, + "PATH": f"{tools}:{os.environ['PATH']}", + "RUNNER_TEMP": str(root), + "CALL_LOG": str(call_log), + } + for name in ("az", "python3", "rcodesign"): + body = ( + "exit 0" + if name == "az" + else f'printf "{name} %s\\n" "$*" >> "$CALL_LOG"' + ) + executable = tools / name + executable.write_text( + f"#!/usr/bin/env bash\nset -euo pipefail\n{body}\n" + ) + executable.chmod(0o755) + for kind in ("binary", "dmg"): + with self.subTest(kind=kind): + artifact = root / ("codex.dmg" if kind == "dmg" else "codex") + artifact.write_bytes(b"signed release artifact") + subprocess.run( + [ + str( + SIGNING_DIRECTORY / f"notarize_macos_{kind}_with_akv.sh" + ), + f"--{kind}", + str(artifact), + "--report-dir", + str(root / "report"), + ], + env=environment, + check=True, + capture_output=True, + text=True, + ) + calls = call_log.read_text().splitlines() + self.assertIn("notarize_with_akv.py --file", calls[0]) + if kind == "dmg": + self.assertEqual(calls[1], f"rcodesign staple {artifact}") + else: + self.assertEqual(len(calls), 1) + self.assertEqual(list(root.rglob("*.p8")), []) + call_log.unlink() + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/macos-signing/test_provisioned_macos_cli_package.py b/.github/scripts/macos-signing/test_provisioned_macos_cli_package.py new file mode 100644 index 0000000000000000000000000000000000000000..1c6c266b53d89593f1b0368be44e20201d052a7c --- /dev/null +++ b/.github/scripts/macos-signing/test_provisioned_macos_cli_package.py @@ -0,0 +1,217 @@ +"""Exercise profile validation, relocation and signing without release credentials.""" + +import copy +import hashlib +import json +import plistlib +import shutil +import ssl +import subprocess +import tarfile +import tempfile +import unittest +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import provisioned_macos_cli_package as bundle + + +class ProvisioningTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + directory = tempfile.TemporaryDirectory() + cls.addClassCleanup(directory.cleanup) + root = Path(directory.name) + key = root / "test-key.pem" + certificate = root / "test-certificate.pem" + subprocess.run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + str(key), + "-out", + str(certificate), + "-days", + "1", + "-subj", + "/CN=CLI provisioning test/OU=TESTTEAM01", + ], + check=True, + capture_output=True, + ) + cert_der = ssl.PEM_cert_to_DER_cert(certificate.read_text()) + now = datetime.now(timezone.utc).replace(tzinfo=None, microsecond=0) + cls.profile = { + "TeamIdentifier": ["TESTTEAM01"], + "ApplicationIdentifierPrefix": ["TESTTEAM01"], + "Entitlements": { + "com.apple.application-identifier": "TESTTEAM01.com.openai.codex.cli", + "com.apple.developer.team-identifier": "TESTTEAM01", + "keychain-access-groups": ["TESTTEAM01.*"], + }, + "CreationDate": now - timedelta(days=1), + "ExpirationDate": now + timedelta(days=1), + "ProvisionsAllDevices": True, + "DeveloperCertificates": [cert_der], + } + payload = root / "profile.plist" + payload.write_bytes(plistlib.dumps(cls.profile)) + profile_path = root / "test.provisionprofile" + subprocess.run( + [ + "openssl", + "cms", + "-sign", + "-binary", + "-nodetach", + "-outform", + "DER", + "-in", + str(payload), + "-signer", + str(certificate), + "-inkey", + str(key), + "-out", + str(profile_path), + ], + check=True, + capture_output=True, + ) + cls.configuration = bundle.ProfileConfiguration( + profile=profile_path, + profile_sha256=hashlib.sha256(profile_path.read_bytes()).hexdigest(), + certificate_sha256=hashlib.sha256(cert_der).hexdigest(), + team_id="TESTTEAM01", + ) + + +class ProvisionedCliTests(ProvisioningTestCase): + def test_profile_configuration_fails_closed(self): + self.assertEqual(bundle.load_profile(self.configuration), self.profile) + for changes in ( + {"profile_sha256": "0" * 64}, + {"certificate_sha256": "0" * 64}, + {"team_id": "OTHERTEAM1"}, + {"profile_sha256": ""}, + {"certificate_sha256": ""}, + {"team_id": ""}, + ): + with self.subTest(changes=changes), self.assertRaises(ValueError): + bundle.load_profile(replace(self.configuration, **changes)) + + def test_rejects_profile_with_wrong_identity_certificate_or_validity(self): + for field, value in ( + ("TeamIdentifier", ["OTHERTEAM"]), + ("DeveloperCertificates", [b"another certificate"]), + ( + "ExpirationDate", + datetime(2000, 1, 1, tzinfo=timezone.utc).replace(tzinfo=None), + ), + ( + "CreationDate", + datetime(2099, 1, 1, tzinfo=timezone.utc).replace(tzinfo=None), + ), + ("ProvisionsAllDevices", False), + ("ProvisionedDevices", ["device"]), + ): + with self.subTest(field=field): + profile = copy.deepcopy(self.profile) + profile[field] = value + with self.assertRaises(ValueError): + bundle.validate_profile(profile, self.configuration) + for field, value in ( + ( + "com.apple.application-identifier", + f"{self.configuration.team_id}.another.app", + ), + ("keychain-access-groups", [f"{self.configuration.team_id}.another.app"]), + ("get-task-allow", True), + ): + with self.subTest(field=field): + profile = copy.deepcopy(self.profile) + profile["Entitlements"][field] = value + with self.assertRaises(ValueError): + bundle.validate_profile(profile, self.configuration) + + def test_archive_relocation_copy_and_symlink_preserve_launcher(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + package = root / "original package" + metadata = { + "variant": "codex", + "layoutVersion": 1, + "target": "aarch64-apple-darwin", + "entrypoint": "bin/codex", + } + for relative in ("bin/codex", *bundle.HELPERS): + path = package / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('#!/bin/sh\nprintf "%s\\n" "$@"\nexit 23\n') + path.chmod(0o755) + (package / "codex-package.json").write_text(json.dumps(metadata)) + original_binary = (package / "bin/codex").read_bytes() + bundle.prepare(package, root / "reports", self.configuration) + with self.assertRaises(FileExistsError): + bundle.prepare(package, root / "reports", self.configuration) + with self.assertRaisesRegex(ValueError, "Package target differs"): + bundle.verify( + package, root / "reports", "x86_64-apple-darwin", self.configuration + ) + self.assertEqual( + (package / bundle.EXECUTABLE).read_bytes(), original_binary + ) + self.assertEqual( + json.loads((package / "codex-package.json").read_text()), metadata + ) + + archive_path = root / "package.tar.gz" + with tarfile.open(archive_path, "w:gz") as archive: + archive.add(package, arcname=".") + relocated = root / "relocated package" + with tarfile.open(archive_path) as archive: + archive.extractall(relocated, filter="data") + shutil.rmtree(package) + # npm currently dereferences symlinks when copying packages. The + # launcher remains a script and the provisioned executable stays put. + copied = root / "copied package" + shutil.copytree(relocated, copied) + link = root / "installed codex" + link.symlink_to(Path("copied package/bin/codex")) + absolute_link = root / "absolute codex" + absolute_link.symlink_to(link) + for entry in ( + relocated / "bin/codex", + copied / "bin/codex", + link, + absolute_link, + ): + with self.subTest(entry=entry): + result = subprocess.run( + [str(entry), "space argument", "*", "--flag"], + cwd="/", + capture_output=True, + text=True, + check=False, + ) + self.assertEqual( + (result.returncode, result.stdout, result.stderr), + (23, "space argument\n*\n--flag\n", ""), + ) + (copied / bundle.APP / "Contents/embedded.provisionprofile").write_bytes( + b"tampered" + ) + with self.assertRaisesRegex(ValueError, "Embedded profile differs"): + bundle.verify( + copied, root / "reports", "aarch64-apple-darwin", self.configuration + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/macos-signing/test_sign_macos_cli_package.py b/.github/scripts/macos-signing/test_sign_macos_cli_package.py new file mode 100644 index 0000000000000000000000000000000000000000..38efc120983edab2a1a8108697427621c181a5d3 --- /dev/null +++ b/.github/scripts/macos-signing/test_sign_macos_cli_package.py @@ -0,0 +1,293 @@ +"""Exercise package signing and native verification with generated credentials.""" + +import itertools +import json +import os +import shutil +import ssl +import subprocess +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path + +import provisioned_macos_cli_package as bundle +from test_provisioned_macos_cli_package import ProvisioningTestCase + +NATIVE_TOOL = """#!/usr/bin/env python3 +import json, os, plistlib, sys +from pathlib import Path +name = Path(sys.argv[0]).name +args = sys.argv[1:] +with open(os.environ["CALL_LOG"], "a") as log: + log.write(json.dumps([name, *args]) + "\\n") +if name == "codesign": + for arg in args: + if arg.startswith("--extract-certificates="): + Path(arg.split("=", 1)[1] + "0").write_bytes(Path(os.environ["MOCK_CERTIFICATE"]).read_bytes()) + if "--test-requirement" in args and os.environ.get("MOCK_WRONG_SIGNING_IDENTITY"): + print("code failed to satisfy specified code requirement(s)", file=sys.stderr) + sys.exit(1) + if "--entitlements" in args: + target = Path(args[-1]).name + if target == "CodexCLI.app": + source = Path(os.environ["CODEX_REPO_ROOT"]) / "signing-verification/codex-provisioned-entitlements.plist" + elif target in ("codex", "codex-code-mode-host"): + source = Path(os.environ["MOCK_SIGNING_SCRIPTS"]) / (target + ".entitlements.plist") + else: + sys.exit(0) + if os.environ.get("MOCK_WRONG_ENTITLEMENTS"): + sys.stdout.buffer.write(plistlib.dumps({"unexpected": True})) + else: + sys.stdout.buffer.write(source.read_bytes()) +elif name == "plutil": + source = Path(args[-1]) + target = Path(args[args.index("-o") + 1]) if "-o" in args else source + target.write_bytes(plistlib.dumps(plistlib.loads(source.read_bytes()))) +""" + + +class MacosSigningTests(ProvisioningTestCase): + def fixture(self, root, provisioned, target="aarch64-apple-darwin"): + source = Path(bundle.__file__).parent + common = Path(os.path.commonpath((source, bundle.SIGNING))) + helpers = root / source.relative_to(common) + scripts = root / bundle.SIGNING.relative_to(common) + helpers.mkdir(parents=True, exist_ok=True) + scripts.mkdir(parents=True, exist_ok=True) + for name in ("provisioned_macos_cli_package.py", "sign_macos_cli_package.py"): + shutil.copyfile(source / name, helpers / name) + for path in bundle.SIGNING.glob("*.entitlements.plist"): + shutil.copyfile(path, scripts / path.name) + (root / "record.py").write_text( + "import json, os, sys\n" + "with open(os.environ['CALL_LOG'], 'a') as log:\n" + " log.write(json.dumps(sys.argv[1:]) + '\\n')\n" + "if os.environ.get('FAIL_TOOL') == sys.argv[1]:\n" + " print('fixture stdout', flush=True)\n" + " print('fixture stderr', file=sys.stderr, flush=True)\n" + " sys.exit(17)\n" + ) + for name in ("sign_macos_code.sh", "notarize_macos_binary_with_akv.sh"): + (scripts / name).write_text( + 'python3 "$CODEX_REPO_ROOT/record.py" "$(basename "$0")" "$@"\n' + ) + (scripts / "notarize_with_akv.py").write_text( + "import os, runpy, sys\n" + "sys.argv.insert(1, 'notarize_with_akv.py')\n" + "runpy.run_path(os.environ['CODEX_REPO_ROOT'] + '/record.py')\n" + ) + for name in ("rcodesign", "codesign", "lipo", "plutil"): + path = root / name + path.write_text(NATIVE_TOOL) + path.chmod(0o755) + certificate_der = self.profile["DeveloperCertificates"][0] + certificate = root / "certificate.pem" + certificate.write_text(ssl.DER_cert_to_PEM_cert(certificate_der)) + (root / "certificate.der").write_bytes(certificate_der) + package = root / "package with spaces" + for relative in ("bin/codex", *bundle.HELPERS): + path = package / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("#!/bin/sh\nexit 0\n") + path.chmod(0o755) + (package / "codex-package.json").write_text( + json.dumps( + { + "variant": "codex", + "layoutVersion": 1, + "target": target, + "entrypoint": "bin/codex", + } + ) + ) + env = { + key: value + for key, value in os.environ.items() + if key not in ("GITHUB_WORKSPACE", "CODEX_PROVISIONING_HELPER") + } + env.update( + CODEX_REPO_ROOT=str(root), + RUNNER_TEMP=str(root), + TARGET=target, + PATH=f"{root}{os.pathsep}{os.environ['PATH']}", + CALL_LOG=str(root / "calls.jsonl"), + PROVISIONED_MACOS=str(provisioned).lower(), + OAI_AKV_SIGNING_CERTIFICATE_PEM=str(certificate), + MOCK_CERTIFICATE=str(root / "certificate.der"), + MOCK_SIGNING_SCRIPTS=str(scripts), + ) + return package, helpers / "sign_macos_cli_package.py", env + + def run_driver(self, driver, operation, package, env): + return subprocess.run( + [ + sys.executable, + str(driver), + operation, + str(package), + "--profile", + str(self.configuration.profile), + "--profile-sha256", + self.configuration.profile_sha256, + "--certificate-sha256", + self.configuration.certificate_sha256, + "--team-id", + self.configuration.team_id, + ], + env=env, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + def test_standard_and_provisioned_signing_and_verification(self): + for provisioned, target in itertools.product( + (False, True), ("aarch64-apple-darwin", "x86_64-apple-darwin") + ): + with ( + self.subTest(provisioned=provisioned, target=target), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory).resolve() + package, driver, env = self.fixture(root, provisioned, target) + result = self.run_driver(driver, "sign", package, env) + self.assertEqual(result.returncode, 0, result.stderr) + result = self.run_driver(driver, "verify", package, env) + self.assertEqual(result.returncode, 0, result.stderr) + calls = [ + json.loads(line) + for line in (root / "calls.jsonl").read_text().splitlines() + ] + signing = [call for call in calls if call[0] == "sign_macos_code.sh"] + self.assertEqual( + [call[call.index("--target") + 1] for call in signing], + [ + str(package / relative) + for relative in ( + bundle.APP if provisioned else "bin/codex", + *bundle.HELPERS, + ) + ], + ) + self.assertEqual( + signing[0][signing[0].index("--entitlements") + 1], + str( + root + / "signing-verification/codex-provisioned-entitlements.plist" + ) + if provisioned + else str( + Path(env["MOCK_SIGNING_SCRIPTS"]) / "codex.entitlements.plist" + ), + ) + self.assertEqual( + ["--entitlements" in call for call in signing], + [True, True, False, False], + ) + self.assertEqual( + [call[0] for call in calls if call[0].startswith("notarize")], + ["notarize_with_akv.py"] + if provisioned + else ["notarize_macos_binary_with_akv.sh"] * 4, + ) + self.assertEqual( + [call[1:] for call in calls if call[0] == "lipo"], + [ + [ + str(package / relative), + "-verify_arch", + "arm64" if target.startswith("aarch64") else "x86_64", + ] + for relative in ( + bundle.EXECUTABLE if provisioned else "bin/codex", + *bundle.HELPERS, + ) + ], + ) + if provisioned: + with zipfile.ZipFile(root / "provisioned-cli.zip") as archive: + self.assertEqual( + archive.read( + "CodexCLI.app/Contents/embedded.provisionprofile" + ), + self.configuration.profile.read_bytes(), + ) + self.assertEqual( + archive.read("bin/codex").decode(), bundle.LAUNCHER + ) + self.assertTrue( + all( + helper in archive.namelist() + for helper in bundle.HELPERS + ) + ) + + def test_signing_and_notarization_failures_stop_the_driver(self): + for tool in ("sign_macos_code.sh", "notarize_with_akv.py"): + with self.subTest(tool=tool), tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + package, driver, env = self.fixture(root, True) + result = self.run_driver( + driver, "sign", package, {**env, "FAIL_TOOL": tool} + ) + self.assertNotEqual(result.returncode, 0) + calls = [ + json.loads(line) + for line in (root / "calls.jsonl").read_text().splitlines() + ] + self.assertEqual(calls[-1][0], tool) + if tool == "notarize_with_akv.py": + self.assertEqual( + (root / "signing-verification/notarization.log").read_text(), + "fixture stdout\nfixture stderr\n", + ) + self.assertIn("fixture stdout\nfixture stderr\n", result.stdout) + else: + self.assertFalse((root / "provisioned-cli.zip").exists()) + + def test_verification_rejects_wrong_signing_identity_and_entitlements(self): + for provisioned, failure in itertools.product( + (False, True), ("MOCK_WRONG_SIGNING_IDENTITY", "MOCK_WRONG_ENTITLEMENTS") + ): + with ( + self.subTest(provisioned=provisioned, failure=failure), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory).resolve() + package, driver, env = self.fixture(root, provisioned) + signed = self.run_driver(driver, "sign", package, env) + self.assertEqual(signed.returncode, 0, signed.stderr) + result = self.run_driver( + driver, "verify", package, {**env, failure: "true"} + ) + self.assertNotEqual(result.returncode, 0) + if failure == "MOCK_WRONG_SIGNING_IDENTITY": + self.assertIn( + "code failed to satisfy specified code requirement(s)", + result.stderr, + ) + else: + self.assertIn("unexpected", result.stdout) + + def test_cli_requires_independent_profile_configuration(self): + with tempfile.TemporaryDirectory() as directory: + result = subprocess.run( + [ + sys.executable, + str(Path(bundle.__file__).with_name("sign_macos_cli_package.py")), + "verify", + directory, + ], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode, 2) + self.assertIn("the following arguments are required", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/CLA.md b/docs/CLA.md new file mode 100644 index 0000000000000000000000000000000000000000..804f202c0d369d676abefaf0999dfdd1f572127f --- /dev/null +++ b/docs/CLA.md @@ -0,0 +1,49 @@ +# Individual Contributor License Agreement (v1.0, OpenAI) + +_Based on the Apache Software Foundation Individual CLA v 2.2._ + +By commenting **“I have read the CLA Document and I hereby sign the CLA”** +on a Pull Request, **you (“Contributor”) agree to the following terms** for any +past and future “Contributions” submitted to the **OpenAI Codex CLI project +(the “Project”)**. + +--- + +## 1. Definitions + +- **“Contribution”** – any original work of authorship submitted to the Project + (code, documentation, designs, etc.). +- **“You” / “Your”** – the individual (or legal entity) posting the acceptance + comment. + +## 2. Copyright License + +You grant **OpenAI, Inc.** and all recipients of software distributed by the +Project a perpetual, worldwide, non‑exclusive, royalty‑free, irrevocable +license to reproduce, prepare derivative works of, publicly display, publicly +perform, sublicense, and distribute Your Contributions and derivative works. + +## 3. Patent License + +You grant **OpenAI, Inc.** and all recipients of the Project a perpetual, +worldwide, non‑exclusive, royalty‑free, irrevocable (except as below) patent +license to make, have made, use, sell, offer to sell, import, and otherwise +transfer Your Contributions alone or in combination with the Project. + +If any entity brings patent litigation alleging that the Project or a +Contribution infringes a patent, the patent licenses granted by You to that +entity under this CLA terminate. + +## 4. Representations + +1. You are legally entitled to grant the licenses above. +2. Each Contribution is either Your original creation or You have authority to + submit it under this CLA. +3. Your Contributions are provided **“AS IS”** without warranties of any kind. +4. You will notify the Project if any statement above becomes inaccurate. + +## 5. Miscellany + +This Agreement is governed by the laws of the **State of California**, USA, +excluding its conflict‑of‑laws rules. If any provision is held unenforceable, +the remaining provisions remain in force. diff --git a/docs/agents_md.md b/docs/agents_md.md new file mode 100644 index 0000000000000000000000000000000000000000..4fa02abd1d78a779dc8b6295e69429bcb5d7a129 --- /dev/null +++ b/docs/agents_md.md @@ -0,0 +1,3 @@ +# AGENTS.md + +For information about AGENTS.md, see [this documentation](https://developers.openai.com/codex/guides/agents-md). diff --git a/docs/authentication.md b/docs/authentication.md new file mode 100644 index 0000000000000000000000000000000000000000..c30734976685465ee3585e483752f15e90bde6f5 --- /dev/null +++ b/docs/authentication.md @@ -0,0 +1,3 @@ +# Authentication + +For information about Codex CLI authentication, see [this documentation](https://developers.openai.com/codex/auth). diff --git a/docs/config.md b/docs/config.md new file mode 100644 index 0000000000000000000000000000000000000000..d35b0a87e3c2286a4f0d87a62414d0cec4e742b4 --- /dev/null +++ b/docs/config.md @@ -0,0 +1,15 @@ +# Configuration + +For basic configuration instructions, see [this documentation](https://developers.openai.com/codex/config-basic). + +For advanced configuration instructions, see [this documentation](https://developers.openai.com/codex/config-advanced). + +For a full configuration reference, see [this documentation](https://developers.openai.com/codex/config-reference). + +## Lifecycle hooks + +Admins can set top-level `allow_managed_hooks_only = true` in +`requirements.toml` to ignore user, project, and session hook configs while +still allowing managed hooks from requirements and managed config layers. This +setting is only supported in `requirements.toml`; putting it in `config.toml` +does not enable managed-hooks-only mode. diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 0000000000000000000000000000000000000000..67b10a1c9b855bc32082929129b5abbe3c1d7b35 --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,39 @@ +## Contributing + +We welcome community contributions through the [openai/codex issue tracker](https://github.com/openai/codex/issues). Bug reports, root-cause analyses, and feature requests help us understand what matters most and improve Codex. + +**We do not accept external code contributions or pull requests.** + +### Why we do not accept external code contributions + +Effective changes to Codex require architectural context, an understanding of system-level constraints, and visibility into the project's roadmap. External pull requests often focus on issues that are lower priority, affect a small number of users, or need substantial changes to fit the broader system. Reviewing and iterating on those changes can take more time than implementing a fix directly, diverting attention from higher-priority work. + +Community expertise is most valuable when shared through detailed bug reports, reproduction steps, logs, root-cause analysis, and design discussions in issues. Understanding the problem, identifying the right solution, and prioritizing the work are typically the hard parts; implementation is comparatively straightforward with the help of Codex itself. + +For these reasons, we focus community contributions on issue reports, analysis, and feedback, while the Codex team handles code changes. + +### Reporting bugs + +Before opening a new issue, search the issue tracker to see whether the problem has already been reported. If it has, add any new information to the existing issue. + +When reporting a bug, include as much relevant detail as possible: + +- Clear, detailed steps to reproduce the problem. +- Expected and actual behavior. +- Your Codex version, operating system, and other relevant environment details. +- Logs, error messages, or other diagnostic information, with sensitive information removed. +- Root-cause analysis, technical observations, or potential approaches to a fix, if available. + +### Requesting features + +Open a feature request in the issue tracker, or upvote an existing request that describes the same need. Explain your use case, the behavior you would like, and why it would improve your workflow. + +### Community values + +- **Be kind and inclusive.** Treat others with respect; we follow the [Contributor Covenant](https://www.contributor-covenant.org/). +- **Assume good intent.** Written communication is hard, so err on the side of generosity. +- **Share what you learn.** Reproduction details, logs, and analysis help the entire community. + +### Security + +If you discover a security vulnerability, follow the [security policy](../SECURITY.md) instead of reporting it in a public issue. diff --git a/docs/example-config.md b/docs/example-config.md new file mode 100644 index 0000000000000000000000000000000000000000..84b11436c58b86ea9800f5ea72d731a90e3febaf --- /dev/null +++ b/docs/example-config.md @@ -0,0 +1,3 @@ +# Sample configuration + +For a sample configuration file, see [this documentation](https://developers.openai.com/codex/config-sample). diff --git a/docs/exec.md b/docs/exec.md new file mode 100644 index 0000000000000000000000000000000000000000..57e432305bf70930fbbcbe808ab6c160dd266286 --- /dev/null +++ b/docs/exec.md @@ -0,0 +1,3 @@ +# Non-interactive mode + +For information about non-interactive mode, see [this documentation](https://developers.openai.com/codex/noninteractive). diff --git a/docs/execpolicy.md b/docs/execpolicy.md new file mode 100644 index 0000000000000000000000000000000000000000..cafebb32ee4f77d8daed27c11572cbe6c63d501c --- /dev/null +++ b/docs/execpolicy.md @@ -0,0 +1,3 @@ +# Execution policy + +For an overview of execution policy rules, see [this documentation](https://developers.openai.com/codex/exec-policy). diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000000000000000000000000000000000000..b2ea5f26426baeaa551a41c220075b84c4d6c559 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,3 @@ +# Getting started with Codex CLI + +For an overview of Codex CLI features, see [this documentation](https://developers.openai.com/codex/cli/features#running-in-interactive-mode). diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 0000000000000000000000000000000000000000..4f63765b5e72a5085585cb6c5b01eecb04200807 --- /dev/null +++ b/docs/install.md @@ -0,0 +1,65 @@ +## Installing & building + +### System requirements + +| Requirement | Details | +| --------------------------- | --------------------------------------------------------------- | +| Operating systems | macOS 12+, Ubuntu 20.04+/Debian 10+, or Windows 11 **via WSL2** | +| Git (optional, recommended) | 2.23+ for built-in PR helpers | +| RAM | 4-GB minimum (8-GB recommended) | + +### DotSlash + +The GitHub Release also contains a [DotSlash](https://dotslash-cli.com/) file for the Codex CLI named `codex`. Using a DotSlash file makes it possible to make a lightweight commit to source control to ensure all contributors use the same version of an executable, regardless of what platform they use for development. + +### Build from source + +```bash +# Clone the repository and navigate to the root of the Cargo workspace. +git clone https://github.com/openai/codex.git +cd codex/codex-rs + +# Install the Rust toolchain, if necessary. +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +source "$HOME/.cargo/env" +rustup component add rustfmt +rustup component add clippy +# Install helper tools used by the workspace justfile: +cargo install --locked just +# DotSlash fetches pinned development tools such as buildifier on first use. +cargo install --locked dotslash +# Install nextest for the `just test` helper. +cargo install --locked cargo-nextest + +# Build Codex. +cargo build + +# Launch the TUI with a sample prompt. +cargo run --bin codex -- "explain this codebase to me" + +# After making changes, use the root justfile helpers (they default to codex-rs): +just fmt +just fix -p + +# Run the relevant tests (project-specific is fastest), for example: +just test -p codex-tui +# `just test` runs the test suite via nextest: +just test +# Avoid `--all-features` for routine local runs because it increases build +# time and `target/` disk usage by compiling additional feature combinations. +``` + +## Tracing / verbose logging + +Codex is written in Rust, so it honors the `RUST_LOG` environment variable to configure its logging behavior. + +The TUI records diagnostics in bounded local stores by default. Set `log_dir` explicitly to enable a plaintext TUI log for a run: + +```bash +codex -c log_dir=./.codex-log +tail -F ./.codex-log/codex-tui.log +``` + +The non-interactive mode (`codex exec`) defaults to `RUST_LOG=error`, but messages are printed inline, so there is no need to monitor a separate file. + +See the Rust documentation on [`RUST_LOG`](https://docs.rs/env_logger/latest/env_logger/#enabling-logging) for more information on the configuration options. diff --git a/docs/license.md b/docs/license.md new file mode 100644 index 0000000000000000000000000000000000000000..18ad62af03adb533a195b7a6014799831e7cc907 --- /dev/null +++ b/docs/license.md @@ -0,0 +1,3 @@ +## License + +This repository is licensed under the [Apache-2.0 License](../LICENSE). diff --git a/docs/open-source-fund.md b/docs/open-source-fund.md new file mode 100644 index 0000000000000000000000000000000000000000..2da0cdce3e57dfe9f450075cbf9e5e4967031682 --- /dev/null +++ b/docs/open-source-fund.md @@ -0,0 +1,8 @@ +## Codex open source fund + +We're excited to launch a **$1 million initiative** supporting open source projects that use Codex CLI and other OpenAI models. + +- Grants are awarded up to **$25,000** API credits. +- Applications are reviewed **on a rolling basis**. + +**Interested? [Apply here](https://openai.com/form/codex-open-source-fund/).** diff --git a/docs/sandbox.md b/docs/sandbox.md new file mode 100644 index 0000000000000000000000000000000000000000..f5ca8d89bd2ef690e128bb0c299d5e5d9cee5085 --- /dev/null +++ b/docs/sandbox.md @@ -0,0 +1,3 @@ +## Sandbox & approvals + +For information about Codex sandboxing and approvals, see [this documentation](https://developers.openai.com/codex/security). diff --git a/docs/skills.md b/docs/skills.md new file mode 100644 index 0000000000000000000000000000000000000000..a45e7d73917e4a5b67e3f417f2caaa8db6be248d --- /dev/null +++ b/docs/skills.md @@ -0,0 +1,3 @@ +# Skills + +For information about skills, refer to [this documentation](https://developers.openai.com/codex/skills). diff --git a/docs/slash_commands.md b/docs/slash_commands.md new file mode 100644 index 0000000000000000000000000000000000000000..4db63f7f6e6a2d7b8447e311b0dce6279426313c --- /dev/null +++ b/docs/slash_commands.md @@ -0,0 +1,3 @@ +# Slash commands + +For an overview of Codex CLI slash commands, see [this documentation](https://developers.openai.com/codex/cli/slash-commands). diff --git a/patches/BUILD.bazel b/patches/BUILD.bazel new file mode 100644 index 0000000000000000000000000000000000000000..7b58251d8a1c66961e9fb6cc386ef9fced689f16 --- /dev/null +++ b/patches/BUILD.bazel @@ -0,0 +1,32 @@ +exports_files([ + "abseil_windows_gnullvm_thread_identity.patch", + "bzip2_windows_stack_args.patch", + "llvm_rusty_v8_custom_libcxx.patch", + "llvm_windows_arm64_powl.patch", + "llvm_windows_mingw_compat.patch", + "rules_rust_build_script_tools_transition.patch", + "rules_rust_group_build_script_arg_files.patch", + "rules_rust_windows_execroot_separators.patch", + "rules_rust_windows_msvc_direct_link_args.patch", + "rules_rust_windows_process_wrapper_skip_temp_outputs.patch", + "rules_cc_rusty_v8_custom_libcxx.patch", + "rules_foreign_cc_make_cppflags.patch", + "rules_foreign_cc_make_xcompile.patch", + "rules_foreign_cc_make_msvc_stdint.patch", + "rules_rs_build_script_deps_annotation.patch", + "rules_rs_windows_msvc_linker.patch", + "rules_rs_zlib_snapshot_urls.patch", + "rusty_v8_prebuilt_out_dir.patch", + "ring_windows_msvc_include_dirs.patch", + "tikv-jemalloc-sys_make_env.patch", + "tikv-jemalloc-sys_dependency_flags.patch", + "v8_bazel_rules.patch", + "v8_module_deps.patch", + "v8_source_portability.patch", + "webrtc-sys_hermetic_darwin_sysroot.patch", + "windows-link.patch", + "windows-support-remote-case-aliases.patch", + "windows-support-native-tools.patch", + "xz_windows_stack_args.patch", + "zstd-sys_windows_msvc_include_dirs.patch", +]) diff --git a/patches/abseil_windows_gnullvm_thread_identity.patch b/patches/abseil_windows_gnullvm_thread_identity.patch new file mode 100644 index 0000000000000000000000000000000000000000..27c4ecf71f788897115405cf38d0d218695a0efc --- /dev/null +++ b/patches/abseil_windows_gnullvm_thread_identity.patch @@ -0,0 +1,16 @@ +# What: avoid MinGW pthread TLS linkage in Abseil thread identity selection. +# Scope: one MinGW-specific preprocessor branch in Abseil's internal +# thread-identity mode selection; no other Abseil behavior changes. + +diff --git a/absl/base/internal/thread_identity.h b/absl/base/internal/thread_identity.h +--- a/absl/base/internal/thread_identity.h ++++ b/absl/base/internal/thread_identity.h +@@ -217,6 +217,8 @@ + #error ABSL_THREAD_IDENTITY_MODE cannot be directly set + #elif defined(ABSL_FORCE_THREAD_IDENTITY_MODE) + #define ABSL_THREAD_IDENTITY_MODE ABSL_FORCE_THREAD_IDENTITY_MODE ++#elif defined(__MINGW32__) ++#define ABSL_THREAD_IDENTITY_MODE ABSL_THREAD_IDENTITY_MODE_USE_CPP11 + #elif defined(_WIN32) && !defined(__MINGW32__) + #define ABSL_THREAD_IDENTITY_MODE ABSL_THREAD_IDENTITY_MODE_USE_CPP11 + #elif defined(__APPLE__) && defined(ABSL_HAVE_THREAD_LOCAL) diff --git a/patches/bzip2_windows_stack_args.patch b/patches/bzip2_windows_stack_args.patch new file mode 100644 index 0000000000000000000000000000000000000000..2786e908be38fc96d884885585c9f3a87973d400 --- /dev/null +++ b/patches/bzip2_windows_stack_args.patch @@ -0,0 +1,23 @@ +diff --git a/BUILD.bazel b/BUILD.bazel +--- a/BUILD.bazel ++++ b/BUILD.bazel +@@ -28,4 +28,11 @@ cc_library( + defines = [ + "_FILE_OFFSET_BITS=64", + ], ++ copts = select({ ++ "@platforms//os:windows": [ ++ "-fno-stack-protector", ++ "-mno-stack-arg-probe", ++ ], ++ "//conditions:default": [], ++ }), + includes = ["."], +diff --git a/MODULE.bazel b/MODULE.bazel +--- a/MODULE.bazel ++++ b/MODULE.bazel +@@ -4,3 +4,4 @@ module( + ) + + bazel_dep(name = "rules_cc", version = "0.0.10") ++bazel_dep(name = "platforms", version = "1.0.0") diff --git a/patches/llvm_rusty_v8_custom_libcxx.patch b/patches/llvm_rusty_v8_custom_libcxx.patch new file mode 100644 index 0000000000000000000000000000000000000000..dc5f4e652da38c4eceae1296b5ce55a6b13255b9 --- /dev/null +++ b/patches/llvm_rusty_v8_custom_libcxx.patch @@ -0,0 +1,108 @@ +diff --git a/toolchain/BUILD.bazel b/toolchain/BUILD.bazel +index fa00156236..c9e4348172 100644 +--- a/toolchain/BUILD.bazel ++++ b/toolchain/BUILD.bazel +@@ -1,5 +1,6 @@ + load("@bazel_lib//:bzl_library.bzl", "bzl_library") ++load("@bazel_skylib//lib:selects.bzl", "selects") + load("@bazel_skylib//rules:common_settings.bzl", "string_flag", "string_setting") + load("@rules_cc//cc/toolchains:actions.bzl", "cc_action_type_set") + load("@rules_cc//cc/toolchains:artifacts.bzl", "cc_artifact_name_pattern") + load("@rules_cc//cc/toolchains/impl:documented_api.bzl", "cc_args_list") +@@ -64,6 +65,15 @@ config_setting( + visibility = ["//visibility:public"], + ) +- ++ ++selects.config_setting_group( ++ name = "runtimes_all_with_rusty_v8_custom_libcxx", ++ match_all = [ ++ ":runtimes_all", ++ "@@//third_party/v8:use_rusty_v8_custom_libcxx", ++ ], ++ visibility = ["//visibility:public"], ++) ++ + config_setting( + name = "runtimes_stage1", + flag_values = { +@@ -270,11 +280,16 @@ cc_args_list( + + # TODO(cerisier): extract those into proper semantic args list. + # TODO(zbarsky): This must match llvm/toolchains/llvm.bzl ++_DEFAULT_CXXSTDLIB_HEADER_ARGS = select({ ++ "@@//third_party/v8:use_rusty_v8_custom_libcxx": [], ++ "//conditions:default": [ ++ "//toolchain/args:cxxstdlib_headers_include_search_paths", ++ ], ++}) ++ + cc_args_list( + name = "linux_toolchain_args", +- args = [ +- "//toolchain/args:cxxstdlib_headers_include_search_paths", +- ] + select({ ++ args = _DEFAULT_CXXSTDLIB_HEADER_ARGS + select({ + "//platforms/config:musl": [ + "//toolchain/args/linux:kernel_headers_include_search_paths", + "//toolchain/args/linux:musl_libc_headers_include_search_paths", +@@ -302,8 +317,7 @@ cc_args_list( + # TODO(zbarsky): This must match llvm/toolchains/llvm.bzl + cc_args_list( + name = "windows_toolchain_args", +- args = [ +- "//toolchain/args:cxxstdlib_headers_include_search_paths", ++ args = _DEFAULT_CXXSTDLIB_HEADER_ARGS + [ + "//toolchain/args/windows:mingw_headers_include_search_paths", + ], + ) +diff --git a/toolchain/llvm/llvm.bzl b/toolchain/llvm/llvm.bzl +index d36d8b94bd..900aa908d5 100644 +--- a/toolchain/llvm/llvm.bzl ++++ b/toolchain/llvm/llvm.bzl +@@ -297,18 +297,23 @@ def declare_llvm_targets(*, suffix = ""): + ], + ) + ++ default_cxxstdlib_target_headers = select({ ++ "@llvm//toolchain:runtimes_all_with_rusty_v8_custom_libcxx": [], ++ "@llvm//toolchain:runtimes_stage1": [], ++ "@llvm//toolchain:runtimes_stage1_hosted": [], ++ "@llvm//toolchain:runtimes_none": [], ++ "//conditions:default": [ ++ "@llvm//runtimes/cxxstdlib:headers_include_search_directory", ++ "@llvm//runtimes/cxxstdlib:abi_headers_include_search_directory", ++ ], ++ }) ++ + # This must match //toolchain:linux_toolchain_args + include_path( + name = "linux_target_headers", + srcs = [ + ":builtin_resource_dir", +- ] + select({ +- "@llvm//toolchain:runtimes_all": [ +- "@llvm//runtimes/cxxstdlib:headers_include_search_directory", +- "@llvm//runtimes/cxxstdlib:abi_headers_include_search_directory", +- ], +- "//conditions:default": [], +- }) + [ ++ ] + default_cxxstdlib_target_headers + [ + "@kernel_headers//:kernel_headers_directory", + "@llvm//sanitizers:sanitizers_headers_include_search_directory", + ] + select({ +@@ -324,13 +329,7 @@ def declare_llvm_targets(*, suffix = ""): + name = "windows_target_headers", + srcs = [ + ":builtin_resource_dir", +- ] + select({ +- "@llvm//toolchain:runtimes_all": [ +- "@llvm//runtimes/cxxstdlib:headers_include_search_directory", +- "@llvm//runtimes/cxxstdlib:abi_headers_include_search_directory", +- ], +- "//conditions:default": [], +- }) + [ ++ ] + default_cxxstdlib_target_headers + [ + "@mingw//:mingw_generated_headers_crt_directory", + "@mingw//:mingw_w64_headers_include_directory", + "@mingw//:mingw_w64_headers_crt_directory", diff --git a/patches/llvm_windows_arm64_powl.patch b/patches/llvm_windows_arm64_powl.patch new file mode 100644 index 0000000000000000000000000000000000000000..0fd33c97667bcb2300330f6b4f980d4971c47071 --- /dev/null +++ b/patches/llvm_windows_arm64_powl.patch @@ -0,0 +1,25 @@ +# What: provide and link MinGW's ARM64 long-double `powl` compatibility shim. +# Why: Windows ARM64 uses 64-bit long doubles, but LLVM's Bazel source list +# omits the upstream `powl -> pow` implementation and its default link flags +# omit the MinGW extension archive that owns the shim. + +diff --git a/runtimes/mingw/crt_sources.bzl b/runtimes/mingw/crt_sources.bzl +--- a/runtimes/mingw/crt_sources.bzl ++++ b/runtimes/mingw/crt_sources.bzl +@@ -420,4 +420,5 @@ + MINGWEX_ARM64_SRCS = [ ++ "math/arm-common/powl.c", + "math/sincos.c", + "math/sincosf.c", + "math/arm-common/sincosl.c", +diff --git a/toolchain/args/windows/BUILD.bazel b/toolchain/args/windows/BUILD.bazel +--- a/toolchain/args/windows/BUILD.bazel ++++ b/toolchain/args/windows/BUILD.bazel +@@ -45,5 +45,6 @@ cc_args( + args = [ + "-L{mingw_import_library_search_path}", + "-L{mingw_crt_library_search_path}", +- ++ "-lmingwex", ++ + # Clang will respect the user's chosen CRT variant. diff --git a/patches/llvm_windows_mingw_compat.patch b/patches/llvm_windows_mingw_compat.patch new file mode 100644 index 0000000000000000000000000000000000000000..66cf4e7b3446c99838eafd856a63d9047ae35aa1 --- /dev/null +++ b/patches/llvm_windows_mingw_compat.patch @@ -0,0 +1,50 @@ +# What: provide MinGW compatibility archives required by Windows gnullvm links. +# Why: clang and prebuilt MSVC archives can request these names even though the +# hermetic MinGW runtime does not publish them by default. + +diff --git a/runtimes/mingw/BUILD.bazel b/runtimes/mingw/BUILD.bazel +--- a/runtimes/mingw/BUILD.bazel ++++ b/runtimes/mingw/BUILD.bazel +@@ -386,14 +386,42 @@ stub_library( + name = "stdc++", + ) + ++# Clang may inject -lssp and -lssp_nonshared for windows-gnu links. ++# Provide compatibility archives in the MinGW runtime search directory. ++stub_library( ++ name = "ssp", ++) ++ ++stub_library( ++ name = "ssp_nonshared", ++) ++ ++# Prebuilt MSVC archives can carry /DEFAULTLIB:libcmt.lib and ++# /DEFAULTLIB:oldnames.lib directives. Provide MinGW-compatible stand-ins so ++# lld can satisfy the lookup under windows-gnullvm. ++stub_library( ++ name = "libcmt", ++) ++ ++copy_file( ++ name = "oldnames", ++ src = ":moldname", ++ out = "liboldnames.a", ++ visibility = ["//visibility:public"], ++) ++ + copy_to_directory( + name = "mingw_crt_library_search_directory", + srcs = [ ++ ":libcmt", + ":m", + ":mingw32", + ":mingwex", + ":moldname", ++ ":oldnames", + ":pthread", ++ ":ssp", ++ ":ssp_nonshared", + ":stdc++", + ":uuid", + ":winpthread", diff --git a/patches/ring_windows_msvc_include_dirs.patch b/patches/ring_windows_msvc_include_dirs.patch new file mode 100644 index 0000000000000000000000000000000000000000..7bb49cbc2c212f0341df6f55230fd340b061ae0a --- /dev/null +++ b/patches/ring_windows_msvc_include_dirs.patch @@ -0,0 +1,112 @@ +diff --git a/build.rs b/build.rs +--- a/build.rs ++++ b/build.rs +@@ -346,7 +346,27 @@ + // we want to optimize for minimizing the build tools required: No Perl, + // no nasm, etc. + let generated_dir = if !is_git { +- c_root_dir.join(PREGENERATED) ++ let pregenerated_dir = c_root_dir.join(PREGENERATED); ++ if target.os == WINDOWS && target.env == "msvc" ++ { ++ let mut pending = vec![(pregenerated_dir.clone(), out_dir.clone())]; ++ while let Some((src_dir, dst_dir)) = pending.pop() { ++ fs::create_dir_all(&dst_dir).unwrap(); ++ for entry in fs::read_dir(&src_dir).unwrap() { ++ let entry = entry.unwrap(); ++ let src_path = entry.path(); ++ let dst_path = dst_dir.join(entry.file_name()); ++ if entry.file_type().unwrap().is_dir() { ++ pending.push((src_path, dst_path)); ++ } else { ++ fs::copy(&src_path, &dst_path).unwrap(); ++ } ++ } ++ } ++ out_dir.clone() ++ } else { ++ pregenerated_dir ++ } + } else { + generate_sources_and_preassemble( + &out_dir, +@@ -561,7 +581,13 @@ + let compiler = c.get_compiler(); + // FIXME: On Windows AArch64 we currently must use Clang to compile C code + let compiler = if target.os == WINDOWS && target.arch == AARCH64 && !compiler.is_like_clang() { +- let _ = c.compiler("clang"); ++ // Keep inherited MSVC flags compatible with the selected Clang driver. ++ let clang = if compiler.is_like_msvc() { ++ "clang-cl" ++ } else { ++ "clang" ++ }; ++ let _ = c.compiler(clang); + c.get_compiler() + } else { + compiler +@@ -569,6 +595,15 @@ + + let _ = c.include(c_root_dir.join("include")); + let _ = c.include(include_dir); ++ let _ = c.include(c_root_dir.join("third_party").join("fiat")); ++ if compiler.is_like_msvc() { ++ let _ = c.include(c_root_dir.join("crypto").join("curve25519")); ++ let _ = c.include(c_root_dir.join("crypto").join("limbs")); ++ let _ = c.include(c_root_dir.join("crypto").join("fipsmodule").join("aes")); ++ let _ = c.include(c_root_dir.join("crypto").join("fipsmodule").join("bn")); ++ let _ = c.include(c_root_dir.join("crypto").join("fipsmodule").join("ec")); ++ let _ = c.include(c_root_dir.join("crypto").join("poly1305")); ++ } + for f in cpp_flags(&compiler) { + let _ = c.flag(f); + } +diff --git a/crypto/curve25519/curve25519.c b/crypto/curve25519/curve25519.c +index 99d7d7fbb..2f69ad560 100644 +--- a/crypto/curve25519/curve25519.c ++++ b/crypto/curve25519/curve25519.c +@@ -47,11 +47,11 @@ + #if defined(__GNUC__) + #pragma GCC diagnostic ignored "-Wpedantic" + #endif +-#include "../../third_party/fiat/curve25519_64.h" ++#include "curve25519_64.h" + #elif defined(OPENSSL_64_BIT) +-#include "../../third_party/fiat/curve25519_64_msvc.h" ++#include "curve25519_64_msvc.h" + #else +-#include "../../third_party/fiat/curve25519_32.h" ++#include "curve25519_32.h" + #endif + + +diff --git a/crypto/curve25519/curve25519_64_adx.c b/crypto/curve25519/curve25519_64_adx.c +index 88964a9dd..b660f55f4 100644 +--- a/crypto/curve25519/curve25519_64_adx.c ++++ b/crypto/curve25519/curve25519_64_adx.c +@@ -19,5 +19,5 @@ + #pragma GCC diagnostic ignored "-Wpedantic" + #pragma GCC diagnostic ignored "-Wsign-conversion" + +-#include "../../third_party/fiat/curve25519_64_adx.h" ++#include "curve25519_64_adx.h" + #endif +diff --git a/crypto/fipsmodule/ec/p256.c b/crypto/fipsmodule/ec/p256.c +index 0117916da..4224d1192 100644 +--- a/crypto/fipsmodule/ec/p256.c ++++ b/crypto/fipsmodule/ec/p256.c +@@ -50,11 +50,11 @@ + #if defined(__GNUC__) + #pragma GCC diagnostic ignored "-Wpedantic" + #endif +-#include "../../../third_party/fiat/p256_64.h" ++#include "p256_64.h" + #elif defined(OPENSSL_64_BIT) +-#include "../../../third_party/fiat/p256_64_msvc.h" ++#include "p256_64_msvc.h" + #else +-#include "../../../third_party/fiat/p256_32.h" ++#include "p256_32.h" + #endif + + diff --git a/patches/rules_cc_rusty_v8_custom_libcxx.patch b/patches/rules_cc_rusty_v8_custom_libcxx.patch new file mode 100644 index 0000000000000000000000000000000000000000..3efda03525fd4b5e1dd8d1805baa3bce1bc01366 --- /dev/null +++ b/patches/rules_cc_rusty_v8_custom_libcxx.patch @@ -0,0 +1,44 @@ +diff --git a/cc/cc_library.bzl b/cc/cc_library.bzl +index c45c5fd954..2e3a01f12a 100644 +--- a/cc/cc_library.bzl ++++ b/cc/cc_library.bzl +@@ -15,5 +15,39 @@ + + load("@cc_compatibility_proxy//:proxy.bzl", _cc_library = "cc_library") + ++_RUSTY_V8_CUSTOM_LIBCXX_COPTS = select({ ++ "@@//third_party/v8:use_rusty_v8_custom_libcxx": [ ++ "-nostdinc++", ++ "-D_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS", ++ "-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_EXTENSIVE", ++ "-D_LIBCPP_INSTRUMENTED_WITH_ASAN=0", ++ "-D_LIBCXXABI_DISABLE_VISIBILITY_ANNOTATIONS", ++ ], ++ "//conditions:default": [], ++}) ++ ++_RUSTY_V8_CUSTOM_LIBCXX_DEPS = select({ ++ "@@//third_party/v8:use_rusty_v8_custom_libcxx": [ ++ "@@//third_party/v8:rusty_v8_custom_libcxx_headers", ++ ], ++ "//conditions:default": [], ++}) ++ ++_RUSTY_V8_CUSTOM_LIBCXX_REPOS = [ ++ # V8 and ICU are patched in-tree. These external C++ dependencies still ++ # build inside the artifact closure once the LLVM toolchain's default ++ # libc++ headers are suppressed. ++ "@abseil-cpp+", ++ "@v8++http_archive+highway", ++ "@v8++http_archive+simdutf", ++] ++ ++def _should_use_rusty_v8_custom_libcxx(): ++ return native.repository_name() in _RUSTY_V8_CUSTOM_LIBCXX_REPOS ++ + def cc_library(**kwargs): ++ if _should_use_rusty_v8_custom_libcxx(): ++ kwargs["copts"] = (kwargs.get("copts", []) or []) + _RUSTY_V8_CUSTOM_LIBCXX_COPTS ++ kwargs["deps"] = (kwargs.get("deps", []) or []) + _RUSTY_V8_CUSTOM_LIBCXX_DEPS ++ kwargs["features"] = (kwargs.get("features", []) or []) + ["-module_maps"] + _cc_library(**kwargs) diff --git a/patches/rules_foreign_cc_make_cppflags.patch b/patches/rules_foreign_cc_make_cppflags.patch new file mode 100644 index 0000000000000000000000000000000000000000..96cf53df907bf1f03b9bf1a56cbee355a06eb602 --- /dev/null +++ b/patches/rules_foreign_cc_make_cppflags.patch @@ -0,0 +1,15 @@ +diff --git a/foreign_cc/built_tools/make_build.bzl b/foreign_cc/built_tools/make_build.bzl +--- a/foreign_cc/built_tools/make_build.bzl ++++ b/foreign_cc/built_tools/make_build.bzl +@@ -114,6 +114,11 @@ + "ARFLAGS": _join_flags_list(ctx.workspace_name, arflags), + "CC": absolute_cc, + "CFLAGS": _join_flags_list(ctx.workspace_name, non_sysroot_cflags), ++ # Configure's preprocessor checks need the declared header paths too. ++ "CPPFLAGS": " ".join([ ++ "$${CPPFLAGS:-}$$", ++ _join_flags_list(ctx.workspace_name, non_sysroot_cflags), ++ ]).strip(), + "LD": absolute_ld, + "LDFLAGS": _join_flags_list(ctx.workspace_name, non_sysroot_ldflags), + }) diff --git a/patches/rules_foreign_cc_make_msvc_stdint.patch b/patches/rules_foreign_cc_make_msvc_stdint.patch new file mode 100644 index 0000000000000000000000000000000000000000..f3af72841ab0fcd91ebdbe98c44bff4c5e9dd3ea --- /dev/null +++ b/patches/rules_foreign_cc_make_msvc_stdint.patch @@ -0,0 +1,13 @@ +diff --git a/foreign_cc/built_tools/make_build.bzl b/foreign_cc/built_tools/make_build.bzl +--- a/foreign_cc/built_tools/make_build.bzl ++++ b/foreign_cc/built_tools/make_build.bzl +@@ -35,6 +35,9 @@ + build_str += " gcc" + dist_dir = "GccRel" + else: ++ # Modern MSVC provides stdint.h, including the intmax_t types that ++ # Make's legacy Windows config otherwise replaces with macros. ++ build_str = 'export CL="$${CL:-}$$ /DHAVE_STDINT_H=1"\n' + build_str + dist_dir = "WinRel" + + script = [ diff --git a/patches/rules_foreign_cc_make_xcompile.patch b/patches/rules_foreign_cc_make_xcompile.patch new file mode 100644 index 0000000000000000000000000000000000000000..097e87bc184c7b07dc7aba367520379b646f4127 --- /dev/null +++ b/patches/rules_foreign_cc_make_xcompile.patch @@ -0,0 +1,10 @@ +diff --git a/toolchains/private/BUILD.bazel b/toolchains/private/BUILD.bazel +--- a/toolchains/private/BUILD.bazel ++++ b/toolchains/private/BUILD.bazel +@@ -23,5 +23,6 @@ + make_tool( + name = "make_tool", + srcs = "@gnumake_src//:all_srcs", ++ configure_xcompile = True, + tags = ["manual"], + ) diff --git a/patches/rules_rs_build_script_deps_annotation.patch b/patches/rules_rs_build_script_deps_annotation.patch new file mode 100644 index 0000000000000000000000000000000000000000..2f38c2bfedd63184cea5d4842a4c14567e8c4226 --- /dev/null +++ b/patches/rules_rs_build_script_deps_annotation.patch @@ -0,0 +1,58 @@ +# What: restore explicit build-script dependency annotations used by Codex. +# Scope: rules_rs crate-universe annotation plumbing only. + +diff --git a/rs/extensions.bzl b/rs/extensions.bzl +--- a/rs/extensions.bzl ++++ b/rs/extensions.bzl +@@ -321,7 +321,7 @@ def _generate_hub_and_spokes( + kwargs = dict( + hub_name = hub_name, + gen_build_script = annotation.gen_build_script, +- build_script_deps = [], ++ build_script_deps = annotation.build_script_deps, + build_script_deps_select = _select(feature_resolutions.build_deps), + build_script_data = annotation.build_script_data, + build_script_data_select = annotation.build_script_data_select, +@@ -881,9 +881,9 @@ _annotation = tag_class( + "build_script_data_select": _label_list_dict( + doc = "Labels to add to a crate's `cargo_build_script::data` attribute, keyed by platform triplet.", + ), +- # "build_script_deps": attr.label_list( +- # doc = "A list of labels to add to a crate's `cargo_build_script::deps` attribute.", +- # ), ++ "build_script_deps": attr.string_list( ++ doc = "A list of labels to add to a crate's `cargo_build_script::deps` attribute.", ++ ), + "build_script_env": attr.string_dict( + doc = "Additional environment variables to set on a crate's `cargo_build_script::env` attribute.", + ), +diff --git a/rs/private/annotations.bzl b/rs/private/annotations.bzl +--- a/rs/private/annotations.bzl ++++ b/rs/private/annotations.bzl +@@ -2,6 +2,7 @@ def _crate_annotation( + additive_build_file = None, + additive_build_file_content = "", + gen_build_script = "auto", ++ build_script_deps = [], + build_script_data = [], + build_script_data_select = {}, + build_script_env = {}, +@@ -30,6 +31,7 @@ def _crate_annotation( + additive_build_file = additive_build_file, + additive_build_file_content = additive_build_file_content, + gen_build_script = gen_build_script, ++ build_script_deps = build_script_deps, + build_script_data = build_script_data, + build_script_data_select = build_script_data_select, + build_script_env = build_script_env, +diff --git a/rs/private/repository_utils.bzl b/rs/private/repository_utils.bzl +--- a/rs/private/repository_utils.bzl ++++ b/rs/private/repository_utils.bzl +@@ -298,6 +298,6 @@ rust_crate_attrs = { + "hub_name": attr.string(), + "gen_build_script": attr.string(), +- "build_script_deps": attr.label_list(), ++ "build_script_deps": attr.string_list(), + "build_script_deps_select": _label_list_dict(), + "build_script_data": attr.label_list(), + "build_script_data_select": _label_list_dict(), diff --git a/patches/rules_rs_windows_msvc_linker.patch b/patches/rules_rs_windows_msvc_linker.patch new file mode 100644 index 0000000000000000000000000000000000000000..5056ade96bf0841c5517959aca30cb12897ea5bd --- /dev/null +++ b/patches/rules_rs_windows_msvc_linker.patch @@ -0,0 +1,15 @@ +# What: use Rust's bundled direct linker for Windows x86_64 and ARM64 MSVC toolchains. +# Scope: standard and bootstrap rules_rs Rust toolchains targeting x86_64 and ARM64 MSVC. + +diff --git a/rs/toolchains/declare_rustc_toolchains.bzl b/rs/toolchains/declare_rustc_toolchains.bzl +--- a/rs/toolchains/declare_rustc_toolchains.bzl ++++ b/rs/toolchains/declare_rustc_toolchains.bzl +@@ -88,6 +88,8 @@ def declare_rustc_toolchains( + llvm_cov = "@llvm//tools:llvm-cov", + llvm_profdata = "@llvm//tools:llvm-profdata", + linker = select({ ++ "@rules_rs//rs/platforms/config:aarch64-pc-windows-msvc": "{}rust-lld".format(rustc_repo_label), ++ "@rules_rs//rs/platforms/config:x86_64-pc-windows-msvc": "{}rust-lld".format(rustc_repo_label), + "@platforms//cpu:wasm32": "{}rust-lld".format(rustc_repo_label), + "@platforms//cpu:wasm64": "{}rust-lld".format(rustc_repo_label), + "//conditions:default": None, diff --git a/patches/rules_rs_zlib_snapshot_urls.patch b/patches/rules_rs_zlib_snapshot_urls.patch new file mode 100644 index 0000000000000000000000000000000000000000..fffbca8fcd265a4c65f911a07f26434ca4f47188 --- /dev/null +++ b/patches/rules_rs_zlib_snapshot_urls.patch @@ -0,0 +1,21 @@ +# What: fetch the pinned Linux zlib packages from dated Ubuntu snapshots. +# Scope: download URLs only; package versions and checksums are unchanged. +# Backport of https://github.com/hermeticbuild/rules_rs/commit/81336041f1c092b94ba0d35384c4dfc03ce45a07 + +diff --git a/rs/private/rustc_repository.bzl b/rs/private/rustc_repository.bzl +--- a/rs/private/rustc_repository.bzl ++++ b/rs/private/rustc_repository.bzl +@@ -11,11 +11,11 @@ _LINUX_ZLIB = { + "aarch64": struct( + libdir = "usr/lib/aarch64-linux-gnu", + sha256 = "cbe3d39ec32d3cc27c021ae4af11e7c67bdf9d700d573207e0941d4038056278", +- url = "https://ports.ubuntu.com/ubuntu-ports/pool/main/z/zlib/zlib1g_1.3.dfsg-3.1ubuntu2.1_arm64.deb", ++ url = "https://snapshot.ubuntu.com/ubuntu/20260801T000000Z/pool/main/z/zlib/zlib1g_1.3.dfsg-3.1ubuntu2.1_arm64.deb", + ), + "x86_64": struct( + libdir = "usr/lib/x86_64-linux-gnu", + sha256 = "7074b6a2f6367a10d280c00a1cb02e74277709180bab4f2491a2f355ab2d6c20", +- url = "https://archive.ubuntu.com/ubuntu/pool/main/z/zlib/zlib1g_1.3.dfsg-3.1ubuntu2.1_amd64.deb", ++ url = "https://snapshot.ubuntu.com/ubuntu/20260801T000000Z/pool/main/z/zlib/zlib1g_1.3.dfsg-3.1ubuntu2.1_amd64.deb", + ), + } diff --git a/patches/rules_rust_group_build_script_arg_files.patch b/patches/rules_rust_group_build_script_arg_files.patch new file mode 100644 index 0000000000000000000000000000000000000000..5e65873ba529c37e5fdca07617ba452cfcfbda3f --- /dev/null +++ b/patches/rules_rust_group_build_script_arg_files.patch @@ -0,0 +1,12 @@ +diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl +--- a/rust/private/rustc.bzl ++++ b/rust/private/rustc.bzl +@@ -1125,5 +1125,7 @@ + for build_env_file in build_env_files: + process_wrapper_flags.add("--env-file", build_env_file) + +- process_wrapper_flags.add_all(build_flags_files, before_each = "--arg-file") ++ # The wrapper accepts multiple files per flag. Reduce native Windows ++ # command-line length without changing the files or their order. ++ process_wrapper_flags.add_all("--arg-file", build_flags_files) + diff --git a/patches/rules_rust_windows_execroot_separators.patch b/patches/rules_rust_windows_execroot_separators.patch new file mode 100644 index 0000000000000000000000000000000000000000..187b37f48c651eea91d38fed008ad4c3c82c79b0 --- /dev/null +++ b/patches/rules_rust_windows_execroot_separators.patch @@ -0,0 +1,52 @@ +--- a/cargo/private/cargo_build_script_runner/lib.rs ++++ b/cargo/private/cargo_build_script_runner/lib.rs +@@ -337,9 +337,18 @@ + return "${pwd}".to_owned(); + } + +- value ++ let redacted = value + .replace(&format!("{exec_root}/"), "${pwd}/") +- .replace(&format!("{exec_root}\\"), "${pwd}\\") ++ .replace(&format!("{exec_root}\\"), "${pwd}\\"); ++ #[cfg(windows)] ++ let redacted = { ++ // pkg-config can emit forward slashes for the same Windows root. ++ let root = exec_root.replace('\\', "/"); ++ redacted ++ .replace(&format!("{root}/"), "${pwd}/") ++ .replace(&format!("{root}\\"), "${pwd}\\") ++ }; ++ redacted + } + + /// Redact for env vars: uses the generic `${out_dir}` token, resolved +@@ -630,6 +639,28 @@ + link_search_paths: + "-L${pwd}/${bazel-out/cfg/bin/pkg/_bs.out_dir}\n-L${pwd}/other/path".to_owned(), + } ++ ); ++ } ++ ++ #[cfg(windows)] ++ #[test] ++ fn windows_include_metadata_redacts_both_root_spellings() { ++ let root = r"D:\o\execroot\_main"; ++ let forward = root.replace('\\', "/"); ++ let outputs = vec![BuildScriptOutput::DepEnv(format!( ++ "INCLUDE={forward}/include;{root}\\lib;{forward}-other/include" ++ ))]; ++ assert_eq!( ++ BuildScriptOutput::outputs_to_dep_env(&outputs, "glib", root), ++ format!("DEP_GLIB_INCLUDE=${{pwd}}/include;${{pwd}}\\lib;{forward}-other/include") ++ ); ++ let directory = std::env::current_dir().unwrap(); ++ let directory = directory.to_str().unwrap(); ++ let forward = directory.replace('\\', "/"); ++ let outputs = vec![BuildScriptOutput::DepEnv(format!("INCLUDE={forward}/."))]; ++ assert_eq!( ++ BuildScriptOutput::nonhermetic_absolute_paths(&outputs, directory, ""), ++ Vec::::new() + ); + } + diff --git a/patches/rules_rust_windows_msvc_direct_link_args.patch b/patches/rules_rust_windows_msvc_direct_link_args.patch new file mode 100644 index 0000000000000000000000000000000000000000..a8eba2ba6e7b8e6e709693e1b7f6e2a91e6dd5f2 --- /dev/null +++ b/patches/rules_rust_windows_msvc_direct_link_args.patch @@ -0,0 +1,109 @@ +--- a/rust/private/rustc.bzl ++++ b/rust/private/rustc.bzl +@@ -438,4 +438,11 @@ + use_bpf_linker = toolchain.target_arch in ("bpfeb", "bpfel") and toolchain.linker +- if not ld or toolchain.linker_preference == "rust" or use_bpf_linker: ++ use_windows_msvc_linker = ( ++ toolchain.target_arch in ("x86_64", "aarch64") and ++ toolchain.target_os == "windows" and ++ toolchain.target_abi == "msvc" and ++ toolchain.linker != None and ++ toolchain.linker_preference != "cc" ++ ) ++ if not ld or toolchain.linker_preference == "rust" or use_bpf_linker or use_windows_msvc_linker: + ld = toolchain.linker.path + ld_is_direct_driver = toolchain.linker_type == "direct" +@@ -468,11 +468,25 @@ + filtered_args.append(version) + # Keep library search path flags + ++ elif processed_arg == "-L" and i + 1 < len(link_args): ++ path = link_args[i + 1] ++ if ld_is_direct_driver and toolchain.target_os == "windows" and toolchain.target_abi == "msvc": ++ skip_next = True ++ continue ++ filtered_args.extend([processed_arg, path]) ++ skip_next = True ++ + elif processed_arg.startswith("-L"): ++ if ld_is_direct_driver and toolchain.target_os == "windows" and toolchain.target_abi == "msvc": ++ continue + filtered_args.append(processed_arg) + # Keep sysroot flags (as single or two-part arguments) + + elif processed_arg == "--sysroot" or processed_arg.startswith("--sysroot="): ++ if ld_is_direct_driver and toolchain.target_os == "windows" and toolchain.target_abi == "msvc": ++ if processed_arg == "--sysroot" and i + 1 < len(link_args): ++ skip_next = True ++ continue + filtered_args.append(processed_arg) + if processed_arg == "--sysroot" and i + 1 < len(link_args): + # Two-part argument, keep the next arg too +@@ -2943,8 +2957,10 @@ + use_pic, + ambiguous_libs, + get_lib_name, ++ for_windows = False, + for_darwin = False, +- flavor_msvc = False): ++ flavor_msvc = False, ++ use_direct_driver = False): + """_summary_ + + Args: +@@ -2952,8 +2968,10 @@ + use_pic (_type_): _description_ + ambiguous_libs (_type_): _description_ + get_lib_name (_type_): _description_ ++ for_windows (bool, optional): _description_. Defaults to False. + for_darwin (bool, optional): _description_. Defaults to False. + flavor_msvc (bool, optional): _description_. Defaults to False. ++ use_direct_driver (bool, optional): _description_. Defaults to False. + + Returns: + _type_: _description_ +@@ -2997,6 +3015,11 @@ + ): + return [] if for_darwin else ["-lstatic=%s" % get_lib_name(artifact)] + ++ if for_windows and flavor_msvc and use_direct_driver and not artifact.basename.endswith(".lib"): ++ return [ ++ "-Clink-arg={}".format(artifact.path), ++ ] ++ + if flavor_msvc: + return [ + "-lstatic=%s" % get_lib_name(artifact), +@@ -3045,7 +3068,7 @@ + ]) + elif include_link_flags: + get_lib_name = get_lib_name_for_windows if flavor_msvc else get_lib_name_default +- ret.extend(portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name, flavor_msvc = flavor_msvc)) ++ ret.extend(portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name, for_windows = True, flavor_msvc = flavor_msvc, use_direct_driver = use_direct_driver)) + + # Windows toolchains can inherit POSIX defaults like -pthread from C deps, + # which fails to link with the MinGW/LLD toolchain. Drop them here. +@@ -3257,11 +3280,18 @@ + format_each = "-Lnative=%s", + ) + if include_link_flags: +- args.add_all( +- runtime_libs, +- map_each = get_lib_name, +- format_each = static_runtime_link_format, +- ) ++ if toolchain.target_os == "windows" and toolchain.target_abi == "msvc" and use_direct_link_driver: ++ for runtime_lib in runtime_libs.to_list(): ++ if runtime_lib.basename.endswith(".lib"): ++ args.add(get_lib_name(runtime_lib), format = static_runtime_link_format) ++ else: ++ args.add(runtime_lib.path, format = "--codegen=link-arg=%s") ++ else: ++ args.add_all( ++ runtime_libs, ++ map_each = get_lib_name, ++ format_each = static_runtime_link_format, ++ ) + + def _get_dirname(file): + """A helper function for `_add_native_link_flags`. diff --git a/patches/rules_rust_windows_process_wrapper_skip_temp_outputs.patch b/patches/rules_rust_windows_process_wrapper_skip_temp_outputs.patch new file mode 100644 index 0000000000000000000000000000000000000000..52c5ae8652c7f19181928899941f9d35f750214f --- /dev/null +++ b/patches/rules_rust_windows_process_wrapper_skip_temp_outputs.patch @@ -0,0 +1,15 @@ +--- a/util/process_wrapper/main.rs ++++ b/util/process_wrapper/main.rs +@@ -213,6 +213,12 @@ fn consolidate_dependency_search_paths( + let file_name_lower = file_name + .to_string_lossy() + .to_ascii_lowercase(); ++ if file_name_lower.contains(".tmp") || file_name_lower.ends_with(".rcgu.o") { ++ // Native Windows Rust link actions can leave transient `*.exe.tmp*` and ++ // per-codegen-unit `*.rcgu.o` outputs next to the final binary. ++ // Those are not stable linker search-path inputs. ++ continue; ++ } + if !seen.insert(file_name_lower) { + continue; + } diff --git a/patches/rusty_v8_prebuilt_out_dir.patch b/patches/rusty_v8_prebuilt_out_dir.patch new file mode 100644 index 0000000000000000000000000000000000000000..4242a29beb81bc1ce64c061822e8f6257114d1e0 --- /dev/null +++ b/patches/rusty_v8_prebuilt_out_dir.patch @@ -0,0 +1,74 @@ +# What: make `rusty_v8` accept Bazel-provided archives and bindings for the +# windows-gnullvm experiment. +# Scope: `rusty_v8` build.rs only; no V8 source or Bazel rule changes. + +--- a/build.rs ++++ b/build.rs +@@ -543,10 +543,15 @@ + } + + fn static_lib_name(suffix: &str) -> String { +- let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); +- if target_os == "windows" { ++ let target = env::var("TARGET").unwrap(); ++ let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); ++ if target.contains("windows-gnullvm") { ++ // rustc looks for GNU-style archive names when bundling `-lstatic` on the ++ // gnullvm targets, even though the prebuilt release is a COFF `.lib`. ++ format!("librusty_v8{suffix}.a") ++ } else if target_os == "windows" { + format!("rusty_v8{suffix}.lib") + } else { + format!("librusty_v8{suffix}.a") + } + } +@@ -577,7 +577,23 @@ + path + } + ++fn out_dir_abs() -> PathBuf { ++ let cwd = env::current_dir().unwrap(); ++ ++ // target/debug/build/rusty_v8-d9e5a424d4f96994/out/ ++ let out_dir = env::var_os("OUT_DIR").expect( ++ "The 'OUT_DIR' environment is not set (it should be something like \ ++ 'target/debug/rusty_v8-{hash}').", ++ ); ++ ++ cwd.join(out_dir) ++} ++ + fn static_lib_dir() -> PathBuf { ++ if env::var_os("RUSTY_V8_ARCHIVE").is_some() { ++ return out_dir_abs().join("gn_out").join("obj"); ++ } ++ + build_dir().join("gn_out").join("obj") + } + +@@ -794,22 +810,23 @@ + } + + fn print_link_flags() { ++ let target = env::var("TARGET").unwrap(); + println!("cargo:rustc-link-lib=static=rusty_v8"); + let should_dyn_link_libcxx = env::var("CARGO_FEATURE_USE_CUSTOM_LIBCXX") + .is_err() ++ || (target.contains("apple") && env::var("RUSTY_V8_ARCHIVE").is_ok()) + || env::var("GN_ARGS").is_ok_and(|gn_args| { + gn_args + .split_whitespace() + .any(|ba| ba == "use_custom_libcxx=false") + }); + + if should_dyn_link_libcxx { + // Based on https://github.com/alexcrichton/cc-rs/blob/fba7feded71ee4f63cfe885673ead6d7b4f2f454/src/lib.rs#L2462 + if let Ok(stdlib) = env::var("CXXSTDLIB") { + if !stdlib.is_empty() { + println!("cargo:rustc-link-lib=dylib={stdlib}"); + } + } else { +- let target = env::var("TARGET").unwrap(); + if target.contains("msvc") { + // nothing to link to + } else if target.contains("apple") diff --git a/patches/tikv-jemalloc-sys_dependency_flags.patch b/patches/tikv-jemalloc-sys_dependency_flags.patch new file mode 100644 index 0000000000000000000000000000000000000000..9c9d71571850464ddaae0d4cd415c1044e0ae1e1 --- /dev/null +++ b/patches/tikv-jemalloc-sys_dependency_flags.patch @@ -0,0 +1,38 @@ +diff --git a/jemalloc/Makefile.in b/jemalloc/Makefile.in +--- a/jemalloc/Makefile.in ++++ b/jemalloc/Makefile.in +@@ -508,7 +508,7 @@ + @mkdir -p $(@D) + $(CC) $(CFLAGS) -c $(CPPFLAGS) $(CTARGET) $< + ifdef CC_MM +- @$(CC) -MM $(CPPFLAGS) -MT $@ -o $(@:%.$(O)=%.d) $< ++ @$(CC) $(CFLAGS) -MM $(CPPFLAGS) -MT $@ -o $(@:%.$(O)=%.d) $< + endif + + $(C_SYMS): %.sym: +@@ -532,7 +532,7 @@ + @mkdir -p $(@D) + $(CXX) $(CXXFLAGS) -c $(CPPFLAGS) $(CTARGET) $< + ifdef CC_MM +- @$(CXX) -MM $(CPPFLAGS) -MT $@ -o $(@:%.$(O)=%.d) $< ++ @$(CXX) $(CXXFLAGS) -MM $(CPPFLAGS) -MT $@ -o $(@:%.$(O)=%.d) $< + endif + + ifneq ($(SOREV),$(SO)) +@@ -589,14 +589,14 @@ + @mkdir -p $(@D) + $(CC) $(CFLAGS) -c $(CPPFLAGS) -DJEMALLOC_STRESS_TEST -I$(srcroot)test/include -I$(objroot)test/include $(CTARGET) $< + ifdef CC_MM +- @$(CC) -MM $(CPPFLAGS) -DJEMALLOC_STRESS_TEST -I$(srcroot)test/include -I$(objroot)test/include -MT $@ -o $(@:%.$(O)=%.d) $< ++ @$(CC) $(CFLAGS) -MM $(CPPFLAGS) -DJEMALLOC_STRESS_TEST -I$(srcroot)test/include -I$(objroot)test/include -MT $@ -o $(@:%.$(O)=%.d) $< + endif + + $(objroot)test/stress/pa/%.$(O): $(srcroot)test/stress/pa/%.cpp + @mkdir -p $(@D) + $(CXX) $(CXXFLAGS) -c $(CPPFLAGS) -I$(srcroot)test/include -I$(objroot)test/include $(CTARGET) $< + ifdef CC_MM +- @$(CXX) -MM $(CPPFLAGS) -I$(srcroot)test/include -I$(objroot)test/include -MT $@ -o $(@:%.$(O)=%.d) $< ++ @$(CXX) $(CXXFLAGS) -MM $(CPPFLAGS) -I$(srcroot)test/include -I$(objroot)test/include -MT $@ -o $(@:%.$(O)=%.d) $< + endif + + build_lib_shared: $(DSOS) diff --git a/patches/tikv-jemalloc-sys_make_env.patch b/patches/tikv-jemalloc-sys_make_env.patch new file mode 100644 index 0000000000000000000000000000000000000000..305022b3afbab0ad7eb8323779455f3cf109ba21 --- /dev/null +++ b/patches/tikv-jemalloc-sys_make_env.patch @@ -0,0 +1,13 @@ +diff --git a/build.rs b/build.rs +--- a/build.rs ++++ b/build.rs +@@ -331,7 +331,8 @@ + run_and_log(&mut cmd, &build_dir.join("config.log")); + + // Make: +- let make = make_cmd(&host); ++ let make = read_and_watch_env("MAKE").unwrap_or_else(|_| make_cmd(&host).to_owned()); ++ let make = make.as_str(); + run(&mut make_command(make, &build_dir, &num_jobs)); + + // Skip watching this environment variables to avoid rebuild in CI. diff --git a/patches/v8_module_deps.patch b/patches/v8_module_deps.patch new file mode 100644 index 0000000000000000000000000000000000000000..d179617e6302ebe573662bb08e962dc41218dfdd --- /dev/null +++ b/patches/v8_module_deps.patch @@ -0,0 +1,285 @@ +# What: replace upstream V8 module dependency bootstrapping with repository +# declarations and dependency setup that match this Bazel workspace. +# Scope: upstream MODULE.bazel only; affects external repo resolution and Bazel +# module wiring, not V8 source files. + +diff --git a/orig/v8-15.0.245.2/MODULE.bazel b/mod/v8-15.0.245.2/MODULE.bazel +index b8bf8bd..573f463 100644 +--- a/orig/v8-15.0.245.2/MODULE.bazel ++++ b/mod/v8-15.0.245.2/MODULE.bazel +@@ -8,193 +8,67 @@ bazel_dep(name = "rules_cc", version = "0.2.0") + bazel_dep(name = "rules_python", version = "1.0.0") + bazel_dep(name = "platforms", version = "1.0.0") + bazel_dep(name = "abseil-cpp", version = "20250814.0") +-bazel_dep(name = "highway", version = "1.2.0") +- +-pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") +-pip.parse( +- hub_name = "v8_python_deps", +- python_version = "3.11", +- requirements_lock = "//:bazel/requirements.txt", +- extra_pip_args = [ +- "--require-hashes", +- "--index-url=https://pypi.org/simple", +- ], ++bazel_dep(name = "rules_license", version = "0.0.4") ++ ++git_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") ++http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") ++ ++http_archive( ++ name = "highway", ++ patch_args = ["-p1"], ++ patches = ["@v8//:bazel/highway.patch"], ++ sha256 = "7e0be78b8318e8bdbf6fa545d2ecb4c90f947df03f7aadc42c1967f019e63343", ++ strip_prefix = "highway-1.2.0", ++ urls = ["https://github.com/google/highway/archive/refs/tags/1.2.0.tar.gz"], + ) +-use_repo(pip, "v8_python_deps") +- +-# Define the local LLVM toolchain repository +-llvm_toolchain_repository = use_repo_rule("//bazel/toolchain:llvm_repository.bzl", "llvm_toolchain_repository") +- +-llvm_toolchain_repository( +- name = "llvm_toolchain", +- path = "third_party/llvm-build/Release+Asserts", +- config_file_content = """ +-load("@rules_cc//cc:defs.bzl", "CcToolchainConfigInfo", "cc_common") +-load("@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", "feature", "flag_group", "flag_set", "tool_path") + +-def _impl(ctx): +- tool_paths = [ +- tool_path(name = "gcc", path = "bin/clang"), +- tool_path(name = "ld", path = "bin/lld"), +- tool_path(name = "ar", path = "bin/llvm-ar"), +- tool_path(name = "cpp", path = "bin/clang++"), +- tool_path(name = "gcov", path = "/bin/false"), +- tool_path(name = "nm", path = "bin/llvm-nm"), +- tool_path(name = "objdump", path = "bin/llvm-objdump"), +- tool_path(name = "strip", path = "bin/llvm-strip"), +- ] +- +- features = [ +- feature( +- name = "default_compile_flags", +- enabled = True, +- flag_sets = [ +- flag_set( +- actions = [ +- "c-compile", +- "c++-compile", +- "c++-header-parsing", +- "c++-module-compile", +- "c++-module-codegen", +- "linkstamp-compile", +- "assemble", +- "preprocess-assemble", +- ], +- flag_groups = [ +- flag_group( +- flags = [ +- "--sysroot={WORKSPACE_ROOT}/build/linux/debian_bullseye_amd64-sysroot", +- "-nostdinc++", +- "-isystem", +- "{WORKSPACE_ROOT}/buildtools/third_party/libc++", +- "-isystem", +- "{WORKSPACE_ROOT}/third_party/libc++/src/include", +- "-isystem", +- "{WORKSPACE_ROOT}/third_party/libc++abi/src/include", +- "-isystem", +- "{WORKSPACE_ROOT}/third_party/libc++/src/src", +- "-isystem", +- "{WORKSPACE_ROOT}/third_party/llvm-libc/src", +- "-D_LIBCPP_HARDENING_MODE_DEFAULT=_LIBCPP_HARDENING_MODE_NONE", +- "-DLIBC_NAMESPACE=__llvm_libc_cr", +- ], +- ), +- ], +- ), +- ], +- ), +- feature( +- name = "default_linker_flags", +- enabled = True, +- flag_sets = [ +- flag_set( +- actions = [ +- "c++-link-executable", +- "c++-link-dynamic-library", +- "c++-link-nodeps-dynamic-library", +- ], +- flag_groups = [ +- flag_group( +- flags = [ +- "--sysroot={WORKSPACE_ROOT}/build/linux/debian_bullseye_amd64-sysroot", +- "-fuse-ld=lld", +- "-lm", +- "-lpthread", +- ], +- ), +- ], +- ), +- ], +- ), +- ] +- +- return cc_common.create_cc_toolchain_config_info( +- ctx = ctx, +- toolchain_identifier = "local_clang", +- features = features, +- cxx_builtin_include_directories = [ +- "{WORKSPACE_ROOT}/buildtools/third_party/libc++", +- "{WORKSPACE_ROOT}/third_party/libc++/src/include", +- "{WORKSPACE_ROOT}/third_party/libc++abi/src/include", +- "{WORKSPACE_ROOT}/third_party/libc++/src/src", +- "{WORKSPACE_ROOT}/third_party/llvm-libc/src", +- "{WORKSPACE_ROOT}/third_party/llvm-build/Release+Asserts/lib/clang/22/include", +- "{WORKSPACE_ROOT}/third_party/llvm-build/Release+Asserts/lib/clang/23/include", +- "{WORKSPACE_ROOT}/build/linux/debian_bullseye_amd64-sysroot/usr/include", +- "{WORKSPACE_ROOT}/build/linux/debian_bullseye_amd64-sysroot/usr/local/include", +- ], +- host_system_name = "local", +- target_system_name = "local", +- target_cpu = "k8", +- target_libc = "unknown", +- compiler = "clang", +- abi_version = "unknown", +- abi_libc_version = "unknown", +- tool_paths = tool_paths, +- ) +- +-cc_toolchain_config = rule( +- implementation = _impl, +- attrs = {}, +- provides = [CcToolchainConfigInfo], ++git_repository( ++ name = "icu", ++ build_file = "@v8//:bazel/BUILD.icu", ++ commit = "a86a32e67b8d1384b33f8fa48c83a6079b86f8cd", ++ patch_cmds = ["find source -name BUILD.bazel | xargs rm"], ++ patch_cmds_win = ["Get-ChildItem -Path source -File -Include BUILD.bazel -Recurse | Remove-Item"], ++ remote = "https://chromium.googlesource.com/chromium/deps/icu.git", + ) +-""", +- build_file_content = """ +-load("@rules_cc//cc:defs.bzl", "cc_toolchain") +-load(":cc_toolchain_config.bzl", "cc_toolchain_config") +- +-package(default_visibility = ["//visibility:public"]) + +-filegroup( +- name = "all_files", +- srcs = glob(["**/*"]), ++http_archive( ++ name = "fast_float", ++ build_file_content = 'load("@rules_cc//cc:defs.bzl", "cc_library")\n\ncc_library(\n name = "fast_float",\n hdrs = glob(["include/fast_float/*.h"]),\n include_prefix = "third_party/fast_float/src/include",\n strip_include_prefix = "include",\n visibility = ["//visibility:public"],\n)\n', ++ sha256 = "e14a33089712b681d74d94e2a11362643bd7d769ae8f7e7caefe955f57f7eacd", ++ strip_prefix = "fast_float-8.0.2", ++ urls = ["https://github.com/fastfloat/fast_float/archive/refs/tags/v8.0.2.tar.gz"], + ) + +-filegroup(name = "empty") +- +-cc_toolchain_config(name = "k8_toolchain_config") +- +-cc_toolchain( +- name = "k8_toolchain", +- all_files = ":all_files", +- ar_files = ":all_files", +- compiler_files = ":all_files", +- dwp_files = ":empty", +- linker_files = ":all_files", +- objcopy_files = ":all_files", +- strip_files = ":all_files", +- supports_param_files = 0, +- toolchain_config = ":k8_toolchain_config", +- toolchain_identifier = "local_clang", ++http_archive( ++ name = "simdutf", ++ build_file_content = 'load("@rules_cc//cc:defs.bzl", "cc_library")\n\ncc_library(\n name = "simdutf",\n srcs = ["src/simdutf.cpp"],\n hdrs = glob(["include/**/*.h"]),\n textual_hdrs = glob(["src/**/*.h", "src/**/*.cpp"], exclude = ["src/simdutf.cpp"]),\n copts = ["-std=c++20"],\n include_prefix = "third_party/simdutf",\n includes = ["include", "src"],\n strip_include_prefix = "include",\n visibility = ["//visibility:public"],\n)\n', ++ sha256 = "0180de81a1dd48a87b8c0442ffa81734f3db91a7350914107a449935124e3c6f", ++ strip_prefix = "simdutf-7.7.0", ++ urls = ["https://github.com/simdutf/simdutf/archive/refs/tags/v7.7.0.tar.gz"], + ) + +-toolchain( +- name = "cc_toolchain_k8", +- exec_compatible_with = [ +- "@platforms//cpu:x86_64", +- "@platforms//os:linux", +- ], +- target_compatible_with = [ +- "@platforms//cpu:x86_64", +- "@platforms//os:linux", +- ], +- toolchain = ":k8_toolchain", +- toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", +-) +-""", ++git_repository( ++ name = "dragonbox", ++ build_file_content = 'load("@rules_cc//cc:defs.bzl", "cc_library")\n\ncc_library(\n name = "dragonbox",\n hdrs = ["include/dragonbox/dragonbox.h"],\n include_prefix = "third_party/dragonbox/src/include",\n strip_include_prefix = "include",\n visibility = ["//visibility:public"],\n)\n', ++ commit = "beeeef91cf6fef89a4d4ba5e95d47ca64ccb3a44", ++ remote = "https://chromium.googlesource.com/external/github.com/jk-jeon/dragonbox.git", + ) + +-register_toolchains("@llvm_toolchain//:cc_toolchain_k8") +- +-# Define local repository for libc++ from third_party sources +-libcxx_repository = use_repo_rule("//bazel/toolchain:libcxx_repository.bzl", "libcxx_repository") +- +-libcxx_repository( +- name = "libcxx", ++git_repository( ++ name = "fp16", ++ build_file_content = 'load("@rules_cc//cc:defs.bzl", "cc_library")\n\ncc_library(\n name = "fp16",\n hdrs = glob(["include/**/*.h"]),\n include_prefix = "third_party/fp16/src/include",\n includes = ["include"],\n strip_include_prefix = "include",\n visibility = ["//visibility:public"],\n)\n', ++ commit = "3d2de1816307bac63c16a297e8c4dc501b4076df", ++ remote = "https://chromium.googlesource.com/external/github.com/Maratyszcza/FP16.git", + ) + +-single_version_override( +- module_name = "highway", +- patches = ["//bazel/patches:highway_rules_cc.patch"], +- patch_strip = 1, ++pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") ++pip.parse( ++ hub_name = "v8_python_deps", ++ python_version = "3.11", ++ requirements_lock = "//:bazel/requirements.txt", ++ extra_pip_args = [ ++ "--require-hashes", ++ "--index-url=https://pypi.org/simple", ++ ], + ) ++use_repo(pip, "v8_python_deps") +diff --git a/mod/v8-15.0.245.2/bazel/highway.patch b/mod/v8-15.0.245.2/bazel/highway.patch +new file mode 100644 +index 0000000..5a13dd0 +--- /dev/null ++++ b/mod/v8-15.0.245.2/bazel/highway.patch +@@ -0,0 +1,25 @@ ++diff --git a/BUILD b/BUILD ++--- a/BUILD +++++ b/BUILD ++@@ -2,7 +2,7 @@ ++ load("@bazel_skylib//lib:selects.bzl", "selects") ++ load("@rules_license//rules:license.bzl", "license") ++ ++-load("@rules_cc//cc:defs.bzl", "cc_test") +++load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") ++ # Placeholder#2 for Guitar, do not remove ++ ++ package( ++@@ -134,11 +134,7 @@ COPTS = select({ ++ ], ++ }) ++ ++-DEFINES = select({ ++- ":compiler_msvc": ["HWY_SHARED_DEFINE"], ++- ":compiler_clangcl": ["HWY_SHARED_DEFINE"], ++- "//conditions:default": [], ++-}) +++DEFINES = [] ++ ++ # Unused on Bazel builds, where this is not defined/known; Copybara replaces ++ # usages with an empty list. diff --git a/patches/webrtc-sys_hermetic_darwin_sysroot.patch b/patches/webrtc-sys_hermetic_darwin_sysroot.patch new file mode 100644 index 0000000000000000000000000000000000000000..c90edbd08e9a8cd0bb063e4795da9fbe36fc2518 --- /dev/null +++ b/patches/webrtc-sys_hermetic_darwin_sysroot.patch @@ -0,0 +1,75 @@ +diff --git a/build.rs b/build.rs +index 3b4a24a..6aa06ff 100644 +--- a/build.rs ++++ b/build.rs +@@ -15,1 +15,2 @@ +-use std::path::Path; ++use std::fs; ++use std::path::Path; +@@ -118,1 +119,9 @@ +- println!("cargo:rustc-link-search=native={}", webrtc_lib.to_str().unwrap()); ++ let webrtc_link_lib = if env::var_os("WEBRTC_SYS_LINK_OUT_DIR").is_some() { ++ let out_lib = PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("lib"); ++ fs::create_dir_all(&out_lib).unwrap(); ++ fs::copy(webrtc_lib.join("libwebrtc.a"), out_lib.join("libwebrtc.a")).unwrap(); ++ out_lib ++ } else { ++ webrtc_lib ++ }; ++ println!("cargo:rustc-link-search=native={}", webrtc_link_lib.to_str().unwrap()); +@@ -255,1 +264,1 @@ +- println!("cargo:rustc-link-lib=framework=Appkit"); ++ println!("cargo:rustc-link-lib=framework=AppKit"); +@@ -320,1 +329,18 @@ +- builder.warnings(false).compile("webrtcsys-cxx"); ++ builder.warnings(false).compile("webrtcsys-cxx"); ++ let cxxbridge_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("cxxbridge"); ++ let cxx_header = cxxbridge_dir.join("include/rust/cxx.h"); ++ if fs::symlink_metadata(&cxx_header) ++ .is_ok_and(|metadata| metadata.file_type().is_symlink()) ++ { ++ if let Ok(header) = fs::read(&cxx_header) { ++ fs::remove_file(&cxx_header).unwrap(); ++ fs::write(&cxx_header, header).unwrap(); ++ } ++ } ++ let crate_link = cxxbridge_dir.join("crate/webrtc-sys"); ++ if fs::symlink_metadata(&crate_link) ++ .is_ok_and(|metadata| metadata.file_type().is_symlink()) ++ { ++ fs::remove_file(crate_link).unwrap(); ++ } ++ +@@ -375,1 +384,3 @@ +- println!("cargo:rustc-link-lib={}", clang_rt); ++ if env::var_os("WEBRTC_SYS_LINK_OUT_DIR").is_none() { ++ println!("cargo:rustc-link-lib={}", clang_rt); ++ } +@@ -378,6 +389,21 @@ +- let sysroot = Command::new("xcrun").args(["--sdk", sdk, "--show-sdk-path"]).output().unwrap(); +- +- let sysroot = String::from_utf8_lossy(&sysroot.stdout); +- let sysroot = sysroot.trim(); +- +- let search_dirs = Command::new("cc").arg("--print-search-dirs").output().unwrap(); ++ let sysroot = env::var("WEBRTC_SYS_DARWIN_SDK_PATH").unwrap_or_else(|_| { ++ let output = Command::new("xcrun") ++ .args(["--sdk", sdk, "--show-sdk-path"]) ++ .output() ++ .unwrap(); ++ String::from_utf8_lossy(&output.stdout).into_owned() ++ }); ++ ++ let sysroot = sysroot.as_str(); ++ let sysroot = sysroot.trim(); ++ builder.flag(format!("-F{sysroot}/System/Library/Frameworks")); ++ builder.define("__APPLICATIONSERVICES__", None); ++ builder.flag("-include"); ++ builder.flag("CoreGraphics/CoreGraphics.h"); ++ builder.include(format!("{sysroot}/usr/include")); ++ ++ let cc = env::var("CC").unwrap_or_else(|_| "cc".to_string()); ++ let search_dirs = Command::new(cc) ++ .arg("--print-search-dirs") ++ .output() ++ .unwrap(); diff --git a/patches/windows-link.patch b/patches/windows-link.patch new file mode 100644 index 0000000000000000000000000000000000000000..92423d1fe857d95ed6f6cd791c38e367ae0c7c8b --- /dev/null +++ b/patches/windows-link.patch @@ -0,0 +1,10 @@ +diff --git a/src/lib.rs b/src/lib.rs +index 2d5a2a2..6e8c4cd 100644 +--- a/src/lib.rs ++++ b/src/lib.rs +@@ -1,4 +1,4 @@ +-#![doc = include_str!("../readme.md")] ++#![doc = "windows-link"] + #![no_std] + + /// Defines an external function to import. diff --git a/patches/windows-support-native-tools.patch b/patches/windows-support-native-tools.patch new file mode 100644 index 0000000000000000000000000000000000000000..8d480d019e57b2ee615b5d577f193c6f34aa51da --- /dev/null +++ b/patches/windows-support-native-tools.patch @@ -0,0 +1,166 @@ +diff --git a/windows/private/extensions/msvc_runtime.bzl b/windows/private/extensions/msvc_runtime.bzl +--- a/windows/private/extensions/msvc_runtime.bzl ++++ b/windows/private/extensions/msvc_runtime.bzl +@@ -204,13 +204,20 @@ + _keep_only_children( + repository_ctx, + repository_ctx.path("{}/Contents/VC/Tools/MSVC/{}".format(sysroot_dir, msvc_version)), +- ["include", "lib"], ++ ["include", "lib", "bin"], + ) + _keep_only_children( + repository_ctx, + repository_ctx.path("{}/Contents/VC/Tools/MSVC/{}/lib".format(sysroot_dir, msvc_version)), + architectures, + ) ++ ++ # Keep native-host tools and their support files separate from runtime inputs. ++ bin_dir = "{}/Contents/VC/Tools/MSVC/{}/bin".format(sysroot_dir, msvc_version) ++ _keep_only_children(repository_ctx, repository_ctx.path(bin_dir), ["HostArm64" if arch == "arm64" else "Host" + arch for arch in architectures]) ++ for arch in architectures: ++ host = "HostArm64" if arch == "arm64" else "Host" + arch ++ _keep_only_children(repository_ctx, repository_ctx.path(bin_dir + "/" + host), [arch]) + + def _msvc_runtime_repository_impl(repository_ctx): + _check_msvc_license_requirements(repository_ctx) +diff --git a/windows/private/extensions/msvc_runtime.BUILD.bazel b/windows/private/extensions/msvc_runtime.BUILD.bazel +--- a/windows/private/extensions/msvc_runtime.BUILD.bazel ++++ b/windows/private/extensions/msvc_runtime.BUILD.bazel +@@ -17,6 +17,7 @@ + name = "sysroot_files", + srcs = glob( + ["__MSVC_RUNTIME_DIR__/**"], ++ exclude = ["__MSVC_RUNTIME_DIR__/Contents/VC/Tools/MSVC/__MSVC_VERSION__/bin/**"], + allow_empty = True, + ), + ) +@@ -98,6 +99,7 @@ + "**/*.hh", + "**/*.hpp", + ], ++ exclude = ["__MSVC_RUNTIME_DIR__/Contents/VC/Tools/MSVC/__MSVC_VERSION__/bin/**"], + allow_empty = True, + ), + ) +@@ -106,6 +108,37 @@ + name = "libs", + srcs = glob( + ["**/*.lib"], ++ exclude = ["__MSVC_RUNTIME_DIR__/Contents/VC/Tools/MSVC/__MSVC_VERSION__/bin/**"], + allow_empty = True, + ), + ) ++ ++# Native build tools are opt-in; ordinary sysroot users keep only headers/libs. ++ ++directory( ++ name = "msvc_tools_x64_files", ++ srcs = glob( ++ ["__MSVC_RUNTIME_DIR__/Contents/VC/Tools/MSVC/__MSVC_VERSION__/bin/Hostx64/x64/**"], ++ allow_empty = True, ++ ), ++) ++ ++subdirectory( ++ name = "msvc_tools_x64", ++ parent = ":msvc_tools_x64_files", ++ path = "__MSVC_RUNTIME_DIR__/Contents/VC/Tools/MSVC/__MSVC_VERSION__/bin/Hostx64/x64", ++) ++ ++directory( ++ name = "msvc_tools_arm64_files", ++ srcs = glob( ++ ["__MSVC_RUNTIME_DIR__/Contents/VC/Tools/MSVC/__MSVC_VERSION__/bin/HostArm64/arm64/**"], ++ allow_empty = True, ++ ), ++) ++ ++subdirectory( ++ name = "msvc_tools_arm64", ++ parent = ":msvc_tools_arm64_files", ++ path = "__MSVC_RUNTIME_DIR__/Contents/VC/Tools/MSVC/__MSVC_VERSION__/bin/HostArm64/arm64", ++) +diff --git a/windows/private/extensions/windows_sdk.bzl b/windows/private/extensions/windows_sdk.bzl +--- a/windows/private/extensions/windows_sdk.bzl ++++ b/windows/private/extensions/windows_sdk.bzl +@@ -31,13 +31,18 @@ + def _keep_exposed_windows_sdk_files(repository_ctx, sysroot_dir, include_version, architectures): + _keep_only_children(repository_ctx, repository_ctx.path(sysroot_dir), ["base"] + architectures) + _keep_only_children(repository_ctx, repository_ctx.path("{}/base".format(sysroot_dir)), ["c"]) +- _keep_only_children(repository_ctx, repository_ctx.path("{}/base/c".format(sysroot_dir)), ["Include"]) ++ _keep_only_children(repository_ctx, repository_ctx.path("{}/base/c".format(sysroot_dir)), ["Include", "bin"]) + _keep_only_children(repository_ctx, repository_ctx.path("{}/base/c/Include".format(sysroot_dir)), [include_version]) + _keep_only_children( + repository_ctx, + repository_ctx.path("{}/base/c/Include/{}".format(sysroot_dir, include_version)), + ["ucrt", "shared", "um", "winrt"], + ) ++ ++ # Resource and manifest tools share the selected SDK's versioned bin tree. ++ bin_dir = "{}/base/c/bin".format(sysroot_dir) ++ _keep_only_children(repository_ctx, repository_ctx.path(bin_dir), [include_version]) ++ _keep_only_children(repository_ctx, repository_ctx.path(bin_dir + "/" + include_version), architectures) + + seen_architectures = {} + for arch in architectures: +diff --git a/windows/private/extensions/windows_sdk.BUILD.bazel b/windows/private/extensions/windows_sdk.BUILD.bazel +--- a/windows/private/extensions/windows_sdk.BUILD.bazel ++++ b/windows/private/extensions/windows_sdk.BUILD.bazel +@@ -19,6 +19,7 @@ + name = "sysroot_files", + srcs = glob( + ["__WINSDK_DIR__/**"], ++ exclude = ["__WINSDK_DIR__/base/c/bin/__WINSDK_INCLUDE_VERSION__/**"], + allow_empty = True, + ) + _LOGICAL_ALIASES, + ) +@@ -210,6 +211,7 @@ + "**/*.hh", + "**/*.hpp", + ], ++ exclude = ["__WINSDK_DIR__/base/c/bin/__WINSDK_INCLUDE_VERSION__/**"], + allow_empty = True, + ) + [ + f +@@ -222,6 +224,7 @@ + name = "libs", + srcs = glob( + ["**/*.lib"], ++ exclude = ["__WINSDK_DIR__/base/c/bin/__WINSDK_INCLUDE_VERSION__/**"], + allow_empty = True, + ) + [ + f +@@ -229,3 +232,33 @@ + if f.endswith(".lib") + ], + ) ++ ++# Native build tools are opt-in; ordinary sysroot users keep only headers/libs. ++ ++directory( ++ name = "winsdk_tools_x64_files", ++ srcs = glob( ++ ["__WINSDK_DIR__/base/c/bin/__WINSDK_INCLUDE_VERSION__/x64/**"], ++ allow_empty = True, ++ ), ++) ++ ++subdirectory( ++ name = "winsdk_tools_x64", ++ parent = ":winsdk_tools_x64_files", ++ path = "__WINSDK_DIR__/base/c/bin/__WINSDK_INCLUDE_VERSION__/x64", ++) ++ ++directory( ++ name = "winsdk_tools_arm64_files", ++ srcs = glob( ++ ["__WINSDK_DIR__/base/c/bin/__WINSDK_INCLUDE_VERSION__/arm64/**"], ++ allow_empty = True, ++ ), ++) ++ ++subdirectory( ++ name = "winsdk_tools_arm64", ++ parent = ":winsdk_tools_arm64_files", ++ path = "__WINSDK_DIR__/base/c/bin/__WINSDK_INCLUDE_VERSION__/arm64", ++) diff --git a/patches/windows-support-remote-case-aliases.patch b/patches/windows-support-remote-case-aliases.patch new file mode 100644 index 0000000000000000000000000000000000000000..0127a7ef60e18535af460b73d67939f7845df179 --- /dev/null +++ b/patches/windows-support-remote-case-aliases.patch @@ -0,0 +1,235 @@ +diff --git a/windows/private/extensions/windows_sdk.bzl b/windows/private/extensions/windows_sdk.bzl +index 420cee4..16d2cf3 100644 +--- a/windows/private/extensions/windows_sdk.bzl ++++ b/windows/private/extensions/windows_sdk.bzl +@@ -133,19 +133,18 @@ def _repository_fs_is_case_sensitive(repository_ctx): + def _apply_transformations(repository_ctx): + transformations = repository_ctx.attr.transformations + if not transformations: +- return ++ return [] + + sysroot = repository_ctx.path(_SYSROOT_DIR) + if not sysroot.exists: +- return +- if not _repository_fs_is_case_sensitive(repository_ctx): +- # Case-only aliases cannot be materialized on case-insensitive filesystems. +- return ++ return [] ++ repository_fs_is_case_sensitive = _repository_fs_is_case_sensitive(repository_ctx) + + all_files = [] + _collect_files_recursive(sysroot, all_files) + all_rel_files = [_relative_to_repo_root(repository_ctx, f) for f in all_files] + created = {} ++ logical_aliases = [] + + for src_pattern, transform in transformations.items(): + src_pattern = _normalize_relpath(src_pattern) +@@ -165,11 +164,18 @@ def _apply_transformations(repository_ctx): + if not dst_rel or dst_rel == src_rel or created.get(dst_rel, False): + continue + if repository_ctx.path(dst_rel).exists: ++ if not repository_fs_is_case_sensitive and src_rel.lower() == dst_rel.lower(): ++ # Keep the alternate spelling as a Bazel source label. Bazel then ++ # preserves that spelling in a case-sensitive remote input tree. ++ logical_aliases.append(dst_rel) ++ created[dst_rel] = True + continue + + repository_ctx.symlink(src_rel, dst_rel) + created[dst_rel] = True + ++ return sorted(logical_aliases) ++ + def _windows_sdk_repository_impl(repository_ctx): + if not repository_ctx.attr.windows_sdk_version: + fail("windows_sdk_version must be set") +@@ -220,8 +226,8 @@ def _windows_sdk_repository_impl(repository_ctx): + "windows_sdk_include_versions", + ) + +- _apply_transformations(repository_ctx) + _keep_exposed_windows_sdk_files(repository_ctx, _SYSROOT_DIR, include_version, requested_architectures) ++ logical_aliases = _apply_transformations(repository_ctx) + + repository_ctx.template( + "BUILD.bazel", +@@ -229,6 +235,7 @@ def _windows_sdk_repository_impl(repository_ctx): + substitutions = { + "__WINSDK_DIR__": _SYSROOT_DIR, + "__WINSDK_INCLUDE_VERSION__": include_version, ++ "__WINSDK_LOGICAL_ALIASES__": repr(logical_aliases), + }, + ) + +@@ -258,7 +265,9 @@ _WINDOWS_SDK_ATTR = { + "transformations": attr.string_dict( + default = {}, + doc = """ +- Dict of source path patterns to transformations applied when run on case-sensitive filesystems. ++ Dict of source path patterns to transformations applied to extracted Windows SDK files. ++ Case-only transformations are Bazel aliases on case-insensitive filesystems and symlinks on ++ case-sensitive filesystems. + + Supports exact paths and `**/*.ext` patterns. Use value `lowercase` to create lowercase aliases. + +diff --git a/windows/private/extensions/windows_sdk.BUILD.bazel b/windows/private/extensions/windows_sdk.BUILD.bazel +index 0d03f4e..199fdf0 100644 +--- a/windows/private/extensions/windows_sdk.BUILD.bazel ++++ b/windows/private/extensions/windows_sdk.BUILD.bazel +@@ -5,6 +5,8 @@ load("@bazel_skylib//rules/directory:subdirectory.bzl", "subdirectory") + + package(default_visibility = ["//visibility:public"]) + ++_LOGICAL_ALIASES = __WINSDK_LOGICAL_ALIASES__ ++ + config_setting( + name = "windows_aarch64", + constraint_values = [ +@@ -18,7 +20,7 @@ directory( + srcs = glob( + ["__WINSDK_DIR__/**"], + allow_empty = True, +- ), ++ ) + _LOGICAL_ALIASES, + ) + + subdirectory( +@@ -32,7 +34,11 @@ directory( + srcs = glob( + ["__WINSDK_DIR__/base/c/Include/__WINSDK_INCLUDE_VERSION__/ucrt/**"], + allow_empty = True, +- ), ++ ) + [ ++ f ++ for f in _LOGICAL_ALIASES ++ if f.startswith("__WINSDK_DIR__/base/c/Include/__WINSDK_INCLUDE_VERSION__/ucrt/") ++ ], + ) + + subdirectory( +@@ -46,7 +52,11 @@ directory( + srcs = glob( + ["__WINSDK_DIR__/base/c/Include/__WINSDK_INCLUDE_VERSION__/shared/**"], + allow_empty = True, +- ), ++ ) + [ ++ f ++ for f in _LOGICAL_ALIASES ++ if f.startswith("__WINSDK_DIR__/base/c/Include/__WINSDK_INCLUDE_VERSION__/shared/") ++ ], + ) + + subdirectory( +@@ -60,7 +70,11 @@ directory( + srcs = glob( + ["__WINSDK_DIR__/base/c/Include/__WINSDK_INCLUDE_VERSION__/um/**"], + allow_empty = True, +- ), ++ ) + [ ++ f ++ for f in _LOGICAL_ALIASES ++ if f.startswith("__WINSDK_DIR__/base/c/Include/__WINSDK_INCLUDE_VERSION__/um/") ++ ], + ) + + subdirectory( +@@ -74,7 +88,11 @@ directory( + srcs = glob( + ["__WINSDK_DIR__/base/c/Include/__WINSDK_INCLUDE_VERSION__/winrt/**"], + allow_empty = True, +- ), ++ ) + [ ++ f ++ for f in _LOGICAL_ALIASES ++ if f.startswith("__WINSDK_DIR__/base/c/Include/__WINSDK_INCLUDE_VERSION__/winrt/") ++ ], + ) + + subdirectory( +@@ -88,7 +106,11 @@ directory( + srcs = glob( + ["__WINSDK_DIR__/arm64/c/ucrt/arm64/**"], + allow_empty = True, +- ), ++ ) + [ ++ f ++ for f in _LOGICAL_ALIASES ++ if f.startswith("__WINSDK_DIR__/arm64/c/ucrt/arm64/") ++ ], + ) + + subdirectory( +@@ -102,7 +124,11 @@ directory( + srcs = glob( + ["__WINSDK_DIR__/x64/c/ucrt/x64/**"], + allow_empty = True, +- ), ++ ) + [ ++ f ++ for f in _LOGICAL_ALIASES ++ if f.startswith("__WINSDK_DIR__/x64/c/ucrt/x64/") ++ ], + ) + + subdirectory( +@@ -116,7 +142,11 @@ directory( + srcs = glob( + ["__WINSDK_DIR__/arm64/c/um/arm64/**"], + allow_empty = True, +- ), ++ ) + [ ++ f ++ for f in _LOGICAL_ALIASES ++ if f.startswith("__WINSDK_DIR__/arm64/c/um/arm64/") ++ ], + ) + + subdirectory( +@@ -130,7 +160,11 @@ directory( + srcs = glob( + ["__WINSDK_DIR__/x64/c/um/x64/**"], + allow_empty = True, +- ), ++ ) + [ ++ f ++ for f in _LOGICAL_ALIASES ++ if f.startswith("__WINSDK_DIR__/x64/c/um/x64/") ++ ], + ) + + subdirectory( +@@ -160,7 +194,7 @@ filegroup( + srcs = glob( + ["**"], + allow_empty = True, +- ), ++ ) + _LOGICAL_ALIASES, + ) + + filegroup( +@@ -177,7 +211,11 @@ filegroup( + "**/*.hpp", + ], + allow_empty = True, +- ), ++ ) + [ ++ f ++ for f in _LOGICAL_ALIASES ++ if f.endswith(".h") or f.endswith(".hh") or f.endswith(".hpp") ++ ], + ) + + filegroup( +@@ -185,5 +223,9 @@ filegroup( + srcs = glob( + ["**/*.lib"], + allow_empty = True, +- ), ++ ) + [ ++ f ++ for f in _LOGICAL_ALIASES ++ if f.endswith(".lib") ++ ], + ) diff --git a/patches/xz_windows_stack_args.patch b/patches/xz_windows_stack_args.patch new file mode 100644 index 0000000000000000000000000000000000000000..926fb0a6dc90f310730659f44d16dfe0c2937a2c --- /dev/null +++ b/patches/xz_windows_stack_args.patch @@ -0,0 +1,14 @@ +diff --git a/BUILD.bazel b/BUILD.bazel +--- a/BUILD.bazel ++++ b/BUILD.bazel +@@ -154,6 +154,9 @@ cc_library( + ], + copts = select({ +- "@platforms//os:windows": [], ++ "@platforms//os:windows": [ ++ "-fno-stack-protector", ++ "-mno-stack-arg-probe", ++ ], + "//conditions:default": ["-std=c99"], + }), + defines = select({ diff --git a/patches/zstd-sys_windows_msvc_include_dirs.patch b/patches/zstd-sys_windows_msvc_include_dirs.patch new file mode 100644 index 0000000000000000000000000000000000000000..c45a360a46c283b78bc42a8157c6d9c49aea4e3b --- /dev/null +++ b/patches/zstd-sys_windows_msvc_include_dirs.patch @@ -0,0 +1,41 @@ +diff --git a/build.rs b/build.rs +index 912a3bf..e078f0e 100644 +--- a/build.rs ++++ b/build.rs +@@ -85,6 +85,7 @@ fn flag_if_supported_with_fallbacks(config: &mut cc::Build, flags: &[&str]) { + } + + fn compile_zstd() { ++ let target = env::var("TARGET").unwrap_or_default(); + let mut config = cc::Build::new(); + + // Search the following directories for C files to add to the compilation. +@@ -164,6 +165,16 @@ fn compile_zstd() { + // Some extra parameters + config.include("zstd/lib/"); + config.include("zstd/lib/common"); ++ if target.contains("msvc") { ++ config.include("zstd/lib/compress"); ++ config.include("zstd/lib/decompress"); ++ #[cfg(feature = "zdict_builder")] ++ config.include("zstd/lib/dictBuilder"); ++ #[cfg(feature = "legacy")] ++ config.include("zstd/lib/legacy"); ++ #[cfg(feature = "seekable")] ++ config.include("zstd/contrib/seekable_format"); ++ } + config.warnings(false); + + config.define("ZSTD_LIB_DEPRECATED", Some("0")); +@@ -210,7 +221,9 @@ fn compile_zstd() { + + // Hide symbols from resulting library, + // so we can be used with another zstd-linking lib. +- // See https://github.com/gyscos/zstd-rs/issues/58 +- config.flag("-fvisibility=hidden"); ++ // See https://github.com/gyscos/zstd-rs/issues/58. ++ if !target.contains("msvc") { ++ config.flag("-fvisibility=hidden"); ++ } + config.define("XXH_PRIVATE_API", Some("")); + config.define("ZSTDLIB_VISIBILITY", Some("")); diff --git a/scripts/asciicheck.py b/scripts/asciicheck.py new file mode 100644 index 0000000000000000000000000000000000000000..812d1c6b69b4cb8c811d71dd8cd5338b0c0af626 --- /dev/null +++ b/scripts/asciicheck.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +import argparse +import sys +from pathlib import Path + +""" +Utility script that takes a list of files and returns non-zero if any of them +contain non-ASCII characters other than those in the allowed list. + +If --fix is used, it will attempt to replace non-ASCII characters with ASCII +equivalents. + +The motivation behind this script is that characters like U+00A0 (non-breaking +space) can cause regexes not to match and can result in surprising anchor +values for headings when GitHub renders Markdown as HTML. +""" + + +""" +When --fix is used, perform the following substitutions. +""" +substitutions: dict[int, str] = { + 0x00A0: " ", # non-breaking space + 0x2011: "-", # non-breaking hyphen + 0x2013: "-", # en dash + 0x2014: "-", # em dash + 0x2018: "'", # left single quote + 0x2019: "'", # right single quote + 0x201C: '"', # left double quote + 0x201D: '"', # right double quote + 0x2026: "...", # ellipsis + 0x202F: " ", # narrow non-breaking space +} + +""" +Unicode codepoints that are allowed in addition to ASCII. +Be conservative with this list. + +Note that it is always an option to use the hex HTML representation +instead of the character itself so the source code is ASCII-only. +For example, U+2728 (sparkles) can be written as `✨`. +""" +allowed_unicode_codepoints = { + 0x2728, # sparkles +} + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check for non-ASCII characters in files." + ) + parser.add_argument( + "--fix", + action="store_true", + help="Rewrite files, replacing non-ASCII characters with ASCII equivalents, where possible.", + ) + parser.add_argument( + "files", + nargs="+", + help="Files to check for non-ASCII characters.", + ) + args = parser.parse_args() + + has_errors = False + for filename in args.files: + path = Path(filename) + has_errors |= lint_utf8_ascii(path, fix=args.fix) + return 1 if has_errors else 0 + + +def lint_utf8_ascii(filename: Path, fix: bool) -> bool: + """Returns True if an error was printed.""" + try: + with open(filename, "rb") as f: + raw = f.read() + text = raw.decode("utf-8") + except UnicodeDecodeError as e: + print("UTF-8 decoding error:") + print(f" byte offset: {e.start}") + print(f" reason: {e.reason}") + # Attempt to find line/column + partial = raw[: e.start] + line = partial.count(b"\n") + 1 + col = e.start - (partial.rfind(b"\n") if b"\n" in partial else -1) + print(f" location: line {line}, column {col}") + return True + + errors = [] + for lineno, line in enumerate(text.splitlines(keepends=True), 1): + for colno, char in enumerate(line, 1): + codepoint = ord(char) + if char == "\n": + continue + if ( + not (0x20 <= codepoint <= 0x7E) + and codepoint not in allowed_unicode_codepoints + ): + errors.append((lineno, colno, char, codepoint)) + + if errors: + for lineno, colno, char, codepoint in errors: + safe_char = repr(char)[1:-1] # nicely escape things like \u202f + print( + f"Invalid character at line {lineno}, column {colno}: U+{codepoint:04X} ({safe_char})" + ) + + if errors and fix: + print(f"Attempting to fix {filename}...") + num_replacements = 0 + new_contents = "" + for char in text: + codepoint = ord(char) + if codepoint in substitutions: + num_replacements += 1 + new_contents += substitutions[codepoint] + else: + new_contents += char + with open(filename, "w", encoding="utf-8") as f: + f.write(new_contents) + print(f"Fixed {num_replacements} of {len(errors)} errors in {filename}.") + + return bool(errors) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_blob_size.py b/scripts/check_blob_size.py new file mode 100644 index 0000000000000000000000000000000000000000..b63e1d19f0cba28e7954bc41d3800d3f967b6c9b --- /dev/null +++ b/scripts/check_blob_size.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + + +DEFAULT_MAX_BYTES = 500 * 1024 + + +@dataclass(frozen=True) +class ChangedBlob: + path: str + size_bytes: int + is_allowlisted: bool + is_binary: bool + + +def run_git(*args: str) -> str: + result = subprocess.run( + ["git", *args], + check=True, + capture_output=True, + text=True, + ) + return result.stdout + + +def load_allowlist(path: Path) -> set[str]: + allowlist: set[str] = set() + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.split("#", 1)[0].strip() + if line: + allowlist.add(line) + return allowlist + + +def get_changed_paths(base: str, head: str) -> list[str]: + output = run_git( + "diff", + "--name-only", + "--diff-filter=AM", + "--no-renames", + "-z", + base, + head, + ) + return [path for path in output.split("\0") if path] + + +def is_binary_change(base: str, head: str, path: str) -> bool: + output = run_git( + "diff", + "--numstat", + "--diff-filter=AM", + "--no-renames", + base, + head, + "--", + path, + ).strip() + if not output: + return False + + added, deleted, _ = output.split("\t", 2) + return added == "-" and deleted == "-" + + +def blob_size(commit: str, path: str) -> int: + return int(run_git("cat-file", "-s", f"{commit}:{path}").strip()) + + +def collect_changed_blobs( + base: str, head: str, allowlist: set[str] +) -> list[ChangedBlob]: + blobs: list[ChangedBlob] = [] + for path in get_changed_paths(base, head): + blobs.append( + ChangedBlob( + path=path, + size_bytes=blob_size(head, path), + is_allowlisted=path in allowlist, + is_binary=is_binary_change(base, head, path), + ) + ) + return blobs + + +def format_kib(size_bytes: int) -> str: + return f"{size_bytes / 1024:.1f} KiB" + + +def write_step_summary( + max_bytes: int, + blobs: list[ChangedBlob], + violations: list[ChangedBlob], +) -> None: + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_path: + return + + lines = [ + "## Blob Size Policy", + "", + f"Default max: `{max_bytes}` bytes ({format_kib(max_bytes)})", + f"Changed files checked: `{len(blobs)}`", + f"Violations: `{len(violations)}`", + "", + ] + + if blobs: + lines.extend( + [ + "| Path | Kind | Size | Status |", + "| --- | --- | ---: | --- |", + ] + ) + for blob in blobs: + status = "allowlisted" if blob.is_allowlisted else "ok" + if blob in violations: + status = "blocked" + kind = "binary" if blob.is_binary else "non-binary" + lines.append( + f"| `{blob.path}` | {kind} | `{blob.size_bytes}` bytes ({format_kib(blob.size_bytes)}) | {status} |" + ) + else: + lines.append("No changed files were detected.") + + lines.append("") + Path(summary_path).write_text("\n".join(lines), encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Fail if changed blobs exceed the configured size budget." + ) + parser.add_argument( + "--base", required=True, help="Base git revision to diff against." + ) + parser.add_argument("--head", required=True, help="Head git revision to inspect.") + parser.add_argument( + "--max-bytes", + type=int, + default=DEFAULT_MAX_BYTES, + help=f"Maximum allowed blob size in bytes. Default: {DEFAULT_MAX_BYTES}.", + ) + parser.add_argument( + "--allowlist", + type=Path, + required=True, + help="Path to the newline-delimited allowlist file.", + ) + args = parser.parse_args() + + allowlist = load_allowlist(args.allowlist) + blobs = collect_changed_blobs(args.base, args.head, allowlist) + violations = [ + blob + for blob in blobs + if blob.size_bytes > args.max_bytes and not blob.is_allowlisted + ] + + write_step_summary(args.max_bytes, blobs, violations) + + if not blobs: + print("No changed files were detected.") + return 0 + + print( + f"Checked {len(blobs)} changed file(s) against the {args.max_bytes}-byte limit." + ) + for blob in blobs: + status = "allowlisted" if blob.is_allowlisted else "ok" + if blob in violations: + status = "blocked" + kind = "binary" if blob.is_binary else "non-binary" + print( + f"- {blob.path}: {blob.size_bytes} bytes ({format_kib(blob.size_bytes)}) [{kind}, {status}]" + ) + + if violations: + print("\nFile(s) exceed the configured limit:") + for blob in violations: + print(f"- {blob.path}: {blob.size_bytes} bytes > {args.max_bytes} bytes") + print( + "\nIf one of these is a real checked-in asset we want to keep, add its " + "repo-relative path to .github/blob-size-allowlist.txt. Otherwise, " + "shrink it or keep it out of git." + ) + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/just-shell.py b/scripts/just-shell.py new file mode 100644 index 0000000000000000000000000000000000000000..bb2db6ebc4312a301ee39d2793fde16289d046d8 --- /dev/null +++ b/scripts/just-shell.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Cross-platform shell launcher for `just` recipes. + +This keeps recipe bodies as normal shell snippets while giving the justfile one +portable placeholder, `{args}`, for forwarding variadic recipe arguments. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys + + +ARGS_TOKEN = "{args}" +STDERR_NULL_TOKEN = "{stderr-null}" +POWERSHELL_ARGS = "@($args | Select-Object -Skip 1)" +POWERSHELL_STDERR_NULL = "2>$null; exit $LASTEXITCODE" +SH_ARGS = '"$@"' +SH_STDERR_NULL = "2>/dev/null" + + +def main() -> int: + if len(sys.argv) < 2: + print("just shell adapter expected a recipe command.", file=sys.stderr) + return 1 + + command = sys.argv[1] + recipe_name = sys.argv[2] if len(sys.argv) > 2 else "" + recipe_args = sys.argv[3:] + + if os.name == "nt": + return run_powershell(command, recipe_name, recipe_args) + else: + return run_sh(command, recipe_name, recipe_args) + + +def run_sh(command: str, recipe_name: str, recipe_args: list[str]) -> int: + command = command.replace(ARGS_TOKEN, SH_ARGS) + command = command.replace(STDERR_NULL_TOKEN, SH_STDERR_NULL) + os.execvp("sh", ["sh", "-cu", command, recipe_name, *recipe_args]) + + +def run_powershell(command: str, recipe_name: str, recipe_args: list[str]) -> int: + pwsh = shutil.which("pwsh.exe") or shutil.which("pwsh") + if pwsh is None: + print( + "PowerShell ('pwsh') is required for Windows just recipes. " + "Run 'just install' to install it.", + file=sys.stderr, + ) + return 1 + + command = command.replace(ARGS_TOKEN, POWERSHELL_ARGS) + command = command.replace(STDERR_NULL_TOKEN, POWERSHELL_STDERR_NULL) + return subprocess.run( + [ + pwsh, + "-NoLogo", + "-NoProfile", + "-CommandWithArgs", + command, + recipe_name, + *recipe_args, + ], + check=False, + ).returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_tui_with_exec_server.sh b/scripts/run_tui_with_exec_server.sh new file mode 100644 index 0000000000000000000000000000000000000000..db926fc74b65d86875eaffb00ad7c607b0e533e2 --- /dev/null +++ b/scripts/run_tui_with_exec_server.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cargo_root="$repo_root/codex-rs" +listen_url="${CODEX_EXEC_SERVER_LISTEN_URL:-ws://127.0.0.1:0}" +start_timeout_seconds="${CODEX_EXEC_SERVER_START_TIMEOUT_SECONDS:-120}" +tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/codex-tui-with-exec-server.XXXXXX")" +stdout_log="$tmp_dir/exec-server.stdout" +stderr_log="$tmp_dir/exec-server.stderr" +server_pid="" +exec_server_url="" + +cleanup() { + if [[ -n "$server_pid" ]]; then + kill "$server_pid" >/dev/null 2>&1 || true + wait "$server_pid" >/dev/null 2>&1 || true + fi + rm -rf "$tmp_dir" +} + +trap cleanup EXIT INT TERM HUP + +( + cd "$cargo_root" + cargo run -p codex-cli --bin codex -- exec-server --listen "$listen_url" +) >"$stdout_log" 2>"$stderr_log" & +server_pid="$!" + +# Wait for the server to print its bound websocket URL before launching the TUI. +for _ in $(seq 1 "$((start_timeout_seconds * 20))"); do + if [[ -s "$stdout_log" ]]; then + exec_server_url="$(head -n 1 "$stdout_log" | tr -d '\r')" + if [[ "$exec_server_url" == ws://* ]]; then + break + fi + fi + + if ! kill -0 "$server_pid" >/dev/null 2>&1; then + cat "$stderr_log" >&2 || true + cat "$stdout_log" >&2 || true + echo "failed to start codex exec-server" >&2 + exit 1 + fi + + sleep 0.05 +done + +if [[ -z "$exec_server_url" ]]; then + cat "$stderr_log" >&2 || true + cat "$stdout_log" >&2 || true + echo "timed out waiting ${start_timeout_seconds}s for codex exec-server to report its websocket URL" >&2 + exit 1 +fi + +export CODEX_EXEC_SERVER_URL="$exec_server_url" +echo "Starting codex-tui with CODEX_EXEC_SERVER_URL=$CODEX_EXEC_SERVER_URL" >&2 + +cd "$cargo_root" +cargo run -p codex-tui --bin codex-tui -- -c mcp_oauth_credentials_store=file "$@" diff --git a/scripts/stage_npm_packages.py b/scripts/stage_npm_packages.py new file mode 100644 index 0000000000000000000000000000000000000000..b258b4c94626562ef675302ceec1ee36ccdb6619 --- /dev/null +++ b/scripts/stage_npm_packages.py @@ -0,0 +1,632 @@ +#!/usr/bin/env python3 +"""Stage one or more Codex npm packages for release.""" + +import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import contextmanager +from dataclasses import dataclass +import importlib.util +import json +import os +import shutil +import subprocess +import tarfile +import tempfile +from pathlib import Path +from typing import Sequence + + +REPO_ROOT = Path(__file__).resolve().parent.parent +BUILD_SCRIPT = REPO_ROOT / "codex-cli" / "scripts" / "build_npm_package.py" +WORKFLOW_NAME = ".github/workflows/rust-release.yml" +GITHUB_REPO = "openai/codex" +BINARY_TARGETS = ( + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-musl", + "x86_64-apple-darwin", + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc", + "aarch64-pc-windows-msvc", +) + +_SPEC = importlib.util.spec_from_file_location("codex_build_npm_package", BUILD_SCRIPT) +if _SPEC is None or _SPEC.loader is None: + raise RuntimeError(f"Unable to load module from {BUILD_SCRIPT}") +_BUILD_MODULE = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_BUILD_MODULE) +PACKAGE_NATIVE_COMPONENTS = getattr(_BUILD_MODULE, "PACKAGE_NATIVE_COMPONENTS", {}) +PACKAGE_EXPANSIONS = getattr(_BUILD_MODULE, "PACKAGE_EXPANSIONS", {}) +CODEX_PLATFORM_PACKAGES = getattr(_BUILD_MODULE, "CODEX_PLATFORM_PACKAGES", {}) +CODEX_PACKAGE_COMPONENT = getattr( + _BUILD_MODULE, "CODEX_PACKAGE_COMPONENT", "codex-package" +) + + +@dataclass(frozen=True) +class BinaryComponent: + artifact_prefix: str + dest_dir: str + binary_basename: str + + +@dataclass(frozen=True) +class WorkflowArtifact: + name: str + size_in_bytes: int + + +BINARY_COMPONENTS = { + "codex-responses-api-proxy": BinaryComponent( + artifact_prefix="codex-responses-api-proxy", + dest_dir="codex-responses-api-proxy", + binary_basename="codex-responses-api-proxy", + ), +} + + +def _gha_enabled() -> bool: + return os.environ.get("GITHUB_ACTIONS") == "true" + + +def _gha_escape(value: str) -> str: + return value.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +@contextmanager +def _gha_group(title: str): + if _gha_enabled(): + print(f"::group::{_gha_escape(title)}", flush=True) + try: + yield + finally: + if _gha_enabled(): + print("::endgroup::", flush=True) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--release-version", + required=True, + help="Version to stage (e.g. 0.1.0 or 0.1.0-alpha.1).", + ) + parser.add_argument( + "--package", + dest="packages", + action="append", + required=True, + help="Package name to stage. May be provided multiple times.", + ) + parser.add_argument( + "--workflow-url", + help="Optional workflow URL to reuse for native artifacts.", + ) + parser.add_argument( + "--artifacts-dir", + type=Path, + help="Directory containing previously downloaded workflow artifacts.", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=None, + help="Directory where npm tarballs should be written (default: dist/npm).", + ) + parser.add_argument( + "--keep-staging-dirs", + action="store_true", + help="Retain temporary staging directories instead of deleting them.", + ) + return parser.parse_args() + + +def native_components_for_package(package: str) -> tuple[str, ...]: + return tuple(sorted(PACKAGE_NATIVE_COMPONENTS.get(package, []))) + + +def collect_native_component_sets(packages: list[str]) -> list[tuple[str, ...]]: + component_sets: list[tuple[str, ...]] = [] + seen: set[tuple[str, ...]] = set() + for package in packages: + components = native_components_for_package(package) + if not components or components in seen: + continue + seen.add(components) + component_sets.append(components) + return component_sets + + +def expand_packages(packages: list[str]) -> list[str]: + expanded: list[str] = [] + for package in packages: + for expanded_package in PACKAGE_EXPANSIONS.get(package, [package]): + if expanded_package in expanded: + continue + expanded.append(expanded_package) + return expanded + + +def resolve_release_workflow(version: str) -> dict: + stdout = subprocess.check_output( + [ + "gh", + "run", + "list", + "--branch", + f"rust-v{version}", + "--json", + "workflowName,url,headSha", + "--workflow", + WORKFLOW_NAME, + "--jq", + "first(.[])", + ], + cwd=REPO_ROOT, + text=True, + ) + workflow = json.loads(stdout or "null") + if not workflow: + raise RuntimeError( + f"Unable to find rust-release workflow for version {version}." + ) + return workflow + + +def resolve_workflow_url(version: str, override: str | None) -> tuple[str, str | None]: + if override: + return override, None + + workflow = resolve_release_workflow(version) + return workflow["url"], workflow.get("headSha") + + +def install_native_components( + workflow_url: str | None, + components: set[str], + vendor_root: Path, + artifacts_dir: Path, +) -> None: + if not components: + return + + vendor_dir = vendor_root / "vendor" + vendor_dir.mkdir(parents=True, exist_ok=True) + + if workflow_url is None: + with _gha_group("Install downloaded native artifacts"): + install_from_downloaded_artifacts( + artifacts_dir, sorted(components), vendor_dir + ) + else: + workflow_id = workflow_url.rstrip("/").split("/")[-1] + print( + f"Downloading native artifacts from workflow {workflow_id}...", flush=True + ) + with _gha_group(f"Download native artifacts from workflow {workflow_id}"): + artifacts_dir.mkdir(parents=True, exist_ok=True) + install_from_workflow_artifacts( + workflow_id, + artifacts_dir, + sorted(components), + vendor_dir, + ) + print(f"Installed native dependencies into {vendor_dir}", flush=True) + + +def install_from_workflow_artifacts( + workflow_id: str, + artifacts_dir: Path, + components: Sequence[str], + vendor_dir: Path, +) -> None: + artifacts = select_target_artifacts(workflow_id, components) + download_artifacts(workflow_id, artifacts_dir, artifacts) + install_from_downloaded_artifacts(artifacts_dir, components, vendor_dir) + + +def install_from_downloaded_artifacts( + artifacts_dir: Path, + components: Sequence[str], + vendor_dir: Path, +) -> None: + if CODEX_PACKAGE_COMPONENT in components: + install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS) + install_binary_components( + artifacts_dir, + vendor_dir, + [BINARY_COMPONENTS[name] for name in components if name in BINARY_COMPONENTS], + ) + + +def select_target_artifacts( + workflow_id: str, + components: Sequence[str], +) -> list[WorkflowArtifact]: + needs_target_artifacts = CODEX_PACKAGE_COMPONENT in components or any( + component in BINARY_COMPONENTS for component in components + ) + if not needs_target_artifacts: + return [] + + artifacts_by_name = { + artifact.name: artifact for artifact in list_workflow_artifacts(workflow_id) + } + selected_artifacts: list[WorkflowArtifact] = [] + for target in BINARY_TARGETS: + for artifact_name in [target, f"{target}-unsigned"]: + artifact = artifacts_by_name.get(artifact_name) + if artifact is not None: + selected_artifacts.append(artifact) + break + else: + raise FileNotFoundError( + f"Expected workflow artifact not found for target {target}" + ) + + return selected_artifacts + + +def list_workflow_artifacts(workflow_id: str) -> list[WorkflowArtifact]: + stdout = subprocess.check_output( + [ + "gh", + "api", + f"repos/{GITHUB_REPO}/actions/runs/{workflow_id}/artifacts", + "--paginate", + "--jq", + ".artifacts[] | [.name, .size_in_bytes] | @tsv", + ], + text=True, + ) + artifacts: list[WorkflowArtifact] = [] + for line in stdout.splitlines(): + name, size_in_bytes = line.split("\t", 1) + artifacts.append(WorkflowArtifact(name=name, size_in_bytes=int(size_in_bytes))) + return artifacts + + +def download_artifacts( + workflow_id: str, + dest_dir: Path, + artifacts: Sequence[WorkflowArtifact], +) -> None: + total_bytes = sum(artifact.size_in_bytes for artifact in artifacts) + print( + f"Downloading {len(artifacts)} artifacts ({format_bytes(total_bytes)})", + flush=True, + ) + for artifact in artifacts: + artifact_dir = dest_dir / artifact.name + if artifact_dir.is_dir() and any(artifact_dir.iterdir()): + print( + f" using cached {artifact.name} ({format_bytes(artifact.size_in_bytes)})", + flush=True, + ) + continue + + artifact_dir.mkdir(parents=True, exist_ok=True) + print( + f" downloading {artifact.name} ({format_bytes(artifact.size_in_bytes)})", + flush=True, + ) + subprocess.check_call( + [ + "gh", + "run", + "download", + "--name", + artifact.name, + "--dir", + str(artifact_dir), + "--repo", + GITHUB_REPO, + workflow_id, + ] + ) + + +def install_codex_package_archives( + artifacts_dir: Path, + vendor_dir: Path, + targets: Sequence[str], +) -> None: + if not targets: + return + + print( + "Installing Codex package archives for targets: " + ", ".join(targets), + flush=True, + ) + max_workers = min(len(targets), max(1, (os.cpu_count() or 1))) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit( + install_single_codex_package_archive, + artifacts_dir, + vendor_dir, + target, + ): target + for target in targets + } + for future in as_completed(futures): + installed_path = future.result() + print(f" installed {installed_path}", flush=True) + + +def install_single_codex_package_archive( + artifacts_dir: Path, + vendor_dir: Path, + target: str, +) -> Path: + artifact_subdir = artifact_dir_for_target(artifacts_dir, target) + archive_path = artifact_subdir / f"codex-package-{target}.tar.gz" + if not archive_path.exists(): + raise FileNotFoundError(f"Expected package archive not found: {archive_path}") + + dest_dir = vendor_dir / target + if dest_dir.exists(): + shutil.rmtree(dest_dir) + dest_dir.mkdir(parents=True, exist_ok=True) + + with tarfile.open(archive_path, "r:gz") as archive: + archive.extractall(dest_dir, filter="data") + + return dest_dir + + +def install_binary_components( + artifacts_dir: Path, + vendor_dir: Path, + selected_components: Sequence[BinaryComponent], +) -> None: + for component in selected_components: + component_targets = list(BINARY_TARGETS) + + print( + f"Installing {component.binary_basename} binaries for targets: " + + ", ".join(component_targets), + flush=True, + ) + max_workers = min(len(component_targets), max(1, (os.cpu_count() or 1))) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit( + install_single_binary, + artifacts_dir, + vendor_dir, + target, + component, + ): target + for target in component_targets + } + for future in as_completed(futures): + installed_path = future.result() + print(f" installed {installed_path}", flush=True) + + +def install_single_binary( + artifacts_dir: Path, + vendor_dir: Path, + target: str, + component: BinaryComponent, +) -> Path: + artifact_subdir = artifact_dir_for_target(artifacts_dir, target) + archive_path = binary_archive_path( + artifact_subdir, component.artifact_prefix, target + ) + + dest_dir = vendor_dir / target / component.dest_dir + dest_dir.mkdir(parents=True, exist_ok=True) + + binary_name = ( + f"{component.binary_basename}.exe" + if "windows" in target + else component.binary_basename + ) + dest = dest_dir / binary_name + dest.unlink(missing_ok=True) + extract_zstd_archive(archive_path, dest) + if "windows" not in target: + dest.chmod(0o755) + return dest + + +def binary_archive_path(artifact_dir: Path, artifact_prefix: str, target: str) -> Path: + archive_names = [archive_name_for_target(artifact_prefix, target)] + if artifact_dir.name == f"{target}-unsigned": + archive_names.append( + archive_name_for_target(artifact_prefix, f"{target}-unsigned") + ) + + for archive_name in archive_names: + archive_path = artifact_dir / archive_name + if archive_path.exists(): + return archive_path + + raise FileNotFoundError( + f"Expected artifact not found: {artifact_dir / archive_names[0]}" + ) + + +def archive_name_for_target(artifact_prefix: str, target: str) -> str: + if "windows" in target: + return f"{artifact_prefix}-{target}.exe.zst" + return f"{artifact_prefix}-{target}.zst" + + +def artifact_dir_for_target(artifacts_dir: Path, target: str) -> Path: + for artifact_name in [target, f"{target}-unsigned"]: + artifact_dir = artifacts_dir / artifact_name + if artifact_dir.is_dir(): + return artifact_dir + + return artifacts_dir / target + + +def extract_zstd_archive(archive_path: Path, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + + output_path = archive_path.parent / dest.name + subprocess.check_call( + ["zstd", "-f", "-d", str(archive_path), "-o", str(output_path)] + ) + shutil.move(str(output_path), dest) + + +def format_bytes(size_in_bytes: int) -> str: + value = float(size_in_bytes) + for unit in ["B", "KiB", "MiB"]: + if value < 1024: + return f"{value:.1f} {unit}" + value /= 1024 + return f"{value:.1f} GiB" + + +def run_command(cmd: list[str]) -> None: + print("+", " ".join(cmd), flush=True) + subprocess.run(cmd, cwd=REPO_ROOT, check=True) + + +def tarball_name_for_package(package: str, version: str) -> str: + if package in CODEX_PLATFORM_PACKAGES: + platform = package.removeprefix("codex-") + return f"codex-npm-{platform}-{version}.tgz" + return f"{package}-npm-{version}.tgz" + + +def main() -> int: + args = parse_args() + + output_dir = args.output_dir or (REPO_ROOT / "dist" / "npm") + output_dir.mkdir(parents=True, exist_ok=True) + + runner_temp = Path(os.environ.get("RUNNER_TEMP", tempfile.gettempdir())) + + packages = expand_packages(list(args.packages)) + native_component_sets = collect_native_component_sets(packages) + print("Expanded packages: " + ", ".join(packages), flush=True) + if native_component_sets: + component_sets = [ + "(" + ", ".join(components) + ")" for components in native_component_sets + ] + print( + "Native component sets: " + ", ".join(component_sets), + flush=True, + ) + + vendor_temp_roots: list[Path] = [] + vendor_src_by_components: dict[tuple[str, ...], Path] = {} + artifacts_temp_root: Path | None = None + remove_artifacts_temp_root = False + resolved_head_sha: str | None = None + staging_jobs: list[tuple[Path, list[str], str]] = [] + + try: + if native_component_sets: + if args.artifacts_dir is not None: + artifacts_temp_root = args.artifacts_dir.resolve() + artifacts_temp_root.mkdir(parents=True, exist_ok=True) + else: + artifacts_temp_root = Path( + tempfile.mkdtemp(prefix="npm-native-artifacts-", dir=runner_temp) + ) + remove_artifacts_temp_root = True + + target_artifact_dirs = [ + artifact_dir_for_target(artifacts_temp_root, target) + for target in BINARY_TARGETS + ] + if args.artifacts_dir is not None and all( + artifact_dir.is_dir() and any(artifact_dir.iterdir()) + for artifact_dir in target_artifact_dirs + ): + workflow_url = None + else: + workflow_url, resolved_head_sha = resolve_workflow_url( + args.release_version, args.workflow_url + ) + print(f"Using native artifacts from {workflow_url}", flush=True) + + print(f"Using artifact cache at {artifacts_temp_root}", flush=True) + for components in native_component_sets: + vendor_temp_root = Path( + tempfile.mkdtemp(prefix="npm-native-", dir=runner_temp) + ) + vendor_temp_roots.append(vendor_temp_root) + print( + "Installing native components " + + ", ".join(components) + + f" into {vendor_temp_root}", + flush=True, + ) + install_native_components( + workflow_url, + set(components), + vendor_temp_root, + artifacts_temp_root, + ) + vendor_src_by_components[components] = vendor_temp_root / "vendor" + + if resolved_head_sha: + print(f"should `git checkout {resolved_head_sha}`", flush=True) + + for package in packages: + staging_dir = Path( + tempfile.mkdtemp(prefix=f"npm-stage-{package}-", dir=runner_temp) + ) + pack_output = output_dir / tarball_name_for_package( + package, args.release_version + ) + print(f"Staging {package} in {staging_dir}", flush=True) + + cmd = [ + str(BUILD_SCRIPT), + "--package", + package, + "--release-version", + args.release_version, + "--staging-dir", + str(staging_dir), + "--pack-output", + str(pack_output), + ] + + vendor_src = vendor_src_by_components.get( + native_components_for_package(package) + ) + if vendor_src is not None: + cmd.extend(["--vendor-src", str(vendor_src)]) + + staging_jobs.append( + (staging_dir, cmd, f"Staged {package} at {pack_output}") + ) + + max_workers = min(len(staging_jobs), os.cpu_count() or 1) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit(run_command, cmd): staging_dir + for staging_dir, cmd, _message in staging_jobs + } + for future in as_completed(futures): + try: + future.result() + finally: + if not args.keep_staging_dirs: + shutil.rmtree(futures[future], ignore_errors=True) + finally: + if not args.keep_staging_dirs: + for staging_dir, _cmd, _message in staging_jobs: + shutil.rmtree(staging_dir, ignore_errors=True) + for vendor_temp_root in vendor_temp_roots: + shutil.rmtree(vendor_temp_root, ignore_errors=True) + if remove_artifacts_temp_root and artifacts_temp_root is not None: + shutil.rmtree(artifacts_temp_root, ignore_errors=True) + + for _staging_dir, _cmd, message in staging_jobs: + print(message, flush=True) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test-remote-env.sh b/scripts/test-remote-env.sh new file mode 100644 index 0000000000000000000000000000000000000000..7fcdd1ca37b53cb3bf275ed967f569f8ddffb124 --- /dev/null +++ b/scripts/test-remote-env.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash + +# Remote-env setup script for codex-rs integration tests. +# +# Usage (source-only): +# source scripts/test-remote-env.sh +# cd codex-rs +# just test -p codex-core --test all remote_test_env_can_connect_and_use_filesystem +# codex_remote_env_cleanup + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +is_sourced() { + [[ "${BASH_SOURCE[0]}" != "$0" ]] +} + +setup_remote_env() { + local container_name + local codex_binary_path + local container_ip + local remote_codex_path + local remote_exec_server_pid + local remote_exec_server_port + local remote_exec_server_stdout_path + + container_name="${CODEX_TEST_REMOTE_ENV_CONTAINER_NAME:-codex-remote-test-env-local-$(date +%s)-${RANDOM}}" + codex_binary_path="${CARGO_TARGET_DIR:-${REPO_ROOT}/codex-rs/target}/debug/codex" + + if ! command -v docker >/dev/null 2>&1; then + echo "docker is required (Colima or Docker Desktop)" >&2 + return 1 + fi + + if ! docker info >/dev/null 2>&1; then + echo "docker daemon is not reachable; for Colima run: colima start" >&2 + return 1 + fi + + if ! command -v cargo >/dev/null 2>&1; then + echo "cargo is required to build codex" >&2 + return 1 + fi + + ( + cd "${REPO_ROOT}/codex-rs" + cargo build -p codex-cli --bin codex + ) + + if [[ ! -f "${codex_binary_path}" ]]; then + echo "codex binary not found at ${codex_binary_path}" >&2 + return 1 + fi + + docker rm -f "${container_name}" >/dev/null 2>&1 || true + # bubblewrap needs mount propagation inside the remote test container. + docker run -d \ + --name "${container_name}" \ + --privileged \ + --security-opt seccomp=unconfined \ + ubuntu:24.04 sleep infinity >/dev/null + if ! docker exec "${container_name}" sh -lc "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python3 zsh bubblewrap"; then + docker rm -f "${container_name}" >/dev/null 2>&1 || true + return 1 + fi + + if [[ -z "${CODEX_TEST_REMOTE_EXEC_SERVER_URL:-}" ]]; then + remote_codex_path="/tmp/codex-remote-env/codex" + remote_exec_server_port="31987" + remote_exec_server_stdout_path="/tmp/codex-remote-env/exec-server.stdout" + docker exec "${container_name}" sh -lc "mkdir -p /tmp/codex-remote-env" + docker cp "${codex_binary_path}" "${container_name}:${remote_codex_path}" + docker exec "${container_name}" chmod +x "${remote_codex_path}" + remote_exec_server_pid="$( + docker exec "${container_name}" sh -lc \ + "rm -f ${remote_exec_server_stdout_path}; nohup ${remote_codex_path} exec-server --listen ws://0.0.0.0:${remote_exec_server_port} > ${remote_exec_server_stdout_path} 2>&1 & echo \$!" + )" + wait_for_remote_exec_server_port "${container_name}" "${remote_exec_server_port}" "${remote_exec_server_stdout_path}" + container_ip="$( + docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${container_name}" + )" + if [[ -z "${container_ip}" ]]; then + echo "container ${container_name} has no IP address" >&2 + docker rm -f "${container_name}" >/dev/null 2>&1 || true + return 1 + fi + export CODEX_TEST_REMOTE_EXEC_SERVER_PID="${remote_exec_server_pid}" + export CODEX_TEST_REMOTE_EXEC_SERVER_URL="ws://${container_ip}:${remote_exec_server_port}" + fi + + export CODEX_TEST_REMOTE_ENV="${container_name}" + export CODEX_TEST_REMOTE_ENV_CONTAINER_NAME="${container_name}" + export CODEX_TEST_ENVIRONMENT="docker" +} + +wait_for_remote_exec_server_port() { + local container_name="$1" + local port="$2" + local stdout_path="$3" + local deadline=$((SECONDS + 5)) + + while (( SECONDS < deadline )); do + if docker exec "${container_name}" python3 -c "import socket; socket.create_connection(('127.0.0.1', ${port}), timeout=0.2).close()" >/dev/null 2>&1; then + return 0 + fi + sleep 0.025 + done + + echo "timed out waiting for remote exec-server on ${container_name}:${port}" >&2 + docker exec "${container_name}" sh -lc "cat ${stdout_path} 2>/dev/null || true" >&2 || true + return 1 +} + +codex_remote_env_cleanup() { + if [[ -n "${CODEX_TEST_REMOTE_ENV:-}" ]]; then + docker rm -f "${CODEX_TEST_REMOTE_ENV}" >/dev/null 2>&1 || true + unset CODEX_TEST_REMOTE_ENV + fi + unset CODEX_TEST_REMOTE_ENV_CONTAINER_NAME + unset CODEX_TEST_REMOTE_EXEC_SERVER_PID + unset CODEX_TEST_REMOTE_EXEC_SERVER_URL + unset CODEX_TEST_ENVIRONMENT +} + +if ! is_sourced; then + echo "source this script instead of executing it: source scripts/test-remote-env.sh" >&2 + exit 1 +fi + +old_shell_options="$(set +o)" +set -euo pipefail +if setup_remote_env; then + status=0 + echo "CODEX_TEST_REMOTE_ENV=${CODEX_TEST_REMOTE_ENV}" + echo "CODEX_TEST_ENVIRONMENT=${CODEX_TEST_ENVIRONMENT}" + echo "CODEX_TEST_REMOTE_EXEC_SERVER_URL=${CODEX_TEST_REMOTE_EXEC_SERVER_URL}" + echo "Remote env ready. Run your command, then call: codex_remote_env_cleanup" +else + status=$? +fi +eval "${old_shell_options}" +return "${status}" diff --git a/scripts/uv.lock b/scripts/uv.lock new file mode 100644 index 0000000000000000000000000000000000000000..6c0c3bd0810badf5f901317b61d642d1dc84e026 --- /dev/null +++ b/scripts/uv.lock @@ -0,0 +1,43 @@ +version = 1 +revision = 3 +requires-python = ">=3.10" + +[options] +exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer-span = "P7D" + +[[package]] +name = "codex-scripts" +version = "0.0.0" +source = { virtual = "." } +dependencies = [ + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [{ name = "ruff", specifier = ">=0.15.8" }] + +[[package]] +name = "ruff" +version = "0.15.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/21/a7d5c126d5b557715ef81098f3db2fe20f622a039ff2e626af28d674ab80/ruff-0.15.13.tar.gz", hash = "sha256:f9d89f17f7ba7fb2ed42921f0df75da797a9a5d71bc39049e2c687cf2baf44b7", size = 4678180, upload-time = "2026-05-14T13:44:37.869Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/61/11d458dc6ac22504fd8e237b29dfd40504c7fbbcc8930402cfe51a8e63ed/ruff-0.15.13-py3-none-linux_armv6l.whl", hash = "sha256:444b580fc72fd6887e650acd3e575e18cdc79dbcf42fb4030b491057921f61f8", size = 10738279, upload-time = "2026-05-14T13:44:18.7Z" }, + { url = "https://files.pythonhosted.org/packages/86/ca/caa871ee7be718c45256fada4e16a218ee3e33f0c4a46b729a60a24912e6/ruff-0.15.13-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6590d009e7cb7ebf36f83dbdd44a3fa48a0994ff6f1cdc1b08006abe58f98dc7", size = 11124798, upload-time = "2026-05-14T13:44:06.427Z" }, + { url = "https://files.pythonhosted.org/packages/d3/19/43f5f2e568dddde567fc41f8471f9432c09563e19d3e617a48cfa52f8f0a/ruff-0.15.13-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1c26d2f66163deeb6e08d8b39fbbe983ce3c71cea06a6d7591cfd1421793c629", size = 10460761, upload-time = "2026-05-14T13:44:04.375Z" }, + { url = "https://files.pythonhosted.org/packages/99/df/cf938cd6de3003178f03ad7c1ea2a6c099468c03a35037985070b37e76be/ruff-0.15.13-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dbd6f94b434f896308e4d57fb7bfde0d02b99f7a64b3bdab0fdfa6a864203a5", size = 10804451, upload-time = "2026-05-14T13:44:25.221Z" }, + { url = "https://files.pythonhosted.org/packages/c7/7d/5d0973129b154ded2225729169d7068f26b467760b146493fde138415f23/ruff-0.15.13-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf3259f3be4d181bda591da5db2571aed6853c6a048157756448020bc6c5cd22", size = 10534285, upload-time = "2026-05-14T13:44:08.888Z" }, + { url = "https://files.pythonhosted.org/packages/1f/e3/6b999bbc66cd51e5f073842bc2a3995e99c5e0e72e16b15e7261f7abf57a/ruff-0.15.13-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae9c17e5eb4430c154e76abc25d79a318190f5a997f38fb6b114416c5319ffc9", size = 11312063, upload-time = "2026-05-14T13:44:11.274Z" }, + { url = "https://files.pythonhosted.org/packages/af/5a/642639e9f5db04f1e97fbd6e091c6fd20725bdf072fb114d00eefb9e6eb8/ruff-0.15.13-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e2e39bff6c341f4b577a21b801326fab0b11847f48fcaa83f00a113c9b3cb55", size = 12183079, upload-time = "2026-05-14T13:44:01.634Z" }, + { url = "https://files.pythonhosted.org/packages/19/4c/7585735f6b53b0f12de13618b2f7d250a844f018822efc899df2e7b8295f/ruff-0.15.13-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e8d9a8e08013542e94d3220bc5b62cc3e5ef87c5f74bff367d3fac14fab013e6", size = 11440833, upload-time = "2026-05-14T13:43:59.043Z" }, + { url = "https://files.pythonhosted.org/packages/e8/31/bf1a0803d077e679cfeee5f2f67290a0fa79c7385b5d9a8c17b9db2c48f0/ruff-0.15.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc411dfebe5eebe55ce041c6ae080eb7668955e866daa2fbb16692a784f1c4ca", size = 11434486, upload-time = "2026-05-14T13:44:27.761Z" }, + { url = "https://files.pythonhosted.org/packages/e1/4e/62c9b999875d4f14db80f277c030578f5e249c9852d65b7ac7ad0b43c041/ruff-0.15.13-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:768494eb08b9cee54e2fd27969966f74db5a57f6eaa7a90fcb3306af34dfc4bd", size = 11385189, upload-time = "2026-05-14T13:44:13.704Z" }, + { url = "https://files.pythonhosted.org/packages/fc/89/7e959047a104df3eb12863447c110140191fc5b6c4f379ea2e803fcdb0e4/ruff-0.15.13-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:fb75f9a3a7e42ffe117d734494e6c5e5cb3565d66e12612cb63d0e572a41a5b6", size = 10781380, upload-time = "2026-05-14T13:43:56.734Z" }, + { url = "https://files.pythonhosted.org/packages/ff/52/5fd18f3b88cab63e88aa11516b3b4e1e5f720e5c330f8dbe5c26210f41f8/ruff-0.15.13-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8cb74dd33bb2f6613faf7fc03b660053b5ac4f80e706d5788c6335e2a8048d51", size = 10540605, upload-time = "2026-05-14T13:44:20.748Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e0/9e35f338990d3e41a82875ff7053ffe97541dae81c9d02143177f381d572/ruff-0.15.13-py3-none-musllinux_1_2_i686.whl", hash = "sha256:7ef823f817fcd191dc934e984be9cf4094f808effa16f2542ad8e821ba02bbf2", size = 11036554, upload-time = "2026-05-14T13:44:16.256Z" }, + { url = "https://files.pythonhosted.org/packages/c2/13/070fb048c24080fba188f66371e2a92785be257ad02242066dc7255ac6e9/ruff-0.15.13-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:f345a13937bd7f09f6f5d19fa0721b0c103e00e7f62bc67089a8e5e037719e0b", size = 11528133, upload-time = "2026-05-14T13:44:22.808Z" }, + { url = "https://files.pythonhosted.org/packages/6b/8c/b1e1666aef7fc6555094d73ae6cd981701781ae85b97ceefc0eebd0b4668/ruff-0.15.13-py3-none-win32.whl", hash = "sha256:4044f94208b3b05ba0fc4a4abd0558cf4d6459bd18325eead7fd8cc66f909b41", size = 10721455, upload-time = "2026-05-14T13:44:35.697Z" }, + { url = "https://files.pythonhosted.org/packages/ab/a6/870a3e8a50590bb92be184ad928c2922f088b00d9dc5c5ec7b924ee08c22/ruff-0.15.13-py3-none-win_amd64.whl", hash = "sha256:7064884d442b7d477b4e7473d12da7f08851d2b1982763c5d3f388a19468a1a4", size = 11900409, upload-time = "2026-05-14T13:44:30.389Z" }, + { url = "https://files.pythonhosted.org/packages/9b/36/9c015cd052fca743dae8cb2aeb16b551444787467db42ceab0fc968865af/ruff-0.15.13-py3-none-win_arm64.whl", hash = "sha256:2471da9bd1068c8c064b5fd9c0c4b6dddffd6369cb1cd68b29993b1709ff1b21", size = 11179336, upload-time = "2026-05-14T13:44:33.026Z" }, +] diff --git a/scripts/workspace-status.sh b/scripts/workspace-status.sh new file mode 100644 index 0000000000000000000000000000000000000000..ba4cd3869e1b1b3690bb90d258d575a174bffe22 --- /dev/null +++ b/scripts/workspace-status.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ -n "${STABLE_GIT_COMMIT:-}" ]]; then + build_commit="${STABLE_GIT_COMMIT}" +elif build_commit="$(git rev-parse --verify HEAD 2>/dev/null)"; then + : +elif [[ -n "${GITHUB_SHA:-}" ]]; then + build_commit="${GITHUB_SHA}" +else + build_commit="unknown" +fi + +printf 'STABLE_GIT_COMMIT %s\n' "${build_commit}"