src/tools/build_wheel.sh · CIIL/Anuvani-Diarization at main

Anuvani-Diarization / src /tools /build_wheel.sh

Anuvani-Diarization 1.0.0

63dd946 16 days ago

8.76 kB

#!/usr/bin/env bash
# Copyright (c) 2026 The Anuvani-Diarization authors
# SPDX-License-Identifier: MIT OR Apache-2.0
# Build the self-contained wheel: CPython >= 3.10 (abi3), manylinux_2_28 x86_64, inside a pinned container.
#
#   tools/build_wheel.sh        ->  artifacts/wheels/anuvani_diar-<version>-cp310-abi3-manylinux_2_28_x86_64.whl
#                                   artifacts/wheels/<wheel>.build.json   (what was built, from what, and the checks)
#
# Pinned: the build image by digest, rustup-init 1.29.1 by sha256, rustc 1.94.1 (minimal profile), maturin 1.15.0,
# the crates by Cargo.lock (--locked), file times by SOURCE_DATE_EPOCH (the commit's time).
#
# 1. tools/stage_wheel_files.py stages the bundled libraries, their licence texts and the assets (they come from the
#    reference environment) and verifies them.
# 2. The build context is the files git tracks for the build (working-tree content) plus the staged files, copied
#    to artifacts/wheel_build/anuvani_diar_rs, so nothing else in this tree (target/, .venv/, a stray _core.so) can
#    enter the wheel. The build starts from an empty target directory.
# 3. In the container: maturin build --release --locked --strip --compatibility manylinux_2_28, with every source
#    path remapped (no host or build path in _core), and --auditwheel check (verifies the tag, changes nothing).
# 4. Checks, also in the container: `auditwheel show` (never `repair`: it would rewrite the bundled libraries and
#    break their sha256); the bundled files' sha256 in the wheel; `strings` on _core for host or build
#    paths; the glibc symbol versions _core needs (<= 2.28) and its NEEDED libraries.
#
# The image is pulled once (docker pull); rustup, the toolchain, the crates and maturin are cached in
# artifacts/wheel_build/cache (git-ignored) after the first build.
set -euo pipefail

PROJ=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
IMAGE="quay.io/pypa/manylinux_2_28_x86_64@sha256:39df0042d5cc900b085aa25a0659368b42a0006c54c474299b785b44c1b4ff82"
RUSTUP_VERSION=1.29.1
RUSTUP_SHA256=dda7234360b7f578ca8b0ddcb80145646fa61a67c1720a5abc7051b35c9fcb71
RUST_VERSION=1.94.1
MATURIN_VERSION=1.15.0
WORK="$PROJ/artifacts/wheel_build"
OUT="$PROJ/artifacts/wheels"

cd "$PROJ"
python3 tools/stage_wheel_files.py "$@"

COMMIT=$(git rev-parse HEAD)
DIRTY=$(git status --porcelain -- . | grep -q . && echo true || echo false)
SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)

rm -rf "$WORK/anuvani_diar_rs" "$WORK/target" "$WORK/dist"
mkdir -p "$WORK/anuvani_diar_rs" "$WORK/cache" "$OUT"
git ls-files -z -- Cargo.toml Cargo.lock pyproject.toml crates python tools/wheel_files.json assets/MANIFEST.json \
  LICENSE-MIT LICENSE-APACHE THIRD_PARTY_NOTICES.md LICENSES \
  | tar --null -T - -cf - | tar -xf - -C "$WORK/anuvani_diar_rs"
cp -a python/anuvani_diar/_libs python/anuvani_diar/assets "$WORK/anuvani_diar_rs/python/anuvani_diar/"

START=$(date +%s)
docker run --rm -i --user "$(id -u):$(id -g)" -v "$WORK:/build" -v "$WORK/cache:/cache" -v "$OUT:/out" \
  -e IMAGE="$IMAGE" -e RUSTUP_VERSION="$RUSTUP_VERSION" -e RUSTUP_SHA256="$RUSTUP_SHA256" -e RUST_VERSION="$RUST_VERSION" \
  -e MATURIN_VERSION="$MATURIN_VERSION" -e SOURCE_DATE_EPOCH="$SOURCE_DATE_EPOCH" -e COMMIT="$COMMIT" -e DIRTY="$DIRTY" \
  "$IMAGE" bash -s <<'IN_CONTAINER'
set -euo pipefail
export HOME=/tmp/home RUSTUP_HOME=/cache/rustup CARGO_HOME=/cache/cargo
export PATH=/cache/cargo/bin:$PATH
mkdir -p "$HOME"
if [[ ! -x /cache/cargo/bin/rustup ]]; then
  curl -sSf -o /tmp/rustup-init "https://static.rust-lang.org/rustup/archive/$RUSTUP_VERSION/x86_64-unknown-linux-gnu/rustup-init"
  echo "$RUSTUP_SHA256  /tmp/rustup-init" | sha256sum -c --quiet -
  chmod +x /tmp/rustup-init
  /tmp/rustup-init -y -q --no-modify-path --profile minimal --default-toolchain none
fi
rustup toolchain install "$RUST_VERSION" --profile minimal >/dev/null 2>&1 || rustup toolchain install "$RUST_VERSION" --profile minimal
export RUSTUP_TOOLCHAIN="$RUST_VERSION"
M=/cache/maturin-$MATURIN_VERSION
if [[ ! -x $M/bin/maturin ]]; then
  /opt/python/cp312-cp312/bin/python -m venv "$M"
  "$M/bin/pip" install -q "maturin==$MATURIN_VERSION"
fi
cd /build/anuvani_diar_rs
# no path of the build machine in the binary: sources, the crate registry, the toolchain
export RUSTFLAGS="--remap-path-prefix=/build/anuvani_diar_rs=/anuvani_diar_rs --remap-path-prefix=/cache/cargo=/cargo --remap-path-prefix=/cache/rustup=/rustup"
export CARGO_TARGET_DIR=/build/target
if ! "$M/bin/maturin" build --release --locked --strip --compatibility manylinux_2_28 --auditwheel check \
     -i /opt/python/cp310-cp310/bin/python3.10 --out /build/dist > /build/maturin.log 2>&1; then
  tail -40 /build/maturin.log
  exit 1
fi
grep -v "^\s*\(Compiling\|Downloaded\|Downloading\) " /build/maturin.log | tail -8
WHEEL=$(ls /build/dist/*.whl)
cp "$WHEEL" /out/
W=/out/$(basename "$WHEEL")

# ---- checks
auditwheel show "$W" > /build/auditwheel_show.txt 2>&1 || true
/opt/python/cp312-cp312/bin/python - "$W" <<'PY'
import hashlib, json, os, re, subprocess, sys, zipfile
from pathlib import Path
wheel = Path(sys.argv[1])
z = zipfile.ZipFile(wheel)
names = z.namelist()
spec = json.loads(Path("tools/wheel_files.json").read_text())
arrays = json.loads(Path("assets/MANIFEST.json").read_text())["arrays"]
bad = []
for name, e in spec["libs"].items():
    member = f"anuvani_diar/_libs/{name}"
    if member not in names:
        bad.append(f"missing {member}")
    elif e["sha256"] and hashlib.sha256(z.read(member)).hexdigest() != e["sha256"]:
        bad.append(f"{member}: sha256 differs from tools/wheel_files.json")
for name in spec["assets"]:
    member = f"anuvani_diar/assets/{name}"
    if member not in names:
        bad.append(f"missing {member}")
    elif name.endswith(".bin") and hashlib.sha256(z.read(member)).hexdigest() != arrays[name[:-4]]["sha256"]:
        bad.append(f"{member}: sha256 differs from assets/MANIFEST.json")
core = "anuvani_diar/_core.abi3.so"
Path("/build/_core.abi3.so").write_bytes(z.read(core))
strings = subprocess.run(["strings", "-a", "/build/_core.abi3.so"], capture_output=True, text=True, check=True).stdout
# a host or build path: one of these prefixes where a path starts (std's own "/rustc/<hash>/library/std/
# src/io/..." is not a hit), or a tool directory anywhere
leaks = sorted({m.group(0) for m in re.finditer(
    r"((?<![\w./-])(/home/|/root/|/mnt/|/tmp/|/build/|/cache/|/opt/|/io/|/usr/local/)|anaconda|\.cargo/|\.rustup/)[^\s\"']*", strings)})
prefixes = {}
for m in re.finditer(r"(?<![\w./-])/[\w.-]+/[\w.-]+", strings):
    prefixes[m.group(0)] = prefixes.get(m.group(0), 0) + 1
if leaks:
    bad.append(f"_core.abi3.so names build or host paths: {leaks[:10]}")
glibc = sorted(set(re.findall(r"GLIBC_([0-9.]+)", subprocess.run(["objdump", "-T", "/build/_core.abi3.so"], capture_output=True,
               text=True, check=True).stdout)), key=lambda v: tuple(map(int, v.split("."))))
if glibc and tuple(map(int, glibc[-1].split("."))) > (2, 28):
    bad.append(f"_core.abi3.so needs GLIBC_{glibc[-1]} (> 2.28)")
needed = re.findall(r"\(NEEDED\).*\[(.*)\]", subprocess.run(["readelf", "-d", "/build/_core.abi3.so"], capture_output=True,
                    text=True, check=True).stdout)
rustc = subprocess.run(["rustc", "-vV"], capture_output=True, text=True, check=True).stdout
maturin = subprocess.run([f"/cache/maturin-{os.environ['MATURIN_VERSION']}/bin/maturin", "--version"], capture_output=True,
                         text=True, check=True).stdout.strip()
record = dict(
    wheel=wheel.name, sha256=hashlib.sha256(wheel.read_bytes()).hexdigest(), bytes=wheel.stat().st_size,
    commit=os.environ["COMMIT"], dirty=os.environ["DIRTY"] == "true", source_date_epoch=int(os.environ["SOURCE_DATE_EPOCH"]),
    image=os.environ["IMAGE"], rustc=rustc.splitlines()[0], rustc_host=re.search(r"host: (.*)", rustc).group(1),
    maturin=maturin, core_bytes=len(z.read(core)), core_glibc_max=glibc[-1] if glibc else None, core_needed=needed,
    files=len(names), path_strings_in_core=leaks, path_prefixes_in_core=dict(sorted(prefixes.items())), auditwheel_show=Path("/build/auditwheel_show.txt").read_text(),
    problems=bad)
Path(f"/out/{wheel.name}.build.json").write_text(json.dumps(record, indent=1) + "\n")
print(json.dumps({k: v for k, v in record.items() if k != "auditwheel_show"}, indent=1))
print(record["auditwheel_show"])
sys.exit(1 if bad else 0)
PY
IN_CONTAINER

# the container must have written this run's record (a container that ran nothing would also exit 0)
RECORD=$(ls -t "$OUT"/*.build.json 2>/dev/null | head -1)
if [[ -z $RECORD || $(stat -c %Y "$RECORD") -lt $START ]]; then
  echo "build_wheel.sh: no wheel was built" >&2
  exit 1
fi
echo "built ${RECORD%.build.json} (record: $RECORD)"

	#!/usr/bin/env bash
	# Copyright (c) 2026 The Anuvani-Diarization authors
	# SPDX-License-Identifier: MIT OR Apache-2.0
	# Build the self-contained wheel: CPython >= 3.10 (abi3), manylinux_2_28 x86_64, inside a pinned container.
	#
	# tools/build_wheel.sh -> artifacts/wheels/anuvani_diar-<version>-cp310-abi3-manylinux_2_28_x86_64.whl
	# artifacts/wheels/<wheel>.build.json (what was built, from what, and the checks)
	#
	# Pinned: the build image by digest, rustup-init 1.29.1 by sha256, rustc 1.94.1 (minimal profile), maturin 1.15.0,
	# the crates by Cargo.lock (--locked), file times by SOURCE_DATE_EPOCH (the commit's time).
	#
	# 1. tools/stage_wheel_files.py stages the bundled libraries, their licence texts and the assets (they come from the
	# reference environment) and verifies them.
	# 2. The build context is the files git tracks for the build (working-tree content) plus the staged files, copied
	# to artifacts/wheel_build/anuvani_diar_rs, so nothing else in this tree (target/, .venv/, a stray _core.so) can
	# enter the wheel. The build starts from an empty target directory.
	# 3. In the container: maturin build --release --locked --strip --compatibility manylinux_2_28, with every source
	# path remapped (no host or build path in _core), and --auditwheel check (verifies the tag, changes nothing).
	# 4. Checks, also in the container: `auditwheel show` (never `repair`: it would rewrite the bundled libraries and
	# break their sha256); the bundled files' sha256 in the wheel; `strings` on _core for host or build
	# paths; the glibc symbol versions _core needs (<= 2.28) and its NEEDED libraries.
	#
	# The image is pulled once (docker pull); rustup, the toolchain, the crates and maturin are cached in
	# artifacts/wheel_build/cache (git-ignored) after the first build.
	set -euo pipefail

	PROJ=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
	IMAGE="quay.io/pypa/manylinux_2_28_x86_64@sha256:39df0042d5cc900b085aa25a0659368b42a0006c54c474299b785b44c1b4ff82"
	RUSTUP_VERSION=1.29.1
	RUSTUP_SHA256=dda7234360b7f578ca8b0ddcb80145646fa61a67c1720a5abc7051b35c9fcb71
	RUST_VERSION=1.94.1
	MATURIN_VERSION=1.15.0
	WORK="$PROJ/artifacts/wheel_build"
	OUT="$PROJ/artifacts/wheels"

	cd "$PROJ"
	python3 tools/stage_wheel_files.py "$@"

	COMMIT=$(git rev-parse HEAD)
	DIRTY=$(git status --porcelain -- . \| grep -q . && echo true \|\| echo false)
	SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)

	rm -rf "$WORK/anuvani_diar_rs" "$WORK/target" "$WORK/dist"
	mkdir -p "$WORK/anuvani_diar_rs" "$WORK/cache" "$OUT"
	git ls-files -z -- Cargo.toml Cargo.lock pyproject.toml crates python tools/wheel_files.json assets/MANIFEST.json \
	LICENSE-MIT LICENSE-APACHE THIRD_PARTY_NOTICES.md LICENSES \
	\| tar --null -T - -cf - \| tar -xf - -C "$WORK/anuvani_diar_rs"
	cp -a python/anuvani_diar/_libs python/anuvani_diar/assets "$WORK/anuvani_diar_rs/python/anuvani_diar/"

	START=$(date +%s)
	docker run --rm -i --user "$(id -u):$(id -g)" -v "$WORK:/build" -v "$WORK/cache:/cache" -v "$OUT:/out" \
	-e IMAGE="$IMAGE" -e RUSTUP_VERSION="$RUSTUP_VERSION" -e RUSTUP_SHA256="$RUSTUP_SHA256" -e RUST_VERSION="$RUST_VERSION" \
	-e MATURIN_VERSION="$MATURIN_VERSION" -e SOURCE_DATE_EPOCH="$SOURCE_DATE_EPOCH" -e COMMIT="$COMMIT" -e DIRTY="$DIRTY" \
	"$IMAGE" bash -s <<'IN_CONTAINER'
	set -euo pipefail
	export HOME=/tmp/home RUSTUP_HOME=/cache/rustup CARGO_HOME=/cache/cargo
	export PATH=/cache/cargo/bin:$PATH
	mkdir -p "$HOME"
	if [[ ! -x /cache/cargo/bin/rustup ]]; then
	curl -sSf -o /tmp/rustup-init "https://static.rust-lang.org/rustup/archive/$RUSTUP_VERSION/x86_64-unknown-linux-gnu/rustup-init"
	echo "$RUSTUP_SHA256 /tmp/rustup-init" \| sha256sum -c --quiet -
	chmod +x /tmp/rustup-init
	/tmp/rustup-init -y -q --no-modify-path --profile minimal --default-toolchain none
	fi
	rustup toolchain install "$RUST_VERSION" --profile minimal >/dev/null 2>&1 \|\| rustup toolchain install "$RUST_VERSION" --profile minimal
	export RUSTUP_TOOLCHAIN="$RUST_VERSION"
	M=/cache/maturin-$MATURIN_VERSION
	if [[ ! -x $M/bin/maturin ]]; then
	/opt/python/cp312-cp312/bin/python -m venv "$M"
	"$M/bin/pip" install -q "maturin==$MATURIN_VERSION"
	fi
	cd /build/anuvani_diar_rs
	# no path of the build machine in the binary: sources, the crate registry, the toolchain
	export RUSTFLAGS="--remap-path-prefix=/build/anuvani_diar_rs=/anuvani_diar_rs --remap-path-prefix=/cache/cargo=/cargo --remap-path-prefix=/cache/rustup=/rustup"
	export CARGO_TARGET_DIR=/build/target
	if ! "$M/bin/maturin" build --release --locked --strip --compatibility manylinux_2_28 --auditwheel check \
	-i /opt/python/cp310-cp310/bin/python3.10 --out /build/dist > /build/maturin.log 2>&1; then
	tail -40 /build/maturin.log
	exit 1
	fi
	grep -v "^\s*\(Compiling\\|Downloaded\\|Downloading\) " /build/maturin.log \| tail -8
	WHEEL=$(ls /build/dist/*.whl)
	cp "$WHEEL" /out/
	W=/out/$(basename "$WHEEL")

	# ---- checks
	auditwheel show "$W" > /build/auditwheel_show.txt 2>&1 \|\| true
	/opt/python/cp312-cp312/bin/python - "$W" <<'PY'
	import hashlib, json, os, re, subprocess, sys, zipfile
	from pathlib import Path
	wheel = Path(sys.argv[1])
	z = zipfile.ZipFile(wheel)
	names = z.namelist()
	spec = json.loads(Path("tools/wheel_files.json").read_text())
	arrays = json.loads(Path("assets/MANIFEST.json").read_text())["arrays"]
	bad = []
	for name, e in spec["libs"].items():
	member = f"anuvani_diar/_libs/{name}"
	if member not in names:
	bad.append(f"missing {member}")
	elif e["sha256"] and hashlib.sha256(z.read(member)).hexdigest() != e["sha256"]:
	bad.append(f"{member}: sha256 differs from tools/wheel_files.json")
	for name in spec["assets"]:
	member = f"anuvani_diar/assets/{name}"
	if member not in names:
	bad.append(f"missing {member}")
	elif name.endswith(".bin") and hashlib.sha256(z.read(member)).hexdigest() != arrays[name[:-4]]["sha256"]:
	bad.append(f"{member}: sha256 differs from assets/MANIFEST.json")
	core = "anuvani_diar/_core.abi3.so"
	Path("/build/_core.abi3.so").write_bytes(z.read(core))
	strings = subprocess.run(["strings", "-a", "/build/_core.abi3.so"], capture_output=True, text=True, check=True).stdout
	# a host or build path: one of these prefixes where a path starts (std's own "/rustc/<hash>/library/std/
	# src/io/..." is not a hit), or a tool directory anywhere
	leaks = sorted({m.group(0) for m in re.finditer(
	r"((?<![\w./-])(/home/\|/root/\|/mnt/\|/tmp/\|/build/\|/cache/\|/opt/\|/io/\|/usr/local/)\|anaconda\|\.cargo/\|\.rustup/)[^\s\"']*", strings)})
	prefixes = {}
	for m in re.finditer(r"(?<![\w./-])/[\w.-]+/[\w.-]+", strings):
	prefixes[m.group(0)] = prefixes.get(m.group(0), 0) + 1
	if leaks:
	bad.append(f"_core.abi3.so names build or host paths: {leaks[:10]}")
	glibc = sorted(set(re.findall(r"GLIBC_([0-9.]+)", subprocess.run(["objdump", "-T", "/build/_core.abi3.so"], capture_output=True,
	text=True, check=True).stdout)), key=lambda v: tuple(map(int, v.split("."))))
	if glibc and tuple(map(int, glibc[-1].split("."))) > (2, 28):
	bad.append(f"_core.abi3.so needs GLIBC_{glibc[-1]} (> 2.28)")
	needed = re.findall(r"\(NEEDED\).\[(.)\]", subprocess.run(["readelf", "-d", "/build/_core.abi3.so"], capture_output=True,
	text=True, check=True).stdout)
	rustc = subprocess.run(["rustc", "-vV"], capture_output=True, text=True, check=True).stdout
	maturin = subprocess.run([f"/cache/maturin-{os.environ['MATURIN_VERSION']}/bin/maturin", "--version"], capture_output=True,
	text=True, check=True).stdout.strip()
	record = dict(
	wheel=wheel.name, sha256=hashlib.sha256(wheel.read_bytes()).hexdigest(), bytes=wheel.stat().st_size,
	commit=os.environ["COMMIT"], dirty=os.environ["DIRTY"] == "true", source_date_epoch=int(os.environ["SOURCE_DATE_EPOCH"]),
	image=os.environ["IMAGE"], rustc=rustc.splitlines()[0], rustc_host=re.search(r"host: (.*)", rustc).group(1),
	maturin=maturin, core_bytes=len(z.read(core)), core_glibc_max=glibc[-1] if glibc else None, core_needed=needed,
	files=len(names), path_strings_in_core=leaks, path_prefixes_in_core=dict(sorted(prefixes.items())), auditwheel_show=Path("/build/auditwheel_show.txt").read_text(),
	problems=bad)
	Path(f"/out/{wheel.name}.build.json").write_text(json.dumps(record, indent=1) + "\n")
	print(json.dumps({k: v for k, v in record.items() if k != "auditwheel_show"}, indent=1))
	print(record["auditwheel_show"])
	sys.exit(1 if bad else 0)
	PY
	IN_CONTAINER

	# the container must have written this run's record (a container that ran nothing would also exit 0)
	RECORD=$(ls -t "$OUT"/*.build.json 2>/dev/null \| head -1)
	if [[ -z $RECORD \|\| $(stat -c %Y "$RECORD") -lt $START ]]; then
	echo "build_wheel.sh: no wheel was built" >&2
	exit 1
	fi
	echo "built ${RECORD%.build.json} (record: $RECORD)"