computaco-tribe / encode.py
jacob-valdez's picture
ComputaCo's inference handler for TRIBE v2, with attribution
6df569c verified
Raw History Blame Contribute Delete
12 kB
"""Predict cortical responses to a stimulus with TRIBE v2 (facebook/tribev2, CC BY-NC 4.0).
A perception runs `encode_stimulus` on the model's inference endpoint (scripts/brain/tribe/handler.py,
docs/harnesses.md § Perception). The command line is the same computation, for studies run offline:
~/tribe-env/bin/python scripts/brain/encode.py --text page.txt --out run/
~/tribe-env/bin/python scripts/brain/encode.py --video clip.mp4 --out run/
~/tribe-env/bin/python scripts/brain/encode.py --screen frame.png --seconds 20 --out run/
A text is spoken (text-to-speech) and transcribed back to timed words, as TRIBE's own
demo does; a screen is a still held for `--seconds` as a silent video; a video or audio
file is taken as it is. The model answers one prediction a second (its TR) on the
fsaverage5 cortical mesh (20,484 vertices, left then right), for the average subject, in
z-units of BOLD, shifted 5 s into the past for the haemodynamic lag.
Writes into --out:
preds.npz preds (T, 20484) float16 and times (T,) seconds
rois.json every Destrieux parcel's time-mean and peak, per hemisphere, with its name
features.npz the backbone features the model was given (time-mean per modality), for
matched-control analyses: the same stimulus without the brain projection
map.png the time-mean prediction on the inflated surface, four views
summary.json what was run (model, checkpoint digest, stimulus, seconds, segments,
duration), the hemisphere means, and the ten strongest parcels
It fails plainly without the model, the GPU or the stimulus; it never writes a result it
did not compute.
"""
import argparse
import hashlib
import json
import os
import pathlib
import subprocess
import time
MODEL_ID = "facebook/tribev2"
MESH = "fsaverage5"
N_VERTICES = 20484
def sha256(path):
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def ffmpeg():
import imageio_ffmpeg
return imageio_ffmpeg.get_ffmpeg_exe()
def screen_video(png, seconds, out):
"""A still image held for `seconds`, with a silent audio track so the audio extractor has one."""
subprocess.run([ffmpeg(), "-y", "-loglevel", "error", "-loop", "1", "-framerate", "4", "-i", str(png),
"-f", "lavfi", "-i", "anullsrc=r=16000:cl=mono", "-t", str(seconds),
"-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2,format=yuv420p", "-c:v", "libx264", "-preset", "fast", "-crf", "20",
"-c:a", "aac", "-shortest", str(out)], check=True)
return out
def transcribe_on_cpu():
"""TRIBE transcribes with whisperx under `uvx` on CUDA in float16; CTranslate2's aarch64 wheel has
no CUDA, so here whisperx runs on the CPU in int8 (BRAIN_WHISPER_DEVICE=cuda keeps TRIBE's own)."""
if os.environ.get("BRAIN_WHISPER_DEVICE", "cpu") == "cuda":
return
import json as _json
import subprocess as _sp
import tempfile
from pathlib import Path
import pandas as pd
from tribev2 import eventstransforms as et
codes = dict(english="en", french="fr", spanish="es", dutch="nl", chinese="zh")
def get_transcript(wav_filename, language):
if language not in codes:
raise ValueError(f"Language {language} not supported")
with tempfile.TemporaryDirectory() as out:
cmd = ["uvx", "whisperx", str(wav_filename), "--model", "large-v3", "--language", codes[language], "--device", "cpu",
"--compute_type", "int8", "--batch_size", "16", "--output_dir", out, "--output_format", "json"]
if language == "english":
cmd += ["--align_model", "WAV2VEC2_ASR_LARGE_LV60K_960H"]
env = {k: v for k, v in os.environ.items() if k != "MPLBACKEND"}
r = _sp.run(cmd, capture_output=True, text=True, env=env)
if r.returncode != 0:
raise RuntimeError(f"whisperx failed:\n{r.stderr[-2000:]}")
transcript = _json.loads((Path(out) / f"{Path(wav_filename).stem}.json").read_text())
words = []
for i, seg in enumerate(transcript["segments"]):
sentence = seg["text"].replace('"', "")
for w in seg["words"]:
if "start" in w:
words.append({"text": w["word"].replace('"', ""), "start": w["start"], "duration": w["end"] - w["start"], "sequence_id": i, "sentence": sentence})
return pd.DataFrame(words)
et.ExtractWordsFromAudio._get_transcript_from_audio = staticmethod(get_transcript)
def load_model(cache):
from tribev2.demo_utils import TribeModel
transcribe_on_cpu()
return TribeModel.from_pretrained(MODEL_ID, cache_folder=str(cache))
def checkpoint_digest():
from huggingface_hub import hf_hub_download
return sha256(hf_hub_download(MODEL_ID, "best.ckpt"))
class FeatureTap:
"""Catches the per-modality backbone features each batch carries into the model."""
def __init__(self, model):
self.sums, self.counts = {}, {}
self.handle = model.register_forward_pre_hook(self.hook, with_kwargs=False)
def hook(self, module, args):
import torch
batch = args[0]
for modality in ("text", "audio", "video"):
if modality not in batch.data:
continue
x = batch.data[modality].to(torch.float32) # B, L, D, T (or B, D, T)
if x.ndim == 3:
x = x.unsqueeze(1)
x = x.mean(dim=1) # mean over layers: B, D, T
self.sums[modality] = self.sums.get(modality, 0) + x.sum(dim=(0, 2)).cpu().numpy()
self.counts[modality] = self.counts.get(modality, 0) + x.shape[0] * x.shape[2]
def means(self):
return {m: self.sums[m] / max(self.counts[m], 1) for m in self.sums}
def atlas():
"""Destrieux parcels on fsaverage5: (labels_left, labels_right, names)."""
from nilearn import datasets
a = datasets.fetch_atlas_surf_destrieux()
names = [n.decode() if isinstance(n, bytes) else str(n) for n in a["labels"]]
return a["map_left"], a["map_right"], names
def parcels(mean, peak, labels_l, labels_r, names):
import numpy as np
out = []
half = N_VERTICES // 2
for hemi, labels, lo in (("L", labels_l, 0), ("R", labels_r, half)):
for idx, name in enumerate(names):
if name in ("Unknown", "Medial_wall"):
continue
sel = np.where(labels == idx)[0]
if not len(sel):
continue
out.append({"id": f"{hemi}_{name}", "hemisphere": hemi, "name": name, "vertices": int(len(sel)),
"mean": float(mean[lo + sel].mean()), "peak": float(peak[lo + sel].mean())})
return out
def plot_map(mean, out):
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
from nilearn import datasets, plotting
fs = datasets.fetch_surf_fsaverage(MESH)
half = N_VERTICES // 2
v = float(np.percentile(np.abs(mean), 99)) or 1.0
fig, axes = plt.subplots(1, 4, figsize=(16, 4), subplot_kw={"projection": "3d"})
views = [("left", "lateral", mean[:half]), ("left", "medial", mean[:half]), ("right", "lateral", mean[half:]), ("right", "medial", mean[half:])]
for ax, (hemi, view, data) in zip(axes, views):
plotting.plot_surf_stat_map(fs[f"infl_{hemi}"], data, hemi=hemi, view=view, bg_map=fs[f"sulc_{hemi}"], cmap="RdBu_r",
vmax=v, symmetric_cbar=True, colorbar=(view == "medial" and hemi == "right"), axes=ax, threshold=v * 0.1)
ax.set_title(f"{hemi} {view}", fontsize=10)
fig.savefig(out, dpi=110, bbox_inches="tight")
plt.close(fig)
def encode_stimulus(model, kind, path, out, seconds=20.0, draw_map=True):
"""Predict the response to one stimulus with a loaded model, and write the files above into `out`.
`kind` is text, video, audio or screen; `path` the file. → the summary."""
import numpy as np
path = pathlib.Path(path)
out = pathlib.Path(out)
out.mkdir(parents=True, exist_ok=True)
t0 = time.time()
stimulus = {"kind": kind, "source": str(path), "sha256": sha256(path)}
if kind == "screen":
clip = out / "stimulus.mp4"
screen_video(path, seconds, clip)
stimulus["seconds"] = seconds
events_arg = {"video_path": str(clip)}
elif kind == "video":
events_arg = {"video_path": str(path.resolve())}
elif kind == "audio":
events_arg = {"audio_path": str(path.resolve())}
elif kind == "text":
stimulus["words"] = len(path.read_text(encoding="utf-8").split())
events_arg = {"text_path": str(path.resolve())}
else:
raise ValueError(f"no stimulus of the kind {kind}")
tap = FeatureTap(model._model)
events = model.get_events_dataframe(**events_arg)
preds, segments = model.predict(events, verbose=False)
tap.handle.remove()
if preds.shape[1] != N_VERTICES:
raise RuntimeError(f"expected {N_VERTICES} vertices, got {preds.shape[1]}")
times = np.array([float(s.start) for s in segments])
order = np.argsort(times)
preds, times = preds[order], times[order]
mean = preds.mean(axis=0)
peak = preds.max(axis=0)
np.savez_compressed(out / "preds.npz", preds=preds.astype(np.float16), times=times.astype(np.float32))
features = tap.means()
np.savez_compressed(out / "features.npz", **{k: v.astype(np.float32) for k, v in features.items()})
labels_l, labels_r, names = atlas()
rows = parcels(mean, peak, labels_l, labels_r, names)
(out / "rois.json").write_text(json.dumps({"atlas": "Destrieux 2009 on fsaverage5 (nilearn)", "parcels": rows}, indent=1))
if draw_map:
plot_map(mean, out / "map.png")
half = N_VERTICES // 2
top = sorted(rows, key=lambda r: -r["mean"])[:10]
summary = {
"model": MODEL_ID, "checkpoint_sha256": checkpoint_digest(), "mesh": MESH, "vertices": N_VERTICES,
"stimulus": stimulus, "segments": int(preds.shape[0]), "seconds_predicted": float(times[-1] - times[0] + 1) if len(times) else 0.0,
"modalities": sorted(features), "feature_dims": {k: int(v.shape[0]) for k, v in features.items()},
"hemisphere_mean": {"left": float(mean[:half].mean()), "right": float(mean[half:].mean())},
"global_mean": float(mean.mean()), "global_peak": float(peak.max()),
"top_parcels": [{"id": r["id"], "mean": r["mean"]} for r in top],
"runtime_s": round(time.time() - t0, 1),
"files": ["preds.npz", "rois.json", "features.npz"] + (["map.png"] if draw_map else []),
}
(out / "summary.json").write_text(json.dumps(summary, indent=1))
return summary
def main():
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
src = ap.add_mutually_exclusive_group(required=True)
src.add_argument("--text", help="a .txt file: spoken and transcribed")
src.add_argument("--video", help="an .mp4/.mov/.webm")
src.add_argument("--audio", help="a .wav/.mp3")
src.add_argument("--screen", help="a .png held still for --seconds")
ap.add_argument("--seconds", type=float, default=20.0, help="how long a --screen is shown")
ap.add_argument("--out", required=True)
ap.add_argument("--cache", default=os.environ.get("TRIBE_CACHE", str(pathlib.Path.home() / ".cache/tribe-features")))
ap.add_argument("--no-map", action="store_true")
args = ap.parse_args()
kind = next(k for k in ("text", "video", "audio", "screen") if getattr(args, k))
cache = pathlib.Path(args.cache)
cache.mkdir(parents=True, exist_ok=True)
summary = encode_stimulus(load_model(cache), kind, getattr(args, kind), args.out, args.seconds, not args.no_map)
print(json.dumps({k: summary[k] for k in ("segments", "seconds_predicted", "modalities", "runtime_s", "top_parcels")}))
if __name__ == "__main__":
main()