Download encode.py from jacob-valdez/computaco-tribe: direct link, hf CLI and curl.
- Browser
- Download file 12 kB
-
https://huggingface.co/jacob-valdez/computaco-tribe/resolve/main/encode.py
- Command line
-
hf download hf://jacob-valdez/computaco-tribe/encode.py
-
curl -L -o encode.py https://huggingface.co/jacob-valdez/computaco-tribe/resolve/main/encode.py
12 kB
| """Predict cortical responses to a stimulus with TRIBE v2 (facebook/tribev2, CC BY-NC 4.0). | |
| A perception runs `encode_stimulus` on the model's inference endpoint (scripts/brain/tribe/handler.py, | |
| docs/harnesses.md § Perception). The command line is the same computation, for studies run offline: | |
| ~/tribe-env/bin/python scripts/brain/encode.py --text page.txt --out run/ | |
| ~/tribe-env/bin/python scripts/brain/encode.py --video clip.mp4 --out run/ | |
| ~/tribe-env/bin/python scripts/brain/encode.py --screen frame.png --seconds 20 --out run/ | |
| A text is spoken (text-to-speech) and transcribed back to timed words, as TRIBE's own | |
| demo does; a screen is a still held for `--seconds` as a silent video; a video or audio | |
| file is taken as it is. The model answers one prediction a second (its TR) on the | |
| fsaverage5 cortical mesh (20,484 vertices, left then right), for the average subject, in | |
| z-units of BOLD, shifted 5 s into the past for the haemodynamic lag. | |
| Writes into --out: | |
| preds.npz preds (T, 20484) float16 and times (T,) seconds | |
| rois.json every Destrieux parcel's time-mean and peak, per hemisphere, with its name | |
| features.npz the backbone features the model was given (time-mean per modality), for | |
| matched-control analyses: the same stimulus without the brain projection | |
| map.png the time-mean prediction on the inflated surface, four views | |
| summary.json what was run (model, checkpoint digest, stimulus, seconds, segments, | |
| duration), the hemisphere means, and the ten strongest parcels | |
| It fails plainly without the model, the GPU or the stimulus; it never writes a result it | |
| did not compute. | |
| """ | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| import pathlib | |
| import subprocess | |
| import time | |
| MODEL_ID = "facebook/tribev2" | |
| MESH = "fsaverage5" | |
| N_VERTICES = 20484 | |
| def sha256(path): | |
| h = hashlib.sha256() | |
| with open(path, "rb") as f: | |
| for chunk in iter(lambda: f.read(1 << 20), b""): | |
| h.update(chunk) | |
| return h.hexdigest() | |
| def ffmpeg(): | |
| import imageio_ffmpeg | |
| return imageio_ffmpeg.get_ffmpeg_exe() | |
| def screen_video(png, seconds, out): | |
| """A still image held for `seconds`, with a silent audio track so the audio extractor has one.""" | |
| subprocess.run([ffmpeg(), "-y", "-loglevel", "error", "-loop", "1", "-framerate", "4", "-i", str(png), | |
| "-f", "lavfi", "-i", "anullsrc=r=16000:cl=mono", "-t", str(seconds), | |
| "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2,format=yuv420p", "-c:v", "libx264", "-preset", "fast", "-crf", "20", | |
| "-c:a", "aac", "-shortest", str(out)], check=True) | |
| return out | |
| def transcribe_on_cpu(): | |
| """TRIBE transcribes with whisperx under `uvx` on CUDA in float16; CTranslate2's aarch64 wheel has | |
| no CUDA, so here whisperx runs on the CPU in int8 (BRAIN_WHISPER_DEVICE=cuda keeps TRIBE's own).""" | |
| if os.environ.get("BRAIN_WHISPER_DEVICE", "cpu") == "cuda": | |
| return | |
| import json as _json | |
| import subprocess as _sp | |
| import tempfile | |
| from pathlib import Path | |
| import pandas as pd | |
| from tribev2 import eventstransforms as et | |
| codes = dict(english="en", french="fr", spanish="es", dutch="nl", chinese="zh") | |
| def get_transcript(wav_filename, language): | |
| if language not in codes: | |
| raise ValueError(f"Language {language} not supported") | |
| with tempfile.TemporaryDirectory() as out: | |
| cmd = ["uvx", "whisperx", str(wav_filename), "--model", "large-v3", "--language", codes[language], "--device", "cpu", | |
| "--compute_type", "int8", "--batch_size", "16", "--output_dir", out, "--output_format", "json"] | |
| if language == "english": | |
| cmd += ["--align_model", "WAV2VEC2_ASR_LARGE_LV60K_960H"] | |
| env = {k: v for k, v in os.environ.items() if k != "MPLBACKEND"} | |
| r = _sp.run(cmd, capture_output=True, text=True, env=env) | |
| if r.returncode != 0: | |
| raise RuntimeError(f"whisperx failed:\n{r.stderr[-2000:]}") | |
| transcript = _json.loads((Path(out) / f"{Path(wav_filename).stem}.json").read_text()) | |
| words = [] | |
| for i, seg in enumerate(transcript["segments"]): | |
| sentence = seg["text"].replace('"', "") | |
| for w in seg["words"]: | |
| if "start" in w: | |
| words.append({"text": w["word"].replace('"', ""), "start": w["start"], "duration": w["end"] - w["start"], "sequence_id": i, "sentence": sentence}) | |
| return pd.DataFrame(words) | |
| et.ExtractWordsFromAudio._get_transcript_from_audio = staticmethod(get_transcript) | |
| def load_model(cache): | |
| from tribev2.demo_utils import TribeModel | |
| transcribe_on_cpu() | |
| return TribeModel.from_pretrained(MODEL_ID, cache_folder=str(cache)) | |
| def checkpoint_digest(): | |
| from huggingface_hub import hf_hub_download | |
| return sha256(hf_hub_download(MODEL_ID, "best.ckpt")) | |
| class FeatureTap: | |
| """Catches the per-modality backbone features each batch carries into the model.""" | |
| def __init__(self, model): | |
| self.sums, self.counts = {}, {} | |
| self.handle = model.register_forward_pre_hook(self.hook, with_kwargs=False) | |
| def hook(self, module, args): | |
| import torch | |
| batch = args[0] | |
| for modality in ("text", "audio", "video"): | |
| if modality not in batch.data: | |
| continue | |
| x = batch.data[modality].to(torch.float32) # B, L, D, T (or B, D, T) | |
| if x.ndim == 3: | |
| x = x.unsqueeze(1) | |
| x = x.mean(dim=1) # mean over layers: B, D, T | |
| self.sums[modality] = self.sums.get(modality, 0) + x.sum(dim=(0, 2)).cpu().numpy() | |
| self.counts[modality] = self.counts.get(modality, 0) + x.shape[0] * x.shape[2] | |
| def means(self): | |
| return {m: self.sums[m] / max(self.counts[m], 1) for m in self.sums} | |
| def atlas(): | |
| """Destrieux parcels on fsaverage5: (labels_left, labels_right, names).""" | |
| from nilearn import datasets | |
| a = datasets.fetch_atlas_surf_destrieux() | |
| names = [n.decode() if isinstance(n, bytes) else str(n) for n in a["labels"]] | |
| return a["map_left"], a["map_right"], names | |
| def parcels(mean, peak, labels_l, labels_r, names): | |
| import numpy as np | |
| out = [] | |
| half = N_VERTICES // 2 | |
| for hemi, labels, lo in (("L", labels_l, 0), ("R", labels_r, half)): | |
| for idx, name in enumerate(names): | |
| if name in ("Unknown", "Medial_wall"): | |
| continue | |
| sel = np.where(labels == idx)[0] | |
| if not len(sel): | |
| continue | |
| out.append({"id": f"{hemi}_{name}", "hemisphere": hemi, "name": name, "vertices": int(len(sel)), | |
| "mean": float(mean[lo + sel].mean()), "peak": float(peak[lo + sel].mean())}) | |
| return out | |
| def plot_map(mean, out): | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| import numpy as np | |
| from nilearn import datasets, plotting | |
| fs = datasets.fetch_surf_fsaverage(MESH) | |
| half = N_VERTICES // 2 | |
| v = float(np.percentile(np.abs(mean), 99)) or 1.0 | |
| fig, axes = plt.subplots(1, 4, figsize=(16, 4), subplot_kw={"projection": "3d"}) | |
| views = [("left", "lateral", mean[:half]), ("left", "medial", mean[:half]), ("right", "lateral", mean[half:]), ("right", "medial", mean[half:])] | |
| for ax, (hemi, view, data) in zip(axes, views): | |
| plotting.plot_surf_stat_map(fs[f"infl_{hemi}"], data, hemi=hemi, view=view, bg_map=fs[f"sulc_{hemi}"], cmap="RdBu_r", | |
| vmax=v, symmetric_cbar=True, colorbar=(view == "medial" and hemi == "right"), axes=ax, threshold=v * 0.1) | |
| ax.set_title(f"{hemi} {view}", fontsize=10) | |
| fig.savefig(out, dpi=110, bbox_inches="tight") | |
| plt.close(fig) | |
| def encode_stimulus(model, kind, path, out, seconds=20.0, draw_map=True): | |
| """Predict the response to one stimulus with a loaded model, and write the files above into `out`. | |
| `kind` is text, video, audio or screen; `path` the file. → the summary.""" | |
| import numpy as np | |
| path = pathlib.Path(path) | |
| out = pathlib.Path(out) | |
| out.mkdir(parents=True, exist_ok=True) | |
| t0 = time.time() | |
| stimulus = {"kind": kind, "source": str(path), "sha256": sha256(path)} | |
| if kind == "screen": | |
| clip = out / "stimulus.mp4" | |
| screen_video(path, seconds, clip) | |
| stimulus["seconds"] = seconds | |
| events_arg = {"video_path": str(clip)} | |
| elif kind == "video": | |
| events_arg = {"video_path": str(path.resolve())} | |
| elif kind == "audio": | |
| events_arg = {"audio_path": str(path.resolve())} | |
| elif kind == "text": | |
| stimulus["words"] = len(path.read_text(encoding="utf-8").split()) | |
| events_arg = {"text_path": str(path.resolve())} | |
| else: | |
| raise ValueError(f"no stimulus of the kind {kind}") | |
| tap = FeatureTap(model._model) | |
| events = model.get_events_dataframe(**events_arg) | |
| preds, segments = model.predict(events, verbose=False) | |
| tap.handle.remove() | |
| if preds.shape[1] != N_VERTICES: | |
| raise RuntimeError(f"expected {N_VERTICES} vertices, got {preds.shape[1]}") | |
| times = np.array([float(s.start) for s in segments]) | |
| order = np.argsort(times) | |
| preds, times = preds[order], times[order] | |
| mean = preds.mean(axis=0) | |
| peak = preds.max(axis=0) | |
| np.savez_compressed(out / "preds.npz", preds=preds.astype(np.float16), times=times.astype(np.float32)) | |
| features = tap.means() | |
| np.savez_compressed(out / "features.npz", **{k: v.astype(np.float32) for k, v in features.items()}) | |
| labels_l, labels_r, names = atlas() | |
| rows = parcels(mean, peak, labels_l, labels_r, names) | |
| (out / "rois.json").write_text(json.dumps({"atlas": "Destrieux 2009 on fsaverage5 (nilearn)", "parcels": rows}, indent=1)) | |
| if draw_map: | |
| plot_map(mean, out / "map.png") | |
| half = N_VERTICES // 2 | |
| top = sorted(rows, key=lambda r: -r["mean"])[:10] | |
| summary = { | |
| "model": MODEL_ID, "checkpoint_sha256": checkpoint_digest(), "mesh": MESH, "vertices": N_VERTICES, | |
| "stimulus": stimulus, "segments": int(preds.shape[0]), "seconds_predicted": float(times[-1] - times[0] + 1) if len(times) else 0.0, | |
| "modalities": sorted(features), "feature_dims": {k: int(v.shape[0]) for k, v in features.items()}, | |
| "hemisphere_mean": {"left": float(mean[:half].mean()), "right": float(mean[half:].mean())}, | |
| "global_mean": float(mean.mean()), "global_peak": float(peak.max()), | |
| "top_parcels": [{"id": r["id"], "mean": r["mean"]} for r in top], | |
| "runtime_s": round(time.time() - t0, 1), | |
| "files": ["preds.npz", "rois.json", "features.npz"] + (["map.png"] if draw_map else []), | |
| } | |
| (out / "summary.json").write_text(json.dumps(summary, indent=1)) | |
| return summary | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) | |
| src = ap.add_mutually_exclusive_group(required=True) | |
| src.add_argument("--text", help="a .txt file: spoken and transcribed") | |
| src.add_argument("--video", help="an .mp4/.mov/.webm") | |
| src.add_argument("--audio", help="a .wav/.mp3") | |
| src.add_argument("--screen", help="a .png held still for --seconds") | |
| ap.add_argument("--seconds", type=float, default=20.0, help="how long a --screen is shown") | |
| ap.add_argument("--out", required=True) | |
| ap.add_argument("--cache", default=os.environ.get("TRIBE_CACHE", str(pathlib.Path.home() / ".cache/tribe-features"))) | |
| ap.add_argument("--no-map", action="store_true") | |
| args = ap.parse_args() | |
| kind = next(k for k in ("text", "video", "audio", "screen") if getattr(args, k)) | |
| cache = pathlib.Path(args.cache) | |
| cache.mkdir(parents=True, exist_ok=True) | |
| summary = encode_stimulus(load_model(cache), kind, getattr(args, kind), args.out, args.seconds, not args.no_map) | |
| print(json.dumps({k: summary[k] for k in ("segments", "seconds_predicted", "modalities", "runtime_s", "top_parcels")})) | |
| if __name__ == "__main__": | |
| main() | |