pytorch-shape-lab / verify_pytorch.py
nityanandmathur's picture
Add a tested PyTorch shape lesson (#1)
d6c87b7
Raw History Blame Contribute Delete
3.88 kB
"""Compare the teaching arithmetic with real CPU operations; no downloads or UI.
Model derived from the MIT tensorViz shape-mismatch example, commit
1baecebf25ff1067a9112bde382c0a398fb99747. See README for the public source.
"""
import json
import platform
from pathlib import Path
import subprocess
import torch
from torch import nn
ROOT = Path(__file__).resolve().parent
CASES = [
{"batch": batch, "height": height, "width": width, "mode": mode}
for batch, height, width in [
(1, 16, 16), (2, 16, 16), (5, 16, 16),
(2, 16, 20), (3, 17, 19), (2, 8, 8), (1, 3, 3),
]
for mode in ["broken", "fixed", "pooled"]
]
NODE_SCRIPT = """
import { inspectShapes } from './shape-math.mjs';
let text = '';
for await (const chunk of process.stdin) text += chunk;
console.log(JSON.stringify(JSON.parse(text).map(inspectShapes)));
"""
def main():
actual_js = json.loads(subprocess.check_output(
["node", "--input-type=module", "-e", NODE_SCRIPT],
input=json.dumps(CASES), text=True, cwd=ROOT,
))
torch.manual_seed(0)
checked = []
with torch.inference_mode():
for case, result in zip(CASES, actual_js, strict=True):
b, h, w = case["batch"], case["height"], case["width"]
x = torch.randn(b, 3, h, w, device="cpu")
features = nn.Sequential(nn.Conv2d(3, 8, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2)).eval()
conv = features[:2](x)
pooled = features[2](conv)
original_flat = nn.Flatten(start_dim=1)(pooled)
assert list(x.shape) == result["input"]
assert list(conv.shape) == result["conv"]
assert list(pooled.shape) == result["pool"]
assert original_flat.shape[0] == b, "Flatten must keep the sample count"
stage = nn.AdaptiveAvgPool2d((1, 1))(pooled) if case["mode"] == "pooled" else pooled
if case["mode"] == "pooled":
assert list(stage.shape) == result["adaptive"]
flat = nn.Flatten(start_dim=1)(stage)
assert list(flat.shape) == result["flattened"]
head = nn.Linear({"broken": 128, "fixed": 512, "pooled": 8}[case["mode"]], 5).eval()
assert list(head.weight.shape) == result["weight"]
try:
out = head(flat)
except RuntimeError as exc:
assert not result["matches"]
assert str(exc) == result["error"]
else:
assert result["matches"]
assert list(out.shape) == result["output"] == [b, 5]
# A batch-preserving path must give one independently computed row per image.
per_image = []
for image in x.split(1):
value = features(image)
if case["mode"] == "pooled":
value = nn.AdaptiveAvgPool2d((1, 1))(value)
per_image.append(head(value.flatten(1)))
torch.testing.assert_close(out, torch.cat(per_image), atol=1e-6, rtol=1e-5)
try:
unsafe = original_flat.reshape(-1, 128)
except RuntimeError:
assert not result["unsafeReshape"]["possible"]
else:
expected = result["unsafeReshape"]
assert list(unsafe.shape) == expected["shape"]
unsafe_out = nn.Linear(128, 5)(unsafe)
assert list(unsafe_out.shape) == expected["output"]
assert (unsafe.shape[0] == b) == expected["preservesBatch"]
checked.append({**case, "flattened": result["flattened"], "output": result["output"], "error": result["error"]})
print(json.dumps({"python": platform.python_version(), "torch": torch.__version__, "device": "cpu", "cases_passed": len(checked), "cases": checked}, indent=2))
if __name__ == "__main__":
main()