Buckets:
| import argparse | |
| import csv | |
| from pathlib import Path | |
| import matplotlib.pyplot as plt | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--directory", type=Path, default=Path(__file__).parent) | |
| parser.add_argument("--output", type=Path, default=Path("memory-comparison.png")) | |
| parser.add_argument("--run-id", action="append") | |
| args = parser.parse_args() | |
| with (args.directory / "comparisons.csv").open() as file: | |
| rows = list(csv.DictReader(file)) | |
| requested = args.run_id or [ | |
| "mm/vllm-a100-llama31-8b/default", | |
| "mm/sglang-a100-llama31-8b/default", | |
| "startup/llamacpp-v-qwen3-8b-q4km", | |
| ] | |
| selected = [next(row for row in rows if row["run_id"] == run_id) for run_id in requested] | |
| figure, axes = plt.subplots(1, 2, figsize=(13, max(3.8, 1.1 * len(selected))), sharey=True, sharex=True) | |
| groups = [ | |
| ( | |
| "GPU ready snapshot · allocated cache pool", | |
| [ | |
| ("weights_reported_bytes", "Engine-reported weights", "0.25", ""), | |
| ("kv_pool_reported_bytes", "Allocated KV pool", "0.6", ""), | |
| ("recurrent_state_reported_bytes", "Recurrent state", "0.8", "xx"), | |
| ("other_device_memory_residual_bytes", "Residual device memory", "0.9", "//"), | |
| ], | |
| ), | |
| ( | |
| "hf-mem 1.0.0 · batch 1, requested context", | |
| [ | |
| ("hf_mem_weights_bytes", "Estimated weights", "0.25", ""), | |
| ("hf_mem_requested_kv_bytes", "Estimated request cache", "0.6", ""), | |
| ("hf_mem_overhead_bytes", "Estimated overhead", "0.9", "//"), | |
| ], | |
| ), | |
| ] | |
| for axis, (title, parts) in zip(axes, groups): | |
| left = [0.0] * len(selected) | |
| for column, label, color, hatch in parts: | |
| values = [float(row.get(column) or 0) / 2**30 for row in selected] | |
| axis.barh( | |
| range(len(selected)), values, left=left, color=color, hatch=hatch, edgecolor="0.15", label=label | |
| ) | |
| left = [a + b for a, b in zip(left, values)] | |
| axis.set_title(title, fontsize=10) | |
| axis.set_xlabel("GiB") | |
| axis.legend(fontsize=7, loc="lower right") | |
| axis.spines[["top", "right"]].set_visible(False) | |
| axes[0].set_yticks( | |
| range(len(selected)), | |
| [f"{r['engine']}\n{r['model'].split('/')[-1]}\ncontext {r['max_model_len']}" for r in selected], | |
| fontsize=8, | |
| ) | |
| axes[0].invert_yaxis() | |
| figure.suptitle("Observed GPU allocation and hf-mem estimation", fontsize=12) | |
| figure.text( | |
| 0.5, | |
| 0.01, | |
| "Cache scopes differ between panels. Residual = device snapshot − reported weights − cache − recurrent state.\nSnapshots are not inference peaks; component logs are rounded. See README for run-specific limitations.", | |
| ha="center", | |
| fontsize=8, | |
| ) | |
| figure.tight_layout(rect=(0, 0.08, 1, 0.94)) | |
| figure.savefig(args.output, dpi=160) | |
| print(args.output) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 3.06 kB
- Xet hash:
- 65298fc269d03e5cb49ce85c1c37121296d270103de7bc6c70171a19a4c63aa9
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.