HDCSTORAGE: hyperdimensional computer, webOS and agentic MCP layer (stdlib-only Python)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- Dockerfile +19 -0
- README.md +1008 -0
- artifacts/demo.hdc +0 -0
- artifacts/demo_run.txt +175 -0
- artifacts/ide.hdc +0 -0
- hdc/__init__.py +96 -0
- hdc/__main__.py +403 -0
- hdc/__pycache__/__init__.cpython-315.pyc +0 -0
- hdc/__pycache__/__main__.cpython-315.pyc +0 -0
- hdc/__pycache__/adder.cpython-315.pyc +0 -0
- hdc/__pycache__/agent.cpython-315.pyc +0 -0
- hdc/__pycache__/assembler.cpython-315.pyc +0 -0
- hdc/__pycache__/assistant.cpython-315.pyc +0 -0
- hdc/__pycache__/autonomy.cpython-315.pyc +0 -0
- hdc/__pycache__/boot.cpython-315.pyc +0 -0
- hdc/__pycache__/debugger.cpython-315.pyc +0 -0
- hdc/__pycache__/entropy.cpython-315.pyc +0 -0
- hdc/__pycache__/fs.cpython-315.pyc +0 -0
- hdc/__pycache__/gfx.cpython-315.pyc +0 -0
- hdc/__pycache__/hypervec.cpython-315.pyc +0 -0
- hdc/__pycache__/ide.cpython-315.pyc +0 -0
- hdc/__pycache__/interpreter.cpython-315.pyc +0 -0
- hdc/__pycache__/isa.cpython-315.pyc +0 -0
- hdc/__pycache__/memory.cpython-315.pyc +0 -0
- hdc/__pycache__/memory_modes.cpython-315.pyc +0 -0
- hdc/__pycache__/os.cpython-315.pyc +0 -0
- hdc/__pycache__/ring.cpython-315.pyc +0 -0
- hdc/__pycache__/semantic.cpython-315.pyc +0 -0
- hdc/__pycache__/shader.cpython-315.pyc +0 -0
- hdc/__pycache__/slicegate.cpython-315.pyc +0 -0
- hdc/__pycache__/state.cpython-315.pyc +0 -0
- hdc/__pycache__/terminal.cpython-315.pyc +0 -0
- hdc/__pycache__/text.cpython-315.pyc +0 -0
- hdc/__pycache__/webui.cpython-315.pyc +0 -0
- hdc/__pycache__/x86.cpython-315.pyc +0 -0
- hdc/adder.py +395 -0
- hdc/agent.py +423 -0
- hdc/assembler.py +392 -0
- hdc/assistant.py +188 -0
- hdc/autonomy.py +381 -0
- hdc/boot.py +263 -0
- hdc/crystal.py +198 -0
- hdc/debugger.py +259 -0
- hdc/demo.py +295 -0
- hdc/entropy.py +140 -0
- hdc/fractal.py +146 -0
- hdc/fs.py +366 -0
- hdc/gfx.py +415 -0
- hdc/hub.py +516 -0
- hdc/hypervec.py +440 -0
Dockerfile
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# HDCSTORAGE runs on the standard library alone: no pip installs, no wheels,
|
| 2 |
+
# no build step. The only bundled asset is the static embedding table the
|
| 3 |
+
# assistant's encoder projects into hypervectors.
|
| 4 |
+
FROM python:3.13-slim
|
| 5 |
+
|
| 6 |
+
ENV PYTHONUNBUFFERED=1 \
|
| 7 |
+
PYTHONDONTWRITEBYTECODE=1
|
| 8 |
+
|
| 9 |
+
WORKDIR /app
|
| 10 |
+
COPY hdc ./hdc
|
| 11 |
+
COPY tests ./tests
|
| 12 |
+
COPY artifacts ./artifacts
|
| 13 |
+
COPY models ./models
|
| 14 |
+
COPY README.md ./README.md
|
| 15 |
+
|
| 16 |
+
# Spaces routes to app_port (7860); the CLI binds 127.0.0.1 by default, so the
|
| 17 |
+
# container has to be told to listen on all interfaces.
|
| 18 |
+
EXPOSE 7860
|
| 19 |
+
CMD ["python", "-m", "hdc", "web", "--host", "0.0.0.0", "--port", "7860"]
|
README.md
ADDED
|
@@ -0,0 +1,1008 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
library_name: hdcstorage
|
| 3 |
+
tags:
|
| 4 |
+
- hyperdimensional-computing
|
| 5 |
+
- vector-symbolic-architecture
|
| 6 |
+
- hdc
|
| 7 |
+
- binary-spatter-code
|
| 8 |
+
- compute-in-memory
|
| 9 |
+
- webos
|
| 10 |
+
- python
|
| 11 |
+
---
|
| 12 |
+
|
| 13 |
+
# HDCSTORAGE
|
| 14 |
+
|
| 15 |
+
A hyperdimensional computer that keeps its entire state in hyperdimensional
|
| 16 |
+
storage, and an agentic MCP tool layer that lives inside that storage.
|
| 17 |
+
|
| 18 |
+
Programs are compiled into **hypervectors**, written into cells of a
|
| 19 |
+
content-addressable HDC memory, fetched back out, and **decoded by cleanup
|
| 20 |
+
memory** — the decoder never reads a fixed bit field, it asks which known
|
| 21 |
+
symbol the fetched vector most resembles. Registers, addresses, stack slots,
|
| 22 |
+
literals, tool records and string constants are all the same kind of object:
|
| 23 |
+
a 10 000-bit vector.
|
| 24 |
+
|
| 25 |
+
```
|
| 26 |
+
.hdcasm source
|
| 27 |
+
| compiler (assembler.py)
|
| 28 |
+
v
|
| 29 |
+
instruction hypervectors ---> HDC storage cells (memory.py)
|
| 30 |
+
|
|
| 31 |
+
| fetch by content, decode by cleanup
|
| 32 |
+
v
|
| 33 |
+
interpreter.py ---> debugger.py / ide.py
|
| 34 |
+
|
|
| 35 |
+
| TOOL opcode
|
| 36 |
+
v
|
| 37 |
+
agent.py (tool records in HDC)
|
| 38 |
+
|
|
| 39 |
+
v
|
| 40 |
+
mcp_server.py (JSON-RPC over stdio)
|
| 41 |
+
```
|
| 42 |
+
|
| 43 |
+
**Standard library only.** No numpy, no third-party packages. Hypervectors are
|
| 44 |
+
Python big integers, so `bind` is one `^`, `similarity` is one `bit_count`,
|
| 45 |
+
and a 10 000-bit vector costs 1 250 bytes.
|
| 46 |
+
|
| 47 |
+
---
|
| 48 |
+
|
| 49 |
+
## Quick start
|
| 50 |
+
|
| 51 |
+
```bash
|
| 52 |
+
cd D:/HDCSTORAGE
|
| 53 |
+
|
| 54 |
+
python -m hdc demo # end-to-end tour, every layer, measured
|
| 55 |
+
python -m hdc run examples/hello.hdcasm
|
| 56 |
+
python -m hdc asm examples/logic.hdcasm # assemble + decode-back report
|
| 57 |
+
python -m hdc ide # the IDE (REPL)
|
| 58 |
+
python -m hdc adder 7 35 # the HDC full adder, stage by stage
|
| 59 |
+
python -m hdc shader # HDC image pipeline: dim, edges, depth, normals
|
| 60 |
+
python -m hdc mcp # MCP server on stdio
|
| 61 |
+
python -m hdc mcp --selftest # protocol self-check, no client needed
|
| 62 |
+
python -m hdc net # BitNet layer self-check
|
| 63 |
+
python -m hdc isa # instruction set reference
|
| 64 |
+
python -m hdc tools # the HDC-resident tool registry
|
| 65 |
+
python -m hdc os # the webOS: desktop, windows, apps, memory map
|
| 66 |
+
python -m hdc boot # POST -> shader splash -> AUTOEXEC, frame by frame
|
| 67 |
+
python -m hdc gfx # shader pass timings on an 80x28 surface
|
| 68 |
+
python -m hdc hub # which embedding table this machine can use
|
| 69 |
+
python -m hdc autonomy # idle + readahead on a simulated session
|
| 70 |
+
python -m hdc assist --autonomous # chat with it; idle ticks between turns
|
| 71 |
+
python -m hdc test # unit tests (over 300)
|
| 72 |
+
```
|
| 73 |
+
|
| 74 |
+
Every command takes `--d <int>` to change dimensionality (default 10 000).
|
| 75 |
+
`hub`, `autonomy` and `assist` are the exceptions that matter: they measure the
|
| 76 |
+
codec rather than the machine, so `autonomy` and `assist` default to `--d 2000`
|
| 77 |
+
(the codec floor) unless a dimension is given. The same simulated session scores
|
| 78 |
+
80% next-intent accuracy at 2000 and 70% at 10 000 — a fixed 21-symbol cleanup
|
| 79 |
+
codebook is diluted by extra bits, so the machine's default would misreport the
|
| 80 |
+
layer.
|
| 81 |
+
|
| 82 |
+
---
|
| 83 |
+
|
| 84 |
+
## 1. The algebra (`hdc/hypervec.py`)
|
| 85 |
+
|
| 86 |
+
Binary spatter codes:
|
| 87 |
+
|
| 88 |
+
| operation | implementation | meaning |
|
| 89 |
+
|---|---|---|
|
| 90 |
+
| bind | `a ^ b` | association; self-inverse, so unbind == bind |
|
| 91 |
+
| bundle | bit-sliced majority | superposition / set |
|
| 92 |
+
| permute | bit rotation in the d-bit mask | order and position |
|
| 93 |
+
| similarity | `(d - 2·popcount(a^b)) / d` | agreement in `[-1, 1]` |
|
| 94 |
+
|
| 95 |
+
Bundling is a real majority, not a sum: every vector is added into a d-lane
|
| 96 |
+
binary counter (one Python int per bit of the count) and a bit-sliced
|
| 97 |
+
comparator thresholds it at `floor(n/2)+1`. That keeps the cost at
|
| 98 |
+
`O(n·log n)` big-integer operations instead of `O(n·d)` element visits, which
|
| 99 |
+
is what makes 10 000-bit bundles cheap in pure Python.
|
| 100 |
+
|
| 101 |
+
The numbers that matter, measured by `python -m hdc demo` on this machine:
|
| 102 |
+
|
| 103 |
+
```
|
| 104 |
+
random pair similarity mean +0.0001 sd 0.0100 <- noise floor
|
| 105 |
+
bundle of 5 members recovered at 0.369 - 0.382 <- signal
|
| 106 |
+
```
|
| 107 |
+
|
| 108 |
+
Everything downstream — cleanup, decoding, dispatch — is a decision made
|
| 109 |
+
between those two numbers.
|
| 110 |
+
|
| 111 |
+
**Integer codec.** A scalar is stored as a superposition of bit-position atoms
|
| 112 |
+
(`bind(NUM, permute(BITBASE, i))` for each set bit), so integers are vectors
|
| 113 |
+
like everything else and decode by probing each bit position. The layout is
|
| 114 |
+
**32-bit two's complement**, because sign-magnitude has no meaningful carry
|
| 115 |
+
and a ripple-carry adder needs one. This codec is *symbolic, not
|
| 116 |
+
similarity-preserving*: integers that share set bits share atoms, so
|
| 117 |
+
`sim(int_to_hv(5), int_to_hv(37))` is high. Read `int_to_hv`'s docstring before
|
| 118 |
+
using distances between encoded integers as if they meant something
|
| 119 |
+
semantically.
|
| 120 |
+
|
| 121 |
+
---
|
| 122 |
+
|
| 123 |
+
## 2. Storage (`hdc/memory.py`)
|
| 124 |
+
|
| 125 |
+
A cell is one hypervector with a name; the name's address is also a vector
|
| 126 |
+
(`hvc("cell/" + name)`).
|
| 127 |
+
|
| 128 |
+
* `write(name, hv)` / `read(name)` — named access.
|
| 129 |
+
* `resolve(addr_hv)` — content addressing: exact hit at confidence 1.0,
|
| 130 |
+
otherwise nearest cell address above a floor. A damaged address still lands
|
| 131 |
+
on the right cell and reports how sure it was (`demo` flips 12 of 10 000
|
| 132 |
+
bits and resolves at ~0.998).
|
| 133 |
+
* `recall(probe, k)` — associative search over cell *values*; the
|
| 134 |
+
compute-in-memory read.
|
| 135 |
+
* `nearest_symbol(hv)` — cleanup memory, the table that turns a noisy decoded
|
| 136 |
+
field back into a name.
|
| 137 |
+
|
| 138 |
+
Persistence is a byte-stable `.hdc` image: magic, header, packed vectors,
|
| 139 |
+
symbol table. Symbolic vectors are regenerated from their names on load, so an
|
| 140 |
+
image is portable at the same `d`.
|
| 141 |
+
|
| 142 |
+
---
|
| 143 |
+
|
| 144 |
+
## 3. Instruction set (`hdc/isa.py`)
|
| 145 |
+
|
| 146 |
+
37 opcodes. An instruction is a bundle of bound fields:
|
| 147 |
+
|
| 148 |
+
```python
|
| 149 |
+
instr = bundle([
|
| 150 |
+
bind(role("op"), hvc("op/ADD")),
|
| 151 |
+
bind(role("ra"), hvc("reg/R2")),
|
| 152 |
+
bind(role("rb"), hvc("reg/R0")),
|
| 153 |
+
bind(role("rc"), hvc("reg/R1")),
|
| 154 |
+
])
|
| 155 |
+
```
|
| 156 |
+
|
| 157 |
+
Decoding unbinds `role("op")`, cleans up over the opcode alphabet, then reads
|
| 158 |
+
exactly the operand slots that opcode declares — and reports the minimum
|
| 159 |
+
similarity across fields as its confidence. Measured on assembled programs:
|
| 160 |
+
opcode-only instructions decode at 1.00, four-field arithmetic at 0.36-0.50,
|
| 161 |
+
against a 0.01 noise floor.
|
| 162 |
+
|
| 163 |
+
| class | opcodes |
|
| 164 |
+
|---|---|
|
| 165 |
+
| control | `NOP HALT JMP JZ JNZ CALL RET` |
|
| 166 |
+
| integer ALU | `LDI MOV ADD SUB MUL DIV AND OR XOR NOT SHL SHR POPC CMP` |
|
| 167 |
+
| hyperdimensional | `BIND BUNDLE PERMUTE SIM CLEAN` |
|
| 168 |
+
| storage | `STORE LOAD RECALL MEMSTAT PUSH POP` |
|
| 169 |
+
| I/O | `OUT OUTT PRINTS` |
|
| 170 |
+
| agentic | `TOOL` |
|
| 171 |
+
| debug | `BREAK` |
|
| 172 |
+
|
| 173 |
+
`python -m hdc isa` prints the full table with descriptions.
|
| 174 |
+
|
| 175 |
+
**Semantics, stated plainly.** Registers hold hypervectors. `BIND`, `BUNDLE`,
|
| 176 |
+
`PERMUTE`, `SIM`, `CLEAN`, `STORE`, `LOAD`, `RECALL`, `TOOL`, and now `ADD` and
|
| 177 |
+
`SUB` never leave hyperdimensional space — the arithmetic runs as the
|
| 178 |
+
ripple-carry circuit described in §6. `MUL`, `DIV`, the bitwise ops and the
|
| 179 |
+
shifts still decode the vector payload to a Python integer, compute, and
|
| 180 |
+
re-encode; that is a conventional ALU behind a hyperdimensional register file.
|
| 181 |
+
Jumps use the decoded cell index, which the assembler bound into the
|
| 182 |
+
instruction.
|
| 183 |
+
|
| 184 |
+
---
|
| 185 |
+
|
| 186 |
+
## 4. Language (`hdc/assembler.py`)
|
| 187 |
+
|
| 188 |
+
```asm
|
| 189 |
+
; comments start with ; # or //
|
| 190 |
+
.str banner "hello from hyperdimensional storage" ; a string cell
|
| 191 |
+
.data counter 0 ; a preloaded integer cell
|
| 192 |
+
.name result ; declare a cell name
|
| 193 |
+
|
| 194 |
+
start: ; labels name program cells
|
| 195 |
+
ldi R0, 21
|
| 196 |
+
mul R2, R0, R1
|
| 197 |
+
store result, R2
|
| 198 |
+
load R3, result
|
| 199 |
+
tool R4, R0, R2, hdc.similarity
|
| 200 |
+
prints banner
|
| 201 |
+
halt
|
| 202 |
+
```
|
| 203 |
+
|
| 204 |
+
Two passes: scan (labels, and the literal/string/name/tool alphabets that
|
| 205 |
+
become cleanup memory) then emit. Assembling runs a **decode-back check** on
|
| 206 |
+
every cell and refuses to emit a program whose cells do not decode back to
|
| 207 |
+
their own opcode — the compiler verifies the image, not just the text.
|
| 208 |
+
|
| 209 |
+
A string operand may be quoted inline or name a `.str` cell declared anywhere
|
| 210 |
+
in the file.
|
| 211 |
+
|
| 212 |
+
---
|
| 213 |
+
|
| 214 |
+
## 5. Interpreter and debugger
|
| 215 |
+
|
| 216 |
+
`Machine` fetches cell `code/<pc>` through the content-addressable path,
|
| 217 |
+
decodes it, and executes. Decoding is memoised by vector value (a pure
|
| 218 |
+
function of vector + alphabet), so fetch still goes through storage every
|
| 219 |
+
step while repeated executions stay fast.
|
| 220 |
+
|
| 221 |
+
`Debugger` adds breakpoints by cell or label, step / step-over, a trace ring
|
| 222 |
+
with per-step confidence, and a register table whose `SYMBOL` column is a
|
| 223 |
+
cleanup query — with the match hidden below the noise band rather than shown
|
| 224 |
+
as a bogus 0.02 hit.
|
| 225 |
+
|
| 226 |
+
```
|
| 227 |
+
hdc> asm
|
| 228 |
+
hdc> bp compute
|
| 229 |
+
hdc> run
|
| 230 |
+
stopped: break at pc=2 (breakpoint)
|
| 231 |
+
hdc> regs
|
| 232 |
+
hdc> disasm
|
| 233 |
+
hdc> bits R3
|
| 234 |
+
```
|
| 235 |
+
|
| 236 |
+
---
|
| 237 |
+
|
| 238 |
+
## 6. Native arithmetic (`hdc/adder.py`)
|
| 239 |
+
|
| 240 |
+
`ADD` and `SUB` do not decode. They run a carry chain whose only primitive is
|
| 241 |
+
binding plus cleanup:
|
| 242 |
+
|
| 243 |
+
* a bit is read by a similarity probe against its position atom;
|
| 244 |
+
* a gate is a truth table **stored as records in HDC storage** — 8 rows
|
| 245 |
+
(`adder/row/000` … `adder/row/111`) for a 1-bit stage, 2**(2k+1) rows for a
|
| 246 |
+
k-bit stage;
|
| 247 |
+
* one stage is `bind → clean up`: form the key for the stage's inputs and ask
|
| 248 |
+
storage which row it resembles, resolving well above the 0.01 noise floor and
|
| 249 |
+
reporting that number per stage;
|
| 250 |
+
* the sum is reassembled by bundling position atoms, and the carry feeds
|
| 251 |
+
forward as the next stage's input;
|
| 252 |
+
* `SUB` is `a + ~b + 1` in two's complement — no sign case analysis anywhere.
|
| 253 |
+
|
| 254 |
+
**Stage count is a tunable.** A gate that consumes k bits per stage trades
|
| 255 |
+
stage depth for table size:
|
| 256 |
+
|
| 257 |
+
| radix | table rows | stages per 32-bit add | measured add, d=10 000 |
|
| 258 |
+
|---|---|---|---|
|
| 259 |
+
| 1 | 8 | 32 | 0.98 ms |
|
| 260 |
+
| 2 | 32 | 16 | — |
|
| 261 |
+
| 4 | 512 | 8 | 0.38 ms |
|
| 262 |
+
| 8 | 131 072 | **4** | **0.38 ms** |
|
| 263 |
+
|
| 264 |
+
The default is **radix 8: four byte-slice stages**. A 131 072-row table at
|
| 265 |
+
d=10 000 would be 164 MB of vectors and ~1.6 s of XOR folding to enumerate,
|
| 266 |
+
so `hdc/slicegate.py` materialises rows **lazily**: a 32-bit add touches four
|
| 267 |
+
byte rows, they are composed once, decoded, written to storage and cached, and
|
| 268 |
+
the worst case is still complete. Gate keys are exact (one atom per slot,
|
| 269 |
+
position and value), so they are looked up in a dictionary — a 131 072-row
|
| 270 |
+
similarity scan would cost more than the ripple it replaces — with the fuzzy
|
| 271 |
+
`resolve_similar` path kept for damaged keys.
|
| 272 |
+
|
| 273 |
+
Two design constraints cost real debugging time and are now regression-tested:
|
| 274 |
+
|
| 275 |
+
**The codec had to become two's complement.** Sign-magnitude has no meaningful
|
| 276 |
+
carry, and negative values in two's complement are *dense* (`-1` sets all 32
|
| 277 |
+
atoms), which forced the probe threshold down to 0.10 and pins the supported
|
| 278 |
+
floor at `d = 2 000`.
|
| 279 |
+
|
| 280 |
+
**Each gate slot needs its own atom pair.** A key built from one shared
|
| 281 |
+
zero/one pair is an XOR of atoms, and XOR is parity: `1^1^1` and `1^0^0`
|
| 282 |
+
collapse to the same vector, leaving 2 distinguishable rows out of 8. Tagging
|
| 283 |
+
inputs with a role before the XOR does *not* help — the role constants cancel
|
| 284 |
+
into a fixed mask. Per-slot pairs (`gate/atom/a/0`, …) restore eight
|
| 285 |
+
near-orthogonal keys (worst cross-similarity −0.02 measured).
|
| 286 |
+
|
| 287 |
+
Cost, measured: **0.38 ms per add at d = 10 000** with radix 8, and 0.98 ms
|
| 288 |
+
with the 1-bit gate, versus ~0.05 ms for the old decode-and-add. The 32-bit
|
| 289 |
+
wrap is now hardware-like instead of an exception. `python -m hdc adder 7 35`
|
| 290 |
+
prints the table and the carry chain.
|
| 291 |
+
|
| 292 |
+
## 7. BitNet base (`hdc/net.py`)
|
| 293 |
+
|
| 294 |
+
1-bit and ternary kernels on the same primitives, because a 1-bit dot product
|
| 295 |
+
*is* an XOR and a popcount:
|
| 296 |
+
|
| 297 |
+
```python
|
| 298 |
+
dot_1bit_packed(a, b, n) == n - 2 * popcount(a ^ b) # == dot_1bit
|
| 299 |
+
```
|
| 300 |
+
|
| 301 |
+
Also `ternarize` (absmean with a dead zone, in the spirit of BitNet b1.58),
|
| 302 |
+
`quantize_absmax`, `ternary_matvec`, `BitLinear`, and `PrototypeNet` — a
|
| 303 |
+
nearest-prototype classifier whose class weights are literally `HDCStorage`
|
| 304 |
+
cells, so they persist and are addressable by content like anything else.
|
| 305 |
+
|
| 306 |
+
`python -m hdc net` verifies the XOR identity at run time and reports measured
|
| 307 |
+
sparsity and accuracy values. Nothing in this repository is a stored
|
| 308 |
+
benchmark; every number is recomputed.
|
| 309 |
+
|
| 310 |
+
---
|
| 311 |
+
|
| 312 |
+
## 8. Agentic tools inside HDC (`hdc/agent.py`)
|
| 313 |
+
|
| 314 |
+
A tool is a record, stored as a cell:
|
| 315 |
+
|
| 316 |
+
```python
|
| 317 |
+
bundle([
|
| 318 |
+
bind(role("name"), hvc("tool/hdc.bind")),
|
| 319 |
+
bind(role("cap"), capability_vector("hdc/bind/construct")),
|
| 320 |
+
bind(role("arity"), hvc("arity/2")),
|
| 321 |
+
])
|
| 322 |
+
```
|
| 323 |
+
|
| 324 |
+
Dispatch is similarity, in three tiers, each reporting confidence:
|
| 325 |
+
|
| 326 |
+
1. exact name → 1.000;
|
| 327 |
+
2. character-superposition similarity, so `hdc.bundl` finds `hdc.bundle`
|
| 328 |
+
(measured 0.74-0.76) — a misspelling is a cleanup problem, not a crash;
|
| 329 |
+
3. capability-path similarity, so `hdc.memory.search` reaches `hdc.recall`
|
| 330 |
+
and `bit.count` reaches `bit.popcount`.
|
| 331 |
+
|
| 332 |
+
The same records are what the `TOOL` instruction calls and what the MCP
|
| 333 |
+
server advertises, so a program running inside HDC and an external agent reach
|
| 334 |
+
identical tools by identical paths.
|
| 335 |
+
|
| 336 |
+
---
|
| 337 |
+
|
| 338 |
+
## 9. MCP server (`hdc/mcp_server.py`)
|
| 339 |
+
|
| 340 |
+
JSON-RPC 2.0 over stdio, both newline-delimited and `Content-Length` framing.
|
| 341 |
+
Methods: `initialize`, `tools/list`, `tools/call`, `resources/list`,
|
| 342 |
+
`resources/read`, `ping`. Resources: `hdc://isa`, `hdc://memory`,
|
| 343 |
+
`hdc://tools`, `hdc://program`.
|
| 344 |
+
|
| 345 |
+
Hypervectors are returned as `0x`-prefixed hex strings and accepted in either
|
| 346 |
+
form, because a 10 000-bit integer does not survive a JSON number in most
|
| 347 |
+
clients.
|
| 348 |
+
|
| 349 |
+
Client configuration:
|
| 350 |
+
|
| 351 |
+
```json
|
| 352 |
+
{
|
| 353 |
+
"mcpServers": {
|
| 354 |
+
"hdcstorage": {
|
| 355 |
+
"command": "python",
|
| 356 |
+
"args": ["-m", "hdc", "mcp"],
|
| 357 |
+
"cwd": "D:/HDCSTORAGE"
|
| 358 |
+
}
|
| 359 |
+
}
|
| 360 |
+
}
|
| 361 |
+
```
|
| 362 |
+
|
| 363 |
+
```
|
| 364 |
+
$ python -m hdc mcp --selftest --d 2000
|
| 365 |
+
hdc.decode_int -> hdc.decode_int conf=1.000 42
|
| 366 |
+
hdc.similarity -> hdc.similarity conf=1.000 1000
|
| 367 |
+
hdc.bundl -> hdc.bundle conf=0.743 0x21c14d6466b6...
|
| 368 |
+
hdc.memory.search -> hdc.recall conf=1.000 9
|
| 369 |
+
```
|
| 370 |
+
|
| 371 |
+
---
|
| 372 |
+
|
| 373 |
+
## 10. HDC shader layer (`hdc/shader.py`)
|
| 374 |
+
|
| 375 |
+
An honest answer to "shaders in hyperdimensional space": a small operator set
|
| 376 |
+
on pixel records, with the exact/approximate boundary marked in the API.
|
| 377 |
+
|
| 378 |
+
**Exact.** Dimming is `shift(hv, ±k)` — multiply or divide a coded value by
|
| 379 |
+
2**k with two rotations and two XORs, *no probe and no decode*. It works
|
| 380 |
+
because the codec's bit atoms are a permutation orbit, so rotating the vector
|
| 381 |
+
rotates every atom index; measured 0 failures over 2 120 shifted values. Sign,
|
| 382 |
+
ordering, `max`, and absolute differences are bit scans plus the slice adder.
|
| 383 |
+
The depth buffer resolves visibility with that exact comparator.
|
| 384 |
+
|
| 385 |
+
**Approximate.** Discontinuity (`1 - sim`) is a cheap edge proxy; `edges()`
|
| 386 |
+
uses the exact difference by default and the two agree at 0.74 vs 0.72 on a
|
| 387 |
+
test step. Normal shading is a LUT: a 64-entry direction codebook cleaned up by
|
| 388 |
+
similarity, with Lambert terms precomputed per entry — exact within the table,
|
| 389 |
+
quantisation error between entries.
|
| 390 |
+
|
| 391 |
+
One finding worth recording: the integer codec is **symbolic**, so `sim` cannot
|
| 392 |
+
rank nearby values (20 and 21 score 0.506, about the same as 20 and 200). A
|
| 393 |
+
normal LUT built on it returned a geometrically much worse entry. Switching the
|
| 394 |
+
codebook to a **thermometer code** — value v is the bundle of atoms 0..v−1, so
|
| 395 |
+
similarity falls monotonically with |v−w| — fixed it: 16/16 stored directions
|
| 396 |
+
recovered, 68–70% exact nearest-entry agreement on random queries with mean
|
| 397 |
+
cosine deficit 0.03. The remaining misses are the same capacity law as
|
| 398 |
+
everywhere else: a 3-axis × 64-step thermometer is a 192-member bundle, whose
|
| 399 |
+
own signal is 1/√192 ≈ 0.07.
|
| 400 |
+
|
| 401 |
+
```python
|
| 402 |
+
fabric = ShaderFabric(d=2000)
|
| 403 |
+
img = step_image(fabric=fabric) # synthetic step edge
|
| 404 |
+
img.dim(1).levels()[0] # [100, 100, 100, 100, 20, ...]
|
| 405 |
+
img.edges().levels()[0] # [0, 0, 0, 160, 0, ...]
|
| 406 |
+
```
|
| 407 |
+
|
| 408 |
+
## 11. Memory modes (`hdc/memory_modes.py`)
|
| 409 |
+
|
| 410 |
+
Three ways to remember one thing, measured rather than asserted.
|
| 411 |
+
|
| 412 |
+
| mode | structure | recall | capacity |
|
| 413 |
+
|---|---|---|---|
|
| 414 |
+
| photographic | one cell per key | bit-exact or a miss | `capacity` cells, oldest evicted |
|
| 415 |
+
| holographic | one majority superposition | cleanup over a codebook | degrades with items |
|
| 416 |
+
| hybrid | photographic buckets + a trace each | exact hit, else trace | eviction keeps an approximate path |
|
| 417 |
+
|
| 418 |
+
**The finding that matters: XOR accumulation does not work.** The obvious
|
| 419 |
+
holographic memory is `trace ^= bind(key, value)`, and it is broken at any
|
| 420 |
+
size — the crosstalk after unbinding is an XOR of random terms, which is
|
| 421 |
+
itself random at every bit. Measured exact recall with an 8-symbol codebook:
|
| 422 |
+
**0/2, 0/4, 2/8 items (chance)**. Majority superposition (a bit is set when
|
| 423 |
+
most bound terms set it) gives **8/8**, and similarity decays smoothly instead
|
| 424 |
+
of collapsing: 0.502 at k=2, 0.278 at k=8, 0.135 at k=32 (d=2000).
|
| 425 |
+
|
| 426 |
+
A second finding, from the ring controller: re-observing the same key must
|
| 427 |
+
*replace* its binding, not add another vote. Piling votes on one pair makes
|
| 428 |
+
the majority represent the observation count rather than the set of keys, and
|
| 429 |
+
recall starts returning a neighbouring symbol. `record` therefore maintains
|
| 430 |
+
per-key state and the trace is `majority` over distinct keys only.
|
| 431 |
+
|
| 432 |
+
```python
|
| 433 |
+
mem = HolographicMemory(d=2000, alphabet=symbols)
|
| 434 |
+
for i in range(8):
|
| 435 |
+
mem.record("slot/%d" % i, symbols[i])
|
| 436 |
+
mem.recall("slot/3") # (symbol vector, cleanup similarity)
|
| 437 |
+
mem.confidence("slot/3") # presence signal; do not compare raw probes to the trace
|
| 438 |
+
```
|
| 439 |
+
|
| 440 |
+
## 12. Geometry, fractal and crystal shaders
|
| 441 |
+
|
| 442 |
+
`hdc/manifold.py` treats a filled voxel set as the manifold it is. Geodesic
|
| 443 |
+
distance is breadth-first **on the surface** (a full 5³ box: corner to corner
|
| 444 |
+
= 12, matching Manhattan exactly; disconnected components return `None`). The
|
| 445 |
+
Laplacian is `|Σ neighbours − k·center|` in coded arithmetic with `k` the real
|
| 446 |
+
in-bounds coordination, so border voxels are not biased by phantom neighbours.
|
| 447 |
+
Signed curvature folds that to ridge/valley/flat and shading lifts ridges and
|
| 448 |
+
drops valleys by one exact shift. It is mean curvature up to a positive factor,
|
| 449 |
+
not Gaussian curvature, and a one-voxel-thick shell reports flat normals under
|
| 450 |
+
a central difference — both stated in the module.
|
| 451 |
+
|
| 452 |
+
`hdc/fractal.py` is integer-only. The Sierpinski triangle is the closed form
|
| 453 |
+
`x & y == 0`; the Barnsley fern runs the chaos game in 1/256 fixed point; fBm
|
| 454 |
+
is hashed value noise interpolated with integer lerps and combined by `max`
|
| 455 |
+
after `dim`-ing each octave. Measured octave peaks: **254, 127, 63, 31** — the
|
| 456 |
+
halving is a checkable property, not an adjective.
|
| 457 |
+
|
| 458 |
+
`hdc/crystal.py` exposes periodicity. Grains grow by multi-source geodesic BFS
|
| 459 |
+
from lattice sites (so grain walls follow the surface); facets are the six
|
| 460 |
+
dominant-gradient normals with ties resolved to the lowest axis; and
|
| 461 |
+
`periodicity()` recovers a banded volume's spacing by autocorrelating
|
| 462 |
+
per-plane occupancy — a 2-plane lattice measures spacing 2 with score > 0.5.
|
| 463 |
+
|
| 464 |
+
## 13. Entropic 4D shaders (`hdc/shader4d.py`, `hdc/entropy.py`)
|
| 465 |
+
|
| 466 |
+
The fourth axis is a *parameter* — time, a scan index — not a spatial axis. A
|
| 467 |
+
`Volume4D` is a stack of volumes; entropy along `u` says whether each voxel is
|
| 468 |
+
static, drifting or churning, measured with the same comparator that builds
|
| 469 |
+
every other histogram (entropy of a flat field is exactly 0 bits, two values
|
| 470 |
+
1.000 bits, eight uniform bins 3.000 bits). Entropic shading composes the 3D
|
| 471 |
+
shade with an integer attenuation derived from that entropy, so flat-in-time
|
| 472 |
+
regions fall away by up to 1/4 and churning ones keep their value.
|
| 473 |
+
|
| 474 |
+
## 14. Predictive state and the ring (`hdc/state.py`, `hdc/ring.py`)
|
| 475 |
+
|
| 476 |
+
A transition is a bound pair in a holographic memory per action; prediction is
|
| 477 |
+
unbind plus cleanup; planning is beam search scored by `sim(predicted, goal)`.
|
| 478 |
+
Surprise is `1 − sim(predicted, actual)` — the temporally measured loss, and
|
| 479 |
+
the only signal that makes this an agent rather than a table.
|
| 480 |
+
|
| 481 |
+
`hdc/ring.py` closes the loop. A `HistoricLedger` is addressed **by count**, so
|
| 482 |
+
a lap-ago entry is recalled by content, not by an offset that shifts as the
|
| 483 |
+
ring wraps; eviction recycles detail into a per-phase summary that outlives it,
|
| 484 |
+
and the whole retained history is also one superposed `timeline_record`.
|
| 485 |
+
Each tick runs PAST → PRESENT → FUTURE → PAST, and an energy budget pays for
|
| 486 |
+
lookahead (`horizon = 1 + 2·energy`, clipped to 3):
|
| 487 |
+
|
| 488 |
+
* periodic world, 12 ticks — surprise falls to **0.000** after the first lap,
|
| 489 |
+
energy saturates at 1.000, mode `stable`, horizon 3;
|
| 490 |
+
* drifting world (a new state every tick) — surprise stays at **1.00**, energy
|
| 491 |
+
falls to 0.000, mode `explore`, horizon 1.
|
| 492 |
+
|
| 493 |
+
That pair is the whole claim: the controller widens when it can predict and
|
| 494 |
+
pulls in when it cannot, using only measured similarity.
|
| 495 |
+
|
| 496 |
+
## 15. Text encoder (`hdc/text.py`)
|
| 497 |
+
|
| 498 |
+
A word is a bundle of *position-bound character n-grams*, so morphology is
|
| 499 |
+
similarity before any training: `walked`~`walking` **+0.320** against
|
| 500 |
+
`walked`~`banana` **+0.085**. Position binding keeps `listen` and `silent`
|
| 501 |
+
distinct despite identical n-gram multisets. A context is a role-bound record
|
| 502 |
+
of the last `span` tokens, which is exactly the key the predictive model wants.
|
| 503 |
+
|
| 504 |
+
Measured on a 25-token corpus: order-1 contexts give **67%** top-1 next-token
|
| 505 |
+
accuracy, order-2 **83%** (the remainder is genuine two-way ambiguity). This is
|
| 506 |
+
what a text-bound ring buys: contexts that are comparable, and a fallback to
|
| 507 |
+
the nearest known form for words never trained on.
|
| 508 |
+
|
| 509 |
+
```python
|
| 510 |
+
ring = TextRing(d=2000, span=2)
|
| 511 |
+
ring.fit(tokenize(corpus))
|
| 512 |
+
ring.predict(tokenize("the cat sat on the")) # ('mat', 0.30)
|
| 513 |
+
```
|
| 514 |
+
|
| 515 |
+
## 16. Threshold as topology (`hdc/topology.py`)
|
| 516 |
+
|
| 517 |
+
The threshold is not a constant to tune; sweep it and the codebook becomes a
|
| 518 |
+
nested family of graphs `G(τ) = { {u,v} : sim(u,v) > τ }`. Component merges are
|
| 519 |
+
0-dimensional persistence, computed with a union-find over exact similarities.
|
| 520 |
+
Three consequences, each measured:
|
| 521 |
+
|
| 522 |
+
* **Self-guiding.** Otsu's criterion on the similarity histogram finds the
|
| 523 |
+
valley between members and crosstalk without a hand-set constant: τ=0.691
|
| 524 |
+
with 70% explained variance on a structured codebook, and on a flat one it
|
| 525 |
+
reports that the distribution is unimodal (no fake threshold).
|
| 526 |
+
* **Self-healing is a projection, not bit recovery.** Bundling the nearest
|
| 527 |
+
survivors returns a cell to its class prototype: 0.665 → **0.701** for severe
|
| 528 |
+
damage on the shared-core codebook, but a lightly damaged 0.960 cell is pulled
|
| 529 |
+
*down* to 0.710. On independent symbols the graph is no denser than chance
|
| 530 |
+
(0.25 vs 0.25) and the routine reports `informative = False`, changing
|
| 531 |
+
nothing. Healing presupposes geometry and drift.
|
| 532 |
+
* **Self-scaling.** The measured member/non-member gap sets the dimension:
|
| 533 |
+
`d' = d·(gap_target/gap_measured)²`. Gap 0.114 at d=2000 asks for 6 156.
|
| 534 |
+
|
| 535 |
+
```python
|
| 536 |
+
components(codebook, tau=0.1) # fragment labels at one threshold
|
| 537 |
+
otsu_threshold(similarities(codebook)) # (tau, explained variance)
|
| 538 |
+
heal(codebook, tau, flips=400) # projection repair, gated on real topology
|
| 539 |
+
```
|
| 540 |
+
|
| 541 |
+
## 17. Semantic vectors without libraries (`hdc/semantic.py`)
|
| 542 |
+
|
| 543 |
+
§15 ends with a caveat: the n-gram encoder is morphological, not semantic. The
|
| 544 |
+
recovery is a **static** embedding model, and the interesting part is that it
|
| 545 |
+
needs no runtime dependency at all.
|
| 546 |
+
|
| 547 |
+
`minishlab/potion-base-8M` (MIT) is a Model2Vec distillation of
|
| 548 |
+
`BAAI/bge-base-en-v1.5` into a token x 256-float **lookup matrix** plus a
|
| 549 |
+
WordPiece vocabulary. Static means: no attention, no layers, no activations —
|
| 550 |
+
a dict from token to vector. So the only obstacle is the file format, and a
|
| 551 |
+
`safetensors` file is an 8-byte header length, a JSON header and a raw f32
|
| 552 |
+
buffer. `struct` reads it. 29 528 tokens x 256 dims, 30 MB on disk:
|
| 553 |
+
|
| 554 |
+
```
|
| 555 |
+
hf download minishlab/potion-base-8M model.safetensors tokenizer.json config.json \
|
| 556 |
+
--local-dir models/potion-base-8M
|
| 557 |
+
python -m hdc semantic
|
| 558 |
+
```
|
| 559 |
+
|
| 560 |
+
Words become hypervectors by **sparse sign random projection** (SimHash):
|
| 561 |
+
`bit_j = 1 iff Σ_k w(j,k)·v_k > 0` with K=12 pseudorandom ±1 weights derived
|
| 562 |
+
from a hash, so nothing is stored. Measured:
|
| 563 |
+
|
| 564 |
+
| pair | model cosine | `sim(hv)` | recovered |
|
| 565 |
+
|---|---|---|---|
|
| 566 |
+
| car~automobile | +0.779 | +0.557 | +0.768 |
|
| 567 |
+
| doctor~physician | +0.677 | +0.472 | +0.675 |
|
| 568 |
+
| big~large | +0.767 | +0.531 | +0.741 |
|
| 569 |
+
| car~banana | +0.044 | +0.015 | +0.024 |
|
| 570 |
+
| dog~tectonics | −0.065 | −0.080 | −0.125 |
|
| 571 |
+
|
| 572 |
+
**Projection fidelity: Pearson r = 0.998.** The attenuation is not noise, it is
|
| 573 |
+
the arcsine law for binarised projections — `sim = (2/π)·arcsin(cos)` — and it
|
| 574 |
+
inverts, which is why the recovered column tracks the true cosine.
|
| 575 |
+
|
| 576 |
+
The projection plan depends only on `(d, sparsity, seed)`, not on the word, so
|
| 577 |
+
it is built once instead of re-hashed per token: **36.3 ms/word → 2.28 ms/word**
|
| 578 |
+
at d=2000 (16×), plan build 0.08 s, and fidelity unchanged — `car`~`automobile`
|
| 579 |
+
recovers +0.768 against a raw +0.779.
|
| 580 |
+
|
| 581 |
+
What it buys downstream, measured by fitting a `TextRing` on one wording and
|
| 582 |
+
scoring a paraphrase: **n-gram 0.50 → semantic 0.67** at span 2 (one case is
|
| 583 |
+
0.50 → 1.00). At span 1 the two encoders tie: with no shared context word the
|
| 584 |
+
key carries the substitution, but a shared word carries it for both, so
|
| 585 |
+
transfer is partial rather than free.
|
| 586 |
+
|
| 587 |
+
The enabling change is in `PredictiveModel.predict`: tables use exact keys
|
| 588 |
+
(similar keys would interfere in a majority trace), plus a **snap** step that
|
| 589 |
+
retrieves with the nearest known key vector when the exact lookup is weak.
|
| 590 |
+
Precision on the exact path, graded recall on the fallback.
|
| 591 |
+
|
| 592 |
+
Caveat worth keeping: with d=2000 and a 256-dim source, binarisation caps a
|
| 593 |
+
0.68 cosine at 0.47 similarity, so semantic margins are compressed. A wider
|
| 594 |
+
code or a thermometer encoding would recover resolution at the cost of bundle
|
| 595 |
+
capacity.
|
| 596 |
+
|
| 597 |
+
## 18. Model discovery (`hdc/hub.py`)
|
| 598 |
+
|
| 599 |
+
§17 assumes a model file exists. This layer decides *which* file, from the Hub,
|
| 600 |
+
with the standard library and a score whose every component is measured.
|
| 601 |
+
|
| 602 |
+
The installed `hf` CLI is 0.36.2 and exposes only
|
| 603 |
+
`auth/cache/download/jobs/repo/repo-files/upload/env/version` — there is no
|
| 604 |
+
`models` or `api` subcommand — so search calls the HTTPS `/api/models` endpoint
|
| 605 |
+
those commands wrap, while downloads still go through `hf download`. Two
|
| 606 |
+
endpoint behaviours cost real debugging, and both return an empty list instead
|
| 607 |
+
of an error: `author` combined with `filter` yields zero records, and
|
| 608 |
+
`filter=library:model2vec` is not a supported filter — the family has to be
|
| 609 |
+
enumerated with `author=minishlab`.
|
| 610 |
+
|
| 611 |
+
Shapes are read without downloading the weights. A `safetensors` file is an
|
| 612 |
+
8-byte little-endian header length, a JSON header, then the buffer, so two HTTP
|
| 613 |
+
Range requests return the dtypes, the shapes and the exact size: **scoring a
|
| 614 |
+
129 MB model downloads no model bytes.**
|
| 615 |
+
|
| 616 |
+
| component | weight | measured by |
|
| 617 |
+
|---|---|---|
|
| 618 |
+
| runtime_free | 0.35 | a single rank-2 tensor is a matrix the CPU can use; hundreds of tensors mean a graph engine |
|
| 619 |
+
| code | 0.30 | tokenizer coverage of a machine-code token list; whole tokens 0.7, stitched fragments 0.3 |
|
| 620 |
+
| size | 0.15 | 1.0 at ≤35 MB, 0.0 at 512 MB |
|
| 621 |
+
| dim | 0.10 | 1.0 at 256 dims — projection cost scales with d |
|
| 622 |
+
| license | 0.10 | permissive licences only |
|
| 623 |
+
|
| 624 |
+
Measured bakeoff, top of the pool (`code` is the coverage component after the
|
| 625 |
+
whole/fragment weighting, which is what separates the tables):
|
| 626 |
+
|
| 627 |
+
| repo | score | code | table | MB |
|
| 628 |
+
|---|---|---|---|---|
|
| 629 |
+
| potion-code-16M-v2 | 0.95 | 0.85 | 63 457 × 256 | 32.5 |
|
| 630 |
+
| potion-science-32M | 0.93 | 0.85 | 63 457 × 256 | 127.4 |
|
| 631 |
+
| potion-retrieval-32M | 0.85 | 0.72 | 63 091 × 512 | 129.2 |
|
| 632 |
+
| potion-base-32M | 0.85 | 0.72 | 29 528 × 512 | 129.2 |
|
| 633 |
+
| potion-base-8M | 0.85 | 0.50 | 29 528 × 256 | 30.2 |
|
| 634 |
+
| M2V_base_output | 0.85 | 0.50 | 29 528 × 256 | 30.2 |
|
| 635 |
+
| potion-base-4M | 0.80 | 0.50 | 29 528 × 128 | 15.1 |
|
| 636 |
+
| potion-base-2M | 0.78 | 0.50 | 29 528 × 64 | 7.6 |
|
| 637 |
+
|
| 638 |
+
The code table wins, which is the point of scoring coverage at all: the general
|
| 639 |
+
table is 30% smaller and would have tied on every other axis.
|
| 640 |
+
|
| 641 |
+
The transformer code models fail the only component that is a hard requirement:
|
| 642 |
+
`jinaai/jina-embeddings-v2-base-code` scores **0.46** at 321.7 MB and cannot
|
| 643 |
+
run here at all, because it needs a deep-learning runtime. Static tables win by
|
| 644 |
+
being data.
|
| 645 |
+
|
| 646 |
+
**Machine code in the same space.** `minishlab/potion-code-16M-v2` (MIT) — the
|
| 647 |
+
winner above — is one of those tables — 63 457 × 256 — but stored as **F16**, which the §17 reader
|
| 648 |
+
would have misread as f32; `hdc.semantic` now decodes F32/F16/BF16 (bf16 by
|
| 649 |
+
shifting 16 bits into the high half of an f32). Measured whole-token coverage
|
| 650 |
+
of the 42-token machine-code list (this project's own mnemonics, x86 names,
|
| 651 |
+
registers, literal forms): **79% for the code model against 29%** for
|
| 652 |
+
potion-base-8M — everything tokenisable either way, nothing missing. The code
|
| 653 |
+
model's only fragment-only tokens are `ldi jnz rbx rcx rdx rdi 0b1011 $0x10
|
| 654 |
+
[rip+4]`.
|
| 655 |
+
|
| 656 |
+
Mnemonics do not cluster semantically (`mov`~`ldi` **+0.017**) but registers do
|
| 657 |
+
(`rax`~`rbx` **+0.413**). That is the right behaviour for a symbol space:
|
| 658 |
+
distinct opcodes, related registers.
|
| 659 |
+
|
| 660 |
+
```bash
|
| 661 |
+
python -m hdc hub # search, score, recommend — no weights downloaded
|
| 662 |
+
```
|
| 663 |
+
|
| 664 |
+
Scores are cached in `models/_hub/scores.json` keyed by `repo@sha`, so a repeat
|
| 665 |
+
run is instant and does not need the network.
|
| 666 |
+
|
| 667 |
+
## 19. Idle, learning, readahead (`hdc/autonomy.py`)
|
| 668 |
+
|
| 669 |
+
The assistant is reactive: it answers what it is asked. This layer adds the
|
| 670 |
+
part that runs when nobody is asking, and its three behaviours are measured.
|
| 671 |
+
|
| 672 |
+
**Learning.** Every turn leaves an intent. The sequence of intents is learned
|
| 673 |
+
into the same holographic transition table §14 uses. Idle ticks *replay* that
|
| 674 |
+
trace, and replay is training rather than decoration: repeated evidence raises
|
| 675 |
+
a majority trace's cleanup confidence, so a cycle that is noisy on the first lap
|
| 676 |
+
sharpens on the next. Measured on a 12-utterance session: correct predictions
|
| 677 |
+
scored 0.18–0.39 after one lap, cold wrong ones 0.00–0.06.
|
| 678 |
+
|
| 679 |
+
**Readahead.** From the last intent, chain the model `horizon` steps; each hop
|
| 680 |
+
carries its own similarity plus a running `path` product, so a three-step guess
|
| 681 |
+
cannot look as certain as a one-step one. Every prediction is graded when the
|
| 682 |
+
next utterance is classified — PAST trace → PRESENT intent → FUTURE lookahead →
|
| 683 |
+
graded back into PAST — and that grading is the accuracy number.
|
| 684 |
+
|
| 685 |
+
**Autonomy, bounded.** Only `SAFE_COMMANDS` may run unasked: read-only
|
| 686 |
+
inspections, never `run`, `step`, `asm`, `boot` or `clear`. A per-tick `budget`
|
| 687 |
+
caps how many, and a confidence floor gates each one. The floor was set from a
|
| 688 |
+
measured sweep rather than from taste — cleanup over a 21-symbol codebook has
|
| 689 |
+
chance level `1/√21 = 0.218`, and correct post-lap predictions scored 0.18–0.39:
|
| 690 |
+
|
| 691 |
+
| floor | turns covered (of 24) | precision |
|
| 692 |
+
|---|---|---|
|
| 693 |
+
| 0.25 | 14 | 1.00 |
|
| 694 |
+
| 0.30 | 9 | 1.00 |
|
| 695 |
+
| 0.45 | 0 | — |
|
| 696 |
+
|
| 697 |
+
0.45 was the first guess and made autonomy idle in name only; **0.25** is the
|
| 698 |
+
default. The worst case of a wrong prediction is wasted idle cycles, because the
|
| 699 |
+
command set is read-only.
|
| 700 |
+
|
| 701 |
+
Measured, 12 utterances at d=2000 with the semantic encoder: **80% next-intent
|
| 702 |
+
accuracy, first half 33% → second half 100%**, 4 autonomous pre-executions,
|
| 703 |
+
4 answers served from the readahead cache. A hit trades idle time for response
|
| 704 |
+
time: `Assistant.chat` takes an `execute` hook, so an answer the machine already
|
| 705 |
+
computed costs a dict lookup instead of a kernel round trip.
|
| 706 |
+
|
| 707 |
+
```bash
|
| 708 |
+
python -m hdc autonomy # simulated session: accuracy, cache, savings
|
| 709 |
+
```
|
| 710 |
+
|
| 711 |
+
## 20. Filesystem in HDC (`hdc/fs.py`)
|
| 712 |
+
|
| 713 |
+
A filesystem has two jobs and they pull apart: finding things is a similarity
|
| 714 |
+
question, and giving text back exactly is not. So the split is explicit.
|
| 715 |
+
|
| 716 |
+
**Hyperdimensional:** every node's *record* is
|
| 717 |
+
`bind(role(name), name) + bind(role(kind), kind) + bind(role(parent), parent)`,
|
| 718 |
+
written into the machine's own `HDCStorage`. `search()` bundles the query's word
|
| 719 |
+
vectors and compares it to each node's own bundle — one `sim` per file, no index.
|
| 720 |
+
Measured on the seeded tree, `search("shader notes about edges")`:
|
| 721 |
+
|
| 722 |
+
| file | score |
|
| 723 |
+
|---|---|
|
| 724 |
+
| `notes-shader.txt` | **+0.492** |
|
| 725 |
+
| `README.txt` | +0.328 |
|
| 726 |
+
| `img/gradient.txt` | +0.281 |
|
| 727 |
+
| `src/hello.hdcasm` | +0.072 |
|
| 728 |
+
|
| 729 |
+
**Exact:** the text itself, addressed by path, exactly as `ring.py` keeps exact
|
| 730 |
+
ledger entries beside its holographic summaries. HDC cannot decode a bundle back
|
| 731 |
+
into text, and pretending otherwise would make "read" a lie. `HDCStorage.remove`
|
| 732 |
+
exists for this layer: deleting a file frees its cell, so `cells in use` stays
|
| 733 |
+
true (measured: 11 cells before a delete, 10 after).
|
| 734 |
+
|
| 735 |
+
`write()` creates intermediate directories, `remove()` refuses a non-empty
|
| 736 |
+
directory unless `recursive=True`, and `rename` rewrites a whole subtree's paths
|
| 737 |
+
and cell names together.
|
| 738 |
+
|
| 739 |
+
## 21. Graphics engine (`hdc/gfx.py`)
|
| 740 |
+
|
| 741 |
+
A `Surface` is a grid of cells whose brightness is a **coded hypervector**. That
|
| 742 |
+
choice is what makes the passes arithmetic rather than array copying: dimming is
|
| 743 |
+
`shift` (exact halving, no decode), blending is `max`/`min` through the fabric's
|
| 744 |
+
similarity comparator, and edge detection has two honest modes — `proxy` (one
|
| 745 |
+
`sim` per cell) and `exact` (`|a-b|` through the HDC adder).
|
| 746 |
+
|
| 747 |
+
Measured on an 80×28 surface (2 240 cells) at d=2000:
|
| 748 |
+
|
| 749 |
+
| pass | ms | cells/s |
|
| 750 |
+
|---|---|---|
|
| 751 |
+
| `flat` (gradient only) | 24.0 | 93 171 |
|
| 752 |
+
| `background` (gradient+noise+scanline+vignette) | 87.6 | 25 563 |
|
| 753 |
+
| `metal` (noise+edges+dim) | 132.5 | 16 909 |
|
| 754 |
+
| `plasma` (noise+max_blend+scanline) | 295.7 | 7 576 |
|
| 755 |
+
| `edges(exact)` | 36 ms per **128** cells | — |
|
| 756 |
+
|
| 757 |
+
The exact edge pass is the adder's table lookups per cell, so a full screen costs
|
| 758 |
+
about a second: it ships as a demo pass, not as part of the desktop's frame.
|
| 759 |
+
|
| 760 |
+
One design note worth stating. Display needs one ramp lookup per cell, and
|
| 761 |
+
decoding a coded value costs 64 similarity probes — a full decode per frame would
|
| 762 |
+
cost seconds. So each surface keeps a plain integer **shadow** used only to pick
|
| 763 |
+
the character, while the coded cells stay the authority for every operation, and
|
| 764 |
+
`verify()` decodes a sample and compares. The cache cannot drift silently; the
|
| 765 |
+
tests assert `verify()` after every shader.
|
| 766 |
+
|
| 767 |
+
## 22. The webOS (`hdc/os.py`)
|
| 768 |
+
|
| 769 |
+
80×30 character cells: a menu bar, tiled windows, a taskbar, and a compositor
|
| 770 |
+
that draws in z-order into a `Surface`. An unfocused window is darkened with
|
| 771 |
+
`dim_region` — an **exact** `shift` on the coded cells, not a multiply on a copy.
|
| 772 |
+
|
| 773 |
+
Programs are the layers this project already had, arranged for a desktop:
|
| 774 |
+
|
| 775 |
+
| app | what it is |
|
| 776 |
+
|---|---|
|
| 777 |
+
| Terminal | the kernel console, running real commands |
|
| 778 |
+
| Files | `hdc/fs.py` browser; Enter opens a directory or loads a file into the editor |
|
| 779 |
+
| Editor | appends lines to files in the same filesystem, round-tripped |
|
| 780 |
+
| IDE | `hdc/ide.py`: assemble, run, step, breakpoints |
|
| 781 |
+
| Shader Lab | `hdc/gfx.py` live in a window; space cycles shaders, `e` toggles edges |
|
| 782 |
+
| Monitor | memory buckets, filesystem size, kernel counters, readahead accuracy |
|
| 783 |
+
|
| 784 |
+
**Memory** is one store, not several: the desktop's filesystem writes into the
|
| 785 |
+
kernel's own `HDCStorage`, so `memory.buckets()` counts what is really resident —
|
| 786 |
+
measured `toolreg 16 + fs 10` on a fresh boot, with 8 166 cells free. `MemoryMap`
|
| 787 |
+
also offers `allocate(tag, n)` / `free(tag)` for apps that want to reserve cells.
|
| 788 |
+
|
| 789 |
+
Measured frame cost: **~108 ms** with three windows, ~138 ms with four, at 80×30
|
| 790 |
+
and d=2000, dominated by the wallpaper shader (`background`). That is a desktop
|
| 791 |
+
redraw, not a GPU frame — see the note in Honest limits.
|
| 792 |
+
|
| 793 |
+
In the browser it is the **DESKTOP** tab: the frame arrives as 80 character rows
|
| 794 |
+
and is painted into the CRT, keys are captured globally (`F1`–`F6`, `Tab`,
|
| 795 |
+
arrows, `Ctrl+W`, `Ctrl+arrows` to move a window), and a click is converted from
|
| 796 |
+
pixels to a character cell and sent as coordinates to focus the window under it.
|
| 797 |
+
The endpoints are `GET /api/os` and `POST /api/os/{key,text,click,launch}`.
|
| 798 |
+
|
| 799 |
+
The CRT never asks you to scroll it. `fitCrt()` derives the font size from the
|
| 800 |
+
measured monospace column width **and** from the box height for the row count on
|
| 801 |
+
screen (25 for the kernel console, 30 for a desktop frame), and `#screen` has a
|
| 802 |
+
definite height rather than a `min`/`max` pair so that measurement is stable.
|
| 803 |
+
Fitting width alone let a 30-row frame run past a 76vh box, and `paint()` then
|
| 804 |
+
scrolled to the bottom — which hid the desktop's menu bar. Measured in headless
|
| 805 |
+
Chrome painting a 30-row frame:
|
| 806 |
+
|
| 807 |
+
| window | font | rows shown | box vs content | scrolls |
|
| 808 |
+
|---|---|---|---|---|
|
| 809 |
+
| 1920×1080 | 21.8 px | 30 | 873 / 873 | no |
|
| 810 |
+
| 1440×900 | 17.3 px | 30 | 693 / 693 | no |
|
| 811 |
+
| 1280×800 | 14.8 px | 30 | 593 / 593 | no |
|
| 812 |
+
| 1024×700 | 12.6 px | 30 | 505 / 505 | no |
|
| 813 |
+
| 800×600 | 10.1 px | 30 | 405 / 405 | no |
|
| 814 |
+
| 640×480 | 8.5 px | 30 | 344 / 344 | no |
|
| 815 |
+
|
| 816 |
+
`A-` still goes below the fit on request, and `overflow:auto` stays as the
|
| 817 |
+
fallback for that: the grid is drawn whole or scrolled deliberately, never
|
| 818 |
+
silently cut. The 8 px floor is why the box has a `min-height` — at 640×480 the
|
| 819 |
+
page scrolls a line, but the terminal does not.
|
| 820 |
+
|
| 821 |
+
### The boot chain (`hdc/boot.py`)
|
| 822 |
+
|
| 823 |
+
Three phases, all rendered by the graphics engine, each doing real work:
|
| 824 |
+
|
| 825 |
+
| phase | what happens |
|
| 826 |
+
|---|---|
|
| 827 |
+
| **POST** | `hdc.x86` runs for real — IVT installed, boot sector read from LBA 0, `int 10h` teletype — and its text buffer is painted onto a phosphor surface with a scanline pass. The lines on screen are the lines the BIOS produced. |
|
| 828 |
+
| **Splash** | gradient + fractal noise + `max_blend` + scanline + vignette, with a title box and a progress bar drawn into coded cells |
|
| 829 |
+
| **AUTOEXEC** | a boot script executed against the live desktop: `MOUNT` really changes the filesystem's `cwd`, `FS.SEARCH` runs the hypervector search, `START` launches the windows, `AUTONOMY` turns on the readahead loop |
|
| 830 |
+
| desktop | the composited desktop the machine is left in |
|
| 831 |
+
|
| 832 |
+
Measured on a cold boot at d=2000: **12 frames, 611 ms of rendering, 2 220 ms
|
| 833 |
+
of playback**. The first splash attempt took 2 404 ms because it re-ran
|
| 834 |
+
`max_blend` per frame — a similarity comparator per cell, ~350 ms a frame. The
|
| 835 |
+
shader base is now rendered **once** and each animation frame copies it and
|
| 836 |
+
applies exact `shift`s for the sweep: the motion is still HDC arithmetic, and
|
| 837 |
+
the whole chain dropped to 611 ms.
|
| 838 |
+
|
| 839 |
+
AUTOEXEC's output is not decoration — this is what it prints, and it is what
|
| 840 |
+
happened:
|
| 841 |
+
|
| 842 |
+
```
|
| 843 |
+
SET DIM 2000
|
| 844 |
+
GFX.INIT 4 shaders, 2400 cells, verify ok
|
| 845 |
+
MOUNT /home/user 5 files, 5 dirs
|
| 846 |
+
FS.SEARCH /home/user/notes-shader.txt (+0.492)
|
| 847 |
+
START Terminal at 1,1 Monitor at 48,1 Files at 1,15
|
| 848 |
+
AUTONOMY readahead on, 21 intents
|
| 849 |
+
```
|
| 850 |
+
|
| 851 |
+
The browser replays the recorded frames at their own delays the first time the
|
| 852 |
+
desktop is opened (`GET /api/boot`), and **any key skips**. It is a recording of
|
| 853 |
+
work that already ran, so skipping loses nothing.
|
| 854 |
+
|
| 855 |
+
```bash
|
| 856 |
+
python -m hdc boot # the chain, frame by frame, then the timeline
|
| 857 |
+
python -m hdc os --boot # frames + report + the desktop it hands over
|
| 858 |
+
python -m hdc web --open # then open http://127.0.0.1:8765/#os
|
| 859 |
+
```
|
| 860 |
+
|
| 861 |
+
```bash
|
| 862 |
+
python -m hdc os # one frame, then memory and session report
|
| 863 |
+
python -m hdc os --tour # scripted session: open apps, run a command, screenshot
|
| 864 |
+
python -m hdc gfx # shader pass timings
|
| 865 |
+
python -m hdc gfx --demo # one of each shader, printed
|
| 866 |
+
```
|
| 867 |
+
|
| 868 |
+
## 23. Honest limits
|
| 869 |
+
|
| 870 |
+
Worth reading before quoting the headline.
|
| 871 |
+
|
| 872 |
+
* **What is genuinely hyperdimensional.** All state (program cells, registers,
|
| 873 |
+
stack, literals, addresses, tool records) is vectors; all retrieval
|
| 874 |
+
(fetch, cleanup, dispatch, recall) is similarity; binding, bundling,
|
| 875 |
+
permutation and the BitNet kernel are bit operations on those vectors.
|
| 876 |
+
* **What is conventional.** The control unit, the cleanup search itself, the
|
| 877 |
+
scalar ALU for `MUL`/`DIV`/bitwise/shift, and the ripple loop's sequencing
|
| 878 |
+
are ordinary Python. "Runs within HDC" describes where the *program and its
|
| 879 |
+
data live and how they are addressed, dispatched and arithmetically
|
| 880 |
+
combined* — not that a Python `for` loop is a hypervector.
|
| 881 |
+
* **The adder is depth-serial.** Four byte-slice stages (or 32 with the 1-bit
|
| 882 |
+
gate), each a storage lookup: correct and genuinely hyperdimensional, but
|
| 883 |
+
slower than one native add. Its interest is structural (the arithmetic is
|
| 884 |
+
data), not speed.
|
| 885 |
+
* **There is no rasteriser.** The shader layer does image *operators* —
|
| 886 |
+
dimming, edges, LUT normals, exact depth ordering for offered fragments — not
|
| 887 |
+
triangle setup, clipping or exact fragment coverage. Claiming otherwise
|
| 888 |
+
would be the failure mode this README argues against.
|
| 889 |
+
* **The shader's normal lookup is 70% exact, not 100%.** Nearest-entry
|
| 890 |
+
agreement on random directions is 68–70% with a mean cosine deficit of 0.03;
|
| 891 |
+
boundary queries and crosstalk account for the rest. The measured numbers are
|
| 892 |
+
in the module docstring rather than rounded up.
|
| 893 |
+
* **Integers are symbolic.** See the codec note above; do not interpret
|
| 894 |
+
similarity between two encoded integers as meaning.
|
| 895 |
+
* **Decode is not error-correcting.** A cell whose vector is damaged can
|
| 896 |
+
decode to a different opcode; the machine reports low confidence rather
|
| 897 |
+
than refusing to run. There is no ECC layer.
|
| 898 |
+
* **Search is linear.** `recall` and `resolve` scan all cells. This is honest
|
| 899 |
+
compute-in-memory (no index to maintain) and also O(n·d/64) per query. It is
|
| 900 |
+
fast at thousands of cells, not millions.
|
| 901 |
+
* **The codec refuses d < 1500.** Measured 0 errors at d=1500/2000/3000, but
|
| 902 |
+
60/456 at d=1000 and 401/456 at d=500: below the floor the bit atoms
|
| 903 |
+
crosstalk and decode silently returns a neighbouring value, so `int_to_hv`
|
| 904 |
+
raises rather than corrupting thresholds downstream.
|
| 905 |
+
* **Holographic capacity is a decay, not a wall.** Recall stays exact to
|
| 906 |
+
k=32 at d=2000 with a 40-symbol codebook, but similarity has already fallen
|
| 907 |
+
from 0.50 to 0.14; past that, cleanup starts choosing neighbours. The hybrid
|
| 908 |
+
mode exists so that decay is a fallback path rather than a cliff.
|
| 909 |
+
* **Curvature is mean curvature, up to scale.** Sign and relative magnitude
|
| 910 |
+
are meaningful; the absolute number is not a curvature in 1/units, and a
|
| 911 |
+
one-voxel shell reads flat under a central difference.
|
| 912 |
+
* **No rasteriser, no 4D perspective.** The 3D/4D layers do operators on voxel
|
| 913 |
+
fields — dilate, bleed, shade, project, entropy — not triangle setup or
|
| 914 |
+
occlusion beyond max-projection.
|
| 915 |
+
* **The ring is first-order and finite.** States are exact symbols from a
|
| 916 |
+
codebook, the transition model is one action deep, and beam search is
|
| 917 |
+
exponential in horizon before the beam caps it. It is an inspectable world
|
| 918 |
+
model, not an LLM.
|
| 919 |
+
* **Energy is bookkeeping.** It is a budget over measured surprise, not a
|
| 920 |
+
physical quantity; the claim is only that horizon and energy move together.
|
| 921 |
+
* **Text encoding is morphological *by default*.** Synonyms are as far apart
|
| 922 |
+
as random words unless §17's model is downloaded; with it, semantics come
|
| 923 |
+
from a distilled lookup table, and paraphrase transfer is partial (0.50 →
|
| 924 |
+
0.67), not solved.
|
| 925 |
+
* **The semantic model is optional data, not a dependency.** No file, no
|
| 926 |
+
semantics — `hdc.semantic.load()` returns `None` and callers fall back to
|
| 927 |
+
n-grams. The 30 MB is not in the repo; the MIT terms and provenance are in
|
| 928 |
+
the module docstring.
|
| 929 |
+
* **"Self-healing" is denoising, not recovery.** See §16: repair projects a
|
| 930 |
+
damaged cell onto its cluster prototype, which helps drifted cells and
|
| 931 |
+
over-smooths healthy ones. There is no error-correcting code anywhere here.
|
| 932 |
+
* **Readahead predicts the next *command*, not the next *words*.** The intent
|
| 933 |
+
set is fixed at construction, the chain is first-order, and the confidence
|
| 934 |
+
figures are cleanup similarities over a 21-symbol codebook — not
|
| 935 |
+
probabilities. It predicts the shape of a session, nothing about what the
|
| 936 |
+
user will say.
|
| 937 |
+
* **Machine-code embeddings are symbolic, not semantic.** Whole-token coverage
|
| 938 |
+
is 79% for the code model, but mnemonics are distinct rather than grouped
|
| 939 |
+
(`mov`~`ldi` +0.017): embedding a listing gives a space where opcodes are
|
| 940 |
+
separable, not an understanding of what the program does.
|
| 941 |
+
* **The desktop is character cells, not pixels.** 80×30 with one glyph per cell,
|
| 942 |
+
no antialiasing, no alpha blending, no triangles. The graphics engine shades
|
| 943 |
+
cells; it is not a rasteriser, and `edges(exact)` costs about a second per
|
| 944 |
+
screen because it is the adder, per cell.
|
| 945 |
+
* **A desktop frame costs ~108 ms**, dominated by the wallpaper shader. Drawing
|
| 946 |
+
happens on demand, so typing feels like a 1990s terminal rather than a GPU
|
| 947 |
+
desktop — which is what it is. The integer grid beside each `Surface` is a
|
| 948 |
+
display cache; `verify()` checks it against the decoded cells, and the tests
|
| 949 |
+
assert that after every shader.
|
| 950 |
+
* **One session, one desktop.** No users, no permissions, no locking, no swap.
|
| 951 |
+
The filesystem is in memory and `save()` persists node vectors, not text.
|
| 952 |
+
* **The boot is a recording.** Frames are rendered when the server starts and
|
| 953 |
+
replayed to the browser; AUTOEXEC has already run by then, so the desktop you
|
| 954 |
+
type into is the result, not a re-run. The splash animates by copying a
|
| 955 |
+
rendered base and applying exact shifts between frames, which is why it costs
|
| 956 |
+
milliseconds rather than re-running the noise pass per frame.
|
| 957 |
+
* **Topology here is 0-dimensional.** Components only — no loops, no higher
|
| 958 |
+
homology, no persistence barcodes.
|
| 959 |
+
* **Single process, single thread**, no garbage collection of cells, no
|
| 960 |
+
incremental persistence.
|
| 961 |
+
* **The MCP server holds one machine.** Tools are stateless with respect to
|
| 962 |
+
program execution; a `tools/call` does not run the loaded program.
|
| 963 |
+
|
| 964 |
+
---
|
| 965 |
+
|
| 966 |
+
## 24. Layout
|
| 967 |
+
|
| 968 |
+
```
|
| 969 |
+
hdc/
|
| 970 |
+
hypervec.py bind / bundle / permute / sim / integer codec
|
| 971 |
+
memory.py HDCStorage, content addressing, cleanup, .hdc persistence
|
| 972 |
+
isa.py opcode table, instruction encoding, compute-in-memory decode
|
| 973 |
+
adder.py hyperdimensional carry-chain full adder (ADD/SUB)
|
| 974 |
+
slicegate.py wide gates: k bits per stage, lazily materialised rows
|
| 975 |
+
shader.py image records, exact dim/edge/depth, LUT normals
|
| 976 |
+
shader3d.py volumes, morphology, normals, shaded projection
|
| 977 |
+
shader4d.py volume stacks over a fourth axis, entropy-weighted
|
| 978 |
+
entropy.py coded histograms and Shannon entropy
|
| 979 |
+
manifold.py geodesic field, Laplacian curvature, curvature shading
|
| 980 |
+
fractal.py dyadic IFS attractors, integer value-noise fBm
|
| 981 |
+
crystal.py lattice grains, facet maps, periodic banding
|
| 982 |
+
memory_modes.py photographic / holographic / hybrid memory
|
| 983 |
+
state.py predictive model, beam planner, surprise, agent loop
|
| 984 |
+
ring.py historic ledger + PAST/PRESENT/FUTURE controller
|
| 985 |
+
text.py character-ngram encoder for the ring
|
| 986 |
+
topology.py threshold graphs, Otsu guiding, repair, scaling rule
|
| 987 |
+
semantic.py static-embedding loader (safetensors via struct), SimHash
|
| 988 |
+
hub.py Hub search, Range header probe, suitability score
|
| 989 |
+
autonomy.py idle ticks, intent learning, graded readahead
|
| 990 |
+
fs.py filesystem: HDC-addressed nodes, exact contents, similarity search
|
| 991 |
+
gfx.py coded-cell surfaces, shader passes, display cache + verify
|
| 992 |
+
os.py webOS: memory map, window manager, apps, composited frames
|
| 993 |
+
boot.py boot chain: BIOS frame, shader splash, AUTOEXEC against the OS
|
| 994 |
+
assembler.py two-pass compiler + decode-back verification
|
| 995 |
+
interpreter.py Machine: fetch, decode, execute; HDC registers and stack
|
| 996 |
+
debugger.py breakpoints, stepping, trace, symbolic registers
|
| 997 |
+
net.py BitNet base: 1-bit / ternary kernels, PrototypeNet
|
| 998 |
+
agent.py tool records in HDC, similarity dispatch
|
| 999 |
+
mcp_server.py MCP JSON-RPC over stdio
|
| 1000 |
+
ide.py native binary-logic IDE
|
| 1001 |
+
demo.py end-to-end tour
|
| 1002 |
+
__main__.py python -m hdc <command>
|
| 1003 |
+
examples/ hello, logic, agent programs
|
| 1004 |
+
models/ downloaded model files (ignored by git, fetched with hf):
|
| 1005 |
+
potion-base-8M, potion-code-16M-v2
|
| 1006 |
+
tests/ over 300 unit tests (python -m hdc test)
|
| 1007 |
+
artifacts/ storage images written by the demo
|
| 1008 |
+
```
|
artifacts/demo.hdc
ADDED
|
Binary file (1.41 kB). View file
|
|
|
artifacts/demo_run.txt
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
HDCSTORAGE d=10000 bits per vector (1250 bytes)
|
| 2 |
+
|
| 3 |
+
==========================================================================
|
| 4 |
+
1. ALGEBRA - binary spatter codes on Python big integers
|
| 5 |
+
==========================================================================
|
| 6 |
+
random pair similarity mean +0.0001 sd 0.0100
|
| 7 |
+
bundle of 5 members recovered at 0.376 0.382 0.379 0.375 0.369
|
| 8 |
+
bind is self-inverse bind(bind(a,b),b) == a : True
|
| 9 |
+
bound pair vs operand 0.0114 (independent vectors only;
|
| 10 |
+
structured codes stay correlated)
|
| 11 |
+
permute preserves similarity -0.0080 == -0.0080
|
| 12 |
+
integer codec 0 -> hv -> 0
|
| 13 |
+
integer codec 1 -> hv -> 1
|
| 14 |
+
integer codec 42 -> hv -> 42
|
| 15 |
+
integer codec -7 -> hv -> -7
|
| 16 |
+
integer codec 65535 -> hv -> 65535
|
| 17 |
+
|
| 18 |
+
==========================================================================
|
| 19 |
+
2. HDC STORAGE - content-addressed cells, cleanup memory
|
| 20 |
+
==========================================================================
|
| 21 |
+
HDC storage 'demo' d=10000 cells=5/4096 symbols=5
|
| 22 |
+
0 sensor/temp 010101100111000100000000010011011001011001111111... pop=5067
|
| 23 |
+
1 sensor/humidity 000011001110010011101100100010111110111001100111... pop=4989
|
| 24 |
+
2 concept/cat 000001001110111100001100100011101000000110000101... pop=4999
|
| 25 |
+
3 concept/dog 101111111001000110110001000100101010101011011100... pop=5079
|
| 26 |
+
4 policy/retry 010010110001111110100101000011011110001111100010... pop=5003
|
| 27 |
+
|
| 28 |
+
associative recall of concept/cat:
|
| 29 |
+
concept/cat 1.0000 <- exact
|
| 30 |
+
sensor/temp 0.0160
|
| 31 |
+
|
| 32 |
+
damaged address (12/10000 bits flipped) -> sensor/humidity at confidence 0.998
|
| 33 |
+
persistence 6412 bytes, 5 cells, identical on reload: True
|
| 34 |
+
|
| 35 |
+
==========================================================================
|
| 36 |
+
3. COMPILER - text becomes instruction hypervectors
|
| 37 |
+
==========================================================================
|
| 38 |
+
cells=13 literals=3 strings=1 cell names=2
|
| 39 |
+
decode confidence min 0.36 mean 0.49
|
| 40 |
+
|
| 41 |
+
first cells of the image (disassembled from the vectors):
|
| 42 |
+
0000 LDI regs=[0] conf=0.50
|
| 43 |
+
0001 LDI regs=[1] conf=0.50
|
| 44 |
+
0002 ADD regs=[2, 0, 1] conf=0.36
|
| 45 |
+
0003 OUT regs=[2] conf=0.49
|
| 46 |
+
0004 BIND regs=[3, 0, 1] conf=0.36
|
| 47 |
+
|
| 48 |
+
cell 0 as raw bit planes (first two of 157):
|
| 49 |
+
0100011111100010111000110011101010110110010110101110101010101001
|
| 50 |
+
0111010011100110011010011010111001110111110101110101011101000100
|
| 51 |
+
|
| 52 |
+
==========================================================================
|
| 53 |
+
4. INTERPRETER - fetch from storage, decode by cleanup, execute
|
| 54 |
+
==========================================================================
|
| 55 |
+
42
|
| 56 |
+
1000
|
| 57 |
+
42
|
| 58 |
+
every value in this machine is a 10000-bit vector
|
| 59 |
+
|
| 60 |
+
status=halted steps=13 cells=30
|
| 61 |
+
trace:
|
| 62 |
+
0000 LDI R0, 5 conf=0.50 ; imm=5
|
| 63 |
+
0001 LDI R1, 37 conf=0.50 ; imm=37
|
| 64 |
+
0002 ADD R2, R0, R1 conf=0.36 ; 5 add 37 = 42
|
| 65 |
+
0003 OUT R2 conf=0.49 ; out 42
|
| 66 |
+
0004 BIND R3, R0, R1 conf=0.36 ; bind
|
| 67 |
+
0005 TOOL R5, R3, R1, hdc.unbind conf=0.36 ; tool=hdc.unbind
|
| 68 |
+
0006 SIM R6, R5, R0 conf=0.37 ; sim=1.0000
|
| 69 |
+
0007 OUT R6 conf=0.48 ; out 1000
|
| 70 |
+
0008 STORE ledger, R2 conf=0.49 ; ->ledger
|
| 71 |
+
0009 LOAD R7, ledger conf=0.49 ; <-ledger
|
| 72 |
+
0010 OUT R7 conf=0.49 ; out 42
|
| 73 |
+
0011 PRINTS "every value in this machine is a 10000-bit vector" conf=0.49 ; prints
|
| 74 |
+
0012 HALT conf=1.00 ; halt
|
| 75 |
+
|
| 76 |
+
==========================================================================
|
| 77 |
+
5. DEBUGGER - breakpoints, stepping, symbolic registers
|
| 78 |
+
==========================================================================
|
| 79 |
+
stopped: break at pc=2 (breakpoint)
|
| 80 |
+
0000 LDI R0, 5 conf=0.50
|
| 81 |
+
0001 LDI R1, 37 conf=0.50
|
| 82 |
+
=>B 0002 ADD R2, R0, R1 conf=0.36
|
| 83 |
+
0003 OUT R2 conf=0.49
|
| 84 |
+
0004 BIND R3, R0, R1 conf=0.36
|
| 85 |
+
0005 TOOL R5, R3, R1, hdc.unbind conf=0.36
|
| 86 |
+
|
| 87 |
+
42
|
| 88 |
+
REG VALUE POP SYMBOL SIM HV(head)
|
| 89 |
+
------------------------------------------------------------------------------
|
| 90 |
+
R0 5 2553 - 0.00 000000000100000001000001
|
| 91 |
+
R1 37 5011 - 0.00 101000010100100001001111
|
| 92 |
+
R2 42 5045 - 0.00 100000010100111111001101
|
| 93 |
+
R3 0 5039 - 0.00 101110101000100100001110
|
| 94 |
+
R4 0 5039 - 0.00 101110101000100100001110
|
| 95 |
+
R5 0 5039 - 0.00 101110101000100100001110
|
| 96 |
+
R6 0 5039 - 0.00 101110101000100100001110
|
| 97 |
+
R7 0 5039 - 0.00 101110101000100100001110
|
| 98 |
+
R8 0 5039 - 0.00 101110101000100100001110
|
| 99 |
+
R9 0 5039 - 0.00 101110101000100100001110
|
| 100 |
+
R10 0 5039 - 0.00 101110101000100100001110
|
| 101 |
+
R11 0 5039 - 0.00 101110101000100100001110
|
| 102 |
+
R12 0 5039 - 0.00 101110101000100100001110
|
| 103 |
+
R13 0 5039 - 0.00 101110101000100100001110
|
| 104 |
+
R14 0 5039 - 0.00 101110101000100100001110
|
| 105 |
+
R15 0 5039 - 0.00 101110101000100100001110
|
| 106 |
+
PC=4 ZF=0 STACK=0 STEPS=4 STATUS=break
|
| 107 |
+
|
| 108 |
+
trace tail:
|
| 109 |
+
0000 LDI R0, 5 conf=0.50 ; imm=5
|
| 110 |
+
0001 LDI R1, 37 conf=0.50 ; imm=37
|
| 111 |
+
0002 ADD R2, R0, R1 conf=0.36 ; 5 add 37 = 42
|
| 112 |
+
0003 OUT R2 conf=0.49 ; out 42
|
| 113 |
+
1000
|
| 114 |
+
42
|
| 115 |
+
every value in this machine is a 10000-bit vector
|
| 116 |
+
|
| 117 |
+
resumed -> halt at pc=13
|
| 118 |
+
|
| 119 |
+
==========================================================================
|
| 120 |
+
6. AGENTIC TOOLS - MCP tools stored inside HDC
|
| 121 |
+
==========================================================================
|
| 122 |
+
TOOL CAPABILITY ARGS DESCRIPTION
|
| 123 |
+
------------------------------------------------------------------------------------------------
|
| 124 |
+
agent.echo agent/plumbing/echo 1 Return the argument unchanged; useful for wiring tests.
|
| 125 |
+
bit.hamming bit/logic/distance 2 Hamming distance between two hypervectors.
|
| 126 |
+
bit.popcount bit/logic/count 1 Population count of a hypervector.
|
| 127 |
+
hdc.bind hdc/bind/construct 2 XOR-bind two hypervectors.
|
| 128 |
+
hdc.bundle hdc/bundle/construct 2 Majority-bundle two hypervectors.
|
| 129 |
+
hdc.cleanup hdc/memory/cleanup 1 Index of the stored symbol nearest to a hypervector.
|
| 130 |
+
hdc.decode_int hdc/codec/decode 1 Decode an integer hypervector.
|
| 131 |
+
hdc.encode_int hdc/codec/encode 1 Encode an integer as a hypervector.
|
| 132 |
+
hdc.permute hdc/order/rotate 2 Rotate a hypervector by k bits.
|
| 133 |
+
hdc.recall hdc/memory/search 2 Index of the stored cell most similar to a hypervector.
|
| 134 |
+
hdc.similarity hdc/bind/similarity 2 Similarity of two hypervectors, scaled to milli-units.
|
| 135 |
+
hdc.unbind hdc/bind/invert 2 Unbind: XOR is self-inverse, so this is bind.
|
| 136 |
+
mem.stat storage/memory/stats 0 Number of occupied HDC cells.
|
| 137 |
+
net.dot1bit bitnet/inference/dot 2 1-bit dot product as agreement minus disagreement.
|
| 138 |
+
|
| 139 |
+
fuzzy and capability dispatch (tool records are vectors):
|
| 140 |
+
hdc.bundl -> hdc.bundle conf=0.759
|
| 141 |
+
hdc.memory.search -> hdc.recall conf=1.000
|
| 142 |
+
bit.count -> bit.popcount conf=0.502
|
| 143 |
+
nonsense.xyz -> unresolved conf=0.024
|
| 144 |
+
|
| 145 |
+
TOOL executed from inside the ISA, per the trace above: ['tool=hdc.unbind']
|
| 146 |
+
|
| 147 |
+
==========================================================================
|
| 148 |
+
7. BITNET BASE - 1-bit and ternary inference on the same primitives
|
| 149 |
+
==========================================================================
|
| 150 |
+
xor_matches_dot True
|
| 151 |
+
xor_trials 25
|
| 152 |
+
ternary_sparsity 0.4248
|
| 153 |
+
proto_density 0.5752
|
| 154 |
+
prototype_accuracy 1.0
|
| 155 |
+
prototype_classes 4
|
| 156 |
+
prototype_dim 2000
|
| 157 |
+
d 10000
|
| 158 |
+
BitLinear 64->16 output [-2, 3, 7, -1, 13, -2, 2, -6]
|
| 159 |
+
ternary weight density 0.568
|
| 160 |
+
PrototypeNet 3 classes, held-out accuracy 1.000
|
| 161 |
+
|
| 162 |
+
==========================================================================
|
| 163 |
+
8. MCP - protocol surface over the same tool records
|
| 164 |
+
==========================================================================
|
| 165 |
+
server hdcstorage, protocol 2024-11-05, 14 tools advertised
|
| 166 |
+
round trip over JSON-RPC: encode(42) -> 0x814fcd4b03e0f70c16d6f5d3148f12b0..., decode -> 42
|
| 167 |
+
|
| 168 |
+
==========================================================================
|
| 169 |
+
9. ARTIFACTS
|
| 170 |
+
==========================================================================
|
| 171 |
+
storage image: D:\HDCSTORAGE\artifacts\demo.hdc (6412 bytes)
|
| 172 |
+
wrote D:\HDCSTORAGE\artifacts\demo.hdc
|
| 173 |
+
|
| 174 |
+
Next: python -m hdc ide (step it yourself)
|
| 175 |
+
python -m hdc mcp (attach an MCP client)
|
artifacts/ide.hdc
ADDED
|
Binary file (6.19 kB). View file
|
|
|
hdc/__init__.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""HDCSTORAGE: a hyperdimensional computer that keeps its whole state in HDC.
|
| 2 |
+
|
| 3 |
+
Layers, bottom to top:
|
| 4 |
+
|
| 5 |
+
hypervec binary spatter-code algebra (bind / bundle / permute / sim)
|
| 6 |
+
memory content-addressable HDC storage: cells, cleanup memory, .hdc file
|
| 7 |
+
isa opcode table whose instructions *are* hypervectors
|
| 8 |
+
assembler text -> hypervector instruction image
|
| 9 |
+
interpreter fetch-from-storage, decode-by-cleanup, execute
|
| 10 |
+
debugger breakpoints, single step, trace, confidence-aware disassembly
|
| 11 |
+
net BitNet-style 1-bit / ternary inference on the same primitives
|
| 12 |
+
semantic static embedding tables projected to hypervectors (no runtime)
|
| 13 |
+
hub Hub search + a measured suitability score for those tables
|
| 14 |
+
autonomy idle ticks, intent learning, graded readahead, bounded action
|
| 15 |
+
agent MCP tool registry stored inside HDC, dispatched by similarity
|
| 16 |
+
mcp_server MCP (JSON-RPC 2.0 over stdio) exposing those HDC-resident tools
|
| 17 |
+
ide native binary-logic IDE: edit, assemble, run, step, inspect bits
|
| 18 |
+
shader coded-image operators, morphology, normals, depth
|
| 19 |
+
shader3d 3D volumes, curvature and shaded projection
|
| 20 |
+
shader4d volume stacks over a fourth (parameter) axis, entropy-weighted
|
| 21 |
+
entropy histogram and Shannon entropy from the bit-scan comparator
|
| 22 |
+
manifold geodesic distance, Laplacian curvature, curvature shading
|
| 23 |
+
fractal dyadic IFS attractors and integer value-noise fBm
|
| 24 |
+
crystal lattice grains, facet maps and periodic banding
|
| 25 |
+
memory_modes photographic, holographic and hybrid memory
|
| 26 |
+
state predictive state model, beam planner, surprise
|
| 27 |
+
ring historic ledger and the PAST/PRESENT/FUTURE controller
|
| 28 |
+
text character-ngram word encoder for the ring
|
| 29 |
+
semantic static-embedding word vectors, parsed without libraries
|
| 30 |
+
topology threshold graphs, Otsu guiding, repair, scaling rule
|
| 31 |
+
|
| 32 |
+
Standard library only. Run ``python -m hdc demo`` for the end-to-end tour.
|
| 33 |
+
"""
|
| 34 |
+
|
| 35 |
+
from .hypervec import (
|
| 36 |
+
DEFAULT_DIM,
|
| 37 |
+
bind,
|
| 38 |
+
bundle,
|
| 39 |
+
hvc,
|
| 40 |
+
int_to_hv,
|
| 41 |
+
hv_to_int,
|
| 42 |
+
hv_from_bits,
|
| 43 |
+
hv_to_bits,
|
| 44 |
+
permute,
|
| 45 |
+
sim,
|
| 46 |
+
)
|
| 47 |
+
from .memory import HDCStorage
|
| 48 |
+
from .isa import OPCODES, decode_instr, disassemble, encode_instr, opcode_table_text
|
| 49 |
+
from .assembler import Program, assemble, assembler
|
| 50 |
+
from .interpreter import Machine
|
| 51 |
+
from .debugger import Debugger
|
| 52 |
+
from .agent import ToolRegistry
|
| 53 |
+
from .adder import HyperAdder
|
| 54 |
+
from .shader import ShaderFabric
|
| 55 |
+
from .memory_modes import HolographicMemory, HybridMemory, PhotographicMemory
|
| 56 |
+
from .state import PredictiveModel, StateAgent
|
| 57 |
+
from .ring import HistoricLedger, RingController
|
| 58 |
+
from .text import TextRing, encode_word
|
| 59 |
+
from .semantic import SemanticEncoder
|
| 60 |
+
|
| 61 |
+
__all__ = [
|
| 62 |
+
"Debugger",
|
| 63 |
+
"ToolRegistry",
|
| 64 |
+
"HyperAdder",
|
| 65 |
+
"ShaderFabric",
|
| 66 |
+
"PhotographicMemory",
|
| 67 |
+
"HolographicMemory",
|
| 68 |
+
"HybridMemory",
|
| 69 |
+
"PredictiveModel",
|
| 70 |
+
"StateAgent",
|
| 71 |
+
"HistoricLedger",
|
| 72 |
+
"RingController",
|
| 73 |
+
"TextRing",
|
| 74 |
+
"encode_word",
|
| 75 |
+
"SemanticEncoder",
|
| 76 |
+
"DEFAULT_DIM",
|
| 77 |
+
"bind",
|
| 78 |
+
"bundle",
|
| 79 |
+
"hvc",
|
| 80 |
+
"int_to_hv",
|
| 81 |
+
"hv_to_int",
|
| 82 |
+
"permute",
|
| 83 |
+
"sim",
|
| 84 |
+
"HDCStorage",
|
| 85 |
+
"OPCODES",
|
| 86 |
+
"Program",
|
| 87 |
+
"assemble",
|
| 88 |
+
"assembler",
|
| 89 |
+
"disassemble",
|
| 90 |
+
"opcode_table_text",
|
| 91 |
+
"encode_instr",
|
| 92 |
+
"decode_instr",
|
| 93 |
+
"Machine",
|
| 94 |
+
]
|
| 95 |
+
|
| 96 |
+
__version__ = "1.0.0"
|
hdc/__main__.py
ADDED
|
@@ -0,0 +1,403 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Command line entry point: ``python -m hdc <command>``."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import sys
|
| 6 |
+
|
| 7 |
+
from .hypervec import DEFAULT_DIM
|
| 8 |
+
|
| 9 |
+
USAGE = """\
|
| 10 |
+
HDCSTORAGE - hyperdimensional storage as a computer
|
| 11 |
+
|
| 12 |
+
python -m hdc run <file.hdcasm> assemble and execute a program
|
| 13 |
+
python -m hdc asm <file.hdcasm> assemble and show the decode-back report
|
| 14 |
+
python -m hdc ide [file] native binary-logic IDE (REPL)
|
| 15 |
+
python -m hdc mcp [--selftest] MCP server on stdio (JSON-RPC 2.0)
|
| 16 |
+
python -m hdc demo end-to-end tour of every layer
|
| 17 |
+
python -m hdc adder [a b] run one addition through the HDC full adder
|
| 18 |
+
python -m hdc shader HDC image pipeline: dim, edges, depth, normals
|
| 19 |
+
python -m hdc memory photographic / holographic / hybrid recall
|
| 20 |
+
python -m hdc geometry geodesic, curvature and manifold shading
|
| 21 |
+
python -m hdc fractal dyadic IFS and fBm detail
|
| 22 |
+
python -m hdc crystal lattice grains, facets and banding
|
| 23 |
+
python -m hdc state predictive state: plan, act, surprise
|
| 24 |
+
python -m hdc ring PAST -> PRESENT -> FUTURE ring controller
|
| 25 |
+
python -m hdc text character-ngram encoder bound to the ring
|
| 26 |
+
python -m hdc topology threshold-quantised topology: guide, heal, scale
|
| 27 |
+
python -m hdc semantic static-embedding word vectors (needs the model)
|
| 28 |
+
python -m hdc web [--port 8765] CRT WebUI: terminal, diagnostics, assistant
|
| 29 |
+
python -m hdc boot BIOS POST + boot sector + kernel handoff
|
| 30 |
+
python -m hdc chat <text...> one-shot assistant turn
|
| 31 |
+
python -m hdc net BitNet layer self-check
|
| 32 |
+
python -m hdc os [--tour|--boot] the webOS: desktop, windows, apps, memory map
|
| 33 |
+
python -m hdc boot the boot chain: POST, shader splash, AUTOEXEC
|
| 34 |
+
python -m hdc gfx [--demo] graphics engine: coded cells and shader passes
|
| 35 |
+
python -m hdc hub [--offline] score embedding tables from the Hub search API
|
| 36 |
+
python -m hdc autonomy idle + learning + readahead, on a simulated session
|
| 37 |
+
python -m hdc assist [--autonomous] chat with the assistant (idle ticks between turns)
|
| 38 |
+
|
| 39 |
+
autonomy, assist, os and gfx default to --d 2000: that is the codec floor the
|
| 40 |
+
documented numbers are taken at, and a desktop frame costs twice as much at
|
| 41 |
+
the machine's 10000
|
| 42 |
+
python -m hdc isa instruction set reference
|
| 43 |
+
python -m hdc tools the HDC-resident tool registry
|
| 44 |
+
python -m hdc test run the unit test suite
|
| 45 |
+
|
| 46 |
+
every command accepts --d <int> hypervector dimensionality (default 10000)
|
| 47 |
+
"""
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _split_dim(argv: list[str]) -> tuple[int, list[str]]:
|
| 51 |
+
d = DEFAULT_DIM
|
| 52 |
+
rest: list[str] = []
|
| 53 |
+
i = 0
|
| 54 |
+
while i < len(argv):
|
| 55 |
+
if argv[i] == "--d" and i + 1 < len(argv):
|
| 56 |
+
d = int(argv[i + 1])
|
| 57 |
+
i += 2
|
| 58 |
+
continue
|
| 59 |
+
rest.append(argv[i])
|
| 60 |
+
i += 1
|
| 61 |
+
return d, rest
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _assist(d: int, argv: list[str]) -> int:
|
| 65 |
+
"""``python -m hdc assist [--autonomous] [--idle N] [--simulate]``.
|
| 66 |
+
|
| 67 |
+
The interactive path runs an idle tick between turns, which is where the
|
| 68 |
+
readahead work happens; ``--simulate`` runs the scripted session instead so
|
| 69 |
+
the same numbers can be seen without typing anything.
|
| 70 |
+
"""
|
| 71 |
+
from .assistant import Assistant
|
| 72 |
+
from .autonomy import AutonomousMode, report, simulate
|
| 73 |
+
from .semantic import load
|
| 74 |
+
from .terminal import Kernel
|
| 75 |
+
|
| 76 |
+
if "--simulate" in argv:
|
| 77 |
+
print(report(d=d))
|
| 78 |
+
return 0
|
| 79 |
+
idle = 1
|
| 80 |
+
if "--idle" in argv:
|
| 81 |
+
idle = int(argv[argv.index("--idle") + 1])
|
| 82 |
+
encoder = load(d=d)
|
| 83 |
+
kernel = Kernel(d=d)
|
| 84 |
+
assistant = Assistant(kernel, d=d, encoder=encoder)
|
| 85 |
+
mode = AutonomousMode(assistant, d=d, enabled="--autonomous" in argv)
|
| 86 |
+
print("HDCSTORAGE assistant (encoder=%s, autonomous=%s)\n"
|
| 87 |
+
% ("semantic" if encoder else "n-gram",
|
| 88 |
+
"on" if mode.enabled else "off - pass --autonomous"))
|
| 89 |
+
mode.idle(idle)
|
| 90 |
+
while True:
|
| 91 |
+
try:
|
| 92 |
+
line = input("you> ").strip()
|
| 93 |
+
except (EOFError, KeyboardInterrupt):
|
| 94 |
+
print()
|
| 95 |
+
break
|
| 96 |
+
if not line:
|
| 97 |
+
continue
|
| 98 |
+
if line in ("exit", "quit"):
|
| 99 |
+
break
|
| 100 |
+
record = mode.chat(line)
|
| 101 |
+
print("\n%s" % record.get("reply", ""))
|
| 102 |
+
if record.get("output"):
|
| 103 |
+
print(record["output"].rstrip())
|
| 104 |
+
if record.get("cached"):
|
| 105 |
+
print("[readahead] answered from the predicted cache (saved %.1f ms)"
|
| 106 |
+
% record.get("saved_ms", 0.0))
|
| 107 |
+
if record.get("graded", {}).get("predicted"):
|
| 108 |
+
verdict = record["graded"]
|
| 109 |
+
print("[predict] %s -> %s %s (%.2f)"
|
| 110 |
+
% (verdict["predicted"], verdict["actual"],
|
| 111 |
+
"hit" if verdict["hit"] else "miss", verdict["confidence"]))
|
| 112 |
+
if mode.readahead:
|
| 113 |
+
print("[ahead] " + " -> ".join(
|
| 114 |
+
"%s(%.2f)" % (item["intent"], item["confidence"])
|
| 115 |
+
for item in mode.readahead))
|
| 116 |
+
mode.idle(idle)
|
| 117 |
+
print()
|
| 118 |
+
print("\n" + mode.report())
|
| 119 |
+
return 0
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def main(argv: list[str] | None = None) -> int:
|
| 123 |
+
argv = list(sys.argv[1:] if argv is None else argv)
|
| 124 |
+
if not argv or argv[0] in ("-h", "--help", "help"):
|
| 125 |
+
print(USAGE)
|
| 126 |
+
return 0
|
| 127 |
+
cmd, rest = argv[0], argv[1:]
|
| 128 |
+
explicit_dim = "--d" in rest
|
| 129 |
+
d, rest = _split_dim(rest)
|
| 130 |
+
# The assistant-facing commands measure the codec, not the machine, so they
|
| 131 |
+
# default to its floor. Measured: the same simulated session scores 80% at
|
| 132 |
+
# d=2000 and 70% at d=10000 — a fixed 21-symbol cleanup codebook is diluted
|
| 133 |
+
# by extra bits, so the ten-thousand-bit default would misreport the layer.
|
| 134 |
+
if cmd in ("autonomy", "assist", "os", "gfx") and not explicit_dim:
|
| 135 |
+
d = 2000
|
| 136 |
+
|
| 137 |
+
if cmd == "version":
|
| 138 |
+
from . import __version__
|
| 139 |
+
|
| 140 |
+
print("hdcstorage %s" % __version__)
|
| 141 |
+
return 0
|
| 142 |
+
|
| 143 |
+
if cmd == "isa":
|
| 144 |
+
from .isa import opcode_table_text
|
| 145 |
+
|
| 146 |
+
print(opcode_table_text())
|
| 147 |
+
return 0
|
| 148 |
+
|
| 149 |
+
if cmd == "shader":
|
| 150 |
+
from .shader import demo_report
|
| 151 |
+
from .shader3d import demo_report_3d
|
| 152 |
+
from .shader4d import demo_report_4d
|
| 153 |
+
|
| 154 |
+
print(demo_report(d=d))
|
| 155 |
+
print()
|
| 156 |
+
print(demo_report_3d(d=d))
|
| 157 |
+
print()
|
| 158 |
+
print(demo_report_4d(d=d))
|
| 159 |
+
return 0
|
| 160 |
+
|
| 161 |
+
if cmd == "memory":
|
| 162 |
+
from .memory_modes import demo_report
|
| 163 |
+
|
| 164 |
+
print(demo_report(d=d))
|
| 165 |
+
return 0
|
| 166 |
+
|
| 167 |
+
if cmd == "geometry":
|
| 168 |
+
from .manifold import report
|
| 169 |
+
from .shader3d import cube_volume
|
| 170 |
+
|
| 171 |
+
print(report(cube_volume(5, 180), seed=(2, 2, 2)))
|
| 172 |
+
return 0
|
| 173 |
+
|
| 174 |
+
if cmd == "fractal":
|
| 175 |
+
from .fractal import report
|
| 176 |
+
|
| 177 |
+
print(report())
|
| 178 |
+
return 0
|
| 179 |
+
|
| 180 |
+
if cmd == "crystal":
|
| 181 |
+
from .crystal import report
|
| 182 |
+
from .shader3d import cube_volume
|
| 183 |
+
|
| 184 |
+
print(report(cube_volume(5, 180)))
|
| 185 |
+
return 0
|
| 186 |
+
|
| 187 |
+
if cmd == "state":
|
| 188 |
+
from .state import report
|
| 189 |
+
|
| 190 |
+
print(report(d=d))
|
| 191 |
+
return 0
|
| 192 |
+
|
| 193 |
+
if cmd == "ring":
|
| 194 |
+
from .ring import report
|
| 195 |
+
|
| 196 |
+
print(report(d=d))
|
| 197 |
+
return 0
|
| 198 |
+
|
| 199 |
+
if cmd == "text":
|
| 200 |
+
from .text import report
|
| 201 |
+
|
| 202 |
+
print(report(d=d))
|
| 203 |
+
return 0
|
| 204 |
+
|
| 205 |
+
if cmd == "topology":
|
| 206 |
+
from .topology import report
|
| 207 |
+
|
| 208 |
+
print(report(d=d))
|
| 209 |
+
return 0
|
| 210 |
+
|
| 211 |
+
if cmd == "semantic":
|
| 212 |
+
from .semantic import report
|
| 213 |
+
|
| 214 |
+
print(report(d=d))
|
| 215 |
+
return 0
|
| 216 |
+
|
| 217 |
+
if cmd == "web":
|
| 218 |
+
from .webui import main as web_main
|
| 219 |
+
|
| 220 |
+
# the UI picks its own default dimension (2000, the codec floor) so a
|
| 221 |
+
# bare `python -m hdc web` responds immediately; --d still overrides
|
| 222 |
+
return web_main(rest)
|
| 223 |
+
|
| 224 |
+
if cmd == "boot":
|
| 225 |
+
from .terminal import Kernel
|
| 226 |
+
from .x86 import Boot
|
| 227 |
+
|
| 228 |
+
machine = Boot()
|
| 229 |
+
for line in machine.post():
|
| 230 |
+
if line.strip():
|
| 231 |
+
print(line)
|
| 232 |
+
result = machine.boot()
|
| 233 |
+
for line in result["screen"]:
|
| 234 |
+
if line.strip():
|
| 235 |
+
print(line)
|
| 236 |
+
print("\n[handoff: %s after %d instructions]"
|
| 237 |
+
% (result["reason"], result["instructions"]))
|
| 238 |
+
kernel = Kernel(d=d)
|
| 239 |
+
kernel.boot(machine)
|
| 240 |
+
print("\n[kernel] cells=%d, prompt: hdc>" % len(kernel.storage))
|
| 241 |
+
return 0
|
| 242 |
+
|
| 243 |
+
if cmd == "chat":
|
| 244 |
+
from .assistant import Assistant
|
| 245 |
+
from .semantic import load as load_semantic
|
| 246 |
+
from .terminal import Kernel
|
| 247 |
+
|
| 248 |
+
text = " ".join(rest) or "help"
|
| 249 |
+
kernel = Kernel(d=d)
|
| 250 |
+
assistant = Assistant(kernel, d=d, encoder=load_semantic(d))
|
| 251 |
+
record = assistant.chat(text)
|
| 252 |
+
print("intent=%s confidence=%.2f" % (record["intent"] or "(none)", record["confidence"]))
|
| 253 |
+
print(record["output"] or record["reply"])
|
| 254 |
+
return 0
|
| 255 |
+
|
| 256 |
+
if cmd == "adder":
|
| 257 |
+
from .adder import HyperAdder, selfcheck, wrap
|
| 258 |
+
|
| 259 |
+
if rest and not rest[0].startswith("-"):
|
| 260 |
+
a = int(rest[0])
|
| 261 |
+
b = int(rest[1]) if len(rest) > 1 else 0
|
| 262 |
+
adder = HyperAdder(d=d)
|
| 263 |
+
# run it first: the trace is a record of what the circuit did
|
| 264 |
+
total = adder.decode(adder.add(a, b))
|
| 265 |
+
print(adder.table_text())
|
| 266 |
+
print()
|
| 267 |
+
print("ripple for %d + %d:" % (a, b))
|
| 268 |
+
print(adder.trace_text(8))
|
| 269 |
+
print(" ... %d more stages" % (adder.width - 8))
|
| 270 |
+
print("\nsum = %d (expected %d) -> %s"
|
| 271 |
+
% (total, wrap(a + b), "ok" if total == wrap(a + b) else "MISMATCH"))
|
| 272 |
+
return 0
|
| 273 |
+
for key, value in selfcheck().items():
|
| 274 |
+
print("%-22s %s" % (key, value))
|
| 275 |
+
return 0
|
| 276 |
+
|
| 277 |
+
if cmd == "net":
|
| 278 |
+
from .net import selfcheck
|
| 279 |
+
|
| 280 |
+
for key, value in selfcheck().items():
|
| 281 |
+
print("%-20s %s" % (key, value))
|
| 282 |
+
return 0
|
| 283 |
+
|
| 284 |
+
if cmd == "hub":
|
| 285 |
+
from .hub import report as hub_report
|
| 286 |
+
|
| 287 |
+
print(hub_report(offline="--offline" in rest, refresh="--refresh" in rest))
|
| 288 |
+
return 0
|
| 289 |
+
|
| 290 |
+
if cmd == "gfx":
|
| 291 |
+
from .gfx import demo, report as gfx_report
|
| 292 |
+
|
| 293 |
+
print(demo(d=d) if "--demo" in rest else gfx_report(d=d))
|
| 294 |
+
return 0
|
| 295 |
+
|
| 296 |
+
if cmd == "boot":
|
| 297 |
+
from .boot import demo
|
| 298 |
+
|
| 299 |
+
print(demo(d))
|
| 300 |
+
return 0
|
| 301 |
+
|
| 302 |
+
if cmd == "os":
|
| 303 |
+
from .boot import BootSequence, render
|
| 304 |
+
from .os import Desktop, boot, tour
|
| 305 |
+
|
| 306 |
+
if "--boot" in rest:
|
| 307 |
+
desktop = Desktop(d=d)
|
| 308 |
+
sequence = BootSequence(desktop, d=d)
|
| 309 |
+
sequence.run()
|
| 310 |
+
print(render(sequence.frames))
|
| 311 |
+
print()
|
| 312 |
+
print(sequence.report())
|
| 313 |
+
print()
|
| 314 |
+
print("\n".join(desktop.frame()))
|
| 315 |
+
return 0
|
| 316 |
+
if "--tour" in rest:
|
| 317 |
+
print(tour(d))
|
| 318 |
+
return 0
|
| 319 |
+
desktop = boot(d)
|
| 320 |
+
print("\n".join(desktop.frame()))
|
| 321 |
+
print()
|
| 322 |
+
print(desktop.report())
|
| 323 |
+
print(desktop.memory.report())
|
| 324 |
+
return 0
|
| 325 |
+
|
| 326 |
+
if cmd == "autonomy":
|
| 327 |
+
from .autonomy import report as autonomy_report
|
| 328 |
+
|
| 329 |
+
print(autonomy_report(d=d))
|
| 330 |
+
return 0
|
| 331 |
+
|
| 332 |
+
if cmd == "assist":
|
| 333 |
+
return _assist(d, rest)
|
| 334 |
+
|
| 335 |
+
if cmd == "tools":
|
| 336 |
+
from .agent import ToolRegistry
|
| 337 |
+
|
| 338 |
+
reg = ToolRegistry(d=d)
|
| 339 |
+
print(reg.specs_text())
|
| 340 |
+
print()
|
| 341 |
+
print(reg.memory_report())
|
| 342 |
+
return 0
|
| 343 |
+
|
| 344 |
+
if cmd == "test":
|
| 345 |
+
import unittest
|
| 346 |
+
|
| 347 |
+
loader = unittest.TestLoader()
|
| 348 |
+
suite = loader.discover("tests", top_level_dir=".")
|
| 349 |
+
result = unittest.TextTestRunner(verbosity=2).run(suite)
|
| 350 |
+
return 0 if result.wasSuccessful() else 1
|
| 351 |
+
|
| 352 |
+
if cmd == "demo":
|
| 353 |
+
from .demo import main as demo_main
|
| 354 |
+
|
| 355 |
+
return demo_main(["--d", str(d)] + rest)
|
| 356 |
+
|
| 357 |
+
if cmd == "mcp":
|
| 358 |
+
from .mcp_server import main as mcp_main
|
| 359 |
+
|
| 360 |
+
return mcp_main(["--d", str(d)] + rest)
|
| 361 |
+
|
| 362 |
+
if cmd == "ide":
|
| 363 |
+
from .ide import main as ide_main
|
| 364 |
+
|
| 365 |
+
return ide_main(["--d", str(d)] + rest)
|
| 366 |
+
|
| 367 |
+
if cmd in ("run", "asm"):
|
| 368 |
+
if not rest:
|
| 369 |
+
print("usage: python -m hdc %s <file.hdcasm>" % cmd, file=sys.stderr)
|
| 370 |
+
return 2
|
| 371 |
+
from .agent import ToolRegistry
|
| 372 |
+
from .assembler import assemble, roundtrip_report
|
| 373 |
+
from .interpreter import Machine
|
| 374 |
+
from .memory import HDCStorage
|
| 375 |
+
|
| 376 |
+
with open(rest[0], "r", encoding="utf-8") as fh:
|
| 377 |
+
source = fh.read()
|
| 378 |
+
program = assemble(source, d=d)
|
| 379 |
+
if cmd == "asm":
|
| 380 |
+
print(roundtrip_report(program))
|
| 381 |
+
print()
|
| 382 |
+
print(program.listing_text())
|
| 383 |
+
return 0
|
| 384 |
+
storage = HDCStorage(d=d, label=rest[0])
|
| 385 |
+
machine = Machine(
|
| 386 |
+
storage=storage,
|
| 387 |
+
d=d,
|
| 388 |
+
tools=ToolRegistry(storage=storage, d=d),
|
| 389 |
+
echo=print,
|
| 390 |
+
)
|
| 391 |
+
machine.load(program)
|
| 392 |
+
result = machine.run()
|
| 393 |
+
print("-- %s --" % result["status"])
|
| 394 |
+
print("steps=%d cells=%d output=%d" % (result["steps"], len(storage), len(result["output"])))
|
| 395 |
+
return 0 if result["status"] == "halted" else 1
|
| 396 |
+
|
| 397 |
+
print("unknown command %r\n" % cmd, file=sys.stderr)
|
| 398 |
+
print(USAGE, file=sys.stderr)
|
| 399 |
+
return 2
|
| 400 |
+
|
| 401 |
+
|
| 402 |
+
if __name__ == "__main__":
|
| 403 |
+
raise SystemExit(main())
|
hdc/__pycache__/__init__.cpython-315.pyc
ADDED
|
Binary file (3.88 kB). View file
|
|
|
hdc/__pycache__/__main__.cpython-315.pyc
ADDED
|
Binary file (17.6 kB). View file
|
|
|
hdc/__pycache__/adder.cpython-315.pyc
ADDED
|
Binary file (24.8 kB). View file
|
|
|
hdc/__pycache__/agent.cpython-315.pyc
ADDED
|
Binary file (31.3 kB). View file
|
|
|
hdc/__pycache__/assembler.cpython-315.pyc
ADDED
|
Binary file (21.4 kB). View file
|
|
|
hdc/__pycache__/assistant.cpython-315.pyc
ADDED
|
Binary file (12.2 kB). View file
|
|
|
hdc/__pycache__/autonomy.cpython-315.pyc
ADDED
|
Binary file (26.5 kB). View file
|
|
|
hdc/__pycache__/boot.cpython-315.pyc
ADDED
|
Binary file (20.2 kB). View file
|
|
|
hdc/__pycache__/debugger.cpython-315.pyc
ADDED
|
Binary file (17.6 kB). View file
|
|
|
hdc/__pycache__/entropy.cpython-315.pyc
ADDED
|
Binary file (7.92 kB). View file
|
|
|
hdc/__pycache__/fs.cpython-315.pyc
ADDED
|
Binary file (30.3 kB). View file
|
|
|
hdc/__pycache__/gfx.cpython-315.pyc
ADDED
|
Binary file (34.5 kB). View file
|
|
|
hdc/__pycache__/hypervec.cpython-315.pyc
ADDED
|
Binary file (23 kB). View file
|
|
|
hdc/__pycache__/ide.cpython-315.pyc
ADDED
|
Binary file (34.6 kB). View file
|
|
|
hdc/__pycache__/interpreter.cpython-315.pyc
ADDED
|
Binary file (30.3 kB). View file
|
|
|
hdc/__pycache__/isa.cpython-315.pyc
ADDED
|
Binary file (22.4 kB). View file
|
|
|
hdc/__pycache__/memory.cpython-315.pyc
ADDED
|
Binary file (21.9 kB). View file
|
|
|
hdc/__pycache__/memory_modes.cpython-315.pyc
ADDED
|
Binary file (28.1 kB). View file
|
|
|
hdc/__pycache__/os.cpython-315.pyc
ADDED
|
Binary file (55.1 kB). View file
|
|
|
hdc/__pycache__/ring.cpython-315.pyc
ADDED
|
Binary file (24.3 kB). View file
|
|
|
hdc/__pycache__/semantic.cpython-315.pyc
ADDED
|
Binary file (26.6 kB). View file
|
|
|
hdc/__pycache__/shader.cpython-315.pyc
ADDED
|
Binary file (45.7 kB). View file
|
|
|
hdc/__pycache__/slicegate.cpython-315.pyc
ADDED
|
Binary file (18.5 kB). View file
|
|
|
hdc/__pycache__/state.cpython-315.pyc
ADDED
|
Binary file (19.5 kB). View file
|
|
|
hdc/__pycache__/terminal.cpython-315.pyc
ADDED
|
Binary file (23.1 kB). View file
|
|
|
hdc/__pycache__/text.cpython-315.pyc
ADDED
|
Binary file (14.6 kB). View file
|
|
|
hdc/__pycache__/webui.cpython-315.pyc
ADDED
|
Binary file (54.9 kB). View file
|
|
|
hdc/__pycache__/x86.cpython-315.pyc
ADDED
|
Binary file (37.2 kB). View file
|
|
|
hdc/adder.py
ADDED
|
@@ -0,0 +1,395 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""A hyperdimensional full adder.
|
| 2 |
+
|
| 3 |
+
The integer ALU used to decode a register to a Python ``int``, add, and
|
| 4 |
+
re-encode. This module removes that step for ``ADD`` and ``SUB``: the sum is
|
| 5 |
+
computed by a ripple-carry circuit whose primitive is *binding plus cleanup*,
|
| 6 |
+
which is the only thing a hypervector machine has.
|
| 7 |
+
|
| 8 |
+
The circuit
|
| 9 |
+
-----------
|
| 10 |
+
A number is a superposition of bit-position atoms (``int_to_hv``). Reading a
|
| 11 |
+
bit is a similarity probe; that is unavoidable, and it is what a bit line is
|
| 12 |
+
in hardware:
|
| 13 |
+
|
| 14 |
+
bit_i(n) == sim(hv, bind(NUM, perm(BITBASE, i))) >= CODEC_THRESHOLD
|
| 15 |
+
|
| 16 |
+
A full-adder row is the classic 8-entry truth table, and each row is stored as
|
| 17 |
+
a record in HDC storage:
|
| 18 |
+
|
| 19 |
+
row = bundle([
|
| 20 |
+
bind(role("key"), key_of(a, b, c)),
|
| 21 |
+
bind(role("sum"), atom("s", a ^ b ^ c)),
|
| 22 |
+
bind(role("carry"), atom("k", majority(a, b, c))),
|
| 23 |
+
])
|
| 24 |
+
|
| 25 |
+
Evaluating one stage is a *content-addressable lookup*: form the query key,
|
| 26 |
+
and ask storage which row it resembles. Each stage reports the similarity it
|
| 27 |
+
resolved at, so a marginal carry chain is visible rather than assumed.
|
| 28 |
+
|
| 29 |
+
Why the key needs one atom pair **per slot**
|
| 30 |
+
-------------------------------------------
|
| 31 |
+
XOR is associative and commutative, so a key built as
|
| 32 |
+
``bind(bind(atom_a, atom_b), atom_c)`` only ever sees
|
| 33 |
+
``atom_a ^ atom_b ^ atom_c``. With a single shared pair for zero and one that
|
| 34 |
+
is *parity*: ``1^1^1`` and ``1^0^0`` both collapse to the same vector, and the
|
| 35 |
+
8-row table degenerates to 2 distinguishable entries. Tagging each input with
|
| 36 |
+
a role before the XOR does not help, for the same reason — the role constants
|
| 37 |
+
cancel into a fixed mask.
|
| 38 |
+
|
| 39 |
+
Giving each slot its own atom pair (``gate/atom/a/0``, ``gate/atom/a/1``, ...)
|
| 40 |
+
makes the eight keys XOR combinations of *independently chosen* constants, so
|
| 41 |
+
they are pairwise near-orthogonal and cleanup resolves them exactly. That is
|
| 42 |
+
the same distinction as using distinct address lines rather than one shared
|
| 43 |
+
bit line.
|
| 44 |
+
|
| 45 |
+
Subtraction is addition of the complement with carry-in 1 — two's complement,
|
| 46 |
+
so there is no sign special case anywhere: ``a - b == a + ~b + 1`` and the
|
| 47 |
+
32-bit wrap is whatever falls out of dropping the final carry.
|
| 48 |
+
|
| 49 |
+
Cost: 32 stages x (8 similarity probes + 4 atom probes) at d=10 000 is roughly
|
| 50 |
+
0.9 ms per operation, versus ~0.05 ms for the old decode-and-add. That is the
|
| 51 |
+
price of keeping the arithmetic in the vector domain, and it is stated in the
|
| 52 |
+
README rather than hidden.
|
| 53 |
+
"""
|
| 54 |
+
|
| 55 |
+
from __future__ import annotations
|
| 56 |
+
|
| 57 |
+
from dataclasses import dataclass
|
| 58 |
+
from typing import Sequence
|
| 59 |
+
|
| 60 |
+
from .hypervec import (
|
| 61 |
+
CODEC_THRESHOLD,
|
| 62 |
+
DEFAULT_DIM,
|
| 63 |
+
MAX_INT_BITS,
|
| 64 |
+
bind,
|
| 65 |
+
bundle,
|
| 66 |
+
hvc,
|
| 67 |
+
hv_bits_of_int,
|
| 68 |
+
hv_from_bits,
|
| 69 |
+
hv_to_bits,
|
| 70 |
+
hv_to_int,
|
| 71 |
+
role,
|
| 72 |
+
sim,
|
| 73 |
+
)
|
| 74 |
+
from .memory import HDCStorage
|
| 75 |
+
|
| 76 |
+
PREFIX = "adder/row/"
|
| 77 |
+
|
| 78 |
+
#: Gate slots: two inputs, the carry in, then the sum and carry out.
|
| 79 |
+
GATE_SLOTS = ("a", "b", "c", "s", "k")
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class AdderError(Exception):
|
| 83 |
+
"""The adder could not resolve a truth-table row."""
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def wrap(value: int, width: int = MAX_INT_BITS) -> int:
|
| 87 |
+
"""Wrap an integer into a ``width``-bit two's complement range."""
|
| 88 |
+
span = 1 << width
|
| 89 |
+
return (value + (span >> 1)) % span - (span >> 1)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
@dataclass(frozen=True)
|
| 93 |
+
class Stage:
|
| 94 |
+
"""One bit slice of the carry chain, as executed."""
|
| 95 |
+
|
| 96 |
+
index: int
|
| 97 |
+
a: int
|
| 98 |
+
b: int
|
| 99 |
+
carry_in: int
|
| 100 |
+
total: int
|
| 101 |
+
carry_out: int
|
| 102 |
+
confidence: float
|
| 103 |
+
|
| 104 |
+
def render(self) -> str:
|
| 105 |
+
return (
|
| 106 |
+
"bit %2d a=%d b=%d cin=%d -> sum=%d cout=%d (lookup sim %.3f)"
|
| 107 |
+
% (self.index, self.a, self.b, self.carry_in, self.total, self.carry_out,
|
| 108 |
+
self.confidence)
|
| 109 |
+
)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
@dataclass(frozen=True)
|
| 113 |
+
class Row:
|
| 114 |
+
"""One truth-table row, and the record that stores it."""
|
| 115 |
+
|
| 116 |
+
index: int
|
| 117 |
+
a: int
|
| 118 |
+
b: int
|
| 119 |
+
c: int
|
| 120 |
+
total: int
|
| 121 |
+
carry: int
|
| 122 |
+
key: int
|
| 123 |
+
cell: str
|
| 124 |
+
|
| 125 |
+
def render(self) -> str:
|
| 126 |
+
return "a=%d b=%d cin=%d | sum=%d carry=%d | %s" % (
|
| 127 |
+
self.a, self.b, self.c, self.total, self.carry, self.cell
|
| 128 |
+
)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
class HyperAdder:
|
| 132 |
+
"""Ripple-carry adder whose gates are HDC bindings and cleanups."""
|
| 133 |
+
|
| 134 |
+
def __init__(
|
| 135 |
+
self,
|
| 136 |
+
d: int = DEFAULT_DIM,
|
| 137 |
+
storage: HDCStorage | None = None,
|
| 138 |
+
width: int = MAX_INT_BITS,
|
| 139 |
+
threshold: float = CODEC_THRESHOLD,
|
| 140 |
+
radix: int = 1,
|
| 141 |
+
):
|
| 142 |
+
self.d = d
|
| 143 |
+
self.width = min(width, MAX_INT_BITS)
|
| 144 |
+
self.threshold = threshold
|
| 145 |
+
#: Bits per stage. 1 is the 8-row full adder (32 stages); 8 is a
|
| 146 |
+
#: 131 072-row byte gate (4 stages), materialised lazily.
|
| 147 |
+
self.radix = max(1, min(radix, 8))
|
| 148 |
+
#: One atom pair per slot — see the module docstring on why sharing
|
| 149 |
+
#: a single pair would collapse the key to parity.
|
| 150 |
+
self.atoms = {
|
| 151 |
+
(slot, bit): hvc("gate/atom/%s/%d" % (slot, bit), d)
|
| 152 |
+
for slot in GATE_SLOTS
|
| 153 |
+
for bit in (0, 1)
|
| 154 |
+
}
|
| 155 |
+
self.storage = storage if storage is not None else HDCStorage(
|
| 156 |
+
d=d, capacity=64, label="adder"
|
| 157 |
+
)
|
| 158 |
+
self.rows: list[Row] = []
|
| 159 |
+
self.last_stages: list = []
|
| 160 |
+
self.stages_executed = 0
|
| 161 |
+
self.gate = None
|
| 162 |
+
if self.radix == 1:
|
| 163 |
+
self._build_table()
|
| 164 |
+
else:
|
| 165 |
+
from .slicegate import SliceGate
|
| 166 |
+
|
| 167 |
+
self.gate = SliceGate(radix=self.radix, d=d, threshold=threshold)
|
| 168 |
+
|
| 169 |
+
# ------------------------------------------------------------ the gate
|
| 170 |
+
def atom(self, slot: str, bit: int) -> int:
|
| 171 |
+
"""The vector that represents ``bit`` in a given gate slot."""
|
| 172 |
+
return self.atoms[(slot, 1 if bit else 0)]
|
| 173 |
+
|
| 174 |
+
def key_of(self, a: int, b: int, c: int) -> int:
|
| 175 |
+
"""Exact gate address: XOR of three role-specific atoms."""
|
| 176 |
+
return bind(
|
| 177 |
+
bind(self.atom("a", a), self.atom("b", b), self.d),
|
| 178 |
+
self.atom("c", c),
|
| 179 |
+
self.d,
|
| 180 |
+
)
|
| 181 |
+
|
| 182 |
+
def _build_table(self) -> None:
|
| 183 |
+
"""Materialise the 8 full-adder rows as cells in HDC storage."""
|
| 184 |
+
for a in (0, 1):
|
| 185 |
+
for b in (0, 1):
|
| 186 |
+
for c in (0, 1):
|
| 187 |
+
total = a ^ b ^ c
|
| 188 |
+
carry = 1 if (a + b + c) >= 2 else 0
|
| 189 |
+
key = self.key_of(a, b, c)
|
| 190 |
+
record = bundle(
|
| 191 |
+
[
|
| 192 |
+
bind(role("key", self.d), key, self.d),
|
| 193 |
+
bind(role("sum", self.d), self.atom("s", total), self.d),
|
| 194 |
+
bind(role("carry", self.d), self.atom("k", carry), self.d),
|
| 195 |
+
],
|
| 196 |
+
self.d,
|
| 197 |
+
)
|
| 198 |
+
cell = PREFIX + "%d%d%d" % (a, b, c)
|
| 199 |
+
self.storage.write(cell, record, symbol=True)
|
| 200 |
+
self.rows.append(
|
| 201 |
+
Row(a * 4 + b * 2 + c, a, b, c, total, carry, key, cell)
|
| 202 |
+
)
|
| 203 |
+
|
| 204 |
+
def _pick(self, probe: int, slot: str) -> tuple[int, float]:
|
| 205 |
+
"""Cleanup an output field to 0 or 1; returns (bit, margin)."""
|
| 206 |
+
s0 = sim(probe, self.atom(slot, 0), self.d)
|
| 207 |
+
s1 = sim(probe, self.atom(slot, 1), self.d)
|
| 208 |
+
if s1 >= s0:
|
| 209 |
+
return 1, s1 - s0
|
| 210 |
+
return 0, s0 - s1
|
| 211 |
+
|
| 212 |
+
def full_adder(self, a: int, b: int, c: int) -> tuple[int, int, float]:
|
| 213 |
+
"""One stage: bind the inputs, clean up over the 8 stored rows."""
|
| 214 |
+
query = self.key_of(a, b, c)
|
| 215 |
+
# The stored row holds bind(role("key"), key), so the probe carries the
|
| 216 |
+
# same role; a bare key against the bundle would sit at the noise floor.
|
| 217 |
+
probe = bind(role("key", self.d), query, self.d)
|
| 218 |
+
hits = self.storage.recall(probe, k=1, prefix=PREFIX)
|
| 219 |
+
if not hits or hits[0][2] < 0.20:
|
| 220 |
+
raise AdderError(
|
| 221 |
+
"no truth-table row resolved (best similarity %.3f)"
|
| 222 |
+
% (hits[0][2] if hits else -1.0)
|
| 223 |
+
)
|
| 224 |
+
record, confidence = hits[0][1], hits[0][2]
|
| 225 |
+
total, _ = self._pick(bind(record, role("sum", self.d), self.d), "s")
|
| 226 |
+
carry, _ = self._pick(bind(record, role("carry", self.d), self.d), "k")
|
| 227 |
+
return total, carry, confidence
|
| 228 |
+
|
| 229 |
+
# --------------------------------------------------------- the ripple
|
| 230 |
+
def add_bits(
|
| 231 |
+
self, a_bits: Sequence[int], b_bits: Sequence[int], carry_in: int = 0
|
| 232 |
+
) -> tuple[list[int], list]:
|
| 233 |
+
"""Carry across the operand, one gate stage at a time."""
|
| 234 |
+
if self.gate is not None:
|
| 235 |
+
return self._add_sliced(a_bits, b_bits, carry_in)
|
| 236 |
+
carry = 1 if carry_in else 0
|
| 237 |
+
out: list[int] = []
|
| 238 |
+
stages: list[Stage] = []
|
| 239 |
+
for i in range(self.width):
|
| 240 |
+
# A short slice is zero-extended, exactly as a partial bit vector
|
| 241 |
+
# is on a real adder input.
|
| 242 |
+
a = 1 if i < len(a_bits) and a_bits[i] else 0
|
| 243 |
+
b = 1 if i < len(b_bits) and b_bits[i] else 0
|
| 244 |
+
total, carry_next, confidence = self.full_adder(a, b, carry)
|
| 245 |
+
stages.append(Stage(i, a, b, carry, total, carry_next, confidence))
|
| 246 |
+
out.append(total)
|
| 247 |
+
carry = carry_next
|
| 248 |
+
self.stages_executed += len(stages)
|
| 249 |
+
self.last_stages = stages
|
| 250 |
+
return out, stages
|
| 251 |
+
|
| 252 |
+
def _add_sliced(
|
| 253 |
+
self, a_bits: Sequence[int], b_bits: Sequence[int], carry_in: int
|
| 254 |
+
) -> tuple[list[int], list]:
|
| 255 |
+
"""Width-k stages: each gate call consumes ``radix`` bits plus carry."""
|
| 256 |
+
from .slicegate import SliceStep
|
| 257 |
+
|
| 258 |
+
k = self.radix
|
| 259 |
+
out = [0] * self.width
|
| 260 |
+
steps: list[SliceStep] = []
|
| 261 |
+
carry = 1 if carry_in else 0
|
| 262 |
+
for start in range(0, self.width, k):
|
| 263 |
+
a_val = 0
|
| 264 |
+
b_val = 0
|
| 265 |
+
for i in range(k):
|
| 266 |
+
pos = start + i
|
| 267 |
+
if pos < self.width:
|
| 268 |
+
a_val |= (1 if pos < len(a_bits) and a_bits[pos] else 0) << i
|
| 269 |
+
b_val |= (1 if pos < len(b_bits) and b_bits[pos] else 0) << i
|
| 270 |
+
total, carry_next, confidence = self.gate.evaluate(a_val, b_val, carry)
|
| 271 |
+
for i in range(k):
|
| 272 |
+
pos = start + i
|
| 273 |
+
if pos < self.width:
|
| 274 |
+
out[pos] = (total >> i) & 1
|
| 275 |
+
steps.append(
|
| 276 |
+
SliceStep(start, a_val, b_val, carry, total, carry_next, confidence, k)
|
| 277 |
+
)
|
| 278 |
+
carry = carry_next
|
| 279 |
+
self.stages_executed += len(steps)
|
| 280 |
+
self.last_stages = steps
|
| 281 |
+
return out, steps
|
| 282 |
+
|
| 283 |
+
# -------------------------------------------------------- public API
|
| 284 |
+
def add(self, a: int, b: int) -> int:
|
| 285 |
+
"""Integer operands in, hypervector out."""
|
| 286 |
+
bits, _ = self.add_bits(hv_bits_of_int(a), hv_bits_of_int(b), 0)
|
| 287 |
+
return hv_from_bits(bits, self.d)
|
| 288 |
+
|
| 289 |
+
def sub(self, a: int, b: int) -> int:
|
| 290 |
+
"""``a - b`` as ``a + ~b + 1``, with no sign case analysis."""
|
| 291 |
+
inverted = [0 if bit else 1 for bit in hv_bits_of_int(b)]
|
| 292 |
+
bits, _ = self.add_bits(hv_bits_of_int(a), inverted, 1)
|
| 293 |
+
return hv_from_bits(bits, self.d)
|
| 294 |
+
|
| 295 |
+
def add_hv(self, a_hv: int, b_hv: int) -> int:
|
| 296 |
+
"""Hypervector in, hypervector out — no integer is materialised."""
|
| 297 |
+
bits, _ = self.add_bits(
|
| 298 |
+
hv_to_bits(a_hv, self.d, self.threshold),
|
| 299 |
+
hv_to_bits(b_hv, self.d, self.threshold),
|
| 300 |
+
0,
|
| 301 |
+
)
|
| 302 |
+
return hv_from_bits(bits, self.d)
|
| 303 |
+
|
| 304 |
+
def sub_hv(self, a_hv: int, b_hv: int) -> int:
|
| 305 |
+
"""As :meth:`add_hv`, via the complement and carry-in 1."""
|
| 306 |
+
b_bits = hv_to_bits(b_hv, self.d, self.threshold)
|
| 307 |
+
bits, _ = self.add_bits(
|
| 308 |
+
hv_to_bits(a_hv, self.d, self.threshold),
|
| 309 |
+
[0 if bit else 1 for bit in b_bits],
|
| 310 |
+
1,
|
| 311 |
+
)
|
| 312 |
+
return hv_from_bits(bits, self.d)
|
| 313 |
+
|
| 314 |
+
def decode(self, hv: int) -> int:
|
| 315 |
+
"""Decode to an integer — for display and for verifying the circuit."""
|
| 316 |
+
return hv_to_int(hv, self.d, self.threshold)
|
| 317 |
+
|
| 318 |
+
# ------------------------------------------------------------- views
|
| 319 |
+
def table_text(self) -> str:
|
| 320 |
+
rows = ["full-adder truth table, stored as records in %r" % self.storage.label]
|
| 321 |
+
for row in self.rows:
|
| 322 |
+
rows.append(" " + row.render())
|
| 323 |
+
return "\n".join(rows)
|
| 324 |
+
|
| 325 |
+
def trace_text(self, limit: int | None = None) -> str:
|
| 326 |
+
stages = self.last_stages if limit is None else self.last_stages[:limit]
|
| 327 |
+
if not stages:
|
| 328 |
+
return "<no stages executed yet>"
|
| 329 |
+
return "\n".join(stage.render() for stage in stages)
|
| 330 |
+
|
| 331 |
+
def separation(self) -> dict:
|
| 332 |
+
"""Measured similarity between the 8 row keys (signal vs crosstalk)."""
|
| 333 |
+
if self.gate is not None:
|
| 334 |
+
summary = self.gate.summary()
|
| 335 |
+
summary.update({
|
| 336 |
+
"rows": summary["table_rows"],
|
| 337 |
+
"worst_cross_similarity": None,
|
| 338 |
+
"table_cells": summary["cells"],
|
| 339 |
+
"width": self.width,
|
| 340 |
+
"d": self.d,
|
| 341 |
+
})
|
| 342 |
+
return summary
|
| 343 |
+
worst = 2.0
|
| 344 |
+
for i, row_a in enumerate(self.rows):
|
| 345 |
+
for j, row_b in enumerate(self.rows):
|
| 346 |
+
if i != j:
|
| 347 |
+
worst = min(worst, sim(row_a.key, row_b.key, self.d))
|
| 348 |
+
return {
|
| 349 |
+
"rows": len(self.rows),
|
| 350 |
+
"worst_cross_similarity": worst,
|
| 351 |
+
"table_cells": len(self.storage),
|
| 352 |
+
"width": self.width,
|
| 353 |
+
"d": self.d,
|
| 354 |
+
}
|
| 355 |
+
|
| 356 |
+
def verify(self, trials: int = 200, seed: int = 1234) -> dict:
|
| 357 |
+
"""Check the circuit against Python arithmetic, both directions."""
|
| 358 |
+
import random
|
| 359 |
+
|
| 360 |
+
rng = random.Random(seed)
|
| 361 |
+
adds = subs = hv_layer = 0
|
| 362 |
+
for _ in range(trials):
|
| 363 |
+
a = rng.randrange(-(1 << 31), 1 << 31)
|
| 364 |
+
b = rng.randrange(-(1 << 31), 1 << 31)
|
| 365 |
+
if self.decode(self.add(a, b)) == wrap(a + b, self.width):
|
| 366 |
+
adds += 1
|
| 367 |
+
if self.decode(self.sub(a, b)) == wrap(a - b, self.width):
|
| 368 |
+
subs += 1
|
| 369 |
+
if self.decode(self.add_hv(self.add(0, a), self.add(0, b))) == wrap(
|
| 370 |
+
a + b, self.width
|
| 371 |
+
):
|
| 372 |
+
hv_layer += 1
|
| 373 |
+
return {
|
| 374 |
+
"trials": trials,
|
| 375 |
+
"add_correct": adds,
|
| 376 |
+
"sub_correct": subs,
|
| 377 |
+
"hv_to_hv_correct": hv_layer,
|
| 378 |
+
"stages": self.width,
|
| 379 |
+
}
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
def selfcheck(trials: int = 100) -> dict:
|
| 383 |
+
"""Small self-verification used by the demo and the CLI."""
|
| 384 |
+
adder = HyperAdder(d=2000)
|
| 385 |
+
report = adder.separation()
|
| 386 |
+
report.update(adder.verify(trials))
|
| 387 |
+
report["decode_roundtrip"] = all(
|
| 388 |
+
adder.decode(adder.add(0, n)) == n for n in (0, 1, -1, 7, -7, 65535, -12345)
|
| 389 |
+
)
|
| 390 |
+
return report
|
| 391 |
+
|
| 392 |
+
|
| 393 |
+
if __name__ == "__main__": # pragma: no cover - manual inspection
|
| 394 |
+
for key, value in selfcheck().items():
|
| 395 |
+
print("%-22s %s" % (key, value))
|
hdc/agent.py
ADDED
|
@@ -0,0 +1,423 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Agentic MCP tools held *inside* HDC storage.
|
| 2 |
+
|
| 3 |
+
A tool is a record, not a row in a database:
|
| 4 |
+
|
| 5 |
+
record = bundle([
|
| 6 |
+
bind(role("name"), hvc("tool/hdc.bind")),
|
| 7 |
+
bind(role("cap"), capability_vector("hdc/bind/construct")),
|
| 8 |
+
bind(role("arity"), hvc("arity/2")),
|
| 9 |
+
])
|
| 10 |
+
|
| 11 |
+
The record is written to storage as the cell ``toolreg/<name>``. Registration,
|
| 12 |
+
discovery and dispatch therefore all run on the same content-addressable
|
| 13 |
+
machinery as programs and data: a tool is found by *resembling* the query.
|
| 14 |
+
|
| 15 |
+
Dispatch has three tiers, each reporting its confidence:
|
| 16 |
+
|
| 17 |
+
1. exact name — confidence 1.0;
|
| 18 |
+
2. character-superposition similarity, so a misspelled or truncated tool name
|
| 19 |
+
(``hdc.bundl``) still finds ``hdc.bundle`` rather than failing;
|
| 20 |
+
3. capability path similarity, so ``hdc.memory.search`` reaches
|
| 21 |
+
``hdc.recall`` because both carry ``hdc`` + ``memory`` atoms.
|
| 22 |
+
|
| 23 |
+
Tier 2 uses the standard HDC string codec: each character becomes an atom and
|
| 24 |
+
its position is bound in by permutation, so two strings that match in most
|
| 25 |
+
positions have highly similar vectors. That is what makes fuzzy dispatch a
|
| 26 |
+
cleanup operation instead of an edit-distance algorithm.
|
| 27 |
+
|
| 28 |
+
The registry is what the ``TOOL`` instruction calls, and what
|
| 29 |
+
:mod:`hdc.mcp_server` advertises to an outside agent, so a program running
|
| 30 |
+
inside HDC and an MCP client reach the same tools by the same path.
|
| 31 |
+
"""
|
| 32 |
+
|
| 33 |
+
from __future__ import annotations
|
| 34 |
+
|
| 35 |
+
from dataclasses import dataclass, field
|
| 36 |
+
from typing import Callable, Sequence
|
| 37 |
+
|
| 38 |
+
from .hypervec import (
|
| 39 |
+
DEFAULT_DIM,
|
| 40 |
+
bind,
|
| 41 |
+
bundle,
|
| 42 |
+
hvc,
|
| 43 |
+
hv_to_int,
|
| 44 |
+
int_to_hv,
|
| 45 |
+
permute,
|
| 46 |
+
popcount,
|
| 47 |
+
sim,
|
| 48 |
+
)
|
| 49 |
+
from .memory import HDCStorage
|
| 50 |
+
|
| 51 |
+
#: Minimum fuzzy similarity accepted for a non-exact tool name.
|
| 52 |
+
FUZZY_FLOOR = 0.55
|
| 53 |
+
#: Fuzzy matches must beat the runner-up by this much to be trusted.
|
| 54 |
+
FUZZY_MARGIN = 0.04
|
| 55 |
+
|
| 56 |
+
_CAP_CACHE: dict[tuple[str, int], int] = {}
|
| 57 |
+
_STR_CACHE: dict[tuple[str, int], int] = {}
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def capability_vector(capability: str, d: int = DEFAULT_DIM) -> int:
|
| 61 |
+
"""Superpose the path atoms of a capability (``hdc/bind/construct``)."""
|
| 62 |
+
key = (capability, d)
|
| 63 |
+
hit = _CAP_CACHE.get(key)
|
| 64 |
+
if hit is not None:
|
| 65 |
+
return hit
|
| 66 |
+
atoms = [hvc("cap/" + part, d) for part in capability.replace(".", "/").split("/") if part]
|
| 67 |
+
vec = bundle(atoms, d) if atoms else hvc("cap/unknown", d)
|
| 68 |
+
_CAP_CACHE[key] = vec
|
| 69 |
+
return vec
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def string_vector(text: str, d: int = DEFAULT_DIM) -> int:
|
| 73 |
+
"""Character-superposition codec: order-sensitive, near-orthogonal, fuzzy.
|
| 74 |
+
|
| 75 |
+
Each character is bound to its position by permutation and the results are
|
| 76 |
+
bundled, so ``sim(string_vector("hdc.bundle"), string_vector("hdc.bundl"))``
|
| 77 |
+
is high while unrelated strings stay near the noise floor.
|
| 78 |
+
"""
|
| 79 |
+
key = (text, d)
|
| 80 |
+
hit = _STR_CACHE.get(key)
|
| 81 |
+
if hit is not None:
|
| 82 |
+
return hit
|
| 83 |
+
if not text:
|
| 84 |
+
vec = 0
|
| 85 |
+
else:
|
| 86 |
+
vec = bundle(
|
| 87 |
+
[permute(hvc("char/" + ch, d), i, d) for i, ch in enumerate(text)], d
|
| 88 |
+
)
|
| 89 |
+
_STR_CACHE[key] = vec
|
| 90 |
+
return vec
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def _json_type(kind: str) -> str:
|
| 94 |
+
return "integer" if kind in ("int", "hv") else "string"
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
@dataclass
|
| 98 |
+
class ToolSpec:
|
| 99 |
+
"""One agentic tool.
|
| 100 |
+
|
| 101 |
+
``returns`` matters because a hypervector and a scalar are both Python
|
| 102 |
+
``int``s: the machine has to know whether to *encode* a result (scalar) or
|
| 103 |
+
store it as a vector in its own right. Without this the two are
|
| 104 |
+
indistinguishable at the call site.
|
| 105 |
+
"""
|
| 106 |
+
|
| 107 |
+
name: str
|
| 108 |
+
capability: str
|
| 109 |
+
params: tuple[tuple[str, str], ...]
|
| 110 |
+
doc: str
|
| 111 |
+
fn: Callable[..., object]
|
| 112 |
+
returns: str = "int"
|
| 113 |
+
calls: int = 0
|
| 114 |
+
|
| 115 |
+
@property
|
| 116 |
+
def kinds(self) -> tuple[str, ...]:
|
| 117 |
+
return tuple(kind for _, kind in self.params)
|
| 118 |
+
|
| 119 |
+
def record(self, d: int) -> int:
|
| 120 |
+
"""The hypervector record that represents this tool in storage."""
|
| 121 |
+
parts = [
|
| 122 |
+
bind(self._role("name", d), hvc("tool/" + self.name, d), d),
|
| 123 |
+
bind(self._role("cap", d), capability_vector(self.capability, d), d),
|
| 124 |
+
bind(self._role("arity", d), hvc("arity/%d" % len(self.params), d), d),
|
| 125 |
+
]
|
| 126 |
+
return bundle(parts, d)
|
| 127 |
+
|
| 128 |
+
@staticmethod
|
| 129 |
+
def _role(name: str, d: int) -> int:
|
| 130 |
+
from .hypervec import role
|
| 131 |
+
|
| 132 |
+
return role(name, d)
|
| 133 |
+
|
| 134 |
+
def input_schema(self) -> dict:
|
| 135 |
+
"""MCP ``inputSchema`` built from the declared parameter kinds."""
|
| 136 |
+
props = {}
|
| 137 |
+
for pname, kind in self.params:
|
| 138 |
+
props[pname] = {
|
| 139 |
+
"type": _json_type(kind),
|
| 140 |
+
"description": (
|
| 141 |
+
"hypervector: pass the integer, or a hex string like 0x1f3a"
|
| 142 |
+
if kind == "hv"
|
| 143 |
+
else "scalar integer"
|
| 144 |
+
),
|
| 145 |
+
}
|
| 146 |
+
return {
|
| 147 |
+
"type": "object",
|
| 148 |
+
"properties": props,
|
| 149 |
+
"required": [pname for pname, _ in self.params],
|
| 150 |
+
"additionalProperties": False,
|
| 151 |
+
}
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
class ToolRegistry:
|
| 155 |
+
"""Tool records in HDC storage, plus the dispatcher that reads them."""
|
| 156 |
+
|
| 157 |
+
def __init__(self, storage: HDCStorage | None = None, d: int = DEFAULT_DIM):
|
| 158 |
+
self.d = d
|
| 159 |
+
self.storage = storage if storage is not None else HDCStorage(d=d, label="tools")
|
| 160 |
+
self.specs: dict[str, ToolSpec] = {}
|
| 161 |
+
self._records: dict[str, int] = {}
|
| 162 |
+
self.last_machine = None
|
| 163 |
+
self.install_builtins()
|
| 164 |
+
|
| 165 |
+
# ---------------------------------------------------------- registration
|
| 166 |
+
def register(
|
| 167 |
+
self,
|
| 168 |
+
name: str,
|
| 169 |
+
capability: str,
|
| 170 |
+
params: Sequence[tuple[str, str]],
|
| 171 |
+
doc: str,
|
| 172 |
+
fn: Callable[..., object],
|
| 173 |
+
returns: str = "int",
|
| 174 |
+
store: bool = True,
|
| 175 |
+
) -> ToolSpec:
|
| 176 |
+
"""Add a tool and write its record into HDC storage."""
|
| 177 |
+
spec = ToolSpec(name, capability, tuple(params), doc, fn, returns=returns)
|
| 178 |
+
self.specs[name] = spec
|
| 179 |
+
rec = spec.record(self.d)
|
| 180 |
+
self._records[name] = rec
|
| 181 |
+
if store:
|
| 182 |
+
self.storage.write("toolreg/" + name, rec, symbol=True)
|
| 183 |
+
return spec
|
| 184 |
+
|
| 185 |
+
def unregister(self, name: str) -> None:
|
| 186 |
+
self.specs.pop(name, None)
|
| 187 |
+
self._records.pop(name, None)
|
| 188 |
+
if "toolreg/" + name in self.storage:
|
| 189 |
+
self.storage.cells.pop("toolreg/" + name, None)
|
| 190 |
+
self.storage.order.remove("toolreg/" + name)
|
| 191 |
+
|
| 192 |
+
# -------------------------------------------------------------- lookup
|
| 193 |
+
def get(self, name: str) -> ToolSpec | None:
|
| 194 |
+
return self.specs.get(name)
|
| 195 |
+
|
| 196 |
+
def params_for(self, name: str) -> tuple[str, ...]:
|
| 197 |
+
spec, _ = self.resolve(name)
|
| 198 |
+
if spec is None:
|
| 199 |
+
raise ToolError("unknown tool: %r" % (name,))
|
| 200 |
+
return spec.kinds
|
| 201 |
+
|
| 202 |
+
def returns_for(self, name: str) -> str:
|
| 203 |
+
spec, _ = self.resolve(name)
|
| 204 |
+
if spec is None:
|
| 205 |
+
raise ToolError("unknown tool: %r" % (name,))
|
| 206 |
+
return spec.returns
|
| 207 |
+
|
| 208 |
+
def resolve(self, query: str | int) -> tuple[ToolSpec | None, float]:
|
| 209 |
+
"""Find a tool by name, by fuzzy name, or by capability resemblance."""
|
| 210 |
+
if isinstance(query, int):
|
| 211 |
+
best_name, best = "", -2.0
|
| 212 |
+
for name, rec in self._records.items():
|
| 213 |
+
s = sim(rec, query, self.d)
|
| 214 |
+
if s > best:
|
| 215 |
+
best_name, best = name, s
|
| 216 |
+
return (self.specs.get(best_name), best) if best >= 0.3 else (None, best)
|
| 217 |
+
|
| 218 |
+
if query in self.specs:
|
| 219 |
+
return self.specs[query], 1.0
|
| 220 |
+
|
| 221 |
+
probe = string_vector(query, self.d)
|
| 222 |
+
scored = sorted(
|
| 223 |
+
((name, sim(probe, string_vector(name, self.d), self.d)) for name in self.specs),
|
| 224 |
+
key=lambda t: t[1],
|
| 225 |
+
reverse=True,
|
| 226 |
+
)
|
| 227 |
+
if scored and scored[0][1] >= FUZZY_FLOOR:
|
| 228 |
+
margin = scored[0][1] - (scored[1][1] if len(scored) > 1 else 0.0)
|
| 229 |
+
if margin >= FUZZY_MARGIN or len(scored) == 1:
|
| 230 |
+
return self.specs[scored[0][0]], scored[0][1]
|
| 231 |
+
|
| 232 |
+
cap_probe = capability_vector(query, self.d)
|
| 233 |
+
scored = sorted(
|
| 234 |
+
(
|
| 235 |
+
(name, sim(cap_probe, capability_vector(s.capability, self.d), self.d))
|
| 236 |
+
for name, s in self.specs.items()
|
| 237 |
+
),
|
| 238 |
+
key=lambda t: t[1],
|
| 239 |
+
reverse=True,
|
| 240 |
+
)
|
| 241 |
+
if scored and scored[0][1] >= 0.30:
|
| 242 |
+
return self.specs[scored[0][0]], scored[0][1]
|
| 243 |
+
return None, (scored[0][1] if scored else 0.0)
|
| 244 |
+
|
| 245 |
+
# --------------------------------------------------------------- calling
|
| 246 |
+
def call(self, name: str, args: Sequence[object], machine=None) -> object:
|
| 247 |
+
"""Invoke a tool. Raises ``ToolError`` for an unresolvable tool."""
|
| 248 |
+
spec, confidence = self.resolve(name)
|
| 249 |
+
if spec is None:
|
| 250 |
+
raise ToolError(
|
| 251 |
+
"no tool resembling %r (best similarity %.3f)" % (name, confidence)
|
| 252 |
+
)
|
| 253 |
+
if len(args) != len(spec.params):
|
| 254 |
+
raise ToolError(
|
| 255 |
+
"tool %s takes %d argument(s), got %d"
|
| 256 |
+
% (spec.name, len(spec.params), len(args))
|
| 257 |
+
)
|
| 258 |
+
self.last_machine = machine
|
| 259 |
+
spec.calls += 1
|
| 260 |
+
return spec.fn(*args)
|
| 261 |
+
|
| 262 |
+
def call_named(self, name: str, arguments: dict, machine=None) -> object:
|
| 263 |
+
"""Invoke a tool with named arguments (the MCP calling convention)."""
|
| 264 |
+
spec, confidence = self.resolve(name)
|
| 265 |
+
if spec is None:
|
| 266 |
+
raise ToolError("unknown tool %r (%.3f)" % (name, confidence))
|
| 267 |
+
args = []
|
| 268 |
+
for pname, kind in spec.params:
|
| 269 |
+
if pname not in arguments:
|
| 270 |
+
raise ToolError("missing argument %r for %s" % (pname, spec.name))
|
| 271 |
+
value = arguments[pname]
|
| 272 |
+
if kind in ("int", "hv") and isinstance(value, str):
|
| 273 |
+
value = int(value, 0)
|
| 274 |
+
if kind == "int":
|
| 275 |
+
value = int(value)
|
| 276 |
+
elif kind == "hv":
|
| 277 |
+
value = int(value)
|
| 278 |
+
args.append(value)
|
| 279 |
+
return self.call(spec.name, args, machine=machine)
|
| 280 |
+
|
| 281 |
+
# ---------------------------------------------------------------- views
|
| 282 |
+
def list_tools(self) -> list[dict]:
|
| 283 |
+
"""MCP ``tools/list`` payload."""
|
| 284 |
+
out = []
|
| 285 |
+
for name in sorted(self.specs):
|
| 286 |
+
spec = self.specs[name]
|
| 287 |
+
out.append(
|
| 288 |
+
{
|
| 289 |
+
"name": spec.name,
|
| 290 |
+
"description": spec.doc,
|
| 291 |
+
"inputSchema": spec.input_schema(),
|
| 292 |
+
"annotations": {"hdc_capability": spec.capability},
|
| 293 |
+
}
|
| 294 |
+
)
|
| 295 |
+
return out
|
| 296 |
+
|
| 297 |
+
def specs_text(self) -> str:
|
| 298 |
+
rows = ["%-16s %-26s %-5s %s" % ("TOOL", "CAPABILITY", "ARGS", "DESCRIPTION")]
|
| 299 |
+
rows.append("-" * 96)
|
| 300 |
+
for name in sorted(self.specs):
|
| 301 |
+
s = self.specs[name]
|
| 302 |
+
rows.append(
|
| 303 |
+
"%-16s %-26s %-5d %s"
|
| 304 |
+
% (s.name, s.capability, len(s.params), s.doc)
|
| 305 |
+
)
|
| 306 |
+
return "\n".join(rows)
|
| 307 |
+
|
| 308 |
+
def memory_report(self) -> str:
|
| 309 |
+
"""Show the tool records as they sit in storage."""
|
| 310 |
+
rows = ["tool records in %r (d=%d)" % (self.storage.label, self.d)]
|
| 311 |
+
for name in sorted(self.specs):
|
| 312 |
+
cell = "toolreg/" + name
|
| 313 |
+
present = cell in self.storage
|
| 314 |
+
hv = self.storage.read(cell, default=0)
|
| 315 |
+
rows.append(
|
| 316 |
+
" %-16s cell=%-18s pop=%-5d %s"
|
| 317 |
+
% (name, cell, popcount(hv), "stored" if present else "MISSING")
|
| 318 |
+
)
|
| 319 |
+
return "\n".join(rows)
|
| 320 |
+
|
| 321 |
+
# ------------------------------------------------------------- builtins
|
| 322 |
+
def install_builtins(self) -> None:
|
| 323 |
+
"""Register the built-in HDC / bit-logic / BitNet tools."""
|
| 324 |
+
d = self.d
|
| 325 |
+
store = self.storage
|
| 326 |
+
|
| 327 |
+
def hdc_similarity(a: int, b: int) -> int:
|
| 328 |
+
return int(round(sim(a, b, d) * 1000))
|
| 329 |
+
|
| 330 |
+
def hdc_bind(a: int, b: int) -> int:
|
| 331 |
+
return bind(a, b, d)
|
| 332 |
+
|
| 333 |
+
def hdc_bundle(a: int, b: int) -> int:
|
| 334 |
+
return bundle([a, b], d)
|
| 335 |
+
|
| 336 |
+
def hdc_permute(v: int, k: int) -> int:
|
| 337 |
+
return permute(v, k, d)
|
| 338 |
+
|
| 339 |
+
def hdc_cleanup(v: int) -> int:
|
| 340 |
+
hits = store.nearest_symbol(v, k=1)
|
| 341 |
+
if not hits:
|
| 342 |
+
return -1
|
| 343 |
+
syms = store.symbols()
|
| 344 |
+
return syms.index(hits[0][0]) if hits[0][0] in syms else -1
|
| 345 |
+
|
| 346 |
+
def hdc_recall(v: int, k: int) -> int:
|
| 347 |
+
hits = store.recall(v, k=max(1, k))
|
| 348 |
+
if not hits:
|
| 349 |
+
return -1
|
| 350 |
+
return store.order.index(hits[0][0]) if hits[0][0] in store.order else -1
|
| 351 |
+
|
| 352 |
+
def hdc_encode_int(n: int) -> int:
|
| 353 |
+
return int_to_hv(n, d)
|
| 354 |
+
|
| 355 |
+
def hdc_decode_int(v: int) -> int:
|
| 356 |
+
return hv_to_int(v, d)
|
| 357 |
+
|
| 358 |
+
def bit_popcount(v: int) -> int:
|
| 359 |
+
return popcount(v)
|
| 360 |
+
|
| 361 |
+
def bit_hamming(a: int, b: int) -> int:
|
| 362 |
+
return (a ^ b).bit_count()
|
| 363 |
+
|
| 364 |
+
def net_dot1bit(a: int, b: int) -> int:
|
| 365 |
+
# A 1-bit dot product is bind-then-count: agree - disagree.
|
| 366 |
+
return d - 2 * (a ^ b).bit_count()
|
| 367 |
+
|
| 368 |
+
def mem_stat() -> int:
|
| 369 |
+
return int(store.stat()["cells_used"])
|
| 370 |
+
|
| 371 |
+
# HDC-native arithmetic, exposed to agents: a ripple-carry circuit over
|
| 372 |
+
# bit atoms rather than a decode-and-add.
|
| 373 |
+
from .adder import HyperAdder
|
| 374 |
+
|
| 375 |
+
adder = HyperAdder(d=d)
|
| 376 |
+
|
| 377 |
+
def hdc_add(a: int, b: int) -> int:
|
| 378 |
+
return adder.add_hv(a, b)
|
| 379 |
+
|
| 380 |
+
def hdc_sub(a: int, b: int) -> int:
|
| 381 |
+
return adder.sub_hv(a, b)
|
| 382 |
+
|
| 383 |
+
def agent_echo(n: int) -> int:
|
| 384 |
+
return int(n)
|
| 385 |
+
|
| 386 |
+
self.register("hdc.similarity", "hdc/bind/similarity", (("a", "hv"), ("b", "hv")),
|
| 387 |
+
"Similarity of two hypervectors, scaled to milli-units.", hdc_similarity)
|
| 388 |
+
self.register("hdc.bind", "hdc/bind/construct", (("a", "hv"), ("b", "hv")),
|
| 389 |
+
"XOR-bind two hypervectors.", hdc_bind, returns="hv")
|
| 390 |
+
self.register("hdc.unbind", "hdc/bind/invert", (("a", "hv"), ("b", "hv")),
|
| 391 |
+
"Unbind: XOR is self-inverse, so this is bind.", hdc_bind, returns="hv")
|
| 392 |
+
self.register("hdc.bundle", "hdc/bundle/construct", (("a", "hv"), ("b", "hv")),
|
| 393 |
+
"Majority-bundle two hypervectors.", hdc_bundle, returns="hv")
|
| 394 |
+
self.register("hdc.permute", "hdc/order/rotate", (("v", "hv"), ("k", "int")),
|
| 395 |
+
"Rotate a hypervector by k bits.", hdc_permute, returns="hv")
|
| 396 |
+
self.register("hdc.cleanup", "hdc/memory/cleanup", (("v", "hv"),),
|
| 397 |
+
"Index of the stored symbol nearest to a hypervector.", hdc_cleanup)
|
| 398 |
+
self.register("hdc.recall", "hdc/memory/search", (("v", "hv"), ("k", "int")),
|
| 399 |
+
"Index of the stored cell most similar to a hypervector.", hdc_recall)
|
| 400 |
+
self.register("hdc.encode_int", "hdc/codec/encode", (("n", "int"),),
|
| 401 |
+
"Encode an integer as a hypervector.", hdc_encode_int, returns="hv")
|
| 402 |
+
self.register("hdc.decode_int", "hdc/codec/decode", (("v", "hv"),),
|
| 403 |
+
"Decode an integer hypervector.", hdc_decode_int)
|
| 404 |
+
self.register("bit.popcount", "bit/logic/count", (("v", "hv"),),
|
| 405 |
+
"Population count of a hypervector.", bit_popcount)
|
| 406 |
+
self.register("bit.hamming", "bit/logic/distance", (("a", "hv"), ("b", "hv")),
|
| 407 |
+
"Hamming distance between two hypervectors.", bit_hamming)
|
| 408 |
+
self.register("net.dot1bit", "bitnet/inference/dot", (("a", "hv"), ("b", "hv")),
|
| 409 |
+
"1-bit dot product as agreement minus disagreement.", net_dot1bit)
|
| 410 |
+
self.register("mem.stat", "storage/memory/stats", (),
|
| 411 |
+
"Number of occupied HDC cells.", mem_stat)
|
| 412 |
+
self.register("hdc.add", "hdc/arithmetic/add", (("a", "hv"), ("b", "hv")),
|
| 413 |
+
"Add two integer hypervectors with a ripple-carry HDC circuit.",
|
| 414 |
+
hdc_add, returns="hv")
|
| 415 |
+
self.register("hdc.sub", "hdc/arithmetic/subtract", (("a", "hv"), ("b", "hv")),
|
| 416 |
+
"Subtract two integer hypervectors (two's complement, HDC circuit).",
|
| 417 |
+
hdc_sub, returns="hv")
|
| 418 |
+
self.register("agent.echo", "agent/plumbing/echo", (("n", "int"),),
|
| 419 |
+
"Return the argument unchanged; useful for wiring tests.", agent_echo)
|
| 420 |
+
|
| 421 |
+
|
| 422 |
+
class ToolError(Exception):
|
| 423 |
+
"""A tool could not be resolved or invoked."""
|
hdc/assembler.py
ADDED
|
@@ -0,0 +1,392 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Compiler: ``.hdcasm`` source -> hypervector program image.
|
| 2 |
+
|
| 3 |
+
Two passes, because labels are forward references:
|
| 4 |
+
|
| 5 |
+
1. **scan** — strip comments, record label -> cell index, collect the alphabets
|
| 6 |
+
the decoder will later be allowed to use (integer literals, string literals,
|
| 7 |
+
tool names, target cells). These alphabets are *cleanup memory*: they are
|
| 8 |
+
what makes decoding a bounded argmax instead of an open search.
|
| 9 |
+
2. **emit** — turn each line into one composed hypervector and, on install,
|
| 10 |
+
write it into HDC storage as a cell named ``code/<n>``.
|
| 11 |
+
|
| 12 |
+
Nothing about an instruction survives as text in the image. The symbolic
|
| 13 |
+
parts survive only as *names*, from which their vectors are regenerated
|
| 14 |
+
deterministically; the composed instruction vectors are what get stored. That
|
| 15 |
+
is why an image is a genuine program in HDC and not a text file with extra
|
| 16 |
+
steps.
|
| 17 |
+
|
| 18 |
+
.data tbl0 42 ; preload a storage cell with an integer
|
| 19 |
+
.str hello "hi there" ; a string literal
|
| 20 |
+
start: ; a label is a cell index
|
| 21 |
+
ldi R0, 5
|
| 22 |
+
bind R1, R0, R0
|
| 23 |
+
prints hello
|
| 24 |
+
halt
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
from __future__ import annotations
|
| 28 |
+
|
| 29 |
+
import re
|
| 30 |
+
from dataclasses import dataclass, field
|
| 31 |
+
|
| 32 |
+
from .hypervec import DEFAULT_DIM, hvc, int_to_hv
|
| 33 |
+
from .isa import (
|
| 34 |
+
OPCODES,
|
| 35 |
+
REG_NAMES,
|
| 36 |
+
DecodeContext,
|
| 37 |
+
DecodedInstr,
|
| 38 |
+
Opcode,
|
| 39 |
+
decode_instr,
|
| 40 |
+
encode_instr,
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class AsmError(Exception):
|
| 45 |
+
"""Assembly failed; message carries the line number."""
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
_COMMENT = re.compile(r"(;|#|//).*$")
|
| 49 |
+
_LABEL = re.compile(r"^([A-Za-z_.][\w.]*)\s*:\s*(.*)$")
|
| 50 |
+
_DIRECTIVE = re.compile(r"^\.(\w+)\s*(.*)$", re.S)
|
| 51 |
+
_INT = re.compile(r"^-?(0[xX][0-9a-fA-F]+|\d+)$")
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
@dataclass
|
| 55 |
+
class SourceLine:
|
| 56 |
+
number: int
|
| 57 |
+
text: str
|
| 58 |
+
#: Labels that sit on or immediately above this instruction. A label-only
|
| 59 |
+
#: line carries forward to the next instruction, because that is the cell
|
| 60 |
+
#: the label actually names.
|
| 61 |
+
labels: tuple[str, ...] = ()
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
@dataclass
|
| 65 |
+
class Program:
|
| 66 |
+
"""An assembled, hypervector-encoded program."""
|
| 67 |
+
|
| 68 |
+
d: int = DEFAULT_DIM
|
| 69 |
+
instrs: list[int] = field(default_factory=list)
|
| 70 |
+
source: list[SourceLine] = field(default_factory=list)
|
| 71 |
+
labels: dict[str, int] = field(default_factory=dict)
|
| 72 |
+
literals: list[int] = field(default_factory=list)
|
| 73 |
+
strings: list[str] = field(default_factory=list)
|
| 74 |
+
tools: list[str] = field(default_factory=list)
|
| 75 |
+
cell_names: list[str] = field(default_factory=list)
|
| 76 |
+
data: dict[str, str] = field(default_factory=dict)
|
| 77 |
+
#: Decoded view, filled by :func:`assembler` for the IDE's listing.
|
| 78 |
+
listing: list[DecodedInstr] = field(default_factory=list)
|
| 79 |
+
|
| 80 |
+
# ------------------------------------------------------------- install
|
| 81 |
+
def install(self, storage, prefix: str = "code") -> list[str]:
|
| 82 |
+
"""Write the program into HDC storage. Returns the code cell names."""
|
| 83 |
+
names: list[str] = []
|
| 84 |
+
for i, hv in enumerate(self.instrs):
|
| 85 |
+
name = "%s/%d" % (prefix, i)
|
| 86 |
+
storage.write(name, hv)
|
| 87 |
+
names.append(name)
|
| 88 |
+
storage.write("program/length", int_to_hv(len(self.instrs), self.d))
|
| 89 |
+
for name, spec in self.data.items():
|
| 90 |
+
kind, _, value = spec.partition(":")
|
| 91 |
+
if kind == "str":
|
| 92 |
+
storage.write(name, hvc("str/" + value, self.d), symbol=True)
|
| 93 |
+
else:
|
| 94 |
+
storage.write(name, int_to_hv(int(value), self.d), symbol=True)
|
| 95 |
+
for name in self.cell_names:
|
| 96 |
+
if name not in storage:
|
| 97 |
+
storage.write(name, 0, symbol=True)
|
| 98 |
+
for label, idx in self.labels.items():
|
| 99 |
+
storage.define("label/" + label)
|
| 100 |
+
return names
|
| 101 |
+
|
| 102 |
+
def context(self) -> DecodeContext:
|
| 103 |
+
return DecodeContext(
|
| 104 |
+
d=self.d,
|
| 105 |
+
imms=tuple(self.literals),
|
| 106 |
+
tgts=tuple(range(len(self.instrs))),
|
| 107 |
+
names=tuple(self.cell_names),
|
| 108 |
+
strs=tuple(self.strings),
|
| 109 |
+
tools=tuple(self.tools),
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
def listing_text(self) -> str:
|
| 113 |
+
lines = []
|
| 114 |
+
for i, line in enumerate(self.source):
|
| 115 |
+
for lbl in line.labels:
|
| 116 |
+
lines.append("%s:" % lbl)
|
| 117 |
+
body = " " + line.text
|
| 118 |
+
dec = self.listing[i] if i < len(self.listing) else None
|
| 119 |
+
conf = "" if dec is None else " ; conf %.2f" % dec.confidence
|
| 120 |
+
lines.append("%-44s%s" % (body, conf))
|
| 121 |
+
return "\n".join(lines)
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
# --------------------------------------------------------------------------
|
| 125 |
+
# parsing helpers
|
| 126 |
+
# --------------------------------------------------------------------------
|
| 127 |
+
def split_operands(text: str) -> list[str]:
|
| 128 |
+
"""Split an operand list on commas, respecting double-quoted strings."""
|
| 129 |
+
if not text.strip():
|
| 130 |
+
return []
|
| 131 |
+
parts: list[str] = []
|
| 132 |
+
cur: list[str] = []
|
| 133 |
+
quoted = False
|
| 134 |
+
for ch in text:
|
| 135 |
+
if ch == '"':
|
| 136 |
+
quoted = not quoted
|
| 137 |
+
cur.append(ch)
|
| 138 |
+
elif ch == "," and not quoted:
|
| 139 |
+
parts.append("".join(cur).strip())
|
| 140 |
+
cur = []
|
| 141 |
+
else:
|
| 142 |
+
cur.append(ch)
|
| 143 |
+
tail = "".join(cur).strip()
|
| 144 |
+
if tail:
|
| 145 |
+
parts.append(tail)
|
| 146 |
+
return parts
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def _parse_int(token: str) -> int:
|
| 150 |
+
if not _INT.match(token):
|
| 151 |
+
raise ValueError("not an integer literal: %r" % (token,))
|
| 152 |
+
return int(token, 0) if token.lower().startswith(("0x", "-0x")) else int(token)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def _parse_reg(token: str) -> int:
|
| 156 |
+
t = token.strip().upper()
|
| 157 |
+
if t.startswith("R"):
|
| 158 |
+
t = t[1:]
|
| 159 |
+
if not t.isdigit():
|
| 160 |
+
raise ValueError("not a register: %r" % (token,))
|
| 161 |
+
idx = int(t)
|
| 162 |
+
if not 0 <= idx < len(REG_NAMES):
|
| 163 |
+
raise ValueError("register out of range: %r" % (token,))
|
| 164 |
+
return idx
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def _strip_string(token: str) -> str:
|
| 168 |
+
t = token.strip()
|
| 169 |
+
if len(t) >= 2 and t[0] == '"' and t[-1] == '"':
|
| 170 |
+
return t[1:-1]
|
| 171 |
+
raise ValueError("expected a quoted string, got %r" % (token,))
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def _resolve_text(token: str, data: dict) -> str:
|
| 175 |
+
"""A string operand is either quoted inline or the name of a .str cell.
|
| 176 |
+
|
| 177 |
+
Deferring this to after pass 1 is what lets ``prints greeting`` refer to a
|
| 178 |
+
``.str greeting "..."`` declared further down the file.
|
| 179 |
+
"""
|
| 180 |
+
t = token.strip()
|
| 181 |
+
if len(t) >= 2 and t[0] == '"' and t[-1] == '"':
|
| 182 |
+
return t[1:-1]
|
| 183 |
+
spec = data.get(t, "")
|
| 184 |
+
if isinstance(spec, str) and spec.startswith("str:"):
|
| 185 |
+
return spec[4:]
|
| 186 |
+
raise ValueError(
|
| 187 |
+
"unquoted string operand %r is not a declared .str cell" % (t,)
|
| 188 |
+
)
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def _scan_operand(
|
| 192 |
+
literals: dict,
|
| 193 |
+
strings: dict,
|
| 194 |
+
names: dict,
|
| 195 |
+
tools: dict,
|
| 196 |
+
str_tokens: list,
|
| 197 |
+
kind: str,
|
| 198 |
+
token: str,
|
| 199 |
+
) -> None:
|
| 200 |
+
"""Record a token into the alphabet pass-2 decoding will need.
|
| 201 |
+
|
| 202 |
+
These four sets become the cleanup alphabets for the whole program, so
|
| 203 |
+
they must be complete before the first instruction is composed. Labels
|
| 204 |
+
need no entry: pass 1 has already seen every definition.
|
| 205 |
+
"""
|
| 206 |
+
if kind == "imm":
|
| 207 |
+
literals.setdefault(_parse_int(token), None)
|
| 208 |
+
elif kind == "str":
|
| 209 |
+
# deferred: .str declarations anywhere in the file are visible
|
| 210 |
+
str_tokens.append(token)
|
| 211 |
+
elif kind == "name":
|
| 212 |
+
names.setdefault(token, None)
|
| 213 |
+
elif kind == "tool":
|
| 214 |
+
tools.setdefault(token, None)
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
# --------------------------------------------------------------------------
|
| 218 |
+
# the compiler
|
| 219 |
+
# --------------------------------------------------------------------------
|
| 220 |
+
def assembler(source: str | list[str], d: int = DEFAULT_DIM, strict: bool = True) -> Program:
|
| 221 |
+
"""Compile assembler text into a :class:`Program`."""
|
| 222 |
+
if isinstance(source, str):
|
| 223 |
+
raw_lines = source.splitlines()
|
| 224 |
+
else:
|
| 225 |
+
raw_lines = list(source)
|
| 226 |
+
|
| 227 |
+
# ---------------------------------------------------------- pass 1
|
| 228 |
+
prog = Program(d=d)
|
| 229 |
+
labels: dict[str, int] = {}
|
| 230 |
+
literals: dict[int, None] = {}
|
| 231 |
+
strings: dict[str, None] = {}
|
| 232 |
+
names: dict[str, None] = {}
|
| 233 |
+
tools: dict[str, None] = {}
|
| 234 |
+
str_tokens: list[str] = []
|
| 235 |
+
pending_labels: list[str] = []
|
| 236 |
+
#: Instruction lines only, so ``labels[name]`` is an instruction index and
|
| 237 |
+
#: ``body[i]`` is the i-th instruction. Directives emit no cells.
|
| 238 |
+
body: list[SourceLine] = []
|
| 239 |
+
|
| 240 |
+
for n, raw in enumerate(raw_lines, 1):
|
| 241 |
+
text = _COMMENT.sub("", raw).strip()
|
| 242 |
+
if not text:
|
| 243 |
+
continue
|
| 244 |
+
label = None
|
| 245 |
+
m = _LABEL.match(text)
|
| 246 |
+
if m:
|
| 247 |
+
label = m.group(1)
|
| 248 |
+
text = m.group(2).strip()
|
| 249 |
+
if label in labels:
|
| 250 |
+
raise AsmError("line %d: duplicate label %r" % (n, label))
|
| 251 |
+
labels[label] = len(body)
|
| 252 |
+
pending_labels.append(label)
|
| 253 |
+
if not text:
|
| 254 |
+
continue
|
| 255 |
+
|
| 256 |
+
dm = _DIRECTIVE.match(text)
|
| 257 |
+
if dm:
|
| 258 |
+
dname, rest = dm.group(1).lower(), dm.group(2).strip()
|
| 259 |
+
if dname == "str":
|
| 260 |
+
# directives are whitespace separated: .str NAME "some text"
|
| 261 |
+
parts = rest.split(None, 1)
|
| 262 |
+
if len(parts) != 2:
|
| 263 |
+
raise AsmError('line %d: .str NAME "text"' % n)
|
| 264 |
+
txt = _strip_string(parts[1])
|
| 265 |
+
strings.setdefault(txt, None)
|
| 266 |
+
names.setdefault(parts[0], None)
|
| 267 |
+
prog.data[parts[0]] = "str:" + txt
|
| 268 |
+
elif dname == "data":
|
| 269 |
+
parts = rest.split(None, 1)
|
| 270 |
+
if len(parts) != 2:
|
| 271 |
+
raise AsmError("line %d: .data NAME VALUE" % n)
|
| 272 |
+
names.setdefault(parts[0], None)
|
| 273 |
+
prog.data[parts[0]] = "int:%d" % _parse_int(parts[1])
|
| 274 |
+
elif dname == "name":
|
| 275 |
+
names.setdefault(rest, None)
|
| 276 |
+
elif dname == "tool":
|
| 277 |
+
tools.setdefault(rest, None)
|
| 278 |
+
else:
|
| 279 |
+
raise AsmError("line %d: unknown directive .%s" % (n, dname))
|
| 280 |
+
# A label on a directive names no program cell.
|
| 281 |
+
pending_labels.clear()
|
| 282 |
+
continue
|
| 283 |
+
|
| 284 |
+
mn, _, operand_text = text.partition(" ")
|
| 285 |
+
mn = mn.upper()
|
| 286 |
+
if mn not in OPCODES:
|
| 287 |
+
raise AsmError("line %d: unknown opcode %r" % (n, mn))
|
| 288 |
+
ops = split_operands(operand_text)
|
| 289 |
+
spec = OPCODES[mn]
|
| 290 |
+
want = spec.operand_list
|
| 291 |
+
if len(ops) != len(want):
|
| 292 |
+
raise AsmError(
|
| 293 |
+
"line %d: %s takes %d operand(s) (%s), got %d"
|
| 294 |
+
% (n, mn, len(want), spec.operands, len(ops))
|
| 295 |
+
)
|
| 296 |
+
try:
|
| 297 |
+
for kind, tok in zip(want, ops):
|
| 298 |
+
_scan_operand(literals, strings, names, tools, str_tokens, kind, tok)
|
| 299 |
+
except ValueError as exc:
|
| 300 |
+
raise AsmError("line %d: %s" % (n, exc)) from exc
|
| 301 |
+
body.append(SourceLine(n, text, tuple(pending_labels)))
|
| 302 |
+
pending_labels.clear()
|
| 303 |
+
|
| 304 |
+
# Deferred resolution: string operands may name a .str cell declared later,
|
| 305 |
+
# and .data values join the literal alphabet so a value preloaded into
|
| 306 |
+
# storage can also be decoded out of an instruction field.
|
| 307 |
+
for tok in str_tokens:
|
| 308 |
+
try:
|
| 309 |
+
strings.setdefault(_resolve_text(tok, prog.data), None)
|
| 310 |
+
except ValueError as exc:
|
| 311 |
+
raise AsmError(str(exc)) from exc
|
| 312 |
+
for spec in prog.data.values():
|
| 313 |
+
if spec.startswith("int:"):
|
| 314 |
+
literals.setdefault(int(spec[4:]), None)
|
| 315 |
+
elif spec.startswith("str:"):
|
| 316 |
+
strings.setdefault(spec[4:], None)
|
| 317 |
+
|
| 318 |
+
prog.labels = labels
|
| 319 |
+
prog.literals = sorted(literals)
|
| 320 |
+
prog.strings = sorted(strings)
|
| 321 |
+
prog.tools = sorted(tools)
|
| 322 |
+
prog.cell_names = sorted(names)
|
| 323 |
+
|
| 324 |
+
# ---------------------------------------------------------- pass 2
|
| 325 |
+
for line in body:
|
| 326 |
+
text = line.text
|
| 327 |
+
if _DIRECTIVE.match(text):
|
| 328 |
+
continue
|
| 329 |
+
mn, _, operand_text = text.partition(" ")
|
| 330 |
+
mn = mn.upper()
|
| 331 |
+
spec: Opcode = OPCODES[mn]
|
| 332 |
+
ops = split_operands(operand_text)
|
| 333 |
+
regs: list[int] = []
|
| 334 |
+
kwargs: dict = {}
|
| 335 |
+
try:
|
| 336 |
+
for kind, tok in zip(spec.operand_list, ops):
|
| 337 |
+
if kind == "reg":
|
| 338 |
+
regs.append(_parse_reg(tok))
|
| 339 |
+
elif kind == "imm":
|
| 340 |
+
kwargs["imm"] = _parse_int(tok)
|
| 341 |
+
elif kind == "label":
|
| 342 |
+
if tok.startswith("@"):
|
| 343 |
+
kwargs["tgt"] = int(tok[1:])
|
| 344 |
+
elif tok in labels:
|
| 345 |
+
kwargs["tgt"] = labels[tok]
|
| 346 |
+
else:
|
| 347 |
+
raise ValueError("undefined label %r" % (tok,))
|
| 348 |
+
elif kind == "name":
|
| 349 |
+
kwargs["name"] = tok
|
| 350 |
+
elif kind == "str":
|
| 351 |
+
kwargs["text"] = _resolve_text(tok, prog.data)
|
| 352 |
+
elif kind == "tool":
|
| 353 |
+
kwargs["tool"] = tok
|
| 354 |
+
hv = encode_instr(mn, regs=regs, d=d, **kwargs)
|
| 355 |
+
except (ValueError, KeyError) as exc:
|
| 356 |
+
raise AsmError("line %d: %s" % (line.number, exc)) from exc
|
| 357 |
+
prog.instrs.append(hv)
|
| 358 |
+
|
| 359 |
+
# ------------------------------------------------- decode-back check
|
| 360 |
+
prog.source = body
|
| 361 |
+
ctx = prog.context()
|
| 362 |
+
for i, hv in enumerate(prog.instrs):
|
| 363 |
+
try:
|
| 364 |
+
dec = decode_instr(hv, ctx)
|
| 365 |
+
except Exception as exc: # pragma: no cover - defensive
|
| 366 |
+
raise AsmError("internal: cell %d failed to decode: %s" % (i, exc)) from exc
|
| 367 |
+
if strict and dec.confidence < 0.15:
|
| 368 |
+
raise AsmError(
|
| 369 |
+
"cell %d decoded weakly (conf %.2f); check operands" % (i, dec.confidence)
|
| 370 |
+
)
|
| 371 |
+
prog.listing.append(dec)
|
| 372 |
+
return prog
|
| 373 |
+
|
| 374 |
+
|
| 375 |
+
def assemble(source: str | list[str], d: int = DEFAULT_DIM, strict: bool = True) -> Program:
|
| 376 |
+
"""Alias kept for the public API."""
|
| 377 |
+
return assembler(source, d=d, strict=strict)
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
def roundtrip_report(prog: Program) -> str:
|
| 381 |
+
"""Self-check: does every stored cell decode back to its source line?"""
|
| 382 |
+
ctx = prog.context()
|
| 383 |
+
rows = []
|
| 384 |
+
bad = 0
|
| 385 |
+
for i, hv in enumerate(prog.instrs):
|
| 386 |
+
dec = decode_instr(hv, ctx)
|
| 387 |
+
src = prog.source[i].text.strip()
|
| 388 |
+
rows.append("%3d %-30s -> %-30s conf=%.2f" % (i, src, dec.op, dec.confidence))
|
| 389 |
+
if dec.op not in src.upper():
|
| 390 |
+
bad += 1
|
| 391 |
+
head = "decode-back check: %d/%d cells round-trip" % (len(prog.instrs) - bad, len(prog.instrs))
|
| 392 |
+
return "\n".join([head] + rows)
|
hdc/assistant.py
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The assistant: TextRing encoding + predictive model + real tool calls.
|
| 2 |
+
|
| 3 |
+
Honest about what it is. This is **not** an LLM. It has three parts, each of
|
| 4 |
+
which already existed in the project:
|
| 5 |
+
|
| 6 |
+
1. **Encoding.** An utterance is a bundle of its word vectors —
|
| 7 |
+
:mod:`hdc.text` n-grams by default, or the semantic encoder from
|
| 8 |
+
:mod:`hdc.semantic` when the model file is present. That bundle is the
|
| 9 |
+
utterance's hypervector.
|
| 10 |
+
2. **Classification.** A :class:`~hdc.state.PredictiveModel` maps utterance
|
| 11 |
+
vectors to intent vectors, with key snapping so a paraphrase can resolve to
|
| 12 |
+
a trained intent. The output is an intent plus a cleanup similarity, and
|
| 13 |
+
the similarity is reported to the user rather than hidden.
|
| 14 |
+
3. **Agency.** An intent names a kernel command. The assistant *runs it* and
|
| 15 |
+
returns what the machine actually printed, so a reply can carry a decoded
|
| 16 |
+
instruction listing, a run result, or a layer report instead of prose.
|
| 17 |
+
|
| 18 |
+
That structure is why it is useful inside this machine and also why it is
|
| 19 |
+
limited: the intent set is fixed at construction, the encoder knows nothing a
|
| 20 |
+
half-second of training cannot teach, and when no intent clears the threshold
|
| 21 |
+
it says so instead of inventing an answer.
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
from __future__ import annotations
|
| 25 |
+
|
| 26 |
+
from typing import Sequence
|
| 27 |
+
|
| 28 |
+
from .hypervec import DEFAULT_DIM, bundle, hvc, sim
|
| 29 |
+
from .state import PredictiveModel
|
| 30 |
+
from .text import encode_word, tokenize
|
| 31 |
+
|
| 32 |
+
#: intent -> (example utterances, kernel command to run)
|
| 33 |
+
INTENTS: dict[str, tuple[list[str], str]] = {
|
| 34 |
+
"greet": (["hello", "hi there", "good morning", "hey", "greetings"],
|
| 35 |
+
""),
|
| 36 |
+
"help": (["what can you do", "show me the commands", "help me", "list commands"],
|
| 37 |
+
"help"),
|
| 38 |
+
"assemble": (["assemble a program", "compile this assembly", "build the demo program",
|
| 39 |
+
"assemble code"], "asm"),
|
| 40 |
+
"run": (["run the program", "execute it", "start the program", "run the code now"],
|
| 41 |
+
"run"),
|
| 42 |
+
"step": (["step one instruction", "single step", "advance the machine",
|
| 43 |
+
"step the cpu"], "step 1"),
|
| 44 |
+
"registers": (["show the registers", "what is in the registers", "register file",
|
| 45 |
+
"print regs", "what registers exist", "register contents",
|
| 46 |
+
"dump registers"], "regs"),
|
| 47 |
+
"disassemble": (["disassemble the program", "show the next instructions",
|
| 48 |
+
"what is the next instruction"], "disasm"),
|
| 49 |
+
"storage": (["show memory", "what is in storage", "dump the cells", "list cells"],
|
| 50 |
+
"mem"),
|
| 51 |
+
"status": (["what is the status", "machine status", "how many cells",
|
| 52 |
+
"show statistics"], "stat"),
|
| 53 |
+
"isa": (["show the instruction set", "what opcodes exist", "list opcodes",
|
| 54 |
+
"isa reference"], "isa"),
|
| 55 |
+
"shader": (["run the shader demo", "show the image pipeline", "edges and depth",
|
| 56 |
+
"shader report"], "shader"),
|
| 57 |
+
"ring": (["run the ring controller", "past present future", "show the history ledger",
|
| 58 |
+
"ring report"], "ring"),
|
| 59 |
+
"state": (["show the predictive state", "plan a path", "state report",
|
| 60 |
+
"what does the model predict"], "state"),
|
| 61 |
+
"memory_modes": (["photographic memory", "holographic recall", "memory modes report",
|
| 62 |
+
"how does recall work"], "memory"),
|
| 63 |
+
"topology": (["show the threshold topology", "component analysis", "self healing report",
|
| 64 |
+
"topology report"], "topology"),
|
| 65 |
+
"text": (["encode some text", "text encoder report", "character ngrams",
|
| 66 |
+
"how do you encode words"], "text"),
|
| 67 |
+
"semantic": (["semantic vectors", "word meanings", "embedding report",
|
| 68 |
+
"how close are synonyms"], "semantic"),
|
| 69 |
+
"boot": (["boot the machine", "run the bios", "start over", "reboot"],
|
| 70 |
+
"boot"),
|
| 71 |
+
"tools": (["show the tools", "what tools are registered", "tool registry"],
|
| 72 |
+
"tools"),
|
| 73 |
+
"clear": (["clear the screen", "wipe the console", "cls"], "clear"),
|
| 74 |
+
"thanks": (["thanks", "thank you", "nice", "great work"], ""),
|
| 75 |
+
}
|
| 76 |
+
|
| 77 |
+
#: Function words carry no intent and, bundled, they dominate the vector —
|
| 78 |
+
#: every utterance shares them, so prototypes collide on "the" and "of".
|
| 79 |
+
STOPWORDS = {
|
| 80 |
+
"the", "a", "an", "me", "my", "is", "are", "am", "was", "what", "how",
|
| 81 |
+
"do", "does", "it", "this", "that", "these", "those", "to", "of", "in",
|
| 82 |
+
"on", "for", "and", "or", "now", "some", "please", "can", "could", "you",
|
| 83 |
+
"i", "we", "us", "with", "at", "be", "there", "here", "up",
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
REPLIES = {
|
| 87 |
+
"greet": "Kernel is up. I can assemble, run, step and inspect — say what you want.",
|
| 88 |
+
"thanks": "Understood.",
|
| 89 |
+
"": "",
|
| 90 |
+
}
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
class Assistant:
|
| 94 |
+
"""A small, inspectable assistant over the kernel's own commands."""
|
| 95 |
+
|
| 96 |
+
def __init__(self, kernel, d: int = DEFAULT_DIM, encoder=None, threshold: float = 0.42):
|
| 97 |
+
self.kernel = kernel
|
| 98 |
+
self.d = d
|
| 99 |
+
self.encoder = encoder
|
| 100 |
+
# 0.42 keeps genuine paraphrases (measured 0.50-0.84) and rejects
|
| 101 |
+
# unrelated input (measured <= 0.38); reported, not tuned per-query
|
| 102 |
+
self.threshold = threshold
|
| 103 |
+
self.intent_vectors: dict[str, int] = {}
|
| 104 |
+
self.history: list[dict] = []
|
| 105 |
+
self.trained = 0
|
| 106 |
+
# One *prototype* per intent: the majority bundle of its examples.
|
| 107 |
+
# A single transition table holding all 82 utterances was measured at
|
| 108 |
+
# 0.05-0.09 confidence — past the majority trace's capacity — while a
|
| 109 |
+
# centroid of 4-6 members stays at ~0.5. Same rule as PrototypeNet.
|
| 110 |
+
self.prototypes: dict[str, int] = {}
|
| 111 |
+
for name, (examples, _command) in INTENTS.items():
|
| 112 |
+
self.intent_vectors[name] = hvc("intent/" + name, d)
|
| 113 |
+
self.prototypes[name] = bundle([self.encode(text) for text in examples], d)
|
| 114 |
+
self.trained += len(examples)
|
| 115 |
+
# kept for conversational follow-ups ("do that again")
|
| 116 |
+
self.model = PredictiveModel(d=d, alphabet=list(self.intent_vectors.values()))
|
| 117 |
+
for name, (examples, _command) in INTENTS.items():
|
| 118 |
+
for text in examples:
|
| 119 |
+
self.model.learn(self.encode(text), "intent", self.intent_vectors[name])
|
| 120 |
+
|
| 121 |
+
# ----------------------------------------------------------- encoding
|
| 122 |
+
def word(self, token: str) -> int:
|
| 123 |
+
if self.encoder is not None:
|
| 124 |
+
return self.encoder.encode(token)
|
| 125 |
+
return encode_word(token, self.d)
|
| 126 |
+
|
| 127 |
+
def encode(self, text: str) -> int:
|
| 128 |
+
"""An utterance as one bundled hypervector, content words only."""
|
| 129 |
+
tokens = [t for t in tokenize(text) if t not in STOPWORDS]
|
| 130 |
+
if not tokens:
|
| 131 |
+
tokens = tokenize(text) or [text.strip().lower()]
|
| 132 |
+
return bundle([self.word(token) for token in tokens], self.d)
|
| 133 |
+
|
| 134 |
+
# ------------------------------------------------------- classification
|
| 135 |
+
def classify(self, text: str) -> tuple[str, float]:
|
| 136 |
+
"""Nearest intent prototype and its similarity.
|
| 137 |
+
|
| 138 |
+
The score is a plain ``sim`` against a centroid, so it means the same
|
| 139 |
+
thing everywhere else in this library: 0.5 is "most of the members are
|
| 140 |
+
there", 0.05 is "unrelated". No softmax is applied to make it look
|
| 141 |
+
like a probability it is not.
|
| 142 |
+
"""
|
| 143 |
+
vector = self.encode(text)
|
| 144 |
+
best, best_score = "", -2.0
|
| 145 |
+
for name, prototype in self.prototypes.items():
|
| 146 |
+
score = sim(vector, prototype, self.d)
|
| 147 |
+
if score > best_score:
|
| 148 |
+
best, best_score = name, score
|
| 149 |
+
return best, best_score
|
| 150 |
+
|
| 151 |
+
# --------------------------------------------------------------- chat
|
| 152 |
+
def chat(self, text: str, execute=None) -> dict:
|
| 153 |
+
"""Classify, act, and return both the reply and what was executed.
|
| 154 |
+
|
| 155 |
+
``execute`` overrides how the named command is run. The autonomy layer
|
| 156 |
+
passes a cache-backed runner, so an answer the machine already predicted
|
| 157 |
+
costs a dict lookup instead of a kernel round trip; the default is the
|
| 158 |
+
kernel, so nothing changes for every other caller.
|
| 159 |
+
"""
|
| 160 |
+
intent, confidence = self.classify(text)
|
| 161 |
+
record = {"input": text, "intent": intent, "confidence": round(confidence, 3)}
|
| 162 |
+
if not intent or confidence < self.threshold:
|
| 163 |
+
record["reply"] = ("No intent cleared the threshold (best %.2f, need %.2f). "
|
| 164 |
+
"Try 'help' or one of the pipeline commands."
|
| 165 |
+
% (confidence, self.threshold))
|
| 166 |
+
record["output"] = ""
|
| 167 |
+
record["executed"] = ""
|
| 168 |
+
self.history.append(record)
|
| 169 |
+
return record
|
| 170 |
+
|
| 171 |
+
command = INTENTS[intent][1]
|
| 172 |
+
output = ""
|
| 173 |
+
if command:
|
| 174 |
+
output = (execute or self.kernel.execute)(command)
|
| 175 |
+
reply = REPLIES.get(intent) or ("Running '%s'. Confidence %.2f." % (command, confidence))
|
| 176 |
+
record.update({"reply": reply, "output": output, "executed": command})
|
| 177 |
+
self.history.append(record)
|
| 178 |
+
return record
|
| 179 |
+
|
| 180 |
+
# -------------------------------------------------------------- report
|
| 181 |
+
def report(self) -> str:
|
| 182 |
+
lines = ["assistant: %d intents, %d utterances trained, encoder=%s"
|
| 183 |
+
% (len(INTENTS), self.trained,
|
| 184 |
+
"semantic" if self.encoder is not None else "n-gram")]
|
| 185 |
+
for text in ("run the program", "show me the registers", "tell me a joke"):
|
| 186 |
+
intent, confidence = self.classify(text)
|
| 187 |
+
lines.append(" %-28s -> %-12s %.2f" % (text, intent or "(none)", confidence))
|
| 188 |
+
return "\n".join(lines)
|
hdc/autonomy.py
ADDED
|
@@ -0,0 +1,381 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Idle, learning, readahead: the assistant's autonomous mode.
|
| 2 |
+
|
| 3 |
+
The assistant on its own is reactive — it answers what it is asked. This module
|
| 4 |
+
adds the three behaviours that make it *predictive* instead, and each one is a
|
| 5 |
+
measured quantity rather than a mood:
|
| 6 |
+
|
| 7 |
+
**Idle.** Between utterances there is time the machine would otherwise waste.
|
| 8 |
+
An idle tick spends it on work that is useful whichever way the conversation
|
| 9 |
+
goes: replaying history, computing what comes next, and pre-running the
|
| 10 |
+
read-only inspection commands that are most likely to be asked for.
|
| 11 |
+
|
| 12 |
+
**Learning.** Every turn leaves an intent. The sequence of intents is a trace
|
| 13 |
+
of behaviour, and ``(previous, next)`` pairs are learned into a
|
| 14 |
+
:class:`~hdc.state.PredictiveModel` — the same holographic transition table the
|
| 15 |
+
state layer uses. Idle *replays* that trace, and replay is not decorative: a
|
| 16 |
+
majority trace's cleanup confidence rises with repeated evidence, so a cycle
|
| 17 |
+
that was noisy after one lap sharpens on the next. Measured: see
|
| 18 |
+
:meth:`AutonomousMode.report`.
|
| 19 |
+
|
| 20 |
+
**Readahead.** From the last intent, predict the next ``horizon`` intents by
|
| 21 |
+
chaining the transition model, each hop cleaned up against the intent codebook
|
| 22 |
+
with its own similarity. ``PAST (trace) -> PRESENT (last intent) -> FUTURE
|
| 23 |
+
(readahead) -> PAST (graded on the next turn)``: every prediction is stored and
|
| 24 |
+
graded when the actual next utterance is classified. That grading is the
|
| 25 |
+
learning signal, and its rate is the number the report leads with.
|
| 26 |
+
|
| 27 |
+
**Autonomy, bounded.** Only commands on :data:`SAFE_COMMANDS` may be executed
|
| 28 |
+
without being asked, with a per-idle-call budget and a confidence floor. They
|
| 29 |
+
are read-only inspections, so the worst case of a wrong prediction is wasted
|
| 30 |
+
cycles, never a changed machine. When the user does ask for one of them, the
|
| 31 |
+
answer is already computed — which is the point: readahead trades idle time for
|
| 32 |
+
response time.
|
| 33 |
+
|
| 34 |
+
Honest limits: intents are a fixed set, so this predicts *which* command comes
|
| 35 |
+
next and not what the user will say; a first-order chain has no notion of a
|
| 36 |
+
conversation's subject, only of its recent shape; and the confidence numbers are
|
| 37 |
+
cleanup similarities, not probabilities.
|
| 38 |
+
"""
|
| 39 |
+
|
| 40 |
+
from __future__ import annotations
|
| 41 |
+
|
| 42 |
+
import time
|
| 43 |
+
from typing import Sequence
|
| 44 |
+
|
| 45 |
+
from .hypervec import DEFAULT_DIM, bundle, hvc, permute
|
| 46 |
+
from .state import PredictiveModel
|
| 47 |
+
|
| 48 |
+
#: Commands the autonomous mode may run unasked: read-only inspection only.
|
| 49 |
+
#: ``run``, ``step``, ``asm``, ``boot`` and ``clear`` are deliberately absent —
|
| 50 |
+
#: predicting the wrong one of those would change the machine.
|
| 51 |
+
SAFE_COMMANDS = frozenset({
|
| 52 |
+
"help", "isa", "stat", "regs", "mem", "disasm", "tools", "shader", "ring",
|
| 53 |
+
"state", "memory", "topology", "text", "semantic",
|
| 54 |
+
})
|
| 55 |
+
|
| 56 |
+
#: Confidence a prediction needs before it may execute during idle.
|
| 57 |
+
#:
|
| 58 |
+
#: Set from a measured session, not from taste. Cleanup over a 21-symbol
|
| 59 |
+
#: codebook has a chance level of ``1/sqrt(21) = 0.218``, and after one lap of
|
| 60 |
+
#: learning this model's correct next-intent predictions scored 0.18-0.39
|
| 61 |
+
#: (wrong ones, made cold, scored 0.00-0.06). Sweeping the floor over that
|
| 62 |
+
#: trace: 0.25 covers 14 of 24 turns at precision 1.00, 0.30 covers 9, and 0.45
|
| 63 |
+
#: covers none at all — which is why the earlier 0.45 made autonomy do nothing.
|
| 64 |
+
#: Executing a wrong prediction costs idle cycles and nothing else: the command
|
| 65 |
+
#: set below is read-only.
|
| 66 |
+
ACT_FLOOR = 0.25
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
class AutonomousMode:
|
| 70 |
+
"""Idle ticks, transition learning, and graded readahead over the assistant."""
|
| 71 |
+
|
| 72 |
+
def __init__(self, assistant, *, d: int | None = None, horizon: int = 3,
|
| 73 |
+
order: int = 1, budget: int = 4, act_floor: float = ACT_FLOOR,
|
| 74 |
+
enabled: bool = False):
|
| 75 |
+
self.assistant = assistant
|
| 76 |
+
self.d = d or assistant.d
|
| 77 |
+
self.horizon = horizon
|
| 78 |
+
self.order = order
|
| 79 |
+
self.budget = budget
|
| 80 |
+
self.act_floor = act_floor
|
| 81 |
+
self.enabled = enabled
|
| 82 |
+
|
| 83 |
+
self.intents: dict[str, int] = dict(assistant.intent_vectors)
|
| 84 |
+
self._names: dict[int, str] = {vector: name for name, vector in self.intents.items()}
|
| 85 |
+
self.model = PredictiveModel(d=self.d, alphabet=list(self.intents.values()))
|
| 86 |
+
self.sequence: list[str] = [] # observed intents, oldest first
|
| 87 |
+
self._learned = 0 # pairs already recorded
|
| 88 |
+
|
| 89 |
+
self.readahead: list[dict] = []
|
| 90 |
+
self.graded: int = 0
|
| 91 |
+
self.hits: int = 0
|
| 92 |
+
self.ticks: int = 0
|
| 93 |
+
self.replays: int = 0
|
| 94 |
+
self.actions: list[dict] = []
|
| 95 |
+
self.cache: dict[str, dict] = {} # command -> {"output", "ms"}
|
| 96 |
+
self.served: int = 0
|
| 97 |
+
self.ms_saved: float = 0.0
|
| 98 |
+
self.at_floor_skips: int = 0
|
| 99 |
+
self.log: list[dict] = []
|
| 100 |
+
|
| 101 |
+
# ------------------------------------------------------------- learning
|
| 102 |
+
def _key(self, window: Sequence[int]) -> int:
|
| 103 |
+
"""Context for an ``order``-long window of intent vectors.
|
| 104 |
+
|
| 105 |
+
Position-permuted so ``help->run`` and ``run->help`` are different keys;
|
| 106 |
+
with ``order=1`` it collapses to the intent itself, which is what a
|
| 107 |
+
deterministic command cycle needs.
|
| 108 |
+
"""
|
| 109 |
+
if len(window) == 1:
|
| 110 |
+
return window[0]
|
| 111 |
+
return bundle([permute(vector, -i, self.d) for i, vector in enumerate(reversed(window))],
|
| 112 |
+
self.d)
|
| 113 |
+
|
| 114 |
+
def _window(self) -> list[int]:
|
| 115 |
+
return [self.intents[name] for name in self.sequence[-self.order:]]
|
| 116 |
+
|
| 117 |
+
def learn_pending(self, epochs: int = 1) -> int:
|
| 118 |
+
"""Learn every pair not yet recorded, ``epochs`` times over.
|
| 119 |
+
|
| 120 |
+
Replay is genuine training for a majority trace: the same pair recorded
|
| 121 |
+
again increases its weight against crosstalk, so cleanup confidence — and
|
| 122 |
+
therefore prediction accuracy — improves with repetition.
|
| 123 |
+
"""
|
| 124 |
+
start = max(self._learned, self.order - 1)
|
| 125 |
+
learned = 0
|
| 126 |
+
for _ in range(max(1, epochs)):
|
| 127 |
+
for i in range(start, len(self.sequence) - 1):
|
| 128 |
+
window = [self.intents[name] for name in self.sequence[i - self.order + 1:i + 1]]
|
| 129 |
+
self.model.learn(self._key(window), "next", self.intents[self.sequence[i + 1]])
|
| 130 |
+
learned += 1
|
| 131 |
+
self.replays += 1
|
| 132 |
+
self._learned = max(self._learned, len(self.sequence) - 1)
|
| 133 |
+
return learned
|
| 134 |
+
|
| 135 |
+
def observe(self, intent: str, *, learn: bool = True) -> dict:
|
| 136 |
+
"""Record one observed intent and grade the prediction made for it.
|
| 137 |
+
|
| 138 |
+
Returns the grading: which intent was predicted, whether it was right,
|
| 139 |
+
and the confidence that prediction carried before the answer was known.
|
| 140 |
+
"""
|
| 141 |
+
expected = self.readahead[0] if self.readahead else None
|
| 142 |
+
verdict = {"predicted": (expected or {}).get("intent"), "actual": intent,
|
| 143 |
+
"confidence": (expected or {}).get("confidence", 0.0),
|
| 144 |
+
"hit": bool(expected and expected.get("intent") == intent)}
|
| 145 |
+
if expected is not None:
|
| 146 |
+
self.graded += 1
|
| 147 |
+
if verdict["hit"]:
|
| 148 |
+
self.hits += 1
|
| 149 |
+
self.sequence.append(intent)
|
| 150 |
+
if learn:
|
| 151 |
+
self.learn_pending()
|
| 152 |
+
self.readahead = self.predict()
|
| 153 |
+
return verdict
|
| 154 |
+
|
| 155 |
+
# ----------------------------------------------------------- prediction
|
| 156 |
+
def name_of(self, vector: int | None) -> str | None:
|
| 157 |
+
return self._names.get(vector) if vector is not None else None
|
| 158 |
+
|
| 159 |
+
def predict(self, horizon: int | None = None) -> list[dict]:
|
| 160 |
+
"""Chain the transition model forward from the last observed intent.
|
| 161 |
+
|
| 162 |
+
Each hop is cleaned up against the intent codebook; ``path`` is the
|
| 163 |
+
running confidence product, ``confidence`` the raw hop similarity, so a
|
| 164 |
+
three-step guess cannot look as certain as a one-step one.
|
| 165 |
+
"""
|
| 166 |
+
horizon = self.horizon if horizon is None else horizon
|
| 167 |
+
window = self._window()
|
| 168 |
+
if not window:
|
| 169 |
+
return []
|
| 170 |
+
state = self._key(window)
|
| 171 |
+
out: list[dict] = []
|
| 172 |
+
seen: set[str] = set()
|
| 173 |
+
path = 1.0
|
| 174 |
+
for depth in range(1, horizon + 1):
|
| 175 |
+
predicted, confidence = self.model.predict(state, "next")
|
| 176 |
+
name = self.name_of(predicted)
|
| 177 |
+
if name is None or confidence <= 0.0:
|
| 178 |
+
break
|
| 179 |
+
path *= confidence
|
| 180 |
+
out.append({"intent": name, "command": self._command(name),
|
| 181 |
+
"confidence": round(confidence, 3), "path": round(path, 3),
|
| 182 |
+
"depth": depth, "cached": self._command(name) in self.cache})
|
| 183 |
+
seen.add(name)
|
| 184 |
+
# chain from the *predicted* vector, not from a re-encode, so the
|
| 185 |
+
# whole lookahead stays in the vector space the model was built in
|
| 186 |
+
state = predicted
|
| 187 |
+
if len(seen) == len(self.intents):
|
| 188 |
+
break
|
| 189 |
+
return out
|
| 190 |
+
|
| 191 |
+
def _command(self, intent: str) -> str:
|
| 192 |
+
from .assistant import INTENTS
|
| 193 |
+
|
| 194 |
+
return INTENTS.get(intent, ([], ""))[1]
|
| 195 |
+
|
| 196 |
+
# ---------------------------------------------------------------- idle
|
| 197 |
+
def tick(self) -> dict:
|
| 198 |
+
"""One idle cycle: replay, predict, and pre-execute what is safe."""
|
| 199 |
+
self.ticks += 1
|
| 200 |
+
learned = self.learn_pending()
|
| 201 |
+
self.readahead = self.predict()
|
| 202 |
+
acted = []
|
| 203 |
+
if self.enabled:
|
| 204 |
+
acted = self._act()
|
| 205 |
+
entry = {"tick": self.ticks, "learned": learned, "acted": acted,
|
| 206 |
+
"readahead": [item["intent"] for item in self.readahead[:3]]}
|
| 207 |
+
self.log.append(entry)
|
| 208 |
+
return entry
|
| 209 |
+
|
| 210 |
+
def idle(self, ticks: int = 1) -> list[dict]:
|
| 211 |
+
return [self.tick() for _ in range(max(0, ticks))]
|
| 212 |
+
|
| 213 |
+
def _act(self, budget: int | None = None) -> list[dict]:
|
| 214 |
+
"""Pre-run the predicted read-only commands, within the budget."""
|
| 215 |
+
budget = self.budget if budget is None else budget
|
| 216 |
+
acted: list[dict] = []
|
| 217 |
+
for prediction in self.readahead:
|
| 218 |
+
if len(acted) >= budget:
|
| 219 |
+
break
|
| 220 |
+
command = prediction["command"]
|
| 221 |
+
if not command or command.split()[0] not in SAFE_COMMANDS:
|
| 222 |
+
continue
|
| 223 |
+
if command in self.cache:
|
| 224 |
+
continue
|
| 225 |
+
if prediction["confidence"] < self.act_floor:
|
| 226 |
+
self.at_floor_skips += 1
|
| 227 |
+
continue
|
| 228 |
+
started = time.perf_counter()
|
| 229 |
+
output = self.assistant.kernel.execute(command)
|
| 230 |
+
elapsed = (time.perf_counter() - started) * 1000.0
|
| 231 |
+
self.cache[command] = {"output": output, "ms": elapsed,
|
| 232 |
+
"confidence": prediction["confidence"]}
|
| 233 |
+
record = {"command": command, "intent": prediction["intent"],
|
| 234 |
+
"confidence": prediction["confidence"], "ms": round(elapsed, 2)}
|
| 235 |
+
acted.append(record)
|
| 236 |
+
self.actions.append(record)
|
| 237 |
+
return acted
|
| 238 |
+
|
| 239 |
+
# ----------------------------------------------------------------- chat
|
| 240 |
+
def chat(self, text: str) -> dict:
|
| 241 |
+
"""``Assistant.chat`` with the readahead cache in front of execution."""
|
| 242 |
+
intent, confidence = self.assistant.classify(text)
|
| 243 |
+
command = self._command(intent) if intent else ""
|
| 244 |
+
served = command and command in self.cache and confidence >= self.assistant.threshold
|
| 245 |
+
if served:
|
| 246 |
+
cached = self.cache[command]
|
| 247 |
+
self.served += 1
|
| 248 |
+
self.ms_saved += cached["ms"]
|
| 249 |
+
record = self.assistant.chat(text, execute=self._executor if served else None)
|
| 250 |
+
if served:
|
| 251 |
+
record["output"] = self.cache[command]["output"]
|
| 252 |
+
record["cached"] = True
|
| 253 |
+
record["saved_ms"] = round(self.cache[command]["ms"], 2)
|
| 254 |
+
verdict = self.observe(intent) if intent else None
|
| 255 |
+
record["graded"] = verdict
|
| 256 |
+
if self.enabled:
|
| 257 |
+
self.tick()
|
| 258 |
+
return record
|
| 259 |
+
|
| 260 |
+
def _executor(self, command: str) -> str:
|
| 261 |
+
cached = self.cache.get(command)
|
| 262 |
+
if cached is not None:
|
| 263 |
+
return cached["output"]
|
| 264 |
+
return self.assistant.kernel.execute(command)
|
| 265 |
+
|
| 266 |
+
# --------------------------------------------------------------- report
|
| 267 |
+
def chance(self) -> float:
|
| 268 |
+
"""Cleanup similarity a random symbol scores against the intent codebook."""
|
| 269 |
+
return 1.0 / max(1.0, len(self.intents) ** 0.5)
|
| 270 |
+
|
| 271 |
+
def stats(self) -> dict:
|
| 272 |
+
accuracy = (self.hits / self.graded) if self.graded else 0.0
|
| 273 |
+
return {
|
| 274 |
+
"enabled": self.enabled, "ticks": self.ticks, "replays": self.replays,
|
| 275 |
+
"observed": len(self.sequence), "transitions": self.model.transitions,
|
| 276 |
+
"graded": self.graded, "hits": self.hits, "accuracy": round(accuracy, 3),
|
| 277 |
+
"chance": round(self.chance(), 3),
|
| 278 |
+
"readahead": self.readahead, "cached_commands": sorted(self.cache),
|
| 279 |
+
"autonomous_actions": len(self.actions), "served_from_readahead": self.served,
|
| 280 |
+
"ms_saved": round(self.ms_saved, 1), "act_floor": self.act_floor,
|
| 281 |
+
"budget": self.budget,
|
| 282 |
+
}
|
| 283 |
+
|
| 284 |
+
def report(self) -> str:
|
| 285 |
+
stats = self.stats()
|
| 286 |
+
lines = ["autonomy: %s, horizon %d, order %d, %.2f confidence floor"
|
| 287 |
+
% ("enabled" if self.enabled else "disabled",
|
| 288 |
+
self.horizon, self.order, self.act_floor)]
|
| 289 |
+
lines.append(" idle ticks %-3d replays %-3d transitions %-3d observed %d"
|
| 290 |
+
% (stats["ticks"], stats["replays"], stats["transitions"],
|
| 291 |
+
stats["observed"]))
|
| 292 |
+
lines.append(" readahead graded %d, correct %d -> %.1f%% next-intent accuracy "
|
| 293 |
+
"(chance %.2f)"
|
| 294 |
+
% (stats["graded"], stats["hits"], 100.0 * stats["accuracy"],
|
| 295 |
+
stats["chance"]))
|
| 296 |
+
if self.readahead:
|
| 297 |
+
lines.append(" current lookahead: " + " -> ".join(
|
| 298 |
+
"%s(%.2f)" % (item["intent"], item["confidence"]) for item in self.readahead))
|
| 299 |
+
else:
|
| 300 |
+
lines.append(" current lookahead: none (observation needed first)")
|
| 301 |
+
lines.append(" autonomous actions %d (%d skipped under the floor), cached %d, "
|
| 302 |
+
"served %d, %.1f ms saved"
|
| 303 |
+
% (stats["autonomous_actions"], self.at_floor_skips,
|
| 304 |
+
len(self.cache), stats["served_from_readahead"], stats["ms_saved"]))
|
| 305 |
+
return "\n".join(lines)
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
#: A session with a shape: the same four-command cycle three times, then a break.
|
| 309 |
+
SESSION = (
|
| 310 |
+
"show the statistics", "show the registers", "show the instruction set",
|
| 311 |
+
"what can you do",
|
| 312 |
+
"show the statistics", "show the registers", "show the instruction set",
|
| 313 |
+
"what can you do",
|
| 314 |
+
"show the statistics", "show the registers", "show the instruction set",
|
| 315 |
+
"what can you do",
|
| 316 |
+
)
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
def simulate(session: Sequence[str] = SESSION, *, d: int = 2000, encoder=None,
|
| 320 |
+
enabled: bool = True, warmup: int = 1) -> dict:
|
| 321 |
+
"""Run a scripted session through idle + chat and report what it learned.
|
| 322 |
+
|
| 323 |
+
The point of the simulation is that the numbers in :meth:`AutonomousMode.report`
|
| 324 |
+
are measured on a conversation, not asserted. ``warmup`` idle ticks run
|
| 325 |
+
before the first utterance, so the machine starts in a cold state and has to
|
| 326 |
+
earn its accuracy.
|
| 327 |
+
"""
|
| 328 |
+
from .assistant import Assistant
|
| 329 |
+
from .terminal import Kernel
|
| 330 |
+
|
| 331 |
+
kernel = Kernel(d=d)
|
| 332 |
+
assistant = Assistant(kernel, d=d, encoder=encoder)
|
| 333 |
+
mode = AutonomousMode(assistant, d=d, enabled=enabled)
|
| 334 |
+
mode.idle(warmup)
|
| 335 |
+
|
| 336 |
+
per_turn: list[dict] = []
|
| 337 |
+
for text in session:
|
| 338 |
+
record = mode.chat(text)
|
| 339 |
+
per_turn.append({"input": text, "intent": record.get("intent"),
|
| 340 |
+
"graded": record.get("graded"),
|
| 341 |
+
"cached": record.get("cached", False),
|
| 342 |
+
"saved_ms": record.get("saved_ms", 0.0)})
|
| 343 |
+
half = len(per_turn) // 2
|
| 344 |
+
early = [t for t in per_turn[:half] if t["graded"]]
|
| 345 |
+
late = [t for t in per_turn[half:] if t["graded"]]
|
| 346 |
+
return {
|
| 347 |
+
"mode": mode, "turns": per_turn,
|
| 348 |
+
"early_accuracy": _rate(early), "late_accuracy": _rate(late),
|
| 349 |
+
"stats": mode.stats(), "report": mode.report(),
|
| 350 |
+
}
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
def _rate(turns: Sequence[dict]) -> float:
|
| 354 |
+
if not turns:
|
| 355 |
+
return 0.0
|
| 356 |
+
return sum(1 for turn in turns if turn["graded"]["hit"]) / len(turns)
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
def report(d: int = DEFAULT_DIM) -> str:
|
| 360 |
+
"""The printable form: a simulated session, its accuracy and its savings."""
|
| 361 |
+
from .semantic import load
|
| 362 |
+
|
| 363 |
+
encoder = load(d=d)
|
| 364 |
+
result = simulate(d=d, encoder=encoder)
|
| 365 |
+
lines = ["autonomy report (simulated session, %d utterances, encoder=%s)"
|
| 366 |
+
% (len(SESSION), "semantic" if encoder else "n-gram")]
|
| 367 |
+
lines.append(result["report"])
|
| 368 |
+
lines.append(" next-intent accuracy: first half %.0f%% -> second half %.0f%%"
|
| 369 |
+
% (100.0 * result["early_accuracy"], 100.0 * result["late_accuracy"]))
|
| 370 |
+
for turn in result["turns"][-4:]:
|
| 371 |
+
graded = turn["graded"] or {}
|
| 372 |
+
lines.append(" %-26s -> %-10s predicted %-10s %s%s"
|
| 373 |
+
% (turn["input"][:26], turn["intent"],
|
| 374 |
+
graded.get("predicted") or "-",
|
| 375 |
+
"hit" if graded.get("hit") else "miss",
|
| 376 |
+
" [readahead]" if turn["cached"] else ""))
|
| 377 |
+
return "\n".join(lines)
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
if __name__ == "__main__": # pragma: no cover - manual inspection
|
| 381 |
+
print(report())
|
hdc/boot.py
ADDED
|
@@ -0,0 +1,263 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The boot chain: POST screen, shader splash, AUTOEXEC, desktop.
|
| 2 |
+
|
| 3 |
+
Three phases, each rendered by the graphics engine rather than assembled as
|
| 4 |
+
strings, and each doing real work rather than miming it:
|
| 5 |
+
|
| 6 |
+
1. **POST.** The 16-bit BIOS in :mod:`hdc.x86` runs for real (IVT installed,
|
| 7 |
+
boot sector read from LBA 0, `int 10h` teletype) and its text buffer is painted
|
| 8 |
+
onto a phosphor surface with a scanline pass. The lines on screen are the
|
| 9 |
+
lines the BIOS produced.
|
| 10 |
+
2. **Splash.** A shader frame: gradient, fractal noise, a `max_blend` of a second
|
| 11 |
+
noise field, scanline and vignette — regenerated per frame with a shifted seed,
|
| 12 |
+
so the animation *is* the shader pipeline running, not a GIF. A title and a
|
| 13 |
+
progress bar are drawn into the coded cells with exact levels.
|
| 14 |
+
3. **AUTOEXEC.** A boot script whose every line is executed against the live
|
| 15 |
+
desktop: `GFX.INIT` measures the shader pass, `MOUNT` changes the filesystem's
|
| 16 |
+
working directory, `FS.SEARCH` runs the real hypervector search, `START`
|
| 17 |
+
launches the real windows, `AUTONOMY` turns the readahead loop on.
|
| 18 |
+
|
| 19 |
+
``BootSequence.run()`` records the frames it drew, so a client can replay the
|
| 20 |
+
boot without re-running it, and the desktop it hands over is the state that
|
| 21 |
+
actually resulted from those commands.
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
from __future__ import annotations
|
| 25 |
+
|
| 26 |
+
import time
|
| 27 |
+
|
| 28 |
+
from .fs import tokenize # noqa: F401 (kept for symmetry with fs.search's contract)
|
| 29 |
+
from .gfx import Surface
|
| 30 |
+
from .hypervec import DEFAULT_DIM
|
| 31 |
+
|
| 32 |
+
#: Lines executed in order; the handler returns the text shown beside each.
|
| 33 |
+
AUTOEXEC: tuple[str, ...] = (
|
| 34 |
+
"SET DIM %d",
|
| 35 |
+
"GFX.INIT render shader passes",
|
| 36 |
+
"MOUNT /home/user",
|
| 37 |
+
"FS.SEARCH shader notes about edges",
|
| 38 |
+
"START terminal",
|
| 39 |
+
"START monitor",
|
| 40 |
+
"START files",
|
| 41 |
+
"AUTONOMY enable readahead",
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
#: Splash animation frames and how long each is held.
|
| 45 |
+
SPLASH_FRAMES = 6
|
| 46 |
+
SPLASH_MS = 110
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
class BootSequence:
|
| 50 |
+
"""Run and record the boot: BIOS, shader splash, AUTOEXEC."""
|
| 51 |
+
|
| 52 |
+
def __init__(self, desktop, d: int | None = None):
|
| 53 |
+
self.desktop = desktop
|
| 54 |
+
self.d = d or desktop.d
|
| 55 |
+
self.frames: list[dict] = []
|
| 56 |
+
self.log: list[dict] = []
|
| 57 |
+
self.started = 0.0
|
| 58 |
+
self.elapsed_ms = 0.0
|
| 59 |
+
|
| 60 |
+
# ---------------------------------------------------------------- phases
|
| 61 |
+
def _post_lines(self) -> list[str]:
|
| 62 |
+
from .x86 import Boot
|
| 63 |
+
|
| 64 |
+
boot = Boot()
|
| 65 |
+
boot.post()
|
| 66 |
+
result = boot.boot()
|
| 67 |
+
lines = [line for line in result["screen"] if line.strip()]
|
| 68 |
+
self.log.append({"phase": "post", "lines": len(lines),
|
| 69 |
+
"stage": result["stage"],
|
| 70 |
+
"instructions": result["instructions"]})
|
| 71 |
+
return lines
|
| 72 |
+
|
| 73 |
+
def _post_frame(self, lines: list[str]) -> dict:
|
| 74 |
+
surface = Surface(self.desktop.cols, self.desktop.rows, d=self.d)
|
| 75 |
+
surface.clear(4)
|
| 76 |
+
surface.scanline(1)
|
| 77 |
+
for index, line in enumerate(lines[: self.desktop.rows - 2]):
|
| 78 |
+
surface.text_at(1, 1 + index, line, 200)
|
| 79 |
+
surface.text_at(1, self.desktop.rows - 2,
|
| 80 |
+
"Press F1-F6 after boot | HDCSTORAGE BIOS", 120)
|
| 81 |
+
return {"phase": "post", "lines": surface.rows(), "delay": 260,
|
| 82 |
+
"status": "BIOS POST complete: %d lines" % len(lines)}
|
| 83 |
+
|
| 84 |
+
def _splash_base(self) -> Surface:
|
| 85 |
+
"""The expensive part of the splash, rendered once.
|
| 86 |
+
|
| 87 |
+
``max_blend`` resolves each cell with a similarity comparator (up to 64
|
| 88 |
+
probes), which measured ~350 ms for a full frame — 2.1 s across six
|
| 89 |
+
frames. The base is computed once here and every animation frame copies
|
| 90 |
+
it and applies exact shifts, so the motion is still HDC arithmetic
|
| 91 |
+
(``shift``) while the boot stays under a second.
|
| 92 |
+
"""
|
| 93 |
+
cols, rows = self.desktop.cols, self.desktop.rows
|
| 94 |
+
surface = Surface(cols, rows, d=self.d)
|
| 95 |
+
surface.gradient(top=6, bottom=34, vertical=True)
|
| 96 |
+
surface.noise(scale=9, octaves=3, seed=21, amplitude=120)
|
| 97 |
+
surface.max_blend(scale=5, seed=24, amplitude=70)
|
| 98 |
+
surface.scanline(1)
|
| 99 |
+
surface.vignette(2)
|
| 100 |
+
self.base_ms = 0.0
|
| 101 |
+
return surface
|
| 102 |
+
|
| 103 |
+
def _splash_frames(self) -> list[dict]:
|
| 104 |
+
out: list[dict] = []
|
| 105 |
+
cols, rows = self.desktop.cols, self.desktop.rows
|
| 106 |
+
title = "H D C S T O R A G E"
|
| 107 |
+
subtitle = "hyperdimensional computer | everything is a vector"
|
| 108 |
+
box_x, box_w = cols // 2 - 19, 39
|
| 109 |
+
top = max(2, rows // 2 - 4)
|
| 110 |
+
bar_width = 33
|
| 111 |
+
base = self._splash_base()
|
| 112 |
+
for index in range(SPLASH_FRAMES):
|
| 113 |
+
progress = (index + 1) / float(SPLASH_FRAMES)
|
| 114 |
+
surface = base.copy()
|
| 115 |
+
# a bright band sweeping down the screen: two exact shifts, and the
|
| 116 |
+
# reason the splash looks like it is doing something
|
| 117 |
+
band = 2 + (index * (rows - 8)) // max(1, SPLASH_FRAMES - 1)
|
| 118 |
+
surface.brighten_region(1, 0, band, cols, 2)
|
| 119 |
+
surface.fill(box_x, top - 1, box_w, 9, 12)
|
| 120 |
+
surface.frame(box_x, top - 1, box_w, 9, 190)
|
| 121 |
+
surface.text_at((cols - len(title)) // 2, top, title, 250)
|
| 122 |
+
surface.text_at((cols - len(subtitle)) // 2, top + 2, subtitle, 170)
|
| 123 |
+
filled = int(bar_width * progress)
|
| 124 |
+
surface.fill(box_x + 4, top + 4, max(0, filled), 1, 245)
|
| 125 |
+
surface.text_at(box_x + 3, top + 4, "[", 180)
|
| 126 |
+
surface.text_at(box_x + 4 + bar_width, top + 4, "]", 180)
|
| 127 |
+
percent = "%3d%%" % int(progress * 100)
|
| 128 |
+
surface.text_at((cols - 12) // 2, top + 6, percent + " gfx engine", 220)
|
| 129 |
+
out.append({"phase": "splash", "lines": surface.rows(),
|
| 130 |
+
"delay": SPLASH_MS,
|
| 131 |
+
"status": "splash %s shader noise+max_blend, sweep row %d"
|
| 132 |
+
% (percent, band)})
|
| 133 |
+
return out
|
| 134 |
+
|
| 135 |
+
# -------------------------------------------------------------- autoexec
|
| 136 |
+
def _autoexec_frames(self) -> list[dict]:
|
| 137 |
+
out: list[dict] = []
|
| 138 |
+
shown: list[str] = ["AUTOEXEC.BAT - HDCSTORAGE boot script", ""]
|
| 139 |
+
for index, template in enumerate(AUTOEXEC):
|
| 140 |
+
command = template % self.d if "%d" in template else template
|
| 141 |
+
result = self._run_autoexec(command)
|
| 142 |
+
shown.append("%-12s %s" % (command.split(None, 1)[0], result))
|
| 143 |
+
self.log.append({"phase": "autoexec", "command": command, "result": result})
|
| 144 |
+
if index % 2 == 1 or index == len(AUTOEXEC) - 1:
|
| 145 |
+
out.append(self._text_frame(shown, "autoexec: " + result))
|
| 146 |
+
return out
|
| 147 |
+
|
| 148 |
+
def _run_autoexec(self, command: str) -> str:
|
| 149 |
+
"""Execute one boot line against the live desktop; return what happened."""
|
| 150 |
+
words = command.split()
|
| 151 |
+
verb = words[0].upper() if words else ""
|
| 152 |
+
desktop = self.desktop
|
| 153 |
+
try:
|
| 154 |
+
if verb == "SET":
|
| 155 |
+
return "d=%d" % self.d if len(words) < 2 else " ".join(words[1:])
|
| 156 |
+
if verb == "GFX.INIT":
|
| 157 |
+
from .gfx import SHADERS
|
| 158 |
+
|
| 159 |
+
surface = Surface(desktop.cols, desktop.rows, d=self.d).apply("background")
|
| 160 |
+
check = surface.verify()
|
| 161 |
+
return "%d shaders, %d cells, verify %s" % (
|
| 162 |
+
len(SHADERS), desktop.cols * desktop.rows,
|
| 163 |
+
"ok" if check["ok"] else "FAILED")
|
| 164 |
+
if verb == "MOUNT":
|
| 165 |
+
target = words[1] if len(words) > 1 else "/"
|
| 166 |
+
desktop.fs.chdir(target)
|
| 167 |
+
stat = desktop.fs.stat()
|
| 168 |
+
return "%s %d files, %d dirs" % (target, stat["files"], stat["dirs"])
|
| 169 |
+
if verb == "FS.SEARCH":
|
| 170 |
+
query = " ".join(words[1:]) or "readme"
|
| 171 |
+
hits = desktop.fs.search(query, limit=1)
|
| 172 |
+
if not hits:
|
| 173 |
+
return "no match for %r" % query
|
| 174 |
+
return "%s (%+.3f)" % (hits[0][0], hits[0][1])
|
| 175 |
+
if verb == "START":
|
| 176 |
+
key = words[1] if len(words) > 1 else "terminal"
|
| 177 |
+
# tiled, not cascaded: a booted desktop should look arranged
|
| 178 |
+
spots = {"terminal": (1, 1), "monitor": (48, 1), "files": (1, 15)}
|
| 179 |
+
spot = spots.get(key)
|
| 180 |
+
window = (desktop.launch(key, *spot) if spot else desktop.launch(key))
|
| 181 |
+
return "%s at %d,%d" % (window.title, window.x, window.y)
|
| 182 |
+
if verb == "AUTONOMY":
|
| 183 |
+
if desktop.autonomy is None:
|
| 184 |
+
return "no assistant attached"
|
| 185 |
+
desktop.autonomy.enabled = True
|
| 186 |
+
desktop.autonomy.tick()
|
| 187 |
+
return "readahead on, %d intents" % len(desktop.autonomy.intents)
|
| 188 |
+
except Exception as error: # a boot line must not abort boot
|
| 189 |
+
return "[error] %s" % error
|
| 190 |
+
return "unknown boot command %r" % command
|
| 191 |
+
|
| 192 |
+
def _text_frame(self, lines: list[str], status: str) -> dict:
|
| 193 |
+
surface = Surface(self.desktop.cols, self.desktop.rows, d=self.d)
|
| 194 |
+
surface.clear(5)
|
| 195 |
+
surface.scanline(1)
|
| 196 |
+
for index, line in enumerate(lines[: self.desktop.rows - 2]):
|
| 197 |
+
surface.text_at(2, 1 + index, line, 210)
|
| 198 |
+
surface.text_at(2, self.desktop.rows - 2, "AUTOEXEC running ...", 140)
|
| 199 |
+
return {"phase": "autoexec", "lines": surface.rows(), "delay": 220,
|
| 200 |
+
"status": status}
|
| 201 |
+
|
| 202 |
+
# ------------------------------------------------------------------- run
|
| 203 |
+
def run(self) -> dict:
|
| 204 |
+
"""Play every phase against the live desktop and record the frames."""
|
| 205 |
+
started = time.perf_counter()
|
| 206 |
+
self.frames = []
|
| 207 |
+
self.log = []
|
| 208 |
+
self.frames.append(self._post_frame(self._post_lines()))
|
| 209 |
+
self.frames.extend(self._splash_frames())
|
| 210 |
+
self.frames.extend(self._autoexec_frames())
|
| 211 |
+
self.frames.append({"phase": "desktop", "lines": self.desktop.frame(),
|
| 212 |
+
"delay": 420, "status": "desktop ready"})
|
| 213 |
+
self.elapsed_ms = (time.perf_counter() - started) * 1000.0
|
| 214 |
+
self.started = time.time()
|
| 215 |
+
return self.result()
|
| 216 |
+
|
| 217 |
+
def result(self) -> dict:
|
| 218 |
+
phases: list[str] = []
|
| 219 |
+
for frame in self.frames:
|
| 220 |
+
if not phases or phases[-1] != frame["phase"]:
|
| 221 |
+
phases.append(frame["phase"])
|
| 222 |
+
return {"frames": self.frames, "phases": phases, "log": self.log,
|
| 223 |
+
"elapsed_ms": round(self.elapsed_ms, 1),
|
| 224 |
+
"frame_count": len(self.frames),
|
| 225 |
+
"total_ms": sum(int(frame["delay"]) for frame in self.frames)}
|
| 226 |
+
|
| 227 |
+
def report(self) -> str:
|
| 228 |
+
result = self.result()
|
| 229 |
+
lines = ["boot: %d frames in %.0f ms of rendering, %d ms of playback"
|
| 230 |
+
% (result["frame_count"], result["elapsed_ms"], result["total_ms"]),
|
| 231 |
+
" phases: " + " -> ".join(result["phases"])]
|
| 232 |
+
for entry in self.log:
|
| 233 |
+
if entry["phase"] == "autoexec":
|
| 234 |
+
lines.append(" %-10s %s" % (entry["command"].split()[0], entry["result"]))
|
| 235 |
+
elif entry["phase"] == "post":
|
| 236 |
+
lines.append(" POST %d lines, stage %s, %d instructions"
|
| 237 |
+
% (entry["lines"], entry["stage"], entry["instructions"]))
|
| 238 |
+
return "\n".join(lines)
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def render(frames: list[dict], marks: bool = True) -> str:
|
| 242 |
+
"""Printable form: frames separated by a phase banner (for the CLI)."""
|
| 243 |
+
out: list[str] = []
|
| 244 |
+
for index, frame in enumerate(frames):
|
| 245 |
+
if marks:
|
| 246 |
+
out.append("--- frame %d/%d [%s] %s ---"
|
| 247 |
+
% (index + 1, len(frames), frame["phase"], frame["status"]))
|
| 248 |
+
out.extend(frame["lines"])
|
| 249 |
+
return "\n".join(out)
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
def demo(d: int = DEFAULT_DIM) -> str:
|
| 253 |
+
"""Boot a throwaway desktop and print what the sequence produced."""
|
| 254 |
+
from .os import Desktop
|
| 255 |
+
|
| 256 |
+
desktop = Desktop(d=d)
|
| 257 |
+
sequence = BootSequence(desktop)
|
| 258 |
+
sequence.run()
|
| 259 |
+
return "\n".join([render(sequence.frames), "", sequence.report()])
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
if __name__ == "__main__": # pragma: no cover - manual inspection
|
| 263 |
+
print(demo(2000))
|
hdc/crystal.py
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Crystalline shaders: lattice, grains, facets and banding.
|
| 2 |
+
|
| 3 |
+
A crystal is a periodic arrangement, so the operators that matter are the ones
|
| 4 |
+
that expose periodicity:
|
| 5 |
+
|
| 6 |
+
* **Lattice sites** — a Bravais lattice as integer basis vectors. Three common
|
| 7 |
+
cells are provided (simple cubic, body-centred, hexagonal in-plane) because
|
| 8 |
+
the shading they produce differs by symmetry, not by detail.
|
| 9 |
+
* **Crystallisation** — grow grains by multi-source BFS from the lattice sites
|
| 10 |
+
through occupied voxels. Each voxel takes the id of the site that reaches it
|
| 11 |
+
first, which is a Voronoi partition under the *geodesic* metric — grain
|
| 12 |
+
boundaries therefore follow the surface, not straight-line space.
|
| 13 |
+
* **Facet map** — each surface voxel's dominant gradient direction, quantised
|
| 14 |
+
to the six axis normals. Ties resolve to the lowest axis; a finer codebook
|
| 15 |
+
would resolve them smoothly, but six facets is what a coarse lattice has.
|
| 16 |
+
* **Banding** — brightness modulated every ``spacing`` planes along one axis,
|
| 17 |
+
which is the visual signature of a lattice plane stack.
|
| 18 |
+
|
| 19 |
+
Measured: :func:`periodicity` recovers the spacing of a banded volume by
|
| 20 |
+
autocorrelating per-plane occupancy, so "it looks periodic" is checkable.
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
from __future__ import annotations
|
| 24 |
+
|
| 25 |
+
from collections import deque
|
| 26 |
+
from typing import Sequence
|
| 27 |
+
|
| 28 |
+
from .shader import ShaderFabric
|
| 29 |
+
from .shader3d import Volume
|
| 30 |
+
|
| 31 |
+
LATTICES = {
|
| 32 |
+
"cubic": (1, 1, 1),
|
| 33 |
+
"bcc": (2, 2, 2),
|
| 34 |
+
"fcc": (2, 2, 2),
|
| 35 |
+
"hex": (2, 1, 1),
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def lattice_sites(size: int, kind: str = "cubic", origin: tuple[int, int, int] = (0, 0, 0)) -> list[tuple[int, int, int]]:
|
| 40 |
+
"""Integer lattice sites inside a ``size`` cube.
|
| 41 |
+
|
| 42 |
+
``cubic``: every integer point. ``bcc``/``fcc``: the conventional cell with
|
| 43 |
+
its centring points. ``hex``: in-plane triangular rows.
|
| 44 |
+
"""
|
| 45 |
+
if kind not in LATTICES:
|
| 46 |
+
raise ValueError("unknown lattice %r" % (kind,))
|
| 47 |
+
ox, oy, oz = origin
|
| 48 |
+
sites: list[tuple[int, int, int]] = []
|
| 49 |
+
if kind == "cubic":
|
| 50 |
+
for z in range(size):
|
| 51 |
+
for y in range(size):
|
| 52 |
+
for x in range(size):
|
| 53 |
+
sites.append((x + ox, y + oy, z + oz))
|
| 54 |
+
elif kind == "hex":
|
| 55 |
+
for z in range(0, size, 2):
|
| 56 |
+
for y in range(0, size, 2):
|
| 57 |
+
for x in range(0, size, 2):
|
| 58 |
+
sites.append((x + ox, y + oy, z + oz))
|
| 59 |
+
sites.append((x + 1 + ox, y + oy, z + oz))
|
| 60 |
+
sites.append((x + ox, y + 1 + oy, z + oz))
|
| 61 |
+
else: # bcc / fcc share the conventional cell corners plus centres
|
| 62 |
+
step = LATTICES[kind][0]
|
| 63 |
+
for z in range(0, size, step):
|
| 64 |
+
for y in range(0, size, step):
|
| 65 |
+
for x in range(0, size, step):
|
| 66 |
+
sites.append((x + ox, y + oy, z + oz))
|
| 67 |
+
sites.append((x + 1 + ox, y + 1 + oy, z + 1 + oz))
|
| 68 |
+
return [s for s in sites
|
| 69 |
+
if 0 <= s[0] < size and 0 <= s[1] < size and 0 <= s[2] < size]
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def crystallize(volume: Volume, sites: Sequence[tuple[int, int, int]], cap: int = 255) -> Volume:
|
| 73 |
+
"""Grow grains from ``sites`` by geodesic BFS; value is the grain id."""
|
| 74 |
+
fabric = volume.fabric
|
| 75 |
+
levels = volume.levels()
|
| 76 |
+
out = Volume(volume.w, volume.h, volume.depth, fabric, fill=cap)
|
| 77 |
+
seen: dict[tuple[int, int, int], int] = {}
|
| 78 |
+
queue: deque[tuple[int, int, int]] = deque()
|
| 79 |
+
for grain, (x, y, z) in enumerate(sites):
|
| 80 |
+
if 0 <= x < volume.w and 0 <= y < volume.h and 0 <= z < volume.depth:
|
| 81 |
+
if levels[z][y][x] > 0 and (x, y, z) not in seen:
|
| 82 |
+
seen[(x, y, z)] = grain % cap
|
| 83 |
+
queue.append((x, y, z))
|
| 84 |
+
while queue:
|
| 85 |
+
cx, cy, cz = queue.popleft()
|
| 86 |
+
grain = seen[(cx, cy, cz)]
|
| 87 |
+
for dx, dy, dz in ((1, 0, 0), (-1, 0, 0), (0, 1, 0), (0, -1, 0), (0, 0, 1), (0, 0, -1)):
|
| 88 |
+
nx, ny, nz = cx + dx, cy + dy, cz + dz
|
| 89 |
+
if not (0 <= nx < volume.w and 0 <= ny < volume.h and 0 <= nz < volume.depth):
|
| 90 |
+
continue
|
| 91 |
+
if (nx, ny, nz) in seen or levels[nz][ny][nx] <= 0:
|
| 92 |
+
continue
|
| 93 |
+
seen[(nx, ny, nz)] = grain
|
| 94 |
+
queue.append((nx, ny, nz))
|
| 95 |
+
for (x, y, z), grain in seen.items():
|
| 96 |
+
out.voxels[z][y][x] = fabric.level(grain)
|
| 97 |
+
return out
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def grain_boundaries(grains: Volume) -> Volume:
|
| 101 |
+
"""Bright where a voxel's grain differs from a neighbour's (grain walls)."""
|
| 102 |
+
fabric = grains.fabric
|
| 103 |
+
levels = grains.levels()
|
| 104 |
+
out = Volume(grains.w, grains.h, grains.depth, fabric)
|
| 105 |
+
for z in range(grains.depth):
|
| 106 |
+
for y in range(grains.h):
|
| 107 |
+
for x in range(grains.w):
|
| 108 |
+
here = levels[z][y][x]
|
| 109 |
+
for dx, dy, dz in ((1, 0, 0), (0, 1, 0), (0, 0, 1)):
|
| 110 |
+
nx, ny, nz = x + dx, y + dy, z + dz
|
| 111 |
+
if 0 <= nx < grains.w and 0 <= ny < grains.h and 0 <= nz < grains.depth:
|
| 112 |
+
if levels[nz][ny][nx] != here:
|
| 113 |
+
out.voxels[z][y][x] = fabric.level(255)
|
| 114 |
+
return out
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def facet_histogram(volume: Volume) -> dict[int, int]:
|
| 118 |
+
"""Count surface voxels per quantised normal code."""
|
| 119 |
+
hist: dict[int, int] = {}
|
| 120 |
+
for plane in volume.normals():
|
| 121 |
+
for row in plane:
|
| 122 |
+
for code in row:
|
| 123 |
+
hist[code] = hist.get(code, 0) + 1
|
| 124 |
+
return dict(sorted(hist.items()))
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def plane_counts(volume: Volume, axis: int = 0) -> list[int]:
|
| 128 |
+
"""Occupied voxels per plane along ``axis``."""
|
| 129 |
+
levels = volume.levels()
|
| 130 |
+
if axis == 0:
|
| 131 |
+
return [sum(1 for z in range(volume.depth) for y in range(volume.h) if levels[z][y][x] > 0)
|
| 132 |
+
for x in range(volume.w)]
|
| 133 |
+
if axis == 1:
|
| 134 |
+
return [sum(1 for z in range(volume.depth) for x in range(volume.w) if levels[z][y][x] > 0)
|
| 135 |
+
for y in range(volume.h)]
|
| 136 |
+
return [sum(1 for y in range(volume.h) for x in range(volume.w) if levels[z][y][x] > 0)
|
| 137 |
+
for z in range(volume.depth)]
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def periodicity(volume: Volume, axis: int = 0, max_spacing: int = 6) -> tuple[int, float]:
|
| 141 |
+
"""Recover the dominant plane spacing by autocorrelation of occupancy.
|
| 142 |
+
|
| 143 |
+
Returns ``(spacing, score)``; a score near 1.0 means the volume repeats
|
| 144 |
+
almost exactly at that spacing.
|
| 145 |
+
"""
|
| 146 |
+
counts = plane_counts(volume, axis)
|
| 147 |
+
if len(counts) < 3:
|
| 148 |
+
return 0, 0.0
|
| 149 |
+
mean = sum(counts) / len(counts)
|
| 150 |
+
variance = sum((c - mean) ** 2 for c in counts)
|
| 151 |
+
if variance == 0:
|
| 152 |
+
return 0, 0.0
|
| 153 |
+
best, best_score = 0, -1.0
|
| 154 |
+
for lag in range(1, min(max_spacing, len(counts) - 1) + 1):
|
| 155 |
+
num = sum((counts[i] - mean) * (counts[i + lag] - mean) for i in range(len(counts) - lag))
|
| 156 |
+
# normalise by the whole-series variance so the score stays in [-1, 1]
|
| 157 |
+
score = num / (variance or 1.0)
|
| 158 |
+
if score > best_score:
|
| 159 |
+
best, best_score = lag, score
|
| 160 |
+
return best, best_score
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def lattice_shade(volume: Volume, light: tuple[float, float, float] = (0.0, 0.0, 1.0),
|
| 164 |
+
spacing: int = 2, axis: int = 0) -> Volume:
|
| 165 |
+
"""Lambert shading with a brightness step on every ``spacing``-th plane."""
|
| 166 |
+
fabric = volume.fabric
|
| 167 |
+
shaded = volume.shade(light)
|
| 168 |
+
out = Volume(volume.w, volume.h, volume.depth, fabric)
|
| 169 |
+
for z in range(volume.depth):
|
| 170 |
+
for y in range(volume.h):
|
| 171 |
+
for x in range(volume.w):
|
| 172 |
+
coord = x if axis == 0 else y if axis == 1 else z
|
| 173 |
+
base = shaded.voxels[z][y][x]
|
| 174 |
+
out.voxels[z][y][x] = (
|
| 175 |
+
fabric.dim(base, 1) if spacing > 1 and coord % spacing else base
|
| 176 |
+
)
|
| 177 |
+
return out
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def report(volume: Volume, size: int | None = None) -> str:
|
| 181 |
+
"""Printable crystal tour over ``volume``."""
|
| 182 |
+
size = size or max(volume.w, volume.h, volume.depth)
|
| 183 |
+
sites = lattice_sites(size, "cubic")
|
| 184 |
+
grains = crystallize(volume, sites)
|
| 185 |
+
boundaries = grain_boundaries(grains)
|
| 186 |
+
walls = sum(1 for plane in boundaries.levels() for row in plane for v in row if v == 255)
|
| 187 |
+
spacing, score = periodicity(volume, 0)
|
| 188 |
+
mid = volume.depth // 2
|
| 189 |
+
lines = [
|
| 190 |
+
"crystal: %d lattice sites, %d grain-wall voxels" % (len(sites), walls),
|
| 191 |
+
"facets (normal code -> voxel count): %s" % facet_histogram(volume),
|
| 192 |
+
"dominant plane spacing on x: %d (autocorrelation %.2f)" % (spacing, score),
|
| 193 |
+
"grain walls at z=%d:" % mid,
|
| 194 |
+
boundaries.ascii_slice(mid),
|
| 195 |
+
"lattice banding (spacing 2) at z=%d:" % mid,
|
| 196 |
+
lattice_shade(volume, spacing=2).ascii_slice(mid),
|
| 197 |
+
]
|
| 198 |
+
return "\n".join(lines)
|
hdc/debugger.py
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The debugger.
|
| 2 |
+
|
| 3 |
+
A debugger for this machine has an unusual job: the instruction stream is not
|
| 4 |
+
readable by construction, so inspection *is* decoding. Every view here goes
|
| 5 |
+
through cleanup and carries its confidence, which means a low-confidence
|
| 6 |
+
decode is visible as a fact about the program rather than hidden behind a
|
| 7 |
+
plausible-looking mnemonic.
|
| 8 |
+
|
| 9 |
+
What it offers:
|
| 10 |
+
|
| 11 |
+
* breakpoints by cell index or by label;
|
| 12 |
+
* step / step-over, with the PC held at a BREAK trap;
|
| 13 |
+
* a trace ring of executed instructions with decode confidence and the
|
| 14 |
+
registers each one wrote;
|
| 15 |
+
* symbolic register inspection — for each register, the nearest named symbol
|
| 16 |
+
and its similarity, which is the HDC equivalent of a type readout;
|
| 17 |
+
* raw bit-plane views of registers and storage cells.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
from dataclasses import dataclass
|
| 23 |
+
|
| 24 |
+
from .isa import REG_NAMES, format_operands
|
| 25 |
+
from .interpreter import Machine, TraceEntry
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
@dataclass
|
| 29 |
+
class StopReason:
|
| 30 |
+
"""Why execution stopped."""
|
| 31 |
+
|
| 32 |
+
reason: str # 'break' | 'halt' | 'budget' | 'step' | 'error'
|
| 33 |
+
pc: int
|
| 34 |
+
detail: str = ""
|
| 35 |
+
|
| 36 |
+
def __str__(self) -> str:
|
| 37 |
+
return "%s at pc=%d%s" % (self.reason, self.pc, (" (%s)" % self.detail) if self.detail else "")
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class Debugger:
|
| 41 |
+
"""Interactive control of a :class:`~hdc.interpreter.Machine`."""
|
| 42 |
+
|
| 43 |
+
def __init__(self, machine: Machine):
|
| 44 |
+
self.m = machine
|
| 45 |
+
self.breakpoints: set[int] = set()
|
| 46 |
+
self.labels: dict[str, int] = {}
|
| 47 |
+
self._trap_pc: int | None = None
|
| 48 |
+
|
| 49 |
+
# ---------------------------------------------------------- breakpoints
|
| 50 |
+
def attach_program(self, program) -> "Debugger":
|
| 51 |
+
"""Take label -> cell mapping from an assembled program."""
|
| 52 |
+
self.labels = dict(program.labels)
|
| 53 |
+
self.breakpoints.clear()
|
| 54 |
+
self._trap_pc = None
|
| 55 |
+
return self
|
| 56 |
+
|
| 57 |
+
def add_break(self, where: int | str) -> int:
|
| 58 |
+
"""Set a breakpoint at a cell index or a label name."""
|
| 59 |
+
pc = self._where(where)
|
| 60 |
+
self.breakpoints.add(pc)
|
| 61 |
+
return pc
|
| 62 |
+
|
| 63 |
+
def remove_break(self, where: int | str) -> bool:
|
| 64 |
+
pc = self._where(where)
|
| 65 |
+
if pc in self.breakpoints:
|
| 66 |
+
self.breakpoints.discard(pc)
|
| 67 |
+
return True
|
| 68 |
+
return False
|
| 69 |
+
|
| 70 |
+
def _where(self, where: int | str) -> int:
|
| 71 |
+
if isinstance(where, int):
|
| 72 |
+
return where
|
| 73 |
+
if where in self.labels:
|
| 74 |
+
return self.labels[where]
|
| 75 |
+
if where.startswith("@"):
|
| 76 |
+
return int(where[1:])
|
| 77 |
+
raise KeyError("no such label: %r" % (where,))
|
| 78 |
+
|
| 79 |
+
def check(self) -> StopReason | None:
|
| 80 |
+
"""Does the current PC sit on a fresh breakpoint?"""
|
| 81 |
+
pc = self.m.pc
|
| 82 |
+
if pc == self._trap_pc:
|
| 83 |
+
return None
|
| 84 |
+
if pc in self.breakpoints:
|
| 85 |
+
self._trap_pc = pc
|
| 86 |
+
return StopReason("break", pc, "breakpoint")
|
| 87 |
+
return None
|
| 88 |
+
|
| 89 |
+
# -------------------------------------------------------------- running
|
| 90 |
+
def step(self) -> TraceEntry | None:
|
| 91 |
+
"""Execute exactly one instruction, ignoring breakpoints."""
|
| 92 |
+
entry = self.m.step()
|
| 93 |
+
if self.m.status == "break":
|
| 94 |
+
self._trap_pc = self.m.pc
|
| 95 |
+
else:
|
| 96 |
+
self._trap_pc = None
|
| 97 |
+
return entry
|
| 98 |
+
|
| 99 |
+
def step_over(self) -> TraceEntry | None:
|
| 100 |
+
"""Step one instruction; a CALL runs to completion first."""
|
| 101 |
+
entry = self.m.step()
|
| 102 |
+
if entry is not None and entry.op == "CALL":
|
| 103 |
+
depth = len(self.m.stack)
|
| 104 |
+
guard = 0
|
| 105 |
+
while len(self.m.stack) >= depth and not self.m.halted and guard < 100_000:
|
| 106 |
+
self.m.step()
|
| 107 |
+
guard += 1
|
| 108 |
+
self._trap_pc = None
|
| 109 |
+
return entry
|
| 110 |
+
|
| 111 |
+
def continue_(self, max_steps: int = 100_000) -> StopReason:
|
| 112 |
+
"""Run until the next breakpoint, HALT, fault or budget."""
|
| 113 |
+
m = self.m
|
| 114 |
+
if m.halted:
|
| 115 |
+
return StopReason("halt", m.pc, "already halted")
|
| 116 |
+
if m.status == "ready":
|
| 117 |
+
m.status = "running"
|
| 118 |
+
# If we are parked on a trap, allow that cell to execute once.
|
| 119 |
+
if m.pc == self._trap_pc:
|
| 120 |
+
m.skip_break = True
|
| 121 |
+
self._trap_pc = None
|
| 122 |
+
while m.steps < max_steps:
|
| 123 |
+
stop = self.check()
|
| 124 |
+
if stop is not None:
|
| 125 |
+
m.status = "break"
|
| 126 |
+
return stop
|
| 127 |
+
if m.halted:
|
| 128 |
+
return StopReason("halt", m.pc)
|
| 129 |
+
if m.pc >= m.prog_len:
|
| 130 |
+
m.halted = True
|
| 131 |
+
m.status = "halted"
|
| 132 |
+
return StopReason("halt", m.pc, "past end of program")
|
| 133 |
+
before = m.pc
|
| 134 |
+
entry = m.step()
|
| 135 |
+
if entry is None:
|
| 136 |
+
return StopReason("halt", m.pc)
|
| 137 |
+
if m.status == "error":
|
| 138 |
+
return StopReason("error", m.pc, self.trace_tail(1))
|
| 139 |
+
if entry.op == "BREAK" and m.status == "break":
|
| 140 |
+
self._trap_pc = before
|
| 141 |
+
return StopReason("break", before, "BREAK trap")
|
| 142 |
+
m.status = "budget"
|
| 143 |
+
return StopReason("budget", m.pc, "step budget exhausted")
|
| 144 |
+
|
| 145 |
+
# ---------------------------------------------------------------- views
|
| 146 |
+
def trace_tail(self, n: int = 10) -> str:
|
| 147 |
+
rows = self.m.trace[-n:]
|
| 148 |
+
if not rows:
|
| 149 |
+
return "<no steps yet>"
|
| 150 |
+
return "\n".join(e.render() for e in rows)
|
| 151 |
+
|
| 152 |
+
def disasm(self, start: int | None = None, count: int = 12) -> str:
|
| 153 |
+
"""Disassemble cells, marking the PC and any breakpoints."""
|
| 154 |
+
m = self.m
|
| 155 |
+
if start is None:
|
| 156 |
+
start = max(0, m.pc - 2)
|
| 157 |
+
rows = []
|
| 158 |
+
for pc in range(start, min(start + count, m.prog_len)):
|
| 159 |
+
dec = m.decode_at(pc)
|
| 160 |
+
marks = []
|
| 161 |
+
if pc == m.pc:
|
| 162 |
+
marks.append("=>")
|
| 163 |
+
else:
|
| 164 |
+
marks.append(" ")
|
| 165 |
+
if pc in self.breakpoints:
|
| 166 |
+
marks.append("B")
|
| 167 |
+
else:
|
| 168 |
+
marks.append(" ")
|
| 169 |
+
rows.append(
|
| 170 |
+
"%s%s %04d %-8s %-26s conf=%.2f"
|
| 171 |
+
% ("".join(marks), "", pc, dec.op, format_operands(dec), dec.confidence)
|
| 172 |
+
)
|
| 173 |
+
return "\n".join(rows) if rows else "<empty program>"
|
| 174 |
+
|
| 175 |
+
def regs(self) -> str:
|
| 176 |
+
"""Registers: integer payload, and nearest named symbol (cleanup)."""
|
| 177 |
+
m = self.m
|
| 178 |
+
rows = [
|
| 179 |
+
"REG VALUE POP SYMBOL SIM HV(head)"
|
| 180 |
+
,
|
| 181 |
+
"-" * 78,
|
| 182 |
+
]
|
| 183 |
+
for i in range(len(m.regs)):
|
| 184 |
+
val = m.get_int(i)
|
| 185 |
+
hits = m.storage.nearest_symbol(m.regs[i], k=1)
|
| 186 |
+
sym, s = (hits[0] if hits else ("", 0.0))
|
| 187 |
+
if s < 0.15:
|
| 188 |
+
# Below the noise band there is no nearest symbol worth
|
| 189 |
+
# naming; printing the argmax anyway would imply a match.
|
| 190 |
+
sym, s = "", 0.0
|
| 191 |
+
from .hypervec import bits
|
| 192 |
+
|
| 193 |
+
rows.append(
|
| 194 |
+
"%-5s %-10d %-6d %-25s %-5.2f %s"
|
| 195 |
+
% (
|
| 196 |
+
REG_NAMES[i],
|
| 197 |
+
val,
|
| 198 |
+
m.regs[i].bit_count(),
|
| 199 |
+
sym[:25] or "-",
|
| 200 |
+
s,
|
| 201 |
+
bits(m.regs[i], m.d)[:24],
|
| 202 |
+
)
|
| 203 |
+
)
|
| 204 |
+
rows.append(
|
| 205 |
+
"PC=%d ZF=%d STACK=%d STEPS=%d STATUS=%s"
|
| 206 |
+
% (m.pc, int(m.flag_z), len(m.stack), m.steps, m.status)
|
| 207 |
+
)
|
| 208 |
+
return "\n".join(rows)
|
| 209 |
+
|
| 210 |
+
def sim_table(self, regs: list[int] | None = None) -> str:
|
| 211 |
+
"""Pairwise register similarity — the 'how HDC is my state' view."""
|
| 212 |
+
m = self.m
|
| 213 |
+
idx = regs if regs is not None else list(range(len(m.regs)))
|
| 214 |
+
head = " " + " ".join("%6s" % REG_NAMES[i] for i in idx)
|
| 215 |
+
rows = [head]
|
| 216 |
+
for i in idx:
|
| 217 |
+
cells = []
|
| 218 |
+
for j in idx:
|
| 219 |
+
cells.append("%6.3f" % m.get_sim(i, j))
|
| 220 |
+
rows.append("%5s " % REG_NAMES[i] + " ".join(cells))
|
| 221 |
+
return "\n".join(rows)
|
| 222 |
+
|
| 223 |
+
def bits(self, reg: int) -> str:
|
| 224 |
+
"""Bit planes of one register."""
|
| 225 |
+
from .hypervec import bitplanes
|
| 226 |
+
|
| 227 |
+
return "\n".join(bitplanes(self.m.regs[reg], self.m.d))
|
| 228 |
+
|
| 229 |
+
def cell_bits(self, name: str) -> str:
|
| 230 |
+
"""Bit planes of a storage cell."""
|
| 231 |
+
return "\n".join(self.m.storage.bitplanes(name))
|
| 232 |
+
|
| 233 |
+
def summary(self) -> str:
|
| 234 |
+
m = self.m
|
| 235 |
+
return (
|
| 236 |
+
"status=%-8s pc=%-4d steps=%-5d halted=%-5s zf=%d bps=%s\n%s"
|
| 237 |
+
% (
|
| 238 |
+
m.status,
|
| 239 |
+
m.pc,
|
| 240 |
+
m.steps,
|
| 241 |
+
m.halted,
|
| 242 |
+
int(m.flag_z),
|
| 243 |
+
sorted(self.breakpoints) or "[]",
|
| 244 |
+
self.disasm(count=5),
|
| 245 |
+
)
|
| 246 |
+
)
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def debug_program(machine: Machine, program, breakpoints: list[int | str] = ()) -> str:
|
| 250 |
+
"""Convenience: attach, set breaks, run, and return a session transcript."""
|
| 251 |
+
dbg = Debugger(machine).attach_program(program)
|
| 252 |
+
for b in breakpoints:
|
| 253 |
+
dbg.add_break(b)
|
| 254 |
+
lines = [dbg.summary()]
|
| 255 |
+
stop = dbg.continue_()
|
| 256 |
+
lines.append("stopped: %s" % stop)
|
| 257 |
+
lines.append(dbg.regs())
|
| 258 |
+
lines.append("trace:\n" + dbg.trace_tail(8))
|
| 259 |
+
return "\n".join(lines)
|
hdc/demo.py
ADDED
|
@@ -0,0 +1,295 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""End-to-end tour: ``python -m hdc demo``.
|
| 2 |
+
|
| 3 |
+
Every number printed here is measured at run time by the code being described —
|
| 4 |
+
there are no stored benchmark results in this project.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import os
|
| 10 |
+
import random
|
| 11 |
+
import sys
|
| 12 |
+
|
| 13 |
+
from .adder import HyperAdder
|
| 14 |
+
from .agent import ToolRegistry
|
| 15 |
+
from .assembler import assemble
|
| 16 |
+
from .debugger import Debugger
|
| 17 |
+
from .hypervec import (
|
| 18 |
+
DEFAULT_DIM,
|
| 19 |
+
bind,
|
| 20 |
+
bitplanes,
|
| 21 |
+
bundle,
|
| 22 |
+
hvc,
|
| 23 |
+
hv_to_int,
|
| 24 |
+
int_to_hv,
|
| 25 |
+
permute,
|
| 26 |
+
popcount,
|
| 27 |
+
random_hv,
|
| 28 |
+
sim,
|
| 29 |
+
)
|
| 30 |
+
from .interpreter import Machine
|
| 31 |
+
from .memory import HDCStorage
|
| 32 |
+
from .net import BitLinear, PrototypeNet, selfcheck as net_selfcheck
|
| 33 |
+
from .shader import DepthBuffer, NormalCodebook, ShaderFabric, step_image
|
| 34 |
+
|
| 35 |
+
PROGRAM = """\
|
| 36 |
+
; the demonstration program: stored, decoded and executed as hypervectors
|
| 37 |
+
.str banner "every value in this machine is a 10000-bit vector"
|
| 38 |
+
.data ledger 0
|
| 39 |
+
|
| 40 |
+
start:
|
| 41 |
+
ldi R0, 5
|
| 42 |
+
ldi R1, 37
|
| 43 |
+
compute:
|
| 44 |
+
add R2, R0, R1 ; integer payloads, HDC registers
|
| 45 |
+
out R2
|
| 46 |
+
bind R3, R0, R1 ; XOR binding of the two vectors
|
| 47 |
+
tool R5, R3, R1, hdc.unbind
|
| 48 |
+
sim R6, R5, R0 ; unbinding recovers the operand exactly
|
| 49 |
+
out R6
|
| 50 |
+
store ledger, R2
|
| 51 |
+
load R7, ledger
|
| 52 |
+
out R7
|
| 53 |
+
prints banner
|
| 54 |
+
halt
|
| 55 |
+
"""
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _rule(title: str) -> None:
|
| 59 |
+
print()
|
| 60 |
+
print("=" * 74)
|
| 61 |
+
print(title)
|
| 62 |
+
print("=" * 74)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _flip(hv: int, count: int, rng: random.Random, d: int) -> int:
|
| 66 |
+
for _ in range(count):
|
| 67 |
+
hv ^= 1 << rng.randrange(d)
|
| 68 |
+
return hv
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def main(argv: list[str] | None = None) -> int:
|
| 72 |
+
argv = list(sys.argv[1:] if argv is None else argv)
|
| 73 |
+
d = DEFAULT_DIM
|
| 74 |
+
if "--d" in argv:
|
| 75 |
+
d = int(argv[argv.index("--d") + 1])
|
| 76 |
+
rng = random.Random(20261103)
|
| 77 |
+
artifacts = os.path.join(os.getcwd(), "artifacts")
|
| 78 |
+
written: list[str] = []
|
| 79 |
+
|
| 80 |
+
print("HDCSTORAGE d=%d bits per vector (%.0f bytes)" % (d, d / 8))
|
| 81 |
+
|
| 82 |
+
# ------------------------------------------------------------------ 1
|
| 83 |
+
_rule("1. ALGEBRA - binary spatter codes on Python big integers")
|
| 84 |
+
pairs = [(random_hv(d, rng), random_hv(d, rng)) for _ in range(2000)]
|
| 85 |
+
sims = [sim(a, b, d) for a, b in pairs]
|
| 86 |
+
mean = sum(sims) / len(sims)
|
| 87 |
+
sd = (sum((s - mean) ** 2 for s in sims) / len(sims)) ** 0.5
|
| 88 |
+
print("random pair similarity mean %+.4f sd %.4f" % (mean, sd))
|
| 89 |
+
members = [random_hv(d, rng) for _ in range(5)]
|
| 90 |
+
packed = bundle(members, d)
|
| 91 |
+
print(
|
| 92 |
+
"bundle of 5 members recovered at %s"
|
| 93 |
+
% " ".join("%.3f" % sim(packed, m, d) for m in members)
|
| 94 |
+
)
|
| 95 |
+
a, b = random_hv(d, rng), random_hv(d, rng)
|
| 96 |
+
print("bind is self-inverse bind(bind(a,b),b) == a : %s"
|
| 97 |
+
% (bind(bind(a, b, d), b, d) == a))
|
| 98 |
+
print("bound pair vs operand %.4f (independent vectors only;"
|
| 99 |
+
% sim(bind(a, b, d), a, d))
|
| 100 |
+
print(" structured codes stay correlated)")
|
| 101 |
+
print("permute preserves similarity %.4f == %.4f"
|
| 102 |
+
% (sim(permute(a, 7, d), permute(b, 7, d), d), sim(a, b, d)))
|
| 103 |
+
for value in (0, 1, 42, -7, 65535):
|
| 104 |
+
coded = int_to_hv(value, d)
|
| 105 |
+
print("integer codec %6d -> hv -> %d" % (value, hv_to_int(coded, d)))
|
| 106 |
+
|
| 107 |
+
# ------------------------------------------------------------------ 2
|
| 108 |
+
_rule("2. HDC STORAGE - content-addressed cells, cleanup memory")
|
| 109 |
+
store = HDCStorage(d=d, capacity=4096, label="demo")
|
| 110 |
+
for name in ("sensor/temp", "sensor/humidity", "concept/cat", "concept/dog", "policy/retry"):
|
| 111 |
+
store.write(name, hvc("record/" + name, d), symbol=True)
|
| 112 |
+
print(store.dump(limit=8))
|
| 113 |
+
hits = store.recall(store.read("concept/cat"), k=3)
|
| 114 |
+
print("\nassociative recall of concept/cat:")
|
| 115 |
+
for name, _, s in hits:
|
| 116 |
+
print(" %-20s %.4f %s" % (name, s, "<- exact" if s > 0.99 else ""))
|
| 117 |
+
addr = store.addr("sensor/humidity")
|
| 118 |
+
damaged = _flip(addr, 12, rng, d)
|
| 119 |
+
resolved = store.resolve(damaged)
|
| 120 |
+
print(
|
| 121 |
+
"\ndamaged address (12/%d bits flipped) -> %s at confidence %.3f"
|
| 122 |
+
% (d, resolved[0] if resolved else None, resolved[1] if resolved else 0.0)
|
| 123 |
+
)
|
| 124 |
+
os.makedirs(artifacts, exist_ok=True)
|
| 125 |
+
image = os.path.join(artifacts, "demo.hdc")
|
| 126 |
+
size = store.save(image)
|
| 127 |
+
written.append(image)
|
| 128 |
+
reloaded = HDCStorage.load(image)
|
| 129 |
+
same = all(reloaded.read(n) == store.read(n) for n, _ in store)
|
| 130 |
+
print("persistence %d bytes, %d cells, identical on reload: %s"
|
| 131 |
+
% (size, len(reloaded), same))
|
| 132 |
+
|
| 133 |
+
# ------------------------------------------------------------------ 3
|
| 134 |
+
_rule("3. COMPILER - text becomes instruction hypervectors")
|
| 135 |
+
program = assemble(PROGRAM, d=d)
|
| 136 |
+
confidences = [d_.confidence for d_ in program.listing]
|
| 137 |
+
print("cells=%d literals=%d strings=%d cell names=%d"
|
| 138 |
+
% (len(program.instrs), len(program.literals), len(program.strings),
|
| 139 |
+
len(program.cell_names)))
|
| 140 |
+
print("decode confidence min %.2f mean %.2f"
|
| 141 |
+
% (min(confidences), sum(confidences) / len(confidences)))
|
| 142 |
+
print("\nfirst cells of the image (disassembled from the vectors):")
|
| 143 |
+
for i in range(min(5, len(program.instrs))):
|
| 144 |
+
dec = program.listing[i]
|
| 145 |
+
print(" %04d %-8s regs=%-10s conf=%.2f"
|
| 146 |
+
% (i, dec.op, dec.regs, dec.confidence))
|
| 147 |
+
print("\ncell 0 as raw bit planes (first two of %d):" % ((d + 63) // 64))
|
| 148 |
+
for plane in bitplanes(program.instrs[0], d)[:2]:
|
| 149 |
+
print(" " + plane)
|
| 150 |
+
|
| 151 |
+
# ------------------------------------------------------------------ 4
|
| 152 |
+
_rule("4. INTERPRETER - fetch from storage, decode by cleanup, execute")
|
| 153 |
+
machine_store = HDCStorage(d=d, capacity=4096, label="machine")
|
| 154 |
+
registry = ToolRegistry(storage=machine_store, d=d)
|
| 155 |
+
machine = Machine(storage=machine_store, d=d, tools=registry, echo=print)
|
| 156 |
+
machine.load(program)
|
| 157 |
+
result = machine.run()
|
| 158 |
+
print("\nstatus=%s steps=%d cells=%d" % (result["status"], result["steps"], len(machine_store)))
|
| 159 |
+
print("trace:")
|
| 160 |
+
for entry in machine.trace:
|
| 161 |
+
print(" " + entry.render())
|
| 162 |
+
|
| 163 |
+
# ------------------------------------------------------------------ 5
|
| 164 |
+
_rule("5. DEBUGGER - breakpoints, stepping, symbolic registers")
|
| 165 |
+
machine.reset()
|
| 166 |
+
dbg = Debugger(machine).attach_program(program)
|
| 167 |
+
dbg.add_break("compute")
|
| 168 |
+
stop = dbg.continue_()
|
| 169 |
+
print("stopped: %s" % stop)
|
| 170 |
+
print(dbg.disasm(count=6))
|
| 171 |
+
print()
|
| 172 |
+
dbg.step()
|
| 173 |
+
dbg.step()
|
| 174 |
+
print(dbg.regs())
|
| 175 |
+
print("\ntrace tail:")
|
| 176 |
+
print(dbg.trace_tail(4))
|
| 177 |
+
stop = dbg.continue_()
|
| 178 |
+
print("\nresumed -> %s" % stop)
|
| 179 |
+
|
| 180 |
+
# ------------------------------------------------------------------ 6
|
| 181 |
+
_rule("6. AGENTIC TOOLS - MCP tools stored inside HDC")
|
| 182 |
+
print(registry.specs_text())
|
| 183 |
+
print("\nfuzzy and capability dispatch (tool records are vectors):")
|
| 184 |
+
for query in ("hdc.bundl", "hdc.memory.search", "bit.count", "nonsense.xyz"):
|
| 185 |
+
spec, conf = registry.resolve(query)
|
| 186 |
+
print(" %-20s -> %-16s conf=%.3f"
|
| 187 |
+
% (query, spec.name if spec else "unresolved", conf))
|
| 188 |
+
print("\nTOOL executed from inside the ISA, per the trace above: %s"
|
| 189 |
+
% [e.note for e in machine.trace if e.op == "TOOL"])
|
| 190 |
+
|
| 191 |
+
# ------------------------------------------------------------------ 7
|
| 192 |
+
_rule("7. HDC FULL ADDER - ADD and SUB without leaving vector space")
|
| 193 |
+
adder = HyperAdder(d=d)
|
| 194 |
+
print(adder.table_text())
|
| 195 |
+
sep = adder.separation()
|
| 196 |
+
print(
|
| 197 |
+
"\n%d row keys, worst cross-similarity %+.4f (noise floor ~0.01)"
|
| 198 |
+
% (sep["rows"], sep["worst_cross_similarity"])
|
| 199 |
+
)
|
| 200 |
+
adder.add(7, 35)
|
| 201 |
+
print("\nripple for 7 + 35, one line per bit slice:")
|
| 202 |
+
print(adder.trace_text(10))
|
| 203 |
+
print(" ... 20 more stages")
|
| 204 |
+
total = adder.decode(adder.add(7, 35))
|
| 205 |
+
checks = adder.verify(20)
|
| 206 |
+
print(
|
| 207 |
+
"sum = %d, and %d/%d random pairs agree with Python arithmetic"
|
| 208 |
+
% (total, checks["add_correct"], checks["trials"])
|
| 209 |
+
)
|
| 210 |
+
print(
|
| 211 |
+
"ISA trace now reports: %s"
|
| 212 |
+
% [e.note for e in machine.trace if e.op in ("ADD", "SUB")]
|
| 213 |
+
)
|
| 214 |
+
|
| 215 |
+
# ------------------------------------------------------------------ 8
|
| 216 |
+
_rule("8. BITNET BASE - 1-bit and ternary inference on the same primitives")
|
| 217 |
+
for key, value in net_selfcheck().items():
|
| 218 |
+
print(" %-22s %s" % (key, value))
|
| 219 |
+
layer = BitLinear(in_features=64, out_features=16, seed=3)
|
| 220 |
+
out = layer.forward([(1 if rng.random() < 0.5 else -1) for _ in range(64)])
|
| 221 |
+
print(" BitLinear 64->16 output %s" % (out[:8],))
|
| 222 |
+
print(" ternary weight density %.3f" % layer.density())
|
| 223 |
+
|
| 224 |
+
net = PrototypeNet(storage=HDCStorage(d=d, label="proto"), d=d)
|
| 225 |
+
prototypes = {label: random_hv(d, rng) for label in ("alpha", "beta", "gamma")}
|
| 226 |
+
for label, proto in prototypes.items():
|
| 227 |
+
for _ in range(5):
|
| 228 |
+
net.fit(label, _flip(proto, 40, rng, d))
|
| 229 |
+
tests = [(label, _flip(proto, 40, rng, d)) for label, proto in prototypes.items() for _ in range(4)]
|
| 230 |
+
print(" PrototypeNet %d classes, held-out accuracy %.3f"
|
| 231 |
+
% (len(net.classes()), net.accuracy(tests)))
|
| 232 |
+
|
| 233 |
+
# ------------------------------------------------------------------ 9
|
| 234 |
+
_rule("9. MCP - protocol surface over the same tool records")
|
| 235 |
+
from .mcp_server import MCPServer
|
| 236 |
+
|
| 237 |
+
server = MCPServer(d=d)
|
| 238 |
+
init = server.handle(
|
| 239 |
+
{"jsonrpc": "2.0", "id": 1, "method": "initialize",
|
| 240 |
+
"params": {"protocolVersion": "2024-11-05"}}
|
| 241 |
+
)
|
| 242 |
+
tools = server.handle({"jsonrpc": "2.0", "id": 2, "method": "tools/list"})
|
| 243 |
+
print("server %s, protocol %s, %d tools advertised"
|
| 244 |
+
% (init["result"]["serverInfo"]["name"],
|
| 245 |
+
init["result"]["protocolVersion"],
|
| 246 |
+
len(tools["result"]["tools"])))
|
| 247 |
+
encoded = server.handle(
|
| 248 |
+
{"jsonrpc": "2.0", "id": 3, "method": "tools/call",
|
| 249 |
+
"params": {"name": "hdc.encode_int", "arguments": {"n": 42}}}
|
| 250 |
+
)["result"]["structuredContent"]["hv"]
|
| 251 |
+
decoded = server.handle(
|
| 252 |
+
{"jsonrpc": "2.0", "id": 4, "method": "tools/call",
|
| 253 |
+
"params": {"name": "hdc.decode_int", "arguments": {"v": encoded}}}
|
| 254 |
+
)["result"]["content"][0]["text"]
|
| 255 |
+
print("round trip over JSON-RPC: encode(42) -> %s..., decode -> %s"
|
| 256 |
+
% (encoded[:34], decoded))
|
| 257 |
+
|
| 258 |
+
# ------------------------------------------------------------------ 10
|
| 259 |
+
_rule("10. HDC SHADER - image records, dimming, edges, depth")
|
| 260 |
+
fabric = ShaderFabric(d=d)
|
| 261 |
+
frame = step_image(8, 4, fabric=fabric, bright_b=200)
|
| 262 |
+
print("input row 0 %s" % frame.levels()[0])
|
| 263 |
+
print("dim(1) %s exact halving: 2 rotations + 2 XORs" % frame.dim(1).levels()[0])
|
| 264 |
+
print("edges() row 0 %s exact |a-b| through the slice adder" % frame.edges().levels()[0])
|
| 265 |
+
print("discontinuity %s 1 - sim, no arithmetic at all"
|
| 266 |
+
% [round(v, 2) for v in frame.discontinuities()[0]])
|
| 267 |
+
print(frame.ascii())
|
| 268 |
+
codebook = NormalCodebook(entries=64, d=d, fabric=fabric)
|
| 269 |
+
entry, match = codebook.nearest((0.0, 0.0, 1.0))
|
| 270 |
+
exact_rate = 0
|
| 271 |
+
for probe_entry in codebook.entries[:16]:
|
| 272 |
+
got, _ = codebook.nearest(probe_entry.direction)
|
| 273 |
+
exact_rate += 1 if got.index == probe_entry.index else 0
|
| 274 |
+
print(
|
| 275 |
+
"normal LUT (0,0,1) -> entry %d (sim %.3f, lambert %.3f); "
|
| 276 |
+
"stored directions recovered %d/16"
|
| 277 |
+
% (entry.index, match, codebook.lambert(entry, (0.0, 0.0, 1.0)), exact_rate)
|
| 278 |
+
)
|
| 279 |
+
zbuf = DepthBuffer(2, 1, fabric)
|
| 280 |
+
accepted = [zbuf.test(0, 0, fabric.level(v)) for v in (20, 50, 5)]
|
| 281 |
+
print("z-buffer 20:%s 50:%s 5:%s -> visible depth %s (exact comparator)"
|
| 282 |
+
% (accepted[0], accepted[1], accepted[2], zbuf.depths()[0][0]))
|
| 283 |
+
|
| 284 |
+
# ------------------------------------------------------------------ 11
|
| 285 |
+
_rule("11. ARTIFACTS")
|
| 286 |
+
print("storage image: %s (%d bytes)" % (image, os.path.getsize(image)))
|
| 287 |
+
for path in written:
|
| 288 |
+
print(" wrote %s" % path)
|
| 289 |
+
print("\nNext: python -m hdc ide (step it yourself)")
|
| 290 |
+
print(" python -m hdc mcp (attach an MCP client)")
|
| 291 |
+
return 0
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
if __name__ == "__main__":
|
| 295 |
+
raise SystemExit(main())
|
hdc/entropy.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Entropy over coded values.
|
| 2 |
+
|
| 3 |
+
Shannon entropy needs counts, and counting needs an ordering. This module gets
|
| 4 |
+
that ordering from the same bit-scan comparator the shaders use, so a
|
| 5 |
+
histogram can be built from hypervectors without decoding them:
|
| 6 |
+
|
| 7 |
+
bucket(v) = number of bucket edges e with v >= e
|
| 8 |
+
|
| 9 |
+
Each of those tests is ``ShaderFabric.compare``, which is exact. The result is
|
| 10 |
+
an ordinary integer count vector, from which entropy in bits follows.
|
| 11 |
+
|
| 12 |
+
Why this matters for shading
|
| 13 |
+
----------------------------
|
| 14 |
+
Entropy is the natural weight for "how much is happening here": a flat region
|
| 15 |
+
has all its values in one bucket (0 bits), an edge or a noisy patch spreads
|
| 16 |
+
across buckets (up to log2(buckets) bits). ``entropic_octave`` turns that into
|
| 17 |
+
a power-of-two attenuation, which the shader applies with ``shift`` — so an
|
| 18 |
+
entropy-weighted shader is still exact arithmetic, not a float multiply.
|
| 19 |
+
|
| 20 |
+
The one decoded step is :func:`local_entropy`, which is a *measurement* of an
|
| 21 |
+
already-decoded grid; it is used for reporting and for computing the
|
| 22 |
+
attenuation table, never inside the pixel path.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
from __future__ import annotations
|
| 26 |
+
|
| 27 |
+
import math
|
| 28 |
+
from typing import Sequence
|
| 29 |
+
|
| 30 |
+
from .shader import ShaderFabric
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def coded_histogram(
|
| 34 |
+
vectors: Sequence[int],
|
| 35 |
+
buckets: int = 8,
|
| 36 |
+
lo: int = 0,
|
| 37 |
+
hi: int = 255,
|
| 38 |
+
fabric: ShaderFabric | None = None,
|
| 39 |
+
) -> list[int]:
|
| 40 |
+
"""Count *coded* values into ``buckets`` bins using only the comparator.
|
| 41 |
+
|
| 42 |
+
``vectors`` must be hypervectors from ``fabric.level``; passing raw
|
| 43 |
+
integers is a silent error, since a small integer is also a valid vector
|
| 44 |
+
word. Use :func:`level_histogram` for raw levels.
|
| 45 |
+
"""
|
| 46 |
+
if buckets < 1:
|
| 47 |
+
raise ValueError("buckets must be positive")
|
| 48 |
+
fabric = fabric or ShaderFabric()
|
| 49 |
+
span = (hi - lo) / buckets
|
| 50 |
+
edges = [fabric.level(lo + int(round(span * (i + 1)))) for i in range(buckets - 1)]
|
| 51 |
+
counts = [0] * buckets
|
| 52 |
+
for vector in vectors:
|
| 53 |
+
idx = 0
|
| 54 |
+
for edge in edges:
|
| 55 |
+
if fabric.compare(vector, edge) >= 0:
|
| 56 |
+
idx += 1
|
| 57 |
+
else:
|
| 58 |
+
break
|
| 59 |
+
counts[idx] += 1
|
| 60 |
+
return counts
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def level_histogram(
|
| 64 |
+
levels: Sequence[int],
|
| 65 |
+
buckets: int = 8,
|
| 66 |
+
lo: int = 0,
|
| 67 |
+
hi: int = 255,
|
| 68 |
+
fabric: ShaderFabric | None = None,
|
| 69 |
+
) -> list[int]:
|
| 70 |
+
"""Encode raw levels, then histogram them."""
|
| 71 |
+
fabric = fabric or ShaderFabric()
|
| 72 |
+
return coded_histogram([fabric.level(v) for v in levels], buckets, lo, hi, fabric)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def shannon(counts: Sequence[int]) -> float:
|
| 76 |
+
"""Shannon entropy of a count vector, in bits."""
|
| 77 |
+
total = sum(counts)
|
| 78 |
+
if total <= 0:
|
| 79 |
+
return 0.0
|
| 80 |
+
bits = 0.0
|
| 81 |
+
for count in counts:
|
| 82 |
+
if count:
|
| 83 |
+
p = count / total
|
| 84 |
+
bits -= p * math.log2(p)
|
| 85 |
+
return bits
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def entropy_bits(
|
| 89 |
+
levels: Sequence[int],
|
| 90 |
+
buckets: int = 8,
|
| 91 |
+
lo: int = 0,
|
| 92 |
+
hi: int = 255,
|
| 93 |
+
fabric: ShaderFabric | None = None,
|
| 94 |
+
) -> tuple[float, list[int]]:
|
| 95 |
+
"""Entropy of a set of raw levels, plus the histogram it came from."""
|
| 96 |
+
counts = level_histogram(levels, buckets, lo, hi, fabric)
|
| 97 |
+
return shannon(counts), counts
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def local_entropy(
|
| 101 |
+
levels: Sequence[Sequence[int]], radius: int = 1, buckets: int = 8, hi: int = 255
|
| 102 |
+
) -> list[list[float]]:
|
| 103 |
+
"""Per-pixel entropy of a decoded grid inside a (2r+1) window.
|
| 104 |
+
|
| 105 |
+
A measurement of an already-decoded grid — used for reporting and for
|
| 106 |
+
choosing attenuation, not inside the exact pixel path.
|
| 107 |
+
"""
|
| 108 |
+
height = len(levels)
|
| 109 |
+
width = len(levels[0]) if height else 0
|
| 110 |
+
out = [[0.0] * width for _ in range(height)]
|
| 111 |
+
span = hi / buckets
|
| 112 |
+
for y in range(height):
|
| 113 |
+
for x in range(width):
|
| 114 |
+
counts = [0] * buckets
|
| 115 |
+
for dy in range(-radius, radius + 1):
|
| 116 |
+
for dx in range(-radius, radius + 1):
|
| 117 |
+
ny, nx = y + dy, x + dx
|
| 118 |
+
if 0 <= ny < height and 0 <= nx < width:
|
| 119 |
+
idx = min(buckets - 1, int(levels[ny][nx] / span))
|
| 120 |
+
counts[idx] += 1
|
| 121 |
+
out[y][x] = shannon(counts)
|
| 122 |
+
return out
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def entropic_octave(bits: float, max_bits: float, octaves: int = 4) -> int:
|
| 126 |
+
"""Map entropy in bits to an attenuation exponent.
|
| 127 |
+
|
| 128 |
+
Low entropy means a flat region, which is dimmed most; high entropy keeps
|
| 129 |
+
the value. The mapping is deliberately coarse: it lands on an integer
|
| 130 |
+
shift, which is what makes entropy-weighted shading exact.
|
| 131 |
+
"""
|
| 132 |
+
if max_bits <= 0:
|
| 133 |
+
return octaves
|
| 134 |
+
ratio = max(0.0, min(1.0, bits / max_bits))
|
| 135 |
+
return max(0, min(octaves, int(round((1.0 - ratio) * octaves))))
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def max_bits(buckets: int = 8) -> float:
|
| 139 |
+
"""Entropy ceiling for a histogram with ``buckets`` bins."""
|
| 140 |
+
return math.log2(max(1, buckets))
|
hdc/fractal.py
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Fractal shaders built from exact integer maps.
|
| 2 |
+
|
| 3 |
+
Two families, both avoiding floating point entirely:
|
| 4 |
+
|
| 5 |
+
* **Dyadic IFS** — an iterated function system whose coefficients are fixed
|
| 6 |
+
point with an 8-bit fraction. Every step is a multiply by a small integer
|
| 7 |
+
and a shift, so the attractor is reproduced bit-identically on any machine.
|
| 8 |
+
``sierpinski`` is the closed form of the same idea (``x & y == 0``).
|
| 9 |
+
* **Value-noise fBm** — a lattice of hashed values, interpolated with integer
|
| 10 |
+
lerps, summed over octaves whose amplitude halves by ``dim`` (an exact shift
|
| 11 |
+
of the coded level). Amplitude halving is what makes the result fractal; the
|
| 12 |
+
report prints the measured per-octave peak so the claim is checkable.
|
| 13 |
+
|
| 14 |
+
Both produce ordinary :class:`~hdc.shader.Image` objects, so fractal detail
|
| 15 |
+
composes with the morphological and entropic passes: add detail, dilate it,
|
| 16 |
+
then attenuate by entropy.
|
| 17 |
+
|
| 18 |
+
Honest limit: this is detail synthesis over a 2D grid, not a renderer for a
|
| 19 |
+
fractal surface. There is no ray-marching and no shading across scales.
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
from __future__ import annotations
|
| 23 |
+
|
| 24 |
+
from typing import Sequence
|
| 25 |
+
|
| 26 |
+
from .hypervec import hv_to_int
|
| 27 |
+
from .shader import Image, ShaderFabric
|
| 28 |
+
|
| 29 |
+
# classic Barnsley fern: (fx, fy, gx, gy, ox, oy, cumulative % of 100),
|
| 30 |
+
# coefficients scaled by 256 so every step is integer fixed point.
|
| 31 |
+
_FERN = (
|
| 32 |
+
(0, 0, 0, 0, 0, 0, 1), # stem: (x, y) -> (0, 0) 1%
|
| 33 |
+
(218, 10, -10, 218, 0, 410, 86), # 0.85/0.04, +1.6 85%
|
| 34 |
+
(51, -67, 59, 56, 0, 410, 93), # 0.20/-0.26, +1.6 7%
|
| 35 |
+
(-38, 72, 66, 61, 0, 113, 100), # -0.15/0.28, +0.44 7%
|
| 36 |
+
)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _hash(x: int, y: int, seed: int) -> int:
|
| 40 |
+
h = (x * 374761393 + y * 668265263 + seed * 1442695040888963407) & 0xFFFFFFFF
|
| 41 |
+
h = (h ^ (h >> 13)) * 1274126177 & 0xFFFFFFFF
|
| 42 |
+
return (h ^ (h >> 16)) & 0xFF
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def sierpinski(order: int, value: int = 255, fabric: ShaderFabric | None = None) -> Image:
|
| 46 |
+
"""Sierpinski triangle as ``x & y == 0`` — exact, no iteration needed."""
|
| 47 |
+
fabric = fabric or ShaderFabric()
|
| 48 |
+
size = 1 << order
|
| 49 |
+
rows = [[value if (x & y) == 0 else 0 for x in range(size)] for y in range(size)]
|
| 50 |
+
return Image.from_levels(rows, fabric)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def fern(iterations: int = 4000, width: int = 60, height: int = 70,
|
| 54 |
+
fabric: ShaderFabric | None = None, seed: int = 1) -> Image:
|
| 55 |
+
"""Barnsley fern by the chaos game, in 1/256 fixed point."""
|
| 56 |
+
fabric = fabric or ShaderFabric()
|
| 57 |
+
state = seed & 0xFFFFFFFF
|
| 58 |
+
x = y = 0 # fixed point, 256 = 1.0
|
| 59 |
+
hits: dict[tuple[int, int], int] = {}
|
| 60 |
+
for _ in range(iterations):
|
| 61 |
+
state = (1103515245 * state + 12345) & 0x7FFFFFFF
|
| 62 |
+
r = state % 100
|
| 63 |
+
for fx, fy, gx, gy, ox, oy, threshold in _FERN:
|
| 64 |
+
if r < threshold:
|
| 65 |
+
break
|
| 66 |
+
nx = (fx * x + fy * y) >> 8
|
| 67 |
+
ny = (gx * x + gy * y) >> 8
|
| 68 |
+
x, y = nx + ox, ny + oy
|
| 69 |
+
# fern spans x in [-3, 3], y in [0, 10]; mapping is integer too
|
| 70 |
+
px = ((x + 768) * (width - 1)) // (6 * 256)
|
| 71 |
+
py = (height - 1) - ((y * (height - 1)) // (10 * 256))
|
| 72 |
+
if 0 <= px < width and 0 <= py < height:
|
| 73 |
+
hits[(px, py)] = min(255, hits.get((px, py), 0) + 60)
|
| 74 |
+
rows = [[hits.get((x_, y_), 0) for x_ in range(width)] for y_ in range(height)]
|
| 75 |
+
return Image.from_levels(rows, fabric)
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def value_noise(width: int, height: int, cell: int, seed: int, amplitude: int = 255) -> list[list[int]]:
|
| 79 |
+
"""Bilinear value noise on an integer lattice, exact interpolation."""
|
| 80 |
+
rows: list[list[int]] = []
|
| 81 |
+
for y in range(height):
|
| 82 |
+
row = []
|
| 83 |
+
gy, fy = divmod(y, max(1, cell))
|
| 84 |
+
for x in range(width):
|
| 85 |
+
gx, fx = divmod(x, max(1, cell))
|
| 86 |
+
v00 = _hash(gx, gy, seed)
|
| 87 |
+
v10 = _hash(gx + 1, gy, seed)
|
| 88 |
+
v01 = _hash(gx, gy + 1, seed)
|
| 89 |
+
v11 = _hash(gx + 1, gy + 1, seed)
|
| 90 |
+
top = (v00 * (cell - fx) + v10 * fx) // max(1, cell)
|
| 91 |
+
bottom = (v01 * (cell - fx) + v11 * fx) // max(1, cell)
|
| 92 |
+
row.append(((top * (cell - fy) + bottom * fy) // max(1, cell)) * amplitude // 255)
|
| 93 |
+
rows.append(row)
|
| 94 |
+
return rows
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def fbm(width: int = 32, height: int = 32, octaves: int = 4, cell: int = 8, seed: int = 7,
|
| 98 |
+
fabric: ShaderFabric | None = None) -> Image:
|
| 99 |
+
"""Fractal Brownian motion: octaves at halving amplitude, combined by max.
|
| 100 |
+
|
| 101 |
+
``max`` rather than a sum keeps every octave's peak, which is what makes the
|
| 102 |
+
mask usable directly as a shader input; a sum would need a divide to fit the
|
| 103 |
+
coded range.
|
| 104 |
+
"""
|
| 105 |
+
fabric = fabric or ShaderFabric()
|
| 106 |
+
result = Image(width, height, fabric)
|
| 107 |
+
for octave in range(octaves):
|
| 108 |
+
step = max(1, cell >> octave)
|
| 109 |
+
field = value_noise(width, height, step, seed + octave)
|
| 110 |
+
layer = Image.from_levels(field, fabric).dim(octave)
|
| 111 |
+
result = result.max_with(layer)
|
| 112 |
+
return result
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def detail(base: Image, octaves: int = 3, cell: int = 4, seed: int = 3,
|
| 116 |
+
strength: int = 2) -> Image:
|
| 117 |
+
"""Add fBm detail to an image, then keep the maximum.
|
| 118 |
+
|
| 119 |
+
``strength`` is a shift, so the detail lands at ``2**-strength`` of full
|
| 120 |
+
amplitude — exact, no multiply.
|
| 121 |
+
"""
|
| 122 |
+
fabric = base.fabric
|
| 123 |
+
noise = fbm(base.w, base.h, octaves, cell, seed, fabric)
|
| 124 |
+
return base.max_with(noise.dim(strength))
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def octave_peaks(image: Image, octaves: int = 4) -> list[int]:
|
| 128 |
+
"""Measured peak level per octave of an fBm image, for the halving claim."""
|
| 129 |
+
peaks = []
|
| 130 |
+
for _ in range(octaves):
|
| 131 |
+
peaks.append(max(v for row in image.levels() for v in row))
|
| 132 |
+
image = image.dim(1)
|
| 133 |
+
return peaks
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def report(fabric: ShaderFabric | None = None) -> str:
|
| 137 |
+
"""Printable fractal tour."""
|
| 138 |
+
fabric = fabric or ShaderFabric(d=2000)
|
| 139 |
+
lines = ["fractal shaders (dyadic, integer-only)"]
|
| 140 |
+
lines.append("sierpinski(4):")
|
| 141 |
+
lines.append(sierpinski(4, fabric=fabric).ascii())
|
| 142 |
+
lines.append("fbm 32x32, 4 octaves, octave peaks: %s"
|
| 143 |
+
% octave_peaks(fbm(32, 32, 4, 8, 7, fabric)))
|
| 144 |
+
lines.append("fern(3000) on 26x30:")
|
| 145 |
+
lines.append(fern(3000, 26, 30, fabric).ascii())
|
| 146 |
+
return "\n".join(lines)
|
hdc/fs.py
ADDED
|
@@ -0,0 +1,366 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""A filesystem whose directory entries live in HDC.
|
| 2 |
+
|
| 3 |
+
A filesystem has two jobs and they pull in opposite directions:
|
| 4 |
+
|
| 5 |
+
* **Find things.** "Where did I put the shader notes?" is a similarity question,
|
| 6 |
+
and similarity is what this machine is made of. So every node's *record* is a
|
| 7 |
+
hypervector — `bind(role(name), name_vector) + bind(role(kind), kind_vector) +
|
| 8 |
+
bind(role(parent), parent_vector)` — stored in :class:`~hdc.memory.HDCStorage`
|
| 9 |
+
and searchable by ``sim``. :meth:`FileSystem.search` answers "which files are
|
| 10 |
+
near this description" with no index beyond the vectors themselves.
|
| 11 |
+
* **Give back exactly what was written.** A file's bytes have to round-trip, and
|
| 12 |
+
HDC cannot decode a bundle back into text: superposition is lossy by design.
|
| 13 |
+
So contents are kept **exact**, addressed by path, exactly as
|
| 14 |
+
:class:`~hdc.ring.HistoricLedger` keeps exact tick entries beside its
|
| 15 |
+
holographic summaries. Claiming otherwise would be the dishonest version of
|
| 16 |
+
this module.
|
| 17 |
+
|
| 18 |
+
What is genuinely hyperdimensional: path identity (``bind`` of the parent and the
|
| 19 |
+
name), directory listing (cleanup over the parent's children), and content search
|
| 20 |
+
(each file carries a bundle of its own word vectors, so a query is one compare
|
| 21 |
+
per file). What is conventional: the text itself, and the Python dict that maps
|
| 22 |
+
a path to its node for O(1) reads.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
from __future__ import annotations
|
| 26 |
+
|
| 27 |
+
import posixpath
|
| 28 |
+
import time
|
| 29 |
+
from typing import Iterable, Sequence
|
| 30 |
+
|
| 31 |
+
from .hypervec import DEFAULT_DIM, bind, bundle, hvc, sim
|
| 32 |
+
from .memory import HDCStorage
|
| 33 |
+
from .text import encode_word, tokenize
|
| 34 |
+
|
| 35 |
+
KINDS = ("dir", "file")
|
| 36 |
+
|
| 37 |
+
#: How many content tokens contribute to a file's search vector. A long file
|
| 38 |
+
#: bundled whole would swamp its own signal (the capacity law again), so the
|
| 39 |
+
#: first tokens plus the file name are what search actually sees.
|
| 40 |
+
SEARCH_TOKENS = 160
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
class Node:
|
| 44 |
+
"""One directory entry: exact text plus its hypervector records."""
|
| 45 |
+
|
| 46 |
+
__slots__ = ("path", "name", "kind", "lines", "vector", "search", "created", "modified")
|
| 47 |
+
|
| 48 |
+
def __init__(self, path: str, name: str, kind: str, lines: list[str] | None = None):
|
| 49 |
+
self.path = path
|
| 50 |
+
self.name = name
|
| 51 |
+
self.kind = kind
|
| 52 |
+
self.lines = lines if lines is not None else []
|
| 53 |
+
self.vector = 0
|
| 54 |
+
self.search = 0
|
| 55 |
+
stamp = time.time()
|
| 56 |
+
self.created = stamp
|
| 57 |
+
self.modified = stamp
|
| 58 |
+
|
| 59 |
+
@property
|
| 60 |
+
def size(self) -> int:
|
| 61 |
+
"""Bytes of content, newline-terminated like any text file."""
|
| 62 |
+
return sum(len(line) + 1 for line in self.lines)
|
| 63 |
+
|
| 64 |
+
def text(self) -> str:
|
| 65 |
+
return "\n".join(self.lines)
|
| 66 |
+
|
| 67 |
+
def as_dict(self) -> dict:
|
| 68 |
+
return {"name": self.name, "path": self.path, "kind": self.kind,
|
| 69 |
+
"size": self.size, "lines": len(self.lines),
|
| 70 |
+
"modified": round(self.modified, 3)}
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
class FileSystem:
|
| 74 |
+
"""Directories by binding, contents by exact address."""
|
| 75 |
+
|
| 76 |
+
def __init__(self, d: int = DEFAULT_DIM, storage: HDCStorage | None = None):
|
| 77 |
+
self.d = d
|
| 78 |
+
# not ``storage or ...``: HDCStorage defines __len__, so a brand-new
|
| 79 |
+
# (empty) store is falsy and would be silently replaced by a second one.
|
| 80 |
+
# That bug split the desktop's memory map from its filesystem.
|
| 81 |
+
self.storage = storage if storage is not None else HDCStorage(d=d, label="fs")
|
| 82 |
+
self.nodes: dict[str, Node] = {}
|
| 83 |
+
self.cwd = "/"
|
| 84 |
+
root = Node("/", "", "dir")
|
| 85 |
+
self._index(root)
|
| 86 |
+
self.writes = 0
|
| 87 |
+
self.searches = 0
|
| 88 |
+
|
| 89 |
+
# ------------------------------------------------------------- plumbing
|
| 90 |
+
def _vector(self, name: str, kind: str, parent: str) -> int:
|
| 91 |
+
return bundle([
|
| 92 |
+
bind(hvc("fs/role/name", self.d), hvc("fs/name/" + name, self.d), self.d),
|
| 93 |
+
bind(hvc("fs/role/kind", self.d), hvc("fs/kind/" + kind, self.d), self.d),
|
| 94 |
+
bind(hvc("fs/role/parent", self.d), hvc("fs/parent/" + parent, self.d), self.d),
|
| 95 |
+
], self.d)
|
| 96 |
+
|
| 97 |
+
def _search_vector(self, name: str, lines: Sequence[str]) -> int:
|
| 98 |
+
words = tokenize(name.replace(".", " ").replace("/", " "))
|
| 99 |
+
for line in lines[:SEARCH_TOKENS]:
|
| 100 |
+
words.extend(tokenize(line))
|
| 101 |
+
if not words:
|
| 102 |
+
words = ["empty"]
|
| 103 |
+
return bundle([encode_word(word, self.d) for word in words[:SEARCH_TOKENS * 2]], self.d)
|
| 104 |
+
|
| 105 |
+
def _index(self, node: Node) -> Node:
|
| 106 |
+
parent = posixpath.dirname(node.path) or "/"
|
| 107 |
+
node.vector = self._vector(node.name, node.kind, parent)
|
| 108 |
+
node.search = self._search_vector(node.name, node.lines)
|
| 109 |
+
self.nodes[node.path] = node
|
| 110 |
+
self.storage.write("fs/" + node.path, node.vector)
|
| 111 |
+
return node
|
| 112 |
+
|
| 113 |
+
def normalise(self, path: str) -> str:
|
| 114 |
+
"""Absolute, normalised path; ``.`` and ``..`` resolved against ``cwd``."""
|
| 115 |
+
if not path:
|
| 116 |
+
return self.cwd
|
| 117 |
+
if not path.startswith("/"):
|
| 118 |
+
path = posixpath.join(self.cwd, path)
|
| 119 |
+
out = posixpath.normpath(path)
|
| 120 |
+
return "/" if out in (".", "") else out
|
| 121 |
+
|
| 122 |
+
def parent(self, path: str) -> str:
|
| 123 |
+
return posixpath.dirname(self.normalise(path)) or "/"
|
| 124 |
+
|
| 125 |
+
# -------------------------------------------------------------- queries
|
| 126 |
+
def exists(self, path: str) -> bool:
|
| 127 |
+
return self.normalise(path) in self.nodes
|
| 128 |
+
|
| 129 |
+
def isdir(self, path: str) -> bool:
|
| 130 |
+
node = self.nodes.get(self.normalise(path))
|
| 131 |
+
return bool(node and node.kind == "dir")
|
| 132 |
+
|
| 133 |
+
def get(self, path: str) -> Node:
|
| 134 |
+
node = self.nodes.get(self.normalise(path))
|
| 135 |
+
if node is None:
|
| 136 |
+
raise FileNotFoundError(self.normalise(path))
|
| 137 |
+
return node
|
| 138 |
+
|
| 139 |
+
def children(self, path: str) -> list[Node]:
|
| 140 |
+
directory = self.normalise(path)
|
| 141 |
+
if not self.isdir(directory):
|
| 142 |
+
raise NotADirectoryError(directory)
|
| 143 |
+
prefix = "/" if directory == "/" else directory + "/"
|
| 144 |
+
out = [node for node in self.nodes.values()
|
| 145 |
+
if node.path != directory and posixpath.dirname(node.path) ==
|
| 146 |
+
(directory if directory != "/" else "/")]
|
| 147 |
+
return sorted(out, key=lambda node: (node.kind != "dir", node.name))
|
| 148 |
+
|
| 149 |
+
def ls(self, path: str = ".") -> list[dict]:
|
| 150 |
+
return [node.as_dict() for node in self.children(path)]
|
| 151 |
+
|
| 152 |
+
def walk(self, path: str = "/") -> list[Node]:
|
| 153 |
+
out = [self.get(path)]
|
| 154 |
+
for child in self.children(path):
|
| 155 |
+
out.extend(self.walk(child.path) if child.kind == "dir" else [child])
|
| 156 |
+
return out
|
| 157 |
+
|
| 158 |
+
def tree(self, path: str = "/") -> str:
|
| 159 |
+
lines: list[str] = [self.normalise(path)]
|
| 160 |
+
self._tree(self.normalise(path), "", lines)
|
| 161 |
+
return "\n".join(lines)
|
| 162 |
+
|
| 163 |
+
def _tree(self, directory: str, indent: str, lines: list[str]) -> None:
|
| 164 |
+
entries = self.children(directory)
|
| 165 |
+
for index, node in enumerate(entries):
|
| 166 |
+
last = index == len(entries) - 1
|
| 167 |
+
lines.append("%s%s %s%s" % (indent, "`-" if last else "|-", node.name,
|
| 168 |
+
"/" if node.kind == "dir" else ""))
|
| 169 |
+
if node.kind == "dir":
|
| 170 |
+
self._tree(node.path, indent + (" " if last else "| "), lines)
|
| 171 |
+
|
| 172 |
+
# ------------------------------------------------------------ mutation
|
| 173 |
+
def mkdir(self, path: str, parents: bool = True) -> Node:
|
| 174 |
+
target = self.normalise(path)
|
| 175 |
+
if target == "/":
|
| 176 |
+
return self.nodes["/"]
|
| 177 |
+
if target in self.nodes:
|
| 178 |
+
if self.nodes[target].kind == "dir":
|
| 179 |
+
return self.nodes[target]
|
| 180 |
+
raise FileExistsError(target)
|
| 181 |
+
parent = self.parent(target)
|
| 182 |
+
if parent not in self.nodes:
|
| 183 |
+
if not parents:
|
| 184 |
+
raise FileNotFoundError(parent)
|
| 185 |
+
self.mkdir(parent, parents=True)
|
| 186 |
+
if not self.isdir(parent):
|
| 187 |
+
raise NotADirectoryError(parent)
|
| 188 |
+
return self._index(Node(target, posixpath.basename(target), "dir"))
|
| 189 |
+
|
| 190 |
+
def write(self, path: str, text: str) -> Node:
|
| 191 |
+
"""Create or replace a file; the text is stored exactly.
|
| 192 |
+
|
| 193 |
+
Intermediate directories are created, like ``mkdir -p``. An editor
|
| 194 |
+
saving to ``/home/user/src/new/thing.hdcasm`` should not have to make two
|
| 195 |
+
directories first, and the failure it would otherwise raise
|
| 196 |
+
(``FileNotFoundError``) reads as a bug in the editor rather than as the
|
| 197 |
+
caller's mistake. Writing *through* a file still raises, because the
|
| 198 |
+
parent then exists and is not a directory.
|
| 199 |
+
"""
|
| 200 |
+
target = self.normalise(path)
|
| 201 |
+
parent = self.parent(target)
|
| 202 |
+
if not self.exists(parent):
|
| 203 |
+
self.mkdir(parent, parents=True)
|
| 204 |
+
if not self.isdir(parent):
|
| 205 |
+
raise NotADirectoryError(parent)
|
| 206 |
+
existing = self.nodes.get(target)
|
| 207 |
+
if existing is not None:
|
| 208 |
+
if existing.kind == "dir":
|
| 209 |
+
raise IsADirectoryError(target)
|
| 210 |
+
existing.lines = text.splitlines()
|
| 211 |
+
existing.modified = time.time()
|
| 212 |
+
self._index(existing)
|
| 213 |
+
else:
|
| 214 |
+
existing = self._index(
|
| 215 |
+
Node(target, posixpath.basename(target), "file", text.splitlines()))
|
| 216 |
+
self.writes += 1
|
| 217 |
+
return existing
|
| 218 |
+
|
| 219 |
+
def append(self, path: str, text: str) -> Node:
|
| 220 |
+
node = self.get(path)
|
| 221 |
+
if node.kind != "file":
|
| 222 |
+
raise IsADirectoryError(node.path)
|
| 223 |
+
node.lines.extend(text.splitlines())
|
| 224 |
+
node.modified = time.time()
|
| 225 |
+
return self._index(node)
|
| 226 |
+
|
| 227 |
+
def read(self, path: str) -> str:
|
| 228 |
+
node = self.get(path)
|
| 229 |
+
if node.kind != "file":
|
| 230 |
+
raise IsADirectoryError(node.path)
|
| 231 |
+
return node.text()
|
| 232 |
+
|
| 233 |
+
def remove(self, path: str, recursive: bool = False) -> int:
|
| 234 |
+
"""Remove a file, or a directory when empty (or ``recursive``)."""
|
| 235 |
+
node = self.get(path)
|
| 236 |
+
if node.path == "/":
|
| 237 |
+
raise PermissionError("cannot remove the root directory")
|
| 238 |
+
victims = [node]
|
| 239 |
+
if node.kind == "dir":
|
| 240 |
+
victims = self.walk(node.path)
|
| 241 |
+
if len(victims) > 1 and not recursive:
|
| 242 |
+
raise OSError("directory not empty: " + node.path)
|
| 243 |
+
for victim in victims:
|
| 244 |
+
self.nodes.pop(victim.path, None)
|
| 245 |
+
self.storage.remove("fs/" + victim.path)
|
| 246 |
+
return len(victims)
|
| 247 |
+
|
| 248 |
+
def rename(self, old: str, new: str) -> Node:
|
| 249 |
+
node = self.get(old)
|
| 250 |
+
target = self.normalise(new)
|
| 251 |
+
if target in self.nodes:
|
| 252 |
+
raise FileExistsError(target)
|
| 253 |
+
if not self.isdir(self.parent(target)):
|
| 254 |
+
raise FileNotFoundError(self.parent(target))
|
| 255 |
+
moved = [node] + (self.walk(node.path)[1:] if node.kind == "dir" else [])
|
| 256 |
+
for victim in moved:
|
| 257 |
+
suffix = victim.path[len(node.path):]
|
| 258 |
+
self.nodes.pop(victim.path, None)
|
| 259 |
+
self.storage.remove("fs/" + victim.path)
|
| 260 |
+
victim.path = target + suffix
|
| 261 |
+
victim.name = posixpath.basename(victim.path)
|
| 262 |
+
self._index(victim)
|
| 263 |
+
return node
|
| 264 |
+
|
| 265 |
+
def copy(self, old: str, new: str) -> Node:
|
| 266 |
+
source = self.get(old)
|
| 267 |
+
if source.kind == "file":
|
| 268 |
+
return self.write(new, source.text())
|
| 269 |
+
self.mkdir(new)
|
| 270 |
+
for child in self.children(source.path):
|
| 271 |
+
self.copy(child.path, posixpath.join(self.normalise(new), child.name))
|
| 272 |
+
return self.get(new)
|
| 273 |
+
|
| 274 |
+
def chdir(self, path: str) -> str:
|
| 275 |
+
target = self.normalise(path)
|
| 276 |
+
if not self.isdir(target):
|
| 277 |
+
raise NotADirectoryError(target)
|
| 278 |
+
self.cwd = target
|
| 279 |
+
return self.cwd
|
| 280 |
+
|
| 281 |
+
# -------------------------------------------------------------- search
|
| 282 |
+
def search(self, query: str, limit: int = 8) -> list[tuple[str, float]]:
|
| 283 |
+
"""Rank every node by similarity to a description — one compare each.
|
| 284 |
+
|
| 285 |
+
This is the payoff of keeping records as vectors: no index to build, no
|
| 286 |
+
tokeniser to consult. A query is a bundle of its word vectors and each
|
| 287 |
+
file is compared against it.
|
| 288 |
+
"""
|
| 289 |
+
words = tokenize(query) or [query]
|
| 290 |
+
probe = bundle([encode_word(word, self.d) for word in words], self.d)
|
| 291 |
+
scored = [(node.path, sim(probe, node.search, self.d))
|
| 292 |
+
for node in self.nodes.values() if node.path != "/"]
|
| 293 |
+
scored.sort(key=lambda item: -item[1])
|
| 294 |
+
self.searches += 1
|
| 295 |
+
return scored[:limit]
|
| 296 |
+
|
| 297 |
+
def similar(self, path: str, limit: int = 8) -> list[tuple[str, float]]:
|
| 298 |
+
node = self.get(path)
|
| 299 |
+
scored = [(other.path, sim(node.search, other.search, self.d))
|
| 300 |
+
for other in self.nodes.values()
|
| 301 |
+
if other.path not in ("/", node.path)]
|
| 302 |
+
scored.sort(key=lambda item: -item[1])
|
| 303 |
+
return scored[:limit]
|
| 304 |
+
|
| 305 |
+
# -------------------------------------------------------------- report
|
| 306 |
+
def stat(self) -> dict:
|
| 307 |
+
files = [node for node in self.nodes.values() if node.kind == "file"]
|
| 308 |
+
return {"files": len(files), "dirs": len(self.nodes) - len(files),
|
| 309 |
+
"bytes": sum(node.size for node in files),
|
| 310 |
+
"cells": len(self.storage), "writes": self.writes,
|
| 311 |
+
"searches": self.searches, "cwd": self.cwd,
|
| 312 |
+
"dim": self.d}
|
| 313 |
+
|
| 314 |
+
def save(self, path: str) -> int:
|
| 315 |
+
"""Persist the node vectors (the exact text stays with the caller)."""
|
| 316 |
+
return self.storage.save(path)
|
| 317 |
+
|
| 318 |
+
def seed(self) -> None:
|
| 319 |
+
"""A small tree, so a fresh desktop has something in it."""
|
| 320 |
+
if "/home" in self.nodes:
|
| 321 |
+
return
|
| 322 |
+
self.mkdir("/home")
|
| 323 |
+
self.mkdir("/home/user")
|
| 324 |
+
self.mkdir("/home/user/src")
|
| 325 |
+
self.mkdir("/home/user/img")
|
| 326 |
+
self.write("/home/user/README.txt",
|
| 327 |
+
"HDCSTORAGE webOS\n"
|
| 328 |
+
"Files are exact; their addresses are hypervectors.\n"
|
| 329 |
+
"Try: find shader notes")
|
| 330 |
+
self.write("/home/user/src/hello.hdcasm",
|
| 331 |
+
"; add two numbers and halt\n"
|
| 332 |
+
"LDI R1, 7\n"
|
| 333 |
+
"LDI R2, 35\n"
|
| 334 |
+
"ADD R3, R1, R2\n"
|
| 335 |
+
"HLT")
|
| 336 |
+
self.write("/home/user/src/loop.hdcasm",
|
| 337 |
+
"; count down from five\n"
|
| 338 |
+
"LDI R1, 5\n"
|
| 339 |
+
"loop: JZ R1, done\n"
|
| 340 |
+
"SUB R1, R1, 1\n"
|
| 341 |
+
"JMP loop\n"
|
| 342 |
+
"done: HLT")
|
| 343 |
+
self.write("/home/user/notes-shader.txt",
|
| 344 |
+
"shader notes: dimming is exact, edges are |a-b|.\n"
|
| 345 |
+
"depth test compares coded values directly.")
|
| 346 |
+
self.write("/home/user/img/gradient.txt", "0 32 64 96 128 160 192 224 255")
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
def seeded(d: int = DEFAULT_DIM, storage: HDCStorage | None = None) -> FileSystem:
|
| 350 |
+
"""A filesystem with the starter tree already in it.
|
| 351 |
+
|
| 352 |
+
Pass the machine's storage when the desktop boots, so the filesystem's cells
|
| 353 |
+
are the kernel's cells and the memory map counts them — two stores would make
|
| 354 |
+
every "free cells" figure a lie.
|
| 355 |
+
"""
|
| 356 |
+
filesystem = FileSystem(d=d, storage=storage)
|
| 357 |
+
filesystem.seed()
|
| 358 |
+
return filesystem
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
if __name__ == "__main__": # pragma: no cover - manual inspection
|
| 362 |
+
filesystem = seeded(2000)
|
| 363 |
+
print(filesystem.tree("/home"))
|
| 364 |
+
print()
|
| 365 |
+
for path, score in filesystem.search("shader notes about edges"):
|
| 366 |
+
print(" %+.3f %s" % (score, path))
|
hdc/gfx.py
ADDED
|
@@ -0,0 +1,415 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The graphics engine: coded cells, shader passes, and a text overlay.
|
| 2 |
+
|
| 3 |
+
This is the raster end of the machine. A :class:`Surface` is a grid of cells and
|
| 4 |
+
every cell's brightness is a **coded hypervector** (``int_to_hv``), not an
|
| 5 |
+
integer — which is what makes the passes here arithmetic in HDC rather than array
|
| 6 |
+
manipulation:
|
| 7 |
+
|
| 8 |
+
* **Dimming is exact and free.** ``shift(hv, -k)`` halves a value: two XORs and
|
| 9 |
+
a rotation, no decode. A scanline or a vignette is a shift per cell.
|
| 10 |
+
* **Blending is ``max``/``min``**, which the fabric resolves with similarity
|
| 11 |
+
probes on the coded values — the same comparator the depth buffer uses.
|
| 12 |
+
* **Edge detection** has two modes and the difference is stated rather than
|
| 13 |
+
hidden: ``proxy`` is one ``sim`` per cell (cheap, approximate), ``exact`` is
|
| 14 |
+
``|a-b|`` through the hyperdimensional adder (correct, and two orders of
|
| 15 |
+
magnitude slower — see :func:`benchmark`).
|
| 16 |
+
|
| 17 |
+
Text is an *overlay*, not coded: a character cell is a character. Representing
|
| 18 |
+
glyphs as bundled vectors would be decoration, and a layer either does the thing
|
| 19 |
+
or says it does not.
|
| 20 |
+
|
| 21 |
+
**Why there is an integer grid beside the coded one.** Display has to be one
|
| 22 |
+
ramp lookup per cell; decoding a coded value costs 64 similarity probes, so a
|
| 23 |
+
full decode per frame would cost seconds. So each surface keeps a plain integer
|
| 24 |
+
shadow used *only* to choose the ramp character, while the coded cells remain the
|
| 25 |
+
authority for every operation. :meth:`Surface.verify` decodes a sample and
|
| 26 |
+
compares it to the shadow, so the cache cannot drift silently — and the tests use
|
| 27 |
+
it. This is what a framebuffer is; the arithmetic that fills it is HDC.
|
| 28 |
+
|
| 29 |
+
Honest limits: no rasteriser (no triangles, no clipping, no antialiasing), the
|
| 30 |
+
framebuffer is character-cell resolution, ``edges(exact)`` is slow by
|
| 31 |
+
construction, and the integer shadow is a cache rather than a claim.
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
from __future__ import annotations
|
| 35 |
+
|
| 36 |
+
import math
|
| 37 |
+
import time
|
| 38 |
+
|
| 39 |
+
from .hypervec import DEFAULT_DIM, hv_to_int
|
| 40 |
+
from .shader import ShaderFabric
|
| 41 |
+
|
| 42 |
+
#: Brightness ramp for a monochrome cell display, darkest to brightest.
|
| 43 |
+
RAMP = " .:-=+*#%@"
|
| 44 |
+
|
| 45 |
+
#: Shader programs: named pass chains, applied in order over the whole surface.
|
| 46 |
+
SHADERS: dict[str, list[tuple[str, dict]]] = {
|
| 47 |
+
"background": [
|
| 48 |
+
("gradient", {"top": 26, "bottom": 96, "vertical": True}),
|
| 49 |
+
("noise", {"scale": 7, "octaves": 3, "seed": 5, "amplitude": 54}),
|
| 50 |
+
("scanline", {"steps": 1}),
|
| 51 |
+
("vignette", {"strength": 2}),
|
| 52 |
+
],
|
| 53 |
+
"metal": [
|
| 54 |
+
("noise", {"scale": 4, "octaves": 4, "seed": 11, "amplitude": 200}),
|
| 55 |
+
("edges", {"mode": "proxy", "gain": 3}),
|
| 56 |
+
("dim_region", {"steps": 1}),
|
| 57 |
+
],
|
| 58 |
+
"plasma": [
|
| 59 |
+
("noise", {"scale": 6, "octaves": 2, "seed": 3, "amplitude": 140}),
|
| 60 |
+
("max_blend", {"scale": 5, "seed": 9, "amplitude": 120}),
|
| 61 |
+
("scanline", {"steps": 2}),
|
| 62 |
+
],
|
| 63 |
+
"flat": [("gradient", {"top": 12, "bottom": 28, "vertical": True})],
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _hash2(x: int, y: int, seed: int) -> int:
|
| 68 |
+
h = (x * 0x27D4EB2D ^ y * 0x165667B1 ^ seed * 0x9E3779B1) & 0xFFFFFFFF
|
| 69 |
+
h ^= h >> 15
|
| 70 |
+
h = (h * 0x2545F491) & 0xFFFFFFFF
|
| 71 |
+
return (h ^ (h >> 13)) & 0xFFFFFFFF
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def value_noise(width: int, height: int, scale: int, seed: int,
|
| 75 |
+
amplitude: int = 255) -> list[list[int]]:
|
| 76 |
+
"""Smooth integer value noise — bilinear interpolation of a coarse lattice."""
|
| 77 |
+
scale = max(1, scale)
|
| 78 |
+
out: list[list[int]] = []
|
| 79 |
+
for y in range(height):
|
| 80 |
+
row: list[int] = []
|
| 81 |
+
gy, fy = divmod(y, scale)
|
| 82 |
+
fy /= scale
|
| 83 |
+
for x in range(width):
|
| 84 |
+
gx, fx = divmod(x, scale)
|
| 85 |
+
fx /= scale
|
| 86 |
+
a = _hash2(gx, gy, seed) & 0xFF
|
| 87 |
+
b = _hash2(gx + 1, gy, seed) & 0xFF
|
| 88 |
+
c = _hash2(gx, gy + 1, seed) & 0xFF
|
| 89 |
+
d = _hash2(gx + 1, gy + 1, seed) & 0xFF
|
| 90 |
+
top = a + (b - a) * fx
|
| 91 |
+
bottom = c + (d - c) * fx
|
| 92 |
+
row.append(int(max(0.0, min(255.0, (top + (bottom - top) * fy) * amplitude / 255.0))))
|
| 93 |
+
out.append(row)
|
| 94 |
+
return out
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
class Surface:
|
| 98 |
+
"""A coded-cell framebuffer with a character overlay and a display cache."""
|
| 99 |
+
|
| 100 |
+
def __init__(self, cols: int, rows: int, fabric: ShaderFabric | None = None,
|
| 101 |
+
d: int = DEFAULT_DIM):
|
| 102 |
+
if cols < 1 or rows < 1:
|
| 103 |
+
raise ValueError("surface needs at least one cell")
|
| 104 |
+
self.cols = cols
|
| 105 |
+
self.rows_ = rows
|
| 106 |
+
self.d = d
|
| 107 |
+
self.fabric = fabric or ShaderFabric(d=d)
|
| 108 |
+
self.cells = [[0] * cols for _ in range(rows)]
|
| 109 |
+
self.shadow = [[0] * cols for _ in range(rows)] # ramp cache only
|
| 110 |
+
self.text = [[""] * cols for _ in range(rows)]
|
| 111 |
+
self.passes = 0
|
| 112 |
+
self.coded = 0
|
| 113 |
+
self.clear(0)
|
| 114 |
+
|
| 115 |
+
# ------------------------------------------------------------ geometry
|
| 116 |
+
def inside(self, x: int, y: int) -> bool:
|
| 117 |
+
return 0 <= x < self.cols and 0 <= y < self.rows_
|
| 118 |
+
|
| 119 |
+
def clamp_rect(self, x: int, y: int, w: int, h: int) -> tuple[int, int, int, int]:
|
| 120 |
+
x0, y0 = max(0, x), max(0, y)
|
| 121 |
+
x1, y1 = min(self.cols, x + w), min(self.rows_, y + h)
|
| 122 |
+
return x0, y0, max(0, x1 - x0), max(0, y1 - y0)
|
| 123 |
+
|
| 124 |
+
# ---------------------------------------------------------- primitives
|
| 125 |
+
def clear(self, level: int = 0) -> "Surface":
|
| 126 |
+
cell = self.fabric.level(level)
|
| 127 |
+
for y in range(self.rows_):
|
| 128 |
+
for x in range(self.cols):
|
| 129 |
+
self.cells[y][x] = cell
|
| 130 |
+
self.shadow[y][x] = level
|
| 131 |
+
self.text[y][x] = ""
|
| 132 |
+
self.coded += self.cols * self.rows_
|
| 133 |
+
return self
|
| 134 |
+
|
| 135 |
+
def put(self, x: int, y: int, level: int) -> None:
|
| 136 |
+
if self.inside(x, y):
|
| 137 |
+
self.cells[y][x] = self.fabric.level(level)
|
| 138 |
+
self.shadow[y][x] = level
|
| 139 |
+
self.coded += 1
|
| 140 |
+
|
| 141 |
+
def fill(self, x: int, y: int, w: int, h: int, level: int) -> None:
|
| 142 |
+
cell = self.fabric.level(level)
|
| 143 |
+
x0, y0, w, h = self.clamp_rect(x, y, w, h)
|
| 144 |
+
for row in range(y0, y0 + h):
|
| 145 |
+
line, cache = self.cells[row], self.shadow[row]
|
| 146 |
+
for col in range(x0, x0 + w):
|
| 147 |
+
line[col] = cell
|
| 148 |
+
cache[col] = level
|
| 149 |
+
self.coded += w * h
|
| 150 |
+
|
| 151 |
+
def frame(self, x: int, y: int, w: int, h: int, level: int) -> None:
|
| 152 |
+
x0, y0, w, h = self.clamp_rect(x, y, w, h)
|
| 153 |
+
if not w or not h:
|
| 154 |
+
return
|
| 155 |
+
self.fill(x0, y0, w, 1, level)
|
| 156 |
+
self.fill(x0, y0 + h - 1, w, 1, level)
|
| 157 |
+
self.fill(x0, y0, 1, h, level)
|
| 158 |
+
self.fill(x0 + w - 1, y0, 1, h, level)
|
| 159 |
+
|
| 160 |
+
def text_at(self, x: int, y: int, text: str, level: int | None = None) -> None:
|
| 161 |
+
"""Write characters left to right; clipped, never wrapped."""
|
| 162 |
+
cell = self.fabric.level(level) if level is not None else None
|
| 163 |
+
for index, char in enumerate(text):
|
| 164 |
+
col = x + index
|
| 165 |
+
if not self.inside(col, y):
|
| 166 |
+
break
|
| 167 |
+
self.text[y][col] = char
|
| 168 |
+
if cell is not None:
|
| 169 |
+
self.cells[y][col] = cell
|
| 170 |
+
self.shadow[y][col] = level
|
| 171 |
+
|
| 172 |
+
def level_at(self, x: int, y: int) -> int:
|
| 173 |
+
"""Decode one cell — the real value, not the cache."""
|
| 174 |
+
return hv_to_int(self.cells[y][x], self.d)
|
| 175 |
+
|
| 176 |
+
def copy(self) -> "Surface":
|
| 177 |
+
"""An independent copy, coded cells and overlay included.
|
| 178 |
+
|
| 179 |
+
Used by the boot splash: the expensive shader base (noise plus a
|
| 180 |
+
``max_blend`` pass, ~90-350 ms) is rendered once and each animation
|
| 181 |
+
frame copies it and applies cheap exact shifts, instead of re-running the
|
| 182 |
+
whole pipeline per frame.
|
| 183 |
+
"""
|
| 184 |
+
clone = Surface(self.cols, self.rows_, fabric=self.fabric, d=self.d)
|
| 185 |
+
clone.cells = [row[:] for row in self.cells]
|
| 186 |
+
clone.shadow = [row[:] for row in self.shadow]
|
| 187 |
+
clone.text = [row[:] for row in self.text]
|
| 188 |
+
clone.passes = self.passes
|
| 189 |
+
clone.coded = self.coded
|
| 190 |
+
return clone
|
| 191 |
+
|
| 192 |
+
# ------------------------------------------------------------- shaders
|
| 193 |
+
def gradient(self, top: int = 0, bottom: int = 255, vertical: bool = True) -> "Surface":
|
| 194 |
+
span = max(1, (self.rows_ if vertical else self.cols) - 1)
|
| 195 |
+
steps = self.rows_ if vertical else self.cols
|
| 196 |
+
levels = [top + (bottom - top) * i // span for i in range(steps)]
|
| 197 |
+
for y in range(self.rows_):
|
| 198 |
+
for x in range(self.cols):
|
| 199 |
+
level = levels[y if vertical else x]
|
| 200 |
+
self.cells[y][x] = self.fabric.level(level)
|
| 201 |
+
self.shadow[y][x] = level
|
| 202 |
+
self.coded += self.cols * self.rows_
|
| 203 |
+
self.passes += 1
|
| 204 |
+
return self
|
| 205 |
+
|
| 206 |
+
def noise(self, scale: int = 6, octaves: int = 3, seed: int = 7,
|
| 207 |
+
amplitude: int = 60) -> "Surface":
|
| 208 |
+
"""Additive fractal noise: each octave halves in scale and in weight."""
|
| 209 |
+
field = [[0] * self.cols for _ in range(self.rows_)]
|
| 210 |
+
weight = amplitude
|
| 211 |
+
cell = max(1, scale)
|
| 212 |
+
for octave in range(max(1, octaves)):
|
| 213 |
+
layer = value_noise(self.cols, self.rows_, cell, seed + octave, weight)
|
| 214 |
+
for y in range(self.rows_):
|
| 215 |
+
target, source = field[y], layer[y]
|
| 216 |
+
for x in range(self.cols):
|
| 217 |
+
target[x] += source[x]
|
| 218 |
+
cell = max(1, cell // 2)
|
| 219 |
+
weight = max(1, weight // 2)
|
| 220 |
+
for y in range(self.rows_):
|
| 221 |
+
for x in range(self.cols):
|
| 222 |
+
level = min(255, max(0, field[y][x]))
|
| 223 |
+
self.cells[y][x] = self.fabric.level(level)
|
| 224 |
+
self.shadow[y][x] = level
|
| 225 |
+
self.coded += self.cols * self.rows_
|
| 226 |
+
self.passes += 1
|
| 227 |
+
return self
|
| 228 |
+
|
| 229 |
+
def dim_region(self, steps: int = 1, x: int = 0, y: int = 0,
|
| 230 |
+
w: int | None = None, h: int | None = None) -> "Surface":
|
| 231 |
+
"""Exact halving: ``shift`` each coded value, no decode anywhere."""
|
| 232 |
+
w = self.cols if w is None else w
|
| 233 |
+
h = self.rows_ if h is None else h
|
| 234 |
+
x0, y0, w, h = self.clamp_rect(x, y, w, h)
|
| 235 |
+
for row in range(y0, y0 + h):
|
| 236 |
+
line, cache = self.cells[row], self.shadow[row]
|
| 237 |
+
for col in range(x0, x0 + w):
|
| 238 |
+
line[col] = self.fabric.dim(line[col], steps)
|
| 239 |
+
cache[col] = cache[col] >> steps
|
| 240 |
+
self.passes += 1
|
| 241 |
+
return self
|
| 242 |
+
|
| 243 |
+
def brighten_region(self, steps: int = 1, x: int = 0, y: int = 0,
|
| 244 |
+
w: int | None = None, h: int | None = None) -> "Surface":
|
| 245 |
+
w = self.cols if w is None else w
|
| 246 |
+
h = self.rows_ if h is None else h
|
| 247 |
+
x0, y0, w, h = self.clamp_rect(x, y, w, h)
|
| 248 |
+
for row in range(y0, y0 + h):
|
| 249 |
+
line, cache = self.cells[row], self.shadow[row]
|
| 250 |
+
for col in range(x0, x0 + w):
|
| 251 |
+
line[col] = self.fabric.brighten(line[col], steps)
|
| 252 |
+
cache[col] = min(255, cache[col] << steps)
|
| 253 |
+
self.passes += 1
|
| 254 |
+
return self
|
| 255 |
+
|
| 256 |
+
def scanline(self, steps: int = 1, offset: int = 1) -> "Surface":
|
| 257 |
+
"""Dim alternating rows — the CRT look, done with exact shifts."""
|
| 258 |
+
for y in range(offset, self.rows_, 2):
|
| 259 |
+
self.dim_region(steps, 0, y, self.cols, 1)
|
| 260 |
+
self.passes += 1
|
| 261 |
+
return self
|
| 262 |
+
|
| 263 |
+
def vignette(self, strength: int = 2, cx: float | None = None,
|
| 264 |
+
cy: float | None = None) -> "Surface":
|
| 265 |
+
"""Darken by squared distance from the centre, in ``strength`` buckets."""
|
| 266 |
+
cx = (self.cols - 1) / 2.0 if cx is None else cx
|
| 267 |
+
cy = (self.rows_ - 1) / 2.0 if cy is None else cy
|
| 268 |
+
span = math.hypot(max(cx, self.cols - cx), max(cy, self.rows_ - cy)) or 1.0
|
| 269 |
+
for y in range(self.rows_):
|
| 270 |
+
for x in range(self.cols):
|
| 271 |
+
distance = math.hypot(x - cx, y - cy) / span
|
| 272 |
+
steps = int(distance * distance * max(0, strength) * 3)
|
| 273 |
+
if steps:
|
| 274 |
+
self.cells[y][x] = self.fabric.dim(self.cells[y][x], steps)
|
| 275 |
+
self.shadow[y][x] >>= steps
|
| 276 |
+
self.passes += 1
|
| 277 |
+
return self
|
| 278 |
+
|
| 279 |
+
def edges(self, mode: str = "proxy", gain: int = 2,
|
| 280 |
+
region: tuple[int, int, int, int] | None = None) -> "Surface":
|
| 281 |
+
"""Edge map: the cheap similarity proxy, or the exact difference.
|
| 282 |
+
|
| 283 |
+
``proxy`` is ``1 - sim`` against the right neighbour, scaled into a
|
| 284 |
+
level — one 2000-bit compare per cell. ``exact`` runs ``|a-b|`` through
|
| 285 |
+
the hyperdimensional adder, which is the honest operation and about two
|
| 286 |
+
orders of magnitude slower at this resolution.
|
| 287 |
+
"""
|
| 288 |
+
if mode not in ("proxy", "exact"):
|
| 289 |
+
raise ValueError("mode must be 'proxy' or 'exact'")
|
| 290 |
+
if region is None:
|
| 291 |
+
x0, y0, w, h = 0, 0, self.cols, self.rows_
|
| 292 |
+
else:
|
| 293 |
+
x0, y0, w, h = self.clamp_rect(*region)
|
| 294 |
+
previous = [row[:] for row in self.cells]
|
| 295 |
+
cache = [row[:] for row in self.shadow]
|
| 296 |
+
for y in range(y0, y0 + h):
|
| 297 |
+
for x in range(x0, min(x0 + w, self.cols - 1)):
|
| 298 |
+
here, right = previous[y][x], previous[y][x + 1]
|
| 299 |
+
if mode == "proxy":
|
| 300 |
+
level = min(255, int(self.fabric.discontinuity(here, right) * 255 * gain))
|
| 301 |
+
self.cells[y][x] = self.fabric.level(level)
|
| 302 |
+
else:
|
| 303 |
+
self.cells[y][x] = self.fabric.gradient(here, right)
|
| 304 |
+
level = abs(cache[y][x] - cache[y][x + 1])
|
| 305 |
+
self.shadow[y][x] = level
|
| 306 |
+
self.passes += 1
|
| 307 |
+
return self
|
| 308 |
+
|
| 309 |
+
def max_blend(self, scale: int = 5, seed: int = 9, amplitude: int = 120) -> "Surface":
|
| 310 |
+
"""Blend in a second noise field with ``max`` — probed, not decoded."""
|
| 311 |
+
field = value_noise(self.cols, self.rows_, scale, seed, amplitude)
|
| 312 |
+
for y in range(self.rows_):
|
| 313 |
+
for x in range(self.cols):
|
| 314 |
+
candidate = self.fabric.level(field[y][x])
|
| 315 |
+
self.cells[y][x] = self.fabric.max_of(self.cells[y][x], candidate)
|
| 316 |
+
self.shadow[y][x] = max(self.shadow[y][x], field[y][x])
|
| 317 |
+
self.passes += 1
|
| 318 |
+
return self
|
| 319 |
+
|
| 320 |
+
# --------------------------------------------------------------- output
|
| 321 |
+
def apply(self, name: str, **params) -> "Surface":
|
| 322 |
+
"""Run a named shader program from :data:`SHADERS`."""
|
| 323 |
+
program = SHADERS.get(name)
|
| 324 |
+
if program is None:
|
| 325 |
+
raise KeyError("unknown shader %r" % (name,))
|
| 326 |
+
for operation, defaults in program:
|
| 327 |
+
getattr(self, operation)(**dict(defaults, **params))
|
| 328 |
+
return self
|
| 329 |
+
|
| 330 |
+
def levels(self) -> list[list[int]]:
|
| 331 |
+
"""Decode **every** cell — the authority, and slow on purpose."""
|
| 332 |
+
return [[hv_to_int(cell, self.d) for cell in row] for row in self.cells]
|
| 333 |
+
|
| 334 |
+
def verify(self, sample: int = 24) -> dict:
|
| 335 |
+
"""Decode a sample and check the display cache against the coded cells."""
|
| 336 |
+
checked = matches = 0
|
| 337 |
+
step = max(1, (self.cols * self.rows_) // max(1, sample))
|
| 338 |
+
for index in range(0, self.cols * self.rows_, step):
|
| 339 |
+
y, x = divmod(index, self.cols)
|
| 340 |
+
checked += 1
|
| 341 |
+
if self.level_at(x, y) == self.shadow[y][x]:
|
| 342 |
+
matches += 1
|
| 343 |
+
return {"checked": checked, "matches": matches,
|
| 344 |
+
"ok": checked == matches and checked > 0}
|
| 345 |
+
|
| 346 |
+
def rows(self, ramp: str = RAMP) -> list[str]:
|
| 347 |
+
"""The character grid: text overlay first, then the brightness ramp."""
|
| 348 |
+
top = len(ramp) - 1
|
| 349 |
+
out: list[str] = []
|
| 350 |
+
for y in range(self.rows_):
|
| 351 |
+
chars: list[str] = []
|
| 352 |
+
cache = self.shadow[y]
|
| 353 |
+
for x in range(self.cols):
|
| 354 |
+
char = self.text[y][x]
|
| 355 |
+
if char:
|
| 356 |
+
chars.append(char)
|
| 357 |
+
else:
|
| 358 |
+
chars.append(ramp[cache[x] * top // 255])
|
| 359 |
+
out.append("".join(chars).rstrip())
|
| 360 |
+
return out
|
| 361 |
+
|
| 362 |
+
def ascii(self, ramp: str = RAMP) -> str:
|
| 363 |
+
return "\n".join(self.rows(ramp))
|
| 364 |
+
|
| 365 |
+
def stat(self) -> dict:
|
| 366 |
+
return {"cols": self.cols, "rows": self.rows_, "cells": self.cols * self.rows_,
|
| 367 |
+
"passes": self.passes, "coded": self.coded, "d": self.d}
|
| 368 |
+
|
| 369 |
+
|
| 370 |
+
def demo(d: int = 2000, cols: int = 48, rows: int = 14) -> str:
|
| 371 |
+
lines: list[str] = []
|
| 372 |
+
for name in SHADERS:
|
| 373 |
+
surface = Surface(cols, rows, d=d).apply(name)
|
| 374 |
+
lines.append("[%s]" % name)
|
| 375 |
+
lines.append(surface.ascii())
|
| 376 |
+
lines.append("")
|
| 377 |
+
return "\n".join(lines)
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
def benchmark(d: int = 2000, cols: int = 80, rows: int = 28) -> dict:
|
| 381 |
+
"""Measure the passes, so "fast enough for a desktop frame" is a number."""
|
| 382 |
+
timings: dict[str, float] = {}
|
| 383 |
+
for name in SHADERS:
|
| 384 |
+
surface = Surface(cols, rows, d=d)
|
| 385 |
+
started = time.perf_counter()
|
| 386 |
+
surface.apply(name)
|
| 387 |
+
timings[name] = (time.perf_counter() - started) * 1000.0
|
| 388 |
+
# the exact edge pass is measured on a small tile: per-cell cost is what
|
| 389 |
+
# matters and a full screen would just multiply it by 50
|
| 390 |
+
tile = Surface(16, 8, d=d).apply("metal")
|
| 391 |
+
started = time.perf_counter()
|
| 392 |
+
tile.edges(mode="exact")
|
| 393 |
+
exact_ms = (time.perf_counter() - started) * 1000.0
|
| 394 |
+
return {"shaders": timings, "cols": cols, "rows": rows, "cells": cols * rows,
|
| 395 |
+
"d": d, "exact_edges_ms_per_128_cells": exact_ms}
|
| 396 |
+
|
| 397 |
+
|
| 398 |
+
def report(d: int = 2000) -> str:
|
| 399 |
+
stats = benchmark(d=d)
|
| 400 |
+
lines = ["gfx: %dx%d cells (%d), d=%d"
|
| 401 |
+
% (stats["cols"], stats["rows"], stats["cells"], d)]
|
| 402 |
+
for name, ms in sorted(stats["shaders"].items(), key=lambda item: item[1]):
|
| 403 |
+
rate = stats["cells"] / (ms / 1000.0) if ms else float("inf")
|
| 404 |
+
lines.append(" %-14s %8.1f ms %10.0f cells/s" % (name, ms, rate))
|
| 405 |
+
lines.append(" edges(exact) %.0f ms per 128 cells -> a full screen would take "
|
| 406 |
+
"%.0f s, so it ships as a demo pass only"
|
| 407 |
+
% (stats["exact_edges_ms_per_128_cells"],
|
| 408 |
+
stats["exact_edges_ms_per_128_cells"] * stats["cells"] / 128.0 / 1000.0))
|
| 409 |
+
lines.append(" dim/scanline/vignette are exact shifts; max_blend uses similarity "
|
| 410 |
+
"probes; the integer shadow is display cache, checked by verify()")
|
| 411 |
+
return "\n".join(lines)
|
| 412 |
+
|
| 413 |
+
|
| 414 |
+
if __name__ == "__main__": # pragma: no cover - manual inspection
|
| 415 |
+
print(report())
|
hdc/hub.py
ADDED
|
@@ -0,0 +1,516 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Model discovery: the Hub search API, and a suitability score that is measured.
|
| 2 |
+
|
| 3 |
+
The question this module answers is narrow: *which published embedding table can
|
| 4 |
+
this machine actually use?* Not "which is best on MTEB" — which one can be read
|
| 5 |
+
by ``struct``, projected into hypervectors, and driven from the standard library,
|
| 6 |
+
including on **machine code** (this project's own ISA mnemonics, x86 opcodes,
|
| 7 |
+
registers and hex literals).
|
| 8 |
+
|
| 9 |
+
Two axes, both measured rather than assumed:
|
| 10 |
+
|
| 11 |
+
**Runtime-free.** A static embedding model is a ``token x dim`` matrix. There is
|
| 12 |
+
no interpreter, no attention, no graph — the file *is* the program, and the CPU
|
| 13 |
+
executes the projection directly. That is exactly the property this project
|
| 14 |
+
needs, and it is readable from the safetensors header: a static model has a
|
| 15 |
+
single rank-2 tensor. A transformer has hundreds of tensors and needs a deep
|
| 16 |
+
learning runtime, so it cannot run here at all.
|
| 17 |
+
|
| 18 |
+
**Machine-code coverage.** The encoder can only embed a token it has a row for.
|
| 19 |
+
So the tokenizer's vocabulary is fetched and probed against
|
| 20 |
+
:data:`MACHINE_CODE` — the mnemonics of this project's own assembler, a set of
|
| 21 |
+
x86-64 opcodes/registers, and literal forms. Coverage is a fraction, reported
|
| 22 |
+
with the tokens that went missing, because a mnemonic that tokenises to ``[UNK]``
|
| 23 |
+
gets the *same* vector as every other missing token and silently collapses the
|
| 24 |
+
program space.
|
| 25 |
+
|
| 26 |
+
Search itself: ``hf`` (huggingface_hub) up to 0.36.x exposes only
|
| 27 |
+
``auth/cache/download/jobs/repo/repo-files/upload/env/version`` — there is no
|
| 28 |
+
``models`` or ``api`` subcommand, verified against the installed build — so
|
| 29 |
+
:func:`search` calls the HTTPS endpoint that those commands wrap, and
|
| 30 |
+
:func:`fetch` hands actual downloads back to ``hf download``, which is the part
|
| 31 |
+
the CLI does handle. Both paths are stdlib.
|
| 32 |
+
|
| 33 |
+
Nothing here is required at runtime: the machine runs on ``models/`` files that
|
| 34 |
+
were fetched once. This module is the *chooser* and the *provenance record*.
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
from __future__ import annotations
|
| 38 |
+
|
| 39 |
+
import json
|
| 40 |
+
import os
|
| 41 |
+
import shutil
|
| 42 |
+
import struct
|
| 43 |
+
import subprocess
|
| 44 |
+
import urllib.error
|
| 45 |
+
import urllib.parse
|
| 46 |
+
import urllib.request
|
| 47 |
+
|
| 48 |
+
HUB = "https://huggingface.co"
|
| 49 |
+
API = HUB + "/api/models"
|
| 50 |
+
CACHE_DIR = os.path.join("models", "_hub")
|
| 51 |
+
|
| 52 |
+
#: Tokens a hypervector machine-code encoder has to be able to represent.
|
| 53 |
+
#: First group: this project's own assembler mnemonics (``hdc/isa.py``).
|
| 54 |
+
#: Second: x86-64 names an assembler listing would carry.
|
| 55 |
+
#: Third: operand and literal forms.
|
| 56 |
+
MACHINE_CODE = [
|
| 57 |
+
"ldi", "sta", "lda", "add", "sub", "and", "or", "xor", "jmp", "jz", "jnz",
|
| 58 |
+
"hlt", "nop", "push", "pop", "call", "ret", "cmp", "mov", "lea", "shl",
|
| 59 |
+
"shr", "inc", "dec", "tst", "not",
|
| 60 |
+
"rax", "rbx", "rcx", "rdx", "rdi", "rsi", "rip", "eax", "esi", "edi",
|
| 61 |
+
"0x1f", "0b1011", "$0x10", "[rip+4]", "r1", "r2",
|
| 62 |
+
]
|
| 63 |
+
|
| 64 |
+
#: How much each measured component counts toward the score. Sums to 1.0.
|
| 65 |
+
WEIGHTS = {
|
| 66 |
+
"runtime_free": 0.35, # single rank-2 tensor: no interpreter needed
|
| 67 |
+
"code": 0.30, # measured tokenizer coverage of MACHINE_CODE
|
| 68 |
+
"size": 0.15, # bytes on disk (download + resident cost)
|
| 69 |
+
"dim": 0.10, # projection cost per word (d output bits each)
|
| 70 |
+
"license": 0.10, # can the vectors be shipped?
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
PERMISSIVE = ("mit", "apache-2.0", "cc-by-4.0", "bsd-3-clause", "cc0-1.0", "wtfpl")
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
# ---------------------------------------------------------------- the client
|
| 77 |
+
def hf_api_available() -> bool:
|
| 78 |
+
"""True when the installed ``hf`` CLI can run raw API calls.
|
| 79 |
+
|
| 80 |
+
It cannot in 0.36.x — that is why :func:`search` speaks HTTPS directly. The
|
| 81 |
+
check is kept because the CLI does gain the subcommand in later builds, and
|
| 82 |
+
when it exists there is no reason not to use it.
|
| 83 |
+
"""
|
| 84 |
+
hf = shutil.which("hf")
|
| 85 |
+
if not hf:
|
| 86 |
+
return False
|
| 87 |
+
try:
|
| 88 |
+
out = subprocess.run([hf, "--help"], capture_output=True, text=True, timeout=20)
|
| 89 |
+
except (OSError, subprocess.SubprocessError):
|
| 90 |
+
return False
|
| 91 |
+
text = (out.stdout or "") + (out.stderr or "")
|
| 92 |
+
return "\n api " in text or " api " in text.split("positional arguments")[-1]
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _request(url: str, headers: dict | None = None, timeout: int = 45) -> bytes:
|
| 96 |
+
request = urllib.request.Request(url, headers=dict(
|
| 97 |
+
{"User-Agent": "hdcstorage/1.0 (content-addressed hypervectors)"},
|
| 98 |
+
**(headers or {})))
|
| 99 |
+
with urllib.request.urlopen(request, timeout=timeout) as response:
|
| 100 |
+
return response.read()
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def _get_json(url: str, timeout: int = 45):
|
| 104 |
+
try:
|
| 105 |
+
return json.loads(_request(url, timeout=timeout))
|
| 106 |
+
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, OSError):
|
| 107 |
+
return None
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def search(text: str | None = None, *, task: str | None = "feature-extraction",
|
| 111 |
+
author: str | None = None, library: str | None = None,
|
| 112 |
+
limit: int = 25, sort: str = "downloads", direction: int = -1) -> list[dict]:
|
| 113 |
+
"""Query the Hub model index; returns normalised records, newest API shape.
|
| 114 |
+
|
| 115 |
+
Two measured quirks of the endpoint, both of which silently return an empty
|
| 116 |
+
list rather than an error:
|
| 117 |
+
|
| 118 |
+
* ``author`` combined with ``filter`` yields zero records — so the task
|
| 119 |
+
filter is dropped when an author is named.
|
| 120 |
+
* ``filter=library:model2vec`` is not a supported filter; the whole family is
|
| 121 |
+
enumerated with ``author=minishlab`` instead.
|
| 122 |
+
|
| 123 |
+
An empty pool looks exactly like "nothing suitable exists", so these are
|
| 124 |
+
handled here rather than left to the caller.
|
| 125 |
+
"""
|
| 126 |
+
query: dict[str, str] = {"limit": str(limit), "sort": sort, "direction": str(direction)}
|
| 127 |
+
if text:
|
| 128 |
+
query["search"] = text
|
| 129 |
+
if task and not author:
|
| 130 |
+
query["filter"] = task
|
| 131 |
+
if author:
|
| 132 |
+
query["author"] = author
|
| 133 |
+
payload = _get_json(API + "?" + urllib.parse.urlencode(query))
|
| 134 |
+
if not isinstance(payload, list):
|
| 135 |
+
return []
|
| 136 |
+
records = [normalise(record) for record in payload]
|
| 137 |
+
if library:
|
| 138 |
+
records = [r for r in records if r.get("library") == library]
|
| 139 |
+
return records
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def normalise(record: dict) -> dict:
|
| 143 |
+
"""One API record reduced to the fields this module uses."""
|
| 144 |
+
tags = list(record.get("tags") or [])
|
| 145 |
+
licence = next((tag.split(":", 1)[1] for tag in tags if tag.startswith("license:")), "")
|
| 146 |
+
return {
|
| 147 |
+
"id": record.get("id") or record.get("modelId") or "",
|
| 148 |
+
"downloads": int(record.get("downloads") or 0),
|
| 149 |
+
"likes": int(record.get("likes") or 0),
|
| 150 |
+
"library": record.get("library_name") or "",
|
| 151 |
+
"task": record.get("pipeline_tag") or "",
|
| 152 |
+
"license": licence,
|
| 153 |
+
"tags": tags,
|
| 154 |
+
"files": [s.get("rfilename", "") for s in record.get("siblings") or []],
|
| 155 |
+
"gated": bool(record.get("gated")),
|
| 156 |
+
"sha": record.get("sha") or "",
|
| 157 |
+
}
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
# --------------------------------------------------- header without download
|
| 161 |
+
def safetensors(repo: str, filename: str = "model.safetensors") -> dict | None:
|
| 162 |
+
"""Read a safetensors *header* with two HTTP Range requests.
|
| 163 |
+
|
| 164 |
+
The format is an 8-byte little-endian header length, a JSON header, then the
|
| 165 |
+
raw tensor buffer. Nothing after the header is needed to know the dtypes,
|
| 166 |
+
the shapes and the exact on-disk size, so scoring a 129 MB model costs two
|
| 167 |
+
requests of a few kilobytes instead of a 129 MB download.
|
| 168 |
+
"""
|
| 169 |
+
url = "%s/%s/resolve/main/%s" % (HUB, repo, filename)
|
| 170 |
+
try:
|
| 171 |
+
head = _request(url, {"Range": "bytes=0-7"})
|
| 172 |
+
if len(head) < 8:
|
| 173 |
+
return None
|
| 174 |
+
length = struct.unpack("<Q", head)[0]
|
| 175 |
+
if not 8 <= length <= 100_000_000:
|
| 176 |
+
return None
|
| 177 |
+
header = json.loads(_request(url, {"Range": "bytes=8-%d" % (8 + length - 1)}))
|
| 178 |
+
except (urllib.error.URLError, TimeoutError, struct.error,
|
| 179 |
+
json.JSONDecodeError, OSError):
|
| 180 |
+
return None
|
| 181 |
+
tensors = {name: {"dtype": body.get("dtype"), "shape": body.get("shape") or []}
|
| 182 |
+
for name, body in header.items() if name != "__metadata__"}
|
| 183 |
+
total = max((body.get("data_offsets", [0, 0])[1] for name, body in header.items()
|
| 184 |
+
if name != "__metadata__"), default=0)
|
| 185 |
+
return {"repo": repo, "tensors": tensors, "bytes": total,
|
| 186 |
+
"params": sum(_shape_product(t["shape"]) for t in tensors.values())}
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def _shape_product(shape: list) -> int:
|
| 190 |
+
out = 1
|
| 191 |
+
for value in shape:
|
| 192 |
+
out *= int(value)
|
| 193 |
+
return out
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def static_table(header: dict | None) -> tuple[bool, int, int]:
|
| 197 |
+
"""``(is_static, rows, dim)`` — a static model is one rank-2 tensor.
|
| 198 |
+
|
| 199 |
+
That single test separates "a matrix the CPU can multiply" from "a model that
|
| 200 |
+
needs a graph engine", and it is read from the file itself rather than from a
|
| 201 |
+
tag someone else applied.
|
| 202 |
+
"""
|
| 203 |
+
if not header or len(header.get("tensors", {})) != 1:
|
| 204 |
+
return False, 0, 0
|
| 205 |
+
shape = next(iter(header["tensors"].values()))["shape"]
|
| 206 |
+
if len(shape) != 2:
|
| 207 |
+
return False, 0, 0
|
| 208 |
+
return True, int(shape[0]), int(shape[1])
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
# ------------------------------------------------------- machine-code coverage
|
| 212 |
+
def tokenizer(repo: str, filename: str = "tokenizer.json", limit: int = 8_000_000) -> dict | None:
|
| 213 |
+
"""Fetch a tokenizer file, with a size guard.
|
| 214 |
+
|
| 215 |
+
``tokenizer.json`` for a large multilingual vocabulary can be tens of
|
| 216 |
+
megabytes; above ``limit`` the candidate is left unprobed rather than
|
| 217 |
+
spending that on a score.
|
| 218 |
+
"""
|
| 219 |
+
url = "%s/%s/resolve/main/%s" % (HUB, repo, filename)
|
| 220 |
+
try:
|
| 221 |
+
size = _head_length(url)
|
| 222 |
+
if size and size > limit:
|
| 223 |
+
return None
|
| 224 |
+
payload = json.loads(_request(url, timeout=90))
|
| 225 |
+
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, OSError):
|
| 226 |
+
return None
|
| 227 |
+
model = (payload or {}).get("model") or {}
|
| 228 |
+
vocab = model.get("vocab") or {}
|
| 229 |
+
if isinstance(vocab, dict):
|
| 230 |
+
return {"vocab": vocab, "prefix": model.get("continuing_subword_prefix", "##")}
|
| 231 |
+
# fast-tokenizer BPE stores merges; token strings live in `added_tokens`
|
| 232 |
+
return None
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
def _head_length(url: str) -> int:
|
| 236 |
+
try:
|
| 237 |
+
request = urllib.request.Request(
|
| 238 |
+
url, method="HEAD",
|
| 239 |
+
headers={"User-Agent": "hdcstorage/1.0"})
|
| 240 |
+
with urllib.request.urlopen(request, timeout=30) as response:
|
| 241 |
+
return int(response.headers.get("Content-Length") or 0)
|
| 242 |
+
except (urllib.error.URLError, TimeoutError, ValueError, OSError):
|
| 243 |
+
return 0
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def code_coverage(vocab: dict, prefix: str = "##",
|
| 247 |
+
tokens: list[str] | None = None) -> dict:
|
| 248 |
+
"""Fraction of :data:`MACHINE_CODE` that has a token id of its own.
|
| 249 |
+
|
| 250 |
+
A word is *covered* when the model can tokenise it without falling back to
|
| 251 |
+
``[UNK]``: either the whole token is in the vocabulary, or it splits into
|
| 252 |
+
known pieces. Piecewise coverage is weaker evidence — ``0x1f`` as ``0x`` +
|
| 253 |
+
``1f`` is two lookups averaged, not a learned literal — so whole-token hits
|
| 254 |
+
are counted separately.
|
| 255 |
+
"""
|
| 256 |
+
tokens = list(MACHINE_CODE if tokens is None else tokens)
|
| 257 |
+
whole, piecewise, missing = [], [], []
|
| 258 |
+
for token in tokens:
|
| 259 |
+
text = token.lower()
|
| 260 |
+
if text in vocab:
|
| 261 |
+
whole.append(token)
|
| 262 |
+
elif _pieces(text, vocab, prefix):
|
| 263 |
+
piecewise.append(token)
|
| 264 |
+
else:
|
| 265 |
+
missing.append(token)
|
| 266 |
+
total = len(tokens) or 1
|
| 267 |
+
any_fraction = (len(whole) + len(piecewise)) / total
|
| 268 |
+
whole_fraction = len(whole) / total
|
| 269 |
+
return {"tokens": len(tokens), "whole": whole, "pieces": piecewise, "missing": missing,
|
| 270 |
+
# a whole token is a learned vector; pieces are an average of
|
| 271 |
+
# fragment vectors, so they count for less than half
|
| 272 |
+
"coverage": 0.7 * whole_fraction + 0.3 * any_fraction,
|
| 273 |
+
"any_fraction": any_fraction,
|
| 274 |
+
"whole_fraction": whole_fraction}
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
def _pieces(text: str, vocab: dict, prefix: str) -> bool:
|
| 278 |
+
start = 0
|
| 279 |
+
while start < len(text):
|
| 280 |
+
end = len(text)
|
| 281 |
+
found = None
|
| 282 |
+
while start < end:
|
| 283 |
+
candidate = text[start:end] if not start else prefix + text[start:end]
|
| 284 |
+
if candidate in vocab:
|
| 285 |
+
found = candidate
|
| 286 |
+
break
|
| 287 |
+
end -= 1
|
| 288 |
+
if found is None:
|
| 289 |
+
return False
|
| 290 |
+
start = end
|
| 291 |
+
return True
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
# --------------------------------------------------------------------- score
|
| 295 |
+
def score(record: dict, header: dict | None = None, coverage: dict | None = None) -> dict:
|
| 296 |
+
"""Component scores plus the weighted total; every part is inspectable."""
|
| 297 |
+
is_static, rows, dim = static_table(header)
|
| 298 |
+
components = {
|
| 299 |
+
"runtime_free": 1.0 if is_static else 0.0,
|
| 300 |
+
"code": float(coverage["coverage"]) if coverage else 0.0,
|
| 301 |
+
"size": _size_score(header["bytes"] if header else 0),
|
| 302 |
+
"dim": _dim_score(dim),
|
| 303 |
+
"license": 1.0 if (record.get("license") or "").lower() in PERMISSIVE else 0.2,
|
| 304 |
+
}
|
| 305 |
+
total = sum(WEIGHTS[name] * value for name, value in components.items())
|
| 306 |
+
return {"components": {k: round(v, 3) for k, v in components.items()},
|
| 307 |
+
"total": round(total, 3), "static": is_static,
|
| 308 |
+
"rows": rows, "dim": dim,
|
| 309 |
+
"bytes": header["bytes"] if header else 0,
|
| 310 |
+
"coverage": coverage}
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
def _size_score(size: int) -> float:
|
| 314 |
+
"""1.0 up to 35 MB, linear to 0.0 at 512 MB."""
|
| 315 |
+
if size <= 0:
|
| 316 |
+
return 0.0
|
| 317 |
+
if size <= 35_000_000:
|
| 318 |
+
return 1.0
|
| 319 |
+
if size >= 512_000_000:
|
| 320 |
+
return 0.0
|
| 321 |
+
return 1.0 - (size - 35_000_000) / (512_000_000 - 35_000_000)
|
| 322 |
+
|
| 323 |
+
|
| 324 |
+
def _dim_score(dim: int) -> float:
|
| 325 |
+
"""256 is the sweet spot: 128 loses distinctions, 768+ costs projection."""
|
| 326 |
+
if dim <= 0:
|
| 327 |
+
return 0.0
|
| 328 |
+
if dim == 256:
|
| 329 |
+
return 1.0
|
| 330 |
+
if dim < 256:
|
| 331 |
+
return max(0.0, dim / 256.0)
|
| 332 |
+
return max(0.0, 1.0 - (dim - 256) / 768.0)
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
# ------------------------------------------------------------------- fetch
|
| 336 |
+
def fetch(repo: str, files: list[str], local_dir: str) -> dict:
|
| 337 |
+
"""Download files with ``hf download`` — the part the CLI does implement.
|
| 338 |
+
|
| 339 |
+
Falls back to HTTPS when ``hf`` is missing, so a fresh machine can still get
|
| 340 |
+
the model; each file lands under ``local_dir`` with its repository name.
|
| 341 |
+
"""
|
| 342 |
+
os.makedirs(local_dir, exist_ok=True)
|
| 343 |
+
hf = shutil.which("hf")
|
| 344 |
+
if hf:
|
| 345 |
+
command = [hf, "download", repo, *files, "--local-dir", local_dir]
|
| 346 |
+
try:
|
| 347 |
+
done = subprocess.run(command, capture_output=True, text=True, timeout=1800)
|
| 348 |
+
return {"ok": done.returncode == 0, "via": "hf download",
|
| 349 |
+
"command": " ".join(command), "output": (done.stdout or "")[-600:],
|
| 350 |
+
"error": (done.stderr or "")[-600:]}
|
| 351 |
+
except (OSError, subprocess.SubprocessError) as error:
|
| 352 |
+
return {"ok": False, "via": "hf download", "error": str(error)}
|
| 353 |
+
for name in files:
|
| 354 |
+
url = "%s/%s/resolve/main/%s" % (HUB, repo, name)
|
| 355 |
+
target = os.path.join(local_dir, name)
|
| 356 |
+
try:
|
| 357 |
+
os.makedirs(os.path.dirname(target) or ".", exist_ok=True)
|
| 358 |
+
with open(target, "wb") as fh:
|
| 359 |
+
fh.write(_request(url, timeout=600))
|
| 360 |
+
except (urllib.error.URLError, TimeoutError, OSError) as error:
|
| 361 |
+
return {"ok": False, "via": "https", "error": "%s: %s" % (name, error)}
|
| 362 |
+
return {"ok": True, "via": "https"}
|
| 363 |
+
|
| 364 |
+
|
| 365 |
+
# ----------------------------------------------------------------- bakeoff
|
| 366 |
+
#: The queries behind :func:`bakeoff`. ``author`` enumerates the static-embedding
|
| 367 |
+
#: family; the text queries reach the code and retrieval-tuned tables.
|
| 368 |
+
QUERIES = (
|
| 369 |
+
{"author": "minishlab", "limit": 40},
|
| 370 |
+
{"text": "static embeddings", "limit": 20},
|
| 371 |
+
{"text": "code embeddings", "limit": 20},
|
| 372 |
+
{"text": "assembly machine code embeddings", "limit": 20},
|
| 373 |
+
{"text": "sentence embeddings", "limit": 20},
|
| 374 |
+
)
|
| 375 |
+
|
| 376 |
+
|
| 377 |
+
def candidate_pool(limit: int = 60) -> list[dict]:
|
| 378 |
+
"""Union of the search queries, de-duplicated by repository id."""
|
| 379 |
+
seen: dict[str, dict] = {}
|
| 380 |
+
for query in QUERIES:
|
| 381 |
+
records = search(query.get("text"), author=query.get("author"),
|
| 382 |
+
limit=int(query.get("limit", 20)))
|
| 383 |
+
for record in records:
|
| 384 |
+
if record["id"] and record["id"] not in seen:
|
| 385 |
+
seen[record["id"]] = record
|
| 386 |
+
pool = sorted(seen.values(), key=lambda r: -r["downloads"])
|
| 387 |
+
return pool[:max(1, limit)]
|
| 388 |
+
|
| 389 |
+
|
| 390 |
+
def cache_path() -> str:
|
| 391 |
+
return os.path.join(CACHE_DIR, "scores.json")
|
| 392 |
+
|
| 393 |
+
|
| 394 |
+
def _cache_usable(entry: dict) -> bool:
|
| 395 |
+
"""False for entries written by an older scoring scheme.
|
| 396 |
+
|
| 397 |
+
The cache outlives the code that wrote it, so a coverage entry missing a
|
| 398 |
+
field the report now prints has to be dropped rather than trusted — an
|
| 399 |
+
older run of ``--offline`` hit exactly that and raised ``KeyError``.
|
| 400 |
+
"""
|
| 401 |
+
coverage = entry.get("coverage")
|
| 402 |
+
return coverage is None or ("any_fraction" in coverage and "whole_fraction" in coverage)
|
| 403 |
+
|
| 404 |
+
|
| 405 |
+
def load_cache() -> dict:
|
| 406 |
+
try:
|
| 407 |
+
with open(cache_path(), "r", encoding="utf-8") as fh:
|
| 408 |
+
data = json.load(fh)
|
| 409 |
+
except (OSError, json.JSONDecodeError):
|
| 410 |
+
return {}
|
| 411 |
+
if not isinstance(data, dict):
|
| 412 |
+
return {}
|
| 413 |
+
return {key: value for key, value in data.items()
|
| 414 |
+
if isinstance(value, dict) and _cache_usable(value)}
|
| 415 |
+
|
| 416 |
+
|
| 417 |
+
def save_cache(cache: dict) -> None:
|
| 418 |
+
os.makedirs(CACHE_DIR, exist_ok=True)
|
| 419 |
+
with open(cache_path(), "w", encoding="utf-8") as fh:
|
| 420 |
+
json.dump(cache, fh, indent=1, sort_keys=True)
|
| 421 |
+
|
| 422 |
+
|
| 423 |
+
def probe(record: dict, cache: dict | None = None, offline: bool = False) -> dict:
|
| 424 |
+
"""Measure one candidate, reusing the cache keyed by ``repo@sha``."""
|
| 425 |
+
key = "%s@%s" % (record["id"], record.get("sha") or "head")
|
| 426 |
+
cache = {} if cache is None else cache
|
| 427 |
+
cached = cache.get(key)
|
| 428 |
+
if cached is not None:
|
| 429 |
+
return dict(cached, cached=True)
|
| 430 |
+
if offline:
|
| 431 |
+
return dict(record, score=None, cached=False, note="not probed (offline)")
|
| 432 |
+
header = safetensors(record["id"])
|
| 433 |
+
token = tokenizer(record["id"], limit=4_000_000) if header else None
|
| 434 |
+
coverage = code_coverage(token["vocab"], token.get("prefix", "##")) if token else None
|
| 435 |
+
graded = score(record, header, coverage)
|
| 436 |
+
out = dict(record, **graded, cached=False)
|
| 437 |
+
cache[key] = {k: v for k, v in out.items() if k != "cached"}
|
| 438 |
+
return out
|
| 439 |
+
|
| 440 |
+
|
| 441 |
+
def bakeoff(limit: int = 10, pool_limit: int = 24, offline: bool = False,
|
| 442 |
+
refresh: bool = False) -> dict:
|
| 443 |
+
"""Score the pool and rank it. Returns a dict, not prose, so callers can use it.
|
| 444 |
+
|
| 445 |
+
Probing costs two Range requests plus one tokenizer fetch per candidate, so
|
| 446 |
+
the pool is bounded and results are cached by ``repo@sha``; a repeat run is
|
| 447 |
+
instant and needs no network.
|
| 448 |
+
"""
|
| 449 |
+
cache = {} if refresh else load_cache()
|
| 450 |
+
pool = [] if offline else candidate_pool(limit=pool_limit)
|
| 451 |
+
if offline:
|
| 452 |
+
# nothing to search: everything known is already in the cache
|
| 453 |
+
pool = [{"id": key.split("@")[0], "sha": key.split("@")[1], "downloads": 0,
|
| 454 |
+
"likes": 0, "library": "", "task": "", "license": "", "tags": [],
|
| 455 |
+
"files": [], "gated": False}
|
| 456 |
+
for key in cache]
|
| 457 |
+
graded = [probe(record, cache) for record in pool]
|
| 458 |
+
graded = [row for row in graded if row.get("score", row.get("total")) is not None]
|
| 459 |
+
graded.sort(key=lambda row: (-float(row.get("total") or 0), -row.get("downloads", 0)))
|
| 460 |
+
if not offline:
|
| 461 |
+
save_cache(cache)
|
| 462 |
+
return {"pool": len(pool), "graded": graded[:limit], "cache": cache_path()}
|
| 463 |
+
|
| 464 |
+
|
| 465 |
+
def best(result: dict) -> dict | None:
|
| 466 |
+
rows = result.get("graded") or []
|
| 467 |
+
return rows[0] if rows else None
|
| 468 |
+
|
| 469 |
+
|
| 470 |
+
# ------------------------------------------------------------------ report
|
| 471 |
+
def report(limit: int = 8, offline: bool = False, refresh: bool = False) -> str:
|
| 472 |
+
"""The printable form: what was searched, what scored what, and why."""
|
| 473 |
+
lines = ["hub: search via %s%s"
|
| 474 |
+
% (API, "" if not hf_api_available() else " (hf api available)")]
|
| 475 |
+
lines.append("score weights: " + ", ".join("%s %.2f" % item for item in WEIGHTS.items()))
|
| 476 |
+
result = bakeoff(limit=limit, offline=offline, refresh=refresh)
|
| 477 |
+
if not result["graded"]:
|
| 478 |
+
return "\n".join(lines + ["no candidates scored — run `python -m hdc hub` with the network up"])
|
| 479 |
+
lines.append("%-42s %5s %6s %6s %6s %6s %s"
|
| 480 |
+
% ("repo", "score", "code", "static", "dim", "MB", "licence"))
|
| 481 |
+
for row in result["graded"]:
|
| 482 |
+
components = row.get("components") or {}
|
| 483 |
+
lines.append("%-42s %5.2f %6.2f %6s %6d %6.1f %s"
|
| 484 |
+
% (row["id"][:42], float(row.get("total") or 0),
|
| 485 |
+
float(components.get("code") or 0),
|
| 486 |
+
"yes" if row.get("static") else "no",
|
| 487 |
+
int(row.get("dim") or 0),
|
| 488 |
+
(row.get("bytes") or 0) / 1e6,
|
| 489 |
+
row.get("license") or "-"))
|
| 490 |
+
winner = best(result)
|
| 491 |
+
if winner:
|
| 492 |
+
coverage = winner.get("coverage") or {}
|
| 493 |
+
lines.append("")
|
| 494 |
+
lines.append("best: %s score %.2f %d x %d %.1f MB"
|
| 495 |
+
% (winner["id"], float(winner.get("total") or 0),
|
| 496 |
+
int(winner.get("rows") or 0), int(winner.get("dim") or 0),
|
| 497 |
+
(winner.get("bytes") or 0) / 1e6))
|
| 498 |
+
if coverage:
|
| 499 |
+
lines.append(" machine-code: %.0f%% whole-token, %.0f%% tokenisable, "
|
| 500 |
+
"%d of %d missing"
|
| 501 |
+
% (100.0 * coverage.get("whole_fraction", 0.0),
|
| 502 |
+
100.0 * coverage.get("any_fraction", 0.0),
|
| 503 |
+
len(coverage.get("missing") or []),
|
| 504 |
+
coverage.get("tokens", 0)))
|
| 505 |
+
if coverage["missing"]:
|
| 506 |
+
lines.append(" missing: " + " ".join(coverage["missing"][:24]))
|
| 507 |
+
lines.append(" fetch: hf download %s model.safetensors tokenizer.json config.json "
|
| 508 |
+
"--local-dir models/%s" % (winner["id"], winner["id"].split("/")[-1]))
|
| 509 |
+
lines.append("")
|
| 510 |
+
lines.append("probe: safetensors header via two Range requests; no model bytes "
|
| 511 |
+
"downloaded to score")
|
| 512 |
+
return "\n".join(lines)
|
| 513 |
+
|
| 514 |
+
|
| 515 |
+
if __name__ == "__main__": # pragma: no cover - manual inspection
|
| 516 |
+
print(report())
|
hdc/hypervec.py
ADDED
|
@@ -0,0 +1,440 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Binary hypervector algebra for HDCSTORAGE.
|
| 2 |
+
|
| 3 |
+
Everything in this machine is a *bit-packed hypervector*: a Python ``int``
|
| 4 |
+
interpreted as ``d`` bits, bit 0 being the least significant. Representation
|
| 5 |
+
choices, and why:
|
| 6 |
+
|
| 7 |
+
* **No numpy.** Dense HDC is normally an array of +/-1 floats or of packed
|
| 8 |
+
uint64 words. A Python ``int`` of 10 000 bits is one object whose bitwise
|
| 9 |
+
ops run in C over whole machine words, so ``bind`` is a single ``^`` and
|
| 10 |
+
``popcount`` is ``int.bit_count()``. Memory per vector is d/8 bytes plus
|
| 11 |
+
object overhead, no dependency, and no per-element Python loop.
|
| 12 |
+
* **Binary spatter codes (BSC).** bind = XOR (self-inverse, so unbind ==
|
| 13 |
+
bind), bundle = per-bit majority, permute = rotate within the d-bit mask.
|
| 14 |
+
|
| 15 |
+
Algebraic facts this module relies on, used everywhere downstream:
|
| 16 |
+
|
| 17 |
+
bind(a, b) ^ b == a (self-inverse)
|
| 18 |
+
sim(bind(a, x), bind(b, x)) ~= sim(a, b) (binding preserves similarity)
|
| 19 |
+
sim(permute(a, k), permute(b, k)) == sim(a, b)
|
| 20 |
+
sim(bundle(S), x) ~ sqrt(1/|S|) for x in S -> cleanup recovers members
|
| 21 |
+
E[sim(a, b)] == 0 for independent random a, b (sd ~ 1/sqrt(d))
|
| 22 |
+
|
| 23 |
+
The last line is the reason d is large: it sets the noise floor that cleanup
|
| 24 |
+
memory and the instruction decoder have to beat.
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
from __future__ import annotations
|
| 28 |
+
|
| 29 |
+
import hashlib
|
| 30 |
+
import math
|
| 31 |
+
from typing import Iterable, Sequence
|
| 32 |
+
|
| 33 |
+
__all__ = [
|
| 34 |
+
"DEFAULT_DIM",
|
| 35 |
+
"mask",
|
| 36 |
+
"popcount",
|
| 37 |
+
"hamming",
|
| 38 |
+
"random_hv",
|
| 39 |
+
"hvc",
|
| 40 |
+
"bind",
|
| 41 |
+
"unbind",
|
| 42 |
+
"bundle",
|
| 43 |
+
"permute",
|
| 44 |
+
"permute_inv",
|
| 45 |
+
"sim",
|
| 46 |
+
"sim_matrix",
|
| 47 |
+
"int_to_hv",
|
| 48 |
+
"hv_to_int",
|
| 49 |
+
"hv_bits_of_int",
|
| 50 |
+
"hv_from_bits",
|
| 51 |
+
"hv_to_bits",
|
| 52 |
+
"bit_probe",
|
| 53 |
+
"shift",
|
| 54 |
+
"CODEC_THRESHOLD",
|
| 55 |
+
"MAX_INT_BITS",
|
| 56 |
+
"density",
|
| 57 |
+
"bits",
|
| 58 |
+
"bitplanes",
|
| 59 |
+
"role",
|
| 60 |
+
"ROLES",
|
| 61 |
+
"clear_cache",
|
| 62 |
+
]
|
| 63 |
+
|
| 64 |
+
#: Default dimensionality. 10 000 bits is the classic Kanerva/VSA size: the
|
| 65 |
+
#: noise floor 1/sqrt(d) ~ 0.01 is far below the ~0.4 signal of a bundle
|
| 66 |
+
#: member, so 4-6 field records decode reliably while 1 250 bytes per vector
|
| 67 |
+
#: still fits a program image in a few hundred kilobytes.
|
| 68 |
+
DEFAULT_DIM = 10_000
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
# --------------------------------------------------------------------------
|
| 72 |
+
# primitive bit operations
|
| 73 |
+
# --------------------------------------------------------------------------
|
| 74 |
+
def mask(d: int = DEFAULT_DIM) -> int:
|
| 75 |
+
"""The all-ones mask covering ``d`` bits."""
|
| 76 |
+
return (1 << d) - 1
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def popcount(x: int) -> int:
|
| 80 |
+
"""Number of set bits."""
|
| 81 |
+
return x.bit_count()
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def hamming(a: int, b: int) -> int:
|
| 85 |
+
"""Hamming distance between two hypervectors."""
|
| 86 |
+
return (a ^ b).bit_count()
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def density(v: int, d: int = DEFAULT_DIM) -> float:
|
| 90 |
+
"""Fraction of set bits (should hover near 0.5 for random vectors)."""
|
| 91 |
+
return popcount(v & mask(d)) / d
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def bits(v: int, d: int = DEFAULT_DIM) -> str:
|
| 95 |
+
"""Bit string, most-significant bit first (the usual left-to-right view)."""
|
| 96 |
+
return format(v & mask(d), "0%db" % d)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def bitplanes(v: int, d: int = DEFAULT_DIM, width: int = 64) -> list[str]:
|
| 100 |
+
"""Split a vector into fixed-width bit planes, WSB first."""
|
| 101 |
+
s = bits(v, d)
|
| 102 |
+
return [s[i : i + width] for i in range(0, len(s), width)]
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
# --------------------------------------------------------------------------
|
| 106 |
+
# deterministic symbol -> hypervector
|
| 107 |
+
# --------------------------------------------------------------------------
|
| 108 |
+
_CACHE: dict[tuple[str, int], int] = {}
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def clear_cache() -> None:
|
| 112 |
+
_CACHE.clear()
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def hvc(name: str, d: int = DEFAULT_DIM) -> int:
|
| 116 |
+
"""Deterministic hypervector for a symbol name.
|
| 117 |
+
|
| 118 |
+
The same name always maps to the same vector, in-process and across runs,
|
| 119 |
+
on any machine. That is what makes a stored program image portable: the
|
| 120 |
+
symbolic layer (opcodes, register names, literals, tool names) is
|
| 121 |
+
*regenerated* from names rather than serialised, and only the composed
|
| 122 |
+
instruction vectors live in the storage file.
|
| 123 |
+
|
| 124 |
+
Uses BLAKE2b expanded over a 4-byte block counter, then masked to d bits.
|
| 125 |
+
"""
|
| 126 |
+
key = (name, d)
|
| 127 |
+
cached = _CACHE.get(key)
|
| 128 |
+
if cached is not None:
|
| 129 |
+
return cached
|
| 130 |
+
out = 0
|
| 131 |
+
block = 0
|
| 132 |
+
h = hashlib.blake2b(name.encode("utf-8"), digest_size=64)
|
| 133 |
+
seed = h.digest()
|
| 134 |
+
while out.bit_length() < d:
|
| 135 |
+
chunk = hashlib.blake2b(
|
| 136 |
+
seed + block.to_bytes(4, "little"), digest_size=64
|
| 137 |
+
).digest()
|
| 138 |
+
out = (out << 512) | int.from_bytes(chunk, "big")
|
| 139 |
+
block += 1
|
| 140 |
+
out &= mask(d)
|
| 141 |
+
_CACHE[key] = out
|
| 142 |
+
return out
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def random_hv(d: int = DEFAULT_DIM, rng=None) -> int:
|
| 146 |
+
"""Random balanced hypervector (about d/2 set bits)."""
|
| 147 |
+
if rng is None:
|
| 148 |
+
import random
|
| 149 |
+
|
| 150 |
+
rng = random
|
| 151 |
+
return rng.getrandbits(d) & mask(d)
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
# Roles: named slots used to build records. BSC's canonical record is
|
| 155 |
+
# record = bundle([bind(role_i, value_i) for i in fields])
|
| 156 |
+
# and a field is read back with bind(record, role_i) -> noisy value_i.
|
| 157 |
+
ROLES: dict[str, str] = {
|
| 158 |
+
"op": "role/operation",
|
| 159 |
+
"ra": "role/dest-register",
|
| 160 |
+
"rb": "role/source-register-b",
|
| 161 |
+
"rc": "role/source-register-c",
|
| 162 |
+
"imm": "role/immediate",
|
| 163 |
+
"tgt": "role/target-pc",
|
| 164 |
+
"addr": "role/address",
|
| 165 |
+
"val": "role/value",
|
| 166 |
+
"name": "role/name",
|
| 167 |
+
"cap": "role/capability",
|
| 168 |
+
"doc": "role/documentation",
|
| 169 |
+
"arity": "role/arity",
|
| 170 |
+
"tool": "role/tool",
|
| 171 |
+
"arg": "role/argument",
|
| 172 |
+
"tag": "role/tag",
|
| 173 |
+
"kind": "role/kind",
|
| 174 |
+
}
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def role(name: str, d: int = DEFAULT_DIM) -> int:
|
| 178 |
+
"""Hypervector for a role slot (``"op"``, ``"ra"``, ...)."""
|
| 179 |
+
return hvc(ROLES.get(name, name), d)
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
# --------------------------------------------------------------------------
|
| 183 |
+
# the algebra
|
| 184 |
+
# --------------------------------------------------------------------------
|
| 185 |
+
def bind(a: int, b: int, d: int | None = None) -> int:
|
| 186 |
+
"""Binding = XOR. Involutive: ``bind(bind(a, b), b) == a``."""
|
| 187 |
+
v = a ^ b
|
| 188 |
+
return v & mask(d) if d is not None else v
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def unbind(record: int, key: int, d: int | None = None) -> int:
|
| 192 |
+
"""Unbinding is binding (XOR is self-inverse); named for readability."""
|
| 193 |
+
return bind(record, key, d)
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def permute(v: int, k: int = 1, d: int = DEFAULT_DIM) -> int:
|
| 197 |
+
"""Cyclic bit rotation left by ``k`` (a rotation, so no bits are lost).
|
| 198 |
+
|
| 199 |
+
Permutation is how this machine encodes *order*: position p of a sequence
|
| 200 |
+
is ``permute(item, p)``, which keeps items near-orthogonal while making
|
| 201 |
+
"same item, different position" a different address.
|
| 202 |
+
"""
|
| 203 |
+
m = mask(d)
|
| 204 |
+
v &= m
|
| 205 |
+
k %= d
|
| 206 |
+
if k == 0:
|
| 207 |
+
return v
|
| 208 |
+
return ((v << k) | (v >> (d - k))) & m
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def permute_inv(v: int, k: int = 1, d: int = DEFAULT_DIM) -> int:
|
| 212 |
+
"""Inverse rotation (rotate right by ``k``)."""
|
| 213 |
+
return permute(v, -k, d)
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
def _add_bitplane(plates: list[int], v: int) -> None:
|
| 217 |
+
"""Ripple-carry add of one bit-plane into a binary counter."""
|
| 218 |
+
i = 0
|
| 219 |
+
carry = v
|
| 220 |
+
while carry:
|
| 221 |
+
if i == len(plates):
|
| 222 |
+
plates.append(carry)
|
| 223 |
+
return
|
| 224 |
+
nxt = plates[i] & carry
|
| 225 |
+
plates[i] ^= carry
|
| 226 |
+
carry = nxt
|
| 227 |
+
i += 1
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def _plane_ge(plates: Sequence[int], threshold: int, d: int) -> int:
|
| 231 |
+
"""Bit-sliced ``count >= threshold`` over the counter planes.
|
| 232 |
+
|
| 233 |
+
Walks planes from most to least significant holding one predicate, ``eq``
|
| 234 |
+
("equal to the threshold so far"), and one accumulator, ``gt`` ("already
|
| 235 |
+
greater, therefore certainly >= threshold"). Lanes that fall below the
|
| 236 |
+
threshold at some position are dropped from ``eq`` and die there, which is
|
| 237 |
+
exactly a ripple comparator built from AND/NOT — the same gate logic a
|
| 238 |
+
hardware majority element uses, widened to d lanes at once.
|
| 239 |
+
"""
|
| 240 |
+
m = mask(d)
|
| 241 |
+
nbits = max(len(plates), threshold.bit_length()) or 1
|
| 242 |
+
eq = m
|
| 243 |
+
gt = 0
|
| 244 |
+
for i in range(nbits - 1, -1, -1):
|
| 245 |
+
p = plates[i] if i < len(plates) else 0
|
| 246 |
+
if (threshold >> i) & 1:
|
| 247 |
+
eq &= p # count bit 0 where threshold bit is 1 -> too small
|
| 248 |
+
else:
|
| 249 |
+
gt |= eq & p # count bit 1 where threshold bit is 0 -> greater
|
| 250 |
+
eq &= ~p & m
|
| 251 |
+
if eq == 0:
|
| 252 |
+
break
|
| 253 |
+
return (gt | eq) & m
|
| 254 |
+
|
| 255 |
+
|
| 256 |
+
def bundle(vecs: Iterable[int], d: int = DEFAULT_DIM) -> int:
|
| 257 |
+
"""Majority bundling: per-bit vote, ties resolve to 0.
|
| 258 |
+
|
| 259 |
+
Implemented bit-sliced: every vector is added into a d-lane binary
|
| 260 |
+
counter (one Python int per bit position of the count), then a single
|
| 261 |
+
vectorised comparator thresholds it at ``floor(n/2)+1``. Cost is
|
| 262 |
+
O(n * log n) big-int operations rather than O(n * d) element visits,
|
| 263 |
+
which is what keeps a 10 000-bit bundle at tens of microseconds.
|
| 264 |
+
"""
|
| 265 |
+
items = list(vecs)
|
| 266 |
+
if not items:
|
| 267 |
+
return 0
|
| 268 |
+
if len(items) == 1:
|
| 269 |
+
return items[0] & mask(d)
|
| 270 |
+
plates: list[int] = []
|
| 271 |
+
for v in items:
|
| 272 |
+
_add_bitplane(plates, v & mask(d))
|
| 273 |
+
return _plane_ge(plates, len(items) // 2 + 1, d)
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
def sim(a: int, b: int, d: int = DEFAULT_DIM) -> float:
|
| 277 |
+
"""Cosine-style similarity in [-1, 1]: agreement of bits, rescaled.
|
| 278 |
+
|
| 279 |
+
``sim = (d - 2*hamming) / d``; +1 identical, 0 orthogonal, -1 complement.
|
| 280 |
+
"""
|
| 281 |
+
if d <= 0:
|
| 282 |
+
raise ValueError("d must be positive")
|
| 283 |
+
return (d - 2 * (a ^ b).bit_count()) / d
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def sim_matrix(vecs: Sequence[int], d: int = DEFAULT_DIM) -> list[list[float]]:
|
| 287 |
+
"""All-pairs similarity matrix."""
|
| 288 |
+
return [[sim(a, b, d) for b in vecs] for a in vecs]
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
# --------------------------------------------------------------------------
|
| 292 |
+
# integers stored *as* hypervectors
|
| 293 |
+
# --------------------------------------------------------------------------
|
| 294 |
+
# A scalar is represented by superposition of bit-position atoms:
|
| 295 |
+
# int_to_hv(n) = bundle([bind(NUM, permute(BITBASE, i)) for i where n has bit i])
|
| 296 |
+
# so an integer occupies a vector like every other value and can be recovered
|
| 297 |
+
# by cleanup (one similarity probe per bit position). This is the pixel that
|
| 298 |
+
# makes "all state is hyperdimensional" true rather than decorative.
|
| 299 |
+
#
|
| 300 |
+
# The layout is **two's complement**, 32 bits: bit MAX_INT_BITS-1 is the sign,
|
| 301 |
+
# and a negative value is dense where a sign-magnitude encoding would be
|
| 302 |
+
# sparse. That choice is what lets arithmetic happen as a ripple-carry
|
| 303 |
+
# circuit over the bit atoms (see hdc/adder.py): two's complement makes
|
| 304 |
+
# addition a pure function of the bits, with no sign case analysis anywhere.
|
| 305 |
+
_NUM = hvc("codec/integer")
|
| 306 |
+
_BITBASE = hvc("codec/bit-position")
|
| 307 |
+
|
| 308 |
+
#: Width of the integer payload, in bits, two's complement.
|
| 309 |
+
MAX_INT_BITS = 32
|
| 310 |
+
|
| 311 |
+
#: Similarity above which a bit atom counts as present in a bundle.
|
| 312 |
+
#:
|
| 313 |
+
#: Measured on this implementation (40 random atom subsets per size, both
|
| 314 |
+
#: dimensions): a member of a k-atom bundle recovers at 1/sqrt(k) — 0.35 at
|
| 315 |
+
#: k=4 down to 0.136 at k=32 — while a non-member stays below 0.044 at
|
| 316 |
+
#: d=10 000 and below 0.093 at d=2 000. 0.10 is the value that separates all
|
| 317 |
+
#: of those cases and round-trips 415/415 sampled 32-bit values at both
|
| 318 |
+
#: dimensions; d=2 000 is therefore the supported floor for the codec.
|
| 319 |
+
CODEC_THRESHOLD = 0.10
|
| 320 |
+
|
| 321 |
+
#: Smallest dimension at which the integer codec round-trips reliably.
|
| 322 |
+
#:
|
| 323 |
+
#: Measured over all 256 byte values plus 200 random bytes: 0 errors at
|
| 324 |
+
#: d=1500, 2000 and 3000; **60 errors at d=1000** and 401 at d=500, with a
|
| 325 |
+
#: worst delta of 512. Below this floor the bit atoms crosstalk enough that a
|
| 326 |
+
#: decode silently returns a neighbouring value, which would corrupt every
|
| 327 |
+
#: threshold and shift downstream, so the codec refuses instead.
|
| 328 |
+
MIN_CODEC_DIM = 1500
|
| 329 |
+
|
| 330 |
+
|
| 331 |
+
def _check_dim(d: int) -> None:
|
| 332 |
+
if d < MIN_CODEC_DIM:
|
| 333 |
+
raise ValueError(
|
| 334 |
+
"integer codec needs d >= %d (got %d): bit atoms crosstalk below that "
|
| 335 |
+
"and values decode wrong" % (MIN_CODEC_DIM, d)
|
| 336 |
+
)
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
def _bit_probe(i: int, d: int) -> int:
|
| 340 |
+
return bind(_NUM, permute(_BITBASE, i, d), d)
|
| 341 |
+
|
| 342 |
+
|
| 343 |
+
def bit_probe(i: int, d: int = DEFAULT_DIM) -> int:
|
| 344 |
+
"""The atom for bit position ``i``, for probing a coded value directly.
|
| 345 |
+
|
| 346 |
+
Exposed because other layers (the shader) need single-bit tests — sign,
|
| 347 |
+
ordering — without decoding the whole integer.
|
| 348 |
+
"""
|
| 349 |
+
return _bit_probe(i, d)
|
| 350 |
+
|
| 351 |
+
|
| 352 |
+
def int_to_hv(n: int, d: int = DEFAULT_DIM) -> int:
|
| 353 |
+
"""Encode a signed 32-bit integer as a hypervector (two's complement).
|
| 354 |
+
|
| 355 |
+
Zero encodes to the all-zero vector (the empty bundle), so a cleared
|
| 356 |
+
register is literally a vector with no atoms in it.
|
| 357 |
+
|
| 358 |
+
This codec is *symbolic, not similarity-preserving*: two integers that
|
| 359 |
+
share set bits share bit-position atoms, so ``sim(int_to_hv(5),
|
| 360 |
+
int_to_hv(37))`` is high even though 5 and 37 have nothing to do with each
|
| 361 |
+
other. That is deliberate — decode needs the atoms to be shared and
|
| 362 |
+
probeable, exactly like a bit field. Do not read a distance between two
|
| 363 |
+
encoded integers as a semantic relationship, and do not expect binding two
|
| 364 |
+
of them to decorrelate the way binding two random vectors does.
|
| 365 |
+
"""
|
| 366 |
+
_check_dim(d)
|
| 367 |
+
lo, hi = -(1 << (MAX_INT_BITS - 1)), (1 << (MAX_INT_BITS - 1)) - 1
|
| 368 |
+
if not lo <= n <= hi:
|
| 369 |
+
raise ValueError("integer outside the %d-bit codec range: %r" % (MAX_INT_BITS, n))
|
| 370 |
+
return hv_from_bits(hv_bits_of_int(n), d)
|
| 371 |
+
|
| 372 |
+
|
| 373 |
+
def hv_bits_of_int(n: int) -> list[int]:
|
| 374 |
+
"""The two's complement bit pattern of an integer, least significant first."""
|
| 375 |
+
pattern = n & ((1 << MAX_INT_BITS) - 1)
|
| 376 |
+
return [(pattern >> i) & 1 for i in range(MAX_INT_BITS)]
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
def hv_from_bits(bits: Sequence[int], d: int = DEFAULT_DIM) -> int:
|
| 380 |
+
"""Assemble a hypervector from bit values, purely by bundling atoms.
|
| 381 |
+
|
| 382 |
+
No integer arithmetic is involved: each set position contributes its atom
|
| 383 |
+
to a majority bundle. Two's complement sign extension is the caller's
|
| 384 |
+
concern, exactly as it is in hardware.
|
| 385 |
+
"""
|
| 386 |
+
parts = [_bit_probe(i, d) for i, bit in enumerate(bits) if bit]
|
| 387 |
+
return bundle(parts, d) if parts else 0
|
| 388 |
+
|
| 389 |
+
|
| 390 |
+
def hv_to_bits(v: int, d: int = DEFAULT_DIM, threshold: float = CODEC_THRESHOLD) -> list[int]:
|
| 391 |
+
"""Read the bit pattern back with one similarity probe per position."""
|
| 392 |
+
return [1 if sim(v, _bit_probe(i, d), d) >= threshold else 0 for i in range(MAX_INT_BITS)]
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
def shift(v: int, k: int, d: int = DEFAULT_DIM) -> int:
|
| 396 |
+
"""Multiply or divide a coded integer by 2**k, without decoding it.
|
| 397 |
+
|
| 398 |
+
This works only because the bit atoms are a *permutation orbit*:
|
| 399 |
+
``atom(i) = NUM ^ perm(BITBASE, i)``, so rotating the whole vector by k
|
| 400 |
+
rotates every atom's index by k. Rotation commutes with bundling (it is a
|
| 401 |
+
bit permutation, and majority is per-bit), so
|
| 402 |
+
|
| 403 |
+
rotate(hv, k) = bundle_i( perm(NUM,k) ^ atom(i+k) )
|
| 404 |
+
|
| 405 |
+
and binding away the ``perm(NUM,k) ^ NUM`` offset leaves exactly the
|
| 406 |
+
encoding of the shifted value. Two XORs and two rotations, no similarity
|
| 407 |
+
probe, no threshold, no decode.
|
| 408 |
+
|
| 409 |
+
Positive ``k`` shifts left (multiply), negative shifts right (divide).
|
| 410 |
+
Exact for non-negative values; for negative values the shift is logical,
|
| 411 |
+
so the sign bit is not extended.
|
| 412 |
+
"""
|
| 413 |
+
if k == 0:
|
| 414 |
+
return v
|
| 415 |
+
return bind(permute(v, k, d), permute(_NUM, k, d) ^ _NUM, d)
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
def hv_to_int(v: int, d: int = DEFAULT_DIM, threshold: float = CODEC_THRESHOLD) -> int:
|
| 419 |
+
"""Decode an integer hypervector by per-bit cleanup (two's complement)."""
|
| 420 |
+
_check_dim(d)
|
| 421 |
+
m = 0
|
| 422 |
+
for i, bit in enumerate(hv_to_bits(v, d, threshold)):
|
| 423 |
+
if bit:
|
| 424 |
+
m |= 1 << i
|
| 425 |
+
if m & (1 << (MAX_INT_BITS - 1)):
|
| 426 |
+
m -= 1 << MAX_INT_BITS
|
| 427 |
+
return m
|
| 428 |
+
|
| 429 |
+
|
| 430 |
+
# --------------------------------------------------------------------------
|
| 431 |
+
# diagnostics
|
| 432 |
+
# --------------------------------------------------------------------------
|
| 433 |
+
def capacity_bits(d: int = DEFAULT_DIM) -> float:
|
| 434 |
+
"""Information content of one vector, in bits (log2 of sign patterns)."""
|
| 435 |
+
return float(d)
|
| 436 |
+
|
| 437 |
+
|
| 438 |
+
def orthogonal_capacity(d: int = DEFAULT_DIM, noise: float = 0.35) -> float:
|
| 439 |
+
"""Rough count of distinguishable near-orthogonal items (~ d / (2 log d))."""
|
| 440 |
+
return d / (2.0 * math.log2(d))
|