hdcstorage / hdc /fs.py
AEUPH's picture
HDCSTORAGE: hyperdimensional computer, webOS and agentic MCP layer (stdlib-only Python)
ad474c9 verified
Raw History Blame Contribute Delete
15.3 kB
"""A filesystem whose directory entries live in HDC.
A filesystem has two jobs and they pull in opposite directions:
* **Find things.** "Where did I put the shader notes?" is a similarity question,
and similarity is what this machine is made of. So every node's *record* is a
hypervector — `bind(role(name), name_vector) + bind(role(kind), kind_vector) +
bind(role(parent), parent_vector)` — stored in :class:`~hdc.memory.HDCStorage`
and searchable by ``sim``. :meth:`FileSystem.search` answers "which files are
near this description" with no index beyond the vectors themselves.
* **Give back exactly what was written.** A file's bytes have to round-trip, and
HDC cannot decode a bundle back into text: superposition is lossy by design.
So contents are kept **exact**, addressed by path, exactly as
:class:`~hdc.ring.HistoricLedger` keeps exact tick entries beside its
holographic summaries. Claiming otherwise would be the dishonest version of
this module.
What is genuinely hyperdimensional: path identity (``bind`` of the parent and the
name), directory listing (cleanup over the parent's children), and content search
(each file carries a bundle of its own word vectors, so a query is one compare
per file). What is conventional: the text itself, and the Python dict that maps
a path to its node for O(1) reads.
"""
from __future__ import annotations
import posixpath
import time
from typing import Iterable, Sequence
from .hypervec import DEFAULT_DIM, bind, bundle, hvc, sim
from .memory import HDCStorage
from .text import encode_word, tokenize
KINDS = ("dir", "file")
#: How many content tokens contribute to a file's search vector. A long file
#: bundled whole would swamp its own signal (the capacity law again), so the
#: first tokens plus the file name are what search actually sees.
SEARCH_TOKENS = 160
class Node:
"""One directory entry: exact text plus its hypervector records."""
__slots__ = ("path", "name", "kind", "lines", "vector", "search", "created", "modified")
def __init__(self, path: str, name: str, kind: str, lines: list[str] | None = None):
self.path = path
self.name = name
self.kind = kind
self.lines = lines if lines is not None else []
self.vector = 0
self.search = 0
stamp = time.time()
self.created = stamp
self.modified = stamp
@property
def size(self) -> int:
"""Bytes of content, newline-terminated like any text file."""
return sum(len(line) + 1 for line in self.lines)
def text(self) -> str:
return "\n".join(self.lines)
def as_dict(self) -> dict:
return {"name": self.name, "path": self.path, "kind": self.kind,
"size": self.size, "lines": len(self.lines),
"modified": round(self.modified, 3)}
class FileSystem:
"""Directories by binding, contents by exact address."""
def __init__(self, d: int = DEFAULT_DIM, storage: HDCStorage | None = None):
self.d = d
# not ``storage or ...``: HDCStorage defines __len__, so a brand-new
# (empty) store is falsy and would be silently replaced by a second one.
# That bug split the desktop's memory map from its filesystem.
self.storage = storage if storage is not None else HDCStorage(d=d, label="fs")
self.nodes: dict[str, Node] = {}
self.cwd = "/"
root = Node("/", "", "dir")
self._index(root)
self.writes = 0
self.searches = 0
# ------------------------------------------------------------- plumbing
def _vector(self, name: str, kind: str, parent: str) -> int:
return bundle([
bind(hvc("fs/role/name", self.d), hvc("fs/name/" + name, self.d), self.d),
bind(hvc("fs/role/kind", self.d), hvc("fs/kind/" + kind, self.d), self.d),
bind(hvc("fs/role/parent", self.d), hvc("fs/parent/" + parent, self.d), self.d),
], self.d)
def _search_vector(self, name: str, lines: Sequence[str]) -> int:
words = tokenize(name.replace(".", " ").replace("/", " "))
for line in lines[:SEARCH_TOKENS]:
words.extend(tokenize(line))
if not words:
words = ["empty"]
return bundle([encode_word(word, self.d) for word in words[:SEARCH_TOKENS * 2]], self.d)
def _index(self, node: Node) -> Node:
parent = posixpath.dirname(node.path) or "/"
node.vector = self._vector(node.name, node.kind, parent)
node.search = self._search_vector(node.name, node.lines)
self.nodes[node.path] = node
self.storage.write("fs/" + node.path, node.vector)
return node
def normalise(self, path: str) -> str:
"""Absolute, normalised path; ``.`` and ``..`` resolved against ``cwd``."""
if not path:
return self.cwd
if not path.startswith("/"):
path = posixpath.join(self.cwd, path)
out = posixpath.normpath(path)
return "/" if out in (".", "") else out
def parent(self, path: str) -> str:
return posixpath.dirname(self.normalise(path)) or "/"
# -------------------------------------------------------------- queries
def exists(self, path: str) -> bool:
return self.normalise(path) in self.nodes
def isdir(self, path: str) -> bool:
node = self.nodes.get(self.normalise(path))
return bool(node and node.kind == "dir")
def get(self, path: str) -> Node:
node = self.nodes.get(self.normalise(path))
if node is None:
raise FileNotFoundError(self.normalise(path))
return node
def children(self, path: str) -> list[Node]:
directory = self.normalise(path)
if not self.isdir(directory):
raise NotADirectoryError(directory)
prefix = "/" if directory == "/" else directory + "/"
out = [node for node in self.nodes.values()
if node.path != directory and posixpath.dirname(node.path) ==
(directory if directory != "/" else "/")]
return sorted(out, key=lambda node: (node.kind != "dir", node.name))
def ls(self, path: str = ".") -> list[dict]:
return [node.as_dict() for node in self.children(path)]
def walk(self, path: str = "/") -> list[Node]:
out = [self.get(path)]
for child in self.children(path):
out.extend(self.walk(child.path) if child.kind == "dir" else [child])
return out
def tree(self, path: str = "/") -> str:
lines: list[str] = [self.normalise(path)]
self._tree(self.normalise(path), "", lines)
return "\n".join(lines)
def _tree(self, directory: str, indent: str, lines: list[str]) -> None:
entries = self.children(directory)
for index, node in enumerate(entries):
last = index == len(entries) - 1
lines.append("%s%s %s%s" % (indent, "`-" if last else "|-", node.name,
"/" if node.kind == "dir" else ""))
if node.kind == "dir":
self._tree(node.path, indent + (" " if last else "| "), lines)
# ------------------------------------------------------------ mutation
def mkdir(self, path: str, parents: bool = True) -> Node:
target = self.normalise(path)
if target == "/":
return self.nodes["/"]
if target in self.nodes:
if self.nodes[target].kind == "dir":
return self.nodes[target]
raise FileExistsError(target)
parent = self.parent(target)
if parent not in self.nodes:
if not parents:
raise FileNotFoundError(parent)
self.mkdir(parent, parents=True)
if not self.isdir(parent):
raise NotADirectoryError(parent)
return self._index(Node(target, posixpath.basename(target), "dir"))
def write(self, path: str, text: str) -> Node:
"""Create or replace a file; the text is stored exactly.
Intermediate directories are created, like ``mkdir -p``. An editor
saving to ``/home/user/src/new/thing.hdcasm`` should not have to make two
directories first, and the failure it would otherwise raise
(``FileNotFoundError``) reads as a bug in the editor rather than as the
caller's mistake. Writing *through* a file still raises, because the
parent then exists and is not a directory.
"""
target = self.normalise(path)
parent = self.parent(target)
if not self.exists(parent):
self.mkdir(parent, parents=True)
if not self.isdir(parent):
raise NotADirectoryError(parent)
existing = self.nodes.get(target)
if existing is not None:
if existing.kind == "dir":
raise IsADirectoryError(target)
existing.lines = text.splitlines()
existing.modified = time.time()
self._index(existing)
else:
existing = self._index(
Node(target, posixpath.basename(target), "file", text.splitlines()))
self.writes += 1
return existing
def append(self, path: str, text: str) -> Node:
node = self.get(path)
if node.kind != "file":
raise IsADirectoryError(node.path)
node.lines.extend(text.splitlines())
node.modified = time.time()
return self._index(node)
def read(self, path: str) -> str:
node = self.get(path)
if node.kind != "file":
raise IsADirectoryError(node.path)
return node.text()
def remove(self, path: str, recursive: bool = False) -> int:
"""Remove a file, or a directory when empty (or ``recursive``)."""
node = self.get(path)
if node.path == "/":
raise PermissionError("cannot remove the root directory")
victims = [node]
if node.kind == "dir":
victims = self.walk(node.path)
if len(victims) > 1 and not recursive:
raise OSError("directory not empty: " + node.path)
for victim in victims:
self.nodes.pop(victim.path, None)
self.storage.remove("fs/" + victim.path)
return len(victims)
def rename(self, old: str, new: str) -> Node:
node = self.get(old)
target = self.normalise(new)
if target in self.nodes:
raise FileExistsError(target)
if not self.isdir(self.parent(target)):
raise FileNotFoundError(self.parent(target))
moved = [node] + (self.walk(node.path)[1:] if node.kind == "dir" else [])
for victim in moved:
suffix = victim.path[len(node.path):]
self.nodes.pop(victim.path, None)
self.storage.remove("fs/" + victim.path)
victim.path = target + suffix
victim.name = posixpath.basename(victim.path)
self._index(victim)
return node
def copy(self, old: str, new: str) -> Node:
source = self.get(old)
if source.kind == "file":
return self.write(new, source.text())
self.mkdir(new)
for child in self.children(source.path):
self.copy(child.path, posixpath.join(self.normalise(new), child.name))
return self.get(new)
def chdir(self, path: str) -> str:
target = self.normalise(path)
if not self.isdir(target):
raise NotADirectoryError(target)
self.cwd = target
return self.cwd
# -------------------------------------------------------------- search
def search(self, query: str, limit: int = 8) -> list[tuple[str, float]]:
"""Rank every node by similarity to a description — one compare each.
This is the payoff of keeping records as vectors: no index to build, no
tokeniser to consult. A query is a bundle of its word vectors and each
file is compared against it.
"""
words = tokenize(query) or [query]
probe = bundle([encode_word(word, self.d) for word in words], self.d)
scored = [(node.path, sim(probe, node.search, self.d))
for node in self.nodes.values() if node.path != "/"]
scored.sort(key=lambda item: -item[1])
self.searches += 1
return scored[:limit]
def similar(self, path: str, limit: int = 8) -> list[tuple[str, float]]:
node = self.get(path)
scored = [(other.path, sim(node.search, other.search, self.d))
for other in self.nodes.values()
if other.path not in ("/", node.path)]
scored.sort(key=lambda item: -item[1])
return scored[:limit]
# -------------------------------------------------------------- report
def stat(self) -> dict:
files = [node for node in self.nodes.values() if node.kind == "file"]
return {"files": len(files), "dirs": len(self.nodes) - len(files),
"bytes": sum(node.size for node in files),
"cells": len(self.storage), "writes": self.writes,
"searches": self.searches, "cwd": self.cwd,
"dim": self.d}
def save(self, path: str) -> int:
"""Persist the node vectors (the exact text stays with the caller)."""
return self.storage.save(path)
def seed(self) -> None:
"""A small tree, so a fresh desktop has something in it."""
if "/home" in self.nodes:
return
self.mkdir("/home")
self.mkdir("/home/user")
self.mkdir("/home/user/src")
self.mkdir("/home/user/img")
self.write("/home/user/README.txt",
"HDCSTORAGE webOS\n"
"Files are exact; their addresses are hypervectors.\n"
"Try: find shader notes")
self.write("/home/user/src/hello.hdcasm",
"; add two numbers and halt\n"
"LDI R1, 7\n"
"LDI R2, 35\n"
"ADD R3, R1, R2\n"
"HLT")
self.write("/home/user/src/loop.hdcasm",
"; count down from five\n"
"LDI R1, 5\n"
"loop: JZ R1, done\n"
"SUB R1, R1, 1\n"
"JMP loop\n"
"done: HLT")
self.write("/home/user/notes-shader.txt",
"shader notes: dimming is exact, edges are |a-b|.\n"
"depth test compares coded values directly.")
self.write("/home/user/img/gradient.txt", "0 32 64 96 128 160 192 224 255")
def seeded(d: int = DEFAULT_DIM, storage: HDCStorage | None = None) -> FileSystem:
"""A filesystem with the starter tree already in it.
Pass the machine's storage when the desktop boots, so the filesystem's cells
are the kernel's cells and the memory map counts them — two stores would make
every "free cells" figure a lie.
"""
filesystem = FileSystem(d=d, storage=storage)
filesystem.seed()
return filesystem
if __name__ == "__main__": # pragma: no cover - manual inspection
filesystem = seeded(2000)
print(filesystem.tree("/home"))
print()
for path, score in filesystem.search("shader notes about edges"):
print(" %+.3f %s" % (score, path))