Download hdc/fs.py from AEUPH/hdcstorage: direct link, hf CLI and curl.
- Browser
- Download file 15.3 kB
-
https://huggingface.co/AEUPH/hdcstorage/resolve/main/hdc/fs.py
- Command line
-
hf download hf://AEUPH/hdcstorage/hdc/fs.py
-
curl -L -o fs.py https://huggingface.co/AEUPH/hdcstorage/resolve/main/hdc/fs.py
15.3 kB
| """A filesystem whose directory entries live in HDC. | |
| A filesystem has two jobs and they pull in opposite directions: | |
| * **Find things.** "Where did I put the shader notes?" is a similarity question, | |
| and similarity is what this machine is made of. So every node's *record* is a | |
| hypervector — `bind(role(name), name_vector) + bind(role(kind), kind_vector) + | |
| bind(role(parent), parent_vector)` — stored in :class:`~hdc.memory.HDCStorage` | |
| and searchable by ``sim``. :meth:`FileSystem.search` answers "which files are | |
| near this description" with no index beyond the vectors themselves. | |
| * **Give back exactly what was written.** A file's bytes have to round-trip, and | |
| HDC cannot decode a bundle back into text: superposition is lossy by design. | |
| So contents are kept **exact**, addressed by path, exactly as | |
| :class:`~hdc.ring.HistoricLedger` keeps exact tick entries beside its | |
| holographic summaries. Claiming otherwise would be the dishonest version of | |
| this module. | |
| What is genuinely hyperdimensional: path identity (``bind`` of the parent and the | |
| name), directory listing (cleanup over the parent's children), and content search | |
| (each file carries a bundle of its own word vectors, so a query is one compare | |
| per file). What is conventional: the text itself, and the Python dict that maps | |
| a path to its node for O(1) reads. | |
| """ | |
| from __future__ import annotations | |
| import posixpath | |
| import time | |
| from typing import Iterable, Sequence | |
| from .hypervec import DEFAULT_DIM, bind, bundle, hvc, sim | |
| from .memory import HDCStorage | |
| from .text import encode_word, tokenize | |
| KINDS = ("dir", "file") | |
| #: How many content tokens contribute to a file's search vector. A long file | |
| #: bundled whole would swamp its own signal (the capacity law again), so the | |
| #: first tokens plus the file name are what search actually sees. | |
| SEARCH_TOKENS = 160 | |
| class Node: | |
| """One directory entry: exact text plus its hypervector records.""" | |
| __slots__ = ("path", "name", "kind", "lines", "vector", "search", "created", "modified") | |
| def __init__(self, path: str, name: str, kind: str, lines: list[str] | None = None): | |
| self.path = path | |
| self.name = name | |
| self.kind = kind | |
| self.lines = lines if lines is not None else [] | |
| self.vector = 0 | |
| self.search = 0 | |
| stamp = time.time() | |
| self.created = stamp | |
| self.modified = stamp | |
| def size(self) -> int: | |
| """Bytes of content, newline-terminated like any text file.""" | |
| return sum(len(line) + 1 for line in self.lines) | |
| def text(self) -> str: | |
| return "\n".join(self.lines) | |
| def as_dict(self) -> dict: | |
| return {"name": self.name, "path": self.path, "kind": self.kind, | |
| "size": self.size, "lines": len(self.lines), | |
| "modified": round(self.modified, 3)} | |
| class FileSystem: | |
| """Directories by binding, contents by exact address.""" | |
| def __init__(self, d: int = DEFAULT_DIM, storage: HDCStorage | None = None): | |
| self.d = d | |
| # not ``storage or ...``: HDCStorage defines __len__, so a brand-new | |
| # (empty) store is falsy and would be silently replaced by a second one. | |
| # That bug split the desktop's memory map from its filesystem. | |
| self.storage = storage if storage is not None else HDCStorage(d=d, label="fs") | |
| self.nodes: dict[str, Node] = {} | |
| self.cwd = "/" | |
| root = Node("/", "", "dir") | |
| self._index(root) | |
| self.writes = 0 | |
| self.searches = 0 | |
| # ------------------------------------------------------------- plumbing | |
| def _vector(self, name: str, kind: str, parent: str) -> int: | |
| return bundle([ | |
| bind(hvc("fs/role/name", self.d), hvc("fs/name/" + name, self.d), self.d), | |
| bind(hvc("fs/role/kind", self.d), hvc("fs/kind/" + kind, self.d), self.d), | |
| bind(hvc("fs/role/parent", self.d), hvc("fs/parent/" + parent, self.d), self.d), | |
| ], self.d) | |
| def _search_vector(self, name: str, lines: Sequence[str]) -> int: | |
| words = tokenize(name.replace(".", " ").replace("/", " ")) | |
| for line in lines[:SEARCH_TOKENS]: | |
| words.extend(tokenize(line)) | |
| if not words: | |
| words = ["empty"] | |
| return bundle([encode_word(word, self.d) for word in words[:SEARCH_TOKENS * 2]], self.d) | |
| def _index(self, node: Node) -> Node: | |
| parent = posixpath.dirname(node.path) or "/" | |
| node.vector = self._vector(node.name, node.kind, parent) | |
| node.search = self._search_vector(node.name, node.lines) | |
| self.nodes[node.path] = node | |
| self.storage.write("fs/" + node.path, node.vector) | |
| return node | |
| def normalise(self, path: str) -> str: | |
| """Absolute, normalised path; ``.`` and ``..`` resolved against ``cwd``.""" | |
| if not path: | |
| return self.cwd | |
| if not path.startswith("/"): | |
| path = posixpath.join(self.cwd, path) | |
| out = posixpath.normpath(path) | |
| return "/" if out in (".", "") else out | |
| def parent(self, path: str) -> str: | |
| return posixpath.dirname(self.normalise(path)) or "/" | |
| # -------------------------------------------------------------- queries | |
| def exists(self, path: str) -> bool: | |
| return self.normalise(path) in self.nodes | |
| def isdir(self, path: str) -> bool: | |
| node = self.nodes.get(self.normalise(path)) | |
| return bool(node and node.kind == "dir") | |
| def get(self, path: str) -> Node: | |
| node = self.nodes.get(self.normalise(path)) | |
| if node is None: | |
| raise FileNotFoundError(self.normalise(path)) | |
| return node | |
| def children(self, path: str) -> list[Node]: | |
| directory = self.normalise(path) | |
| if not self.isdir(directory): | |
| raise NotADirectoryError(directory) | |
| prefix = "/" if directory == "/" else directory + "/" | |
| out = [node for node in self.nodes.values() | |
| if node.path != directory and posixpath.dirname(node.path) == | |
| (directory if directory != "/" else "/")] | |
| return sorted(out, key=lambda node: (node.kind != "dir", node.name)) | |
| def ls(self, path: str = ".") -> list[dict]: | |
| return [node.as_dict() for node in self.children(path)] | |
| def walk(self, path: str = "/") -> list[Node]: | |
| out = [self.get(path)] | |
| for child in self.children(path): | |
| out.extend(self.walk(child.path) if child.kind == "dir" else [child]) | |
| return out | |
| def tree(self, path: str = "/") -> str: | |
| lines: list[str] = [self.normalise(path)] | |
| self._tree(self.normalise(path), "", lines) | |
| return "\n".join(lines) | |
| def _tree(self, directory: str, indent: str, lines: list[str]) -> None: | |
| entries = self.children(directory) | |
| for index, node in enumerate(entries): | |
| last = index == len(entries) - 1 | |
| lines.append("%s%s %s%s" % (indent, "`-" if last else "|-", node.name, | |
| "/" if node.kind == "dir" else "")) | |
| if node.kind == "dir": | |
| self._tree(node.path, indent + (" " if last else "| "), lines) | |
| # ------------------------------------------------------------ mutation | |
| def mkdir(self, path: str, parents: bool = True) -> Node: | |
| target = self.normalise(path) | |
| if target == "/": | |
| return self.nodes["/"] | |
| if target in self.nodes: | |
| if self.nodes[target].kind == "dir": | |
| return self.nodes[target] | |
| raise FileExistsError(target) | |
| parent = self.parent(target) | |
| if parent not in self.nodes: | |
| if not parents: | |
| raise FileNotFoundError(parent) | |
| self.mkdir(parent, parents=True) | |
| if not self.isdir(parent): | |
| raise NotADirectoryError(parent) | |
| return self._index(Node(target, posixpath.basename(target), "dir")) | |
| def write(self, path: str, text: str) -> Node: | |
| """Create or replace a file; the text is stored exactly. | |
| Intermediate directories are created, like ``mkdir -p``. An editor | |
| saving to ``/home/user/src/new/thing.hdcasm`` should not have to make two | |
| directories first, and the failure it would otherwise raise | |
| (``FileNotFoundError``) reads as a bug in the editor rather than as the | |
| caller's mistake. Writing *through* a file still raises, because the | |
| parent then exists and is not a directory. | |
| """ | |
| target = self.normalise(path) | |
| parent = self.parent(target) | |
| if not self.exists(parent): | |
| self.mkdir(parent, parents=True) | |
| if not self.isdir(parent): | |
| raise NotADirectoryError(parent) | |
| existing = self.nodes.get(target) | |
| if existing is not None: | |
| if existing.kind == "dir": | |
| raise IsADirectoryError(target) | |
| existing.lines = text.splitlines() | |
| existing.modified = time.time() | |
| self._index(existing) | |
| else: | |
| existing = self._index( | |
| Node(target, posixpath.basename(target), "file", text.splitlines())) | |
| self.writes += 1 | |
| return existing | |
| def append(self, path: str, text: str) -> Node: | |
| node = self.get(path) | |
| if node.kind != "file": | |
| raise IsADirectoryError(node.path) | |
| node.lines.extend(text.splitlines()) | |
| node.modified = time.time() | |
| return self._index(node) | |
| def read(self, path: str) -> str: | |
| node = self.get(path) | |
| if node.kind != "file": | |
| raise IsADirectoryError(node.path) | |
| return node.text() | |
| def remove(self, path: str, recursive: bool = False) -> int: | |
| """Remove a file, or a directory when empty (or ``recursive``).""" | |
| node = self.get(path) | |
| if node.path == "/": | |
| raise PermissionError("cannot remove the root directory") | |
| victims = [node] | |
| if node.kind == "dir": | |
| victims = self.walk(node.path) | |
| if len(victims) > 1 and not recursive: | |
| raise OSError("directory not empty: " + node.path) | |
| for victim in victims: | |
| self.nodes.pop(victim.path, None) | |
| self.storage.remove("fs/" + victim.path) | |
| return len(victims) | |
| def rename(self, old: str, new: str) -> Node: | |
| node = self.get(old) | |
| target = self.normalise(new) | |
| if target in self.nodes: | |
| raise FileExistsError(target) | |
| if not self.isdir(self.parent(target)): | |
| raise FileNotFoundError(self.parent(target)) | |
| moved = [node] + (self.walk(node.path)[1:] if node.kind == "dir" else []) | |
| for victim in moved: | |
| suffix = victim.path[len(node.path):] | |
| self.nodes.pop(victim.path, None) | |
| self.storage.remove("fs/" + victim.path) | |
| victim.path = target + suffix | |
| victim.name = posixpath.basename(victim.path) | |
| self._index(victim) | |
| return node | |
| def copy(self, old: str, new: str) -> Node: | |
| source = self.get(old) | |
| if source.kind == "file": | |
| return self.write(new, source.text()) | |
| self.mkdir(new) | |
| for child in self.children(source.path): | |
| self.copy(child.path, posixpath.join(self.normalise(new), child.name)) | |
| return self.get(new) | |
| def chdir(self, path: str) -> str: | |
| target = self.normalise(path) | |
| if not self.isdir(target): | |
| raise NotADirectoryError(target) | |
| self.cwd = target | |
| return self.cwd | |
| # -------------------------------------------------------------- search | |
| def search(self, query: str, limit: int = 8) -> list[tuple[str, float]]: | |
| """Rank every node by similarity to a description — one compare each. | |
| This is the payoff of keeping records as vectors: no index to build, no | |
| tokeniser to consult. A query is a bundle of its word vectors and each | |
| file is compared against it. | |
| """ | |
| words = tokenize(query) or [query] | |
| probe = bundle([encode_word(word, self.d) for word in words], self.d) | |
| scored = [(node.path, sim(probe, node.search, self.d)) | |
| for node in self.nodes.values() if node.path != "/"] | |
| scored.sort(key=lambda item: -item[1]) | |
| self.searches += 1 | |
| return scored[:limit] | |
| def similar(self, path: str, limit: int = 8) -> list[tuple[str, float]]: | |
| node = self.get(path) | |
| scored = [(other.path, sim(node.search, other.search, self.d)) | |
| for other in self.nodes.values() | |
| if other.path not in ("/", node.path)] | |
| scored.sort(key=lambda item: -item[1]) | |
| return scored[:limit] | |
| # -------------------------------------------------------------- report | |
| def stat(self) -> dict: | |
| files = [node for node in self.nodes.values() if node.kind == "file"] | |
| return {"files": len(files), "dirs": len(self.nodes) - len(files), | |
| "bytes": sum(node.size for node in files), | |
| "cells": len(self.storage), "writes": self.writes, | |
| "searches": self.searches, "cwd": self.cwd, | |
| "dim": self.d} | |
| def save(self, path: str) -> int: | |
| """Persist the node vectors (the exact text stays with the caller).""" | |
| return self.storage.save(path) | |
| def seed(self) -> None: | |
| """A small tree, so a fresh desktop has something in it.""" | |
| if "/home" in self.nodes: | |
| return | |
| self.mkdir("/home") | |
| self.mkdir("/home/user") | |
| self.mkdir("/home/user/src") | |
| self.mkdir("/home/user/img") | |
| self.write("/home/user/README.txt", | |
| "HDCSTORAGE webOS\n" | |
| "Files are exact; their addresses are hypervectors.\n" | |
| "Try: find shader notes") | |
| self.write("/home/user/src/hello.hdcasm", | |
| "; add two numbers and halt\n" | |
| "LDI R1, 7\n" | |
| "LDI R2, 35\n" | |
| "ADD R3, R1, R2\n" | |
| "HLT") | |
| self.write("/home/user/src/loop.hdcasm", | |
| "; count down from five\n" | |
| "LDI R1, 5\n" | |
| "loop: JZ R1, done\n" | |
| "SUB R1, R1, 1\n" | |
| "JMP loop\n" | |
| "done: HLT") | |
| self.write("/home/user/notes-shader.txt", | |
| "shader notes: dimming is exact, edges are |a-b|.\n" | |
| "depth test compares coded values directly.") | |
| self.write("/home/user/img/gradient.txt", "0 32 64 96 128 160 192 224 255") | |
| def seeded(d: int = DEFAULT_DIM, storage: HDCStorage | None = None) -> FileSystem: | |
| """A filesystem with the starter tree already in it. | |
| Pass the machine's storage when the desktop boots, so the filesystem's cells | |
| are the kernel's cells and the memory map counts them — two stores would make | |
| every "free cells" figure a lie. | |
| """ | |
| filesystem = FileSystem(d=d, storage=storage) | |
| filesystem.seed() | |
| return filesystem | |
| if __name__ == "__main__": # pragma: no cover - manual inspection | |
| filesystem = seeded(2000) | |
| print(filesystem.tree("/home")) | |
| print() | |
| for path, score in filesystem.search("shader notes about edges"): | |
| print(" %+.3f %s" % (score, path)) | |