Upload folder using huggingface_hub
Browse files- .gitattributes +1 -0
- .gitignore +22 -0
- .pytest_cache/.gitignore +2 -0
- .pytest_cache/CACHEDIR.TAG +4 -0
- .pytest_cache/README.md +8 -0
- .pytest_cache/v/cache/nodeids +3 -0
- CONTRIBUTING.md +13 -0
- LICENSE +9 -0
- README.md +81 -0
- benchmarks/train_image.py +57 -0
- configs/base.json +1 -0
- configs/large.json +1 -0
- configs/small.json +1 -0
- configs/tiny.json +1 -0
- docs/SCALING_PROTOCOL.md +16 -0
- docs/manuscript.pdf +3 -0
- hilbert_mi/__init__.py +17 -0
- hilbert_mi/datasets.py +33 -0
- hilbert_mi/learnable.py +181 -0
- hilbert_mi/metrics.py +26 -0
- hilbert_mi/operators.py +112 -0
- hilbert_mi/reasoning.py +74 -0
- hilbert_mi/scalable.py +63 -0
- hilbert_mi/transforms.py +153 -0
- hilbert_mi/utils.py +35 -0
- hilbert_operator_intelligence_benchmark.egg-info/PKG-INFO +98 -0
- hilbert_operator_intelligence_benchmark.egg-info/SOURCES.txt +18 -0
- hilbert_operator_intelligence_benchmark.egg-info/dependency_links.txt +1 -0
- hilbert_operator_intelligence_benchmark.egg-info/requires.txt +9 -0
- hilbert_operator_intelligence_benchmark.egg-info/top_level.txt +1 -0
- pyproject.toml +14 -0
- requirements.txt +6 -0
- results/README.md +2 -0
- results/smoke.json +20 -0
- scripts/run_scaling_sweep.sh +8 -0
- scripts/summarize_scaling.py +9 -0
- tests/test_scalable.py +8 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
docs/manuscript.pdf filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.py[cod]
|
| 3 |
+
*.so
|
| 4 |
+
|
| 5 |
+
.venv/
|
| 6 |
+
venv/
|
| 7 |
+
env/
|
| 8 |
+
.env
|
| 9 |
+
|
| 10 |
+
.pytest_cache/
|
| 11 |
+
.mypy_cache/
|
| 12 |
+
.ruff_cache/
|
| 13 |
+
|
| 14 |
+
wandb/
|
| 15 |
+
runs/
|
| 16 |
+
checkpoints/
|
| 17 |
+
|
| 18 |
+
*.pt
|
| 19 |
+
*.pth
|
| 20 |
+
*.ckpt
|
| 21 |
+
|
| 22 |
+
.DS_Store
|
.pytest_cache/.gitignore
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Created by pytest automatically.
|
| 2 |
+
*
|
.pytest_cache/CACHEDIR.TAG
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Signature: 8a477f597d28d172789f06886806bc55
|
| 2 |
+
# This file is a cache directory tag created by pytest.
|
| 3 |
+
# For information about cache directory tags, see:
|
| 4 |
+
# https://bford.info/cachedir/spec.html
|
.pytest_cache/README.md
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# pytest cache directory #
|
| 2 |
+
|
| 3 |
+
This directory contains data from the pytest's cache plugin,
|
| 4 |
+
which provides the `--lf` and `--ff` options, as well as the `cache` fixture.
|
| 5 |
+
|
| 6 |
+
**Do not** commit this to version control.
|
| 7 |
+
|
| 8 |
+
See [the docs](https://docs.pytest.org/en/stable/how-to/cache.html) for more information.
|
.pytest_cache/v/cache/nodeids
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
"tests/test_scalable.py::test_forward_and_backward"
|
| 3 |
+
]
|
CONTRIBUTING.md
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Contributing GPU scaling results
|
| 2 |
+
|
| 3 |
+
Thank you for contributing compute. The most valuable contribution is a reproducible run, including a negative result.
|
| 4 |
+
|
| 5 |
+
1. Fork the Hugging Face repository and record the commit hash used for training.
|
| 6 |
+
2. Install with `pip install -e '.[benchmark]'` and run `pytest -q`.
|
| 7 |
+
3. Do not change a named tier config for an official comparison. New configs belong under `configs/community/` with a descriptive name.
|
| 8 |
+
4. Run at least seeds 7, 17, and 27 when compute permits. Use AMP consistently and report whether it was enabled.
|
| 9 |
+
5. Submit generated JSON result files and a short note with GPU count/model, VRAM, CUDA/driver, any OOMs, and code changes.
|
| 10 |
+
6. For new baselines, match data split, augmentation policy, epochs, and evaluation metrics. Report parameter count, throughput, peak memory, wall time, accuracy and NLL.
|
| 11 |
+
7. Do not replace or cherry-pick only favorable seeds. Report all attempted runs.
|
| 12 |
+
|
| 13 |
+
Suggested PR title: `results: CIFAR-10 base on 4x H100, seeds 7/17/27`.
|
LICENSE
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Apache License 2.0
|
| 2 |
+
|
| 3 |
+
Copyright 2026 Andrew Kiruluta and contributors
|
| 4 |
+
|
| 5 |
+
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
|
| 6 |
+
|
| 7 |
+
https://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
|
| 9 |
+
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
|
README.md
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
tags:
|
| 3 |
+
- benchmark
|
| 4 |
+
- pytorch
|
| 5 |
+
- spectral-learning
|
| 6 |
+
- operator-learning
|
| 7 |
+
- hilbert-space
|
| 8 |
+
- fourier-neural-operator
|
| 9 |
+
license: apache-2.0
|
| 10 |
+
---
|
| 11 |
+
# Hilbert Operator Intelligence — Scaling Benchmark
|
| 12 |
+
|
| 13 |
+
This repository turns **Operator-Based Machine Intelligence: A Hilbert Space Framework for Spectral Learning and Symbolic Reasoning** into a contributor-oriented scaling benchmark. The research program represents inputs in Hilbert spaces, learns regularized task/reasoning operators, and uses spectral bases and modulation rather than treating every transformation as an opaque dense layer.
|
| 14 |
+
|
| 15 |
+
## Why this benchmark exists
|
| 16 |
+
|
| 17 |
+
The original reproducibility code is intentionally CPU-sized (`sklearn` Digits). The next scientific question is **scaling**: do structured spectral operators retain accuracy/efficiency advantages as width, depth, data, and GPU compute increase? This repo gives contributors one command, fixed configs, machine-readable metrics, and a common result schema so runs on larger GPUs can be compared.
|
| 18 |
+
|
| 19 |
+
## Benchmark task
|
| 20 |
+
|
| 21 |
+
The primary public benchmark is CIFAR-10, with CIFAR-100 available as a harder extension. `synthetic` is included only as an offline smoke test. The model lifts images into a channel-valued function space, applies repeated truncated 2-D Fourier operators, adds a local residual nonlinearity, and classifies the pooled representation. Low-rank factorization is supported for the spectral operator tensor.
|
| 22 |
+
|
| 23 |
+
| Tier | Width | Depth | Modes | Spectral rank | Intended use |
|
| 24 |
+
|---|---:|---:|---:|---:|---|
|
| 25 |
+
| tiny | 32 | 2 | 8 | full | laptop / smoke |
|
| 26 |
+
| small | 64 | 4 | 12 | 16 | single consumer GPU |
|
| 27 |
+
| base | 128 | 6 | 16 | 32 | high-memory GPU |
|
| 28 |
+
| large | 256 | 8 | 16 | 64 | contributor scaling run |
|
| 29 |
+
|
| 30 |
+
The table defines benchmark configurations, not claimed performance. Contributors should submit the generated JSON/CSV results rather than hand-editing numbers.
|
| 31 |
+
|
| 32 |
+
## Quick start
|
| 33 |
+
|
| 34 |
+
```bash
|
| 35 |
+
python -m venv .venv
|
| 36 |
+
source .venv/bin/activate
|
| 37 |
+
pip install -e '.[benchmark]'
|
| 38 |
+
pytest -q
|
| 39 |
+
|
| 40 |
+
# Offline smoke test
|
| 41 |
+
python benchmarks/train_image.py --config configs/tiny.json --dataset synthetic \
|
| 42 |
+
--epochs 1 --train-samples 256 --test-samples 128 --image-size 16 \
|
| 43 |
+
--output results/smoke.json
|
| 44 |
+
|
| 45 |
+
# Standard GPU run
|
| 46 |
+
python benchmarks/train_image.py --config configs/base.json --dataset cifar10 \
|
| 47 |
+
--seed 7 --amp --output results/cifar10_base_s7.json
|
| 48 |
+
```
|
| 49 |
+
|
| 50 |
+
For a three-seed sweep:
|
| 51 |
+
|
| 52 |
+
```bash
|
| 53 |
+
DATASET=cifar10 ./scripts/run_scaling_sweep.sh
|
| 54 |
+
```
|
| 55 |
+
|
| 56 |
+
## What to contribute
|
| 57 |
+
|
| 58 |
+
We are specifically seeking contributors with substantial GPU compute to run **base/large and larger variants**, repeat runs across seeds, and test scaling laws over width/depth/modes/rank. Useful contributions include multi-GPU training, larger image datasets, optimized FFT kernels, low-rank/structured operator parameterizations, learned bases/wavelet packets, and fair Transformer/CNN/FNO baselines under matched parameter/FLOP budgets.
|
| 59 |
+
|
| 60 |
+
Please preserve the benchmark protocol and submit: config, seed, commit hash, hardware/GPU model, PyTorch/CUDA versions, wall-clock training time, peak GPU memory, throughput, parameter count, test accuracy and NLL. See `CONTRIBUTING.md` and `docs/SCALING_PROTOCOL.md`.
|
| 61 |
+
|
| 62 |
+
## Scientific hypotheses to test
|
| 63 |
+
|
| 64 |
+
1. **Spectral scaling:** increasing retained modes improves accuracy until task-relevant bandwidth saturates.
|
| 65 |
+
2. **Structured efficiency:** low-rank spectral operators improve accuracy per parameter and per GPU-byte at larger width.
|
| 66 |
+
3. **Depth/composition:** deeper operator composition improves representation without requiring quadratic token attention.
|
| 67 |
+
4. **Hardware fit:** FFT-heavy implementations become increasingly attractive on GPUs as spatial resolution grows.
|
| 68 |
+
|
| 69 |
+
These are hypotheses, not established results. Negative results are welcome and should be reported.
|
| 70 |
+
|
| 71 |
+
## Relationship to the manuscript
|
| 72 |
+
|
| 73 |
+
The manuscript formulates supervised learning as regularized operator estimation between Hilbert spaces, develops Fourier/wavelet spectral representations and learnable spectral modulation, and frames reasoning as operator composition. It also explicitly identifies high-dimensional scalability and GPU-friendly FFT/DWT implementations as open work. This benchmark focuses on that unresolved scaling question.
|
| 74 |
+
|
| 75 |
+
## Hugging Face collaboration call
|
| 76 |
+
|
| 77 |
+
**Call for GPU Scaling Contributors — Hilbert Operator Intelligence.** We invite researchers and engineers with high-end GPUs or clusters to run the standardized scaling matrix, add matched baselines, and explore larger operator models. The goal is a transparent community scaling study: every result should be reproducible, hardware-labeled, and accompanied by the exact config. Open an Issue titled `Scaling run: <GPU> / <tier> / <dataset>` or submit a PR containing result JSON files plus any reproducibility notes.
|
| 78 |
+
|
| 79 |
+
## Citation
|
| 80 |
+
|
| 81 |
+
See the manuscript supplied with the project and the original implementation referenced there. If you build on this benchmark, cite the manuscript and clearly distinguish new community scaling results from the manuscript's reported/comparative results.
|
benchmarks/train_image.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
"""Reproducible GPU/CPU scaling benchmark for Hilbert Operator Intelligence."""
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
import argparse, json, os, random, time
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch
|
| 8 |
+
from torch import nn
|
| 9 |
+
from torch.utils.data import DataLoader, Subset, TensorDataset
|
| 10 |
+
ROOT=Path(__file__).resolve().parents[1]
|
| 11 |
+
import sys; sys.path.insert(0,str(ROOT))
|
| 12 |
+
from hilbert_mi.scalable import build_model
|
| 13 |
+
|
| 14 |
+
def seed_all(s): random.seed(s); np.random.seed(s); torch.manual_seed(s); torch.cuda.manual_seed_all(s)
|
| 15 |
+
def synthetic(n, classes, size, seed):
|
| 16 |
+
g=torch.Generator().manual_seed(seed); y=torch.randint(classes,(n,),generator=g); x=torch.randn(n,3,size,size,generator=g)*.15
|
| 17 |
+
# deterministic class-dependent spectral/spatial signal
|
| 18 |
+
for i in range(n):
|
| 19 |
+
k=int(y[i]); x[i,:, (k*3)%size::10, :] += .6; x[i,:, :, (k*5)%size::11] += .35
|
| 20 |
+
return TensorDataset(x,y)
|
| 21 |
+
def datasets(name, root, ntrain, ntest, size, seed):
|
| 22 |
+
if name=="synthetic": return synthetic(ntrain,10,size,seed), synthetic(ntest,10,size,seed+1),10
|
| 23 |
+
try:
|
| 24 |
+
import torchvision.transforms as T; import torchvision.datasets as D
|
| 25 |
+
except ImportError as e: raise SystemExit("Install benchmark extras: pip install -e '.[benchmark]'") from e
|
| 26 |
+
tf=T.Compose([T.Resize((size,size)),T.ToTensor()])
|
| 27 |
+
cls={"cifar10":D.CIFAR10,"cifar100":D.CIFAR100}[name]; nc=10 if name=="cifar10" else 100
|
| 28 |
+
tr=cls(root=root,train=True,download=True,transform=tf); te=cls(root=root,train=False,download=True,transform=tf)
|
| 29 |
+
if ntrain and ntrain<len(tr): tr=Subset(tr,range(ntrain))
|
| 30 |
+
if ntest and ntest<len(te): te=Subset(te,range(ntest))
|
| 31 |
+
return tr,te,nc
|
| 32 |
+
def evaluate(m,loader,dev):
|
| 33 |
+
m.eval(); ok=n=0; loss=0.; ce=nn.CrossEntropyLoss(reduction="sum")
|
| 34 |
+
with torch.no_grad():
|
| 35 |
+
for x,y in loader:
|
| 36 |
+
x,y=x.to(dev),y.to(dev); z=m(x); loss+=ce(z,y).item(); ok+=(z.argmax(1)==y).sum().item(); n+=len(y)
|
| 37 |
+
return {"accuracy":ok/n,"nll":loss/n}
|
| 38 |
+
def main():
|
| 39 |
+
p=argparse.ArgumentParser(); p.add_argument("--config",required=True); p.add_argument("--dataset",default="cifar10",choices=["cifar10","cifar100","synthetic"]); p.add_argument("--epochs",type=int,default=None); p.add_argument("--batch-size",type=int,default=None); p.add_argument("--train-samples",type=int,default=0); p.add_argument("--test-samples",type=int,default=0); p.add_argument("--image-size",type=int,default=32); p.add_argument("--seed",type=int,default=7); p.add_argument("--output",default="results/run.json"); p.add_argument("--amp",action="store_true"); a=p.parse_args()
|
| 40 |
+
cfg=json.load(open(a.config)); seed_all(a.seed); dev=torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 41 |
+
tr,te,nc=datasets(a.dataset,str(ROOT/"data"),a.train_samples,a.test_samples,a.image_size,a.seed)
|
| 42 |
+
bs=a.batch_size or cfg.get("batch_size",128); epochs=a.epochs or cfg.get("epochs",20)
|
| 43 |
+
dl=DataLoader(tr,batch_size=bs,shuffle=True,num_workers=cfg.get("workers",2),pin_memory=dev.type=="cuda"); tl=DataLoader(te,batch_size=bs,shuffle=False,num_workers=cfg.get("workers",2),pin_memory=dev.type=="cuda")
|
| 44 |
+
model=build_model(cfg,nc).to(dev); opt=torch.optim.AdamW(model.parameters(),lr=cfg.get("lr",3e-4),weight_decay=cfg.get("weight_decay",1e-4)); ce=nn.CrossEntropyLoss(); scaler=torch.amp.GradScaler("cuda",enabled=a.amp and dev.type=="cuda")
|
| 45 |
+
t0=time.perf_counter(); seen=0
|
| 46 |
+
for ep in range(epochs):
|
| 47 |
+
model.train()
|
| 48 |
+
for x,y in dl:
|
| 49 |
+
x,y=x.to(dev,non_blocking=True),y.to(dev,non_blocking=True); opt.zero_grad(set_to_none=True)
|
| 50 |
+
with torch.autocast(device_type=dev.type,enabled=a.amp and dev.type=="cuda"):
|
| 51 |
+
loss=ce(model(x),y)
|
| 52 |
+
scaler.scale(loss).backward(); scaler.step(opt); scaler.update(); seen+=len(y)
|
| 53 |
+
if dev.type=="cuda": torch.cuda.synchronize()
|
| 54 |
+
sec=time.perf_counter()-t0; met=evaluate(model,tl,dev)
|
| 55 |
+
result={"benchmark":"hoi-image-v1","dataset":a.dataset,"config":Path(a.config).stem,"seed":a.seed,"epochs":epochs,"train_samples":len(tr),"test_samples":len(te),"image_size":a.image_size,"params":model.param_count,"device":str(dev),"gpu":torch.cuda.get_device_name(0) if dev.type=="cuda" else None,"torch":torch.__version__,"amp":bool(a.amp and dev.type=="cuda"),"train_seconds":sec,"samples_per_second":seen/sec,"peak_gpu_memory_gb":torch.cuda.max_memory_allocated()/1e9 if dev.type=="cuda" else 0.0,**met}
|
| 56 |
+
out=ROOT/a.output; out.parent.mkdir(parents=True,exist_ok=True); out.write_text(json.dumps(result,indent=2)+"\n"); print(json.dumps(result,indent=2))
|
| 57 |
+
if __name__=="__main__": main()
|
configs/base.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"width":128,"depth":6,"modes":16,"rank":32,"batch_size":128,"epochs":50,"lr":0.0003,"weight_decay":0.0001,"workers":4}
|
configs/large.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"width":256,"depth":8,"modes":16,"rank":64,"batch_size":64,"epochs":80,"lr":0.0003,"weight_decay":0.0001,"workers":4}
|
configs/small.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"width":64,"depth":4,"modes":12,"rank":16,"batch_size":128,"epochs":30,"lr":0.0003,"weight_decay":0.0001,"workers":4}
|
configs/tiny.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"width":32,"depth":2,"modes":8,"rank":0,"batch_size":64,"epochs":10,"lr":0.0003,"weight_decay":0.0001,"workers":4}
|
docs/SCALING_PROTOCOL.md
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# HOI Scaling Protocol v1
|
| 2 |
+
|
| 3 |
+
## Primary axis
|
| 4 |
+
Dataset × model tier × seed. Primary dataset: CIFAR-10. Extension: CIFAR-100. Seeds: 7, 17, 27.
|
| 5 |
+
|
| 6 |
+
## Controlled variables
|
| 7 |
+
Use the checked-in config, 32×32 input for official CIFAR runs, AdamW, no unreported augmentation, and the benchmark's test evaluation. If a change is scientifically useful, give it a new config name rather than overwriting an official tier.
|
| 8 |
+
|
| 9 |
+
## Required metrics
|
| 10 |
+
Accuracy, NLL, train wall seconds, samples/second, peak allocated GPU memory, parameter count, device name, PyTorch version, AMP status, train/test sample counts, epochs, seed.
|
| 11 |
+
|
| 12 |
+
## Scaling extensions
|
| 13 |
+
Recommended high-compute experiments: width {128,256,512}; depth {6,8,12}; modes {8,12,16}; spectral rank {16,32,64,full}; resolution {32,64,128} on an appropriate dataset. Change one axis at a time before factorial sweeps.
|
| 14 |
+
|
| 15 |
+
## Fair comparisons
|
| 16 |
+
Add CNN/ViT/FNO baselines only with the same data and evaluation protocol. For efficiency claims, provide both parameter-matched and measured-throughput comparisons. Report failed/OOM runs because the feasible frontier is itself a scaling result.
|
docs/manuscript.pdf
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:14599872fd9198c7041f55f98ffd67b0f8f46b1729ecff4c9a6a153ec33b85b9
|
| 3 |
+
size 396233
|
hilbert_mi/__init__.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hilbert-space operator learning and spectral reasoning package."""
|
| 2 |
+
|
| 3 |
+
from .transforms import (
|
| 4 |
+
pixel_features,
|
| 5 |
+
fourier_features,
|
| 6 |
+
haar2d_batch,
|
| 7 |
+
scattering_like_features,
|
| 8 |
+
)
|
| 9 |
+
from .operators import HilbertSchmidtOperatorClassifier
|
| 10 |
+
|
| 11 |
+
__all__ = [
|
| 12 |
+
"pixel_features",
|
| 13 |
+
"fourier_features",
|
| 14 |
+
"haar2d_batch",
|
| 15 |
+
"scattering_like_features",
|
| 16 |
+
"HilbertSchmidtOperatorClassifier",
|
| 17 |
+
]
|
hilbert_mi/datasets.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Dataset utilities."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from dataclasses import dataclass
|
| 6 |
+
import numpy as np
|
| 7 |
+
from sklearn.datasets import load_digits
|
| 8 |
+
from sklearn.model_selection import train_test_split
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
@dataclass
|
| 12 |
+
class SplitData:
|
| 13 |
+
X_train: np.ndarray
|
| 14 |
+
y_train: np.ndarray
|
| 15 |
+
X_val: np.ndarray
|
| 16 |
+
y_val: np.ndarray
|
| 17 |
+
X_test: np.ndarray
|
| 18 |
+
y_test: np.ndarray
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def load_digits_splits(seed: int = 7, test_size: float = 0.20, val_size: float = 0.20) -> SplitData:
|
| 22 |
+
"""Load standard sklearn Digits and create stratified train/val/test splits."""
|
| 23 |
+
digits = load_digits()
|
| 24 |
+
X = digits.images.astype(np.float32) / 16.0
|
| 25 |
+
y = digits.target.astype(np.int64)
|
| 26 |
+
X_trainval, X_test, y_trainval, y_test = train_test_split(
|
| 27 |
+
X, y, test_size=test_size, random_state=seed, stratify=y
|
| 28 |
+
)
|
| 29 |
+
val_fraction_of_trainval = val_size / (1.0 - test_size)
|
| 30 |
+
X_train, X_val, y_train, y_val = train_test_split(
|
| 31 |
+
X_trainval, y_trainval, test_size=val_fraction_of_trainval, random_state=seed, stratify=y_trainval
|
| 32 |
+
)
|
| 33 |
+
return SplitData(X_train, y_train, X_val, y_val, X_test, y_test)
|
hilbert_mi/learnable.py
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Learnable spectral modulation model.
|
| 2 |
+
|
| 3 |
+
This implements the paper's learnable spectral nonlinearity idea using Haar
|
| 4 |
+
coefficients, coefficient-wise shrinkage, gain/phase-like scaling, and a final
|
| 5 |
+
linear task operator.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
from dataclasses import dataclass
|
| 11 |
+
import numpy as np
|
| 12 |
+
import torch
|
| 13 |
+
from torch import nn
|
| 14 |
+
from torch.utils.data import DataLoader, TensorDataset
|
| 15 |
+
|
| 16 |
+
from .transforms import torch_wavelet_features
|
| 17 |
+
from .metrics import nll_from_proba
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class LearnableSpectralOperator(nn.Module):
|
| 21 |
+
"""Haar coefficients + learnable soft-threshold/gain + task operator."""
|
| 22 |
+
|
| 23 |
+
def __init__(self, input_shape=(1, 8, 8), n_classes: int = 10, levels: int = 3):
|
| 24 |
+
super().__init__()
|
| 25 |
+
self.levels = levels
|
| 26 |
+
n_coeff = int(np.prod(input_shape))
|
| 27 |
+
self.log_theta = nn.Parameter(torch.full((n_coeff,), -2.0))
|
| 28 |
+
self.gain = nn.Parameter(torch.ones(n_coeff))
|
| 29 |
+
self.operator = nn.Linear(n_coeff, n_classes)
|
| 30 |
+
|
| 31 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 32 |
+
coeff = torch_wavelet_features(x, levels=self.levels)
|
| 33 |
+
theta = torch.nn.functional.softplus(self.log_theta) + 1e-6
|
| 34 |
+
shrink = coeff.abs() / (coeff.abs() + theta)
|
| 35 |
+
modulated = self.gain * shrink * coeff
|
| 36 |
+
return self.operator(modulated)
|
| 37 |
+
|
| 38 |
+
@property
|
| 39 |
+
def param_count(self) -> int:
|
| 40 |
+
return sum(p.numel() for p in self.parameters())
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
@dataclass
|
| 44 |
+
class TorchModelWrapper:
|
| 45 |
+
model: LearnableSpectralOperator
|
| 46 |
+
device: str = "cpu"
|
| 47 |
+
|
| 48 |
+
@property
|
| 49 |
+
def param_count_(self) -> int:
|
| 50 |
+
return self.model.param_count
|
| 51 |
+
|
| 52 |
+
def _tensor(self, X):
|
| 53 |
+
X = np.asarray(X, dtype=np.float32)
|
| 54 |
+
if X.ndim == 3:
|
| 55 |
+
X = X[:, None, :, :]
|
| 56 |
+
if X.max() > 1.0:
|
| 57 |
+
X = X / 16.0
|
| 58 |
+
return torch.from_numpy(X).to(self.device)
|
| 59 |
+
|
| 60 |
+
def decision_function(self, X):
|
| 61 |
+
self.model.eval()
|
| 62 |
+
with torch.no_grad():
|
| 63 |
+
logits = self.model(self._tensor(X)).cpu().numpy()
|
| 64 |
+
return logits
|
| 65 |
+
|
| 66 |
+
def predict_proba(self, X):
|
| 67 |
+
logits = self.decision_function(X)
|
| 68 |
+
logits = logits - logits.max(axis=1, keepdims=True)
|
| 69 |
+
e = np.exp(logits)
|
| 70 |
+
return e / e.sum(axis=1, keepdims=True)
|
| 71 |
+
|
| 72 |
+
def predict(self, X):
|
| 73 |
+
return self.decision_function(X).argmax(axis=1)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _torch_metrics(wrapper: TorchModelWrapper, X, y, prefix: str) -> dict:
|
| 77 |
+
pred = wrapper.predict(X)
|
| 78 |
+
proba = wrapper.predict_proba(X)
|
| 79 |
+
return {
|
| 80 |
+
f"{prefix}_accuracy": float((pred == y).mean()),
|
| 81 |
+
f"{prefix}_macro_f1": float(__import__("sklearn.metrics").metrics.f1_score(y, pred, average="macro")),
|
| 82 |
+
f"{prefix}_nll": nll_from_proba(proba, y),
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def train_learnable_spectral(
|
| 87 |
+
X_train,
|
| 88 |
+
y_train,
|
| 89 |
+
X_val,
|
| 90 |
+
y_val,
|
| 91 |
+
epochs: int = 30,
|
| 92 |
+
lr: float = 3e-2,
|
| 93 |
+
weight_decay: float = 1e-4,
|
| 94 |
+
seed: int = 7,
|
| 95 |
+
batch_size: int = 128,
|
| 96 |
+
device: str = "cpu",
|
| 97 |
+
):
|
| 98 |
+
torch.manual_seed(seed)
|
| 99 |
+
model = LearnableSpectralOperator().to(device)
|
| 100 |
+
opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay)
|
| 101 |
+
loss_fn = nn.CrossEntropyLoss()
|
| 102 |
+
|
| 103 |
+
Xtr = np.asarray(X_train, dtype=np.float32)
|
| 104 |
+
Xva = np.asarray(X_val, dtype=np.float32)
|
| 105 |
+
if Xtr.ndim == 3:
|
| 106 |
+
Xtr = Xtr[:, None, :, :]
|
| 107 |
+
if Xva.ndim == 3:
|
| 108 |
+
Xva = Xva[:, None, :, :]
|
| 109 |
+
ytr = np.asarray(y_train, dtype=np.int64)
|
| 110 |
+
ds = TensorDataset(torch.from_numpy(Xtr), torch.from_numpy(ytr))
|
| 111 |
+
loader = DataLoader(ds, batch_size=batch_size, shuffle=True)
|
| 112 |
+
|
| 113 |
+
history = []
|
| 114 |
+
for epoch in range(1, epochs + 1):
|
| 115 |
+
model.train()
|
| 116 |
+
total = 0.0
|
| 117 |
+
seen = 0
|
| 118 |
+
for xb, yb in loader:
|
| 119 |
+
xb = xb.to(device)
|
| 120 |
+
yb = yb.to(device)
|
| 121 |
+
opt.zero_grad(set_to_none=True)
|
| 122 |
+
logits = model(xb)
|
| 123 |
+
# mild sparsity prior on spectral gains/shrinkage, matching the paper's regularized spectral learning theme
|
| 124 |
+
loss = loss_fn(logits, yb) + 1e-4 * model.gain.abs().mean()
|
| 125 |
+
loss.backward()
|
| 126 |
+
opt.step()
|
| 127 |
+
total += float(loss.item()) * len(xb)
|
| 128 |
+
seen += len(xb)
|
| 129 |
+
wrapper = TorchModelWrapper(model, device=device)
|
| 130 |
+
val_metrics = _torch_metrics(wrapper, X_val, y_val, "val")
|
| 131 |
+
history.append({"epoch": epoch, "train_loss": total / seen, **val_metrics})
|
| 132 |
+
return TorchModelWrapper(model, device=device), history
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def fit_learnable_threshold_operator(X_train, y_train, X_val, y_val, lam: float = 1e-3, thresholds=None):
|
| 136 |
+
"""Fast validation-learned spectral soft-threshold operator.
|
| 137 |
+
|
| 138 |
+
This dependency-light variant learns a global spectral threshold by validation
|
| 139 |
+
search, then estimates the task operator with Hilbert-Schmidt ridge. It is
|
| 140 |
+
used in the default benchmark so the repo runs quickly on CPU.
|
| 141 |
+
"""
|
| 142 |
+
from .transforms import haar2d_batch
|
| 143 |
+
from .operators import HilbertSchmidtOperatorClassifier
|
| 144 |
+
|
| 145 |
+
if thresholds is None:
|
| 146 |
+
thresholds = [0.0, 0.005, 0.01, 0.025, 0.05, 0.10, 0.20]
|
| 147 |
+
|
| 148 |
+
Ctr = haar2d_batch(X_train)
|
| 149 |
+
Cva = haar2d_batch(X_val)
|
| 150 |
+
|
| 151 |
+
def shrink(C, theta):
|
| 152 |
+
return C * (np.abs(C) / (np.abs(C) + theta + 1e-8))
|
| 153 |
+
|
| 154 |
+
best = None
|
| 155 |
+
for theta in thresholds:
|
| 156 |
+
Ztr = shrink(Ctr, theta)
|
| 157 |
+
Zva = shrink(Cva, theta)
|
| 158 |
+
clf = HilbertSchmidtOperatorClassifier(ridge_lambda=lam).fit(Ztr, y_train)
|
| 159 |
+
val_acc = float((clf.predict(Zva) == y_val).mean())
|
| 160 |
+
if best is None or val_acc > best[0]:
|
| 161 |
+
best = (val_acc, theta, clf)
|
| 162 |
+
|
| 163 |
+
class ThresholdSpectralWrapper:
|
| 164 |
+
theta_ = best[1]
|
| 165 |
+
clf_ = best[2]
|
| 166 |
+
param_count_ = int(best[2].param_count_ + 1)
|
| 167 |
+
|
| 168 |
+
def _features(self, X):
|
| 169 |
+
C = haar2d_batch(X)
|
| 170 |
+
return shrink(C, self.theta_)
|
| 171 |
+
|
| 172 |
+
def decision_function(self, X):
|
| 173 |
+
return self.clf_.decision_function(self._features(X))
|
| 174 |
+
|
| 175 |
+
def predict_proba(self, X):
|
| 176 |
+
return self.clf_.predict_proba(self._features(X))
|
| 177 |
+
|
| 178 |
+
def predict(self, X):
|
| 179 |
+
return self.clf_.predict(self._features(X))
|
| 180 |
+
|
| 181 |
+
return ThresholdSpectralWrapper(), {"selected_threshold": float(best[1]), "val_accuracy": float(best[0])}
|
hilbert_mi/metrics.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Metrics for classification and operator benchmarking."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
from sklearn.metrics import accuracy_score, f1_score, confusion_matrix
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def nll_from_proba(proba: np.ndarray, y: np.ndarray) -> float:
|
| 10 |
+
proba = np.asarray(proba, dtype=np.float64)
|
| 11 |
+
y = np.asarray(y, dtype=np.int64)
|
| 12 |
+
return float(-np.mean(np.log(proba[np.arange(len(y)), y] + 1e-12)))
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def classification_metrics(model, X: np.ndarray, y: np.ndarray, prefix: str) -> dict:
|
| 16 |
+
pred = model.predict(X)
|
| 17 |
+
proba = model.predict_proba(X)
|
| 18 |
+
return {
|
| 19 |
+
f"{prefix}_accuracy": float(accuracy_score(y, pred)),
|
| 20 |
+
f"{prefix}_macro_f1": float(f1_score(y, pred, average="macro")),
|
| 21 |
+
f"{prefix}_nll": nll_from_proba(proba, y),
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def confusion(y_true: np.ndarray, y_pred: np.ndarray) -> list[list[int]]:
|
| 26 |
+
return confusion_matrix(y_true, y_pred).astype(int).tolist()
|
hilbert_mi/operators.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hilbert-Schmidt-regularized operator estimators."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from dataclasses import dataclass
|
| 6 |
+
import numpy as np
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def _one_hot(y: np.ndarray, n_classes: int) -> np.ndarray:
|
| 10 |
+
y = np.asarray(y, dtype=np.int64)
|
| 11 |
+
out = np.zeros((len(y), n_classes), dtype=np.float64)
|
| 12 |
+
out[np.arange(len(y)), y] = 1.0
|
| 13 |
+
return out
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
@dataclass
|
| 17 |
+
class HilbertSchmidtOperatorClassifier:
|
| 18 |
+
"""Closed-form task operator T: H_X -> R^C with HS/Frobenius regularization.
|
| 19 |
+
|
| 20 |
+
Given feature vectors Phi(x_i), the classifier solves
|
| 21 |
+
min_W ||X W - Y||_F^2 + lambda ||W||_F^2.
|
| 22 |
+
This is the finite-dimensional realization of the Hilbert-space operator
|
| 23 |
+
estimation objective in the paper.
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
ridge_lambda: float = 1e-3
|
| 27 |
+
standardize: bool = True
|
| 28 |
+
add_bias: bool = True
|
| 29 |
+
|
| 30 |
+
def fit(self, X: np.ndarray, y: np.ndarray) -> "HilbertSchmidtOperatorClassifier":
|
| 31 |
+
X = np.asarray(X, dtype=np.float64)
|
| 32 |
+
y = np.asarray(y, dtype=np.int64)
|
| 33 |
+
self.classes_ = np.unique(y)
|
| 34 |
+
if not np.array_equal(self.classes_, np.arange(len(self.classes_))):
|
| 35 |
+
raise ValueError("Labels must be contiguous from 0 to C-1")
|
| 36 |
+
self.n_classes_ = len(self.classes_)
|
| 37 |
+
|
| 38 |
+
if self.standardize:
|
| 39 |
+
self.mean_ = X.mean(axis=0, keepdims=True)
|
| 40 |
+
self.std_ = X.std(axis=0, keepdims=True) + 1e-8
|
| 41 |
+
Xs = (X - self.mean_) / self.std_
|
| 42 |
+
else:
|
| 43 |
+
self.mean_ = np.zeros((1, X.shape[1]))
|
| 44 |
+
self.std_ = np.ones((1, X.shape[1]))
|
| 45 |
+
Xs = X
|
| 46 |
+
if self.add_bias:
|
| 47 |
+
Xs = np.concatenate([Xs, np.ones((Xs.shape[0], 1))], axis=1)
|
| 48 |
+
|
| 49 |
+
Y = _one_hot(y, self.n_classes_)
|
| 50 |
+
# Solve (X^T X + lambda I) W = X^T Y
|
| 51 |
+
A = Xs.T @ Xs
|
| 52 |
+
reg = self.ridge_lambda * np.eye(A.shape[0], dtype=np.float64)
|
| 53 |
+
if self.add_bias:
|
| 54 |
+
reg[-1, -1] = 0.0 # do not regularize bias
|
| 55 |
+
B = Xs.T @ Y
|
| 56 |
+
self.W_ = np.linalg.solve(A + reg, B)
|
| 57 |
+
self.param_count_ = int(self.W_.size)
|
| 58 |
+
return self
|
| 59 |
+
|
| 60 |
+
def decision_function(self, X: np.ndarray) -> np.ndarray:
|
| 61 |
+
X = np.asarray(X, dtype=np.float64)
|
| 62 |
+
Xs = (X - self.mean_) / self.std_
|
| 63 |
+
if self.add_bias:
|
| 64 |
+
Xs = np.concatenate([Xs, np.ones((Xs.shape[0], 1))], axis=1)
|
| 65 |
+
return Xs @ self.W_
|
| 66 |
+
|
| 67 |
+
def predict_proba(self, X: np.ndarray) -> np.ndarray:
|
| 68 |
+
logits = self.decision_function(X)
|
| 69 |
+
logits = logits - logits.max(axis=1, keepdims=True)
|
| 70 |
+
exp = np.exp(logits)
|
| 71 |
+
return exp / exp.sum(axis=1, keepdims=True)
|
| 72 |
+
|
| 73 |
+
def predict(self, X: np.ndarray) -> np.ndarray:
|
| 74 |
+
return self.decision_function(X).argmax(axis=1)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def fit_rbf_kernel_ridge(X_train: np.ndarray, y_train: np.ndarray, gamma: float = 0.03, lam: float = 1e-2, n_landmarks: int = 256, seed: int = 7):
|
| 78 |
+
"""Fast RKHS baseline using Nystrom RBF features and operator ridge.
|
| 79 |
+
|
| 80 |
+
A full kernel solve is intentionally avoided so the repository remains fast
|
| 81 |
+
on CPU-only environments. The learned operator still lives in an explicit
|
| 82 |
+
finite-dimensional RKHS approximation.
|
| 83 |
+
"""
|
| 84 |
+
rng = np.random.default_rng(seed)
|
| 85 |
+
X_train = np.asarray(X_train, dtype=np.float64)
|
| 86 |
+
y_train = np.asarray(y_train, dtype=np.int64)
|
| 87 |
+
n = len(X_train)
|
| 88 |
+
m = min(n_landmarks, n)
|
| 89 |
+
idx = rng.choice(n, size=m, replace=False)
|
| 90 |
+
landmarks = X_train[idx]
|
| 91 |
+
|
| 92 |
+
def features(X):
|
| 93 |
+
X = np.asarray(X, dtype=np.float64)
|
| 94 |
+
sq = ((X[:, None, :] - landmarks[None, :, :]) ** 2).sum(axis=-1)
|
| 95 |
+
return np.exp(-gamma * sq)
|
| 96 |
+
|
| 97 |
+
Z = features(X_train)
|
| 98 |
+
clf = HilbertSchmidtOperatorClassifier(ridge_lambda=lam, standardize=True).fit(Z, y_train)
|
| 99 |
+
|
| 100 |
+
class RBFKernelRidge:
|
| 101 |
+
param_count_ = int(clf.param_count_ + landmarks.size)
|
| 102 |
+
|
| 103 |
+
def decision_function(self, X):
|
| 104 |
+
return clf.decision_function(features(X))
|
| 105 |
+
|
| 106 |
+
def predict_proba(self, X):
|
| 107 |
+
return clf.predict_proba(features(X))
|
| 108 |
+
|
| 109 |
+
def predict(self, X):
|
| 110 |
+
return clf.predict(features(X))
|
| 111 |
+
|
| 112 |
+
return RBFKernelRidge()
|
hilbert_mi/reasoning.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Operator-based symbolic reasoning demo in Hilbert space."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from dataclasses import dataclass
|
| 6 |
+
import numpy as np
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def one_hot_digits(labels: np.ndarray, n_classes: int = 10) -> np.ndarray:
|
| 10 |
+
labels = np.asarray(labels, dtype=np.int64)
|
| 11 |
+
X = np.zeros((len(labels), n_classes), dtype=np.float64)
|
| 12 |
+
X[np.arange(len(labels)), labels] = 1.0
|
| 13 |
+
return X
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
@dataclass
|
| 17 |
+
class RelationOperator:
|
| 18 |
+
"""Closed-form relation operator T_r f_x ~= f_y."""
|
| 19 |
+
|
| 20 |
+
lam: float = 1e-3
|
| 21 |
+
|
| 22 |
+
def fit(self, X_src: np.ndarray, X_tgt: np.ndarray) -> "RelationOperator":
|
| 23 |
+
X_src = np.asarray(X_src, dtype=np.float64)
|
| 24 |
+
X_tgt = np.asarray(X_tgt, dtype=np.float64)
|
| 25 |
+
A = X_src.T @ X_src + self.lam * np.eye(X_src.shape[1])
|
| 26 |
+
B = X_src.T @ X_tgt
|
| 27 |
+
self.W_ = np.linalg.solve(A, B)
|
| 28 |
+
return self
|
| 29 |
+
|
| 30 |
+
def apply(self, X: np.ndarray) -> np.ndarray:
|
| 31 |
+
return np.asarray(X, dtype=np.float64) @ self.W_
|
| 32 |
+
|
| 33 |
+
def predict_digit(self, X: np.ndarray) -> np.ndarray:
|
| 34 |
+
return self.apply(X).argmax(axis=1)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def build_relation_pairs(labels: np.ndarray, relation: str) -> tuple[np.ndarray, np.ndarray]:
|
| 38 |
+
labels = np.asarray(labels, dtype=np.int64)
|
| 39 |
+
if relation == "successor":
|
| 40 |
+
targets = (labels + 1) % 10
|
| 41 |
+
elif relation == "predecessor":
|
| 42 |
+
targets = (labels - 1) % 10
|
| 43 |
+
elif relation == "plus_two":
|
| 44 |
+
targets = (labels + 2) % 10
|
| 45 |
+
elif relation == "mirror":
|
| 46 |
+
targets = 9 - labels
|
| 47 |
+
else:
|
| 48 |
+
raise ValueError(f"Unknown relation: {relation}")
|
| 49 |
+
return one_hot_digits(labels), one_hot_digits(targets)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def evaluate_reasoning(seed: int = 7, lam: float = 1e-3) -> dict:
|
| 53 |
+
rng = np.random.default_rng(seed)
|
| 54 |
+
train_labels = rng.integers(0, 10, size=500)
|
| 55 |
+
test_labels = np.arange(10).repeat(20)
|
| 56 |
+
relations = ["successor", "predecessor", "plus_two", "mirror"]
|
| 57 |
+
metrics = {}
|
| 58 |
+
ops = {}
|
| 59 |
+
for rel in relations:
|
| 60 |
+
Xs, Xt = build_relation_pairs(train_labels, rel)
|
| 61 |
+
op = RelationOperator(lam=lam).fit(Xs, Xt)
|
| 62 |
+
ops[rel] = op
|
| 63 |
+
Xs_test, Xt_test = build_relation_pairs(test_labels, rel)
|
| 64 |
+
y_true = Xt_test.argmax(axis=1)
|
| 65 |
+
y_pred = op.predict_digit(Xs_test)
|
| 66 |
+
metrics[f"{rel}_accuracy"] = float((y_true == y_pred).mean())
|
| 67 |
+
|
| 68 |
+
# Composition test: successor after successor should equal plus_two.
|
| 69 |
+
X = one_hot_digits(test_labels)
|
| 70 |
+
composed = ops["successor"].apply(ops["successor"].apply(X))
|
| 71 |
+
y_comp = composed.argmax(axis=1)
|
| 72 |
+
y_true = ((test_labels + 2) % 10)
|
| 73 |
+
metrics["composition_successor_successor_accuracy"] = float((y_comp == y_true).mean())
|
| 74 |
+
return metrics
|
hilbert_mi/scalable.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""GPU-scalable Hilbert/spectral operator models for contributor benchmarks."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
import math
|
| 4 |
+
import torch
|
| 5 |
+
from torch import nn
|
| 6 |
+
|
| 7 |
+
class SpectralOperatorBlock(nn.Module):
|
| 8 |
+
"""Truncated 2-D Fourier operator + pointwise residual nonlinearity.
|
| 9 |
+
|
| 10 |
+
The learned complex multiplier is a finite-dimensional realization of a
|
| 11 |
+
bounded spectral operator. Only low modes are parameterized, making the
|
| 12 |
+
compute/memory scaling explicit and measurable.
|
| 13 |
+
"""
|
| 14 |
+
def __init__(self, width: int, modes: int, rank: int = 0):
|
| 15 |
+
super().__init__()
|
| 16 |
+
self.width, self.modes, self.rank = width, modes, rank
|
| 17 |
+
if rank and rank < width:
|
| 18 |
+
self.left = nn.Parameter(torch.randn(modes, modes, width, rank, 2) / math.sqrt(width))
|
| 19 |
+
self.right = nn.Parameter(torch.randn(modes, modes, rank, width, 2) / math.sqrt(rank))
|
| 20 |
+
self.weight = None
|
| 21 |
+
else:
|
| 22 |
+
self.weight = nn.Parameter(torch.randn(modes, modes, width, width, 2) / math.sqrt(width))
|
| 23 |
+
self.left = self.right = None
|
| 24 |
+
self.local = nn.Conv2d(width, width, 1)
|
| 25 |
+
self.norm = nn.GroupNorm(1, width)
|
| 26 |
+
self.act = nn.GELU()
|
| 27 |
+
|
| 28 |
+
@staticmethod
|
| 29 |
+
def _complex(p):
|
| 30 |
+
return torch.view_as_complex(p.contiguous())
|
| 31 |
+
|
| 32 |
+
def forward(self, x):
|
| 33 |
+
b,c,h,w = x.shape
|
| 34 |
+
z = torch.fft.rfft2(x, norm="ortho")
|
| 35 |
+
mh = min(self.modes, h)
|
| 36 |
+
mw = min(self.modes, z.shape[-1])
|
| 37 |
+
zin = z[:, :, :mh, :mw]
|
| 38 |
+
if self.weight is not None:
|
| 39 |
+
W = self._complex(self.weight[:mh,:mw])
|
| 40 |
+
zout = torch.einsum("bihw,hwi o->bohw", zin, W) if False else torch.einsum("bihw,hwio->bohw", zin, W)
|
| 41 |
+
else:
|
| 42 |
+
L = self._complex(self.left[:mh,:mw]); R = self._complex(self.right[:mh,:mw])
|
| 43 |
+
mid = torch.einsum("bihw,hwir->brhw", zin, L)
|
| 44 |
+
zout = torch.einsum("brhw,hwro->bohw", mid, R)
|
| 45 |
+
full = torch.zeros((b,c,h,z.shape[-1]), dtype=z.dtype, device=x.device)
|
| 46 |
+
full[:, :, :mh, :mw] = zout
|
| 47 |
+
y = torch.fft.irfft2(full, s=(h,w), norm="ortho")
|
| 48 |
+
return self.act(self.norm(y + self.local(x)))
|
| 49 |
+
|
| 50 |
+
class HilbertOperatorClassifier(nn.Module):
|
| 51 |
+
"""Scalable image classifier built from spectral operator blocks."""
|
| 52 |
+
def __init__(self, in_channels=3, num_classes=10, width=64, depth=4, modes=12, rank=0):
|
| 53 |
+
super().__init__()
|
| 54 |
+
self.lift = nn.Conv2d(in_channels, width, 1)
|
| 55 |
+
self.blocks = nn.Sequential(*[SpectralOperatorBlock(width, modes, rank) for _ in range(depth)])
|
| 56 |
+
self.head = nn.Sequential(nn.AdaptiveAvgPool2d(1), nn.Flatten(), nn.Linear(width, num_classes))
|
| 57 |
+
def forward(self, x):
|
| 58 |
+
return self.head(self.blocks(self.lift(x)))
|
| 59 |
+
@property
|
| 60 |
+
def param_count(self): return sum(p.numel() for p in self.parameters())
|
| 61 |
+
|
| 62 |
+
def build_model(cfg, num_classes=10, in_channels=3):
|
| 63 |
+
return HilbertOperatorClassifier(in_channels, num_classes, cfg["width"], cfg["depth"], cfg["modes"], cfg.get("rank",0))
|
hilbert_mi/transforms.py
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Spectral feature maps for Hilbert-space learning.
|
| 2 |
+
|
| 3 |
+
The paper emphasizes Fourier, wavelet, scattering, and learnable spectral
|
| 4 |
+
representations. This module implements compact CPU-friendly versions for the
|
| 5 |
+
8x8 Digits benchmark.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
import torch
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def _as_images(X: np.ndarray) -> np.ndarray:
|
| 15 |
+
"""Return images as (n, 8, 8) float32 in [0, 1]."""
|
| 16 |
+
X = np.asarray(X, dtype=np.float32)
|
| 17 |
+
if X.ndim == 2 and X.shape[1] == 64:
|
| 18 |
+
X = X.reshape(-1, 8, 8)
|
| 19 |
+
elif X.ndim != 3:
|
| 20 |
+
raise ValueError(f"Expected (n,64) or (n,8,8), got {X.shape}")
|
| 21 |
+
# sklearn digits are in [0,16]. If already normalized this is harmless.
|
| 22 |
+
if X.min() >= 0.0 and X.max() > 1.0:
|
| 23 |
+
X = X / 16.0
|
| 24 |
+
return X.astype(np.float32)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def pixel_features(X: np.ndarray) -> np.ndarray:
|
| 28 |
+
"""Identity Hilbert embedding over pixels."""
|
| 29 |
+
imgs = _as_images(X)
|
| 30 |
+
return imgs.reshape(imgs.shape[0], -1).astype(np.float32)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def fourier_features(X: np.ndarray) -> np.ndarray:
|
| 34 |
+
"""Real-valued 2D Fourier feature map.
|
| 35 |
+
|
| 36 |
+
We concatenate real and imaginary FFT coefficients. This is a spectral
|
| 37 |
+
embedding in a global orthonormal basis.
|
| 38 |
+
"""
|
| 39 |
+
imgs = _as_images(X)
|
| 40 |
+
fft = np.fft.fft2(imgs, axes=(-2, -1), norm="ortho")
|
| 41 |
+
feat = np.concatenate([fft.real.reshape(len(imgs), -1), fft.imag.reshape(len(imgs), -1)], axis=1)
|
| 42 |
+
return feat.astype(np.float32)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def haar2d_single(img: np.ndarray, levels: int = 3) -> np.ndarray:
|
| 46 |
+
"""Orthonormal 2D Haar wavelet transform for square power-of-two images.
|
| 47 |
+
|
| 48 |
+
Coefficients are packed in-place using the standard quadrant layout.
|
| 49 |
+
For 8x8 images, levels=3 yields a full multiresolution transform.
|
| 50 |
+
"""
|
| 51 |
+
x = np.asarray(img, dtype=np.float32).copy()
|
| 52 |
+
h, w = x.shape
|
| 53 |
+
if h != w or (h & (h - 1)) != 0:
|
| 54 |
+
raise ValueError("haar2d_single expects square power-of-two images")
|
| 55 |
+
size = h
|
| 56 |
+
for _ in range(levels):
|
| 57 |
+
if size < 2:
|
| 58 |
+
break
|
| 59 |
+
block = x[:size, :size]
|
| 60 |
+
a = block[0::2, 0::2]
|
| 61 |
+
b = block[0::2, 1::2]
|
| 62 |
+
c = block[1::2, 0::2]
|
| 63 |
+
d = block[1::2, 1::2]
|
| 64 |
+
ll = (a + b + c + d) / 2.0
|
| 65 |
+
lh = (a - b + c - d) / 2.0
|
| 66 |
+
hl = (a + b - c - d) / 2.0
|
| 67 |
+
hh = (a - b - c + d) / 2.0
|
| 68 |
+
half = size // 2
|
| 69 |
+
x[:half, :half] = ll
|
| 70 |
+
x[:half, half:size] = lh
|
| 71 |
+
x[half:size, :half] = hl
|
| 72 |
+
x[half:size, half:size] = hh
|
| 73 |
+
size = half
|
| 74 |
+
return x
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def inverse_haar2d_single(coeff: np.ndarray, levels: int = 3) -> np.ndarray:
|
| 78 |
+
"""Inverse of haar2d_single."""
|
| 79 |
+
x = np.asarray(coeff, dtype=np.float32).copy()
|
| 80 |
+
h, w = x.shape
|
| 81 |
+
max_levels = int(np.log2(h))
|
| 82 |
+
levels = min(levels, max_levels)
|
| 83 |
+
sizes = [2 ** i for i in range(1, levels + 1)]
|
| 84 |
+
for size in sizes:
|
| 85 |
+
half = size // 2
|
| 86 |
+
ll = x[:half, :half].copy()
|
| 87 |
+
lh = x[:half, half:size].copy()
|
| 88 |
+
hl = x[half:size, :half].copy()
|
| 89 |
+
hh = x[half:size, half:size].copy()
|
| 90 |
+
a = (ll + lh + hl + hh) / 2.0
|
| 91 |
+
b = (ll - lh + hl - hh) / 2.0
|
| 92 |
+
c = (ll + lh - hl - hh) / 2.0
|
| 93 |
+
d = (ll - lh - hl + hh) / 2.0
|
| 94 |
+
block = np.zeros((size, size), dtype=np.float32)
|
| 95 |
+
block[0::2, 0::2] = a
|
| 96 |
+
block[0::2, 1::2] = b
|
| 97 |
+
block[1::2, 0::2] = c
|
| 98 |
+
block[1::2, 1::2] = d
|
| 99 |
+
x[:size, :size] = block
|
| 100 |
+
return x
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def haar2d_batch(X: np.ndarray, levels: int = 3) -> np.ndarray:
|
| 104 |
+
"""Batch Haar wavelet features."""
|
| 105 |
+
imgs = _as_images(X)
|
| 106 |
+
coeffs = np.stack([haar2d_single(img, levels=levels).reshape(-1) for img in imgs], axis=0)
|
| 107 |
+
return coeffs.astype(np.float32)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def scattering_like_features(X: np.ndarray, levels: int = 3) -> np.ndarray:
|
| 111 |
+
"""A compact deterministic scattering-like feature map.
|
| 112 |
+
|
| 113 |
+
True scattering networks require a bank of complex wavelets. For a small
|
| 114 |
+
offline repo, we approximate the paper's wavelet/scattering idea by using
|
| 115 |
+
Haar coefficients, coefficient magnitudes, and energy coefficients. This
|
| 116 |
+
gives deformation-stable, modulus-style spectral features while remaining
|
| 117 |
+
dependency-free.
|
| 118 |
+
"""
|
| 119 |
+
c = haar2d_batch(X, levels=levels)
|
| 120 |
+
return np.concatenate([c, np.abs(c), c * c], axis=1).astype(np.float32)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def torch_haar2d(x: torch.Tensor, levels: int = 3) -> torch.Tensor:
|
| 124 |
+
"""Differentiable 2D Haar transform for tensors shaped (B, 1, 8, 8).
|
| 125 |
+
|
| 126 |
+
This version avoids in-place writes so autograd remains fast and stable.
|
| 127 |
+
It returns coefficients flattened in multiresolution order rather than
|
| 128 |
+
quadrant-packed image order.
|
| 129 |
+
"""
|
| 130 |
+
if x.ndim != 4 or x.shape[1] != 1:
|
| 131 |
+
raise ValueError("Expected tensor of shape (B,1,H,W)")
|
| 132 |
+
current = x
|
| 133 |
+
details = []
|
| 134 |
+
for _ in range(levels):
|
| 135 |
+
if current.shape[-1] < 2 or current.shape[-2] < 2:
|
| 136 |
+
break
|
| 137 |
+
a = current[:, :, 0::2, 0::2]
|
| 138 |
+
b = current[:, :, 0::2, 1::2]
|
| 139 |
+
c = current[:, :, 1::2, 0::2]
|
| 140 |
+
d = current[:, :, 1::2, 1::2]
|
| 141 |
+
ll = (a + b + c + d) / 2.0
|
| 142 |
+
lh = (a - b + c - d) / 2.0
|
| 143 |
+
hl = (a + b - c - d) / 2.0
|
| 144 |
+
hh = (a - b - c + d) / 2.0
|
| 145 |
+
details.extend([lh.flatten(1), hl.flatten(1), hh.flatten(1)])
|
| 146 |
+
current = ll
|
| 147 |
+
details.append(current.flatten(1))
|
| 148 |
+
return torch.cat(details, dim=1)
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def torch_wavelet_features(x: torch.Tensor, levels: int = 3) -> torch.Tensor:
|
| 152 |
+
"""Flattened differentiable Haar features for learnable spectral operator."""
|
| 153 |
+
return torch_haar2d(x, levels=levels)
|
hilbert_mi/utils.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""General helpers."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
import os
|
| 7 |
+
import random
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
import numpy as np
|
| 10 |
+
import torch
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def set_seed(seed: int) -> None:
|
| 14 |
+
random.seed(seed)
|
| 15 |
+
np.random.seed(seed)
|
| 16 |
+
torch.manual_seed(seed)
|
| 17 |
+
torch.cuda.manual_seed_all(seed)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def ensure_dir(path: str | os.PathLike) -> Path:
|
| 21 |
+
p = Path(path)
|
| 22 |
+
p.mkdir(parents=True, exist_ok=True)
|
| 23 |
+
return p
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def save_json(obj, path: str | os.PathLike) -> None:
|
| 27 |
+
path = Path(path)
|
| 28 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 29 |
+
with open(path, "w", encoding="utf-8") as f:
|
| 30 |
+
json.dump(obj, f, indent=2)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def load_json(path: str | os.PathLike):
|
| 34 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 35 |
+
return json.load(f)
|
hilbert_operator_intelligence_benchmark.egg-info/PKG-INFO
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Metadata-Version: 2.4
|
| 2 |
+
Name: hilbert-operator-intelligence-benchmark
|
| 3 |
+
Version: 0.2.0
|
| 4 |
+
Summary: Scaling benchmark for Hilbert-space spectral operator learning
|
| 5 |
+
Requires-Python: >=3.10
|
| 6 |
+
Description-Content-Type: text/markdown
|
| 7 |
+
License-File: LICENSE
|
| 8 |
+
Requires-Dist: numpy>=1.23
|
| 9 |
+
Requires-Dist: scikit-learn>=1.2
|
| 10 |
+
Requires-Dist: torch>=2.1
|
| 11 |
+
Requires-Dist: pandas>=1.5
|
| 12 |
+
Requires-Dist: matplotlib>=3.6
|
| 13 |
+
Requires-Dist: pytest>=7.0
|
| 14 |
+
Provides-Extra: benchmark
|
| 15 |
+
Requires-Dist: torchvision>=0.16; extra == "benchmark"
|
| 16 |
+
Dynamic: license-file
|
| 17 |
+
|
| 18 |
+
---
|
| 19 |
+
tags:
|
| 20 |
+
- benchmark
|
| 21 |
+
- pytorch
|
| 22 |
+
- spectral-learning
|
| 23 |
+
- operator-learning
|
| 24 |
+
- hilbert-space
|
| 25 |
+
- fourier-neural-operator
|
| 26 |
+
license: apache-2.0
|
| 27 |
+
---
|
| 28 |
+
# Hilbert Operator Intelligence — Scaling Benchmark
|
| 29 |
+
|
| 30 |
+
This repository turns **Operator-Based Machine Intelligence: A Hilbert Space Framework for Spectral Learning and Symbolic Reasoning** into a contributor-oriented scaling benchmark. The research program represents inputs in Hilbert spaces, learns regularized task/reasoning operators, and uses spectral bases and modulation rather than treating every transformation as an opaque dense layer.
|
| 31 |
+
|
| 32 |
+
## Why this benchmark exists
|
| 33 |
+
|
| 34 |
+
The original reproducibility code is intentionally CPU-sized (`sklearn` Digits). The next scientific question is **scaling**: do structured spectral operators retain accuracy/efficiency advantages as width, depth, data, and GPU compute increase? This repo gives contributors one command, fixed configs, machine-readable metrics, and a common result schema so runs on larger GPUs can be compared.
|
| 35 |
+
|
| 36 |
+
## Benchmark task
|
| 37 |
+
|
| 38 |
+
The primary public benchmark is CIFAR-10, with CIFAR-100 available as a harder extension. `synthetic` is included only as an offline smoke test. The model lifts images into a channel-valued function space, applies repeated truncated 2-D Fourier operators, adds a local residual nonlinearity, and classifies the pooled representation. Low-rank factorization is supported for the spectral operator tensor.
|
| 39 |
+
|
| 40 |
+
| Tier | Width | Depth | Modes | Spectral rank | Intended use |
|
| 41 |
+
|---|---:|---:|---:|---:|---|
|
| 42 |
+
| tiny | 32 | 2 | 8 | full | laptop / smoke |
|
| 43 |
+
| small | 64 | 4 | 12 | 16 | single consumer GPU |
|
| 44 |
+
| base | 128 | 6 | 16 | 32 | high-memory GPU |
|
| 45 |
+
| large | 256 | 8 | 16 | 64 | contributor scaling run |
|
| 46 |
+
|
| 47 |
+
The table defines benchmark configurations, not claimed performance. Contributors should submit the generated JSON/CSV results rather than hand-editing numbers.
|
| 48 |
+
|
| 49 |
+
## Quick start
|
| 50 |
+
|
| 51 |
+
```bash
|
| 52 |
+
python -m venv .venv
|
| 53 |
+
source .venv/bin/activate
|
| 54 |
+
pip install -e '.[benchmark]'
|
| 55 |
+
pytest -q
|
| 56 |
+
|
| 57 |
+
# Offline smoke test
|
| 58 |
+
python benchmarks/train_image.py --config configs/tiny.json --dataset synthetic \
|
| 59 |
+
--epochs 1 --train-samples 256 --test-samples 128 --image-size 16 \
|
| 60 |
+
--output results/smoke.json
|
| 61 |
+
|
| 62 |
+
# Standard GPU run
|
| 63 |
+
python benchmarks/train_image.py --config configs/base.json --dataset cifar10 \
|
| 64 |
+
--seed 7 --amp --output results/cifar10_base_s7.json
|
| 65 |
+
```
|
| 66 |
+
|
| 67 |
+
For a three-seed sweep:
|
| 68 |
+
|
| 69 |
+
```bash
|
| 70 |
+
DATASET=cifar10 ./scripts/run_scaling_sweep.sh
|
| 71 |
+
```
|
| 72 |
+
|
| 73 |
+
## What to contribute
|
| 74 |
+
|
| 75 |
+
We are specifically seeking contributors with substantial GPU compute to run **base/large and larger variants**, repeat runs across seeds, and test scaling laws over width/depth/modes/rank. Useful contributions include multi-GPU training, larger image datasets, optimized FFT kernels, low-rank/structured operator parameterizations, learned bases/wavelet packets, and fair Transformer/CNN/FNO baselines under matched parameter/FLOP budgets.
|
| 76 |
+
|
| 77 |
+
Please preserve the benchmark protocol and submit: config, seed, commit hash, hardware/GPU model, PyTorch/CUDA versions, wall-clock training time, peak GPU memory, throughput, parameter count, test accuracy and NLL. See `CONTRIBUTING.md` and `docs/SCALING_PROTOCOL.md`.
|
| 78 |
+
|
| 79 |
+
## Scientific hypotheses to test
|
| 80 |
+
|
| 81 |
+
1. **Spectral scaling:** increasing retained modes improves accuracy until task-relevant bandwidth saturates.
|
| 82 |
+
2. **Structured efficiency:** low-rank spectral operators improve accuracy per parameter and per GPU-byte at larger width.
|
| 83 |
+
3. **Depth/composition:** deeper operator composition improves representation without requiring quadratic token attention.
|
| 84 |
+
4. **Hardware fit:** FFT-heavy implementations become increasingly attractive on GPUs as spatial resolution grows.
|
| 85 |
+
|
| 86 |
+
These are hypotheses, not established results. Negative results are welcome and should be reported.
|
| 87 |
+
|
| 88 |
+
## Relationship to the manuscript
|
| 89 |
+
|
| 90 |
+
The manuscript formulates supervised learning as regularized operator estimation between Hilbert spaces, develops Fourier/wavelet spectral representations and learnable spectral modulation, and frames reasoning as operator composition. It also explicitly identifies high-dimensional scalability and GPU-friendly FFT/DWT implementations as open work. This benchmark focuses on that unresolved scaling question.
|
| 91 |
+
|
| 92 |
+
## Hugging Face collaboration call
|
| 93 |
+
|
| 94 |
+
**Call for GPU Scaling Contributors — Hilbert Operator Intelligence.** We invite researchers and engineers with high-end GPUs or clusters to run the standardized scaling matrix, add matched baselines, and explore larger operator models. The goal is a transparent community scaling study: every result should be reproducible, hardware-labeled, and accompanied by the exact config. Open an Issue titled `Scaling run: <GPU> / <tier> / <dataset>` or submit a PR containing result JSON files plus any reproducibility notes.
|
| 95 |
+
|
| 96 |
+
## Citation
|
| 97 |
+
|
| 98 |
+
See the manuscript supplied with the project and the original implementation referenced there. If you build on this benchmark, cite the manuscript and clearly distinguish new community scaling results from the manuscript's reported/comparative results.
|
hilbert_operator_intelligence_benchmark.egg-info/SOURCES.txt
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
LICENSE
|
| 2 |
+
README.md
|
| 3 |
+
pyproject.toml
|
| 4 |
+
hilbert_mi/__init__.py
|
| 5 |
+
hilbert_mi/datasets.py
|
| 6 |
+
hilbert_mi/learnable.py
|
| 7 |
+
hilbert_mi/metrics.py
|
| 8 |
+
hilbert_mi/operators.py
|
| 9 |
+
hilbert_mi/reasoning.py
|
| 10 |
+
hilbert_mi/scalable.py
|
| 11 |
+
hilbert_mi/transforms.py
|
| 12 |
+
hilbert_mi/utils.py
|
| 13 |
+
hilbert_operator_intelligence_benchmark.egg-info/PKG-INFO
|
| 14 |
+
hilbert_operator_intelligence_benchmark.egg-info/SOURCES.txt
|
| 15 |
+
hilbert_operator_intelligence_benchmark.egg-info/dependency_links.txt
|
| 16 |
+
hilbert_operator_intelligence_benchmark.egg-info/requires.txt
|
| 17 |
+
hilbert_operator_intelligence_benchmark.egg-info/top_level.txt
|
| 18 |
+
tests/test_scalable.py
|
hilbert_operator_intelligence_benchmark.egg-info/dependency_links.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
|
hilbert_operator_intelligence_benchmark.egg-info/requires.txt
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
numpy>=1.23
|
| 2 |
+
scikit-learn>=1.2
|
| 3 |
+
torch>=2.1
|
| 4 |
+
pandas>=1.5
|
| 5 |
+
matplotlib>=3.6
|
| 6 |
+
pytest>=7.0
|
| 7 |
+
|
| 8 |
+
[benchmark]
|
| 9 |
+
torchvision>=0.16
|
hilbert_operator_intelligence_benchmark.egg-info/top_level.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
hilbert_mi
|
pyproject.toml
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires=["setuptools>=67","wheel"]
|
| 3 |
+
build-backend="setuptools.build_meta"
|
| 4 |
+
[project]
|
| 5 |
+
name="hilbert-operator-intelligence-benchmark"
|
| 6 |
+
version="0.2.0"
|
| 7 |
+
description="Scaling benchmark for Hilbert-space spectral operator learning"
|
| 8 |
+
readme="README.md"
|
| 9 |
+
requires-python=">=3.10"
|
| 10 |
+
dependencies=["numpy>=1.23","scikit-learn>=1.2","torch>=2.1","pandas>=1.5","matplotlib>=3.6","pytest>=7.0"]
|
| 11 |
+
[project.optional-dependencies]
|
| 12 |
+
benchmark=["torchvision>=0.16"]
|
| 13 |
+
[tool.setuptools.packages.find]
|
| 14 |
+
include=["hilbert_mi*"]
|
requirements.txt
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
numpy>=1.23
|
| 2 |
+
scikit-learn>=1.2
|
| 3 |
+
torch>=2.0
|
| 4 |
+
pandas>=1.5
|
| 5 |
+
matplotlib>=3.6
|
| 6 |
+
pytest>=7.0
|
results/README.md
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Results
|
| 2 |
+
`smoke.json` is an implementation check, not a scientific result. Community benchmark outputs should be generated by the scripts and committed without manual metric edits.
|
results/smoke.json
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"benchmark": "hoi-image-v1",
|
| 3 |
+
"dataset": "synthetic",
|
| 4 |
+
"config": "tiny",
|
| 5 |
+
"seed": 7,
|
| 6 |
+
"epochs": 1,
|
| 7 |
+
"train_samples": 128,
|
| 8 |
+
"test_samples": 64,
|
| 9 |
+
"image_size": 16,
|
| 10 |
+
"params": 264842,
|
| 11 |
+
"device": "cpu",
|
| 12 |
+
"gpu": null,
|
| 13 |
+
"torch": "2.10.0+cpu",
|
| 14 |
+
"amp": false,
|
| 15 |
+
"train_seconds": 1.4467030469999997,
|
| 16 |
+
"samples_per_second": 88.47703767917758,
|
| 17 |
+
"peak_gpu_memory_gb": 0.0,
|
| 18 |
+
"accuracy": 0.0625,
|
| 19 |
+
"nll": 2.374997615814209
|
| 20 |
+
}
|
scripts/run_scaling_sweep.sh
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
DATASET=${DATASET:-cifar10}; SEEDS=${SEEDS:-"7 17 27"}; TIERS=${TIERS:-"tiny small base large"}
|
| 4 |
+
mkdir -p results/scaling
|
| 5 |
+
for tier in $TIERS; do for seed in $SEEDS; do
|
| 6 |
+
python benchmarks/train_image.py --config configs/${tier}.json --dataset "$DATASET" --seed "$seed" --amp --output results/scaling/${DATASET}_${tier}_s${seed}.json
|
| 7 |
+
done; done
|
| 8 |
+
python scripts/summarize_scaling.py results/scaling/*.json
|
scripts/summarize_scaling.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python
|
| 2 |
+
import json,sys,csv
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
rows=[json.load(open(p)) for p in sys.argv[1:]]
|
| 5 |
+
if not rows: raise SystemExit("Pass result JSON files")
|
| 6 |
+
cols=list(rows[0]); out=Path("results/scaling_summary.csv"); out.parent.mkdir(exist_ok=True)
|
| 7 |
+
with out.open("w",newline="") as f:
|
| 8 |
+
w=csv.DictWriter(f,fieldnames=cols); w.writeheader(); w.writerows(rows)
|
| 9 |
+
print(out)
|
tests/test_scalable.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from hilbert_mi.scalable import HilbertOperatorClassifier
|
| 3 |
+
|
| 4 |
+
def test_forward_and_backward():
|
| 5 |
+
m=HilbertOperatorClassifier(width=8,depth=1,modes=4,rank=2)
|
| 6 |
+
x=torch.randn(2,3,16,16); y=m(x)
|
| 7 |
+
assert y.shape==(2,10); y.sum().backward()
|
| 8 |
+
assert all(p.grad is not None for p in m.parameters())
|