diff --git a/.gitattributes b/.gitattributes index d203298edec602a0e646c01fa53e1e7a2084952b..79db9811852a0eca2633825759e33bfd8976dd0d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -45,3 +45,4 @@ videos/WISP_zh_mimo_voice.mp4 filter=lfs diff=lfs merge=lfs -text videos/WISP_zh_mimo_voice.wav filter=lfs diff=lfs merge=lfs -text videos/WISP_en_mimo_voice.mp4 filter=lfs diff=lfs merge=lfs -text videos/WISP_en_mimo_voice.wav filter=lfs diff=lfs merge=lfs -text +dist/wisp_har-0.2.0-cp312-cp312-linux_x86_64.whl filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md index 422fc75f69cc3f93a21b7cf426cde8f75b09b67f..bf6330c84c0b960f441768c45ac3096f62f92e68 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ python -m pip install --no-deps . wisp list ``` -Only NumPy, SciPy, scikit-learn and its runtime dependencies are installed. No PyTorch/CUDA, aeon, Mamba or baseline source download is needed. [The recorded software gate](validation/software_validation.json) passed 55 tests and verified wheel installation/import from outside the source tree using Python 3.12.13. See [environment notes](environments/README.md) and the release audit for the exact resolved environment and final regression checks. +Only NumPy, SciPy, scikit-learn and its runtime dependencies are installed. No PyTorch/CUDA, aeon, Mamba or baseline source download is needed. Installing from source compiles small optional C kernels when a C compiler is available; without one, the same code runs on slower pure-Python/NumPy paths. A prebuilt wheel for CPython 3.12 on Linux x86_64 is in `dist/`. [The software gate recorded for the first release](validation/software_validation.json) passed 55 tests and verified wheel installation/import from outside the source tree using Python 3.12.13; the current version's test suite passes with its wheel installed the same way. See [environment notes](environments/README.md) and the release audit for the exact resolved environment and final regression checks. ## Export a benchmark fold (optional) @@ -152,6 +152,47 @@ Pickle checkpoints are executable serialized objects. Load only trusted releases `select-fit` fits candidates on training participants, selects using validation only, then fits a fresh selected model on train+validation once. It does not accept test input. See [method roles and selection rule](docs/METHODS.md). +All three training commands accept the two options below. Both default to the original behaviour, so commands without them run as before. + +### Speed options + +- `--speed exact` (default) uses the original Ridge solver. Outputs are identical to the original implementation. +- `--speed fast` uses a screened Ridge solver: the regularisation strength is chosen on a 512-row subsample of the training data, then the classifier is fitted on all rows. It is faster, and results can change slightly. Only models with a Ridge classifier are affected (WISP-RC, WISP-CIS and the RC/CIS parts of the ensembles and search candidates). + +On 12 benchmark datasets (single thread), exact mode was 2.6x faster than the previous implementation with identical outputs; fast mode was about 3.6x faster. + +### HMM smoothing sequence + +The optional HMM smoother decodes windows as sequences. There are two ways to form them: + +- `--sequence subject` (default): all windows of one participant form one chain, ordered by `time_index`. This is the original behaviour and suits continuous, all-day wear. +- `--sequence recording`: each recording forms its own chain. A new recording starts whenever `time_index` goes backwards within a participant's rows. This suits data collected as separately recorded clips. The confidence scaling of classifier scores is chosen on the validation split, and smoothing is skipped when most training recordings contain a single activity, because there are then no within-recording transitions to learn. With `family-fit`, supply the validation split through `--valid`. + +```bash +wisp family-fit --method wisp_cis \ + --input inputs/adl_wrist_accel_v1/fold0/train.npz \ + --valid inputs/adl_wrist_accel_v1/fold0/valid.npz \ + --output outputs/adl_cis_fold0_seed42.pkl \ + --seed 42 --n-jobs 1 --speed fast --sequence recording --execute +``` + +The same choices in Python: + +```python +from wisp_release import build_family +from wisp_release.data import as_har_data, read_npz + +train, valid, test = (as_har_data(read_npz(f"inputs/adl_wrist_accel_v1/fold0/{part}.npz", require_y=True)) + for part in ("train", "valid", "test")) +model = build_family("wisp_cis", seed=42, speed="fast", sequence="recording") +model.fit(train.X, train.y, groups=train.subject, time_index=train.time_index, + n_classes=train.n_classes) +model.fit_confidence_scale(valid.X, valid.y) # only used with sequence="recording" +labels = model.predict(test.X, groups=test.subject, time_index=test.time_index) +``` + +`wisp_release.fit_selector(..., speed=..., sequence=...)` takes the same options. Saved metadata records both choices. Retained checkpoints keep the decoder they were fitted with; these options apply when fitting new models. Details: [method options](docs/METHODS.md#speed-and-hmm-smoothing-sequence). + These commands are separate from checkpoint inference. Paper fold checkpoints correspond to a specific dataset view, label vocabulary, fold and seed. They are not a universal cross-dataset recognizer. A new model fitted to all available participants would be a separate deployment artifact, not the original held-out evaluation checkpoint. For a paper comparison, retain the frozen dataset view, participant-disjoint splits, window/channel ordering, class axis, validation-based selection rule and sequence-decoding contract. Never choose the advertised “best” fold/seed by its held-out test score and present that selection as unbiased. The source-level saving logic and actual retained checkpoint inventory are separate evidence; [checkpoint notes](docs/CHECKPOINTS.md) explain this distinction. diff --git a/RELEASE_SUMMARY.json b/RELEASE_SUMMARY.json index e91d11e5c24da3080fe166cb6951743cd4872e84..cdbf25cd5b449ec7497b2c3500d62daa26935fef 100644 --- a/RELEASE_SUMMARY.json +++ b/RELEASE_SUMMARY.json @@ -14097,5 +14097,30 @@ }, "checkpoint_bytes": 74911659604, "remote_uploaded": false, - "publication_note": "Method-only local delivery. Missing fixed models remain missing; no research reruns." + "publication_note": "Method-only local delivery. Missing fixed models remain missing; no research reruns.", + "code_version": { + "wisp-har": "0.2.0", + "updated_at_utc": "2026-11-07T21:57:57.499881+00:00", + "wheel": "dist/wisp_har-0.2.0-cp312-cp312-linux_x86_64.whl", + "wheel_sha256": "ef623e646aacc3b8e096caa933782d9ad6d6f35a9fbae832d880d4c34f574268", + "wheel_tag": "cp312-cp312-linux_x86_64", + "replaces_wheel": "dist/wisp_har-0.1.0-py3-none-any.whl", + "options": { + "speed": [ + "exact (default)", + "fast" + ], + "sequence": [ + "subject (default)", + "recording" + ] + }, + "software_tests": { + "passed": 101, + "failed": 0, + "python": "3.12.13", + "installed_wheel_outside_source_tree": true + }, + "note": "The software_validation and model_validation entries above describe version 0.1.0 and are kept unchanged. Retained checkpoints and their validation records are not affected by the code update." + } } diff --git a/SHA256SUMS b/SHA256SUMS index 669ecea3ce2fb793bc1fbd12f35c520a6d68b59a..014cea89baa7e255d9c42282ebb5cbdcfe51ad55 100644 --- a/SHA256SUMS +++ b/SHA256SUMS @@ -1,9 +1,9 @@ 22ec17250d6dbf255f5ffe6ce9de5fd7d9752b3e7727f91ea2ccf02cad73b7a1 .dockerignore 3381754b5ce6803518720c82cdc3eaa4855cd6a2e53c0f36954fddc7e07cb448 .gitignore 59e1ab474672b657909afcb346d7c6b2fc0fc627f6518b99925779aa1892b6c4 LICENSE -875a0d43f4fc89ce3d0b62208d1c799da22beeb1dfd786d42a9ce66662d5d2af README.md -6ef0910bd45698e3d0cefd9df6b3601ac055af79e3849e9c5af180c6a0e3f0eb RELEASE_SUMMARY.json -84a9886ef4e8799ee260a4bd7b0def3591f17c4c92852f122522058615337189 SOURCE_PROVENANCE.json +2ae742dce8e2f67661b70fe5ff90c740e82e24668316d3af51c141caf5267107 README.md +43f14bd6fe7e2a93fc525a26fd95ea9cce48cd11b8ad927f1d4e0fbd2147c01c RELEASE_SUMMARY.json +5463d03fb3c8780dc87d585ab51235dde9ee17b9afb1b265dd0fbca63b08dcc3 SOURCE_PROVENANCE.json 7703e8fc152c823ae34336d8bef1cad90941ba722fa117dcf3f9a6d88cc44f38 configs/data_sources.json 4f16e9b993164120d017dea7087db47dda79515931a65b4429d7c687b3482bbc configs/datasets.json 7c486fb00a2fc0fdb82401ff8c1520d964c565cbb342fe66a0d55be6281a99ab configs/ordering/adl_wrist_accel_v1.csv.gz @@ -21,14 +21,14 @@ c717390251fa1febf14bc2a2302e16254927e8b87e9d88e9f7f9e4b471b08ebd configs/orderi a7d4461be0e1f71c4eba257d9748f469cde64c1497ab4e871890884703a93676 configs/ordering_manifest.json e676e4eb85c96eeeeb926c6982da281955f74c85a0735a96d3ee5ed9071bf623 configs/scope.json d54c76a3a7333fe2fe3d851b0ba009de0abb247a7bcf2382bdd42f43ee2d8219 configs/selector_order.json -89c1cccabaf095b66fe8d4b82a007b868316302ca613d6bfe1a9a8b2995876b0 dist/wisp_har-0.1.0-py3-none-any.whl -97ca4c8ca131fad43ddd31abdf587abc2f118eabbd6b54431975bf4f34a1716e docs/CHECKPOINTS.md +ef623e646aacc3b8e096caa933782d9ad6d6f35a9fbae832d880d4c34f574268 dist/wisp_har-0.2.0-cp312-cp312-linux_x86_64.whl +43b7f83d38c327a37006659f42037b504714cdd5f50b9a94c839f0ea8afe8fc2 docs/CHECKPOINTS.md 1ac64b1b9ed7034b5e4f17cd2ee6a6c5d8099163a8e99bf33d677536b8531730 docs/EVALUATION.md -c4f01f072b7fad37655beeedaec72dfb93b4969acb3ffea6a8d301bc3e11af32 docs/INPUTS.md -e9b4701987538a165981eec652ee5a1684071b4141ece9a3cf5f2032621b057e docs/METHODS.md +c26d62945108bcb53e7ddd1e204d1a78c279d08e8763b54ccc919225c02e6b49 docs/INPUTS.md +4ec5f88c378b240f0c8d78e8614e1dfba821a608cd10e242fd9f28005ed6fd66 docs/METHODS.md 37f564e6d4415a602cb4f320e3c68de7d41a58259e32630b23fa293fba41365c docs/MODEL_USE.md -39f0b95082c85cf6d78906736959b8735f0335f750653481b328799a731b7f53 environments/Dockerfile.cpu -15e49eedd14e7f7cfcef615c7a23838265a1029e5fb21bbbf282af7f5e8705fe environments/README.md +352e2d13ee21e1c469f60abf31ae104b5ffbf2d10d61587bb51611fe9215cb0b environments/Dockerfile.cpu +b33f43d99ba44daf9a813f2e8ff69f4b5ef3e5ae13875da1d4339ac9340c1833 environments/README.md 4176347d73ec43f141fac10cfec1d214c80c320c3f4329fbf518fec36c2ec12c environments/cpu.lock.txt 9a0752633ddcb1e528eddf119d6aeed86ec80be6b879f84f4375915d14dc27a7 environments/data.lock.txt 6007cf0450afea5cc50accf42b79e0abedb3ff680539583925b594181cadf366 environments/test.lock.txt @@ -754,11 +754,12 @@ ecbb4e620aef4f605b9a0657077e8589d0c086d0886507bd4d344048bb336d59 models/wisp_ra 57c7e0ea59741b1dc82941501812441efc800bee4b335c72b05ae7bce4cd0c54 models/wisp_random/wisdm_watch_accel_v1/fold4/seed43/model.pkl 19b3fbdc3fd2c757b3ad2c5bdd209650318a60550ee49bb3233f2bceec7a2d66 models/wisp_random/wisdm_watch_accel_v1/fold4/seed44/checkpoint.json f7822a30c9af1ab31909c8eee97cd6685b0825d809e80ae86a3378ea161df9f8 models/wisp_random/wisdm_watch_accel_v1/fold4/seed44/model.pkl -ceaa134057a0b41f3dfccdeeb9329fe938a6410f512e74e8206fa4661f51ecda pyproject.toml +38f0d87a78d4e7f9bbb92c6840f52dff9b66ca931899f442bf96491e82a23d76 pyproject.toml b9ac248916e29564a4393b8096ecb28f211449c5e35f8dff8d7602220dcb78cd results/cis_and_budget_runs.csv 66dc9ee0e1c20ab5ed2a08f27882d949aed69c47c51fd85d7b87ef6908083eb7 results/run_results.csv 918425f34a826207a18599850f01d4b0728016c7ef6bf7e5555deb49c18fccec results/search_selections.csv bfa587f5092287d068cfb96040fb3e5a1befd06133126a66addad436a46a00dd scripts/export_benchmark.py +323f2755c7698fa09b1d1a6f21ddb7b3ce73fb4efe3763539d09becef6dcb032 setup.py 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b src/wisp/__init__.py d92d2cc5a369ca5d6a3baac156355b0b21b7816778ba9e40eb0c898bafaf8354 src/wisp/ablations.py c7b9b262d9bceed5f6a31821a26aeb044350b6af759d100876b3967a36a93be5 src/wisp/cis_algorithms.py @@ -772,26 +773,61 @@ aab2b76f8dbf7d7faf9b4b1785b2596a2596ba4a805a18e0c856b0ea581ef668 src/wisp/cpu/f 1232011f6eac18d45fd13580f96e79b9b8b088f8661a863130fe7084b1e67cc9 src/wisp/cpu/probability.py a7a96dfa10ca77d95032e4e62e1a526fc53503ee4618fa6194a73a97ae523828 src/wisp/registry.py 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b src/wisp/search/__init__.py -2f2d19573a69afe12015657f13a89caf0d28466ffd703a31f2afd951c7d1e245 src/wisp/search/cascade.py -895240d4ba5b79088766a972a76112d15fa40d4e6fa4f5440bf47b6f2a0ec451 src/wisp/search/controller.py +cdaf6c03f502ced2d992dc7b6f7166fd49ecd1c30c688fa2301b28f2fc44d046 src/wisp/search/cascade.py +b244da6b12fd67c9abb3ba3aa4f7a5cef617d8b8a9e880a51ccaaa9d485d74fe src/wisp/search/controller.py 8e976b35e37534c4fd795d529d63ac01cb0e95b87df4f090791d0d6611fb08ca src/wisp/search/dataset_fingerprint.py -a038e582182aa1bed8b03a173f02b6977730dbcde07f44a3cd721f87e6d08971 src/wisp/search/grammar.py +8506aea80be6d2f59cc6aee1faff68ff281c3710579768e1fe1ce59ca197f310 src/wisp/search/grammar.py 5d0375afbb4cec325ccb0d476f27c797d0e335e6d4e292a16b9330f3565a7a44 src/wisp/search/motifs.py -654975111d32566a712dd7ae8e386c577f9fc0196dad8582939ab018f32ff8ec src/wisp/search/operator_program.py -0fb27135a136f9fc77c0bdc6facb959a85587e629e4d99c40f95ff4262507e1f src/wisp/search/posterior.py +eb8c8250bf166d62eb4c53b03253e7c2e1be8504eeb2d72f4d39eccd9e342a84 src/wisp/search/operator_program.py +c3f77ce27909620bd23e074d5dd22f1cd06a5621271cd529f7b5b417ec9e93be src/wisp/search/posterior.py 0acb26c966b72721b42f9e90183186ddb1db713a42d451d886103e0ebf7a16a6 src/wisp/search/profiles.py 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b src/wisp/utils/__init__.py ab9819981cf6f56436edada19b01cfb11d78a3af40d05504c7b9740384441daa src/wisp/utils/jsonl.py -c4c3a0ae71ab4eb449ac357d5cddaa40b897a6bdc683c5713e019f3fcc5027a2 src/wisp_release/__init__.py +6df7c45b5c4fb6578a7b155bd1cb7419e4d62aafb50fbb4c6af92bd6b4aab14b src/wisp_fast/__init__.py +6bf85ad45d36fac91f48eb88764cef64e90a9c9a9b733b768357b813dfea2904 src/wisp_fast/autocorr_lanes.h +98c62c30c9e2567b0020ed3bb52e4ac5bd6f06343af28cb4cb6d7d6ef1f024b9 src/wisp_fast/cache.py +882b278bbb512dc988540f0a17053a528289429428ae8086f0b8c72d3c0b9dd7 src/wisp_fast/cis_conv.c +2f9ed4d52f783c84185af5c131d256923fe6f0a305fd10ea77235cdbccbe7f36 src/wisp_fast/cis_intervals.c +00428cf66c52dc7b19a63f8802a41aaa0b6cf0c70f414af434e4e19fa331bea4 src/wisp_fast/decoder.py +b6d3149cee0d0628f3acef7297155b74496ead17fd9dc8a6415eb3d6f475647d src/wisp_fast/engine.py +e96864aa6831eac552d3c148fc892bddd36d0c6711c2bf6ba935cc6d34a1e7e2 src/wisp_fast/family_kernels.c +faa5d70e561acb2e667634d7868926ea3f18a671158ee2a1ca085682f2308cbd src/wisp_fast/features.py +49e1f0071038f7bcdd68fda0eec9e799a41cec8888d99a2c04681339a226b9db src/wisp_fast/features_kernels.c +8a04e5e3a05489f9999cf633f30d55dc06b8f7304fc9b776bcfccddee1a74578 src/wisp_fast/forest.py +7f2ee234c7f075b875b51e957afbab62f9033f467ee94058e88ad589f4467cc2 src/wisp_fast/forest_kernels.c +06a2c7e0dd3fa3b2ef0751b42f13d17ec3097e61cc54474bbef47159ca30bbdd src/wisp_fast/interval_levels.h +ba379d5cb1f4de8e0a2f1e2385a4a4edaef56745e814caf74c4f5f36b638d971 src/wisp_fast/moments_tiles.h +37273b0b2a5eac371d19bd8be062da0c2a1d198a987c574ce520db26fb38dbea src/wisp_fast/native.py +d04fab57d812b7d770800ab526e906dd91cfe4c0991dc126c8294f784cd98a56 src/wisp_fast/parallel.py +ad0eef8901b5a7f3ed79682204c2c9973956ac175ae3d3169bc298f757582962 src/wisp_fast/ridge_fast.py +f4a28fa322282431209d7236462f02322ecc2e9d5f20819ff97aeb25cb11d188 src/wisp_fast/search.py +ce4a6ca7cecd193852c6966e8096158f51de55e4c141ab3a43bab4bc42ca5709 src/wisp_fast/search_cache.py +c7ada30c8a31f29df5e1c4b597689999a25c79ea58b62c48204049a648744b90 src/wisp_fast/search_forest.py +7b0cf48b25cbe4dd5d2984b0a4d0748ae1820f62d1d8e8af41a9c3115732cebd src/wisp_fast/search_metrics.py +7d7ed2780f4df33a0c7e4a7814ae63b83457c8ae334fc3fd31f917d7c546ad4d src/wisp_fast/selection.py +b0d823bdd280dc8c953620129942c10dfd727e33ea33ec277e064ffcda8ef595 src/wisp_fast/shapelet_tiles.h +2da1b113cab1f80ccd51bdcdc562aa3bde143d9c4955c6c2732c0bede0e95a9b src/wisp_fast/viterbi.c +bebc1a59a528850af786a04a6d71f681950a523d7b5b0dc54da9b53ded3f3995 src/wisp_release/__init__.py 261bf3c45c2ef7f5a21ff188533aa19a46a6774e261fb6ee1d0f25cfb794b47f src/wisp_release/checkpoints.py -4d4f7106bd5eca4c8496c55b0a03c967df589cf1ec2e7f365db0ec6798c980a5 src/wisp_release/cli.py +35e8207dc015afb65c0047b5482009b321fb42540432c37961f8109ef0a47b7c src/wisp_release/cli.py 5dab92de198c7bca28fb620f830168be3bf936bb3c8a8c9ca4dd5762c6cf0a89 src/wisp_release/data.py -2f1eac83ccb9fe7c8d0e2a7520f746df7f522470af8dab65322a261acc17abcd src/wisp_release/methods.py +e5f37c055a6f46019cafdfc923b450daece29583a6cd96e97ab5c1870f66baff src/wisp_release/methods.py aa1e89dd83b7295e0dbe087b441c13bae3247a04129f4710006da9bc49386dc5 src/wisp_release/scoring.py -7077a0bfef5c9ec3371c64b0bfdef1d285ee9f9902142e2f5676d0f0462d72b6 src/wisp_release/selection.py -1a300b9ba2ac5df48ab8bf17f960d2f2c8b21e3bdf9f67fd9632901a3901fbc1 tests/test_method_conformance.py -63f924c3047ab840d14dcd1846ff97ba81ab8ca916f6fdf638945d819206fffd tests/test_release_api.py +41cefadd069009e0cd574d1915c01bbcb8dff5f061b40166f155faad414210eb src/wisp_release/selection.py +05711d66e52059a7afd5a9234f6d1f01bdbdb0c07fa6e76dd8c4c49f112c06a7 tests/test_cis_features.py +6c896f952651453af93a60751712a742f10a1482c44d59a0426aaac68e5d3db5 tests/test_conv_kernels.py +0599ed5bc6cead4c7ad0515612576af284082f76bee45e2e1f63e3edab75f60f tests/test_engine.py +21713fa476ce1c3249a1598cef53f46771b215661ee3298162f10258d500f190 tests/test_family_kernels.py +ce27602e7e335952b6e4c90b7a6f792ffe99fceca30905f6c892a544d7988786 tests/test_features_fast.py +9700b1bf67e554c25fc51944f93ff13ab8b2eafc00b82de08375000468c8b7f5 tests/test_interval_kernels.py +a39c8e5a224903779a38523c123211217569adfe33c8cb45b9e103cbaa35fa42 tests/test_method_conformance.py +7c0b0f4427d74f200584a7a554dcfedb741f55eff52113d72d758fc4efba08e2 tests/test_native_kernels.py +e76d6785da76c35d789ef47427a4f29913be7be0a7653b2cbe7dc45e90178f87 tests/test_release_api.py +168d8eeac0719a4a41f2e494203866e38cf7446b008da4f8bf319fecf91f8b54 tests/test_ridge_screened.py +ae38926198725d66a96038297b792d95bd423e4b781b3acbe479198bbf7ceaf3 tests/test_search_cache.py +5145bc9e2ccc5e3e73e0e1b9776121e52b3653a9afe1c88a0aee41e0d6677cd7 tests/test_search_shared.py c235154fbd26befec3d08b6700769487d535d121c6fb9f2e7fd0f7f7269ade9a tests/test_series_end_to_end.py +d29e2a038d29f7c7d9d58c5301a86e1beb473dbd30e7bbc84fadf8dcb79b6331 tests/test_speed_and_sequence.py 658ccfbb991aad42da596ff6c21ac322cf2cb7b3dffee829338f9ea37525e1ee validation/model_validation.json 26c2159c8c6d9620612b2821e5a7b6752d152475ceff896841e8624e0534b116 validation/software_validation.json 40b444809841d8c452ec78f18cfcd99da706affadeae624ec1efcac2d48163c6 results/external_comparison_by_dataset.csv diff --git a/SOURCE_PROVENANCE.json b/SOURCE_PROVENANCE.json index 502e189ce413555f42fd54d9f23dc9533bf52616..82045b68304f75ef7e607882585ec4bb12896546 100644 --- a/SOURCE_PROVENANCE.json +++ b/SOURCE_PROVENANCE.json @@ -1,7 +1,7 @@ { "release_name": "WISP", "author": "ZipengWu", - "method_code_origin": "previous paper-scoped WISP copy, original operations preserved", + "method_code_origin": "previous paper-scoped WISP copy, original operations preserved; version 0.2.0 replaces five search modules (cascade, controller, grammar, operator_program, posterior) with the accelerated implementation's versions, which accept an injected candidate factory, and adds the accelerated engine in src/wisp_fast", "baseline_assets_included": false, "benchmark_data_duplicated": false, "result_rows": { @@ -77,13 +77,15 @@ }, { "file": "src/wisp/search/cascade.py", - "sha256": "2f2d19573a69afe12015657f13a89caf0d28466ffd703a31f2afd951c7d1e245", - "source_sha256": "2f2d19573a69afe12015657f13a89caf0d28466ffd703a31f2afd951c7d1e245" + "sha256": "cdaf6c03f502ced2d992dc7b6f7166fd49ecd1c30c688fa2301b28f2fc44d046", + "source_sha256": "cdaf6c03f502ced2d992dc7b6f7166fd49ecd1c30c688fa2301b28f2fc44d046", + "source": "accelerated implementation" }, { "file": "src/wisp/search/controller.py", - "sha256": "895240d4ba5b79088766a972a76112d15fa40d4e6fa4f5440bf47b6f2a0ec451", - "source_sha256": "895240d4ba5b79088766a972a76112d15fa40d4e6fa4f5440bf47b6f2a0ec451" + "sha256": "b244da6b12fd67c9abb3ba3aa4f7a5cef617d8b8a9e880a51ccaaa9d485d74fe", + "source_sha256": "b244da6b12fd67c9abb3ba3aa4f7a5cef617d8b8a9e880a51ccaaa9d485d74fe", + "source": "accelerated implementation" }, { "file": "src/wisp/search/dataset_fingerprint.py", @@ -92,8 +94,9 @@ }, { "file": "src/wisp/search/grammar.py", - "sha256": "a038e582182aa1bed8b03a173f02b6977730dbcde07f44a3cd721f87e6d08971", - "source_sha256": "a038e582182aa1bed8b03a173f02b6977730dbcde07f44a3cd721f87e6d08971" + "sha256": "8506aea80be6d2f59cc6aee1faff68ff281c3710579768e1fe1ce59ca197f310", + "source_sha256": "8506aea80be6d2f59cc6aee1faff68ff281c3710579768e1fe1ce59ca197f310", + "source": "accelerated implementation" }, { "file": "src/wisp/search/motifs.py", @@ -102,13 +105,15 @@ }, { "file": "src/wisp/search/operator_program.py", - "sha256": "654975111d32566a712dd7ae8e386c577f9fc0196dad8582939ab018f32ff8ec", - "source_sha256": "654975111d32566a712dd7ae8e386c577f9fc0196dad8582939ab018f32ff8ec" + "sha256": "eb8c8250bf166d62eb4c53b03253e7c2e1be8504eeb2d72f4d39eccd9e342a84", + "source_sha256": "eb8c8250bf166d62eb4c53b03253e7c2e1be8504eeb2d72f4d39eccd9e342a84", + "source": "accelerated implementation" }, { "file": "src/wisp/search/posterior.py", - "sha256": "0fb27135a136f9fc77c0bdc6facb959a85587e629e4d99c40f95ff4262507e1f", - "source_sha256": "0fb27135a136f9fc77c0bdc6facb959a85587e629e4d99c40f95ff4262507e1f" + "sha256": "c3f77ce27909620bd23e074d5dd22f1cd06a5621271cd529f7b5b417ec9e93be", + "source_sha256": "c3f77ce27909620bd23e074d5dd22f1cd06a5621271cd529f7b5b417ec9e93be", + "source": "accelerated implementation" }, { "file": "src/wisp/search/profiles.py", @@ -218,8 +223,9 @@ }, { "file": "tests/test_method_conformance.py", - "sha256": "1a300b9ba2ac5df48ab8bf17f960d2f2c8b21e3bdf9f67fd9632901a3901fbc1", - "source_sha256": "1a300b9ba2ac5df48ab8bf17f960d2f2c8b21e3bdf9f67fd9632901a3901fbc1" + "sha256": "a39c8e5a224903779a38523c123211217569adfe33c8cb45b9e103cbaa35fa42", + "source_sha256": "1a300b9ba2ac5df48ab8bf17f960d2f2c8b21e3bdf9f67fd9632901a3901fbc1", + "modified": "internal module labels renamed; test logic unchanged" }, { "file": "scripts/export_benchmark.py", @@ -246,5 +252,107 @@ "operation": "filter WISP/CIS methods only; values unchanged" } ], - "remote_changes": "none; local preparation only" + "remote_changes": "none; local preparation only", + "accelerated_engine": { + "code_version": "0.2.0", + "files": [ + { + "file": "src/wisp_fast/__init__.py", + "sha256": "6df7c45b5c4fb6578a7b155bd1cb7419e4d62aafb50fbb4c6af92bd6b4aab14b" + }, + { + "file": "src/wisp_fast/autocorr_lanes.h", + "sha256": "6bf85ad45d36fac91f48eb88764cef64e90a9c9a9b733b768357b813dfea2904" + }, + { + "file": "src/wisp_fast/cache.py", + "sha256": "98c62c30c9e2567b0020ed3bb52e4ac5bd6f06343af28cb4cb6d7d6ef1f024b9" + }, + { + "file": "src/wisp_fast/cis_conv.c", + "sha256": "882b278bbb512dc988540f0a17053a528289429428ae8086f0b8c72d3c0b9dd7" + }, + { + "file": "src/wisp_fast/cis_intervals.c", + "sha256": "2f9ed4d52f783c84185af5c131d256923fe6f0a305fd10ea77235cdbccbe7f36" + }, + { + "file": "src/wisp_fast/decoder.py", + "sha256": "00428cf66c52dc7b19a63f8802a41aaa0b6cf0c70f414af434e4e19fa331bea4" + }, + { + "file": "src/wisp_fast/engine.py", + "sha256": "b6d3149cee0d0628f3acef7297155b74496ead17fd9dc8a6415eb3d6f475647d" + }, + { + "file": "src/wisp_fast/family_kernels.c", + "sha256": "e96864aa6831eac552d3c148fc892bddd36d0c6711c2bf6ba935cc6d34a1e7e2" + }, + { + "file": "src/wisp_fast/features.py", + "sha256": "faa5d70e561acb2e667634d7868926ea3f18a671158ee2a1ca085682f2308cbd" + }, + { + "file": "src/wisp_fast/features_kernels.c", + "sha256": "49e1f0071038f7bcdd68fda0eec9e799a41cec8888d99a2c04681339a226b9db" + }, + { + "file": "src/wisp_fast/forest.py", + "sha256": "8a04e5e3a05489f9999cf633f30d55dc06b8f7304fc9b776bcfccddee1a74578" + }, + { + "file": "src/wisp_fast/forest_kernels.c", + "sha256": "7f2ee234c7f075b875b51e957afbab62f9033f467ee94058e88ad589f4467cc2" + }, + { + "file": "src/wisp_fast/interval_levels.h", + "sha256": "06a2c7e0dd3fa3b2ef0751b42f13d17ec3097e61cc54474bbef47159ca30bbdd" + }, + { + "file": "src/wisp_fast/moments_tiles.h", + "sha256": "ba379d5cb1f4de8e0a2f1e2385a4a4edaef56745e814caf74c4f5f36b638d971" + }, + { + "file": "src/wisp_fast/native.py", + "sha256": "37273b0b2a5eac371d19bd8be062da0c2a1d198a987c574ce520db26fb38dbea" + }, + { + "file": "src/wisp_fast/parallel.py", + "sha256": "d04fab57d812b7d770800ab526e906dd91cfe4c0991dc126c8294f784cd98a56" + }, + { + "file": "src/wisp_fast/ridge_fast.py", + "sha256": "ad0eef8901b5a7f3ed79682204c2c9973956ac175ae3d3169bc298f757582962" + }, + { + "file": "src/wisp_fast/search.py", + "sha256": "f4a28fa322282431209d7236462f02322ecc2e9d5f20819ff97aeb25cb11d188" + }, + { + "file": "src/wisp_fast/search_cache.py", + "sha256": "ce4a6ca7cecd193852c6966e8096158f51de55e4c141ab3a43bab4bc42ca5709" + }, + { + "file": "src/wisp_fast/search_forest.py", + "sha256": "c7ada30c8a31f29df5e1c4b597689999a25c79ea58b62c48204049a648744b90" + }, + { + "file": "src/wisp_fast/search_metrics.py", + "sha256": "7b0cf48b25cbe4dd5d2984b0a4d0748ae1820f62d1d8e8af41a9c3115732cebd" + }, + { + "file": "src/wisp_fast/selection.py", + "sha256": "7d7ed2780f4df33a0c7e4a7814ae63b83457c8ae334fc3fd31f917d7c546ad4d" + }, + { + "file": "src/wisp_fast/shapelet_tiles.h", + "sha256": "b0d823bdd280dc8c953620129942c10dfd727e33ea33ec277e064ffcda8ef595" + }, + { + "file": "src/wisp_fast/viterbi.c", + "sha256": "2da1b113cab1f80ccd51bdcdc562aa3bde143d9c4955c6c2732c0bede0e95a9b" + } + ], + "note": "Accelerated engine from the authors' fast implementation, reduced to the modules used by the public API (exact and screened Ridge solvers, subject and recording HMM smoothing sequences, accelerated search)." + } } diff --git a/dist/wisp_har-0.1.0-py3-none-any.whl b/dist/wisp_har-0.1.0-py3-none-any.whl deleted file mode 100644 index 3a3fc56f5f2d0edd61684be4b66f02ed70559973..0000000000000000000000000000000000000000 Binary files a/dist/wisp_har-0.1.0-py3-none-any.whl and /dev/null differ diff --git a/dist/wisp_har-0.2.0-cp312-cp312-linux_x86_64.whl b/dist/wisp_har-0.2.0-cp312-cp312-linux_x86_64.whl new file mode 100644 index 0000000000000000000000000000000000000000..174c5d9f489b309d4a1e736d56f1148962bc908f --- /dev/null +++ b/dist/wisp_har-0.2.0-cp312-cp312-linux_x86_64.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef623e646aacc3b8e096caa933782d9ad6d6f35a9fbae832d880d4c34f574268 +size 605026 diff --git a/docs/CHECKPOINTS.md b/docs/CHECKPOINTS.md index 1612ab8d900b52286d37208cc349bb10b8c3e12b..875941fa1a1b541ed07a5022c44f3eef71be58b7 100644 --- a/docs/CHECKPOINTS.md +++ b/docs/CHECKPOINTS.md @@ -44,7 +44,7 @@ Use compatible Python/NumPy/SciPy/scikit-learn versions and preserve checkpoint- The inference CLI can read an explicitly supplied metadata JSON. If omitted, it looks for a sibling `checkpoint.json`, then `.metadata.json`. Supported validation fields are `input_shape` (time/channel shape, optionally prefixed by the original window count) and `label_names`. A SHA256 stored in a sidecar is descriptive integrity information; the CLI does not treat that file as an independently trusted source for the `--sha256` check. -`wisp family-fit` writes `.metadata.json` with the declared method, input shape, global label names, model seed, fitted-data scope, HMM setting, parameters and output checksum. `wisp select-fit` also writes `.selection.json` for its validation choice. `wisp search` writes `selected_model.metadata.json` for the controller's selected model. A generic new fit does not invent a dataset/fold identity that the caller did not provide. These newly fitted artifacts must be kept distinct from retained paper models. +`wisp family-fit` writes `.metadata.json` with the declared method, input shape, global label names, model seed, fitted-data scope, HMM setting, speed/Ridge solver, HMM smoothing sequence and confidence scaling (see [method options](METHODS.md#speed-and-hmm-smoothing-sequence)), parameters and output checksum. `wisp select-fit` also writes `.selection.json` for its validation choice. `wisp search` writes `selected_model.metadata.json` for the controller's selected model. Models newly fitted by these commands are `wisp_fast.engine.FastFamily` objects; retained paper checkpoints keep their original classes and are loaded and predicted exactly as before. A generic new fit does not invent a dataset/fold identity that the caller did not provide. These newly fitted artifacts must be kept distinct from retained paper models. The manifest inventory is read-only and does not need to deserialize every model. Actual loading/inference verification, hashes, installation and any refit must run on allocated compute nodes. Coverage is reported by the root release audit, not by this explanatory document. diff --git a/docs/INPUTS.md b/docs/INPUTS.md index 11a31d330b803fddc230de663aba582a8badcde8..c547774bbf1150adeca990f41e2ce1b981efd394 100644 --- a/docs/INPUTS.md +++ b/docs/INPUTS.md @@ -42,6 +42,8 @@ For HMM-enabled models, the checkpoint contains a fitted transition matrix and c For new recordings, group identifiers should distinguish independent recording sequences. Do not connect different participants, unrelated sessions or discontinuous recordings into one artificial sequence. Do not shuffle windows, batch a chronological sequence into unrelated chunks or omit metadata and still claim the same HMM prediction protocol. Offline sequence decoding can depend on later windows within a sequence; it is not presented as a causal, streaming recognizer. +Models fitted with `--sequence recording` split each group into recordings wherever `time_index` goes backwards in the supplied row order, and decode each recording separately; keep the original row order for such models. See [method options](METHODS.md#speed-and-hmm-smoothing-sequence). + A direct/argmax model can be run without sequence metadata where supported by its recorded configuration. Disabling HMM or omitting groups is a protocol change, not an equivalent reproduction of an HMM checkpoint. ## Example schema diff --git a/docs/METHODS.md b/docs/METHODS.md index 8cad996617f079be2ffd1945302561201ae5e1b2..0c630a8463deece4bf900026e532d6d30d83bc07 100644 --- a/docs/METHODS.md +++ b/docs/METHODS.md @@ -28,6 +28,22 @@ Implementation coverage, retained checkpoint coverage and verified inference cov Each candidate is fitted on training only and scored on validation. Selection uses, in order, higher macro-F1, higher full-vocabulary worst-class F1, smaller serialized model size using the preserved `estimate_pickle_size_mb` helper with Python's default pickle protocol (protocol 4 in the pinned Python 3.12 runtime), then the canonical order SS, RC, SO, CSE, GIS, CIS, ESE. A fresh winning model is fitted on train+validation once. Test data are not accepted or used by this command. -The selector's size measurement and the new fitted output's serialization protocol are separate: newly saved models use the highest protocol, while the historical size tie-break uses the default protocol. Renaming Python namespaces can itself affect serialized byte counts. The wrapper preserves the helper and ordering, but does not claim byte-identical model sizes or guarantee that an exact size-only tie remains unchanged across renamed namespaces/environments. +The selector's size measurement and the new fitted output's serialization protocol are separate: newly saved models use the highest protocol, while the historical size tie-break uses the default protocol. Size is only needed when two candidates tie on both validation scores, so it is measured only for tied candidates (reported as `null` otherwise), on the reference estimator fitted to the training split, as in the original selector. Renaming Python namespaces can itself affect serialized byte counts. The wrapper preserves the helper and ordering, but does not claim byte-identical model sizes or guarantee that an exact size-only tie remains unchanged across renamed namespaces/environments. With `--execute`, the command saves the fitted output model, `.metadata.json` and `.selection.json`. Without `--execute`, it describes the requested plan only. A newly fitted selector model is a new artifact; it does not fill an original missing paper checkpoint without being labelled as a refit. + +## Speed and HMM smoothing sequence + +`wisp family-fit`, `wisp select-fit` and `wisp search` (and `wisp_release.build_family` / `fit_selector`) take two options. Both default to the original behaviour. + +| Option | Value | Meaning | +| --- | --- | --- | +| `--speed` | `exact` (default) | Original Ridge solver. Outputs are identical to the original implementation. | +| | `fast` | Screened Ridge: the regularisation strength is chosen on a 512-row training subsample, then the Ridge classifier is fitted on all rows. Faster; results can change slightly. Only models with a Ridge classifier change: WISP-RC, WISP-CIS, the RC/CIS members of WISP-CSE and WISP-ESE, and RC/CIS candidates in search. | +| `--sequence` | `subject` (default) | All windows of one participant form one HMM chain, ordered by time index. This is the original behaviour. | +| | `recording` | Each recording forms its own chain. Within one participant's rows, in the order supplied, a new recording starts whenever the time index goes backwards. The confidence scaling of classifier scores (one factor dividing the log-probabilities) is chosen on the validation split by negative log-likelihood. Smoothing is skipped when more than half of the training recordings contain a single activity, because there are then no within-recording transitions to learn. | + +The two sequence options suit different data. `subject` fits continuous wear, where all of a participant's windows belong to one timeline. `recording` fits data collected as separately recorded clips, where consecutive clips of one participant are not connected in time. + +With `--sequence recording`, `family-fit` needs `--valid` to choose the confidence scaling. `select-fit` chooses it on validation for every candidate, and search does the same for every candidate it evaluates. A final train+validation refit reuses the factor chosen for the corresponding train-only model. Saved metadata records `speed`, `sequence` and the chosen `confidence_scale`. + diff --git a/environments/Dockerfile.cpu b/environments/Dockerfile.cpu index c8de837f196576d64bb855c3c3c52c2616bb8b04..ac703749ff91e3a14881a7fcb69b28c90fc26367 100644 --- a/environments/Dockerfile.cpu +++ b/environments/Dockerfile.cpu @@ -6,9 +6,13 @@ ENV OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 WORKDIR /opt/wisp COPY environments/cpu.lock.txt /tmp/wisp-cpu.lock.txt RUN python -m pip install --no-cache-dir -r /tmp/wisp-cpu.lock.txt -COPY pyproject.toml README.md LICENSE ./ +COPY pyproject.toml setup.py README.md LICENSE ./ COPY src ./src -RUN python -m pip install --no-cache-dir --no-deps . +# A C compiler builds the optional native kernels; without it the slower Python paths are used. +RUN apt-get update && apt-get install -y --no-install-recommends gcc libc6-dev \ + && python -m pip install --no-cache-dir "setuptools==80.9.0" "wheel==0.45.1" \ + && python -m pip install --no-cache-dir --no-deps --no-build-isolation . \ + && apt-get purge -y gcc libc6-dev && apt-get autoremove -y && rm -rf /var/lib/apt/lists/* # Mount downloaded checkpoints and inputs; do not bake 75 GB of models into an # environment image. No baseline, CUDA or dataset packages are installed here. diff --git a/environments/README.md b/environments/README.md index 45265acea2eac7fb96a518eb5b1c1fad2722abb7..9f3939bbab77df0e592bd3a7958c147cb52a936a 100644 --- a/environments/README.md +++ b/environments/README.md @@ -28,7 +28,9 @@ python -m pip install -r environments/cpu.lock.txt python -m pip install --no-deps . ``` -For release checks, install `environments/test.lock.txt` in the same environment. The recorded software gate passed 55 tests and verified the installed wheel outside the source tree. The model gate loaded and smoke-tested all 360 retained checkpoints in this environment; six ADL-Wrist models also passed full real-test prediction equivalence. See [software validation](../validation/software_validation.json), [model validation](../validation/model_validation.json) and the root release audit for final regression checks and full coverage boundaries. +Installing from source compiles optional C kernels for the accelerated engine when a C compiler is available (on the cluster, for example after `module load GCC`); without a compiler the build still succeeds and uses slower Python/NumPy paths. Alternatively install the prebuilt CPython 3.12 Linux x86_64 wheel from `dist/` with `python -m pip install --no-deps dist/wisp_har-0.2.0-cp312-cp312-linux_x86_64.whl`. + +For release checks, install `environments/test.lock.txt` in the same environment. The software gate recorded for the first release passed 55 tests and verified the installed wheel outside the source tree; the current version's suite is run the same way. The model gate loaded and smoke-tested all 360 retained checkpoints in this environment; six ADL-Wrist models also passed full real-test prediction equivalence. See [software validation](../validation/software_validation.json), [model validation](../validation/model_validation.json) and the root release audit for final regression checks and full coverage boundaries. The optional `data.lock.txt` adds pandas, Datasets, Hugging Face Hub and PyArrow for the benchmark NPZ export script. These versions match the existing data/analysis environment pins. It is not needed when the input NPZ files are already available, and it adds no baseline implementation. The small lock files pin their listed distributions; the release audit's resolved package record is needed to identify transitive dependencies installed in a particular validation environment. diff --git a/pyproject.toml b/pyproject.toml index 23f2e12fdbdcb815fb7ae42f1f8408477bcd6837..49afe124350239b8fd2002a14fe28dfe2f846e86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "wisp-har" -version = "0.1.0" +version = "0.2.0" description = "WISP representations, search strategies and trained wrist activity recognition pipelines" readme = "README.md" requires-python = ">=3.12.13,<3.13" @@ -21,6 +21,9 @@ wisp = "wisp_release.cli:main" [tool.setuptools.packages.find] where = ["src"] +[tool.setuptools.package-data] +wisp_fast = ["*.c", "*.h"] + [tool.pytest.ini_options] testpaths = ["tests"] addopts = "-ra" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..492567a441ebf50fe32d067ab06a82139c680a1e --- /dev/null +++ b/setup.py @@ -0,0 +1,19 @@ +"""Optional C kernels of the accelerated engine (pure-Python fallbacks are used when a build fails).""" +import os +from setuptools import Extension, setup + +FLAGS = ["/O2", "/fp:strict"] if os.name == "nt" else ["-O3", "-ffp-contract=off"] +SRC = "src/wisp_fast/" + + +def ext(name, sources, depends=()): + return Extension(f"wisp_fast.{name}", [SRC + s for s in sources], depends=[SRC + d for d in depends], + extra_compile_args=FLAGS, optional=True) + + +setup(ext_modules=[ + ext("_kernels", ["features_kernels.c", "viterbi.c", "forest_kernels.c"]), + ext("_family_kernels", ["family_kernels.c"], ["autocorr_lanes.h", "shapelet_tiles.h", "moments_tiles.h"]), + ext("_cis_intervals", ["cis_intervals.c"], ["interval_levels.h"]), + ext("_cis_conv", ["cis_conv.c"]), +]) diff --git a/src/wisp/search/cascade.py b/src/wisp/search/cascade.py index 3ec8fddc93b3c717dffc54397faee20110a21542..7c15a1666219b9ef81423a0bdf7867f2fe282100 100644 --- a/src/wisp/search/cascade.py +++ b/src/wisp/search/cascade.py @@ -8,7 +8,7 @@ from wisp.core.data import HARData from wisp.core.metrics import classification_report_dict, score_from_metrics from wisp.cpu.algorithms import build_cpu_estimator, estimate_pickle_size_mb from wisp.search.grammar import CPU_FAMILIES, CandidateSpec, normalize_spec -from wisp.search.operator_program import TemporalProgram +from wisp.search.operator_program import candidate_certificate @dataclass class StageResult: @@ -47,7 +47,7 @@ def validate_candidate_spec(spec: CandidateSpec) -> StageResult: return StageResult('spec_validity', False, {}, 'n_shapelets outside allowed range') if 'symbolic_bins' in p and (not 2 <= int(p['symbolic_bins']) <= 32): return StageResult('spec_validity', False, {}, 'symbolic_bins outside allowed range') - return StageResult('spec_validity', True, TemporalProgram.from_candidate(spec).certificate()) + return StageResult('spec_validity', True, candidate_certificate(spec)) def _axis_sign_flip(X: np.ndarray) -> np.ndarray: out = X.copy() @@ -71,35 +71,53 @@ def _noise(X: np.ndarray) -> np.ndarray: return X + rng.normal(0.0, 1.0, size=X.shape) * scale STRESS_TRANSFORMS = {'axis_sign_flip': _axis_sign_flip, 'time_shift': _time_shift, 'channel_dropout': _channel_dropout, 'sensor_noise': _noise} -def evaluate_candidate_cascade(spec: CandidateSpec, train: HARData, valid: HARData, *, cost_weight: float=0.01, robustness_weight: float=0.1, max_stress_transforms: int=4) -> CascadeResult: +def evaluate_candidate_cascade(spec: CandidateSpec, train: HARData, valid: HARData, *, cost_weight: float=0.01, robustness_weight: float=0.1, max_stress_transforms: int=4, estimator_factory=build_cpu_estimator, stress_inputs=None, reporter=classification_report_dict, stress_predictor=None, stress_gate=None) -> CascadeResult: spec = normalize_spec(spec) stages: list[StageResult] = [] valid_stage = validate_candidate_spec(spec) stages.append(valid_stage) if not valid_stage.passed: return CascadeResult(False, -1000000000.0, stages, {'candidate': spec.to_dict(), 'score': -1000000000.0}) - model = build_cpu_estimator(spec.to_dict()) + model = estimator_factory(spec.to_dict()) t0 = time.perf_counter() model.fit(train.X, train.y, groups=train.subject, time_index=train.time_index, n_classes=train.n_classes) fit_s = time.perf_counter() - t0 t1 = time.perf_counter() pred = model.predict(valid.X, groups=valid.subject, time_index=valid.time_index) pred_s = time.perf_counter() - t1 - size_mb = estimate_pickle_size_mb(model) - metrics = classification_report_dict(valid.y, pred, fit_seconds=fit_s, predict_seconds=pred_s, model_size_mb=size_mb, label_names=valid.label_names) + size_mb = estimate_pickle_size_mb(model) if stress_gate is None else None + metrics = reporter(valid.y, pred, fit_seconds=fit_s, predict_seconds=pred_s, model_size_mb=size_mb, label_names=valid.label_names) complexity = len(spec.operators) + float(spec.params.get('n_kernels', 0)) / 256.0 + float(spec.params.get('n_shapelets', 0)) / 64.0 metrics['score'] = score_from_metrics(metrics, cost_weight=cost_weight, complexity=complexity) stages.append(StageResult('validation', True, metrics)) + if stress_gate is not None: + certificate = stress_gate(metrics, []) + if certificate is not None: + stages.append(StageResult('stress_upper_bound', True, certificate)) + return CascadeResult(False, -1e9, stages, + {**metrics,'candidate':spec.to_dict(),'prune_certificate':certificate}) + metrics['model_size_mb'] = estimate_pickle_size_mb(model) stress_reports: dict[str, Any] = {} stress_scores = [] + batched = stress_predictor(model) if stress_predictor is not None else None for i, (name, fn) in enumerate(STRESS_TRANSFORMS.items()): if i >= max_stress_transforms: break - Xs = fn(valid.X) - ps = model.predict(Xs, groups=valid.subject, time_index=valid.time_index) - mr = classification_report_dict(valid.y, ps, label_names=valid.label_names) + if batched is None: + Xs = stress_inputs[name] if stress_inputs is not None and name in stress_inputs else fn(valid.X) + ps = model.predict(Xs, groups=valid.subject, time_index=valid.time_index) + else: + ps = batched[name] + mr = reporter(valid.y, ps, label_names=valid.label_names) stress_reports[name] = {'macro_f1': mr['macro_f1'], 'worst_class_f1': mr['worst_class_f1'], 'balanced_accuracy': mr['balanced_accuracy']} stress_scores.append(float(mr['macro_f1'])) + if stress_gate is not None and batched is None and len(stress_scores) CandidateSpec: - bases = initial_candidates(seed=int(rng.integers(0, 2 ** 31 - 1)), include_v2=self.profile.include_v2_families, route='cpu') - spec = CandidateSpec.from_dict(bases[int(rng.integers(0, len(bases)))].to_dict()) + template_seed=int(rng.integers(0, 2 ** 31 - 1)) + template=self._templates[int(rng.integers(0,len(self._templates)))] + params=dict(template.params);params['seed']+=template_seed + spec=CandidateSpec(template.name,params,list(template.operators),template.route,template.parent_id,template.mutation) depth = int(rng.integers(1, 5)) for _ in range(depth): - spec = mutate_candidate(spec, rng=rng, mutation_bias=None, motif_weights=None if fingerprint is None else fingerprint.motif_weights()) + spec = mutate_candidate(spec, rng=rng, mutation_bias=None, motif_weights=self._motif_weights) spec.parent_id = None spec.mutation = f'random_depth_{depth}' constrained = self._constrain_spec(spec) @@ -146,8 +148,11 @@ class WISPSearchController: return constrained def _record(self, *, db_path: Path, candidate_id: str, generation: int, spec: CandidateSpec, status: str, result: dict[str, Any], fingerprint: DatasetFingerprint | None, parent_id: str | None) -> dict[str, Any]: - row = {'id': candidate_id, 'generation': int(generation), 'parent_id': parent_id, 'island': island_key(spec), 'status': status, 'profile': self.profile.to_dict(), 'dataset_fingerprint': None if fingerprint is None else fingerprint.to_dict(), 'motif_weights': None if fingerprint is None else fingerprint.motif_weights(), 'operator_certificate': TemporalProgram.from_candidate(spec).certificate(), 'result': result, 'spec': spec.to_dict()} - append_jsonl(db_path, row) + row = {'id': candidate_id, 'generation': int(generation), 'parent_id': parent_id, 'island': island_key(spec), 'status': status, 'profile': self._profile_dict, 'dataset_fingerprint': self._fingerprint_dict, 'motif_weights': self._motif_weights, 'operator_certificate': candidate_certificate(spec), 'result': result, 'spec': spec.to_dict()} + if self._ledger is None: + append_jsonl(db_path,row) + else: + self._ledger.write(json.dumps(row,ensure_ascii=False,sort_keys=True)+'\n') return row def _final_refit(self, best: dict[str, Any], out_dir: Path) -> dict[str, Any] | None: @@ -170,6 +175,15 @@ class WISPSearchController: return metrics def run(self) -> dict[str, Any]: + self._ledger=None + try: + return self._run() + finally: + if self._ledger is not None: + self._ledger.close() + self._ledger=None + + def _run(self) -> dict[str, Any]: out_dir = Path(self.output) out_dir.mkdir(parents=True, exist_ok=True) db_path = out_dir / 'program_database.jsonl' @@ -181,9 +195,14 @@ class WISPSearchController: return old if db_path.exists() and db_path.stat().st_size: raise RuntimeError(f'Refusing to append to non-empty search database {db_path}') + self._ledger=db_path.open('a',encoding='utf-8') rng = np.random.default_rng(self.seed) fingerprint = compute_dataset_fingerprint(self.train, seed=self.seed) if self.profile.use_fingerprint else None family_transfer, mutation_transfer = ({}, {}) + self._templates=initial_candidates(seed=0,include_v2=self.profile.include_v2_families,route='cpu') + self._motif_weights=None if fingerprint is None else fingerprint.motif_weights() + self._fingerprint_dict=None if fingerprint is None else fingerprint.to_dict() + self._profile_dict=self.profile.to_dict() posterior = MotifPosterior() archive: list[CandidateSpec] = [] population: list[tuple[str, CandidateSpec, dict[str, Any]]] = [] @@ -213,12 +232,15 @@ class WISPSearchController: while evaluated < budget and generation < int(self.generations): generation += 1 remaining = budget - evaluated - proposal_count = min(max(self.population - max(2, self.population // 3), 1), remaining) + width = max(self.population - max(2, self.population // 3), 1) + coalesce = getattr(self, "coalesce_random", False) + proposal_count = min((self.generations-generation+1)*width if coalesce else width, remaining) proposals: list[tuple[str, str | None, CandidateSpec]] = [] attempts = 0 mutation_bias = posterior.mutation_bias(fingerprint, motif_mode=self.profile.motif_mode, transfer_bias=mutation_transfer) if self.profile.use_posterior else mutation_transfer if self.profile.transfer_prior else {} for disabled in self.profile.disabled_mutations: mutation_bias.pop(disabled, None) + draw=prepare_parents(population,use_islands=self.profile.use_islands) if self.profile.search_strategy=='evolution' else None while len(proposals) < proposal_count: attempts += 1 if attempts > proposal_count * 100: @@ -227,10 +249,10 @@ class WISPSearchController: parent_id = None child = self._random_spec(rng, fingerprint) else: - parent_id, parent, parent_result = sample_parent(population, rng, use_islands=self.profile.use_islands, score_key='breeding_score') + parent_id, parent, parent_result = draw(rng) child = None if child is None: - child = mutate_candidate(parent, rng=rng, parent_id=parent_id, mutation_bias=mutation_bias, motif_weights=None if fingerprint is None else fingerprint.motif_weights()) + child = mutate_candidate(parent, rng=rng, parent_id=parent_id, mutation_bias=mutation_bias, motif_weights=self._motif_weights) child = self._constrain_spec(child) if child is None: continue @@ -239,14 +261,15 @@ class WISPSearchController: continue seen.add(key) index = evaluated + len(proposals) - proposals.append((_stable_id(seed=self.seed, generation=generation, index=index, spec=child), parent_id, child)) + logical_generation = generation + len(proposals)//width if coalesce else generation + proposals.append((_stable_id(seed=self.seed, generation=logical_generation, index=index, spec=child), parent_id, child)) evaluated_results = self._evaluate_many([spec for _, _, spec in proposals]) new_items: list[tuple[str, CandidateSpec, dict[str, Any]]] = [] - for (cid, parent_id, spec), (status, result) in zip(proposals, evaluated_results): + for position, ((cid, parent_id, spec), (status, result)) in enumerate(zip(proposals, evaluated_results)): novelty = posterior.novelty_bonus(spec, archive) if self.profile.use_novelty and status == 'ok' else 0.0 result['novelty_bonus'] = float(novelty) result['breeding_score'] = float(result.get('predictive_utility', -1000000000.0)) + float(novelty) - row = self._record(db_path=db_path, candidate_id=cid, generation=generation, spec=spec, status=status, result=result, fingerprint=fingerprint, parent_id=parent_id) + row = self._record(db_path=db_path, candidate_id=cid, generation=generation+position//width if coalesce else generation, spec=spec, status=status, result=result, fingerprint=fingerprint, parent_id=parent_id) rows.append(row) archive.append(spec) if status == 'ok': @@ -262,6 +285,9 @@ class WISPSearchController: population = (population + new_items)[-int(self.population):] if not population: raise RuntimeError(f'No accepted candidate after generation {generation}') + self._ledger.flush() + if getattr(self,'retain_rows',False): + self.evaluated_rows=rows ok_rows = [row for row in rows if row['status'] == 'ok'] if not ok_rows: raise RuntimeError('Search completed without an accepted candidate') diff --git a/src/wisp/search/grammar.py b/src/wisp/search/grammar.py index 70d0525146bef56c5b20f4253ad40cf6d38e5ce4..797e25ae252ae6dd3447a02817cabf1690579b63 100644 --- a/src/wisp/search/grammar.py +++ b/src/wisp/search/grammar.py @@ -1,6 +1,8 @@ # Paper-scoped implementation; see SOURCE_PROVENANCE.json. from __future__ import annotations -from dataclasses import asdict, dataclass, field +from dataclasses import dataclass, field +from copy import deepcopy +from functools import lru_cache from typing import Any, Literal import numpy as np from wisp.search.motifs import FAMILY_MOTIF_WEIGHTS, MOTIF_TO_MUTATION_PRIOR, family_motif_match @@ -16,7 +18,8 @@ class CandidateSpec: mutation: str | None = None def to_dict(self) -> dict[str, Any]: - return asdict(self) + return dict(name=self.name, params=deepcopy(self.params), operators=list(self.operators), + route=self.route, parent_id=self.parent_id, mutation=self.mutation) @staticmethod def from_dict(d: dict[str, Any]) -> 'CandidateSpec': @@ -34,6 +37,15 @@ def initial_cpu_candidates(seed: int=42, *, include_v2: bool=True) -> list[Candi base.extend([CandidateSpec(name='invariant_interval_shape_forest', params={'classifier': 'extratrees', 'use_hmm': True, 'segment_sizes': (4, 8, 16), 'symbolic_bins': 6, 'n_shapelets': 32, 'seed': seed + 6}, operators=['zscore', 'magnitude', 'jerk', 'gram_eig', 'dyadic_intervals', 'quantile_distribution', 'random_shapelet', 'extratrees', 'hmm']), CandidateSpec(name='state_rocket_interval_hmm', params={'n_kernels': 384, 'n_intervals': 24, 'classifier': 'ridge', 'use_hmm': True, 'seed': seed + 7}, operators=['zscore', 'random_conv', 'ppv_pool', 'dyadic_intervals', 'quantile_distribution', 'symbolic_transition', 'ridge', 'hmm']), CandidateSpec(name='posterior_cpu_ensemble', params={'use_hmm': True, 'seed': seed + 8, 'weights': (0.4, 0.35, 0.25)}, operators=['posterior_weighting', 'random_conv', 'invariant_physics', 'symbolic_transition', 'probability_ensemble', 'hmm'])]) return base +# Mutation picks one template. Build its immutable baseline once, then copy its +# public fields and add the requested seed offset, preserving the RNG stream. +_TEMPLATES = initial_cpu_candidates(0) +_FIRST = {t.name:next(i for i,c in enumerate(_TEMPLATES) if c.name==t.name) for t in _TEMPLATES} + +def _template(index, seed): + t=_TEMPLATES[index] + return CandidateSpec(t.name, {**t.params,'seed':t.params['seed']+seed}, list(t.operators)) + def initial_candidates(seed: int=42, *, include_v2: bool=True, route: Route='cpu', population: int | None=None) -> list[CandidateSpec]: if route != 'cpu': raise ValueError('Only CPU search is released') @@ -61,21 +73,30 @@ def _replace_classifier_ops(ops: list[str], classifier: str) -> list[str]: def _weighted_choice(items: list[str], bias: dict[str, float] | None, rng: np.random.Generator) -> str: if not bias: - return str(rng.choice(items)) - weights = np.asarray([max(float(bias.get(item, 0.0)), 0.0) + 1e-06 for item in items], dtype=np.float64) - weights /= weights.sum() - return str(rng.choice(items, p=weights)) + return items[int(rng.integers(len(items)))] + weights=_mutation_weights(tuple(bias.get(item,0.) for item in items)) + return items[int(rng.choice(len(items),p=weights))] + +@lru_cache(maxsize=128) +def _mutation_weights(values): + weights=np.asarray([max(float(v),0.)+1e-6 for v in values]) + return weights/weights.sum() + +@lru_cache(maxsize=32) +def _family_weights(families,weights): + motif_weights=dict(weights) + scores=np.asarray([max(family_motif_match(f,motif_weights).score,0.)+.1 for f in families]) + return scores/scores.sum() def _family_choice(route: Route, motif_weights: dict[str, float] | None, rng: np.random.Generator) -> str: families = CPU_FAMILIES if route == 'cpu' else GPU_FAMILIES if not motif_weights: - return str(rng.choice(families)) - scores = np.asarray([max(family_motif_match(family, motif_weights).score, 0.0) + 0.1 for family in families], dtype=np.float64) - scores /= scores.sum() - return str(rng.choice(families, p=scores)) + return families[int(rng.integers(len(families)))] + p=_family_weights(families,tuple(motif_weights.items())) + return families[int(rng.choice(len(families),p=p))] def mutate_cpu_candidate(parent: CandidateSpec, *, rng: np.random.Generator, parent_id: str | None=None, mutation_bias: dict[str, float] | None=None, motif_weights: dict[str, float] | None=None) -> CandidateSpec: - child = normalize_spec(CandidateSpec.from_dict(parent.to_dict())) + child = normalize_spec(parent) ops = list(child.operators) params = dict(child.params) mutation = _weighted_choice(list(CPU_MUTATIONS), mutation_bias, rng) @@ -110,7 +131,7 @@ def mutate_cpu_candidate(parent: CandidateSpec, *, rng: np.random.Generator, par if child.name == 'state_rocket_interval_hmm': params['n_intervals'] = int(rng.choice([8, 16, 24, 32, 48])) else: - child = CandidateSpec.from_dict(initial_cpu_candidates(seed=int(rng.integers(0, 10000)))[7].to_dict()) + child = _template(7,int(rng.integers(0,10000))) params, ops = (dict(child.params), list(child.operators)) elif mutation == 'change_shapelets': if child.name == 'invariant_interval_shape_forest': @@ -119,23 +140,22 @@ def mutate_cpu_candidate(parent: CandidateSpec, *, rng: np.random.Generator, par values = [8, 16, 24] params['n_shapelets'] = int(rng.choice(values)) else: - child = CandidateSpec.from_dict(initial_cpu_candidates(seed=int(rng.integers(0, 10000)))[6].to_dict()) + child = _template(6,int(rng.integers(0,10000))) params, ops = (dict(child.params), list(child.operators)) elif mutation == 'reweight_ensemble': if child.name != 'posterior_cpu_ensemble': - child = CandidateSpec.from_dict(initial_cpu_candidates(seed=int(rng.integers(0, 10000)))[8].to_dict()) + child = _template(8,int(rng.integers(0,10000))) params, ops = (dict(child.params), list(child.operators)) params['weights'] = tuple((float(x) for x in rng.dirichlet(np.ones(3)).tolist())) elif mutation == 'insert_invariance_motif': - child = CandidateSpec.from_dict(initial_cpu_candidates(seed=int(rng.integers(0, 10000)))[6].to_dict()) + child = _template(6,int(rng.integers(0,10000))) params, ops = (dict(child.params), list(child.operators)) elif mutation == 'insert_state_motif': - child = CandidateSpec.from_dict(initial_cpu_candidates(seed=int(rng.integers(0, 10000)))[7].to_dict()) + child = _template(7,int(rng.integers(0,10000))) params, ops = (dict(child.params), list(child.operators)) elif mutation == 'switch_family': family = _family_choice('cpu', motif_weights, rng) - templates = [s for s in initial_cpu_candidates(seed=int(rng.integers(0, 10000))) if s.name == family] - child = CandidateSpec.from_dict((templates[0] if templates else initial_cpu_candidates(seed=int(rng.integers(0, 10000)))[0]).to_dict()) + child = _template(_FIRST[family],int(rng.integers(0,10000))) params, ops = (dict(child.params), list(child.operators)) params['seed'] = int(params.get('seed', 42)) + int(rng.integers(1, 100000)) return CandidateSpec(child.name, params=params, operators=ops, route='cpu', parent_id=parent_id, mutation=mutation) diff --git a/src/wisp/search/operator_program.py b/src/wisp/search/operator_program.py index c8ff40d441f431346da09a66483d43786b69396a..2b6dc4ee0c6b3400c2e850c3ec1d0dafbac1ae70 100644 --- a/src/wisp/search/operator_program.py +++ b/src/wisp/search/operator_program.py @@ -2,6 +2,7 @@ from __future__ import annotations from dataclasses import asdict, dataclass, field from typing import Any +from functools import lru_cache import numpy as np from wisp.search.grammar import CandidateSpec from wisp.search.motifs import canonicalize_tags, family_motif_match @@ -34,7 +35,7 @@ class TemporalProgram: @classmethod def from_candidate(cls, spec: CandidateSpec) -> 'TemporalProgram': - nodes = [OPERATOR_LIBRARY.get(op, OperatorNode(op, 'unknown', cost=1.0)) for op in spec.operators] + nodes = [OPERATOR_LIBRARY[op] if op in OPERATOR_LIBRARY else OperatorNode(op, 'unknown', cost=1.0) for op in spec.operators] return cls(name=spec.name, route=spec.route, nodes=nodes, params=dict(spec.params), parent_id=spec.parent_id, mutation=spec.mutation) def to_candidate(self) -> CandidateSpec: @@ -89,8 +90,27 @@ class TemporalProgram: return {k: float(min(v, 1.0)) for k, v in score.items()} def jaccard_novelty(a: CandidateSpec, b: CandidateSpec) -> float: - A = set(a.operators) - B = set(b.operators) - if not A and (not B): - return 0.0 - return 1.0 - len(A & B) / max(len(A | B), 1) + return _distance(tuple(a.operators),tuple(b.operators)) + + +@lru_cache(maxsize=4096) +def _certificate(name, route, nodes): + return TemporalProgram(name, route, list(nodes)).certificate() + + +def candidate_certificate(spec): + """Reuse statistics of immutable nodes; returned nested containers are private.""" + nodes=tuple(OPERATOR_LIBRARY.get(op) or OperatorNode(op,'unknown') for op in spec.operators) + cached=_certificate(spec.name,spec.route,nodes) + return {k:(v.copy() if isinstance(v,(dict,list)) else v) for k,v in cached.items()} + + +@lru_cache(maxsize=2048) +def _opset(ops): + return frozenset(ops) + + +@lru_cache(maxsize=8192) +def _distance(a,b): + A,B=_opset(a),_opset(b) + return 1.0-len(A & B)/len(A | B) if A or B else 0. diff --git a/src/wisp/search/posterior.py b/src/wisp/search/posterior.py index e7ac0a3ca8ebfbba40e9d30af0d4be469f833cf0..79e855708095b7da01016604814a3113dcc12601 100644 --- a/src/wisp/search/posterior.py +++ b/src/wisp/search/posterior.py @@ -8,7 +8,7 @@ import numpy as np from wisp.search.dataset_fingerprint import DatasetFingerprint from wisp.search.grammar import CandidateSpec, initial_candidates from wisp.search.motifs import family_motif_match, mutation_prior_from_motifs -from wisp.search.operator_program import TemporalProgram, jaccard_novelty +from wisp.search.operator_program import candidate_certificate, jaccard_novelty from wisp.utils.jsonl import read_jsonl @dataclass @@ -26,7 +26,7 @@ class MotifPosterior: self.operator_scores[op].append(score) if spec.mutation: self.mutation_scores[str(spec.mutation)].append(score) - cert = TemporalProgram.from_candidate(spec).certificate() + cert = candidate_certificate(spec) for motif in cert.get('canonical_motifs', []): self.motif_scores[str(motif)].append(score) self.n_records += 1 @@ -91,7 +91,7 @@ class MotifPosterior: return float(weight * np.mean([jaccard_novelty(spec, other) for other in archive[-50:]])) def island_key(spec: CandidateSpec) -> str: - cert = TemporalProgram.from_candidate(spec).certificate() + cert = candidate_certificate(spec) families = cert.get('family_trace', []) counts = Counter(families) top_family = counts.most_common(1)[0][0] if counts else spec.name @@ -101,18 +101,22 @@ def island_key(spec: CandidateSpec) -> str: motif = motifs[0] if motifs else 'untagged' return f'{spec.route}:{top_family}:{cost_bin}:{motif}' -def sample_parent(population: list[tuple[str, CandidateSpec, dict[str, Any]]], rng: np.random.Generator, *, use_islands: bool=True, score_key: str='breeding_score') -> tuple[str, CandidateSpec, dict[str, Any]]: +def prepare_parents(population, *, use_islands=True, score_key='breeding_score'): + """Build the rank lists once for the lifetime of one fixed generation.""" if not population: raise ValueError('population must be non-empty') - if not use_islands: - members = sorted(population, key=lambda x: float(x[2].get(score_key, -1000000000.0)), reverse=True) - rank = int(min(len(members) - 1, rng.geometric(0.55) - 1)) - return members[rank] - islands: dict[str, list[tuple[str, CandidateSpec, dict[str, Any]]]] = defaultdict(list) + islands=defaultdict(list) for item in population: - islands[island_key(item[1])].append(item) - keys = list(islands) - key = min(keys, key=lambda k: len(islands[k])) if rng.random() < 0.25 else str(rng.choice(keys)) - members = sorted(islands[key], key=lambda x: float(x[2].get(score_key, -1000000000.0)), reverse=True) - rank = int(min(len(members) - 1, rng.geometric(0.55) - 1)) - return members[rank] + islands[island_key(item[1]) if use_islands else None].append(item) + for members in islands.values(): + members.sort(key=lambda x: float(x[2].get(score_key,-1000000000.0)),reverse=True) + keys=list(islands) + smallest=min(keys,key=lambda k:len(islands[k])) + def draw(rng): + key=(smallest if rng.random()<0.25 else str(rng.choice(keys))) if use_islands else None + members=islands[key] + return members[min(len(members)-1,int(rng.geometric(0.55))-1)] + return draw + +def sample_parent(population,rng,*,use_islands=True,score_key='breeding_score'): + return prepare_parents(population,use_islands=use_islands,score_key=score_key)(rng) diff --git a/src/wisp_fast/__init__.py b/src/wisp_fast/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..06cf0d557b89638254650b18db7d915e24fa18f4 --- /dev/null +++ b/src/wisp_fast/__init__.py @@ -0,0 +1,2 @@ +"""Accelerated CPU engine for the WISP method family (fixed families, selection and search).""" +__version__ = "0.2.0" diff --git a/src/wisp_fast/autocorr_lanes.h b/src/wisp_fast/autocorr_lanes.h new file mode 100644 index 0000000000000000000000000000000000000000..ed2aff833b8d2aaf81119df7f11a5d432b6b6dc0 --- /dev/null +++ b/src/wisp_fast/autocorr_lanes.h @@ -0,0 +1,54 @@ +/* Each lane owns one window. The single-channel reduction follows the + contiguous NumPy tree, while strided multi-channel sums stay serial. */ +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) && !defined(_WIN32) +#include +#define WISP_ACF_LANES 1 +__attribute__((target("avx512f"),always_inline)) +static inline __m512d acf_term(const double *a,const double *b,int j){ + __m512d u=_mm512_loadu_pd(a+j*8); + return b?_mm512_mul_pd(u,_mm512_loadu_pd(b+j*8)):u; +} +__attribute__((target("avx512f"))) +static __m512d acf_sum8(const double *a,const double *b,int n,int paired){ + __m512d sum=_mm512_setzero_pd(); + if(!paired || n<8){ + if(paired)sum=_mm512_set1_pd(-0.); + for(int j=0;j128){ + int cut=n/16*8; + return _mm512_add_pd(acf_sum8(a,b,cut,1),acf_sum8(a+8*cut,b?b+8*cut:NULL,n-cut,1)); + } + __m512d s[8]; + for(int q=0;q<8;q++)s[q]=acf_term(a,b,q); + int j=8; + for(;j+8<=n;j+=8)for(int q=0;q<8;q++)s[q]=_mm512_add_pd(s[q],acf_term(a,b,j+q)); + sum=_mm512_add_pd(_mm512_add_pd(_mm512_add_pd(s[0],s[1]),_mm512_add_pd(s[2],s[3])), + _mm512_add_pd(_mm512_add_pd(s[4],s[5]),_mm512_add_pd(s[6],s[7]))); + for(;j0 && lag= 2 and X.flags.c_contiguous and + all(getattr(b,"zscore",False) and hasattr(b,"transform_normalized") for b in blocks)) + if shared: + from wisp.cpu.features import _as_3d + from .features import _normalize + z = _normalize(_as_3d(X)) + return [b.transform_normalized(z) for b in blocks] + return [block.transform(X) for block in blocks] + directory, limit = config + signal = hashlib.sha256(np.ascontiguousarray(X).view(np.uint8)).digest() + features = [] + for block in blocks: + # feature_names_ is display metadata produced during transform. Every + # fit-dependent quantity, including shapelet values, remains in the key. + state = {k:v for k,v in vars(block).items() if k not in ("feature_names_","_conv_plan","_quant_plan")} + key = joblib.hash((type(block).__module__, type(block).__qualname__, state, + X.dtype.str,X.shape,X.strides,signal),hash_name="sha1") + path = directory / (key+".npy") + try: + arr = np.load(path, allow_pickle=False, mmap_mode="r") + except FileNotFoundError: + arr = block.transform(X) + if arr.nbytes <= limit: + tmp = None + try: + with tempfile.NamedTemporaryFile(dir=directory, delete=False) as stream: + tmp = stream.name + np.save(stream, arr, allow_pickle=False) + os.replace(tmp,path) + finally: + if tmp and os.path.exists(tmp): + os.unlink(tmp) + else: + if hasattr(block, "feature_names_") and not block.feature_names_: + block.transform(X[:1]) + features.append(arr) + return features diff --git a/src/wisp_fast/cis_conv.c b/src/wisp_fast/cis_conv.c new file mode 100644 index 0000000000000000000000000000000000000000..ea336b22d0e1345d3999fa81420fbcc4881fc4e8 --- /dev/null +++ b/src/wisp_fast/cis_conv.c @@ -0,0 +1,828 @@ +/* Packed random convolution, preserving the tap accumulation order. The + multichannel exact path reuses the BLAS already linked into NumPy's + core, including its choice of fused operations. */ +#define _GNU_SOURCE +#define PY_SSIZE_T_CLEAN +#include +#include +#include +#include +#include +#include +#ifndef _WIN32 +#include +#endif + +typedef void (*gemv64_t)(int,int,int64_t,int64_t,double,const double*,int64_t, + const double*,int64_t,double,double*,int64_t); +static gemv64_t np_gemv; +static int attempted; +static int bind_numpy(void) { + if (!attempted) { + attempted = 1; +#ifndef _WIN32 + PyObject *core = PyImport_ImportModule("numpy._core._multiarray_umath"); + if (!core) { PyErr_Clear(); return 0; } + PyObject *file = PyModule_GetFilenameObject(core); + Py_DECREF(core); + if (file) { + const char *path = PyUnicode_AsUTF8(file); + void *h = path ? dlopen(path, RTLD_LAZY | RTLD_LOCAL) : NULL; + if (h) { + void *symbol = dlsym(h, "scipy_cblas_dgemv64_"); + if (symbol) memcpy(&np_gemv, &symbol, sizeof np_gemv); + /* The NumPy module keeps this dependency loaded. */ + dlclose(h); + } + Py_DECREF(file); + } + if (PyErr_Occurred()) PyErr_Clear(); +#endif + } + return np_gemv != NULL; +} +static PyObject *has_blas(PyObject *self,PyObject *args) { + return PyBool_FromLong(bind_numpy()); +} +/* This order is specific to the inspected NumPy OpenBLAS SkylakeX kernel + shipped with the pinned Linux x86-64 wheel. Python additionally verifies + the library's full SHA-256. Other builds use actual calls to GEMV. */ +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) && !defined(_WIN32) +#include +#define HAVE_FAST_X86 1 +/* Four vectors share weight broadcasts. Keep pertap contributions in four + registers and only materialize responses once, after the last tap. */ +__attribute__((target("avx2,fma"))) +static void skylakex_project(double *r,const double *base,Py_ssize_t lda,Py_ssize_t frames, + const double *weights,int c,int64_t len,int64_t dilation) { + Py_ssize_t j=0; + for (;j+16<=frames;j+=16) { + __m256d acc[4]={_mm256_setzero_pd(),_mm256_setzero_pd(),_mm256_setzero_pd(),_mm256_setzero_pd()}; + for(int h=0;h1)skylakex_project(r+j,base+j,lda,frames-j,weights,c,len,dilation); + else for(;j=*maximum)*maximum=peak; + } + } + for(;j0.;*special|=isnan(v); + if(v>=*maximum)*maximum=v; + } + total+=v; + } + return total; +} +__attribute__((target("avx512f,popcnt"))) +static double reduce_window(double *r,Py_ssize_t n,double shift,int central, + Py_ssize_t *pos,double *maximum,int *special) { + if(n<=128)return leaf_reduce(r,n,shift,central,pos,maximum,special); + Py_ssize_t m=(n/2)/8*8; + double lo=reduce_window(r,m,shift,central,pos,maximum,special); + return lo+reduce_window(r+m,n-m,shift,central,pos,maximum,special); +} + +/* A SIMD lane owns an independent window. This allows reductions to follow + the reference's exact time order without horizontal-vector reductions. */ +__attribute__((target("avx512f"),always_inline)) +static inline __m512d batch_sum(const double *r,int n){ + if(n<8){ + __m512d acc=_mm512_set1_pd(-0.); + for(int j=0;jgeom,b->geom,sizeof a->geom); +} +__attribute__((target("avx512f,fma"),always_inline)) +static inline void pair_kernel(const double *s0,const double *s1,const double *s2,const double *w,const double *w2, + double bias,double bias2,int c,int len,int dil,int nf,double *r,double *r2,double *out,double *out2,Py_ssize_t jump){ + int main=nf/8*8,j=0; + for(;j1){p=_mm512_fmadd_pd(a,wa,b*wb);z=_mm512_fmadd_pd(a,xa,b*xb);} + if(c==3){p+=e*wc;z+=e*xc;}v[q]+=p;u[q]+=z; + } + } + for(int q=0;q<8;++q){_mm512_storeu_pd(r+8*(j+q),v[q]+bias);_mm512_storeu_pd(r2+8*(j+q),u[q]+bias2);} + } + for(;j1){ + if(j<((nf/4)*4)){p=_mm512_fmadd_pd(a,_mm512_set1_pd(w[3*h]),b*w[3*h+1]);z=_mm512_fmadd_pd(a,_mm512_set1_pd(w2[3*h]),b*w2[3*h+1]);} + else {p=_mm512_fmadd_pd(b,_mm512_set1_pd(w[3*h+1]),p);z=_mm512_fmadd_pd(b,_mm512_set1_pd(w2[3*h+1]),z);} + } + if(c>2){ + if(j<((nf/4)*4)){p+=e*w[3*h+2];z+=e*w2[3*h+2];} + else{p=_mm512_fmadd_pd(e,_mm512_set1_pd(w[3*h+2]),p);z=_mm512_fmadd_pd(e,_mm512_set1_pd(w2[3*h+2]),z);} + } + v+=p;u+=z; + } + _mm512_storeu_pd(r+j*8,v+bias);_mm512_storeu_pd(r2+j*8,u+bias2); + } + finish_projection(r,nf,out,jump);finish_projection(r2,nf,out2,jump); +} + +__attribute__((target("avx512f,fma"),always_inline)) +static inline void batch_kernel(const double *s0,const double *s1,const double *s2, + const double *w,double bias,int c,int len,int dil,int nf, + double *r,double *out,Py_ssize_t outjump){ + int main=nf/8*8,j=0; + for(;j1)p=_mm512_fmadd_pd(_mm512_loadu_pd(s1+off),_mm512_set1_pd(w[3*h+1]),p); + if(c>2)p=_mm512_fmadd_pd(_mm512_loadu_pd(s2+off),_mm512_set1_pd(w[3*h+2]),p); + v=_mm512_add_pd(v,p); + } + _mm512_storeu_pd(r+j*8,_mm512_add_pd(v,_mm512_set1_pd(bias))); + } + finish_projection(r,nf,out,outjump); +} +__attribute__((target("avx512f,fma"))) +static void batch_bank(const double *x,Py_ssize_t ch,int t,const int64_t *rows,const double *weights, + const double *bias,Py_ssize_t k,int maxlen,double *out,double *packed,double *r,const BankItem *items){ + for(Py_ssize_t c=0;c1?packed+8*t*s[4]:a,*e=c>2?packed+8*t*s[5]:a; + if(item+1=16?0xffff:(1u<2)v=_mm512_fmadd_ps(_mm512_loadu_ps(z+off),w2,v); + acc[q]=_mm512_add_ps(acc[q],v); + } + } + } + for(int q=0;q<4;q++){ + __m512 v=_mm512_add_ps(acc[q],_mm512_set1_ps(bias)); + _mm512_storeu_ps(scratch+j+16*q,v); + __mmask16 mask=masks[q]; + mx=_mm512_mask_max_ps(mx,mask,mx,v); + count+=__builtin_popcount((unsigned)(mask&_mm512_cmp_ps_mask(v,_mm512_setzero_ps(),_CMP_GT_OQ))); + } + } + /* The sixteen-window path sums in time order. Preserve that order for + residual windows to make predictions independent of input chunking. */ + float sum=0.f,ss=0.f; + for(int j=0;j1?buf+s[4]*pitch:a,*e=c>2?buf+s[5]*pitch:a; +#define WK(C) small_kernel(a,b,e,w+i*3*maxlen,C,len,d,nf,(float)bias[i],scratch,out+4*i) + if(c==1)WK(1);else if(c==2)WK(2);else WK(3); +#undef WK + } +} +/* Window lanes reuse each coefficient over 16 windows by 8 positions. + Pool each finished tile directly, retaining only the response scratch + required for centered variance. */ +__attribute__((target("avx512f,fma"),always_inline)) +static inline void small_lanes(const float *a,const float *b,const float *e, + const float *w,int c,int len,int d,int nf,float bias,float *work,double *out,int stride){ + __m512 hi=_mm512_set1_ps(-INFINITY),sum=_mm512_setzero_ps(),pos=sum; + __m512 one=_mm512_set1_ps(1.),zero=_mm512_setzero_ps(),v_bias=_mm512_set1_ps(bias); + for(int j=0;j1?buf+pitch*s[4]:a,*e=c>2?buf+pitch*s[5]:a; +#define SB(C) small_lanes(a,b,e,w+i*3*maxlen,C,len,d,nf,(float)bias[i],scratch,out+4*i,4*k) + if(c==1)SB(1);else if(c==2)SB(2);else SB(3); +#undef SB + } +} +static int can_fuse(void){ + if(!bind_numpy() || !__builtin_cpu_supports("avx2") || !__builtin_cpu_supports("fma"))return 0; + Dl_info info; + if(!dladdr((void*)np_gemv,&info))return 0; + void *handle=dlopen(info.dli_fname,RTLD_LAZY|RTLD_LOCAL); + if(!handle)return 0; + const char *(*core)(void)=NULL; + void *fn=dlsym(handle,"scipy_openblas_get_corename64_"); + if(fn)memcpy(&core,&fn,sizeof core); + int ok=core && !strcmp(core(),"SkylakeX"); + dlclose(handle); + return ok; +} +#else +#define HAVE_FAST_X86 0 +#endif +static PyObject *fused_library(PyObject *self,PyObject *args) { +#if HAVE_FAST_X86 + Dl_info info; + if(can_fuse() && dladdr((void*)np_gemv,&info)) + return PyUnicode_DecodeFSDefault(info.dli_fname); +#endif + Py_RETURN_NONE; +} +static int view(PyObject *obj, Py_buffer *v, int rank, int writable, int integer) { + int flags=PyBUF_FORMAT|PyBUF_C_CONTIGUOUS|(writable?PyBUF_WRITABLE:0); + if (PyObject_GetBuffer(obj,v,flags)<0) return -1; + int fmt=integer ? (!strcmp(v->format,"q") || !strcmp(v->format,"l")) + : !strcmp(v->format,"d"); + if (!fmt || v->ndim!=rank || v->itemsize!=8 || (uintptr_t)v->buf%8) { + PyErr_SetString(PyExc_ValueError,"expected aligned contiguous int64 or float64 array of specified rank"); + return -1; + } + return 0; +} +/* Eight independent partial sums and a balanced combination reproduce the + NumPy pairwise reduction tree used by contiguous float64 sums. */ +static double pair_sum(const double *x, Py_ssize_t n) { + if(n<8) { + double s=-0.; + for(Py_ssize_t i=0;i128) { + Py_ssize_t m=(n/2)/8*8; + return pair_sum(x,m)+pair_sum(x+m,n-m); + } + double a[8]; + memcpy(a,x,sizeof a); + Py_ssize_t i=8; + for(;i+8<=n;i+=8) for(int j=0;j<8;j++) a[j]+=x[i+j]; + double s=((a[0]+a[1])+(a[2]+a[3]))+((a[4]+a[5])+(a[6]+a[7])); + for(;i1)v=_mm512_add_pd(v,_mm512_mul_pd(_mm512_maskz_loadu_pd(mask,a+b),w1)); + if(c>2)v=_mm512_add_pd(v,_mm512_mul_pd(_mm512_maskz_loadu_pd(mask,a+g),w2)); + return v; +} + +__attribute__((target("avx512f"))) +static void wide_phase_project(double *r,const double *phase,Py_ssize_t pitch, + Py_ssize_t stride,Py_ssize_t nf,const int64_t *s, + const double *weights){ + const int len=(int)s[0],d=(int)s[1],c=(int)s[2]; + const double *x=phase+s[3]*stride*pitch; + const Py_ssize_t b=c>1?(s[4]-s[3])*stride*pitch:0; + const Py_ssize_t g=c>2?(s[5]-s[3])*stride*pitch:0; + const double *tap[32]; + /* Divide once for the entire filter, including non-power-of-two strides. */ + const Py_ssize_t delta=d%stride, jump=delta*pitch+d/stride; + Py_ssize_t residue=0,off=0,j=0; + for(int h=0;h=stride){off+=1-stride*pitch;residue-=stride;} + } + for(;j+32<=nf;j+=32){ + __m512d a0=_mm512_setzero_pd(),a1=a0,a2=a0,a3=a0; + for(int h=0;h=8)?255:(__mmask8)((1u<<(nf-j))-1u); + __m512d sum=_mm512_setzero_pd(); + for(int h=0;h1?(s[4]-s[3])*stride*pitch:0; + const Py_ssize_t g=c>2?(s[5]-s[3])*stride*pitch:0; + memset(r,0,nf*sizeof(double)); + for(int h=0;h8192 || n<1 || ch<1 || k<1 || m.shape[1]!=6 || + w.shape[0]!=k || w.shape[2]!=3 || maxlen<1 || b.shape[0]!=k || + y.shape[0]!=n || y.shape[1]!=k || y.shape[2]!=4 || stride<1 || stride>t || + (!approximate && stride!=1) || (low && (approximate || stride!=1))) { + PyErr_SetString(PyExc_ValueError,"invalid bank dimensions or stride"); goto finish; + } + Py_buffer *inputs[]={&x,&m,&w,&b}; + for(int i=0;i<4;i++) { + uintptr_t p=(uintptr_t)inputs[i]->buf,q=(uintptr_t)y.buf; + if(plen) { + PyErr_SetString(PyExc_ValueError,"output overlaps input"); goto finish; + } + } + for(Py_ssize_t z=0;zmaxlen || d<1 || len-1>(t-1)/d || c<1 || c>3) { + PyErr_SetString(PyExc_ValueError,"invalid kernel geometry"); goto finish; + } + if(!approximate && c>1 && (!bind_numpy() || t-(len-1)*d<=1)) { + PyErr_SetString(PyExc_ValueError,"unsupported exact BLAS geometry"); goto finish; + } + for(int j=0;j=ch || (j && meta[3+j]<=meta[2+j])) { + PyErr_SetString(PyExc_ValueError,"invalid kernel channels"); goto finish; + } + } +#if HAVE_FAST_X86 + int wide = __builtin_cpu_supports("avx512f") && __builtin_cpu_supports("popcnt"); + fused = fused && !approximate && (__builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma")) && t>=16; +#endif + work=PyMem_Malloc(5*t*sizeof(double)); + if (!work) { PyErr_NoMemory(); goto finish; } + if(low){ + /* Unusual custom coefficients or unnormalized signals can exceed FP32. + The bound covers convolution, reduction and centered squared sums. */ + double xmax=0.,wmax=0.,bmax=0.; + const double *values[3]={x.buf,w.buf,b.buf}; + Py_ssize_t sizes[3]={x.len/8,w.len/8,b.len/8}; + double *maxima[3]={&xmax,&wmax,&bmax}; + for(int j=0;j<3;j++)for(Py_ssize_t i=0;i*maxima[j])*maxima[j]=v; + } + if(xmax>FLT_MAX || wmax>FLT_MAX || bmax>FLT_MAX || + 3.*maxlen*xmax*wmax+bmax>sqrt((double)FLT_MAX/(16.*t)))low=0; + } + if(low){ +#if HAVE_FAST_X86 + if(!wide || !__builtin_cpu_supports("fma")){ +#endif + PyErr_SetString(PyExc_ValueError,"conv32 requires an AVX-512F/FMA/POPCNT CPU");goto finish; +#if HAVE_FAST_X86 + } + if((size_t)(ch+1)>SIZE_MAX/((size_t)t+64)/(16*sizeof(float))){ + PyErr_NoMemory();goto finish; + } + small=PyMem_Calloc(((size_t)ch+1)*(t+64)*(n>=16?16:1),sizeof(float));smallw=PyMem_Malloc(w.len/2); + if(!small||!smallw){PyErr_NoMemory();goto finish;} + for(Py_ssize_t i=0;i1 && ch<=64){ + phase=PyMem_Malloc(ch*stride*pitch*sizeof(double)); + if(!phase){PyErr_NoMemory();goto finish;} + } + Py_ssize_t first=0,block=phase?1:12; +#if HAVE_FAST_X86 + BankItem *items=NULL; + if(!low && wide && fused && n>=8 && ch<=8 && t<=256){ + bt=PyMem_Malloc((ch+2)*t*8*sizeof(double));if(!bt){PyErr_NoMemory();goto finish;} + items=PyMem_Malloc(k*sizeof(*items));if(!items){PyErr_NoMemory();goto finish;} + for(Py_ssize_t z=0;z1?t*(s[4]-a0):t; + Py_ssize_t nf=1+(t-1-(len-1)*d)/stride; + const double *base=X+(i*ch+a0)*t; + if (!phase && c>2 && s[5]-s[4]!=s[4]-a0) { + for(int j=0;j=16) { + int ix=len==7?0:len==9?1:len==11?2:len==15?3:-1; + if(ix>=0)strips[c-1][ix](r,base,lda,(int)nf,W+z*maxlen*3,(int)d); + else wide_project(r,base,lda,nf,W+z*maxlen*3,c,len,d); + } else if(fused && c>1) { + skylakex_project(r,base,lda,nf,W+z*maxlen*3,c,len,d); + } else +#endif + { + memset(r,0,nf*sizeof(double)); + for(int64_t h=0;h=16) { + int special=0; + double mu=reduce_window(r,nf,bias[z],0,&pos,&mx,&special)/nf; + if(special || mx==0.){ + mx=-INFINITY; + for(Py_ssize_t j=0;j=mx || isnan(r[j]))mx=r[j]; + } + double *dst=Y+(i*k+z)*4; + dst[0]=(double)pos/nf;dst[1]=mx;dst[2]=mu; + dst[3]=sqrt(reduce_window(r,nf,mu,1,&pos,&mx,&special)/nf); + continue; + } +#endif + for(Py_ssize_t j=0;j0; + if(r[j]>=mx || isnan(r[j])) mx=r[j]; + } + double mu=pair_sum(r,nf)/nf; + for(Py_ssize_t j=0;j +#include +#include +#include + +/* All arrays originate inside our Python wrapper. Enforce contiguous native + buffers, dtype, rank, extents and index ranges before native indexing. */ +static int view(PyObject *obj, Py_buffer *v, int rank, char type, int write) { + if (PyObject_GetBuffer(obj, v, PyBUF_C_CONTIGUOUS|PyBUF_FORMAT|(write?PyBUF_WRITABLE:0)) < 0) return 0; + int integer = type=='q'; + if (v->ndim!=rank || v->itemsize!=8 || !v->format || + (integer ? strcmp(v->format,"q") && strcmp(v->format,"l") : strcmp(v->format,"d")) || + (uintptr_t)v->buf % 8) { + PyErr_SetString(PyExc_ValueError,"invalid contiguous buffer type or rank");return 0; + } + return 1; +} +static int separate_output(const Py_buffer *views,int output){ + uintptr_t dst=(uintptr_t)views[output].buf; + for(int i=0;i=src ? dst-src<(uintptr_t)views[i].len : src-dst<(uintptr_t)views[output].len)){ + PyErr_SetString(PyExc_ValueError,"output overlaps an input"); + return 0; + } + } + return 1; +} +/* ord is the sorted list of time positions. Select interval membership in + rank order; one packed value per rank, no per-interval indirect gathers. */ +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) && !defined(_WIN32) +#include +__attribute__((target("avx512f,popcnt"))) +static Py_ssize_t filter_wide(const int64_t *ord,const double *rankval,double *out, + Py_ssize_t n,Py_ssize_t a,Py_ssize_t b) { + Py_ssize_t j=0,count=0; + __m512i lo=_mm512_set1_epi64(a),hi=_mm512_set1_epi64(b); + for(;j+8<=n;j+=8){ + __m512i ix=_mm512_loadu_si512(ord+j); + __mmask8 mask=_mm512_cmpge_epi64_mask(ix,lo)&_mm512_cmplt_epi64_mask(ix,hi); + _mm512_mask_compressstoreu_pd(out+count,mask,_mm512_loadu_pd(rankval+j)); + count+=__builtin_popcount((unsigned)mask); + } + for(;j=a && ord[j]=cumulative[w+1])w++; + uint64_t flag=_pdep_u64((uint64_t)1<<(lo-cumulative[w]),bits[w]); + double x=rankval[64*w+__builtin_ctzll(flag)],y=x; + if(lo+1=.5?y-(y-x)*(1.-f):x+(y-x)*f); + } + return 1; +} + + +/* Across-window SIMD leaves every within-interval sum strictly serial. + A small [time][8 windows] tile avoids a gather per scalar visit. */ +__attribute__((target("avx512f"))) +static void numeric_eight(const double *x,const int64_t *bounds,double *out,double *tile, + Py_ssize_t t,Py_ssize_t c,Py_ssize_t m,int width,int turbo){ + for(Py_ssize_t ch=0;chth[l]; + hist[a]++;if(j)hist[k+prev*k+a]++;prev=a; + } +} +#define CAN_FILTER (__builtin_cpu_supports("avx512f") && __builtin_cpu_supports("popcnt")) +#endif + +/* Bucket by value then insertion-sort locally. Equal finite values may be + permuted, but ordered value statistics do not depend on tied time positions. + Defer signed zero / extreme exponents to the NumPy ordering implementation. */ +static PyObject *rank_short(PyObject *self,PyObject *args) { + PyObject *xo,*oo;Py_buffer v[2]={{0}};int ok=1; + if(!PyArg_ParseTuple(args,"OO",&xo,&oo))return NULL; + if(!view(xo,v,3,'d',0)||!view(oo,v+1,3,'q',1)||!separate_output(v,1))goto done; + Py_ssize_t n=v[0].shape[0],t=v[0].shape[1],c=v[0].shape[2]; + if(t<1||t>256||c<1||v[1].shape[0]!=n||v[1].shape[1]!=c||v[1].shape[2]!=t){ + PyErr_SetString(PyExc_ValueError,"invalid short-rank geometry");goto done; + } + const double *x=v[0].buf;int64_t *out=v[1].buf; + Py_BEGIN_ALLOW_THREADS + for(Py_ssize_t i=0;ok && ihi)hi=z;vals[j]=z; + } + if(!ok)break; + if(hi==lo){for(int j=0;j=t)k=(int)t-1; + key[j]=(uint8_t)k;edges[k+1]++; + } + for(int j=1;j<=t;++j)edges[j]+=edges[j-1]; + for(int j=0;jbegin && vals[idx[h-1]]>vals[id]){idx[h]=idx[h-1];--h;}idx[h]=id; + } + begin=end; + } + } + Py_END_ALLOW_THREADS +done: + for(int j=0;j<2;++j)if(v[j].obj)PyBuffer_Release(v+j); + if(PyErr_Occurred())return NULL; + return PyBool_FromLong(ok); +} + +#include "interval_levels.h" +static PyObject *intervals(PyObject *self,PyObject *args) { + PyObject *obj[5]; int turbo; + Py_buffer v[5]={{0}}; + if(!PyArg_ParseTuple(args,"OOOOOi",&obj[0],&obj[1],&obj[2],&obj[3],&obj[4],&turbo)) return NULL; + if(!view(obj[0],v+0,3,'d',0)||!view(obj[1],v+1,2,'q',0)|| + !view(obj[2],v+2,3,'q',0)||!view(obj[3],v+3,1,'d',0)||!view(obj[4],v+4,3,'d',1)) goto finish; + if(!separate_output(v,4))goto finish; + Py_ssize_t n=v[0].shape[0],t=v[0].shape[1],c=v[0].shape[2],m=v[1].shape[0],nq=v[3].shape[0]; + const int width=(turbo?3:5)+(int)nq; + if((turbo && (nq!=1 || ((const double*)v[3].buf)[0]!=0.5)) || t<1 || t>1048576 || c<2 || c>256 || nq<1 || nq>16 || v[1].shape[1]!=2 || + v[2].shape[0]!=n || v[2].shape[1]!=c || v[2].shape[2]!=t || + v[4].shape[0]!=n || v[4].shape[1]!=m || v[4].shape[2]!=width*c) { + PyErr_SetString(PyExc_ValueError,"incompatible interval arguments");goto finish; + } + const double *x=v[0].buf,*q=v[3].buf; double *out=v[4].buf; + const int64_t *bounds=v[1].buf,*order=v[2].buf; + for(Py_ssize_t j=0;j1 || (j && q[j]t || bounds[2*j]>=bounds[2*j+1]){ + PyErr_SetString(PyExc_ValueError,"empty or invalid interval");goto finish; + } + for(Py_ssize_t j=0;j=t){ + PyErr_SetString(PyExc_ValueError,"invalid ordering");goto finish; + } + double *sorted=PyMem_Malloc(3*(size_t)t*sizeof(double)); + if(!sorted){PyErr_NoMemory();goto finish;} + uint64_t *prefix=NULL; +#ifdef CAN_FILTER + int words=(int)((t+63)/64); + if(t<=256 && m>=8 && CAN_FILTER && __builtin_cpu_supports("bmi2")){ + prefix=PyMem_Malloc((t+1)*words*sizeof(uint64_t)); + if(!prefix){PyMem_Free(sorted);PyErr_NoMemory();goto finish;} + } +#endif + double *tile=NULL;Py_ssize_t packed=0; +#ifdef CAN_FILTER + if(n>=8 && t<=1024 && m>=4 && CAN_FILTER){ + tile=PyMem_Malloc(8*t*sizeof(double)); + if(tile)packed=n/8*8; + } +#endif + Py_BEGIN_ALLOW_THREADS +#ifdef CAN_FILTER + for(Py_ssize_t i=0;i=rankval[j-1]); + } +#ifdef CAN_FILTER + if(monotonic && prefix){ + memset(prefix,0,(t+1)*words*sizeof(uint64_t)); + for(Py_ssize_t rank=0;rank=packed){ + double sum=0,power=0; + for(Py_ssize_t j=0;j=16)count=filter_wide(ord,rankval,sorted,t,a,b);else +#endif + for(Py_ssize_t j=0;j=a && ix=0.5?high-delta*(1.0-f):low+delta*f); + } + } + } + Py_END_ALLOW_THREADS + PyMem_Free(tile);PyMem_Free(prefix);PyMem_Free(sorted); +finish: + for(int i=0;i<5;i++)if(v[i].obj)PyBuffer_Release(&v[i]); + if(PyErr_Occurred())return NULL; + Py_RETURN_NONE; +} +static PyObject *state_counts(PyObject *self,PyObject *args){ + PyObject *xx,*qq,*oo;Py_buffer v[3]={{0}}; + if(!PyArg_ParseTuple(args,"OOO",&xx,&qq,&oo))return NULL; + if(!view(xx,&v[0],3,'d',0)||!view(qq,&v[1],3,'d',0)||!view(oo,&v[2],3,'d',1))goto done; + if(!separate_output(v,2))goto done; + Py_ssize_t n=v[0].shape[0],c=v[0].shape[1],t=v[0].shape[2],k=v[1].shape[2]+1; + if(t<1||k<2||k>32||v[1].shape[0]!=n||v[1].shape[1]!=c||v[2].shape[0]!=n|| + v[2].shape[1]!=c||v[2].shape[2]!=k*(1+k)){ + PyErr_SetString(PyExc_ValueError,"incompatible count buffers");goto done; + } + const double *x=v[0].buf,*q=v[1].buf;double *out=v[2].buf; + Py_BEGIN_ALLOW_THREADS + memset(out,0,v[2].len); + for(Py_ssize_t z=0;z=16 && t<=8192 && CAN_FILTER){state_fused(u,th,hist,(int)t,(int)k);continue;} +#endif + int prev=0; + for(Py_ssize_t j=0;jth[l]; + hist[state]++; + if(j)hist[k+prev*k+state]++; + prev=state; + } + } + Py_END_ALLOW_THREADS +done: + for(int i=0;i<3;i++)if(v[i].obj)PyBuffer_Release(&v[i]); + if(PyErr_Occurred())return NULL; + Py_RETURN_NONE; +} +/* Pool normalized histogram, transition probabilities and Shannon terms in + one pass. A small table prepared by NumPy retains the original log and + division for all attainable (numerator, denominator) pairs. */ +static double symbolic_sum(const double *a,int m){ + if(m<8){ double s=-0.;for(int i=0;i128){int cut=m/16*8;return symbolic_sum(a,cut)+symbolic_sum(a+cut,m-cut);} + double sums[8];memcpy(sums,a,sizeof sums); + int j=8; + for(;j+8<=m;j+=8)for(int i=0;i<8;i++)sums[i]+=a[j+i]; + double res=((sums[0]+sums[1])+(sums[2]+sums[3]))+((sums[4]+sums[5])+(sums[6]+sums[7])); + for(;j512 || k<2 || k>16 || v[1].shape[0]!=n || v[1].shape[1]!=c || + v[2].shape[0]!=t+1 || v[2].shape[1]!=t+1 || v[2].shape[2]!=2 || + v[3].shape[0]!=n || v[3].shape[1]!=c || v[3].shape[2]!=k+k*k+1){ + PyErr_SetString(PyExc_ValueError,"invalid bounded symbolic pooling dimensions");goto done; + } + const double *x=v[0].buf,*b=v[1].buf,*tab=v[2].buf;double *out=v[3].buf; + Py_BEGIN_ALLOW_THREADS + for(Py_ssize_t row=0;row=16 && CAN_FILTER)state_fused(src,th,counts,t,k); + else +#endif + { + int prev=0; + for(int j=0;jth[l]; + counts[a]++;if(j)counts[k+prev*k+a]++;prev=a; + } + } + for(int j=0;j|"``. + """ + g = np.asarray(groups).astype(str) + if time_index is None: + return g + t = np.asarray(time_index, dtype=float) + if t.shape != g.shape: + raise ValueError("time_index must have one entry per window") + part = np.zeros(len(t), dtype=np.int64) + for subject in np.unique(g): + rows = np.flatnonzero(g == subject) + part[rows] = np.concatenate([[0], np.cumsum(np.diff(t[rows]) < 0)]) + return np.char.add(np.char.add(g, "|"), part.astype(str)) + + +def single_label_fraction(y, recordings): + """Fraction of recordings with at least two windows whose windows all share one label.""" + _, inverse, counts = np.unique(recordings, return_inverse=True, return_counts=True) + pairs = np.unique(np.stack([inverse.ravel(), np.asarray(y, dtype=np.int64)]), axis=1) + labels_per_recording = np.bincount(pairs[0], minlength=len(counts)) + eligible = counts > 1 + if not eligible.any(): + return 1.0 + return float(1 - np.mean(labels_per_recording[eligible] > 1)) + + +def scale_confidence(proba, scale): + """Divide log-probabilities by ``scale`` and renormalise (scale 1 leaves them unchanged).""" + P = np.asarray(proba, dtype=np.float64) + if scale == 1.0: + return P + Z = np.log(np.clip(P, 1e-12, 1.0)) / scale + Z -= Z.max(axis=1, keepdims=True) + Q = np.exp(Z) + return Q / Q.sum(axis=1, keepdims=True) + + +def fit_confidence_scale(proba, y): + """Choose the factor on CONFIDENCE_SCALE_GRID with the lowest validation NLL.""" + L = np.log(np.clip(np.asarray(proba, dtype=np.float64), 1e-12, 1.0)) + y = np.asarray(y, dtype=int) + best = (np.inf, 1.0) + for scale in CONFIDENCE_SCALE_GRID: + Z = L / scale + Z -= Z.max(1, keepdims=True) + Q = np.exp(Z) + Q /= Q.sum(1, keepdims=True) + nll = -float(np.mean(np.log(np.clip(Q[np.arange(len(y)), y], 1e-12, 1.0)))) + if nll < best[0]: + best = (nll, float(scale)) + return best[1] + + +@lru_cache(maxsize=1) +def _native(): + fn = lookup("wisp_viterbi") + if fn is None: + return None + arr = np.ctypeslib.ndpointer + f64 = arr(dtype=np.float64, flags="C_CONTIGUOUS") + i32 = arr(dtype=np.int32, flags="C_CONTIGUOUS") + i64 = arr(dtype=np.int64, flags="C_CONTIGUOUS") + fn.argtypes = [f64, f64, f64, ctypes.c_int64, ctypes.c_int64, i32, f64, f64, i64] + fn.restype = None + return fn + + +@lru_cache(maxsize=1) +def _native_groups(): + fn = lookup("wisp_viterbi_groups") + if fn is not None: + arr = np.ctypeslib.ndpointer + f64 = arr(dtype=np.float64, flags="C_CONTIGUOUS") + i32 = arr(dtype=np.int32, flags="C_CONTIGUOUS") + i64 = arr(dtype=np.int64, flags="C_CONTIGUOUS") + fn.argtypes = [f64, f64, f64, i64, ctypes.c_int64, ctypes.c_int64, i32, f64, f64, i64] + fn.restype = None + return fn + + +def _order(n, groups, time_index): + """Preserve original order within a group when timestamps are absent or tied.""" + if groups is None: + return np.arange(n), np.array([0, n], dtype=np.intp) + g = np.asarray(groups) + if g.shape != (n,): + raise ValueError("groups must have one entry per window") + if np.any(g != g): + raise ValueError("Missing group ID") + if time_index is None: + order = np.argsort(g, kind="stable") + else: + time = np.asarray(time_index) + if time.shape != (n,): + raise ValueError("time_index must have one entry per window") + order = np.lexsort((time, g)) + sorted_groups = g[order] + bounds = np.r_[0, np.flatnonzero(sorted_groups[1:] != sorted_groups[:-1]) + 1, n] + return order, bounds + + +class FastSmoother: + """HMM smoother. ``sequence`` selects how windows are grouped into chains (see module docs).""" + + def __init__(self, sequence="subject"): + self.sequence = check_sequence(sequence) + + @property + def active(self): + """False when recording-level smoothing was skipped at fit time.""" + return getattr(self, "active_", True) + + @property + def confidence_scale(self): + return getattr(self, "confidence_scale_", 1.0) + + def set_confidence_scale(self, scale): + scale = float(scale) + if not np.isfinite(scale) or scale <= 0: + raise ValueError("confidence scale must be finite and positive") + self.confidence_scale_ = scale + return self + + def fit(self, y, *, groups=None, time_index=None, n_classes=None): + y = np.asarray(y, dtype=np.int64) + k = int(n_classes) if n_classes is not None else int(y.max()) + 1 + if y.ndim != 1 or y.size == 0 or np.any((y < 0) | (y >= k)): + raise ValueError("Invalid y or n_classes") + if getattr(self, "sequence", "subject") == "recording": + self.confidence_scale_ = 1.0 + self.active_ = True + self.single_label_fraction_ = None + if groups is not None: + rec = recording_ids(groups, time_index) + self.single_label_fraction_ = single_label_fraction(y, rec) + self.active_ = self.single_label_fraction_ <= SINGLE_LABEL_RECORDING_LIMIT + groups = rec + A = np.ones((k, k), dtype=np.float64) + if groups is not None: + order, bounds = _order(len(y), groups, time_index) + labels = y[order] + edges = labels[:-1] * k + labels[1:] + valid = np.ones(edges.size, dtype=bool) + valid[bounds[1:-1] - 1] = False + A += np.bincount(edges[valid], minlength=k*k).reshape(k, k) + # Upstream adds persistence twice for ungrouped training and once otherwise. + A.flat[::k + 1] += 4.0 if groups is None else 2.0 + p = np.bincount(y, minlength=k).astype(np.float64) + 1.0 + self.transition_ = A / A.sum(axis=1, keepdims=True) + self.prior_ = p / p.sum() + self.log_A_ = np.log(np.clip(self.transition_, 1e-12, 1.0)) + self.log_pi_ = np.log(np.clip(self.prior_, 1e-12, 1.0)) + return self + + def predict_from_proba(self, proba, *, groups=None, time_index=None): + P = np.asarray(proba, dtype=np.float64) + if P.ndim != 2 or P.shape[1] != len(self.prior_) or not np.isfinite(P).all(): + raise ValueError("Invalid emissions") + n, k = P.shape + if n == 0: + return np.empty(0, dtype=np.int64) + if getattr(self, "sequence", "subject") == "recording": + if not self.active: + return P.argmax(1).astype(np.int64) + P = scale_confidence(P, self.confidence_scale) + if groups is not None: + groups = recording_ids(groups, time_index) + order, bounds = _order(n, groups, time_index) + return self._decode_ordered(P,order,bounds) + + def _decode_ordered(self,P,order,bounds): + n,k=P.shape + P = np.clip(P[order], 1e-12, 1.0) + P /= P.sum(axis=1, keepdims=True) + E = np.ascontiguousarray(np.log(np.clip(P, 1e-12, 1.0))) + back = np.empty((np.diff(bounds).max(), k), dtype=np.int32) + prev, nxt = np.empty(k), np.empty(k) + labels = np.empty(n, dtype=np.int64) + grouped = _native_groups() if _native() is not None else None + if grouped is not None: + grouped(E, self.log_A_, self.log_pi_, np.ascontiguousarray(bounds, dtype=np.int64), + len(bounds)-1, k, back, prev, nxt, labels) + out = np.empty_like(labels) + out[order] = labels + return out + fn = _native() + for lo, hi in zip(bounds[:-1], bounds[1:]): + if fn is not None: + fn(E[lo:hi], self.log_A_, self.log_pi_, hi-lo, k, back, prev, nxt, labels[lo:hi]) + else: + prev[:] = self.log_pi_ + E[lo] + cols = np.arange(k) + for i in range(lo + 1, hi): + scores = prev[:, None] + self.log_A_ + pred = np.argmax(scores, axis=0) + back[i-lo] = pred + prev[:] = scores[pred, cols] + E[i] + labels[hi-1] = np.argmax(prev) + for i in range(hi-2, lo-1, -1): + labels[i] = back[i-lo+1, labels[i+1]] + out = np.empty_like(labels) + out[order] = labels + return out diff --git a/src/wisp_fast/engine.py b/src/wisp_fast/engine.py new file mode 100644 index 0000000000000000000000000000000000000000..4baf55e55736e0f0e4b5e387c148e2c9d399d133 --- /dev/null +++ b/src/wisp_fast/engine.py @@ -0,0 +1,459 @@ +"""Accelerated CPU execution of the seven fixed WISP families and search recipes.""" +from copy import deepcopy +from dataclasses import dataclass +import time + +import numpy as np +from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier +from sklearn.linear_model import RidgeClassifier +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import StandardScaler +from sklearn.utils.class_weight import compute_sample_weight + +from wisp.cpu.algorithms import _classifier, build_cpu_estimator +from wisp.cpu.probability import align_proba, estimator_proba, global_classes, labels_from_proba +from wisp.registry import build_method, list_specs +from wisp_release.methods import build_reference_family as reference_family +from . import features as fx +from .cache import _ACTIVE, _SEARCH, transform_blocks +from .decoder import FastSmoother, check_sequence, fit_confidence_scale +from .forest import ForestPlan, local_proba + +CLASS_TO_ID = {spec.code_class: spec.method_id for spec in list_specs()} +# "original": the paper's RidgeClassifierCV. "screened": Ridge strength chosen on a +# 512-row training subsample, then one Ridge fit on all rows (approximate, faster). +RIDGE_SOLVERS = ("original", "screened") + + +@dataclass(frozen=True) +class TurboBudget: + """Upper limits. The canonical HMM sees all training labels and timestamps.""" + kernels: int = 32 + intervals: int = 4 + shapelets: int = 6 + trees: int = 24 + max_steps: int = 96 + max_train: int = 4096 + ridge_alpha: float = 1.0 + + def __post_init__(self): + for field in ("kernels", "intervals", "shapelets", "trees", "max_steps", "max_train"): + val = getattr(self, field) + if not isinstance(val, (int, np.integer)) or val < 1: + raise ValueError(f"{field} must be a positive integer") + if self.max_steps < 15: + raise ValueError("max_steps must allow the longest standard convolution (15)") + if self.ridge_alpha <= 0 or not np.isfinite(self.ridge_alpha): + raise ValueError("ridge_alpha must be finite and positive") + + +def uses_ridge_head(template): + """True for RC/CIS templates whose classifier is the Ridge head the solver option changes.""" + return (CLASS_TO_ID.get(type(template).__name__) in ("wisp_rc", "wisp_cis") + and getattr(template, "classifier", None) == "ridge") + + +def with_ridge_solver(template, mode="exact", budget=None, ridge_solver="original", sequence="subject", + feature_workers=1): + """Wrap a reference template; apply ``ridge_solver`` wherever a Ridge head exists. + + RC/CIS Ridge models use the solver directly, the CSE/ESE ensembles pass it to their + RC/CIS members, and families without a Ridge head are unaffected by the choice. + """ + if ridge_solver not in RIDGE_SOLVERS: + raise ValueError(f"ridge_solver must be one of {', '.join(RIDGE_SOLVERS)}") + direct = ridge_solver if mode == "exact" and uses_ridge_head(template) else "original" + model = FastFamily(template, mode, budget, ridge_solver=direct, feature_workers=feature_workers, + sequence=sequence) + if mode == "exact" and model.method_id in ("wisp_cse", "wisp_ese"): + model.member_ridge_solver = ridge_solver + return model + + +def build_family(method_id, *, seed=42, n_jobs=1, direct=False, overrides=None, + mode="exact", budget=None, ridge_solver="original", feature_workers=1, + sequence="subject"): + model = reference_family(method_id, seed=seed, n_jobs=n_jobs, direct=direct, + overrides=overrides) + return with_ridge_solver(model, mode, budget, ridge_solver, sequence, feature_workers) + + +def build_recipe(recipe_id, *, seed=42, n_jobs=1, mode="exact", budget=None, ridge_solver="original", + feature_workers=1, sequence="subject"): + """Seven non-empty controlled R/I/S combinations crossed with direct/Viterbi.""" + from wisp.ablations import build_recipe as original_recipe + if n_jobs < 1: + raise ValueError("n_jobs must be positive") + model = FastFamily(original_recipe(recipe_id, seed=seed, n_jobs=n_jobs), mode, budget, + ridge_solver=ridge_solver, feature_workers=feature_workers, sequence=sequence) + model.recipe_id = recipe_id + return model + + +def build_candidate(spec_dict, *, mode="exact", budget=None, ridge_solver="original", sequence="subject"): + if isinstance(spec_dict, dict) and spec_dict.get("route", "cpu") != "cpu": + raise ValueError("Only CPU WISP candidates are released") + return with_ridge_solver(build_cpu_estimator(spec_dict), mode, budget, ridge_solver, sequence) + + +def _training_rows(y, cap, seed): + """Roughly proportional stratified random sample, retaining every observed class.""" + labels, inverse, counts = np.unique(y, return_counts=True, return_inverse=True) + if cap < len(labels): + raise ValueError("max_train is smaller than the number of observed classes") + # Reserve one per class, then allocate remaining slots proportionally to capacity. + quota = (counts - 1) * ((cap - len(labels)) / (len(y) - len(labels))) + floor = np.floor(quota).astype(int) + sizes = floor + 1 + order = np.argsort(-(quota - floor), kind="stable") + sizes[order[:cap - sizes.sum()]] += 1 + rng = np.random.default_rng(seed) + return np.sort(np.concatenate([ + rng.choice(np.flatnonzero(inverse == j), size=size, replace=False) + for j, size in enumerate(sizes)])) + + +class FastFamily: + """fit/predict API of the release. Inference batching never splits a sequence decoder.""" + def __init__(self, template, mode="exact", budget=None, ridge_solver="original", feature_workers=1, + sequence="subject"): + if not isinstance(feature_workers,(int,np.integer)) or feature_workers<1: + raise ValueError("feature_workers must be a positive integer") + self.feature_workers = feature_workers + self.sequence = check_sequence(sequence) + if ridge_solver not in RIDGE_SOLVERS: + raise ValueError("Unknown Ridge solver") + if ridge_solver!="original" and (mode!="exact" or CLASS_TO_ID[type(template).__name__] + not in ("wisp_rc","wisp_cis") or template.classifier!="ridge"): + raise ValueError("Custom solvers are supported for exact RC/CIS Ridge models and CIS recipes") + self.ridge_solver = ridge_solver + if mode not in {"exact", "turbo"}: + raise ValueError("mode must be exact or turbo") + if budget is not None and mode != "turbo": + raise ValueError("budget is only meaningful in turbo mode") + if budget is not None and not isinstance(budget, TurboBudget): + raise TypeError("budget must be a TurboBudget") + self.template = template + self.method_id = CLASS_TO_ID[type(template).__name__] + self.mode = mode + self.budget = (budget or TurboBudget()) if mode == "turbo" else None + self.n_jobs = template.n_jobs + self.smoother_ = None + self.members_ = [] + self.blocks_ = [] + self.model_ = None + self.classes_ = None + self.input_shape_ = None + self.checkpoint_only_ = False + + def __getstate__(self): + # Process-local pointers cannot be serialized or included in a size tiebreak. + state = self.__dict__.copy() + state.pop("_forest_plan", None) + return state + + def design_profile(self): + return self.template.design_profile() + + def _blocks(self): + m, family, b = self.template, self.method_id, self.budget + fast = {"fast_profile": self.mode == "turbo"} + if family == "wisp_ss": + return [fx.WearableFeatureExtractor(segment_sizes=m.segment_sizes, + symbolic_bins=m.symbolic_bins, **fast)] + if family == "wisp_so": + return [fx.WearableFeatureExtractor( + include_raw_stats=True, include_magnitude=True, include_jerk=False, + include_multiscale=True, segment_sizes=(5, 10, 20), include_spectral=False, + include_autocorr=False, include_cross_channel=True, include_symbolic=True, + symbolic_bins=8, **fast)] + if family == "wisp_gis": + return [fx.InvariantPhysicsFeatureExtractor( + segment_sizes=m.segment_sizes, symbolic_bins=m.symbolic_bins, + random_intervals=min(24, b.intervals) if b else 24, + random_state=m.seed, include_axis_features=True, **fast), + fx.RandomShapeletSketch( + n_shapelets=min(m.n_shapelets, b.shapelets) if b else m.n_shapelets, + random_state=m.seed+17, **fast)] + if family in {"wisp_cis", "wisp_rc"}: + kernels = min(m.n_kernels, b.kernels) if b else m.n_kernels + blocks = [] + if getattr(m, "use_rocket", True): + blocks.append(fx.RandomConvSketch(n_kernels=kernels, random_state=m.seed, **fast)) + if family == "wisp_cis" and getattr(m, "use_intervals", True): + blocks.append(fx.IntervalDistributionSketch( + n_random_intervals=min(m.n_intervals, b.intervals) if b else m.n_intervals, + random_state=m.seed+29, **fast)) + if family == "wisp_cis" and getattr(m, "use_state", True): + blocks.append(fx.WearableFeatureExtractor( + include_raw_stats=False, include_magnitude=True, include_jerk=True, + include_multiscale=False, include_spectral=True, include_autocorr=True, + include_cross_channel=True, include_symbolic=True, symbolic_bins=6, **fast)) + return blocks + raise ValueError(f"{family} has no base feature blocks") + + def _inputs(self, X): + X = np.asarray(X) + if X.ndim == 2: + X = X[..., None] + if X.ndim != 3 or min(X.shape) < 1: + raise ValueError("X must have nonempty shape (N,T,C) or (N,T)") + return X + + def _times(self, X): + if self.budget and X.shape[1] > self.budget.max_steps: + def sample(): + take = np.linspace(0, X.shape[1]-1, self.budget.max_steps, dtype=np.intp) + return np.ascontiguousarray(X[:, take, :]) + memory = _SEARCH.get() + return sample() if memory is None else memory.sampled(X, self.budget.max_steps, sample) + return X + + def _fit_base(self, X, y): + self.blocks_ = self._blocks() + for block in self.blocks_: + block.fit(X, y) + self._fit_head(self._features(X), y) + + def _fit_head(self, F, y): + m, b = self.template, self.budget + kind = getattr(m, "classifier", "extratrees") + if self.method_id == "wisp_so": + head = ExtraTreesClassifier(n_estimators=160, max_features="sqrt", + class_weight="balanced", random_state=m.seed, n_jobs=self.n_jobs) + elif b and kind == "ridge": + head = RidgeClassifier(alpha=b.ridge_alpha, solver="cholesky") + elif getattr(self,"ridge_solver","original") == "screened": + from .ridge_fast import ScreenedRidgeClassifierCV + head = ScreenedRidgeClassifierCV(alphas=np.logspace(-3,3,9)) + elif hasattr(self, "recipe_id"): + from wisp.cis_algorithms import _classifier as recipe_classifier + head = recipe_classifier(kind, seed=m.seed, n_jobs=self.n_jobs, + class_balance=m.class_balance) + else: + head = _classifier(kind, seed=m.seed, n_jobs=self.n_jobs) + if b: + if kind in {"rf", "extratrees"}: + head.set_params(n_estimators=min(head.n_estimators, b.trees)) + elif kind == "logreg": + head.set_params(max_iter=250, tol=1e-3) + weighted = kind == "ridge" and getattr(m, "class_balance", "balanced") == "balanced" + kwargs = {"clf__sample_weight": compute_sample_weight("balanced", y)} if weighted else {} + if self.method_id in {"wisp_cis", "wisp_rc"} or kind == "logreg": + head = Pipeline([("scale", StandardScaler()), ("clf", head)]) + # Upstream only uses ridge via the RC/CIS scaled pipeline in its search grammar. + if kind == "ridge" and not isinstance(head, Pipeline): + kwargs = {} + forest = head.steps[-1][1] if isinstance(head, Pipeline) else head + if _SEARCH.get() is not None and type(forest) in (ExtraTreesClassifier,RandomForestClassifier): + from .search_forest import forest_fit_plan + with forest_fit_plan(forest): + self.model_ = head.fit(F, y, **kwargs) + else: + self.model_ = head.fit(F, y, **kwargs) + self.__dict__.pop("_forest_plan", None) + + def _features(self, X): + if getattr(self,"feature_workers",1)>1: + from .parallel import parallel_rows + return parallel_rows(self._feature_batch,X,self.feature_workers) + return self._feature_batch(X) + + def _feature_batch(self, X): + blocks = transform_blocks(self.blocks_, X) + return blocks[0] if len(blocks) == 1 else np.concatenate(blocks, axis=1) + + def _member_models(self): + seed = self.template.seed + if self.method_id == "wisp_cse": + configs = [("wisp_ss", {}), ("wisp_rc", {"n_kernels": 256}), ("wisp_so", {})] + else: + configs = [("wisp_cis", {"n_kernels": 256, "n_intervals": 16}), + ("wisp_gis", {"n_shapelets": 24}), ("wisp_so", {})] + members = [] + for j, (name, params) in enumerate(configs): + template = build_method(name, seed=seed+j, n_jobs=self.n_jobs, + overrides={"use_hmm": False, **params}) + member = FastFamily(template, self.mode, self.budget) + solver = getattr(self, "member_ridge_solver", "original") + if solver != "original" and uses_ridge_head(template): + member.ridge_solver = solver + members.append(member) + return members + + def fit(self, X, y, *, groups=None, time_index=None, n_classes=None): + if self.checkpoint_only_: + raise ValueError("Construct a new family for refitting an imported checkpoint") + start = time.perf_counter() + X = self._inputs(X) + y = np.asarray(y, dtype=np.int64) + if y.shape != (len(X),): + raise ValueError("y must have one label per window") + self.classes_ = global_classes(y, n_classes) + self.trained_classes_ = np.unique(y) + self.input_shape_ = X.shape[1:] + k = len(self.classes_) + self.smoother_ = (FastSmoother(getattr(self, "sequence", "subject")).fit( + y, groups=groups, time_index=time_index, n_classes=k) + if self.template.use_hmm else None) + if self.budget and len(y) > self.budget.max_train: + rows = _training_rows(y, self.budget.max_train, self.template.seed) + X, y = X[rows], y[rows] + self.fit_windows_ = len(y) + if self.method_id in {"wisp_cse", "wisp_ese"}: + self.members_ = self._member_models() + for member in self.members_: + member.fit(X, y, n_classes=k) + else: + self._fit_base(self._times(X), y) + self.fit_seconds_ = time.perf_counter() - start + calibration = self.__dict__.pop("calibration_data", None) + if calibration is not None and groups is not None: + self.fit_confidence_scale(*calibration) + return self + + # ---- recording-level smoothing: confidence scaling chosen on validation data ---- + @property + def use_hmm(self): + return bool(self.template.use_hmm) + + def fit_confidence_scale(self, X_valid, y_valid): + """Choose the confidence-scaling factor on validation windows (``sequence="recording"``). + + Has no effect for ``sequence="subject"``, for models without HMM smoothing, or + when recording-level smoothing was skipped at fit time. + """ + sm = self.smoother_ + if getattr(self, "sequence", "subject") != "recording" or sm is None or not sm.active: + return self + sm.set_confidence_scale(fit_confidence_scale(self.predict_proba(X_valid), y_valid)) + return self + + @property + def confidence_scale(self): + sm = self.smoother_ + if getattr(self, "sequence", "subject") != "recording" or sm is None or not sm.active: + return None + return sm.confidence_scale + + def set_confidence_scale(self, scale): + """Reuse a factor chosen for a train-only fit (e.g. after refitting on train+validation).""" + sm = self.smoother_ + if scale is not None and sm is not None and sm.active and getattr(self, "sequence", "subject") == "recording": + sm.set_confidence_scale(scale) + return self + + def _head_proba(self,features,*,private=False): + head=self.model_ + # Standalone fitted model only. Shared panels, disk cache mappings, + # and externally supplied feature arrays take the read-only path. + if (private and type(head) is Pipeline and len(head.steps)==2 + and type(head.steps[0][1]) is StandardScaler + and features.dtype==np.float64 and features.flags.writeable and features.flags.owndata): + scaled=head.steps[0][1].transform(features,copy=False) + return estimator_proba(head.steps[1][1],scaled,self.classes_) + + if isinstance(head,(ExtraTreesClassifier,RandomForestClassifier)) and head.n_jobs==1: + plan=getattr(self,"_forest_plan",None) + if plan is None:plan=self._forest_plan=ForestPlan(head) + return align_proba(local_proba(head,features,plan),head.classes_,self.classes_) + return estimator_proba(head,features,self.classes_) + + def _proba(self, X): + if not self.members_: + private=(self.method_id in ("wisp_cis","wisp_rc") and len(self.blocks_)>=2 + and _SEARCH.get() is None and _ACTIVE.get() is None) + return self._head_proba(self._features(self._times(X)),private=private) + return self._combine(lambda member:member._proba(X),len(X)) + + def _combine(self,score,n): + # Accumulate in canonical order without retaining all member scores. + if self.method_id == "wisp_cse": + out = score(self.members_[0]).copy() + for member in self.members_[1:]: + out += score(member) + out /= len(self.members_) + else: + w = np.asarray(self.template.weights, dtype=np.float64) + if w.size != len(self.members_): + w = np.ones(len(self.members_)) + w = w / np.maximum(w.sum(), 1e-12) + out = np.zeros((n, len(self.classes_))) + for wi, member in zip(w, self.members_): + if wi != 0: + out += wi * score(member) + return out / np.maximum(out.sum(axis=1, keepdims=True), 1e-12) + + def predict_proba(self, X, *, batch_size=None): + if self.classes_ is None: + raise RuntimeError("Unfitted estimator") + X = self._inputs(X) + if self.input_shape_ is not None and X.shape[1:] != self.input_shape_: + raise ValueError(f"Fitted T,C={self.input_shape_}; got {X.shape[1:]}") + if batch_size is None: + return self._proba(X) + if not isinstance(batch_size, (int, np.integer)) or batch_size < 1: + raise ValueError("batch_size must be a positive integer") + out = np.empty((len(X), len(self.classes_)), dtype=np.float64) + for start in range(0, len(X), batch_size): + stop = min(len(X), start + batch_size) + out[start:stop] = self._proba(X[start:stop]) + return out + + def decode_proba(self, proba, *, groups=None, time_index=None): + if self.smoother_ is not None and groups is not None: + return self.smoother_.predict_from_proba(proba, groups=groups, time_index=time_index) + return labels_from_proba(proba, self.classes_) + + def predict(self, X, *, groups=None, time_index=None, batch_size=None): + return self.decode_proba(self.predict_proba(X, batch_size=batch_size), + groups=groups, time_index=time_index) + + +def _convert_block(block): + cls = getattr(fx, type(block).__name__) + obj = cls(**block.get_params(deep=False)) + for name, value in vars(block).items(): + if name.endswith("_"): + setattr(obj, name, _convert_block(value) if name == "interval_" and value is not None + else deepcopy(value)) + return obj + + +def accelerate_checkpoint(model): + """Convert a trusted, already-loaded release estimator. Does not refit or unpickle. + + Returns an independent inference-only copy, including fitted head and transition + probabilities. For guarded file loading use wisp_release.checkpoints.load_checkpoint. + """ + if isinstance(model, FastFamily): + return deepcopy(model) + if type(model).__name__ not in CLASS_TO_ID or getattr(model, "classes_", None) is None: + raise ValueError("Expected a fitted WISP CPU family from the release") + out = FastFamily(type(model)(**model.get_params(deep=False))) + out.classes_ = model.classes_.copy() + out.trained_classes_ = deepcopy(getattr(model, "trained_classes_", None)) + out.fit_seconds_ = getattr(model, "fit_seconds_", None) + out.checkpoint_only_ = True + if getattr(model, "members_", None): + out.members_ = [accelerate_checkpoint(member) for member in model.members_] + else: + if getattr(model, "model_", None) is None or getattr(model, "use_anchor_stats", False): + raise ValueError("Unsupported or incomplete WISP checkpoint") + out.model_ = deepcopy(model.model_) + names = {"wisp_ss": ("feature_extractor_",), "wisp_so": ("feature_extractor_",), + "wisp_rc": ("sketch_",), "wisp_gis": ("physics_", "shapelets_"), + "wisp_cis": ("rocket_", "interval_", "state_")} + out.blocks_ = [_convert_block(getattr(model, name)) for name in names[out.method_id] + if getattr(model, name, None) is not None] + if not out.blocks_: + raise ValueError("No fitted feature blocks found") + smoother = getattr(model, "smoother_", None) + if smoother is not None: + s = FastSmoother() + s.transition_ = smoother.transition_.copy() + s.prior_ = smoother.prior_.copy() + s.log_A_ = np.ascontiguousarray(np.log(np.clip(s.transition_, 1e-12, 1.0))) + s.log_pi_ = np.ascontiguousarray(np.log(np.clip(s.prior_, 1e-12, 1.0))) + out.smoother_ = s + return out diff --git a/src/wisp_fast/family_kernels.c b/src/wisp_fast/family_kernels.c new file mode 100644 index 0000000000000000000000000000000000000000..d11540d16b6602f9f94387aded1b6840a1ba4791 --- /dev/null +++ b/src/wisp_fast/family_kernels.c @@ -0,0 +1,445 @@ +#define PY_SSIZE_T_CLEAN +#include +#include +#include +#include + +/* Repeat NumPy's double pairwise reduction on a contiguous axis. An + * interleaved channel's stride prevents pairwise summation on the time axis. */ +static double pairwise_sum(const double *x, Py_ssize_t n) { + if (n < 8) { + double s = -0.; + for (Py_ssize_t j = 0; j < n; j++) s += x[j]; + return s; + } + if (n > 128) { + Py_ssize_t mid = (n / 2) / 8 * 8; + return pairwise_sum(x, mid) + pairwise_sum(x + mid, n - mid); + } + double lane[8]; + memcpy(lane, x, sizeof lane); + Py_ssize_t j = 8; + for (; j + 8 <= n; j += 8) + for (int k = 0; k < 8; k++) lane[k] += x[j+k]; + double s = ((lane[0]+lane[1])+(lane[2]+lane[3])) + + ((lane[4]+lane[5])+(lane[6]+lane[7])); + for (; j < n; j++) s += x[j]; + return s; +} + +static double ordered_sum(const double *x, Py_ssize_t n, int contiguous_axis) { + if (contiguous_axis) return pairwise_sum(x, n); + double s = 0.; + for (Py_ssize_t j = 0; j < n; j++) s += x[j]; + return s; +} + +static int get_view(PyObject *obj, Py_buffer *v, int rank, int writable) { + int flags = PyBUF_C_CONTIGUOUS | PyBUF_FORMAT | (writable ? PyBUF_WRITABLE : 0); + if (PyObject_GetBuffer(obj, v, flags) < 0) return 0; + if (v->ndim != rank || v->itemsize != 8 || !v->format || strcmp(v->format, "d") || + (uintptr_t)v->buf % 8) { + PyErr_SetString(PyExc_ValueError, "expected aligned contiguous native float64 array of given rank"); + return 0; + } + return 1; +} + +static int overlaps(const Py_buffer *x, const Py_buffer *y) { + uintptr_t a = (uintptr_t)x->buf, b = (uintptr_t)y->buf; + return x->len && y->len && (a <= b ? b-a < (uintptr_t)x->len : a-b < (uintptr_t)y->len); +} + +static void release_views(Py_buffer *v, int count) { + for (int i = 0; i < count; i++) + if (v[i].obj) PyBuffer_Release(v+i); +} + +#include "moments_tiles.h" + +static PyObject *stats(PyObject *self, PyObject *args) { + PyObject *xo, *yo; + Py_buffer v[2] = {{0}}; + if (!PyArg_ParseTuple(args, "OO", &xo, &yo)) return NULL; + if (!get_view(xo, v, 3, 0) || !get_view(yo, v+1, 3, 1)) goto done; + Py_ssize_t n = v[0].shape[0], t = v[0].shape[1], c = v[0].shape[2]; + if (t < 1 || t > 8192 || c < 1 || v[1].shape[0] != n || + v[1].shape[1] != 11 || v[1].shape[2] != c || overlaps(v, v+1)) { + PyErr_SetString(PyExc_ValueError, "invalid stats geometry"); + goto done; + } + double *signal = PyMem_Malloc(6*t*sizeof(double)); + if (!signal) { PyErr_NoMemory(); goto done; } + double *sq = signal+t, *cube = sq+t, *fourth = cube+t; + double *diff = fourth+t, *diff_sq = diff+t; + const double *x = v[0].buf; + double *out = v[1].buf; + int contiguous = c == 1; + Py_BEGIN_ALLOW_THREADS + Py_ssize_t i=0; +#ifdef MOM512 + if(t<=256 && c<=64 && __builtin_cpu_supports("avx512f")) + for(;i+8<=n;i+=8)stats_tile8(x+i*t*c,(int)t,(int)c,out+i*11*c); +#endif + for (; i < n; i++) for (Py_ssize_t ch = 0; ch < c; ch++) { + const double *src = x + i*t*c + ch; + double *dst = out + i*11*c + ch; + double high = src[0], low = src[0]; + int crossings = 0; + for (Py_ssize_t j = 0; j < t; j++) { + double value = src[c*j]; + signal[j] = value; + sq[j] = value*value; + diff[j] = j ? value-signal[j-1] : 0.; + if (value > high) high = value; + if (value < low) low = value; + if (j) crossings += signbit(value) != signbit(signal[j-1]); + } + double mean = ordered_sum(signal, t, contiguous)/t; + double dmean = ordered_sum(diff, t, contiguous)/t; + double power = ordered_sum(sq, t, contiguous)/t; + double slope = 0., mid = (t-1)*.5; + for (Py_ssize_t j = 0; j < t; j++) { + double centered = signal[j]-mean, delta = diff[j]-dmean; + diff_sq[j] = delta*delta; + sq[j] = centered*centered; + cube[j] = sq[j]*centered; + fourth[j] = sq[j]*sq[j]; + slope += (j-mid)*centered; + } + dst[0] = mean; + dst[c] = ordered_sum(sq, t, contiguous)/t; + dst[2*c] = low; + dst[3*c] = high; + dst[4*c] = power; + dst[5*c] = dmean; + dst[6*c] = sqrt(ordered_sum(diff_sq, t, contiguous)/t); + dst[7*c] = t > 1 ? (double)crossings/(t-1) : 0.; + dst[8*c] = t > 1 ? slope/((double)t*((double)t*t-1)/12.) : 0.; + dst[9*c] = ordered_sum(cube, t, contiguous)/t; + dst[10*c] = ordered_sum(fourth, t, contiguous)/t; + } + Py_END_ALLOW_THREADS + PyMem_Free(signal); +done: + release_views(v, 2); + if (PyErr_Occurred()) return NULL; + Py_RETURN_NONE; +} + +static PyObject *segments(PyObject *self, PyObject *args) { + PyObject *xo, *yo; + Py_buffer v[2] = {{0}}; + int pieces, fast; + if (!PyArg_ParseTuple(args, "OOii", &xo, &yo, &pieces, &fast)) return NULL; + if (!get_view(xo, v, 3, 0) || !get_view(yo, v+1, 4, 1)) goto done; + Py_ssize_t n = v[0].shape[0], t = v[0].shape[1], c = v[0].shape[2], fields = fast ? 3 : 4; + if (t < 1 || t > 8192 || c < 1 || pieces < 1 || pieces > t || v[1].shape[0] != n || + v[1].shape[1] != fields || v[1].shape[2] != pieces || v[1].shape[3] != c || overlaps(v, v+1)) { + PyErr_SetString(PyExc_ValueError, "invalid segment geometry"); + goto done; + } + Py_ssize_t length = (t+pieces-1)/pieces; + double *signal = PyMem_Malloc(2*length*sizeof(double)); + if (!signal) { PyErr_NoMemory(); goto done; } + double *sq = signal+length, *out = v[1].buf; + const double *x = v[0].buf; + Py_BEGIN_ALLOW_THREADS + Py_ssize_t i=0; +#ifdef MOM512 + if(t<=256 && c<=64 && __builtin_cpu_supports("avx512f")) + for(;i+8<=n;i+=8)seg_tile8(x+i*t*c,(int)t,(int)c,pieces,(int)fields,out+i*fields*pieces*c); +#endif + for (; i < n; i++) for (int s = 0; s < pieces; s++) for (Py_ssize_t k = 0; k < c; k++) { + const double *src = x+i*t*c+k; + double *dst = out+(i*fields*pieces+s)*c+k; + Py_ssize_t offset = (Py_ssize_t)s*length, field_stride = pieces*c; + /* The final observed value fills padded segment positions. */ + for (Py_ssize_t j = 0; j < length; j++) + signal[j] = src[(offset+j < t ? offset+j : t-1)*c]; + double mean = ordered_sum(signal, length, c == 1)/length; + double high = signal[0], low = high, slope = 0., mid = (length-1)*.5; + for (Py_ssize_t j = 0; j < length; j++) { + double value = signal[j], centered = value-mean; + if (value > high) high = value; + if (value < low) low = value; + sq[j] = centered*centered; + if (!fast) slope += (j-mid)*centered; + } + dst[0] = mean; + dst[field_stride] = sqrt(ordered_sum(sq, length, c == 1)/length); + dst[2*field_stride] = high-low; + if (!fast) + dst[3*field_stride] = length > 1 ? slope/((double)length*((double)length*length-1)/12.) : 0.; + } + Py_END_ALLOW_THREADS + PyMem_Free(signal); +done: + release_views(v, 2); + if (PyErr_Occurred()) return NULL; + Py_RETURN_NONE; +} + +#include "shapelet_tiles.h" + +static PyObject *shapelets(PyObject *self, PyObject *args) { + PyObject *xo, *wo, *yo; + Py_buffer v[3] = {{0}}; + int dilation, stride; + if (!PyArg_ParseTuple(args, "OOOii", &xo, &wo, &yo, &dilation, &stride)) return NULL; + if (!get_view(xo, v, 2, 0) || !get_view(wo, v+1, 2, 0) || !get_view(yo, v+2, 3, 1)) goto done; + Py_ssize_t n = v[0].shape[0], t = v[0].shape[1], k = v[1].shape[0], length = v[1].shape[1]; + if (t < 1 || length < 1 || length > 256 || k < 1 || stride < 1 || dilation < 1 || + length-1 > (t-1)/dilation || v[2].shape[0] != n || v[2].shape[1] != k || + v[2].shape[2] != 2 || overlaps(v,v+2) || overlaps(v+1,v+2)) { + PyErr_SetString(PyExc_ValueError, "invalid shapelet geometry"); + goto done; + } + double *signal = PyMem_Malloc(2*length*sizeof(double)); + if (!signal) { PyErr_NoMemory(); goto done; } + const double *x = v[0].buf, *weights = v[1].buf; + double *out = v[2].buf, *sq = signal+length; + const Py_ssize_t stop = t - (length-1)*dilation; + Py_BEGIN_ALLOW_THREADS + for (Py_ssize_t row = 0; row < n; row++) { + double *dst = out+row*k*2; + for (Py_ssize_t h = 0; h < k; h++) { + dst[2*h] = INFINITY; + dst[2*h+1] = 0.; + } +#ifdef SHAPE512 + if (length <= 64 && (stop-1)/stride >= 7 && __builtin_cpu_supports("avx512f")) { + shapelet_row(x+row*t,weights,dst,stop,(int)length,dilation,stride,k); + continue; + } +#endif + for (Py_ssize_t a = 0; a < stop; a += stride) { + for (Py_ssize_t j = 0; j < length; j++) signal[j] = x[row*t+a+j*dilation]; + double mean = pairwise_sum(signal, length)/length; + for (Py_ssize_t j = 0; j < length; j++) signal[j] -= mean; + /* NumPy std recomputes the mean of the already centered window. */ + double centered_mean = pairwise_sum(signal, length)/length; + for (Py_ssize_t j = 0; j < length; j++) { + double value = signal[j]-centered_mean; + sq[j] = value*value; + } + double sd = sqrt(pairwise_sum(sq, length)/length); + if (sd < 1e-6) sd = 1.; + for (Py_ssize_t j = 0; j < length; j++) signal[j] /= sd; + /* Normalize once, compare every prototype with this geometry. */ + for (Py_ssize_t h = 0; h < k; h++) { + for (Py_ssize_t j = 0; j < length; j++) { + double d = signal[j]-weights[h*length+j]; + sq[j] = d*d; + } + double value = sqrt(pairwise_sum(sq, length)/length); + if (value < dst[2*h]) { + dst[2*h] = value; + dst[2*h+1] = (double)a/(stop > 1 ? stop-1 : 1); + } + } + } + } + Py_END_ALLOW_THREADS + PyMem_Free(signal); +done: + release_views(v, 3); + if (PyErr_Occurred()) return NULL; + Py_RETURN_NONE; +} + +/* Bounded common geometry. Each scalar uses NumPy's channel-dependent + reduction tree. Return false to the wrapper for non-finite normalization. */ +static PyObject *normalize(PyObject *self, PyObject *args) { + PyObject *xo,*yo;Py_buffer v[2]={{0}}; + int finite=1; + if(!PyArg_ParseTuple(args,"OO",&xo,&yo))return NULL; + if(!get_view(xo,v,3,0)||!get_view(yo,v+1,3,1))goto done; + Py_ssize_t n=v[0].shape[0],t=v[0].shape[1],c=v[0].shape[2]; + if(t<1||t>8192||c<1||c>64||memcmp(v[0].shape,v[1].shape,3*sizeof(Py_ssize_t))||overlaps(v,v+1)){ + PyErr_SetString(PyExc_ValueError,"invalid normalization buffers");goto done; + } + double *tmp=PyMem_Malloc(2*t*sizeof(double)); + if(!tmp){PyErr_NoMemory();goto done;} + double *sq=tmp+t; + const double *x=v[0].buf;double *y=v[1].buf; + Py_BEGIN_ALLOW_THREADS + for(Py_ssize_t i=0;finite && i8192||c<1||k>64||v[1].shape[0]!=n||v[1].shape[2]!=c|| + !PyTuple_Check(lags_obj)||PyTuple_Size(lags_obj)!=k||overlaps(v,v+1)){ + PyErr_SetString(PyExc_ValueError,"invalid autocorrelation buffers");goto done; + } + long lags[64]; + for(Py_ssize_t l=0;l=8 && t<=1024 && c<=16 && __builtin_cpu_supports("avx512f")){ + tile=PyMem_Malloc(8*t*sizeof(double)); + if(!tile){PyMem_Free(tmp);PyErr_NoMemory();goto done;} + } +#endif + Py_BEGIN_ALLOW_THREADS +#ifdef WISP_ACF_LANES + if(tile)for(;first+8<=n;first+=8) + acf_tile(x+first*t*c,t,c,lags,k,tile,co?(double*)v[2].buf+first*t*c:NULL,y+first*k*c); +#endif + for(Py_ssize_t i=first;i0 && lag0 && lag3?3:c; + if(t<1||c<2||v[1].shape[0]!=n||v[1].shape[1]!=t||v[1].shape[2]!=1|| + memcmp(v[1].shape,v[3].shape,3*sizeof(Py_ssize_t))|| + memcmp(v[0].shape,v[2].shape,3*sizeof(Py_ssize_t))){ + PyErr_SetString(PyExc_ValueError,"invalid derived-signal buffers");goto finish; + } + for(int j=0;j<4;j++)for(int h=j+1;h<4;h++)if(overlaps(v+j,v+h)){ + PyErr_SetString(PyExc_ValueError,"overlapping derived-signal buffers");goto finish; + } + const double *x=v[0].buf;double *mag=v[1].buf,*jerk=v[2].buf,*jm=v[3].buf; + Py_BEGIN_ALLOW_THREADS + for(Py_ssize_t i=0;i=3))goto finish; + Py_ssize_t n=v[0].shape[0],f=v[0].shape[1],c=v[0].shape[2],b=v[2].shape[0]; + if(f<2||f>4097||c<1||v[1].shape[0]!=f||v[2].shape[1]!=2|| + v[3].shape[0]!=n||v[3].shape[1]!=f-1||v[3].shape[2]!=c|| + v[4].shape[0]!=n||v[4].shape[1]!=b+3||v[4].shape[2]!=c){ + PyErr_SetString(PyExc_ValueError,"invalid spectral buffers");goto finish; + } + for(int j=3;j<5;j++)for(int h=0;h=0 && bounds[2*j]<=bounds[2*j+1] && + bounds[2*j+1]<=f && bounds[2*j]==floor(bounds[2*j])&&bounds[2*j+1]==floor(bounds[2*j+1]))){ + PyErr_SetString(PyExc_ValueError,"invalid band bounds");goto finish; + } + double *tmp=PyMem_Malloc(2*f*sizeof(double));if(!tmp){PyErr_NoMemory();goto finish;} + double *prob=v[3].buf,*out=v[4].buf,*weighted=tmp+f; + Py_BEGIN_ALLOW_THREADS + for(Py_ssize_t i=0;itmp[peak] || (isnan(tmp[j])&&!isnan(tmp[peak])))peak=j; + } + z[(b+1)*c]=ordered_sum(weighted+1,f-1,c==1)/total; + z[(b+2)*c]=freq[peak]; + } + Py_END_ALLOW_THREADS + PyMem_Free(tmp); +finish: + release_views(v,5);if(PyErr_Occurred())return NULL;Py_RETURN_NONE; +} +static PyObject *clean(PyObject *self,PyObject *args){ + PyObject *o;Py_buffer v={0}; + if(!PyArg_ParseTuple(args,"O",&o))return NULL; + if(!get_view(o,&v,2,1)){if(v.obj)PyBuffer_Release(&v);return NULL;} + double *a=v.buf;Py_ssize_t n=v.len/sizeof(double); + Py_BEGIN_ALLOW_THREADS + for(Py_ssize_t i=0;i border[:, None] + h = np.bincount((states + offset * n_bins).ravel(), minlength=n * n_bins) + trans = (offset * n_bins + states[:, :-1]) * n_bins + states[:, 1:] + mat = np.bincount(trans.ravel(), minlength=n * n_bins * n_bins) + mat = mat.reshape(n, n_bins, n_bins) + flat = (mat / np.maximum(mat.sum(axis=2, keepdims=True), 1)).reshape(n, -1) + ent = -(flat * np.log(flat + 1e-12)).sum(axis=1, keepdims=True) + blocks.extend((h.reshape(n, n_bins) / length, flat, ent)) + return np.concatenate(blocks, axis=1) + + +@lru_cache(maxsize=4) +def _state_table(length): + den = np.maximum(np.arange(length + 1), 1)[:, None] + p = np.arange(length + 1, dtype=np.float64)[None, :] / den + return np.stack((p, p * np.log(p + 1e-12)), axis=-1) + + +@lru_cache(maxsize=128) +def _state_levels(length,bins): + pos = (length-1)*np.linspace(0,1,bins+1)[1:-1] + lo = pos.astype(np.intp) + return lo, np.minimum(lo+1,length-1), pos-lo + + +@lru_cache(maxsize=256) +def _column_names(prefix,width): + return tuple(f"{prefix}_{j}" for j in range(width)) + + +def _symbolic_bundle(variants, bins): + """Quantiles and transition counts for every signal view in the CIS state block.""" + slices = [v[:,:,:3].transpose(0,2,1) for _, v in variants] + n, _, length = slices[0].shape + x = np.empty((n, sum(v.shape[1] for v in slices), length)) + offset = 0 + for v in slices: + x[:,offset:offset+v.shape[1]] = v + offset += v.shape[1] + n, width, length = x.shape + lo, hi, frac = _state_levels(length, bins) + part = (np.sort(x,axis=2) if length <= 512 else + np.partition(x, np.unique(np.r_[lo,hi]), axis=2)) + a, b = np.take(part,lo,axis=2), np.take(part,hi,axis=2) + q = np.ascontiguousarray(np.where(frac>=.5,b-(b-a)*(1-frac),a+(b-a)*frac)) + if length<=512 and bins<=16 and hasattr(_cis_intervals, 'state_pool'): + out = np.empty((n,width,bins+bins*bins+1)) + _cis_intervals.state_pool(x,q,_state_table(length),out) + else: + counts = np.empty((n,width,bins+bins*bins)) + _cis_intervals.state_counts(x,q,counts) + h = counts[:,:,:bins] / length + mat = counts[:,:,bins:].reshape(n,width,bins,bins) + flat = (mat / np.maximum(mat.sum(axis=3,keepdims=True),1)).reshape(n,width,-1) + e = -(flat*np.log(flat+1e-12)).sum(axis=2,keepdims=True) + out = np.concatenate((h,flat,e),axis=2) + boundaries = np.cumsum([v.shape[1] for v in slices])[:-1] + return {name:block.reshape(n,-1) for (name,_),block in + zip(variants,np.split(out,boundaries,axis=1))} + + +try: + from . import _family_kernels +except ImportError: + _family_kernels = None + + +def _normalize(x): + if (_family_kernels is not None and x.dtype == np.float64 and x.flags.c_contiguous + and x.flags.aligned and 0 < x.shape[1] <= 8192 and 0 < x.shape[2] <= 64): + out = np.empty_like(x) + if _family_kernels.normalize(x, out): + return out + return ref._safe_zscore_per_window(x) + + +def _autocorr(v, lags): + if (_family_kernels is not None and v.dtype == np.float64 and v.flags.c_contiguous + and v.flags.aligned and 0 < v.shape[1] <= 8192 and len(lags) <= 64): + out = np.empty((len(v),len(lags),v.shape[2])) + _family_kernels.autocorr(v, tuple(int(lag) for lag in lags), out) + return out.reshape(len(v),len(lags)*v.shape[2]) + return ref._autocorr_features(v, lags) + + +def _clean(out): + if _family_kernels is not None and out.flags.c_contiguous and out.flags.aligned and out.dtype==np.float64 and out.ndim==2: + _family_kernels.clean(out) + return out + return np.nan_to_num(out,copy=False,nan=0.,posinf=0.,neginf=0.) + + +@lru_cache(maxsize=64) +def _frequency_plan(time,bands): + freq=np.fft.rfftfreq(time) + pairs=[(0,0) if np.isnan(lo) or np.isnan(hi) else + (int(freq.searchsorted(lo)),int(freq.searchsorted(hi))) for lo,hi in bands] + return freq,np.array([(l,max(l,r)) for l,r in pairs],dtype=np.float64).reshape(-1,2) + + +def _spectrum_centered(center,bands): + n,t,c=center.shape + freq,bounds=_frequency_plan(t,tuple(map(tuple,bands))) + power=np.abs(np.fft.rfft(center,axis=1)) + np.square(power,out=power);power/=t + prob=np.empty((n,len(freq)-1,c)) + out=np.empty((n,len(bounds)+3,c)) + _family_kernels.spectral(power,freq,bounds,prob,out) + work=np.log(prob+1.e-12);work*=prob + np.copyto(work,0.,where=~(prob>0)) + out[:,len(bounds)]=-work.sum(axis=1)/np.log(max(len(freq)-1,2)) + return out.reshape(n,-1) + + +def _raw_quantiles(variants, bins, need_symbols): + """One time-axis partition supplies the median, quartiles and state boundaries.""" + x = np.ascontiguousarray(np.concatenate([v.transpose(0, 2, 1) for _, v in variants], axis=1)) + n, width, time = x.shape + levels = np.array([.25, .5, .75]) + if need_symbols: + levels = np.r_[levels, np.linspace(0, 1, bins + 1)[1:-1]] + pos = (time - 1) * levels + lower = pos.astype(np.intp) + upper = np.minimum(lower + 1, time - 1) + # Short multi-quantile axes are cheaper to sort. +0 and -0 compare + # equal but may have different median bytes after sort vs partition. + if time<=512 and not np.any(x.view(np.uint64)==np.uint64(1)<= .5, b - (b-a)*(1-frac), a + (b-a)*frac) + median = a[:, :, 1] if time % 2 else (a[:, :, 1] + b[:, :, 1]) * .5 + + pooled = None + if need_symbols: + state_x, thresholds = x, q[:, :, 3:] + if any(v.shape[2] > 3 for _, v in variants): + starts = np.cumsum([0] + [v.shape[2] for _, v in variants[:-1]]) + indices = np.concatenate([np.arange(start, start + min(3, v.shape[2])) + for start, (_, v) in zip(starts, variants)]) + state_x = np.ascontiguousarray(x[:, indices]) + thresholds = thresholds[:, indices] + state_width = state_x.shape[1] + counts = np.empty((n, state_width, bins * (bins+1))) + _cis_intervals.state_counts(state_x, np.ascontiguousarray(thresholds), counts) + hist = counts[:, :, :bins] / time + mat = counts[:, :, bins:].reshape(n, state_width, bins, bins) + flat = (mat / np.maximum(mat.sum(axis=3, keepdims=True), 1)).reshape(n, state_width, -1) + entropy = -(flat * np.log(flat + 1e-12)).sum(axis=-1, keepdims=True) + pooled = np.concatenate((hist, flat, entropy), axis=2) + + quantiles, symbols = {}, {} + start, symbol_start = 0, 0 + for name, v in variants: + end = start + v.shape[2] + quantiles[name] = (median[:, start:end], q[:, start:end, 0], q[:, start:end, 2]) + if pooled is not None: + symbol_end = symbol_start + min(3, v.shape[2]) + symbols[name] = pooled[:, symbol_start:symbol_end].reshape(n, -1) + symbol_start = symbol_end + start = end + return quantiles, symbols + + +def _family_stats(v, quantiles): + n, time, channels = v.shape + out = np.empty((n, 11, channels)) + _family_kernels.stats(v, out) + mean, var, low, high, power, dmean, dstd, zcr, slope, third, fourth = out.transpose(1, 0, 2) + small = var <= (np.finfo(float).eps * mean)**2 + with np.errstate(divide="ignore", invalid="ignore"): + skewness = np.where(small, 0, third / var**1.5) + excess = np.where(small, 0, fourth / var**2 - 3) + if channels == 1: + # NumPy's einsum uses a contiguous dot path on univariate windows. + slope = ref._slope(v) + stats = np.concatenate((mean, np.sqrt(var), low, high, *quantiles, power, dmean, + dstd, zcr, slope, skewness, excess), axis=1) + return np.nan_to_num(stats, copy=False) + + +@dataclass +class WearableFeatureExtractor(ref.WearableFeatureExtractor): + fast_profile: bool = False + + def fit(self, X, y=None): + # No fitted statistics: the first real transform supplies feature names. + return self + + def transform(self, X): + x = ref._as_3d(X) + if not self.fast_profile and not x.flags.c_contiguous: + return super().transform(X) + x = _normalize(x) if self.zscore else np.nan_to_num( + x, nan=0.0, posinf=0.0, neginf=0.0) + return self.transform_normalized(x) + + def transform_normalized(self, x): + n, length, channels = x.shape + variants = [('ch', x)] + if (self.include_jerk and self.include_magnitude and channels>=2 and _family_kernels is not None + and x.dtype==np.float64 and x.flags.c_contiguous and x.flags.aligned): + mag=np.empty((n,length,1));jerk=np.empty_like(x);jm=np.empty_like(mag) + _family_kernels.derived(x,mag,jerk,jm) + variants.extend((('mag',mag),('jerk',jerk),('jerk_mag',jm))) + else: + if self.include_magnitude and channels >= 2: + variants.append(('mag', np.linalg.norm(x[:, :, :3], axis=2, keepdims=True))) + if self.include_jerk: + jerk = np.diff(x, axis=1, prepend=x[:, :1]) + variants.append(('jerk', jerk)) + if channels >= 2: + variants.append(('jerk_mag', np.linalg.norm(jerk[:, :, :3], axis=2, keepdims=True))) + blocks, names = [], [] + bins = min(4, self.symbolic_bins) if self.fast_profile else self.symbolic_bins + raw_native = (self.include_raw_stats and not self.fast_profile and + _family_kernels is not None and 1 <= length <= 8192 and + all(v.flags.c_contiguous and v.flags.aligned for _, v in variants)) + fused_symbols = (self.include_symbolic and length >= 8 and 2 <= bins <= 32 and + _cis_intervals is not None) + symbols = None + if raw_native: + raw_q, sym_q = _raw_quantiles(variants, bins, fused_symbols) + if fused_symbols: + symbols = sym_q + elif fused_symbols: + symbols = _symbolic_bundle(variants, bins) + + def emit(name, block): + blocks.append(block.reshape(n, -1)) + names.extend(_column_names(name, blocks[-1].shape[1])) + + for prefix, v in variants: + if raw_native: + emit(f'{prefix}_stat', _family_stats(v, raw_q[prefix])) + elif self.include_raw_stats: + stats = [v.mean(axis=1), v.std(axis=1), v.min(axis=1), v.max(axis=1)] + if not self.fast_profile: + quartiles = np.quantile(v, [.25, .75], axis=1) + diff = np.diff(v, axis=1, prepend=v[:, :1]) + stats += [np.median(v, axis=1), *quartiles, np.mean(v*v, axis=1), + diff.mean(axis=1), diff.std(axis=1), ref._zero_crossing_rate(v), + ref._slope(v), skew(v, axis=1, nan_policy='omit'), + kurtosis(v, axis=1, nan_policy='omit')] + emit(f'{prefix}_stat', np.concatenate([np.nan_to_num(s) for s in stats], axis=1)) + if self.include_multiscale: + sizes = self.segment_sizes[:1] if self.fast_profile else self.segment_sizes + for w in sizes: + if w > length: + continue + if (_family_kernels is not None and v.flags.c_contiguous and v.flags.aligned + and 1 <= length <= 8192): + nf = 3 if self.fast_profile else 4 + packed = np.empty((n, nf, w, v.shape[2])) + _family_kernels.segments(v, packed, int(w), int(self.fast_profile)) + if nf == 4 and v.shape[2] == 1: + sv = ref._segment_view(v, int(w)) + packed[:, 3] = ref._slope(sv.reshape(n*w, sv.shape[2], 1)).reshape(n, w, 1) + emit(f'{prefix}_seg{w}', packed) + continue + seg = ref._segment_view(v, int(w)) + means = seg.mean(axis=2) + stats = [means, seg.std(axis=2), seg.max(axis=2) - seg.min(axis=2)] + if not self.fast_profile: + # Match _slope's per-segment reduction order. The flattened + # sample+segment axes are independent. + seg_flat = seg.reshape(n*w, seg.shape[2], seg.shape[3]) + stats.append(ref._slope(seg_flat).reshape(means.shape)) + emit(f'{prefix}_seg{w}', np.concatenate([s.reshape(n,-1) for s in stats], axis=1)) + lags=tuple(l for l in self.autocorr_lags if l= 8: + counts = symbols[prefix] if symbols is not None else _state_counts(v[:, :, :3], int(bins)) + emit(f'{prefix}_sym', counts) + if self.include_cross_channel and channels > 1: + cols = [x[:, :, c] - x[:, :, c].mean(axis=1, keepdims=True) for c in range(channels)] + powers = [(col * col).sum(axis=1) for col in cols] + corr = [(cols[i]*cols[j]).sum(axis=1)/(np.sqrt(powers[i]*powers[j])+1e-12) + for i in range(channels) for j in range(i+1, channels)] + emit('corr', np.column_stack(corr)) + if not blocks: + raise ValueError('No feature blocks enabled') + self.feature_names_ = names + return _clean(np.concatenate(blocks, axis=1)) + + +def _pool_response(out, indexes, resp): + # (kernel, sample, time) with contiguous time, as in the original. + out[:, indexes, 0] = (resp > 0.).mean(axis=2).T + out[:, indexes, 1] = resp.max(axis=2).T + out[:, indexes, 2] = resp.mean(axis=2).T + out[:, indexes, 3] = resp.std(axis=2).T + + +try: + from . import _cis_conv +except ImportError: + _cis_conv = None + + +@lru_cache(maxsize=1) +def _verified_conv_blas(): + import hashlib + library = _cis_conv.fused_library() if _cis_conv is not None else None + if not library or np.__version__ != "2.3.5": + return False + try: + with open(library, "rb") as f: + actual = hashlib.file_digest(f, "sha256").hexdigest() + except OSError: + return False + return actual == "ee2039cf13a45e1e89f04d0cea9585657d93ffaa425594e1935161013fdf4e12" + + +def _pack_kernels(kernels): + count = len(kernels) + max_len = max(spec["length"] for spec in kernels) + layout = np.zeros((count, 6), dtype=np.int64) + weights = np.zeros((count, max_len, 3)) + biases = np.empty(count) + for j, spec in enumerate(kernels): + chan = spec["channels"] + length = spec["length"] + layout[j, :3] = (length, spec["dilation"], len(chan)) + layout[j, 3:3+len(chan)] = chan + weights[j, :length, :len(chan)] = spec["weights"] + biases[j] = spec["bias"] + max_offset = int(np.max((layout[:, 0]-1)*layout[:, 1])) + return layout, weights, biases, max_offset, bool(np.any(layout[:, 2] > 1)) + + +@dataclass +class RandomConvSketch(ref.RandomConvSketch): + fast_profile: bool = False + + def fit(self, X, y=None): + self.invalidate_plan() + return super().fit(X, y) + + def invalidate_plan(self): + """Call after custom in-place edits to learned kernel specifications.""" + self.__dict__.pop("_conv_plan", None) + self.__dict__.pop("_quant_plan", None) + + def __getstate__(self): + state = super().__getstate__().copy() + state.pop("_conv_plan", None) + state.pop("_quant_plan", None) + return state + + def transform(self, X): + x = ref._as_3d(X) + if not self.fast_profile and not getattr(self,"response_stride_",None) and not x.flags.c_contiguous: + return super().transform(X) + if self.zscore: + x = _normalize(x) + return self.transform_normalized(x) + + def transform_normalized(self, x): + """Apply the bank to an already prepared float64 time-major window tensor.""" + n, t, _ = x.shape + f = np.empty((n, len(self.kernels_), 4)) + stride = getattr(self,"response_stride_",4 if self.fast_profile else 1) + if (_cis_conv is not None and _kernels is not None and + self.kernels_ and n > 0 and stride <= t <= 8192): + plan = self.__dict__.get("_conv_plan") + if plan is None or plan[0] is not self.kernels_: + plan = (self.kernels_, _pack_kernels(self.kernels_)) + self._conv_plan = plan + layout, weights, biases, max_offset, multi = plan[1] + if t > max_offset and (stride>1 or self.fast_profile or not multi or + (t-max_offset > 1 and _cis_conv.has_blas())): + # A single N,C,T copy is shared by all kernels. Projection and + # pooling scratch storage is per window, independent of bank size. + data = np.ascontiguousarray(x.transpose(0,2,1), dtype=np.float64) + args = (data, layout, weights, biases, stride, self.fast_profile or stride>1, + stride==1 and not self.fast_profile and _verified_conv_blas(), f, + bool(getattr(self, "response_stride_", None))) + if getattr(self, 'projection_precision_', None) == 'float32': + _cis_conv.transform(*args, True) + else: + _cis_conv.transform(*args) + return _clean(f.reshape(n,-1)) + groups = defaultdict(list) + for k, spec in enumerate(self.kernels_): + groups[tuple(spec['channels']), spec['length'], spec['dilation']].append((k, spec)) + for (channels, length, dilation), group in groups.items(): + valid = t - (length - 1)*dilation + if valid <= 0: + raise ValueError('Random convolution kernel exceeds window length') + frames = (valid + stride - 1)//stride + native = _kernels is not None and (stride>1 or self.fast_profile or len(channels) == 1) + if native: + xch = np.ascontiguousarray(x[:, :, channels].transpose(2, 0, 1)) + else: + # Same channel-major advanced-index layout as upstream matmul. + dat = x[:, :, channels] + # Bound simultaneous kernel response storage to about 8 MiB. + block_size = max(1, (1 << 20)//max(n*frames, 1)) + for start in range(0, len(group), block_size): + parts = group[start:start+block_size] + idx = [k for k, _ in parts] + weights = np.stack([spec['weights'] for _, spec in parts]) + biases = np.asarray([spec['bias'] for _, spec in parts]) + resp = np.zeros((len(parts), n, frames)) + if native: + _kernels.project(xch, weights, biases, dilation, stride, resp) + else: + for i in range(length): + sub = dat[None, :, i*dilation:i*dilation+valid:stride, :] + resp += (sub @ weights[:, None, i, :, None])[..., 0] + resp += biases[:, None, None] + _pool_response(f, idx, resp) + return np.nan_to_num(f.reshape(n, -1), copy=False, nan=0., posinf=0., neginf=0.) + + +@dataclass +class IntervalDistributionSketch(ref.IntervalDistributionSketch): + fast_profile: bool = False + + def transform(self, X): + x = ref._as_3d(X) + if not self.fast_profile and not x.flags.c_contiguous: + return super().transform(X) + if self.zscore: + x = _normalize(x) + return self.transform_normalized(x) + + def transform_normalized(self, x, *, native_plan=None): + n, time, channels = x.shape + width = channels * (4 if self.fast_profile else (5 + len(self.quantiles))) + eligible=(_cis_intervals is not None and x.flags.c_contiguous and 2 <= channels <= 256 + and time <= 2048 and 0 < len(self.quantiles) <= 16 + and all(0 <= q <= 1 for q in self.quantiles) + and tuple(sorted(self.quantiles)) == tuple(self.quantiles)) + if eligible: + qs = np.asarray((0.5,) if self.fast_profile else self.quantiles, dtype=np.float64) + if (native_plan is None and n>=8 and time<=255 and 4<=len(self.intervals_)<=65536 + and hasattr(_cis_intervals,"levels")): + bounds=np.ascontiguousarray(self.intervals_,np.int64) + out=np.empty((n,len(bounds),width)) + if _cis_intervals.levels(x,bounds,qs,out,int(self.fast_profile)): + return out.reshape(n,-1) + if eligible and np.isfinite(x).all(): + def rank(): + if time<=256 and n>=8 and hasattr(_cis_intervals,"rank_short"): + ordered=np.empty((n,channels,time),dtype=np.int64) + if _cis_intervals.rank_short(x,ordered): + return ordered + return np.argsort(x.swapaxes(1,2), axis=-1).astype(np.int64,copy=False) + def evaluate(intervals, order): + bounds = np.asarray(intervals, dtype=np.int64).reshape(-1,2) + out = np.empty((n,len(bounds),width)) + _cis_intervals.intervals(x,bounds,order,qs,out,int(self.fast_profile)) + return _clean(out.reshape(n,-1)) + if native_plan is not None: + return native_plan(self, rank, evaluate) + return evaluate(self.intervals_, rank()) + out = np.empty((n, len(self.intervals_), width)) + intervals = defaultdict(list) + for idx, (lo, hi) in enumerate(self.intervals_): + intervals[hi-lo].append((idx, lo, hi)) + for length, group in intervals.items(): + group_size = max(1, (1 << 18) // max(1, n*length*channels)) + for start in range(0, len(group), group_size): + parts = group[start:start + group_size] + if len(parts) == 1: + v = x[:, parts[0][1]:parts[0][2]] + else: + v = np.concatenate([x[:, lo:hi] for _, lo, hi in parts], axis=0) + stats = [v.mean(axis=1), v.std(axis=1), np.ptp(v,axis=1)] + if self.fast_profile: + stats.append(np.median(v, axis=1)) + else: + stats.extend([(v*v).mean(axis=1), ref._slope(v), ref._quantile_block(v, self.quantiles)]) + ix = [idx for idx, _, _ in parts] + out[:,ix] = np.concatenate(stats, axis=1).reshape(len(ix), n, width).swapaxes(0,1) + return np.nan_to_num(out.reshape(n, -1), copy=False, nan=0., posinf=0., neginf=0.) + + +@dataclass +class InvariantPhysicsFeatureExtractor(ref.InvariantPhysicsFeatureExtractor): + fast_profile: bool = False + + def fit(self, X, y=None): + self.interval_ = IntervalDistributionSketch(n_random_intervals=self.random_intervals, + random_state=self.random_state, zscore=self.zscore, fast_profile=self.fast_profile).fit(X, y) + return self + + def transform(self, X): + x = ref._as_3d(X) + if not self.fast_profile and not x.flags.c_contiguous: + return super().transform(X) + x = _normalize(x) if self.zscore else np.nan_to_num(x) + blocks = [] + common = dict(zscore=False, include_magnitude=False, include_jerk=False, + segment_sizes=self.segment_sizes, fast_profile=self.fast_profile) + if self.include_axis_features: + axis = WearableFeatureExtractor(**common, include_spectral=False, + include_autocorr=False, include_symbolic=False) + blocks.append(axis.transform(x)) + base = x[:, :, :3] + mag = np.linalg.norm(base, axis=2, keepdims=True) + jerk = np.diff(base, axis=1, prepend=base[:, :1]) + jmag = np.linalg.norm(jerk, axis=2, keepdims=True) + inv = WearableFeatureExtractor(**common, include_cross_channel=False, symbolic_bins=self.symbolic_bins) + blocks.append(inv.transform(np.concatenate([mag, jmag], axis=2))) + centered = base - base.mean(axis=1,keepdims=True) + gram = centered.swapaxes(1,2) @ centered / max(1, x.shape[1]-1) + vals = np.maximum(np.linalg.eigvalsh(gram)[:, ::-1], 0) + ratios = vals / (vals.sum(axis=1, keepdims=True)+1e-12) + blocks.extend([vals, ratios, np.zeros((len(x), max(6-2*base.shape[2], 0)))]) + blocks.append((jmag*jmag).mean(axis=1)/((mag*mag).mean(axis=1)+1e-12)) + if self.interval_ is not None: + # The upstream interval branch z-scores its already normalized input. + blocks.append(self.interval_.transform(x)) + return _clean(np.concatenate(blocks, axis=1)) + + +@dataclass +class RandomShapeletSketch(ref.RandomShapeletSketch): + fast_profile: bool = False + + def transform(self, X): + x = ref._as_3d(X) + if not self.fast_profile and not x.flags.c_contiguous: + return super().transform(X) + if self.zscore: + x = _normalize(x) + return self.transform_normalized(x) + + def transform_normalized(self, x, *, window_tile=None): + n, time, _ = x.shape + groups = defaultdict(list) + for k, s in enumerate(self.shapelets_): + groups[s['channel'], s['dilation'], len(s['values'])].append((k, s['values'])) + f = np.empty((n, len(self.shapelets_)*2)) + stride = 4 if self.fast_profile else 1 + channels = {} + for (c, d, length), group in groups.items(): + span = (length-1)*d+1 + valid = time-span+1 + if valid < 1: + raise ValueError('Fitted shapelet exceeds input window length') + if _family_kernels is not None and length <= 256 and self.zscore: + if c not in channels: + channels[c] = np.require(x[:, :, c], dtype=np.float64, requirements=("C", "A")) + weights = np.stack([v for _, v in group]) + pooled = np.empty((n, len(group), 2)) + _family_kernels.shapelets(channels[c], weights, pooled, int(d), stride) + for j, (k, _) in enumerate(group): + f[:, 2*k:2*k+2] = pooled[:, j] + continue + grid = sliding_window_view(x[:, :, c], span, axis=1)[:, ::stride, ::d] + # Limit materialized normalized windows; sliding_window_view itself + # does not copy any data. Each row can be discarded after pooling. + chunk = max(1, (1 << 18)//(grid.shape[1]*length)) + for lo in range(0, n, chunk): + def normalize(): + part = np.ascontiguousarray(grid[lo:lo+chunk]) + block = part - part.mean(axis=2, keepdims=True) + sd = block.std(axis=2, keepdims=True) + return block / np.where(sd < 1e-6, 1., sd) + block = (normalize() if window_tile is None else + window_tile((c,d,length,stride,lo,chunk), normalize)) + for k, values in group: + delta = block - values + np.square(delta, out=delta) + dist = np.sqrt(delta.mean(axis=2)) + # NaN distances are skipped by upstream's strict '<' scan. + if not self.zscore: + dist[np.isnan(dist)] = np.inf + positions = dist.argmin(axis=1) + rows = slice(lo, lo+len(block)) + f[rows, 2*k] = dist[np.arange(len(block)), positions] + f[rows, 2*k+1] = positions*stride / max(valid-1, 1) + return np.nan_to_num(f, copy=False, nan=0., posinf=0., neginf=0.) diff --git a/src/wisp_fast/features_kernels.c b/src/wisp_fast/features_kernels.c new file mode 100644 index 0000000000000000000000000000000000000000..af21c8dadbc00ee96d36cd7b5d57710f2813c4c0 --- /dev/null +++ b/src/wisp_fast/features_kernels.c @@ -0,0 +1,88 @@ +#define PY_SSIZE_T_CLEAN +#include +#include + +static int get_doubles(PyObject *obj, Py_buffer *view, int ndim, int writable) { + int flags = PyBUF_FORMAT | PyBUF_C_CONTIGUOUS | (writable ? PyBUF_WRITABLE : 0); + if (PyObject_GetBuffer(obj, view, flags) < 0) + return -1; + if (view->ndim != ndim || view->itemsize != sizeof(double) || + strcmp(view->format, "d") != 0 || + ((uintptr_t)view->buf % _Alignof(double)) != 0) { + PyErr_SetString(PyExc_ValueError, "expected aligned, C-contiguous native doubles with the specified rank"); + return -1; + } + return 0; +} + +/* Project kernels of one geometry. Time-contiguous buffers permit ordinary + compiler vectorization of the inner loop. Pooling stays in NumPy to retain + its exact summation policy. resp is an output (it is cleared below). */ +static PyObject *project(PyObject *self, PyObject *args) { + PyObject *x_obj, *w_obj, *b_obj, *out_obj; + Py_ssize_t dilation, stride; + Py_buffer x = {0}, w = {0}, b = {0}, out = {0}; + if (!PyArg_ParseTuple(args, "OOOnnO", &x_obj, &w_obj, &b_obj, &dilation, &stride, &out_obj)) + return NULL; + if (get_doubles(x_obj, &x, 3, 0) < 0 || get_doubles(w_obj, &w, 3, 0) < 0 || + get_doubles(b_obj, &b, 1, 0) < 0 || get_doubles(out_obj, &out, 3, 1) < 0) + goto done; + + const Py_ssize_t c = x.shape[0], n = x.shape[1], t = x.shape[2]; + const Py_ssize_t k = w.shape[0], length = w.shape[1]; + if (c < 1 || c > 3 || t < 1 || length < 1 || dilation < 1 || stride < 1 || + length-1 > (t-1)/dilation || c != w.shape[2] || b.shape[0] != k || + out.shape[0] != k || out.shape[1] != n) { + PyErr_SetString(PyExc_ValueError, "incompatible projection dimensions or dilation/stride"); + goto done; + } + const Py_ssize_t frames = 1 + (t-1-(length-1)*dilation)/stride; + if (out.shape[2] != frames) { + PyErr_SetString(PyExc_ValueError, "incompatible output frame count"); + goto done; + } + const double *signals = (const double *)x.buf; + const double *weights = (const double *)w.buf; + const double *bias = (const double *)b.buf; + double *result = (double *)out.buf; + Py_BEGIN_ALLOW_THREADS + memset(result, 0, (size_t)out.len); + for (Py_ssize_t kernel = 0; kernel < k; ++kernel) { + for (Py_ssize_t sample = 0; sample < n; ++sample) { + double *r = result + (kernel*n + sample)*frames; + for (Py_ssize_t tap = 0; tap < length; ++tap) { + const double *v = weights + (kernel*length + tap)*c; + const double *a = signals + sample*t + tap*dilation; + if (c == 1) { + if (stride == 1) { + for (Py_ssize_t j=0; j 32768: + return + width = len(head.classes_) + rows = [] + for t in self.trees: + fields = [t.children_left, t.children_right, t.feature, t.threshold, + t.value, getattr(t, "missing_go_to_left", None)] + left, right, feat, thresh, value, missing = fields + if (missing is None or missing.dtype != np.uint8 or thresh.dtype != np.float64 + or any(a.dtype != np.intp for a in [left, right, feat]) + or value.dtype != np.float64 or value.shape != (t.node_count, 1, width) + or value.strides[0] < width * 8 or value.strides[-1] != 8 + or not value.flags.aligned or left.strides[0] <= 0 + or any(a.shape != (t.node_count,) or a.strides != left.strides + for a in (right, feat, thresh, missing))): + return + self._views.append(fields) # keep every buffer alive for the plan's lifetime + rows.append([a.ctypes.data for a in fields] + [left.strides[0], value.strides[0]]) + self.table = np.asarray(rows, dtype=np.uintp) + + +def local_proba(head, features, plan): + # sklearn performs its normal feature-count/dtype and finite input checks. + X = head._validate_X_predict(features) + k = len(head.classes_) + out = np.zeros((len(X), k), dtype=np.float64) + if plan.table is not None: + _native()(np.ascontiguousarray(X), len(X), X.shape[1], plan.table, len(plan.trees), k, out) + else: + for tree in plan.trees: + out += tree.predict(X)[:, :k] + out /= len(plan.trees) + return out diff --git a/src/wisp_fast/forest_kernels.c b/src/wisp_fast/forest_kernels.c new file mode 100644 index 0000000000000000000000000000000000000000..e9320443915652b620b9717b46a9bf77786a3704 --- /dev/null +++ b/src/wisp_fast/forest_kernels.c @@ -0,0 +1,40 @@ +#include +#include +#include + +/* Read the dtype/strides verified by forest.py; never assume an sklearn C struct layout. + The table contains pointers to non-owning array views pinned by the Python plan. + f32 inputs, f64 comparisons and tree-order f64 sums match sklearn's serial path. */ +static intptr_t read_index(uintptr_t p) { + intptr_t out; + memcpy(&out, (const void *)p, sizeof out); + return out; +} +static double read_double(uintptr_t p) { + double out; + memcpy(&out, (const void *)p, sizeof out); + return out; +} +#ifdef _WIN32 +__declspec(dllexport) +#endif +void wisp_forest(const float *x, intptr_t n, intptr_t d, + const uintptr_t *specs, intptr_t nt, intptr_t k, double *out) { + for (intptr_t t = 0; t < nt; ++t) { + const uintptr_t *tree = specs + 8 * t; + const uintptr_t stride = tree[6]; + for (intptr_t i = 0; i < n; ++i) { + intptr_t node = 0, left; + uintptr_t off = 0; + while ((left = read_index(tree[0] + off)) != -1) { + const float val = x[i*d + read_index(tree[2]+off)]; + const int go_left = isnan(val) ? *(const uint8_t *)(tree[5]+off) + : (double)val <= read_double(tree[3]+off); + node = go_left ? left : read_index(tree[1]+off); + off = (uintptr_t)node * stride; + } + const double *leaf = (const double *)(tree[4] + (uintptr_t)node * tree[7]); + for (intptr_t j = 0; j < k; ++j) out[i*k+j] += leaf[j]; + } + } +} diff --git a/src/wisp_fast/interval_levels.h b/src/wisp_fast/interval_levels.h new file mode 100644 index 0000000000000000000000000000000000000000..5d031ce934f4bfef3f28d5ea273bc5693c86e81a --- /dev/null +++ b/src/wisp_fast/interval_levels.h @@ -0,0 +1,84 @@ +/* Each bin may represent exactly one observed double. Detect collisions and + * let the original ranking routine take over. In each time-prefix vector, + * byte b counts samples with value <= bin b (T<=255). */ +#if defined(CAN_FILTER) +typedef struct {unsigned char lo,hi;double frac;} LevelCut; +__attribute__((target("avx512bw,avx512f"))) +static int level_cdf(const double *src,int t,int c,const int64_t *bounds,int m, + const LevelCut *cuts,int nq,int width,int turbo,double *out,int ch){ + double min=src[0],max=src[0],values[64];uint64_t used=0; + for(int j=0;jmax)max=v;} + /* Accepted values also bound every numeric interval moment far below DBL_MAX. */ + if(min < -1.e140 || max > 1.e140)return 0; + double span=max-min,scale=span?63./span:0.; + if(!isfinite(scale)||!isfinite(span))return 0; + unsigned char key[256]; + for(int j=0;j63)k=63; + uint64_t bit=UINT64_C(1)<lo))); + int hi=__builtin_ctzll(_mm512_cmpgt_epu8_mask(f,_mm512_set1_epi8((char)s->hi))); + double x=values[lo],y=values[hi],q=s->frac; + dst[(width-nq)*c+ch*nq+h]=turbo?((n&1)?x:(x+y)*.5):(q>=.5?y-(y-x)*(1.-q):x+(y-x)*q); + } + } + return 1; +} +#endif +static PyObject *interval_levels(PyObject *self,PyObject *args){ + PyObject *obj[4];int turbo;Py_buffer v[4]={{0}};int ok=0; + if(!PyArg_ParseTuple(args,"OOOOi",&obj[0],&obj[1],&obj[2],&obj[3],&turbo))return NULL; + if(!view(obj[0],v,3,'d',0)||!view(obj[1],v+1,2,'q',0)||!view(obj[2],v+2,1,'d',0)|| + !view(obj[3],v+3,3,'d',1)||!separate_output(v,3))goto done; + Py_ssize_t n=v[0].shape[0],t=v[0].shape[1],c=v[0].shape[2],m=v[1].shape[0],nq=v[2].shape[0]; + int width=(turbo?3:5)+(int)nq;const double *q=v[2].buf;const int64_t *bounds=v[1].buf; + if(n<8||t<1||t>255||c<2||c>256||m<1||m>65536||nq<1||nq>16||v[1].shape[1]!=2|| + v[3].shape[0]!=n||v[3].shape[1]!=m||v[3].shape[2]!=c*width|| + (turbo && (nq!=1||q[0]!=.5))){ + PyErr_SetString(PyExc_ValueError,"unsupported bounded CDF geometry");goto done; + } + for(int j=0;j1){PyErr_SetString(PyExc_ValueError,"invalid quantile");goto done;} + for(int j=0;jt||bounds[2*j]>=bounds[2*j+1]){ + PyErr_SetString(PyExc_ValueError,"invalid interval");goto done;} +#ifdef CAN_FILTER + if(__builtin_cpu_supports("avx512bw") && CAN_FILTER){ + LevelCut *cuts=PyMem_Malloc(m*nq*sizeof(*cuts)); + double *tile=PyMem_Malloc(8*t*sizeof(double)),*out=v[3].buf;const double *x=v[0].buf; + if(!cuts || !tile){PyMem_Free(cuts);PyMem_Free(tile);PyErr_NoMemory();goto done;} + for(int j=0;j +#define MOM512 __attribute__((target("avx512f"))) +typedef __m512d MomTile; +static MOM512 inline MomTile gather_rows(const double *x,Py_ssize_t pitch) { + return _mm512_set_pd(x[7*pitch],x[6*pitch],x[5*pitch],x[4*pitch],x[3*pitch],x[2*pitch],x[pitch],x[0]); +} +static MOM512 inline void put_rows(double *out,Py_ssize_t pitch,MomTile value) { + double lane[8];_mm512_storeu_pd(lane,value);for(int i=0;i<8;++i)out[i*pitch]=lane[i]; +} +static MOM512 MomTile sum_rows(const MomTile *x,int n,int paired) { + if(!paired||n<8){ + MomTile s=_mm512_set1_pd(paired?-0.:0.);for(int j=0;j128){int mid=n/2/8*8;return sum_rows(x,mid,1)+sum_rows(x+mid,n-mid,1);} + MomTile a=x[0],b=x[1],c=x[2],d=x[3],e=x[4],f=x[5],g=x[6],h=x[7]; + int j=8; + for(;j+8<=n;j+=8){a+=x[j];b+=x[j+1];c+=x[j+2];d+=x[j+3];e+=x[j+4];f+=x[j+5];g+=x[j+6];h+=x[j+7];} + MomTile s=((a+b)+(c+d))+((e+f)+(g+h));for(;j1){cross/=(double)(t-1);slope/=((double)t*((double)t*t-1)/12.);} + else{cross=zero;slope=zero;} + put_rows(dst+7*c,p,cross);put_rows(dst+8*c,p,slope); + put_rows(dst+9*c,p,sum_rows(cub,t,paired)/(double)t); + put_rows(dst+10*c,p,sum_rows(four,t,paired)/(double)t); + } +} +static MOM512 void seg_tile8(const double *x,int t,int c,int pieces,int fields,double *out) { + int len=(t+pieces-1)/pieces;Py_ssize_t pitch=(Py_ssize_t)t*c,p=fields*pieces*c; + MomTile v[256],sq[256]; + for(int s=0;s1)slope/=((double)len*((double)len*len-1)/12.);else slope=_mm512_setzero_pd();put_rows(dst+3*w,p,slope);} + } +} +#endif diff --git a/src/wisp_fast/native.py b/src/wisp_fast/native.py new file mode 100644 index 0000000000000000000000000000000000000000..12161b54e1926f60e679c1f9364e10986c0e21b2 --- /dev/null +++ b/src/wisp_fast/native.py @@ -0,0 +1,16 @@ +"""Look up optional CPU symbols from the installed extension or standalone C build.""" +import ctypes +from importlib.util import find_spec +from pathlib import Path + + +def lookup(symbol): + ext = find_spec("wisp_fast._kernels") + paths = [ext.origin] if ext is not None and ext.origin else [] + paths.append(Path(__file__).with_name("_wisp_viterbi.so")) + for path in paths: + try: + return getattr(ctypes.CDLL(str(path)), symbol) + except (OSError, AttributeError): + continue + return None diff --git a/src/wisp_fast/parallel.py b/src/wisp_fast/parallel.py new file mode 100644 index 0000000000000000000000000000000000000000..74339697f894107b6289b2ce9b3dd49fa96ecff8 --- /dev/null +++ b/src/wisp_fast/parallel.py @@ -0,0 +1,35 @@ +"""Threaded row-local work; sequence decoding is performed once on complete emissions.""" +from concurrent.futures import ThreadPoolExecutor +from contextvars import copy_context + +import numpy as np +from threadpoolctl import threadpool_limits +from wisp.cpu.probability import labels_from_proba +from wisp_release.checkpoints import require_sequence_metadata + + +def parallel_rows(fn, X, workers): + if not isinstance(workers,(int,np.integer)) or workers < 1: + raise ValueError("workers must be a positive integer") + workers=min(workers,max(1,len(X)//32)) + if workers==1: + return fn(X) + bounds=np.linspace(0,len(X),workers+1,dtype=np.intp) + with threadpool_limits(limits=1, user_api="blas"), ThreadPoolExecutor(max_workers=workers) as pool: + jobs=[pool.submit(copy_context().run,fn,X[a:b]) for a,b in zip(bounds[:-1],bounds[1:])] + first=jobs[0].result() + out=np.empty((len(X),*first.shape[1:]),dtype=first.dtype) + out[:bounds[1]]=first + for a,b,job in zip(bounds[1:-1],bounds[2:],jobs[1:]): + out[a:b]=job.result() + return out + + +def predict_parallel(model,X,*,groups=None,time_index=None,workers=4): + if getattr(model,"feature_workers",1)>1: + raise ValueError("Use either model feature workers or outer prediction workers") + groups,time_index=require_sequence_metadata(model,groups,time_index,n=len(X)) + P=parallel_rows(model.predict_proba,np.asarray(X),workers) + if model.smoother_ is None: + return labels_from_proba(P,model.classes_) + return model.smoother_.predict_from_proba(P,groups=groups,time_index=time_index) diff --git a/src/wisp_fast/ridge_fast.py b/src/wisp_fast/ridge_fast.py new file mode 100644 index 0000000000000000000000000000000000000000..1eb581a731d0132fefe10343b17e36d0f9b9662d --- /dev/null +++ b/src/wisp_fast/ridge_fast.py @@ -0,0 +1,172 @@ +"""Dense weighted leave-one-out Ridge classification for WISP RC/CIS. + +Implements the mean squared, binary-indicator LOO objective of +RidgeClassifierCV(cv=None, scoring=None, fit_intercept=True). Uses the primal +covariance for tall matrices and a centered sample Gram matrix otherwise. +Spectral operations reorder floating-point arithmetic relative to sklearn. +""" +import numpy as np +from scipy.linalg import eigh +from scipy.linalg.blas import dsyrk +from sklearn.base import BaseEstimator, ClassifierMixin +from sklearn.linear_model import RidgeClassifier, RidgeClassifierCV +from sklearn.preprocessing import LabelBinarizer +from sklearn.utils.validation import check_X_y, check_array, check_is_fitted + + +class SpectralRidgeClassifierCV(ClassifierMixin, BaseEstimator): + def __init__(self, *, alphas=(0.1, 1., 10.)): + self.alphas = alphas + + def fit(self, X, y, sample_weight=None): + X, y = check_X_y(X, y, dtype=np.float64, ensure_min_samples=2) + alpha = np.atleast_1d(np.asarray(self.alphas, dtype=np.float64)) + if alpha.ndim != 1 or not len(alpha) or np.any(~np.isfinite(alpha)) or np.any(alpha <= 0): + raise ValueError("alphas must be finite, positive and one-dimensional") + n, d = X.shape + sw = np.ones(n) if sample_weight is None else np.asarray(sample_weight, dtype=np.float64) + if sw.shape != (n,) or not np.isfinite(sw).all() or np.any(sw < 0) or sw.sum() <= 0: + raise ValueError("sample_weight must be nonnegative, finite, positive in total and length N") + self.n_features_in_ = d + encoder = LabelBinarizer(neg_label=-1, pos_label=1) + targets = encoder.fit_transform(y).astype(np.float64) + self.classes_ = encoder.classes_ + def fallback(reason): + # Exact rounding and first-alpha tie policy of the installed reference. + model = RidgeClassifierCV(alphas=alpha).fit(X, y, sample_weight=sw) + self.coef_, self.intercept_ = model.coef_, model.intercept_ + self.alpha_, self.best_score_ = model.alpha_, model.best_score_ + self.solver_used_ = "sklearn_" + reason + self.loo_scores_ = [] + return self + if len(self.classes_) < 2 or np.any(sw == 0): + return fallback("degenerate") + root = np.sqrt(sw) + total = sw.sum() + offset = np.average(X, axis=0, weights=sw) + bias = np.average(targets, axis=0, weights=sw) + A = np.empty((n, d), dtype=np.float64, order="F") + np.subtract(X, offset, out=A) + A *= root[:, None] + B = (targets-bias)*root[:, None] + + # For nearly square designs the sample system avoids multiplying a + # second large eigenvector matrix into every feature row. + primal = 2*d < n + if not primal: + # Orthogonally remove the intercept before building the Gram matrix. + # H = I - 2uu^T maps sqrt(weight) to -||sqrt(weight)|| e0. + # Only rows 1..N-1 of H A and H B are needed for the solve. + u = root.copy() + u[0] += np.sqrt(total) + u /= np.linalg.norm(u) + A[1:] -= (2 * u[1:, None]) * (u @ A) + A = np.asfortranarray(A[1:]) + B = B[1:] - (2 * u[1:, None]) * (u @ B) + gram = dsyrk(1., A, trans=int(primal), lower=1) + try: + vals, vec = eigh(gram, lower=True, overwrite_a=True, check_finite=False, driver="evd") + except np.linalg.LinAlgError: + return fallback("eigensolver") + del gram + if not primal: + reflect_row = (2*u[1:]) @ vec + if np.min(vals) < -1e-4 * np.min(alpha): + return fallback("conditioning") + weights = 1. / (np.maximum(vals, 0.)[:, None] + alpha) + rhs = vec.T @ ((A.T @ B) if primal else B) + projections = (weights[:, :, None] * rhs[:, None, :]).reshape(len(vals), -1) + loss = np.zeros(len(alpha), dtype=float) + for lo in range(0, n, 192): + sl = slice(lo, min(n, lo+192)) + if primal: + Z = A[sl] @ vec + else: + # Selected rows of H [0; Q], expressed in the original sample order. + Z = -u[sl, None] * reflect_row + if lo == 0: + Z[1:] += vec[:len(Z)-1] + else: + Z += vec[lo-1:sl.stop-1] + diagonal = (Z*Z) @ weights + numerator = (Z @ projections).reshape(len(Z), len(alpha), B.shape[1]) + if primal: + diagonal = 1 - sw[sl, None]/total - diagonal + numerator = B[sl, None, :] - numerator + if not np.isfinite(diagonal).all() or np.any(diagonal <= np.finfo(float).eps*32): + return fallback("leverage") + relative = numerator / diagonal[:, :, None] + loss += np.sum(relative*relative, axis=(0, 2)) + loss /= (n * targets.shape[1]) + if not np.isfinite(loss).all(): + return fallback("nonfinite") + winner = int(np.argmin(loss)) + if len(alpha) > 1 and np.partition(loss, 1)[1]-loss[winner] <= 1e-10*(1.+loss[winner]): + return fallback("score_tie") + scaled = weights[:, winner, None] * rhs + coef = vec @ scaled + if not primal: + coef = A.T @ coef + self.coef_ = coef.T.copy() + self.intercept_ = bias - offset @ coef + self.alpha_, self.best_score_ = float(alpha[winner]), float(-loss[winner]) + self.loo_scores_ = -loss + self.solver_used_ = "primal_eigh" if primal else "dual_eigh" + return self + + def decision_function(self, X): + check_is_fitted(self, "coef_") + X = check_array(X, dtype=np.float64) + if X.shape[1] != self.n_features_in_: + raise ValueError("Wrong number of input features") + score = X @ self.coef_.T + self.intercept_ + return score.ravel() if score.ndim == 1 or score.shape[1] == 1 else score + + def predict(self, X): + dec = self.decision_function(X) + if len(self.classes_) == 1: + return np.full(dec.shape[0], self.classes_[0]) + return self.classes_[(dec > 0).astype(int) if dec.ndim == 1 else dec.argmax(axis=1)] + + +class ScreenedRidgeClassifierCV(SpectralRidgeClassifierCV): + """Estimate the alpha on a training subsample, then refit on every row. + + Inclusion weights for each class preserve its total training mass in the + screening step. Alpha selection is approximate when screening_rows < N. + """ + def __init__(self, *, alphas=(0.1, 1., 10.), screening_rows=512, random_state=42): + super().__init__(alphas=alphas) + self.screening_rows = screening_rows + self.random_state = random_state + + def fit(self, X, y, sample_weight=None): + if not isinstance(self.screening_rows, (int, np.integer)) or self.screening_rows < 2: + raise ValueError("screening_rows must be an integer >=2") + X, y = check_X_y(X, y, dtype=np.float64, ensure_min_samples=2) + if len(y) <= self.screening_rows: + self.screening_rows_ = len(y) + return super().fit(X, y, sample_weight=sample_weight) + from .engine import _training_rows + sw = np.ones(len(y)) if sample_weight is None else np.asarray(sample_weight, dtype=np.float64) + if sw.shape != (len(y),) or not np.isfinite(sw).all() or np.any(sw < 0) or sw.sum() <= 0: + raise ValueError("Invalid sample weights") + ids = _training_rows(y, self.screening_rows, self.random_state) + _, encoded = np.unique(y, return_inverse=True) + mass = np.bincount(encoded, weights=sw) + submass = np.bincount(encoded[ids], weights=sw[ids], minlength=len(mass)) + if np.any(submass <= 0): + self.screening_rows_ = len(y) + return super().fit(X, y, sample_weight=sw) + weights = sw[ids] * (mass/submass)[encoded[ids]] + screen = SpectralRidgeClassifierCV(alphas=self.alphas).fit(X[ids], y[ids], sample_weight=weights) + self.alpha_, self.best_score_, self.loo_scores_ = screen.alpha_, screen.best_score_, screen.loo_scores_ + self.screening_rows_ = len(ids) + self.n_features_in_ = X.shape[1] + return self._refit_at_alpha(X, y, sw) + + def _refit_at_alpha(self, X, y, sw): + model = RidgeClassifier(alpha=self.alpha_, solver="cholesky").fit(X, y, sample_weight=sw) + self.coef_, self.intercept_, self.classes_ = model.coef_, model.intercept_, model.classes_ + self.solver_used_ = "screened_loo_full_cholesky" + return self diff --git a/src/wisp_fast/search.py b/src/wisp_fast/search.py new file mode 100644 index 0000000000000000000000000000000000000000..d5db49922e880f12fd09a837b15154bc06635913 --- /dev/null +++ b/src/wisp_fast/search.py @@ -0,0 +1,298 @@ +"""WISP-Random / WISP-Evolution search with split-local feature sharing. + +The proposal grammar, cascade, stress views and utility are those of +``wisp.search.controller``; candidates are fitted with the accelerated engine. +""" +from concurrent.futures import ThreadPoolExecutor +from contextvars import copy_context +from dataclasses import replace +from functools import partial +import json +import pickle +from pathlib import Path +import time + +import numpy as np +from wisp.core.data import concatenate_har +from wisp.core.metrics import classification_report_dict +from wisp.cpu.algorithms import estimate_pickle_size_mb +from wisp.search.cascade import STRESS_TRANSFORMS, evaluate_candidate_cascade +from wisp.search.controller import WISPSearchController +from wisp.search.grammar import CandidateSpec +from wisp.search.profiles import get_profile +from .cache import _SEARCH +from .decoder import check_sequence +from .engine import RIDGE_SOLVERS, build_candidate +from .selection import check_splits, fit_on +from .search_cache import SearchFeatures +from .search_metrics import search_report,SearchConfusion + + +def make_candidate(spec_dict, *, mode="exact", budget=None, ridge_solver="original", sequence="subject", + calibration=None): + """Accelerated estimator for one search candidate. + + ``calibration`` (a validation HARData) is used only with ``sequence="recording"``: + right after fitting, the confidence scaling is chosen on those windows. + """ + if ridge_solver not in RIDGE_SOLVERS or (mode!="exact" and ridge_solver!="original"): + raise ValueError("The screened Ridge solver requires exact full-size features") + model=build_candidate(spec_dict,mode=mode,budget=budget,ridge_solver=ridge_solver,sequence=sequence) + if sequence=="recording" and calibration is not None: + model.calibration_data=(calibration.X,calibration.y) + return model + + +class _AcceleratedSearch(WISPSearchController): + def _evaluate_many(self, specs): + # Random proposals are score-independent. Evolution may prune only after + # generating its terminal batch; every breeding decision is then complete. + terminal = self.rows_seen+len(specs) == self.profile.max_evaluations + eligible = self.profile.search_strategy == "random" or terminal + self.batch_bound = self.incumbent if self.stress_policy=="certified" and eligible else None + # Update only between completed waves. Certificates always reference + # earlier canonical ledger rows, independent of worker completion order. + if self.stress_policy=="frontier" and eligible: + rows=[] + width=max(self._worker_count(),1) + for start in range(0,len(specs),width): + self.batch_bound=self.incumbent + chunk=self._evaluate_batch(specs[start:start+width]) + self._observe(chunk,self.rows_seen+start) + rows.extend(chunk) + else: + rows = self._evaluate_batch(specs) + self._observe(rows,self.rows_seen) + self.rows_seen += len(rows) + return rows + + def _observe(self,rows,start): + for index,(status,result) in enumerate(rows,start): + if status == "ok": + score = result["predictive_utility"] + if self.incumbent is None or score > self.incumbent[0]: + self.incumbent = (score,index) + elif status == "pruned": + self.pruned_count += 1 + self.skipped_views += result["prune_certificate"]["skipped_views"] + + def _stress_gate(self, metrics, observed): + # Macro-F1 <= 1. The stress contribution w*s - .05*max(clean-s,0) + # is monotone on [0,1] for w>=0. No equality pruning, so ties survive. + score,index = self.batch_bound + remaining = self.stress_count-len(observed) + mean_upper = float(np.mean(observed+[1.0]*remaining)) if observed else 1.0 + gap_lower = max(float(metrics["macro_f1"])-mean_upper,0.0) + upper = float(metrics["score"] + self.robustness_weight*mean_upper - .05*gap_lower) + if np.isfinite(upper) and upper < score: + return dict(upper_bound=upper,incumbent_utility=score, + incumbent_ledger_index=index,stress_f1_upper=mean_upper, + observed_stress_f1=list(observed),skipped_views=remaining) + return None + + def _evaluate_batch(self,specs): + if self._worker_count() == 1 or len(specs)<2: + return super()._evaluate_many(specs) + with ThreadPoolExecutor(max_workers=min(self._worker_count(),len(specs))) as pool: + jobs=[pool.submit(copy_context().run,self._evaluate_one,s) for s in specs] + return [job.result() for job in jobs] + + def _batched_stress(self, model): + # Each stress view is a distinct sequence panel. Never Viterbi-decode + # a concatenated panel, even if the same subject IDs appear in each. + P=model.predict_proba(self.stress_bundle) + chunks=np.split(P,self.stress_count) + return {name:model.decode_proba(p,groups=self.valid.subject,time_index=self.valid.time_index) + for name,p in zip(self.stress_names,chunks)} + + def _evaluate_one(self, spec): + token=_SEARCH.set(self.memory if self.memory.limit else None) + built=[] + + def factory(spec_dict): + built.append(self.create_candidate(spec_dict)) + return built[-1] + try: + cascade=evaluate_candidate_cascade( + spec,self.train,self.valid,estimator_factory=factory, + cost_weight=self.cost_weight,robustness_weight=self.robustness_weight, + max_stress_transforms=self.stress_count,stress_inputs=self.stress_inputs, + reporter=self.reporter,stress_gate=self._stress_gate if self.batch_bound is not None else None, + stress_predictor=self._batched_stress if self.stress_bundle is not None else None) + result=cascade.to_dict() + if self.sequence=="recording" and built: + smoother=getattr(built[-1],"smoother_",None) + result["sequence"]="recording" + result["sequence_smoothing"]=bool(smoother is not None and smoother.active) + result["confidence_scale"]=built[-1].confidence_scale + if "prune_certificate" in result: + del result["cascade_score"] + del result["accepted"] # stress and final eligibility were not evaluated + return "pruned",result + result["predictive_utility"]=float(result["cascade_score"]) + return ("ok" if cascade.accepted else "rejected",result) + except Exception as exc: + return ("failed",{"predictive_utility":-1e9,"breeding_score":-1e9, + "error":repr(exc),"candidate":spec.to_dict()}) + finally: + _SEARCH.reset(token) + + def _final_refit(self, best, out_dir): + """Refit the validation winner on train+validation with the accelerated engine.""" + if self.test is None: + return None + model = self.create_candidate(best["spec"]) + model.__dict__.pop("calibration_data", None) + refit = concatenate_har(self.train, self.valid, source="train+validation") + t0 = time.perf_counter() + fit_on(model, refit) + fit_s = time.perf_counter() - t0 + # A train+validation refit reuses the scaling chosen for the train-only candidate. + model.set_confidence_scale(best["result"].get("confidence_scale")) + t1 = time.perf_counter() + pred = model.predict(self.test.X, groups=self.test.subject, time_index=self.test.time_index) + pred_s = time.perf_counter() - t1 + metrics = classification_report_dict(self.test.y, pred, fit_seconds=fit_s, predict_seconds=pred_s, + model_size_mb=estimate_pickle_size_mb(model), + label_names=self.test.label_names) + np.savez_compressed(out_dir / "locked_test_predictions.npz", y_true=self.test.y, y_pred=pred, + subject=self.test.subject, time_index=self.test.time_index) + with (out_dir / "selected_model.pkl").open("wb") as f: + pickle.dump(model, f) + (out_dir / "locked_test_metrics.json").write_text( + json.dumps(metrics, indent=2, ensure_ascii=False, default=str), encoding="utf-8") + return metrics + + +def _configure(search, mode, budget, ridge_solver, cache_mb, stress_cache_mb, stress_count, + stress_policy="complete", sequence="subject"): + if stress_policy not in {"complete","certified","frontier"}: + raise ValueError("stress_policy must be complete, certified or frontier") + if stress_policy in ("certified","frontier") and (stress_count==0 or not np.isfinite(search.robustness_weight) + or search.robustness_weight<0): + raise ValueError("Certified bounds require stress_count>0 and finite nonnegative robustness_weight") + search.stress_policy = stress_policy + search.reporter=SearchConfusion(search.valid.y,search.train.n_classes) if stress_policy=="frontier" else search_report + search.incumbent = search.batch_bound = None + search.pruned_count = search.skipped_views = search.rows_seen = 0 + if mode not in ("exact","turbo") or ridge_solver not in RIDGE_SOLVERS: + raise ValueError("Unknown search execution mode or Ridge solver") + if mode=="turbo" and ridge_solver!="original": + raise ValueError("Turbo uses its fixed Ridge alpha") + if cache_mb<0 or stress_cache_mb<0 or not isinstance(stress_count,int) or not 0<=stress_count<=4: + raise ValueError("Invalid cache size or stress count") + search.sequence=check_sequence(sequence) + search.create_candidate=partial(make_candidate,mode=mode,budget=budget,ridge_solver=ridge_solver, + sequence=sequence,calibration=search.valid) + X=search.valid.X + search.stress_count=stress_count + search.stress_names=list(STRESS_TRANSFORMS)[:stress_count] + search.stress_bundle=None + search.stress_inputs={} + remaining=stress_cache_mb*1024**2 + if stress_count>1 and stress_count*len(X)<=1024 and stress_count*X.nbytes<=min(remaining,16*1024**2): + arr=np.empty((stress_count*len(X),*X.shape[1:]),dtype=X.dtype) + for i,name in enumerate(search.stress_names): + arr[i*len(X):(i+1)*len(X)]=STRESS_TRANSFORMS[name](X) + arr.flags.writeable=False + search.stress_bundle=arr + else: + for name in search.stress_names: + if X.nbytes<=remaining: + transformed=STRESS_TRANSFORMS[name](X) + transformed.flags.writeable=False + search.stress_inputs[name]=transformed + remaining-=transformed.nbytes + arrays=[search.train.X,X,*search.stress_inputs.values()] + if search.stress_bundle is not None:arrays.append(search.stress_bundle) + search.memory=SearchFeatures(cache_mb,arrays) + return search + + +def evaluate_recipes(specs,train,valid,*,mode="exact",budget=None,workers=1,n_jobs=1,seed=42, + cost_weight=.01,robustness_weight=.1,cache_mb=192,stress_cache_mb=128, + ridge_solver="original",stress_count=4,sequence="subject"): + """Evaluate an explicit ordered sequence with fresh split-local caches.""" + check_splits(train,valid) + if min(workers,n_jobs)<1:raise ValueError("Worker counts must be positive") + controller=_AcceleratedSearch(train,valid,".",seed=seed,candidate_workers=workers, + n_jobs_per_candidate=n_jobs,cost_weight=cost_weight,robustness_weight=robustness_weight) + _configure(controller,mode,budget,ridge_solver,cache_mb,stress_cache_mb,stress_count,sequence=sequence) + specs=[s if isinstance(s,CandidateSpec) else CandidateSpec.from_dict(s) for s in specs] + # Preserve recorded effective candidate parameters; supply missing job settings. + specs=[controller._runtime_spec(s) if "n_jobs" not in s.params else s for s in specs] + rows=controller._evaluate_many(specs) + stats=controller.memory.stats() + controller.memory.clear() + return rows, stats + + +def run_search(method, train, valid, test=None, *, output, population, generations, max_evaluations, + seed=42, candidate_workers=1, n_jobs_per_candidate=1, ridge_solver="original", + sequence="subject", cost_weight=0.01, robustness_weight=0.1, cache_mb=192, + stress_cache_mb=128): + """Run WISP-Random or WISP-Evolution with explicit population/generations/budget. + + Selection uses validation only. With ``test``, the winner is refitted once on + train+validation and scored on test (``selected_model.pkl``, ``locked_test_*``). + """ + check_splits(*[part for part in (train, valid, test) if part is not None]) + profile = replace(get_profile(method), max_evaluations=int(max_evaluations)) + search = _AcceleratedSearch(train=train, valid=valid, output=output, profile=profile, test=test, + population=population, generations=generations, seed=seed, + cost_weight=cost_weight, robustness_weight=robustness_weight, + candidate_workers=candidate_workers, n_jobs_per_candidate=n_jobs_per_candidate) + _configure(search, "exact", None, ridge_solver, cache_mb, stress_cache_mb, + 4 if profile.use_stress else 0, "complete", sequence=sequence) + try: + summary = search.run() + finally: + search.memory.clear() + return summary + + +def fit_search(method,train,valid,*,output,mode="exact",seed=42,evaluations=None,workers=1, + n_jobs=1,population=24,cost_weight=0.01,robustness_weight=0.1, + stress_cache_mb=128,budget=None,cache_mb=192,ridge_solver="original", + refit=True,stress_count=4,family_scope="all",stress_policy="complete",sequence="subject"): + """Search with the paper's default budget (329 proposals) and population, then refit. + + Same proposal grammar and default stress/utility as the search controller. Training and + clean prediction times include actual shared-cache costs, so rankings may change.""" + check_splits(train,valid) + if min(workers,n_jobs)<1 or population<3:raise ValueError("Invalid parallelism or population") + scopes={"all":None, "cis":("state_rocket_interval_hmm",), + "cis-related":("state_rocket_interval_hmm","rocket_sketch_ridge_hmm","posterior_cpu_ensemble")} + if family_scope not in scopes:raise ValueError("family_scope must be all, cis or cis-related") + profile=replace(get_profile(method),allowed_cpu_families=scopes[family_scope]) + limit=(profile.max_evaluations if mode=="exact" else 25) if evaluations is None else evaluations + if not isinstance(limit,int) or limit<1:raise ValueError("evaluations must be a positive integer") + out=Path(output) + out.mkdir(parents=True,exist_ok=True) + if any(out.iterdir()):raise ValueError("Search requires a fresh empty output directory per run") + per_generation=max(population-max(2,population//3),1) + generations=max(1,(limit-min(9,limit)+per_generation-1)//per_generation) + search=_AcceleratedSearch(train,valid,out,profile=replace(profile,max_evaluations=limit), + population=population,generations=generations,seed=seed,candidate_workers=workers, + n_jobs_per_candidate=n_jobs,cost_weight=cost_weight,robustness_weight=robustness_weight) + _configure(search,mode,budget,ridge_solver,cache_mb,stress_cache_mb,stress_count,stress_policy,sequence) + summary=search.run() + if summary["n_evaluated"]!=limit:raise RuntimeError("Controller did not complete evaluation budget") + stats=search.memory.stats() + search.memory.clear() + model=None + if refit: + model=search.create_candidate(summary["best"]["spec"]) + model.__dict__.pop("calibration_data",None) + fit_on(model,concatenate_har(train,valid,source="train+validation")) + model.set_confidence_scale(summary["best"]["result"].get("confidence_scale")) + summary.update(backend="wisp-fast-shared",mode=mode,ridge_solver=ridge_solver,sequence=sequence, + family_scope=family_scope, + fit_scope="train+validation" if refit else "candidate selection only",selection_uses_test=False, + cached_stress=search.stress_names if search.stress_bundle is not None else list(search.stress_inputs), + stress_count=stress_count,stress_policy=stress_policy,pruned_candidates=search.pruned_count, + skipped_stress_views=search.skipped_views,batched_stress=search.stress_bundle is not None,feature_cache=stats, + evaluations=limit,winner_recipe=summary["best"]["spec"], + effective_budget=(vars(budget) if budget is not None else "mode defaults")) + (out/"summary.json").write_text(json.dumps(summary,indent=2,default=str)) + return model, summary diff --git a/src/wisp_fast/search_cache.py b/src/wisp_fast/search_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..ed8f5c6581d5fb80c62c803fa3617f564838a8f8 --- /dev/null +++ b/src/wisp_fast/search_cache.py @@ -0,0 +1,166 @@ +"""Search-scoped reuse of deterministic representation components on immutable splits.""" +from collections import OrderedDict +from copy import copy +import hashlib +from threading import RLock +import weakref +from functools import lru_cache + +import numpy as np + +from wisp.cpu import features as ref +from . import features as fx + + +@lru_cache(maxsize=128) +def _dyadic(time): + # Mirrors the release order, including deduplication for short windows. + bounds = [] + for parts in (2, 4, 8): + width = max(2, (time+parts-1)//parts) + bounds.extend((lo,min(lo+width,time)) for lo in range(0,time,width) + if min(lo+width,time)-lo >= 2) + return tuple(dict.fromkeys(bounds)) + + +class SearchFeatures: + """A private per-run LRU; share only deterministic blocks, never train labels or heads.""" + def __init__(self, limit_mb, arrays): + self.limit = int(limit_mb * 1024**2) + self.known = {id(arr): (arr, i) for i, arr in enumerate(arrays)} + self.derived = {} + self.store = OrderedDict() + # Tile data have much greater cardinality than the stable feature + # blocks. Keep a separate bounded LRU to avoid flushing the hot set. + self.tiles = OrderedDict() + self.tile_limit = self.limit // 8 + self.tile_bytes = self.tile_hits = self.rank_hits = self.dyad_hits = 0 + self.bytes = self.peak = self.hits = self.misses = self.evictions = 0 + self.lock = RLock() + + def clear(self): + self.store.clear() + self.known.clear() + self.derived.clear() + self.tiles.clear() + self.tile_bytes = 0 + self.bytes = 0 + + def stats(self): + return dict(limit_bytes=self.limit, peak_bytes=self.peak, hits=self.hits, + misses=self.misses, evictions=self.evictions, shapelet_tile_hits=self.tile_hits, + interval_rank_hits=self.rank_hits, dyadic_block_hits=self.dyad_hits) + + def token(self, x): + val = self.known.get(id(x)) + if val is not None and x is val[0]: + return val[1] + with self.lock: + entry = self.derived.get(id(x)) + if entry is not None and entry[0]() is x: + return entry[1] + # Derived arrays (e.g. train-row cap and temporal downsampling) must + # never be keyed by shape alone. Their allocation may change each fit. + return (x.shape, x.dtype.str, x.strides, + hashlib.blake2b(np.ascontiguousarray(x).view(np.uint8), digest_size=16).digest()) + + def sampled(self, x, steps, sample): + key = (self.token(x),"time_sample",steps) + arr = self.get(key,sample) + # Weak entries tie identity to live allocations, even after eviction. + idx = id(arr) + def expired(ref): + with self.lock: + item = self.derived.get(idx) + if item is not None and item[0] is ref: + self.derived.pop(idx) + with self.lock: + self.derived[idx] = (weakref.ref(arr,expired),key) + return arr + + def get(self, key, fn, *, tile=False): + store = self.tiles if tile else self.store + limit = self.tile_limit if tile else self.limit - self.tile_limit + with self.lock: + if key in store: + self.hits += 1 + if tile:self.tile_hits += 1 + if key[1]=="interval_rank":self.rank_hits += 1 + if key[1]=="dyad":self.dyad_hits += 1 + store.move_to_end(key) + return store[key] + self.misses += 1 + arr = fn() + if arr.nbytes <= limit: + with self.lock: + if key not in store: + used = self.tile_bytes if tile else self.bytes + while used + arr.nbytes > limit: + _, old = store.popitem(last=False) + used -= old.nbytes + self.evictions += 1 + store[key] = arr + used += arr.nbytes + if tile:self.tile_bytes = used + else:self.bytes = used + self.peak = max(self.peak, self.bytes+self.tile_bytes) + return arr + + def transform(self, blocks, x): + token = self.token(x) + def zscore(): + return self.get((token, "z"), lambda: fx._normalize(ref._as_3d(x))) + def interval(block, data, norm): + def native_plan(b, rank, evaluate): + order = self.get((token,"interval_rank",norm),rank) + fixed = _dyadic(data.shape[1]) if b.include_dyadic else () + if not fixed or tuple(b.intervals_[:len(fixed)]) != fixed: + return evaluate(b.intervals_,order) + params = (norm,b.fast_profile,tuple(b.quantiles),fixed) + common = self.get((token,"dyad",params),lambda:evaluate(fixed,order)) + suffix = b.intervals_[len(fixed):] + return np.concatenate((common,evaluate(suffix,order)),axis=1) if suffix else common + return block.transform_normalized(data,native_plan=native_plan) + features=[] + for b in blocks: + if type(b) is fx.WearableFeatureExtractor: + # Stateless window-local features. Constructor parameters fully + # specify columns, including turbo choices. Fitted names are + # cheap to copy and restored on warm hits for pickle-size parity. + params=tuple(b.get_params(deep=False).items()) + key=(token, "wear", params) + def calculate(): + return b.transform_normalized(zscore()) if x.flags.c_contiguous and b.zscore else b.transform(x) + value=self.get(key,calculate) + if not b.feature_names_: + b.transform(x[:1]) + elif type(b) is fx.InvariantPhysicsFeatureExtractor and x.flags.c_contiguous and b.interval_ is not None: + # Upstream concatenates deterministic physics with a separate + # seeded random-interval branch. Share the former across seeds. + key=(token,"physics",b.segment_sizes,b.symbolic_bins,b.zscore,b.fast_profile,b.include_axis_features) + def fixed(): + base=copy(b) + base.interval_=None + return base.transform(x) + common=self.get(key,fixed) + xn=zscore() if b.zscore else np.nan_to_num(ref._as_3d(x)) + if b.interval_.zscore: + xn=self.get((token,"physics_second_z",b.zscore),lambda:fx._normalize(xn)) + intervals=interval(b.interval_,xn,("physics",b.zscore,b.interval_.zscore)) + value=np.concatenate((common,intervals),axis=1) + elif type(b) is fx.IntervalDistributionSketch and x.flags.c_contiguous and b.zscore: + value=interval(b,zscore(),"z") + elif type(b) is fx.RandomShapeletSketch and x.flags.c_contiguous and b.zscore: + def tile(key, fn): + return self.get((token,"shapelet_windows",key),fn,tile=True) + value=b.transform_normalized(zscore(),window_tile=tile) + elif type(b) is fx.RandomConvSketch and x.flags.c_contiguous and b.zscore: + value=b.transform_normalized(zscore()) + else: + # Fitted shapelets depend on actual training examples; they + # are deliberately evaluated with the candidate's own object. + value=b.transform(x) + features.append(value) + capture=getattr(self,"capture",None) + if capture is not None:capture(x,blocks,features) + return features diff --git a/src/wisp_fast/search_forest.py b/src/wisp_fast/search_forest.py new file mode 100644 index 0000000000000000000000000000000000000000..d9ad944f67d40fc84236e186c24ff36ba164d557 --- /dev/null +++ b/src/wisp_fast/search_forest.py @@ -0,0 +1,45 @@ +"""Avoid repeated sklearn parameter introspection for built-in WISP forest trees.""" +from contextlib import contextmanager + +import numpy as np +from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier +from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier +from sklearn.utils import check_random_state + + +@contextmanager +def forest_fit_plan(forest): + """Use the same per-tree constructor parameters and sklearn seed stream. + + The hook is local to this one estimator and removed before it is serialized. + Tree growth, bootstrap samples, sample/class weights and parallel schedules + remain sklearn's own. Only the two built-in release forest types qualify. + """ + if type(forest) not in (RandomForestClassifier, ExtraTreesClassifier): + yield + return + prototype = forest.estimator + if type(prototype) not in (DecisionTreeClassifier,ExtraTreeClassifier): + yield + return + constructor = type(prototype) + params = prototype.get_params(deep=False) + params.update({name:getattr(forest,name) for name in forest.estimator_params}) + max_seed = np.iinfo(np.int32).max + + def make_estimator(append=True,random_state=None): + # sklearn._set_random_states draws one int for these non-nested trees. + if random_state is not None: + params["random_state"] = check_random_state(random_state).randint(max_seed) + else: + params["random_state"] = forest.random_state + tree = constructor(**params) + if append: + forest.estimators_.append(tree) + return tree + + forest._make_estimator = make_estimator + try: + yield + finally: + del forest._make_estimator diff --git a/src/wisp_fast/search_metrics.py b/src/wisp_fast/search_metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..dae0625c83638f14462b68a5d2a1e152a4850bbe --- /dev/null +++ b/src/wisp_fast/search_metrics.py @@ -0,0 +1,48 @@ +"""Single-confusion-matrix form of the *search* scorer, including observed-axis worst F1.""" +import numpy as np + + +def search_report(y_true, y_pred, *, label_names=None, **timing): + truth=np.asarray(y_true,dtype=np.int64) + pred=np.asarray(y_pred,dtype=np.int64) + labels, encoded=np.unique(np.concatenate((truth,pred)),return_inverse=True) + k=len(labels) + if not k: + from wisp.core.metrics import classification_report_dict + return classification_report_dict(truth,pred,label_names=label_names,**timing) + t,p=np.split(encoded,2) + cm=np.bincount(t*k+p,minlength=k*k).reshape(k,k) + return _matrix_report(cm,labels,len(truth),label_names,timing) + + +def _matrix_report(cm,labels,n,label_names,timing): + gt=cm.sum(axis=1); predicted=cm.sum(axis=0); tp=cm.diagonal() + f1=(2.*tp)/(gt+predicted) + present=gt>0 + r=dict(accuracy=float(tp.sum()/n),balanced_accuracy=float(np.mean(tp[present]/gt[present])), + macro_f1=float(f1.mean()),weighted_f1=float(np.average(f1,weights=gt)), + worst_class_f1=float(f1.min()),n=n,n_classes_seen=len(labels), + per_class_f1={str(int(i)):float(f) for i,f in zip(labels,f1)},confusion_matrix=cm.tolist()) + if label_names is not None:r["label_names"]=label_names + r.update({k:float(v) for k,v in timing.items() if v is not None}) + return r + + +class SearchConfusion: + """Reuse the encoded truth offsets for clean and perturbed predictions.""" + def __init__(self,truth,k): + self.truth=truth + self.k=k + self.offset=None + arr=np.asarray(truth) + if 0=0 and arr.max()=self.k): + return search_report(truth,pred,label_names=label_names,**timing) + cm=np.bincount(self.offset+pred,minlength=self.k*self.k).reshape(self.k,self.k) + labels=np.flatnonzero(cm.sum(0)+cm.sum(1)) + return _matrix_report(cm[np.ix_(labels,labels)],labels,len(pred),label_names,timing) diff --git a/src/wisp_fast/selection.py b/src/wisp_fast/selection.py new file mode 100644 index 0000000000000000000000000000000000000000..75f5754241176e6fd67a12f1cc7c0812200bfb53 --- /dev/null +++ b/src/wisp_fast/selection.py @@ -0,0 +1,26 @@ +"""Split checks and the fit call shared by the accelerated search.""" +import numpy as np + + +def check_splits(*parts): + """Require sequence metadata, one global vocabulary and disjoint group IDs.""" + first = parts[0] + if not first.label_names: + raise ValueError("Supply the ordered global label_names") + seen = set() + for data in parts: + if data.label_names != first.label_names or data.X.shape[1:] != first.X.shape[1:]: + raise ValueError("Split vocabularies and time/channel shapes must match") + if data.subject is None or data.time_index is None or not len(data.y): + raise ValueError("Every split needs nonempty windows, subject and time_index") + if np.min(data.y) < 0 or np.max(data.y) >= len(first.label_names): + raise ValueError("Class ID outside supplied global vocabulary") + groups = set(np.asarray(data.subject).astype(str)) + if groups & seen: + raise ValueError("Participant/sequence groups overlap between splits") + seen |= groups + + +def fit_on(model, data): + return model.fit(data.X, data.y, groups=data.subject, time_index=data.time_index, + n_classes=data.n_classes) diff --git a/src/wisp_fast/shapelet_tiles.h b/src/wisp_fast/shapelet_tiles.h new file mode 100644 index 0000000000000000000000000000000000000000..f4927a3d816da0cba0a8d78c9e6af629a0a2d5d2 --- /dev/null +++ b/src/wisp_fast/shapelet_tiles.h @@ -0,0 +1,62 @@ +/* Compare eight consecutive candidate start positions of the same shapelet. + Each lane retains NumPy's internal 8-way double reduction and its second + centering pass in std. Strict chronological comparisons preserve first ties. */ +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) && !defined(_WIN32) +#include +#define SHAPE512 __attribute__((target("avx512f"))) +typedef __m512d PositionTile; +static SHAPE512 inline PositionTile tile_sum(const PositionTile *x, int len) { + if(len<8) { + PositionTile s=_mm512_set1_pd(-0.); + for(int j=0;j8)lanes=8; + __mmask8 active=(__mmask8)((1U<1?stop-1:1); + } + } + } + } + } +} +#endif diff --git a/src/wisp_fast/viterbi.c b/src/wisp_fast/viterbi.c new file mode 100644 index 0000000000000000000000000000000000000000..75f29c55ef45bfb11e501945a74e9efa5830f9d6 --- /dev/null +++ b/src/wisp_fast/viterbi.c @@ -0,0 +1,85 @@ +#include +#include + +/* Inputs are contiguous with predecessor-major A; first index wins ties. + Workspace is sized by the Python caller and reused for every recording. */ +#ifdef _WIN32 +__declspec(dllexport) +#endif +void wisp_viterbi(const double *e, const double *a, const double *pi, + int64_t t_count, int64_t k_count, int32_t *back, + double *prev, double *next, int64_t *path) { + for (int64_t j = 0; j < k_count; ++j) prev[j] = pi[j] + e[j]; + for (int64_t t = 1; t < t_count; ++t) { + int32_t *row = back + t * k_count; + for (int64_t j = 0; j < k_count; ++j) { + next[j] = prev[0] + a[j]; + row[j] = 0; + } + for (int64_t i = 1; i < k_count; ++i) { + const double *arow = a + i * k_count; + for (int64_t j = 0; j < k_count; ++j) { + const double v = prev[i] + arow[j]; + if (v > next[j]) { next[j] = v; row[j] = (int32_t)i; } + } + } + for (int64_t j = 0; j < k_count; ++j) next[j] += e[t * k_count + j]; + double *swap = prev; prev = next; next = swap; + } + int64_t last = 0; + for (int64_t j = 1; j < k_count; ++j) if (prev[j] > prev[last]) last = j; + path[t_count - 1] = last; + for (int64_t t = t_count - 1; t > 0; --t) { + last = back[t * k_count + last]; + path[t - 1] = last; + } +} + +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) && !defined(_WIN32) +#include +/* Hold the winning score and its earliest predecessor in registers while + scanning a transition column tile. No padding or sentinel class needed. */ +__attribute__((target("avx512f"))) +static void viterbi_tile(const double *e,const double *a,const double *pi, + int64_t n,int64_t k,int32_t *back,double *p,double *q,int64_t *path) { + for(int64_t j=0;jp[last])last=j; + path[n-1]=last; + for(int64_t t=n-1;t>0;--t){last=back[t*k+last];path[t-1]=last;} +} +#endif + +#ifdef _WIN32 +__declspec(dllexport) +#endif +void wisp_viterbi_groups(const double *e,const double *a,const double *pi, + const int64_t *bounds,int64_t ng,int64_t k, + int32_t *back,double *prev,double *next,int64_t *path) { + for(int64_t g=0;g=5 && __builtin_cpu_supports("avx512f")) + viterbi_tile(e+start*k,a,pi,n,k,back,prev,next,path+start); + else +#endif + wisp_viterbi(e+start*k,a,pi,n,k,back,prev,next,path+start); + } +} diff --git a/src/wisp_release/__init__.py b/src/wisp_release/__init__.py index 4c3f382d8345ce6b99b6f1fecbc57a28250f1a0a..6e5712ddad0b5cb9ce68897ebca35e8d8b1b6a0e 100644 --- a/src/wisp_release/__init__.py +++ b/src/wisp_release/__init__.py @@ -1,8 +1,9 @@ """Small release interfaces for WISP; importing this package performs no work.""" from .checkpoints import load_checkpoint, predict_checkpoint -from .methods import list_methods +from .methods import SEQUENCES, SPEEDS, build_family, list_methods from .selection import fit_selector -__all__ = ["load_checkpoint", "predict_checkpoint", "list_methods", "fit_selector"] -__version__ = "0.1.0" +__all__ = ["load_checkpoint", "predict_checkpoint", "list_methods", "build_family", "fit_selector", + "SPEEDS", "SEQUENCES"] +__version__ = "0.2.0" diff --git a/src/wisp_release/cli.py b/src/wisp_release/cli.py index 7084d3fec98e662a3f496b7c2b7f2dc6dc2b4547..89617bcd7fb1e35e5ba6079a43e3972a1d9307ba 100644 --- a/src/wisp_release/cli.py +++ b/src/wisp_release/cli.py @@ -13,7 +13,24 @@ from typing import Any from .checkpoints import load_checkpoint, predict_checkpoint, sha256_file from .data import as_har_data, read_npz -from .methods import FIXED_METHODS, SEARCH_METHODS, SELECTION_METHODS, build_family, list_methods +from .methods import (FIXED_METHODS, SEARCH_METHODS, SELECTION_METHODS, SEQUENCES, SPEEDS, + RIDGE_SOLVER_FOR_SPEED, build_family, list_methods) + +__version__ = "0.2.0" + + +def _decoder(model: Any) -> str: + smoother = getattr(model, "smoother_", None) + if smoother is None: + return "direct" + if getattr(smoother, "sequence", "subject") == "recording" and not getattr(smoother, "active", True): + return "direct (recording-level smoothing skipped)" + return "sequence" + + +def _sequence(model: Any) -> str | None: + smoother = getattr(model, "smoother_", None) + return None if smoother is None else getattr(smoother, "sequence", "subject") def _json(payload: Any) -> str: @@ -87,7 +104,7 @@ def _predict(args: argparse.Namespace, *, evaluate: bool = False) -> dict[str, A else int(max(np.max(data["y"]), np.max(prediction))) + 1) result.update(score(data["y"], prediction, n_classes)) result.update({"checkpoint": str(args.checkpoint), "input": str(args.input), - "decoder": "sequence" if getattr(model, "smoother_", None) is not None else "direct", + "decoder": _decoder(model), "sequence": _sequence(model), "metric_contract": "paper: macro_F1 on observed truth/prediction classes; worst_F1 on full global vocabulary", "test_usage": "This command evaluates; it does not select or refit a model."}) _write_new(args.output, _json(result)) @@ -104,7 +121,7 @@ def _predict(args: argparse.Namespace, *, evaluate: bool = False) -> dict[str, A with args.output.open("xb") as stream: np.savez_compressed(stream, **arrays) return {"output": str(args.output), "n_predictions": len(prediction), - "decoder": "sequence" if getattr(model, "smoother_", None) is not None else "direct"} + "decoder": _decoder(model), "sequence": _sequence(model)} def _training_plan(args: argparse.Namespace) -> dict[str, Any]: @@ -128,12 +145,25 @@ def _fit(args: argparse.Namespace) -> dict[str, Any]: if not isinstance(overrides, dict): raise ValueError("--overrides must be a JSON object") model = build_family(args.method, seed=args.seed, n_jobs=args.n_jobs, - direct=args.direct, overrides=overrides) - if getattr(model, "use_hmm", False) and (dataset.subject is None or dataset.time_index is None): + direct=args.direct, overrides=overrides, speed=args.speed, sequence=args.sequence) + if model.use_hmm and (dataset.subject is None or dataset.time_index is None): raise ValueError("Sequence-decoder fitting requires both subject/session IDs and time_index; " "use --direct only if intentionally fitting a different direct-decoder variant") + valid = None + if args.valid is not None: + if args.sequence != "recording" or not model.use_hmm: + raise ValueError("--valid is used only with --sequence recording and HMM smoothing") + valid = as_har_data(read_npz(args.valid, require_y=True), n_classes=dataset.n_classes, + source=str(args.valid)) + if valid.label_names != dataset.label_names or valid.X.shape[1:] != dataset.X.shape[1:]: + raise ValueError("--valid must share the training label_names and time/channel shape") + elif args.sequence == "recording" and model.use_hmm: + raise ValueError("--sequence recording needs --valid: the confidence scaling of classifier " + "scores is chosen on the validation split") model.fit(dataset.X, dataset.y, groups=dataset.subject, time_index=dataset.time_index, n_classes=dataset.n_classes) + if valid is not None: + model.fit_confidence_scale(valid.X, valid.y) args.output.parent.mkdir(parents=True, exist_ok=True) with args.output.open("xb") as stream: pickle.dump(model, stream, protocol=pickle.HIGHEST_PROTOCOL) @@ -141,8 +171,11 @@ def _fit(args: argparse.Namespace) -> dict[str, Any]: "label_names": dataset.label_names, "model_seed": args.seed, "fit_scope": "explicit input NPZ; no automatic benchmark split or refit", "use_hmm": getattr(model, "smoother_", None) is not None, + "speed": args.speed, "ridge_solver": model.ridge_solver, "sequence": args.sequence, + "sequence_smoothing": _decoder(model) == "sequence", + "confidence_scale": model.confidence_scale, "source_status": "newly_fitted", "sha256": sha256_file(args.output), - "params": model.get_params(deep=False)} + "params": model.template.get_params(deep=False)} _write_new(metadata_path, _json(metadata)) return {"checkpoint": str(args.output), "metadata": str(metadata_path), "sha256": metadata["sha256"], "fit_seconds": getattr(model, "fit_seconds_", None)} @@ -160,7 +193,8 @@ def _select_fit(args: argparse.Namespace) -> dict[str, Any]: raise FileExistsError(f"Refusing to overwrite {path}") train, valid = [as_har_data(read_npz(path, require_y=True), n_classes=args.n_classes, source=str(path)) for path in (args.train, args.valid)] - model, report = fit_selector(args.method, train, valid, seed=args.seed, n_jobs=args.n_jobs) + model, report = fit_selector(args.method, train, valid, seed=args.seed, n_jobs=args.n_jobs, + speed=args.speed, sequence=args.sequence) args.output.parent.mkdir(parents=True, exist_ok=True) with args.output.open("xb") as stream: pickle.dump(model, stream, protocol=pickle.HIGHEST_PROTOCOL) @@ -168,6 +202,7 @@ def _select_fit(args: argparse.Namespace) -> dict[str, Any]: "input_shape": list(train.X.shape[1:]), "label_names": train.label_names, "model_seed": args.seed, "fit_scope": "train+validation", "use_hmm": getattr(model, "smoother_", None) is not None, + "speed": args.speed, "sequence": args.sequence, "source_status": "newly_fitted", "sha256": sha256_file(args.output), "selection_uses_test": False} _write_new(metadata_path, _json(metadata)) @@ -180,8 +215,8 @@ def _select_fit(args: argparse.Namespace) -> dict[str, Any]: def _search(args: argparse.Namespace) -> dict[str, Any]: if not args.execute: return _training_plan(args) - from wisp.search.controller import WISPSearchController from wisp.search.profiles import get_profile + from wisp_fast.search import run_search if args.candidate_workers > args.n_jobs: raise ValueError("candidate_workers cannot exceed the explicit total n_jobs budget") @@ -202,13 +237,14 @@ def _search(args: argparse.Namespace) -> dict[str, Any]: # Users supply locked partitions; this command never invents a split, and # selection stays on validation inside the preserved search controller. profile = replace(get_profile(args.method), max_evaluations=args.max_evaluations) - controller = WISPSearchController( - train=parts[0], valid=parts[1], test=parts[2], output=args.output, - profile=profile, population=args.population, generations=args.generations, - seed=args.seed, candidate_workers=args.candidate_workers, + result = run_search( + args.method, parts[0], parts[1], parts[2], output=args.output, + population=args.population, generations=args.generations, + max_evaluations=args.max_evaluations, seed=args.seed, + candidate_workers=args.candidate_workers, n_jobs_per_candidate=max(1, args.n_jobs // args.candidate_workers), + ridge_solver=RIDGE_SOLVER_FOR_SPEED[args.speed], sequence=args.sequence, ) - result = controller.run() model_path = args.output / "selected_model.pkl" if model_path.is_file(): sidecar = args.output / "selected_model.metadata.json" @@ -216,7 +252,8 @@ def _search(args: argparse.Namespace) -> dict[str, Any]: "label_names": parts[0].label_names, "model_seed": args.seed, "fit_scope": "train+validation; selection uses validation only", "sha256": sha256_file(model_path), "source_status": "newly_fitted", - "search_profile": profile.to_dict(), "selection_uses_test": False} + "search_profile": profile.to_dict(), "speed": args.speed, + "sequence": args.sequence, "selection_uses_test": False} if not sidecar.exists(): _write_new(sidecar, _json(metadata)) return result @@ -229,9 +266,24 @@ def _positive(value: str) -> int: return result +SPEED_HELP = ("exact (default): original Ridge solver, outputs identical to the original " + "implementation. fast: Ridge strength chosen on a 512-row training subsample, then " + "fitted on all rows; faster, results can change slightly. Affects Ridge-based models only.") +SEQUENCE_HELP = ("subject (default): all windows of one participant form one HMM chain, ordered by " + "time index (original behaviour). recording: each recording forms its own chain; a new " + "recording starts whenever the time index goes backwards; the confidence scaling of " + "classifier scores is chosen on the validation split; smoothing is skipped when most " + "training recordings contain a single activity.") + + +def _add_options(command: argparse.ArgumentParser) -> None: + command.add_argument("--speed", choices=SPEEDS, default="exact", help=SPEED_HELP) + command.add_argument("--sequence", choices=SEQUENCES, default="subject", help=SEQUENCE_HELP) + + def make_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="wisp", description="WISP method checkpoints and CPU workflows") - parser.add_argument("--version", action="version", version="WISP release 0.1.0") + parser.add_argument("--version", action="version", version=f"WISP release {__version__}") sub = parser.add_subparsers(dest="command", required=True) sub.add_parser("list", help="List the full paper method series; no model loads") inv = sub.add_parser("inventory", help="Read inventory/file presence; no model loads") @@ -255,6 +307,9 @@ def make_parser() -> argparse.ArgumentParser: fit.add_argument("--n-classes", type=_positive) fit.add_argument("--direct", action="store_true", help="Intentionally fit without the sequence decoder") fit.add_argument("--overrides", help="Explicit JSON parameter overrides (changes the trained variant)") + fit.add_argument("--valid", type=Path, + help="Validation NPZ; required with --sequence recording to choose the confidence scaling") + _add_options(fit) fit.add_argument("--execute", action="store_true", help="Actually fit; otherwise print a plan only") select = sub.add_parser("select-fit", help="Plan or explicitly select a fixed family on validation and refit once") select.add_argument("--method", choices=tuple(SELECTION_METHODS), required=True) @@ -264,8 +319,9 @@ def make_parser() -> argparse.ArgumentParser: select.add_argument("--seed", type=int, required=True) select.add_argument("--n-jobs", type=_positive, required=True) select.add_argument("--n-classes", type=_positive) + _add_options(select) select.add_argument("--execute", action="store_true", help="Actually select/refit; otherwise print a plan only") - search = sub.add_parser("search", help="Plan or explicitly run the preserved CPU search controller") + search = sub.add_parser("search", help="Plan or explicitly run the WISP-Random/WISP-Evolution CPU search") search.add_argument("--method", choices=tuple(SEARCH_METHODS), required=True) for part in ("train", "valid", "test"): search.add_argument(f"--{part}", type=Path, required=True) @@ -277,6 +333,7 @@ def make_parser() -> argparse.ArgumentParser: search.add_argument("--n-jobs", type=_positive, required=True, help="Total CPU thread budget") search.add_argument("--candidate-workers", type=_positive, default=1) search.add_argument("--n-classes", type=_positive) + _add_options(search) search.add_argument("--execute", action="store_true", help="Actually search; otherwise print a plan only") return parser diff --git a/src/wisp_release/methods.py b/src/wisp_release/methods.py index 5814791594fc014a45627dc29ccff9a2b5c36267..ebc2952b5590454286375f344b7290b07f55316d 100644 --- a/src/wisp_release/methods.py +++ b/src/wisp_release/methods.py @@ -44,9 +44,22 @@ def list_methods() -> list[dict[str, Any]]: return out -def build_family(method_id: str, *, seed: int, n_jobs: int = 1, - direct: bool = False, overrides: dict[str, Any] | None = None) -> Any: - """Construct one fixed WISP family; construction does not fit it.""" +SPEEDS = ("exact", "fast") +SEQUENCES = ("subject", "recording") +# speed -> Ridge solver of the accelerated engine +RIDGE_SOLVER_FOR_SPEED = {"exact": "original", "fast": "screened"} + + +def check_options(speed: str, sequence: str) -> None: + if speed not in SPEEDS: + raise ValueError(f"speed must be one of {', '.join(SPEEDS)}; got {speed!r}") + if sequence not in SEQUENCES: + raise ValueError(f"sequence must be one of {', '.join(SEQUENCES)}; got {sequence!r}") + + +def build_reference_family(method_id: str, *, seed: int, n_jobs: int = 1, + direct: bool = False, overrides: dict[str, Any] | None = None) -> Any: + """Construct the reference (unaccelerated) estimator of one fixed WISP family.""" if method_id not in FIXED_METHODS: raise ValueError(f"family-fit supports fixed families only: {', '.join(FIXED_METHODS)}") if n_jobs < 1: @@ -57,3 +70,30 @@ def build_family(method_id: str, *, seed: int, n_jobs: int = 1, if direct: params["use_hmm"] = False return build_method(method_id, seed=seed, n_jobs=n_jobs, overrides=params) + + +def build_family(method_id: str, *, seed: int, n_jobs: int = 1, + direct: bool = False, overrides: dict[str, Any] | None = None, + speed: str = "exact", sequence: str = "subject") -> Any: + """Construct one fixed WISP family; construction does not fit it. + + speed: + ``"exact"`` (default) uses the original Ridge solver; outputs match the + original implementation. ``"fast"`` chooses the Ridge regularisation + strength on a 512-row training subsample and then fits on all rows; it is + faster and results can change slightly. Only models with a Ridge + classifier (WISP-RC, WISP-CIS and the RC/CIS members of WISP-CSE and + WISP-ESE) are affected. + sequence: + ``"subject"`` (default): all windows of one participant form one HMM chain + ordered by time index (the original behaviour). ``"recording"``: each + recording forms its own chain, and a new recording starts whenever the + time index goes backwards. Call ``model.fit_confidence_scale(X_valid, + y_valid)`` after fitting to choose the confidence scaling on validation data. + """ + check_options(speed, sequence) + template = build_reference_family(method_id, seed=seed, n_jobs=n_jobs, direct=direct, + overrides=overrides) + from wisp_fast.engine import with_ridge_solver + + return with_ridge_solver(template, ridge_solver=RIDGE_SOLVER_FOR_SPEED[speed], sequence=sequence) diff --git a/src/wisp_release/selection.py b/src/wisp_release/selection.py index c9a395c1cb92ecd18ec438548c7012e536c36e5e..55f67c738c50b0fe8158a6d3604f8267cf14b177 100644 --- a/src/wisp_release/selection.py +++ b/src/wisp_release/selection.py @@ -6,7 +6,7 @@ import math from typing import Any, Callable from .checkpoints import predict_checkpoint -from .methods import FIXED_METHODS, build_family +from .methods import FIXED_METHODS, build_family, build_reference_family, check_options from .scoring import score SELECTOR_ORDER = tuple(FIXED_METHODS) @@ -38,15 +38,28 @@ def _model_size_mb(model: Any) -> float: return estimate_pickle_size_mb(model) +def _quality(record: dict[str, Any]) -> tuple[float, float]: + return float(record["macro_f1"]), float(record["worst_class_f1"]) + + def fit_selector(method_id: str, train: Any, valid: Any, *, seed: int, n_jobs: int = 1, - builder: Callable[..., Any] | None = None) -> tuple[Any, dict[str, Any]]: + builder: Callable[..., Any] | None = None, speed: str = "exact", + sequence: str = "subject") -> tuple[Any, dict[str, Any]]: """Fit/evaluate fixed families on validation, then refit the winner once. No test input is accepted. ``train`` and ``valid`` are HARData objects with the same global vocabulary, shape and disjoint participant/session IDs. ``builder`` is a testing/customization seam; default canonical settings are the preserved paper registry. Changing a builder changes the experiment. + ``speed`` and ``sequence`` are passed to :func:`wisp_release.methods.build_family`. + With ``sequence="recording"`` each candidate's confidence scaling is chosen on + validation, and the final train+validation refit reuses the winner's factor. + + Model size only breaks exact ties on both validation scores. It is measured on + the reference (unaccelerated) estimator fitted on training data, as in the + original selector, and is reported only for tied candidates. """ + check_options(speed, sequence) import numpy as np from wisp.core.data import concatenate_har @@ -64,28 +77,48 @@ def fit_selector(method_id: str, train: Any, valid: Any, *, seed: int, n_jobs: i if train_groups & valid_groups: raise ValueError("Participant/session leakage between train and validation") eligible = SELECT5_FAMILIES if method_id == "wisp_select5" else SELECTOR_ORDER + custom = builder is not None builder = builder or build_family - candidates = [] + candidates, fitted, scales = [], {}, {} for family in eligible: - model = builder(family, seed=seed, n_jobs=n_jobs) + model = builder(family, seed=seed, n_jobs=n_jobs, speed=speed, sequence=sequence) model.fit(train.X, train.y, groups=train.subject, time_index=train.time_index, n_classes=train.n_classes) + if sequence == "recording" and hasattr(model, "fit_confidence_scale"): + model.fit_confidence_scale(valid.X, valid.y) + scales[family] = model.confidence_scale prediction = predict_checkpoint(model, valid.X, subject=valid.subject, time_index=valid.time_index, n_jobs=n_jobs) candidates.append({"method_id": family, **score(valid.y, prediction, valid.n_classes), - "model_size_mb": _model_size_mb(model)}) - winner = choose_validation_winner(candidates) + "model_size_mb": None}) + fitted[family] = model + best = max(map(_quality, candidates)) + tied = [record for record in candidates if _quality(record) == best] + for record in tied if len(tied) > 1 else (): + family = record["method_id"] + if custom: + size_model = fitted[family] + else: + size_model = build_reference_family(family, seed=seed, n_jobs=n_jobs) + size_model.fit(train.X, train.y, groups=train.subject, time_index=train.time_index, + n_classes=train.n_classes) + record["model_size_mb"] = _model_size_mb(size_model) + winner = tied[0]["method_id"] if len(tied) == 1 else choose_validation_winner(tied) combined = concatenate_har(train, valid, source="train+validation") - model = builder(winner, seed=seed, n_jobs=n_jobs) + model = builder(winner, seed=seed, n_jobs=n_jobs, speed=speed, sequence=sequence) model.fit(combined.X, combined.y, groups=combined.subject, time_index=combined.time_index, n_classes=combined.n_classes) + if scales.get(winner) is not None: + model.set_confidence_scale(scales[winner]) report = {"method_id": method_id, "selected_method_id": winner, + "speed": speed, "sequence": sequence, "confidence_scale": scales.get(winner), "validation_candidates": candidates, "fit_scope": "train+validation", "selection_uses_test": False, "model_seed": seed, "tie_break": ["higher validation macro_f1", "higher full-vocabulary worst_class_f1", "smaller fitted model default-protocol pickle size", "canonical family order"], - "serialized_size_caveat": "Module namespace renaming may alter serialized byte counts; " - "refitted models are not claimed binary-identical to historical files.", + "serialized_size_caveat": "Sizes are measured only for candidates tied on both scores, on the " + "reference estimator; refitted models are not claimed binary-identical " + "to historical files.", "canonical_family_order": list(SELECTOR_ORDER)} return model, report diff --git a/tests/test_cis_features.py b/tests/test_cis_features.py new file mode 100644 index 0000000000000000000000000000000000000000..4bdc0ad36d1324597fc092008e7324aff40f2efb --- /dev/null +++ b/tests/test_cis_features.py @@ -0,0 +1,54 @@ +"""Backends for faithful finite normalization, ACF, symbolic thresholds and rank queries.""" +import unittest +from unittest.mock import patch +import warnings + +import numpy as np +from wisp_fast import features as fx + + +class FaithfulCIS(unittest.TestCase): + @unittest.skipIf(fx._family_kernels is None,"native kernels unavailable") + def test_normalization_acf(self): + rng=np.random.default_rng(643) + for t in (1,7,8,9,127,128,129,160,257,2049): + for c in (1,2,3,7): + x=rng.normal(size=(4,t,c));x[1]=3e4;x[2]=np.rint(x[2]) + x[3,::3,0]=-0. + ref=fx.ref._safe_zscore_per_window(x) + np.testing.assert_array_equal(fx._normalize(x),ref) + lags=(1,0,t,3,7,1) + np.testing.assert_array_equal(fx._autocorr(x,lags),fx.ref._autocorr_features(x,lags)) + neg=np.full((2,129,1),-0.) + self.assertEqual(fx._normalize(neg).tobytes(),fx.ref._safe_zscore_per_window(neg).tobytes()) + neg[:,::2]=0. + self.assertEqual(fx._autocorr(neg,(1,2)).tobytes(),fx.ref._autocorr_features(neg,(1,2)).tobytes()) + x=np.array([[[np.nan,0],[np.inf,1],[2,3]],[[0,1],[0,2],[0,3]]]) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", RuntimeWarning) + np.testing.assert_array_equal(fx._normalize(x),fx.ref._safe_zscore_per_window(x)) + with self.assertRaises(ValueError):fx._family_kernels.normalize(ref,ref) + with self.assertRaises(ValueError):fx._family_kernels.autocorr(ref,(1,),ref) + + @unittest.skipIf(fx._cis_intervals is None,"native interval kernels unavailable") + def test_range_select_boundaries_and_degenerate(self): + rng=np.random.default_rng(137) + for t in (17,63,64,65,127,128,129,160,255,256,257): + x=np.round(rng.normal(size=(5,t,3)),1);x[0,:,:]=1.25 + x[1,:,0]=np.linspace(0,100,t);x[2,::2,1]=-0. + for turbo in (True,False): + m=fx.IntervalDistributionSketch(n_random_intervals=17,quantiles=(0.,.1,.25,.5,.75,.9,1.), + fast_profile=turbo,zscore=False,random_state=7).fit(x) + m.intervals_.extend([(0,t),(0,1),(t-1,t),(1,t-1)]) + got=m.transform_normalized(x) + with patch.object(fx,"_cis_intervals",None): + reference=m.transform_normalized(x) + np.testing.assert_array_equal(got,reference) + + @unittest.skipIf(fx._cis_intervals is None,"native symbolic counts unavailable") + def test_short_sort_respects_ties(self): + rng=np.random.default_rng(66) + for t in (8,32,127,256,512,513): + v=np.round(rng.normal(size=(4,t,3)),1);v[0]=-.0 + z=fx._symbolic_bundle([("ch",v)],6)["ch"] + np.testing.assert_array_equal(z,fx._state_counts(v,6)) diff --git a/tests/test_conv_kernels.py b/tests/test_conv_kernels.py new file mode 100644 index 0000000000000000000000000000000000000000..07ed21e65132760d83ba2a1da50f8c2fdf49e618 --- /dev/null +++ b/tests/test_conv_kernels.py @@ -0,0 +1,59 @@ +"""Packed random-convolution parity with the previous release backend.""" +from unittest import TestCase, skipIf +from unittest.mock import patch +import pickle +import warnings +import numpy as np +from wisp_fast import features as fx + + +@skipIf(fx._cis_conv is None or fx._kernels is None, 'native convolution unavailable') +class TestPackedConvolution(TestCase): + def test_exact_tails_and_pairwise_reductions(self): + rng = np.random.default_rng(707) + cases = [15, 16, 17, 18, 19, 23, 32, 35, 37, 63, 96, + 126, 127, 128, 129, 130, 131, 159, 160, 161, 259, 2049] + for index, length in enumerate(cases): + for c in [1, 2, 3, 5]: + with self.subTest(length=length, channels=c): + x = rng.normal(size=(3, length, c)) + if index % 4 == 0: + x = np.rint(x) + if index % 3 == 0: + x[0, 3, 0] = np.nan + x[2, :, -1] = 0. + for turbo in [False, True]: + model = fx.RandomConvSketch(n_kernels=16, kernel_lengths=(1, 5, 7, 9, 15), + fast_profile=turbo, random_state=312) + model.fit(x) + with patch.object(fx, '_cis_conv', None): + old = model.transform(x) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', RuntimeWarning) + out = model.transform(x) + np.testing.assert_array_equal(out, old) + + def test_validation_and_fallback(self): + x = np.random.default_rng(874).normal(size=(3, 32, 3)) + m = fx.RandomConvSketch(n_kernels=8).fit(x) + expected = m.transform(x) + with patch.object(fx, '_verified_conv_blas', return_value=False): + np.testing.assert_array_equal(m.transform(x), expected) + kernel = m.kernels_[0] + kernel['weights'] *= 0. + kernel['bias'] = 0. + m.invalidate_plan() + np.testing.assert_array_equal(m.transform(x)[:, :4], np.zeros((3,4))) + p = pickle.loads(pickle.dumps(m)) + self.assertNotIn('_conv_plan', vars(p)) + np.testing.assert_array_equal(m.transform(x), p.transform(x)) + layout, weights, bias, _, _ = m._conv_plan[1] + for wrong in ('channel', 'length', 'stride', 'output', 'dtype'): + l = layout.copy() + if wrong == 'channel': l[0,3] = 4 + if wrong == 'length': l[1,0] = 0 + out = np.empty((3,8,5 if wrong == 'output' else 4)) + with self.assertRaises((ValueError, BufferError)): + fx._cis_conv.transform(np.ascontiguousarray(x.transpose(0,2,1)),l, + weights.astype('f4') if wrong=='dtype' else weights, + bias,40 if wrong=='stride' else 1,False,False,out) diff --git a/tests/test_engine.py b/tests/test_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..faf1fa06c04d5fc2de8631f3674e3df2e5321134 --- /dev/null +++ b/tests/test_engine.py @@ -0,0 +1,151 @@ +import pickle + +import numpy as np +import unittest +from unittest.mock import patch +from contextlib import nullcontext +from numpy.testing import assert_allclose, assert_array_equal +from threadpoolctl import threadpool_limits + +from wisp.cpu.algorithms import build_cpu_estimator +from wisp.cpu.hmm import TransitionSmoother +from wisp.registry import CANONICAL_METHOD_IDS +from wisp_release.methods import build_reference_family as original +from wisp_fast.decoder import FastSmoother +from wisp_fast.engine import accelerate_checkpoint, build_candidate, build_family, TurboBudget + + +def windows(): + rng = np.random.default_rng(415) + X = rng.normal(size=(38, 42, 3)) + y = np.resize([0, 1, 3], len(X)) + return X[:27], y[:27], X[27:] + + +def check_full_recipe_and_converted_checkpoint(family): + X, y, Q = windows() + g, t = np.arange(len(X)) % 4, np.arange(len(X))[::-1] + gq, tq = np.arange(len(Q)) % 3, np.arange(len(Q))[::-1] + with threadpool_limits(1): + ref = original(family, seed=9).fit(X, y, groups=g, time_index=t, n_classes=5) + fast = build_family(family, seed=9).fit(X, y, groups=g, time_index=t, n_classes=5) + ref_p = ref.predict_proba(Q) + ref_y = ref.predict(Q, groups=gq, time_index=tq) + fast_from_ref = accelerate_checkpoint(ref) + for m in (fast, fast_from_ref): + p = m.predict_proba(Q) + assert_allclose(p, ref_p, atol=2e-10, rtol=1e-9) + assert_array_equal(p[:, [2, 4]], 0) + assert_array_equal(m.decode_proba(p, groups=gq, time_index=tq), ref_y) + assert_array_equal(m.predict(Q, batch_size=3, groups=gq, time_index=tq), ref_y) + assert_array_equal(pickle.loads(pickle.dumps(fast_from_ref)).predict(Q), ref.predict(Q)) + + +def check_turbo_and_chronology(family): + X, y, Q = windows() + g, t = np.arange(len(X)) % 4, np.arange(len(X))[::-1] + b = TurboBudget(kernels=4, intervals=2, shapelets=2, trees=4, max_steps=28, max_train=12) + with threadpool_limits(1): + m = build_family(family, mode="turbo", budget=b).fit(X, y, groups=g, time_index=t, n_classes=5) + assert m.fit_windows_ == 12 + ref_s = TransitionSmoother().fit(y, groups=g, time_index=t, n_classes=5) + assert_array_equal(m.smoother_.transition_, ref_s.transition_) + p = m.predict_proba(Q) + assert_allclose(p.sum(axis=1), 1) + assert_array_equal(p[:, [2, 4]], 0) + + +def check_custom_recipes(): + X, y, Q = windows() + for name, params in [ + ("har_sculpt_forest_hmm", dict(classifier="rf", segment_sizes=(3, 6), symbolic_bins=5)), + ("rocket_sketch_ridge_hmm", dict(classifier="logreg", n_kernels=11)), + ("symbolic_interval_forest", {}), + ("hybrid_cpu_ensemble", {}), + ("invariant_interval_shape_forest", dict(n_shapelets=3)), + ("state_rocket_interval_hmm", dict(n_kernels=9, n_intervals=4)), + ("posterior_cpu_ensemble", dict(weights=(0.0, 0.4, 0.6))), + ]: + spec = dict(name=name, route="cpu", params=dict(seed=33, n_jobs=1, use_hmm=False, **params)) + with threadpool_limits(1): + ref = build_cpu_estimator(spec).fit(X, y) + fast = build_candidate(spec).fit(X, y) + assert_allclose(fast.predict_proba(Q), ref.predict_proba(Q), atol=2e-10, rtol=1e-9) + + +def check_transition_and_tie_order(mode): + rng = np.random.default_rng(911) + g = rng.integers(0, 5, 100) if mode != "none" else None + t = rng.integers(0, 10, 100) if mode == "time" else None + y = rng.integers(0, 4, 100) + a = TransitionSmoother().fit(y, groups=g, time_index=t, n_classes=6) + b = FastSmoother().fit(y, groups=g, time_index=t, n_classes=6) + assert_array_equal(a.prior_, b.prior_) + assert_array_equal(a.transition_, b.transition_) + for p in (rng.uniform(0, 1, (100, 6)), np.ones((100, 6)), np.zeros((100, 6))): + assert_array_equal(a.predict_from_proba(p, groups=g, time_index=t), + b.predict_from_proba(p, groups=g, time_index=t)) + +class EngineTests(unittest.TestCase): + def test_full_recipe_and_checkpoint(self): + for family in CANONICAL_METHOD_IDS: + with self.subTest(family=family): + check_full_recipe_and_converted_checkpoint(family) + + def test_turbo_and_chronology(self): + for family in CANONICAL_METHOD_IDS: + with self.subTest(family=family): + check_turbo_and_chronology(family) + + def test_custom_recipes(self): + check_custom_recipes() + + def test_controlled_recipes(self): + from wisp.ablations import CIS_ABLATIONS, build_recipe as ref_builder + from wisp_fast.engine import build_recipe + X, y, Q = windows() + g, t = np.arange(len(X)) % 3, np.arange(len(X))[::-1] + qg, qt = np.arange(len(Q)) % 3, np.arange(len(Q))[::-1] + for recipe in CIS_ABLATIONS: + with self.subTest(recipe_id=recipe.recipe_id), threadpool_limits(1): + ref = ref_builder(recipe.recipe_id, seed=8, n_jobs=1) + fast = build_recipe(recipe.recipe_id, seed=8) + ref.fit(X, y, groups=g, time_index=t, n_classes=5) + fast.fit(X, y, groups=g, time_index=t, n_classes=5) + assert (fast.smoother_ is not None) == recipe.use_hmm + assert len(fast.blocks_) == sum((recipe.use_rocket, recipe.use_intervals, recipe.use_state)) + assert type(ref.model_.named_steps["clf"]) is type(fast.model_.named_steps["clf"]) + assert_allclose(ref.predict_proba(Q), fast.predict_proba(Q), atol=2e-10, rtol=1e-9) + assert_array_equal(ref.predict(Q, groups=qg, time_index=qt), + fast.predict(Q, groups=qg, time_index=qt)) + + def test_forest_kernel_and_fallback(self): + from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier + from wisp_fast.forest import local_proba, ForestPlan + rng = np.random.default_rng(611) + X = rng.normal(size=(110, 9)) + y = rng.choice([0, 2, 5], size=len(X)) + weight = rng.uniform(0.1, 2, size=len(X)) + Q = rng.normal(size=(27, 9)) + Q[0, 2] = np.nan + for cls in (ExtraTreesClassifier, RandomForestClassifier): + with self.subTest(cls=cls): + model = cls(n_estimators=11, n_jobs=1, random_state=9, + min_samples_leaf=4).fit(X, y, sample_weight=weight) + ref = model.predict_proba(Q) + plan = ForestPlan(model) + assert_array_equal(local_proba(model, Q, plan), ref) + plan.table = None + assert_array_equal(local_proba(model, Q, plan), ref) + + def test_decoder(self): + for backend in ("default", "numpy"): + ctx = patch("wisp_fast.decoder._native", lambda: None) if backend == "numpy" else nullcontext() + with ctx: + for mode in ("none", "group", "time"): + with self.subTest(backend=backend, mode=mode): + check_transition_and_tie_order(mode) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_family_kernels.py b/tests/test_family_kernels.py new file mode 100644 index 0000000000000000000000000000000000000000..e66c1f602612a86e761412657136003bc9da394b --- /dev/null +++ b/tests/test_family_kernels.py @@ -0,0 +1,68 @@ +"""Validate the exact floating-point order and the portable fixed-family fallback.""" +import unittest +import warnings +from unittest.mock import patch + +import numpy as np + +from wisp_fast import features as fx + + +@unittest.skipIf(fx._family_kernels is None, "optional fixed-family extension unavailable") +class TestFixedFamilyKernels(unittest.TestCase): + def test_features_identical_to_v3_fallback(self): + rng = np.random.default_rng(800) + for t, c in ((1, 1), (7, 3), (9, 1), (16, 5), (127, 1), (128, 3), + (129, 4), (256, 1), (383, 3)): + x = rng.normal(size=(4, t, c)) + x[0] = 0 + x[1] = np.round(x[1], 0) + x[2, -1, 0] = np.nan + for fast in (True, False): + for symbols in (True, False): + with self.subTest(t=t, c=c, fast=fast, symbols=symbols), warnings.catch_warnings(): + warnings.simplefilter("ignore", RuntimeWarning) + model = fx.WearableFeatureExtractor(fast_profile=fast, include_symbolic=symbols) + actual = model.transform(x) + names = tuple(model.feature_names_) + with patch.object(fx, "_family_kernels", None): + expected = model.transform(x) + np.testing.assert_array_equal(actual, expected) + self.assertEqual(names, tuple(model.feature_names_)) + + def test_overflow_with_raw_unscaled_data(self): + x = np.random.default_rng(7).normal(size=(3, 31, 3)) * 1e155 + model = fx.WearableFeatureExtractor(zscore=False) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", RuntimeWarning) + actual = model.transform(x) + with patch.object(fx, "_family_kernels", None): + expected = model.transform(x) + np.testing.assert_array_equal(actual, expected) + + def test_shapelet_reductions_and_earliest_rounded_tie(self): + rng = np.random.default_rng(630) + for length in (1, 7, 8, 9, 16, 31, 128, 129, 255, 257): + x = rng.normal(size=(3, 2*length+10, 2)) + x[0] = 0 # Every valid starting position ties. + x[1] = np.round(x[1], 0) + for fast in (True, False): + model = fx.RandomShapeletSketch(fast_profile=fast) + model.shapelets_ = [dict(channel=1, dilation=2, values=rng.normal(size=length)) + for _ in range(3)] + with self.subTest(length=length, fast=fast): + actual = model.transform(x) + with patch.object(fx, "_family_kernels", None): + expected = model.transform(x) + np.testing.assert_equal(actual, expected) + np.testing.assert_equal(actual[0, 1::2], 0.) + + def test_native_buffer_validation(self): + ext = fx._family_kernels + x = np.ones((2, 11, 3)) + with self.assertRaises(ValueError): + ext.stats(x, x) + with self.assertRaises(ValueError): + ext.segments(x, np.empty((2, 4, 0, 3)), 0, 0) + with self.assertRaises(ValueError): + ext.shapelets(x[:, :, 1].copy(), np.ones((1, 12)), np.empty((2, 1, 2)), 1, 1) diff --git a/tests/test_features_fast.py b/tests/test_features_fast.py new file mode 100644 index 0000000000000000000000000000000000000000..8b7d3b4ee1b4465a90eecca7684c67d09d13b1d0 --- /dev/null +++ b/tests/test_features_fast.py @@ -0,0 +1,97 @@ +"""Feature-level release conformance, including discrete and singleton windows.""" +import warnings + +import numpy as np +import unittest +from unittest.mock import patch + +from wisp.cpu import features as ref +from wisp_fast import features as fx + + +IMPLEMENTATIONS = [ + ('WearableFeatureExtractor', {}), + ('IntervalDistributionSketch', {'n_random_intervals': 7}), + ('RandomConvSketch', {'n_kernels': 36}), + ('InvariantPhysicsFeatureExtractor', {'random_intervals': 7}), + ('RandomShapeletSketch', {'n_shapelets': 12}), +] + + +def compare_features(name, kw, case): + rng = np.random.default_rng(781) + x = rng.standard_normal((1 if case == 'one_window' else 5, 37, + 4 if case == 'four_channels' else 3)) + if case == 'ties': + x = np.round(x) + x[:, :, 1] = 2. + if case == 'fortran': + x = np.asfortranarray(x) + if case == 'univariate': + x = x[:, :, 1].copy() + x[0, 5] = np.nan + original = getattr(ref, name)(**kw) + replacement = getattr(fx, name)(**kw) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', RuntimeWarning) + expected = original.fit_transform(x) + result = replacement.fit_transform(x) + np.testing.assert_array_equal(result, expected) + if hasattr(original, 'feature_names_'): + assert replacement.feature_names_ == original.feature_names_ + + +def compare_turbo(name, kw): + x = np.random.default_rng(1024).normal(size=(7, 65, 3)).astype(np.float32) + model = getattr(fx, name)(**kw, fast_profile=True) + result = model.fit_transform(x) + assert len(result) == 7 and np.isfinite(result).all() + np.testing.assert_allclose(result[4:5], model.transform(x[4:5]), rtol=1e-14, atol=1e-14) + np.testing.assert_array_equal(result, model.fit_transform(x)) + + +class TestFeatures(unittest.TestCase): + def test_release_conformance(self): + cases = ['triaxial', 'ties', 'one_window', 'univariate', 'four_channels', 'fortran'] + for name, kw in IMPLEMENTATIONS: + for case in cases: + with self.subTest(name=name, case=case): + compare_features(name, kw, case) + + def test_turbo_singleton_batch_and_refit(self): + for name, kw in IMPLEMENTATIONS: + with self.subTest(name=name): + compare_turbo(name, kw) + + @patch.object(fx, '_kernels', None) + def test_conv_numpy_fallback(self): + x = np.random.default_rng(781).normal(size=(7, 40, 3)) + kw = dict(n_kernels=64, dilations=(1, 2, 5)) + a = ref.RandomConvSketch(**kw).fit_transform(x) + b = fx.RandomConvSketch(**kw).fit_transform(x) + np.testing.assert_array_equal(b, a) + + + @unittest.skipIf(fx._kernels is None, 'optional extension unavailable') + def test_native_dimension_guards(self): + native = fx._kernels + x = np.zeros((3, 5, 37)) + weights = np.zeros((2, 4, 3)) + biases = np.zeros(2) + out = np.zeros((2, 5, 34)) + for args in [(x, weights, biases, 0, 1, out), + (x, weights, biases[:1], 1, 1, out), + (x.astype(np.float32), weights, biases, 1, 1, out), + (x[0:2], weights, biases, 1, 1, out), + (x, weights, biases, 2, 1, out), + (x, weights, biases, 1, 100, out)]: + with self.assertRaises((BufferError, ValueError)): + native.project(*args) + + + def test_identical_shapelet_matches_choose_first_window(self): + x = np.tile(np.arange(8), 7).reshape(1, -1, 1).astype(float) + kw = dict(n_shapelets=12, lengths=(8,), dilations=(1,)) + expected = ref.RandomShapeletSketch(**kw).fit_transform(x) + actual = fx.RandomShapeletSketch(**kw).fit_transform(x) + np.testing.assert_array_equal(actual, expected) diff --git a/tests/test_interval_kernels.py b/tests/test_interval_kernels.py new file mode 100644 index 0000000000000000000000000000000000000000..f88f0959d41103264b86db8a173b6d34d52785be --- /dev/null +++ b/tests/test_interval_kernels.py @@ -0,0 +1,31 @@ +import unittest +from unittest.mock import patch +import numpy as np +from wisp_fast import features as fx +from wisp_fast.cache import transform_blocks +from wisp_fast.parallel import parallel_rows + + +class IntervalsAndSharedTests(unittest.TestCase): + def test_random_shapes_discrete_and_fractional(self): + rng=np.random.default_rng(918) + for n,t,c in ((1,19,3),(8,160,3),(5,64,1),(7,47,6)): + for turbo in (False,True): + for discrete in (False,True): + with self.subTest(n=n,t=t,c=c,turbo=turbo,discrete=discrete): + X=rng.standard_normal((n,t,c)) + if discrete:X=np.round(X,1) + X[0,:,0]=0 + blocks=[fx.IntervalDistributionSketch(fast_profile=turbo).fit(X), + fx.WearableFeatureExtractor(include_raw_stats=False,include_multiscale=False, + fast_profile=turbo).fit(X)] + with patch.object(fx,"_cis_intervals",None): + want=[block.transform(X) for block in blocks] + got=transform_blocks(blocks,X) + for a,b in zip(want,got):np.testing.assert_array_equal(a,b) + + def test_threaded_rows_and_guards(self): + x=np.random.default_rng(7).normal(size=(160,64,3)) + m=fx.IntervalDistributionSketch().fit(x) + np.testing.assert_array_equal(m.transform(x),parallel_rows(m.transform,x,4)) + with self.assertRaises(ValueError):parallel_rows(m.transform,x,0) diff --git a/tests/test_method_conformance.py b/tests/test_method_conformance.py index fb92a067e62d6719c34ed5cff0ae7c302f9960e8..be5660c3d9bdf57d3c4e61ee0e978b65d31c4315 100644 --- a/tests/test_method_conformance.py +++ b/tests/test_method_conformance.py @@ -21,8 +21,8 @@ def load(name, file): spec.loader.exec_module(mod) return mod -F=load('r7_frozen_representations','features.py') -H=load('r7_frozen_hmm','hmm.py') +F=load('frozen_representations','features.py') +H=load('frozen_hmm','hmm.py') def zscore(x): x=np.asarray(x,dtype=float) diff --git a/tests/test_native_kernels.py b/tests/test_native_kernels.py new file mode 100644 index 0000000000000000000000000000000000000000..3176362f8b7e3b5baea0c11dec1b3316a5c27ba7 --- /dev/null +++ b/tests/test_native_kernels.py @@ -0,0 +1,47 @@ +from unittest import TestCase,skipIf +from unittest.mock import patch +import numpy as np +from wisp_fast import features as f + + +@skipIf(f._family_kernels is None,'requires optional native backend') +class NewKernels(TestCase): + def test_spectral_and_center(self): + rng=np.random.default_rng(907) + for n,c,t in ((1,1,160),(9,3,33),(12,1,128),(17,5,513)): + x=rng.normal(size=(n,t,c));x[0,::2]=0. + ac=np.empty((n,3,c));center=np.empty_like(x) + f._family_kernels.autocorr(x,(1,4,8),ac,center) + bands=((0.,.042),(.17,.17),(.5,.51),(.01,.49)) + self.assertEqual(center.tobytes(),(x-x.mean(axis=1,keepdims=True)).tobytes()) + self.assertEqual(f._spectrum_centered(center,bands).tobytes(),f.ref._spectral_features(x,bands).tobytes()) + x=np.zeros((2,31,3)) + with self.assertRaises(ValueError):f._family_kernels.autocorr(x,(1,),np.empty((2,1,3)),x) + with self.assertRaises(ValueError):f._family_kernels.derived(x,np.empty((2,31,1)),x,np.empty((2,31,1))) + + def test_window_lane_r(self): + with np.errstate(over='ignore',invalid='ignore'): + for t,c in ((18,2),(127,5),(160,3),(193,1),(256,3),(257,5)): + x=np.random.default_rng(t).normal(size=(17,t,c));x[1]=0.;x[2]*=1.e-130 + m=f.RandomConvSketch(n_kernels=17,kernel_lengths=(1,5,7,9,11,15),random_state=14).fit(x) + y=m.transform_normalized(x) + with patch.object(f,'_cis_conv',None):ref=m.transform_normalized(x) + np.testing.assert_array_equal(y,ref) + + def test_private_standardization(self): + from wisp_fast.engine import build_family + from sklearn.pipeline import Pipeline + from sklearn.preprocessing import StandardScaler + from sklearn.linear_model import RidgeClassifier + rng=np.random.default_rng(9247);X=rng.normal(size=(20,6));y=np.arange(20)%3 + m=build_family('wisp_cis');m.model_=Pipeline([('s',StandardScaler()),('h',RidgeClassifier())]).fit(X,y) + m.classes_=np.arange(3);orig=X.copy() + a=m._head_proba(X) + self.assertEqual(orig.tobytes(),X.tobytes()) + b=m._head_proba(X.copy(),private=True);np.testing.assert_array_equal(a,b) + X.flags.writeable=False;np.testing.assert_array_equal(a,m._head_proba(X,private=True)) + + def test_clean_and_noncontiguous(self): + a=np.array([[np.nan,np.inf,-0.,-np.inf,1.]]) + check=np.nan_to_num(a,nan=0.,posinf=0.,neginf=0.) + self.assertEqual(f._clean(a).tobytes(),check.tobytes()) diff --git a/tests/test_release_api.py b/tests/test_release_api.py index ee52607c4206fa507408cd3adc9f2a8da18b6d58..60a554548f3ca85960f20517282f65980d6874f1 100644 --- a/tests/test_release_api.py +++ b/tests/test_release_api.py @@ -239,9 +239,9 @@ def test_inventory_does_not_unpickle(tmp_path, capsys): def test_family_constructor_preserves_names_and_forces_positive_jobs(): model = build_family("wisp_cis", seed=7, n_jobs=1, direct=True, overrides={"n_kernels": 8, "n_intervals": 4}) - assert model.seed == 7 + assert model.template.seed == 7 assert model.use_hmm is False - assert model.n_kernels == 8 + assert model.template.n_kernels == 8 with pytest.raises(ValueError, match="fixed families only"): build_family("wisp_random", seed=7) diff --git a/tests/test_ridge_screened.py b/tests/test_ridge_screened.py new file mode 100644 index 0000000000000000000000000000000000000000..06d8e1c3b7795d0537c9592b2e647ba067ffdd9f --- /dev/null +++ b/tests/test_ridge_screened.py @@ -0,0 +1,61 @@ +import pickle +import unittest + +import numpy as np +from sklearn.linear_model import RidgeClassifierCV +from sklearn.utils.class_weight import compute_sample_weight +from threadpoolctl import threadpool_limits + +from wisp_fast.ridge_fast import SpectralRidgeClassifierCV, ScreenedRidgeClassifierCV + + +class RidgeFastTests(unittest.TestCase): + def test_weighted_loo_entire_alpha_path(self): + alphas = np.logspace(-3, 3, 9) + rng = np.random.default_rng(786) + with threadpool_limits(1): + for n, d in [(31, 77), (77, 31), (35, 34), (65, 17)]: + for labels in [(0, 2), (0, 3, 9)]: + with self.subTest(n=n,d=d,labels=labels): + X = rng.normal(size=(n, d)) + X[:, -6:-1] = X[:, :5] + X[:, -1] = 0 + Q = rng.normal(size=(17,d)) + y = np.resize(labels, n) + sw = np.exp(rng.uniform(-1,1,n)) + ref = RidgeClassifierCV(alphas=alphas, store_cv_results=True).fit(X, y, sample_weight=sw) + model = SpectralRidgeClassifierCV(alphas=alphas).fit(X, y, sample_weight=sw) + self.assertIn(model.solver_used_, ("primal_eigh", "dual_eigh")) + np.testing.assert_equal(ref.alpha_, model.alpha_) + errors = ref.cv_results_.reshape(n, -1, len(alphas)) + np.testing.assert_allclose(-errors.mean(axis=(0,1)), model.loo_scores_, atol=3e-9,rtol=3e-9) + np.testing.assert_allclose(ref.decision_function(Q), model.decision_function(Q), atol=1e-8) + np.testing.assert_array_equal(ref.predict(Q), pickle.loads(pickle.dumps(model)).predict(Q)) + + def test_tie_fallback(self): + X = np.ones((15, 5)) + y = np.repeat([3, 12, 18],5) + a = [0.1, 1., 10.] + m = SpectralRidgeClassifierCV(alphas=a).fit(X, y) + ref = RidgeClassifierCV(alphas=a).fit(X,y) + self.assertEqual(m.solver_used_, "sklearn_score_tie") + self.assertEqual(m.alpha_,ref.alpha_) + np.testing.assert_array_equal(m.predict(X), ref.predict(X)) + + def test_screening_sample_fit_and_determinism(self): + rng = np.random.default_rng(24) + X = rng.normal(size=(100, 47)) + y = np.resize([0,1,3], 100) + a = [0.1, 1., 10.] + sw = compute_sample_weight("balanced",y) + m = ScreenedRidgeClassifierCV(alphas=a,screening_rows=30).fit(X, y, sw) + ref = RidgeClassifierCV(alphas=[m.alpha_]).fit(X,y,sample_weight=sw) + self.assertEqual(m.screening_rows_,30) + np.testing.assert_allclose(ref.decision_function(X),m.decision_function(X),atol=3e-9) + self.assertEqual(m.alpha_,ScreenedRidgeClassifierCV(alphas=a,screening_rows=30).fit(X,y,sw).alpha_) + high = ScreenedRidgeClassifierCV(alphas=a,screening_rows=150).fit(X,y,sw) + full = SpectralRidgeClassifierCV(alphas=a).fit(X,y,sw) + np.testing.assert_array_equal(high.decision_function(X),full.decision_function(X)) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_search_cache.py b/tests/test_search_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..918de5e011370ed887b46942ac1e42f0f1e6320c --- /dev/null +++ b/tests/test_search_cache.py @@ -0,0 +1,79 @@ +import json +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest + +import numpy as np +from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier +from threadpoolctl import threadpool_limits + +from wisp_fast import features as fx +from wisp_fast.search import fit_search +from wisp_fast.search_cache import SearchFeatures +from wisp_fast.search_forest import forest_fit_plan +from test_search_shared import splits + + +class SearchPlanTests(unittest.TestCase): + def test_forest_seed_and_tree_parity(self): + rng=np.random.default_rng(76) + x=rng.normal(size=(71,24)) + y=np.resize([0,2,3,3],len(x)) + for kind,weight in ((ExtraTreesClassifier,"balanced"),(RandomForestClassifier,"balanced_subsample")): + for jobs in (1,2): + kw=dict(random_state=7,n_estimators=11,n_jobs=jobs,class_weight=weight) + ref=kind(**kw).fit(x,y) + got=kind(**kw) + with forest_fit_plan(got): + got.fit(x,y) + self.assertNotIn("_make_estimator",got.__dict__) + for a,b in zip(ref.estimators_,got.estimators_): + self.assertEqual(a.get_params(),b.get_params()) + for name in ("children_left","children_right","feature","threshold","value","weighted_n_node_samples"): + np.testing.assert_array_equal(getattr(a.tree_,name),getattr(b.tree_,name)) + np.testing.assert_allclose(ref.predict_proba(x),got.predict_proba(x),atol=1e-15,rtol=0) + + def test_interval_cache_normalization_and_seed(self): + x=np.random.default_rng(22).normal(size=(28,43,3)) + mem=SearchFeatures(4,[x]) + for fast in (True,False): + for seed in (3,11): + for factory in (fx.IntervalDistributionSketch,fx.InvariantPhysicsFeatureExtractor): + b=factory(fast_profile=fast,random_state=seed).fit(x) + np.testing.assert_array_equal(mem.transform([b],x)[0],b.transform(x)) + if fx._cis_intervals is not None: + self.assertGreater(mem.rank_hits,1) + self.assertGreater(mem.dyad_hits,1) + self.assertLessEqual(mem.peak,mem.limit) + + def test_certificates_against_complete_ledgers(self): + train,valid=splits() + with TemporaryDirectory() as tmp,threadpool_limits(1): + for method in ("wisp_random","wisp_evolution"): + arms={} + for policy in ("complete","certified"): + path=Path(tmp)/method/policy + _,s=fit_search(method,train,valid,output=path,mode="turbo",evaluations=25, + population=6,cost_weight=0,refit=False,stress_cache_mb=0, + stress_policy=policy,workers=2) + rows=[json.loads(l) for l in (path/"program_database.jsonl").read_text().splitlines()] + arms[policy]=s,rows + (s,full),(t,short)=arms.values() + self.assertEqual([r["id"] for r in full],[r["id"] for r in short]) + self.assertEqual(s["best"]["id"],t["best"]["id"]) + skipped=0 + for row,original in zip(short,full): + if row["status"]=="pruned": + cert=row["result"]["prune_certificate"] + refscore=original["result"]["predictive_utility"] + idx=cert["incumbent_ledger_index"] + self.assertEqual(short[idx]["result"]["predictive_utility"],cert["incumbent_utility"]) + self.assertLess(cert["upper_bound"],cert["incumbent_utility"]) + self.assertLessEqual(refscore,cert["upper_bound"]) + self.assertNotIn("predictive_utility",row["result"]) + if method=="wisp_evolution":self.assertEqual(row["generation"],short[-1]["generation"]) + skipped+=cert["skipped_views"] + else: + self.assertEqual(row["result"]["predictive_utility"],original["result"]["predictive_utility"]) + self.assertEqual(skipped,t["skipped_stress_views"]) + if method=="wisp_random":self.assertGreater(skipped,0) diff --git a/tests/test_search_shared.py b/tests/test_search_shared.py new file mode 100644 index 0000000000000000000000000000000000000000..f080d1650a9044e2efdd322d29b1351e9fd08f1a --- /dev/null +++ b/tests/test_search_shared.py @@ -0,0 +1,94 @@ +import json +import tempfile +import unittest +from pathlib import Path +import numpy as np +from threadpoolctl import threadpool_limits +from wisp.core.data import HARData +from wisp.core.metrics import classification_report_dict +from wisp.search.cascade import evaluate_candidate_cascade +from wisp.search.grammar import initial_candidates +from wisp_fast.search import evaluate_recipes, fit_search, make_candidate +from wisp_fast.search_metrics import search_report +from wisp_fast.search_cache import SearchFeatures +from wisp_fast import features as fx + + +def splits(): + rng=np.random.default_rng(55) + def build(subject): + y=np.resize([0,0,1,1,3,3],24) + x=rng.normal(size=(len(y),32,3))+np.sin(np.arange(32)[None,:,None]*(1+y[:,None,None])*.4) + return HARData(x,y,subject=np.repeat([subject,subject+1],12), + time_index=np.tile(np.arange(12),2),label_names=list("abcd")) + return build(1),build(3) + + +class SharedSearchTests(unittest.TestCase): + def test_scorer_search_semantics(self): + for a,b in [([0,0,2,3],[0,1,1,2]),([2,2,2],[1,1,1]),([3],[3]),([0,3],[0,3])]: + r=classification_report_dict(a,b) + s=search_report(a,b) + for k in ("weighted_f1","worst_class_f1","accuracy","balanced_accuracy","macro_f1"): + self.assertAlmostEqual(r[k],s[k],places=14) + self.assertEqual(r["confusion_matrix"],s["confusion_matrix"]) + self.assertEqual(r["per_class_f1"],s["per_class_f1"]) + + def test_memoed_physics_is_seed_safe(self): + data,_=splits() + cache=SearchFeatures(16,[data.X]) + for seed in (1,2,1): + model=fx.InvariantPhysicsFeatureExtractor(random_state=seed).fit(data.X) + expect=model.transform(data.X) + got=cache.transform([model],data.X)[0] + np.testing.assert_array_equal(expect,got) + self.assertGreater(cache.hits,1) + + def test_stress_panel_and_full_metric_pairing(self): + a,b=splits() + specs=initial_candidates(seed=42) + with threadpool_limits(1): + scored,stats=evaluate_recipes(specs,a,b,mode="turbo",cost_weight=0,cache_mb=20) + for spec,(status,r) in zip(specs,scored): + # Match controller runtime injection, including one tree job. + spec.params["n_jobs"]=1 + ref=evaluate_candidate_cascade(spec,a,b,cost_weight=0, + estimator_factory=lambda d:make_candidate(d,mode="turbo")).to_dict() + self.assertNotEqual(status,"failed",msg=r) + self.assertEqual(ref["confusion_matrix"],r["confusion_matrix"]) + self.assertAlmostEqual(ref["cascade_score"],r["cascade_score"],places=14) + self.assertEqual(ref["cascade_stages"][2]["metrics"]["stress_reports"], + r["cascade_stages"][2]["metrics"]["stress_reports"]) + self.assertGreater(stats["hits"],0) + + def test_no_refit_and_zero_stress(self): + a,b=splits() + with tempfile.TemporaryDirectory() as tmp,threadpool_limits(1): + m,s=fit_search("wisp_evolution",a,b,mode="turbo",output=Path(tmp)/"run", + evaluations=3,stress_count=0,robustness_weight=0,refit=False) + self.assertIsNone(m) + self.assertEqual(s["stress_count"],0) + self.assertEqual(s["cached_stress"],[]) + self.assertEqual(s["n_evaluated"],3) + + +class ScopeTests(unittest.TestCase): + def test_cis_scope(self): + a,b=splits() + with tempfile.TemporaryDirectory() as tmp,threadpool_limits(1): + for method in ("wisp_random","wisp_evolution"): + _,s=fit_search(method,a,b,mode="turbo",output=Path(tmp)/method,evaluations=11, + population=6,family_scope="cis",refit=False,cost_weight=0) + self.assertEqual(s["family_scope"],"cis") + rows=Path(tmp,method,"program_database.jsonl").read_text().splitlines() + self.assertEqual({json.loads(l)["spec"]["name"] for l in rows},{"state_rocket_interval_hmm"}) + + def test_related_scope(self): + a,b=splits() + allowed={"state_rocket_interval_hmm","rocket_sketch_ridge_hmm","posterior_cpu_ensemble"} + with tempfile.TemporaryDirectory() as tmp,threadpool_limits(1): + for method in ("wisp_random","wisp_evolution"): + fit_search(method,a,b,output=Path(tmp)/method,mode="turbo",evaluations=10,population=6, + family_scope="cis-related",refit=False) + rows=Path(tmp,method,"program_database.jsonl").read_text().splitlines() + self.assertEqual({json.loads(l)["spec"]["name"] for l in rows},allowed) diff --git a/tests/test_speed_and_sequence.py b/tests/test_speed_and_sequence.py new file mode 100644 index 0000000000000000000000000000000000000000..1e635a918ab9ee60c62b41b9780562537e5a26b1 --- /dev/null +++ b/tests/test_speed_and_sequence.py @@ -0,0 +1,124 @@ +"""The two public options: speed (exact/fast) and HMM smoothing sequence (subject/recording).""" + +from __future__ import annotations + +import json + +import numpy as np +import pytest + +from wisp_release.cli import main +from wisp_release.methods import build_family +from wisp_fast.decoder import (CONFIDENCE_SCALE_GRID, FastSmoother, fit_confidence_scale, recording_ids, + scale_confidence) + + +def test_speed_selects_ridge_solver_only_where_a_ridge_head_exists(): + assert build_family("wisp_cis", seed=1).ridge_solver == "original" + assert build_family("wisp_cis", seed=1, speed="fast").ridge_solver == "screened" + assert build_family("wisp_rc", seed=1, speed="fast").ridge_solver == "screened" + assert build_family("wisp_ss", seed=1, speed="fast").ridge_solver == "original" + assert build_family("wisp_ese", seed=1, speed="fast").member_ridge_solver == "screened" + with pytest.raises(ValueError, match="speed"): + build_family("wisp_cis", seed=1, speed="turbo") + with pytest.raises(ValueError, match="sequence"): + build_family("wisp_cis", seed=1, sequence="session") + + +def test_recordings_start_when_time_index_goes_backwards(): + groups = np.array(["a", "a", "a", "b", "a", "a", "b"]) + time = np.array([0, 1, 2, 5, 0, 1, 0]) + assert recording_ids(groups, time).tolist() == ["a|0", "a|0", "a|0", "b|0", "a|1", "a|1", "b|1"] + + +def test_subject_sequence_ignores_recording_boundaries(): + rng = np.random.default_rng(3) + y = rng.integers(0, 3, 40) + groups = np.repeat(["p", "q"], 20) + time = np.tile(np.r_[np.arange(10), np.arange(10)], 2) + P = rng.dirichlet(np.ones(3), 40) + plain = FastSmoother().fit(y, groups=groups, time_index=time, n_classes=3) + explicit = FastSmoother("subject").fit(y, groups=groups, time_index=time, n_classes=3) + np.testing.assert_array_equal(plain.transition_, explicit.transition_) + np.testing.assert_array_equal(plain.predict_from_proba(P, groups=groups, time_index=time), + explicit.predict_from_proba(P, groups=groups, time_index=time)) + recording = FastSmoother("recording").fit(y, groups=groups, time_index=time, n_classes=3) + rec = recording_ids(groups, time) + same = FastSmoother().fit(y, groups=rec, time_index=time, n_classes=3) + np.testing.assert_array_equal(recording.transition_, same.transition_) + assert recording.active + np.testing.assert_array_equal(recording.predict_from_proba(P, groups=groups, time_index=time), + same.predict_from_proba(P, groups=rec, time_index=time)) + + +def test_recording_smoothing_skipped_when_recordings_hold_one_activity(): + groups = np.repeat(["p"], 12) + time = np.tile(np.arange(4), 3) + y = np.repeat([0, 1, 2], 4) + smoother = FastSmoother("recording").fit(y, groups=groups, time_index=time, n_classes=3) + assert smoother.single_label_fraction_ == 1.0 and not smoother.active + P = np.random.default_rng(0).dirichlet(np.ones(3), 12) + np.testing.assert_array_equal(smoother.predict_from_proba(P, groups=groups, time_index=time), P.argmax(1)) + + +def test_confidence_scale_is_chosen_on_the_grid(): + rng = np.random.default_rng(1) + P = rng.dirichlet(np.ones(4), 60) + y = rng.integers(0, 4, 60) + scale = fit_confidence_scale(P, y) + assert np.isclose(CONFIDENCE_SCALE_GRID, scale).any() + np.testing.assert_array_equal(scale_confidence(P, 1.0), P) + np.testing.assert_allclose(scale_confidence(P, 2.0).sum(1), 1.0) + + +def _write(path, rng, n, prefix): + labels = np.repeat(np.arange(3), 2) + y = np.tile(labels, n // 6).astype(np.int64) + X = rng.normal(size=(n, 32, 3)) + y[:, None, None] + np.savez(path, X=X, y=y, subject=np.array([f"{prefix}{i // 12}" for i in range(n)]), + time_index=np.tile(np.arange(6), n // 6), label_names=np.array(["a", "b", "c"])) + + +@pytest.mark.parametrize("speed", ["exact", "fast"]) +def test_cli_recording_fit_requires_validation_and_records_options(tmp_path, capsys, speed): + rng = np.random.default_rng(5) + _write(tmp_path / "train.npz", rng, 36, "t") + _write(tmp_path / "valid.npz", rng, 12, "v") + base = ["family-fit", "--method", "wisp_cis", "--input", str(tmp_path / "train.npz"), "--seed", "1", + "--n-jobs", "1", "--overrides", '{"n_kernels": 8, "n_intervals": 4}', "--speed", speed, + "--sequence", "recording", "--execute"] + with pytest.raises(SystemExit): + main([*base, "--output", str(tmp_path / "missing.pkl")]) + capsys.readouterr() + assert main([*base, "--output", str(tmp_path / "model.pkl"), "--valid", str(tmp_path / "valid.npz")]) == 0 + capsys.readouterr() + metadata = json.loads((tmp_path / "model.metadata.json").read_text()) + assert metadata["speed"] == speed and metadata["sequence"] == "recording" + assert metadata["ridge_solver"] == ("original" if speed == "exact" else "screened") + assert metadata["sequence_smoothing"] is True + assert metadata["confidence_scale"] in CONFIDENCE_SCALE_GRID + + +def test_cli_selector_and_search_accept_both_options(tmp_path, capsys): + rng = np.random.default_rng(7) + for part, n, prefix in (("train", 36, "t"), ("valid", 12, "v"), ("test", 12, "s")): + _write(tmp_path / f"{part}.npz", rng, n, prefix) + paths = {part: str(tmp_path / f"{part}.npz") for part in ("train", "valid", "test")} + options = ["--speed", "fast", "--sequence", "recording", "--seed", "3", "--n-jobs", "1", "--execute"] + assert main(["select-fit", "--method", "wisp_select5", "--train", paths["train"], "--valid", paths["valid"], + "--output", str(tmp_path / "selected.pkl"), *options]) == 0 + capsys.readouterr() + report = json.loads((tmp_path / "selected.selection.json").read_text()) + assert report["speed"] == "fast" and report["sequence"] == "recording" + assert report["selection_uses_test"] is False + output = tmp_path / "search" + assert main(["search", "--method", "wisp_random", "--train", paths["train"], "--valid", paths["valid"], + "--test", paths["test"], "--output", str(output), "--population", "2", "--generations", "1", + "--max-evaluations", "3", *options]) == 0 + capsys.readouterr() + rows = [json.loads(line) for line in (output / "program_database.jsonl").read_text().splitlines()] + assert len(rows) == 3 + assert all(row["result"].get("sequence") == "recording" for row in rows if row["status"] == "ok") + metadata = json.loads((output / "selected_model.metadata.json").read_text()) + assert metadata["speed"] == "fast" and metadata["sequence"] == "recording" + assert (output / "selected_model.pkl").is_file()