| import os |
| os.environ.setdefault('MPLCONFIGDIR',os.path.abspath('.cache/matplotlib')) |
| import json,collections,datetime,hashlib |
| from pathlib import Path |
| import numpy as np,pandas as pd |
| import matplotlib |
| matplotlib.use('Agg') |
| import matplotlib.pyplot as plt |
| from bci_data import LABELS |
| from sklearn.metrics import accuracy_score,balanced_accuracy_score,f1_score,confusion_matrix |
|
|
| def summarize(rows): |
| y=[];p=[];subject=[] |
| for r in rows: |
| z=np.load(f"reports/predictions/{r['name']}.npz");y.extend(z['y']);p.extend(z['pred']);subject.append(float(np.mean(z['y']==z['pred']))) |
| rng=np.random.default_rng(2026);means=np.mean(rng.choice(subject,size=(10000,len(subject)),replace=True),axis=1) |
| return {'subjects':len(rows),'trials':len(y),'accuracy':accuracy_score(y,p),'balanced_accuracy':balanced_accuracy_score(y,p),'macro_f1':f1_score(y,p,average='macro',zero_division=0),'ci_low':float(np.quantile(means,.025)),'ci_high':float(np.quantile(means,.975)),'confusion_matrix':confusion_matrix(y,p,labels=range(5)).tolist(),'subject_accuracy':subject} |
| def pct(x):return f'{100*x:.1f}%' |
| def main(): |
| records=json.load(open('reports/test_subject_results.json'));manifest=json.load(open('reports/frozen_selection.json'));groups=collections.defaultdict(list) |
| for r in records:groups[(r['model'],r['montage'],r['channels'],r['seed'])].append(r) |
| rows=[] |
| for k,rs in sorted(groups.items(),key=lambda x:str(x[0])): |
| m=summarize(rs);rows.append(dict(model=k[0],montage=k[1],channels=k[2],seed=k[3],**{key:value for key,value in m.items() if key not in ['confusion_matrix','subject_accuracy']})) |
| pd.DataFrame(rows).to_csv('reports/results.csv',index=False) |
| primaryrows=json.load(open('reports/demo_subject_models_test.json'));primary=summarize(primaryrows);secondary=summarize(json.load(open('reports/personalized_selection_test.json'))) |
| json.dump({'primary':primary,'personalized_secondary':secondary},open('reports/primary_summary.json','w'),indent=2) |
| Path('reports/figures').mkdir(exist_ok=True) |
| cm=np.array(primary['confusion_matrix']);fig,ax=plt.subplots(figsize=(6,5));im=ax.imshow(cm,cmap='Blues');ax.set(xticks=range(5),yticks=range(5),xticklabels=LABELS,yticklabels=LABELS,xlabel='Predicted',ylabel='True',title='Frozen primary system · official test') |
| plt.setp(ax.get_xticklabels(),rotation=25,ha='right') |
| for i in range(5): |
| for j in range(5):ax.text(j,i,str(cm[i,j]),ha='center',va='center',color='white' if cm[i,j]>cm.max()/2 else 'black') |
| fig.colorbar(im,ax=ax);fig.tight_layout();fig.savefig('reports/figures/primary_confusion.png',dpi=160);plt.close(fig) |
| fig,ax=plt.subplots(figsize=(9,4));ax.bar(range(1,16),primary['subject_accuracy']);ax.axhline(.2,color='gray',ls='--',label='Uniform chance');ax.set(xlabel='Participant',ylabel='Test accuracy',ylim=(0,1),xticks=range(1,16),title='Primary system · all 15 participants');ax.legend();fig.tight_layout();fig.savefig('reports/figures/subjects.png',dpi=160);plt.close(fig) |
| fig,ax=plt.subplots(figsize=(8,5)) |
| for model in ['svm','fbts','eegnet','cbramod','reve_probe','reve_partial','csbrain_probe','csbrain_partial']: |
| rr=sorted([r for r in rows if r['model']==model and r['montage']=='distributed' and r['seed'] in ['deterministic',2026] and r['subjects']==15],key=lambda r:r['channels']) |
| if len(rr)>1:ax.plot([r['channels'] for r in rr],[r['accuracy'] for r in rr],'-o',label=model) |
| ax.axhline(.2,color='gray',ls='--');ax.set(xlabel='Real measured EEG channels (reference/ground additional)',ylabel='Official-test accuracy',xticks=[8,16,32,64],title='Retrained channel reduction');ax.legend();fig.tight_layout();fig.savefig('reports/figures/channel_reduction.png',dpi=160);plt.close(fig) |
| from bci_data import cached,montage |
| import mne |
| _,_,names=cached(1,'train');pos=mne.channels.make_standard_montage('standard_1005').get_positions()['ch_pos'];fig,axs=plt.subplots(1,4,figsize=(15,4)) |
| for ax,n in zip(axs,[64,32,16,8]): |
| idx=montage(names,n);ax.add_patch(plt.Circle((0,0),.105,fill=False,color='gray')) |
| for i in idx: |
| p=pos[names[i]];ax.scatter(p[0],p[1],s=10,color='#146955');ax.text(p[0],p[1],names[i],fontsize=5) |
| ax.set(title=f'{n} measured electrodes',xlim=(-.12,.12),ylim=(-.12,.12),aspect='equal');ax.axis('off') |
| fig.suptitle('Distributed montage · template coordinates, not individual digitization');fig.tight_layout();fig.savefig('reports/figures/montages.png',dpi=180);plt.close(fig) |
| vals=[] |
| for p in Path('experiments').glob('*.json'): |
| try:r=json.load(open(p)) |
| except Exception:continue |
| if not isinstance(r,dict) or 'metrics' not in r:continue |
| vals.append({'name':r.get('name',p.stem),'model':r.get('model'),'channels':r.get('channels'),'split':r.get('split'),'accuracy':r['metrics'].get('accuracy'),'balanced_accuracy':r['metrics'].get('balanced_accuracy'),'macro_f1':r['metrics'].get('macro_f1'),'status':'primary_protocol' if p.name.startswith('strict_') else 'control_or_exploratory'}) |
| pd.DataFrame(vals).to_csv('reports/all_completed_validation_experiments.csv',index=False) |
| table='| Model | Montage | Channels | Seed | Subjects | Test accuracy (subject-bootstrap 95% CI) | Balanced accuracy | Macro-F1 |\n|---|---|---:|---|---:|---|---:|---:|\n' |
| for r in rows:table+=f"| {r['model']} | {r['montage']} | {r['channels']} | {r['seed']} | {r['subjects']} | {pct(r['accuracy'])} ({pct(r['ci_low'])}–{pct(r['ci_high'])}) | {pct(r['balanced_accuracy'])} | {r['macro_f1']:.3f} |\n" |
| Path('reports/RESULTS.md').write_text('# Measured final-test comparisons\n\nPrimary family was frozen before evaluation. Other rows are descriptive secondary comparisons, not permission to choose a new primary model on test results. Confidence intervals resample participants, not EEG windows.\n\n'+table) |
| loso=next((r for r in rows if r['model']=='loso_bandpower'),None);lososvm=next((r for r in rows if r['model']=='loso_svm'),None);cue=next((r for r in rows if r['model']=='precue'),None);perm=next((r for r in rows if r['model']=='permuted'),None) |
| chisco=json.load(open('reports/chisco_test.json')) if Path('reports/chisco_test.json').exists() else [] |
| chisco_selected=manifest.get('chisco_primary_model',{}).get('model','none') |
| chtext='Validation-selected EEG pipeline: '+chisco_selected+'.\n\n'+'\n'.join(f"- {r['model']}: {pct(r['metrics']['accuracy'])} accuracy, {pct(r['metrics']['balanced_accuracy'])} balanced accuracy, macro-F1 {r['metrics']['macro_f1']:.3f}; {r['test_n']} trials, 39 semantic classes; uniform chance 2.56%. Sentence-disjoint test trials: {r['text_disjoint_n']}." for r in chisco) or 'No completed Chisco final-test experiment.' |
| five=json.load(open('reports/chisco_five_test.json')) if Path('reports/chisco_five_test.json').exists() else [] |
| if five: |
| chtext+='\n\nSecondary constrained semantic task: five categories selected solely by training frequency (not EEG performance). This is not five-sentence identification. Validation-selected EEG pipeline: '+manifest.get('chisco_five_primary_model',{}).get('model','none')+'.\n' |
| for r in five: |
| if 'metrics' not in r: |
| chtext+=f"\n- Five-category {r['model']}: no held-out examples in the training-selected vocabulary.";continue |
| chtext+=f"\n- Five-category {r['model']}: {pct(r['metrics']['accuracy'])} accuracy, {pct(r['metrics']['balanced_accuracy'])} balanced accuracy, macro-F1 {r['metrics']['macro_f1']:.3f}; {r['test_n']} test trials, {r['test_category_coverage']}/5 categories observed; block-bootstrap accuracy 95% CI {pct(r['block_bootstrap_accuracy_ci95'][0])}–{pct(r['block_bootstrap_accuracy_ci95'][1])}. Uniform balanced-accuracy chance 20%; train-majority prior accuracy {pct(r['majority_prior_metrics']['accuracy'])}." |
| if Path('reports/chisco_temporal_structure_control.json').exists(): |
| temporal=json.load(open('reports/chisco_temporal_structure_control.json'));chtext+=f"\n\n**Temporal shortcut audit:** using only ordinal trial index, a 5-neighbor classifier achieved {pct(temporal['balanced_accuracy'])} balanced accuracy on an interleaved 80/20 split of blocks 1–36 (1,017 held-out trials), versus 2.56% uniform chance. No EEG or text features were used. This is an analogous diagnostic split, not a reproduction of the published model; it demonstrates a possible temporal shortcut and does not establish that a published model used it. SETU's primary semantic evaluation holds out entire blocks. Final-test blocks were not used in this diagnostic." |
| reference_results=json.load(open('reports/reference3s_test.json')) if Path('reports/reference3s_test.json').exists() else [] |
| if reference_results: |
| pd.DataFrame([{'model':r['model'],'protocol':r['protocol'],'channels':r['channels'],'test_n':r['test_n'],**{k:v for k,v in r['metrics'].items() if k!='confusion_matrix'}} for r in reference_results]).to_csv('reports/reference3s_results.csv',index=False) |
| reference_text='\n'.join(f"- {r['model']}: {pct(r['metrics']['balanced_accuracy'])} measured test balanced accuracy; 64 channels, pooled calibration, 750 held-out trials." for r in reference_results) or 'No completed three-second reproduction test yet.' |
| comparisons=json.load(open('reports/control_comparisons.json')) |
| matched=comparisons.get('matched_late_minus_precue',{}) |
| matched_text=(f"The duration/bandwidth-matched late-imagination minus pre-imagination difference is {100*matched['mean_accuracy_difference']:.1f} percentage points (paired participant-bootstrap 95% CI {100*matched['paired_subject_bootstrap_ci95'][0]:.1f} to {100*matched['paired_subject_bootstrap_ci95'][1]:.1f})." if matched else '') |
| channel_text='\n'.join(f"- Distributed {r['channels']}-channel {r['model']}: {pct(r['accuracy'])} test accuracy, macro-F1 {r['macro_f1']:.3f}." for r in rows if r['model']==manifest['primary_family'][0] and r['montage']=='distributed' and r['seed']==manifest['primary_family'][3]) |
| benchmarks=json.load(open('reports/inference_benchmarks.json'));bench=next((r for r in benchmarks if [r['model'],r['montage'],r['channels'],r['seed']]==manifest['primary_family']),None) |
| latency_text=(f"Representative subject-01 primary checkpoint: {bench['checkpoint_bytes']:,} bytes; warm CPU preprocessing plus inference median {bench['median_preprocess_and_inference_ms']:.2f} ms over five validation trials. This excludes model loading and the two-second EEG acquisition interval." if bench else '') |
| gpu_note=('An additional CUDA run passed an actual GPU tensor computation; see reports/gpu_environment.json and gpu_job_status.json for GPU models and precision. Earlier experiments used CPU. ' if Path('reports/gpu_environment.json').exists() else '') |
| selected=' / '.join(map(str,manifest['primary_family']));start=datetime.datetime.fromisoformat(json.load(open('experiments/run_budget.json'))['start_utc']);now=datetime.datetime.now(datetime.timezone.utc);hours=(now-start).total_seconds()/3600 |
| text=f'''# SETU EEG research report |
| |
| Generated {now.isoformat()}. Elapsed {hours:.2f} hours from run start. Approximate instance time at the user-supplied $3.29/hour rate: ${hours*3.29:.2f}; this is an estimate, not a billing record. |
| |
| ## Strongest supported claim |
| |
| **SETU implemented and evaluated a reproducible, constrained five-command EEG classifier on public imagined-speech recordings. The validation-selected primary system achieved {pct(primary['accuracy'])} official-test accuracy (subject-bootstrap 95% CI {pct(primary['ci_low'])}–{pct(primary['ci_high'])}), versus 20% uniform chance, across 15 previously calibrated participants and {primary['trials']} trials.** |
| |
| Primary configuration: **{selected}**. Balanced accuracy {pct(primary['balanced_accuracy'])}; macro-F1 {primary['macro_f1']:.3f}. Selection used mean validation balanced accuracy across all participants, not test performance. Per-participant hyperparameters were selected on their official validation split. The demo uses this frozen family, with a separate trained checkpoint for each participant unless the selected family is pooled. |
| |
| This establishes retrospective benchmark performance under a cued imagined-speech protocol. It does **not** establish unrestricted mind reading, neural specificity of the decoded signal, live operation, new-person generalization at the same accuracy, or performance with wearable dry electrodes. |
| |
| ## Evaluation and controls |
| |
| - Official train/validation/test counts: 4,500 / 750 / 750. No trial windows were split across partitions. No final-test labels entered training, normalization, feature selection, early stopping or model-family selection. |
| - Transparency exception: an initial shell preview displayed seven subject-01 test labels before training. No model used them. This precludes describing the run as a fully blinded human audit, though the programmatic tuning path excludes test labels. |
| - Primary trial window: [0,2) seconds after imagination onset; samples 128:640 of the original -500 to 2601.5625 ms epochs. Rereference using only selected measured channels, fourth-order 1–40 Hz Butterworth filtering independently per trial, resample 256 to 200 Hz. Pipeline tests prove samples outside the window cannot affect features. |
| - Original subject-01 MATLAB labels exactly match the EDF conversion. Relative conversion RMSE is 0.000252; maximum absolute error is 0.0164 µV. All NEMAR files passed upstream manifest checksums. After selection was frozen, all 750 test labels were checked against the original official competition answer sheet, and subject-01 test signals/time axes were checked against the original MATLAB file; see reports/official_test_verification.json. Test duplicate audit is stored separately. |
| - Classical pipelines use OAS covariance/MDM, Riemannian tangent-space logistic regression, log-bandpower logistic regression, RBF SVM, multiband supervised CSP, and filter-bank Riemannian tangent spaces. All learned transformations fit training data only. |
| - EEGNet and EEG Conformer use Braindecode 1.2.0; Conformer has three transformer blocks. EEGNet uses three seeds at 64 channels. Checkpoints and epoch logs record exact settings. Subject-specific and pooled-subject models remain distinct; pooled models seeing all participants are not cross-person tests. |
| - CBraMod official weights were SHA256-verified against Hugging Face LFS metadata and strictly loaded. Inputs are measured selected channels × two one-second patches × 200 samples, in µV/100; output embeddings have dimension 200. Frozen flattened-feature probes, last-two-transformer-block partial fine-tuning, and full fine-tuning were compared. Source and checkpoint revisions are in reports/sources.json. The GPU extension additionally compares REVE-Base and CSBrain frozen probes, partial adaptation and full adaptation on the identical official splits and real-channel subsets when completed; sources, scaling, positional conventions and provenance limits are in reports/gpu_sources.json. REVE uses the official speech scale µV/1000 and 200-sample patches with 20-sample overlap; its two patches cover 380 of the 400 available samples. CSBrain uses µV/100 and rebuilds region membership/order for selected measured electrodes. |
| - Training-label permutation control: {pct(perm['accuracy']) if perm else 'not completed'} final-test accuracy. This is a single fixed permutation per participant, not a full permutation significance test. |
| - Pre-imagination fixation-window control: {pct(cue['accuracy']) if cue else 'not completed'} final-test accuracy. This short control differs in duration/bandwidth and is not a perfectly matched null. Above-chance pre-imagination decoding would indicate cue memory, anticipation, or correlated artifacts; it prevents attribution solely to imagined articulation. |
| - {matched_text} |
| - Leave-one-subject-out bandpower model: {pct(loso['accuracy']) if loso else 'not completed'} test accuracy. The 32-channel leave-one-subject-out SVM achieved {pct(lososvm['accuracy']) if lososvm else 'not completed'}. Each outer participant was absent from training and hyperparameter selection. Inner validation used other participants; it is not the same estimand as calibrated within-person performance. |
| - Secondary per-subject architecture selection, also frozen before test: {pct(secondary['accuracy'])} accuracy. This has a larger validation search and must not replace the primary endpoint after seeing test outcomes. |
| - Confidence intervals bootstrap 15 participants. They do not turn the 750 correlated trials into 750 independent subjects. LOSO training sets overlap; uncertainty remains substantial. No corrected claim of significance across all exploratory comparisons is made. |
| |
| ## Published-protocol comparison (separate endpoint) |
| |
| Published reference balanced accuracies are REVE-Base 56.35%, CSBrain 60.04% and CBraMod 53.73%; these are external references, never SETU results. Sources and overlap evidence: reports/PRETRAINING_OVERLAP.md and configs/foundation_benchmark.json. |
| |
| {reference_text} |
| |
| The reproduction arm follows the official last-768-samples/Fourier-resample-to-600 recipe, pooled participant calibration and architecture-specific heads. It uses a bounded single-seed training schedule, so this is protocol reproduction with declared optimization differences, not a claim of exact paper replication. That three-second window includes pre/post-imagination data; it is excluded from the stricter primary two-second model selection and demo. Both endpoints freeze validation selection before opening the same official test. Pretraining exclusion is author-reported rather than independently verified against private corpus files. |
| |
| ## Channel reduction and hardware |
| |
| 64, 32, 16 and 8 channels are actual recorded electrodes, with each reduced model retrained. No zero-padded or interpolated virtual electrodes are counted as measurements. Distributed selection uses standard coordinate distances; a left frontotemporal-oriented montage and training-only univariate ranking provide additional comparisons. See reports/figures/montages.png and reports/RESULTS.md. Paired participant-bootstrap estimates of accuracy loss versus 64 channels are in reports/channel_effects.json. Measured CPU latency and checkpoint complexity are in reports/inference_benchmarks.json; frozen CBraMod probe files additionally require the shared 19.8 MB pretrained backbone. Reference and ground are additional physical contacts. Training-only ranking requires high-density calibration and is not proof of a deployable fixed montage. |
| |
| {channel_text} |
| |
| {latency_text} |
| |
| ## Chisco semantic recognition |
| |
| {chtext} |
| |
| These are semantic-category classifiers, not exact sentence identification or text generation. Raw-header audit verifies 1,000 Hz acquisition; the analyzed derivative is 500 Hz with 122 EEG, two EOG and one trigger channel. Subject 01 train blocks 1–27, validation 28–36, test 37–45; reading EEG, EOG and trigger channels are excluded from the EEG classifier. The separate EOG classifier is an artifact control. The neural pipeline uses the final three seconds of the imagined phase. Upstream derivative preprocessing includes operations that may have been fitted per recording block; this is retrospective derivative evaluation, not a prospective train-only preprocessing reproduction. Dataset metadata says CC0 while README adds non-commercial wording: resolve this before commercial redistribution. Chisco data are excluded from the demo. |
| |
| ## Scientific limitations |
| |
| The competition protocol uses four repeated imaginations per auditory cue. Released epochs lack original cue-group identifiers and session chronology; possible shared-cue context between partitions cannot be ruled out. Original acquisition used a common reference; selecting fewer channels cannot simulate independent wearable acquisition noise. BCI data lack EOG/EMG recordings, so residual eye, jaw and muscle signals cannot be fully ruled out. reports/amplitude_quality_analysis.json stratifies performance by a training-derived amplitude threshold; it is descriptive and is not a validated artifact detector. No flagged trials are removed from the primary result. The official benchmark is small, with 50 validation trials per person and a broad exploratory model search. Final-test results are one fixed evaluation, not independent replication. Novel individuals, new days, self-initiated speech and the rest/no-command state require new data. |
| |
| ## Infrastructure and reproducibility |
| |
| The host inventory lists an H100 PCIe, but the execution environment exposes no NVIDIA device files. nvidia-smi fails; compatible PyTorch 2.7.0+cu128 reports CUDA unavailable. **The initial CPU run could not access CUDA.** {gpu_note}Device visibility differs between the Codex sandbox and the normal SSH shell, where the user verified an H100 working with PyTorch. Reinstalling Python CUDA libraries resolved dependency conflicts, not device access. Environment details and pip versions are archived in reports/environment.json and requirements.lock.txt. |
| |
| No configured persistent backup destination was found in the accessible project/home configuration or mounts. Artifacts remain on local instance storage, which may disappear if the instance is terminated. This run did not provision resources, expose credentials, terminate instances or use paid APIs. |
| |
| Run `./setup.sh` for the pinned local environment and `./run_all.sh` to execute/resume the bounded pipeline. `./resume.sh` preserves the original deadline. Completed experiments are skipped; once selection is frozen, these scripts refuse further tuning against the opened test set. Full fresh reproduction requires an empty output workspace with these source/configuration files. The six-hour deadline is stored in experiments/run_budget.json; restartable neural optimizer snapshots end in _last.pt. |
| |
| ## Investor demonstration |
| |
| ```bash |
| cd /home/ubuntu/setu-eeg |
| .venv/bin/python demos/replay.py --subject 1 --trial 0 --count 10 |
| .venv/bin/python demos/server.py --host 127.0.0.1 --port 8080 |
| ``` |
| |
| Open http://127.0.0.1:8080 on the host or use an SSH tunnel. The browser clearly labels recorded EEG, computes actual predictions, reports latency and uncalibrated confidence, then reveals truth. It advances sequentially and retains errors. Selected checkpoint paths are in reports/frozen_selection.json under demo_subject_models. Latency measurements are in reports/demo_validation.json when present; they exclude the two-second signal acquisition interval. |
| |
| ## Artifact index |
| |
| - reports/RESULTS.md and reports/results.csv: every completed frozen model/channel/seed test comparison. |
| - reports/all_completed_validation_experiments.csv: complete validation and exploratory experiment inventory. |
| - reports/predictions/: per-trial outputs; reports/test_subject_results.json: participant results. |
| - reports/figures/: confusion matrix, participant performance, channel-reduction curves and montages. |
| - checkpoints/: trained models and resumable neural snapshots. models/cbramod_pretrained.pth: official pretrained model. |
| - reports/DATASET_INVENTORY.md: provenance, labels, timing, coverage and license limitations. |
| - PROGRESS.md: decisions, failures and changes; NEXT_EXPERIMENTS.md: hardware validation protocol. |
| ''' |
| Path('FINAL_REPORT.md').write_text(text) |
| print(selected,primary,flush=True) |
| if __name__=='__main__':main() |
|
|
