feat(storage): add HfBucketStore with opt-in real-bucket integration test

package-lock.json

@@ -21,7 +21,6 @@
         "svelte": "^5.38.0",
         "svelte-check": "^4.3.0",
         "tailwindcss": "^4.1.0",
-        "tsx": "^4.20.0",
         "typescript": "^5.9.0",
         "vite": "^7.1.0",
         "vitest": "^3.2.0"
@@ -2981,40 +2980,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/tsx": {
-      "version": "4.23.12",
-      "resolved": "https://npm.registries.huggingface.tech/tsx/-/tsx-4.23.12.tgz",
-      "integrity": "sha512-FDf4L4sYzKtzWYhU/Xm0AQFdTjdIxNo9ElTf2mxXM6k8YMHXzYUe4yODVaXP4V9uMFbVg8c0qyBccK2OOxb45Q==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "esbuild": "~0.28.0"
-      },
-      "bin": {
-        "tsx": "dist/cli.mjs"
-      },
-      "engines": {
-        "node": ">=18.0.0"
-      },
-      "optionalDependencies": {
-        "fsevents": "~2.3.3"
-      }
-    },
-    "node_modules/tsx/node_modules/fsevents": {
-      "version": "2.3.3",
-      "resolved": "https://npm.registries.huggingface.tech/fsevents/-/fsevents-2.3.3.tgz",
-      "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
-      "dev": true,
-      "hasInstallScript": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "darwin"
-      ],
-      "engines": {
-        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
-      }
-    },
     "node_modules/typescript": {
       "version": "5.9.3",
       "resolved": "https://npm.registries.huggingface.tech/typescript/-/typescript-5.9.3.tgz",

package.json

@@ -12,7 +12,7 @@
     "test": "vitest run",
     "test:watch": "vitest",
     "test:e2e": "playwright test",
-    "spike": "tsx scripts/spike-bucket.ts"
+    "test:integration": "vitest run tests/integration"
   },
   "dependencies": {
     "@huggingface/hub": "^2.16.1",
@@ -28,7 +28,6 @@
     "svelte": "^5.38.0",
     "svelte-check": "^4.3.0",
     "tailwindcss": "^4.1.0",
-    "tsx": "^4.20.0",
     "typescript": "^5.9.0",
     "vite": "^7.1.0",
     "vitest": "^3.2.0"

src/lib/server/storage/hf-bucket-store.ts

@@ -0,0 +1,103 @@
+import { commit, copyFile, deleteFiles, downloadFile, listFiles, uploadFiles } from '@huggingface/hub';
+import { assertValidPath, type BucketStore, type StoreEntry } from './store';
+
+function statusOf(e: unknown): number | undefined {
+	return typeof e === 'object' && e !== null ? (e as { statusCode?: number }).statusCode : undefined;
+}
+
+function toBlob(bytes: Uint8Array, type?: string): Blob {
+	return new Blob([bytes as unknown as BlobPart], type ? { type } : undefined);
+}
+
+/**
+ * BucketStore backed by a Hugging Face Storage Bucket through `@huggingface/hub`.
+ * `repo` is the full bucket id, e.g. `buckets/assafvayner/splitwise-data`.
+ */
+export class HfBucketStore implements BucketStore {
+	constructor(
+		private readonly repo: `buckets/${string}`,
+		private readonly accessToken: string
+	) {}
+
+	private get auth() {
+		return { repo: this.repo, accessToken: this.accessToken };
+	}
+
+	async get(path: string): Promise<Uint8Array | null> {
+		assertValidPath(path);
+		const blob = await downloadFile({ ...this.auth, path });
+		if (!blob) return null;
+		return new Uint8Array(await blob.arrayBuffer());
+	}
+
+	async put(path: string, bytes: Uint8Array, contentType = 'application/octet-stream'): Promise<void> {
+		assertValidPath(path);
+		await uploadFiles({
+			...this.auth,
+			commitTitle: `put ${path}`,
+			files: [{ path, content: toBlob(bytes, contentType) }]
+		});
+	}
+
+	async append(path: string, bytes: Uint8Array): Promise<void> {
+		assertValidPath(path);
+		const original = await downloadFile({ ...this.auth, path });
+		if (!original) return this.put(path, bytes, 'application/x-ndjson');
+		try {
+			await commit({
+				...this.auth,
+				title: `append ${path}`,
+				operations: [
+					{
+						operation: 'edit',
+						path,
+						originalContent: original,
+						edits: [{ content: toBlob(bytes), start: original.size, end: original.size }]
+					}
+				]
+			});
+		} catch (e) {
+			console.warn(`[hf-bucket-store] edit-append failed for ${path}; falling back to full rewrite`, e);
+			const existing = new Uint8Array(await original.arrayBuffer());
+			const joined = new Uint8Array(existing.length + bytes.length);
+			joined.set(existing, 0);
+			joined.set(bytes, existing.length);
+			await this.put(path, joined, 'application/x-ndjson');
+		}
+	}
+
+	async list(prefix: string): Promise<StoreEntry[]> {
+		const dir = prefix.replace(/\/+$/, '');
+		const out: StoreEntry[] = [];
+		try {
+			for await (const entry of listFiles({ ...this.auth, path: dir || undefined, recursive: true, expand: true })) {
+				if (entry.type !== 'file' || !entry.path.startsWith(prefix)) continue;
+				out.push({
+					path: entry.path,
+					size: entry.size,
+					uploadedAt: entry.uploadedAt ? new Date(entry.uploadedAt) : undefined
+				});
+			}
+		} catch (e) {
+			if (statusOf(e) === 404) return [];
+			throw e;
+		}
+		return out.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+	}
+
+	async delete(paths: string[]): Promise<void> {
+		if (paths.length === 0) return;
+		for (const p of paths) assertValidPath(p);
+		await deleteFiles({ ...this.auth, paths, commitTitle: `delete ${paths.length} object(s)` });
+	}
+
+	async copy(src: string, dst: string): Promise<void> {
+		assertValidPath(src);
+		assertValidPath(dst);
+		await copyFile({
+			accessToken: this.accessToken,
+			source: { repo: this.repo, path: src },
+			destination: { repo: this.repo, path: dst }
+		});
+	}
+}

tests/integration/env.ts

@@ -0,0 +1,23 @@
+import { readFileSync } from 'node:fs';
+
+/** Loads KEY=value lines from .env into process.env without overriding existing variables. */
+export function loadDotEnv(path = '.env'): void {
+	let text: string;
+	try {
+		text = readFileSync(path, 'utf8');
+	} catch {
+		return;
+	}
+	for (const line of text.split('\n')) {
+		const m = /^\s*([A-Z_][A-Z0-9_]*)\s*=\s*(.*?)\s*$/.exec(line);
+		if (m && !(m[1] in process.env)) process.env[m[1]] = m[2].replace(/^["']|["']$/g, '');
+	}
+}
+
+export function realBucket(): { token: string; bucket: `buckets/${string}` } | null {
+	loadDotEnv();
+	const token = process.env.HF_TOKEN;
+	const bucket = process.env.HF_BUCKET;
+	if (!token || !bucket?.startsWith('buckets/')) return null;
+	return { token, bucket: bucket as `buckets/${string}` };
+}

tests/integration/hf-bucket-store.test.ts

@@ -0,0 +1,58 @@
+import { afterAll, describe, expect, it } from 'vitest';
+import { ulid } from '../../src/lib/domain/ulid';
+import { HfBucketStore } from '../../src/lib/server/storage/hf-bucket-store';
+import { PrefixedStore } from '../../src/lib/server/storage/prefixed-store';
+import { decodeText, encodeText } from '../../src/lib/server/storage/store';
+import { realBucket } from './env';
+
+const creds = realBucket();
+
+describe.skipIf(!creds)(
+	'HfBucketStore against the real bucket',
+	() => {
+		const store = new PrefixedStore(new HfBucketStore(creds!.bucket, creds!.token), `test/${ulid()}`);
+
+		afterAll(async () => {
+			await store.wipe();
+			expect(await store.list('')).toEqual([]);
+		}, 60_000);
+
+		it('returns null for a missing object', async () => {
+			expect(await store.get('missing.json')).toBeNull();
+		});
+
+		it('put + get round trip, and read-after-write returns the newest bytes', async () => {
+			await store.put('a.json', encodeText('{"a":1}'), 'application/json');
+			expect(decodeText((await store.get('a.json'))!)).toBe('{"a":1}');
+			for (let i = 0; i < 3; i++) {
+				await store.put('rw.txt', encodeText(`v${i}`));
+				expect(decodeText((await store.get('rw.txt'))!)).toBe(`v${i}`);
+			}
+		});
+
+		it('lists by prefix and returns [] for an unknown prefix', async () => {
+			await store.put('events/one.json', encodeText('1'));
+			const listed = await store.list('events/');
+			expect(listed.map((e) => e.path)).toEqual(['events/one.json']);
+			expect(listed[0].size).toBe(1);
+			expect(await store.list('nothing-here/')).toEqual([]);
+		});
+
+		it('append creates the object, then appends via the edit operation', async () => {
+			await store.append('ledger.jsonl', encodeText('{"n":1}\n'));
+			await store.append('ledger.jsonl', encodeText('{"n":2}\n'));
+			expect(decodeText((await store.get('ledger.jsonl'))!)).toBe('{"n":1}\n{"n":2}\n');
+		});
+
+		it('copies server-side', async () => {
+			await store.copy('ledger.jsonl', 'backups/ledger.jsonl');
+			expect(decodeText((await store.get('backups/ledger.jsonl'))!)).toBe('{"n":1}\n{"n":2}\n');
+		});
+
+		it('deletes, ignoring missing paths', async () => {
+			await store.delete(['a.json', 'never-existed.txt']);
+			expect(await store.get('a.json')).toBeNull();
+		});
+	},
+	60_000
+);

vite.config.ts

@@ -5,7 +5,7 @@ import { defineConfig } from 'vitest/config';
 export default defineConfig({
 	plugins: [tailwindcss(), sveltekit()],
 	test: {
-		include: ['tests/unit/**/*.test.ts'],
+		include: ['tests/unit/**/*.test.ts', 'tests/integration/**/*.test.ts'],
 		environment: 'node'
 	}
 });