/** * HuggingFace Dataset Storage adapter. * Uses @huggingface/hub's uploadFile/listFiles/downloadFile/deleteFile/createRepo. */ import { createRepo, uploadFile, listFiles, downloadFile, deleteFile, } from "@huggingface/hub"; export class HFStorage { constructor({ token, repo } = {}) { this.token = token || process.env.HF_TOKEN; this.repo = repo || process.env.HF_DATASET_REPO; if (!this.token) { throw new Error("HFStorage requires a HF token (set HF_TOKEN env var or pass token in constructor)."); } this.credentials = { accessToken: this.token }; } async ensureRepo(repo = this.repo, { isPrivate = false } = {}) { if (!repo) throw new Error("No HF repo specified."); const [namespace, name] = repo.split("/"); if (!namespace || !name) throw new Error(`Invalid HF repo id: ${repo}`); try { await createRepo({ repo: { type: "dataset", name: repo }, credentials: this.credentials, private: isPrivate, }); } catch (e) { if (!String(e.message).includes("already exists") && e.statusCode !== 409) { console.warn("[HF_ENSURE_REPO]", e.message); } } return true; } async uploadFile(file, path) { if (!this.repo) { throw new Error("HF_DATASET_REPO is not set. Set it to e.g. 'username/hugging-c-ai-assets'."); } await this.ensureRepo(this.repo); await uploadFile({ repo: { type: "dataset", name: this.repo }, path, file, credentials: this.credentials, }); const url = `https://huggingface.co/datasets/${this.repo}/resolve/main/${path}`; return { url, repo: this.repo, path }; } async listFiles(dirPath = "") { if (!this.repo) return []; try { const entries = []; for await (const entry of listFiles({ repo: { type: "dataset", name: this.repo }, path: dirPath, credentials: this.credentials, expand: true, })) { entries.push({ name: entry.path.split("/").pop(), path: entry.path, type: entry.type || "file", size: entry.size, lastModified: entry.lastCommit?.date, }); } return entries; } catch (e) { console.warn("[HF_LIST_FILES_ERROR]", e.message); return []; } } async readFile(path) { return await downloadFile({ repo: { type: "dataset", name: this.repo }, path, credentials: this.credentials, }); } async deleteFile(path) { await deleteFile({ repo: { type: "dataset", name: this.repo }, path, credentials: this.credentials, }); return true; } resolveUrl(repo, path) { return `https://huggingface.co/datasets/${repo}/resolve/main/${path}`; } } export function getHFStorageForUser({ hfAccessToken, hfUsername } = {}) { const token = hfAccessToken || process.env.HF_TOKEN; if (!token) return null; let repo = process.env.HF_DATASET_REPO; if (!repo && hfUsername) repo = `${hfUsername}/hugging-c-ai-assets`; if (!repo) return null; return new HFStorage({ token, repo }); } export async function appendJsonlRow(filename, row, storage) { if (!storage) return false; try { const line = JSON.stringify(row) + "\n"; let existing = ""; try { const blob = await storage.readFile(filename); if (blob && blob.text) existing = await blob.text(); else if (typeof blob === "string") existing = blob; } catch (e) {} await storage.uploadFile(existing + line, filename); return true; } catch (e) { console.warn("[HF_JSONL_APPEND_ERROR]", e.message); return false; } }