src/lib/storage/hf.js

/**
 * HuggingFace Dataset Storage adapter.
 * Uses @huggingface/hub's uploadFile/listFiles/downloadFile/deleteFile/createRepo.
 */
import {
  createRepo, uploadFile, listFiles, downloadFile, deleteFile,
} from "@huggingface/hub";

export class HFStorage {
  constructor({ token, repo } = {}) {
    this.token = token || process.env.HF_TOKEN;
    this.repo = repo || process.env.HF_DATASET_REPO;
    if (!this.token) {
      throw new Error("HFStorage requires a HF token (set HF_TOKEN env var or pass token in constructor).");
    }
    this.credentials = { accessToken: this.token };
  }

  async ensureRepo(repo = this.repo, { isPrivate = false } = {}) {
    if (!repo) throw new Error("No HF repo specified.");
    const [namespace, name] = repo.split("/");
    if (!namespace || !name) throw new Error(`Invalid HF repo id: ${repo}`);
    try {
      await createRepo({
        repo: { type: "dataset", name: repo },
        credentials: this.credentials,
        private: isPrivate,
      });
    } catch (e) {
      if (!String(e.message).includes("already exists") && e.statusCode !== 409) {
        console.warn("[HF_ENSURE_REPO]", e.message);
      }
    }
    return true;
  }

  async uploadFile(file, path) {
    if (!this.repo) {
      throw new Error("HF_DATASET_REPO is not set. Set it to e.g. 'username/hugging-c-ai-assets'.");
    }
    await this.ensureRepo(this.repo);
    await uploadFile({
      repo: { type: "dataset", name: this.repo },
      path, file, credentials: this.credentials,
    });
    const url = `https://huggingface.co/datasets/${this.repo}/resolve/main/${path}`;
    return { url, repo: this.repo, path };
  }

  async listFiles(dirPath = "") {
    if (!this.repo) return [];
    try {
      const entries = [];
      for await (const entry of listFiles({
        repo: { type: "dataset", name: this.repo },
        path: dirPath,
        credentials: this.credentials,
        expand: true,
      })) {
        entries.push({
          name: entry.path.split("/").pop(),
          path: entry.path,
          type: entry.type || "file",
          size: entry.size,
          lastModified: entry.lastCommit?.date,
        });
      }
      return entries;
    } catch (e) {
      console.warn("[HF_LIST_FILES_ERROR]", e.message);
      return [];
    }
  }

  async readFile(path) {
    return await downloadFile({
      repo: { type: "dataset", name: this.repo },
      path, credentials: this.credentials,
    });
  }

  async deleteFile(path) {
    await deleteFile({
      repo: { type: "dataset", name: this.repo },
      path, credentials: this.credentials,
    });
    return true;
  }

  resolveUrl(repo, path) {
    return `https://huggingface.co/datasets/${repo}/resolve/main/${path}`;
  }
}

export function getHFStorageForUser({ hfAccessToken, hfUsername } = {}) {
  const token = hfAccessToken || process.env.HF_TOKEN;
  if (!token) return null;
  let repo = process.env.HF_DATASET_REPO;
  if (!repo && hfUsername) repo = `${hfUsername}/hugging-c-ai-assets`;
  if (!repo) return null;
  return new HFStorage({ token, repo });
}

export async function appendJsonlRow(filename, row, storage) {
  if (!storage) return false;
  try {
    const line = JSON.stringify(row) + "\n";
    let existing = "";
    try {
      const blob = await storage.readFile(filename);
      if (blob && blob.text) existing = await blob.text();
      else if (typeof blob === "string") existing = blob;
    } catch (e) {}
    await storage.uploadFile(existing + line, filename);
    return true;
  } catch (e) {
    console.warn("[HF_JSONL_APPEND_ERROR]", e.message);
    return false;
  }
}