File size: 3,620 Bytes
60faa09 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 | /**
* HuggingFace Dataset Storage adapter.
* Uses @huggingface/hub's uploadFile/listFiles/downloadFile/deleteFile/createRepo.
*/
import {
createRepo, uploadFile, listFiles, downloadFile, deleteFile,
} from "@huggingface/hub";
export class HFStorage {
constructor({ token, repo } = {}) {
this.token = token || process.env.HF_TOKEN;
this.repo = repo || process.env.HF_DATASET_REPO;
if (!this.token) {
throw new Error("HFStorage requires a HF token (set HF_TOKEN env var or pass token in constructor).");
}
this.credentials = { accessToken: this.token };
}
async ensureRepo(repo = this.repo, { isPrivate = false } = {}) {
if (!repo) throw new Error("No HF repo specified.");
const [namespace, name] = repo.split("/");
if (!namespace || !name) throw new Error(`Invalid HF repo id: ${repo}`);
try {
await createRepo({
repo: { type: "dataset", name: repo },
credentials: this.credentials,
private: isPrivate,
});
} catch (e) {
if (!String(e.message).includes("already exists") && e.statusCode !== 409) {
console.warn("[HF_ENSURE_REPO]", e.message);
}
}
return true;
}
async uploadFile(file, path) {
if (!this.repo) {
throw new Error("HF_DATASET_REPO is not set. Set it to e.g. 'username/hugging-c-ai-assets'.");
}
await this.ensureRepo(this.repo);
await uploadFile({
repo: { type: "dataset", name: this.repo },
path, file, credentials: this.credentials,
});
const url = `https://huggingface.co/datasets/${this.repo}/resolve/main/${path}`;
return { url, repo: this.repo, path };
}
async listFiles(dirPath = "") {
if (!this.repo) return [];
try {
const entries = [];
for await (const entry of listFiles({
repo: { type: "dataset", name: this.repo },
path: dirPath,
credentials: this.credentials,
expand: true,
})) {
entries.push({
name: entry.path.split("/").pop(),
path: entry.path,
type: entry.type || "file",
size: entry.size,
lastModified: entry.lastCommit?.date,
});
}
return entries;
} catch (e) {
console.warn("[HF_LIST_FILES_ERROR]", e.message);
return [];
}
}
async readFile(path) {
return await downloadFile({
repo: { type: "dataset", name: this.repo },
path, credentials: this.credentials,
});
}
async deleteFile(path) {
await deleteFile({
repo: { type: "dataset", name: this.repo },
path, credentials: this.credentials,
});
return true;
}
resolveUrl(repo, path) {
return `https://huggingface.co/datasets/${repo}/resolve/main/${path}`;
}
}
export function getHFStorageForUser({ hfAccessToken, hfUsername } = {}) {
const token = hfAccessToken || process.env.HF_TOKEN;
if (!token) return null;
let repo = process.env.HF_DATASET_REPO;
if (!repo && hfUsername) repo = `${hfUsername}/hugging-c-ai-assets`;
if (!repo) return null;
return new HFStorage({ token, repo });
}
export async function appendJsonlRow(filename, row, storage) {
if (!storage) return false;
try {
const line = JSON.stringify(row) + "\n";
let existing = "";
try {
const blob = await storage.readFile(filename);
if (blob && blob.text) existing = await blob.text();
else if (typeof blob === "string") existing = blob;
} catch (e) {}
await storage.uploadFile(existing + line, filename);
return true;
} catch (e) {
console.warn("[HF_JSONL_APPEND_ERROR]", e.message);
return false;
}
}
|