Spaces:
Build error
Build error
File size: 3,608 Bytes
c340098 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 | from huggingface_hub import HfApi
import gradio as gr
import pandas as pd
import os
import hashlib
from datasets import Dataset, concatenate_datasets, load_dataset
from huggingface_hub import login
import json
import uuid
from datetime import datetime, UTC
import torch
from huggingface_hub import snapshot_download
import numpy as np
from config.settings import *
from data.contexts import CONTEXTS
HF_TOKEN = os.environ.get("MyJulySecretToken") # store your token as a secret in Spaces
login(HF_TOKEN)
api = HfApi(token = HF_TOKEN)
def load_hf_dataset():
"""Load existing HF Dataset or create empty one if not exists."""
try:
ds = load_dataset(HF_DATASET_NAME, split="train")
except:
# Dataset does not exist yet
df = pd.DataFrame(columns=["user_id", "gender", "audio_file", "score"])
ds = Dataset.from_pandas(df)
ds.push_to_hub(HF_DATASET_NAME, private=True)
return ds
def upload_generated_audio(
audio_path,
model_name,
context,
comparison_id
):
audio_name = (
f"{model_name}_{context}_{comparison_id}.wav"
)
api.upload_file(
path_or_fileobj = audio_path,
path_in_repo = f"data/{audio_name}",
repo_id = HF_DATASET_NAME,
repo_type = "dataset"
)
return audio_name
def upload_admin_audio(local_path: str, audio_name: str, audio_id: str) -> str:
"""Upload an audio file to the Hugging Face dataset repository."""
repo_path = f"admin_audio/{audio_id}_{audio_name}"
with open(local_path, "rb") as f:
api.upload_file(
path_or_fileobj = f,
path_in_repo = repo_path,
repo_id = HF_DATASET_NAME,
repo_type = "dataset"
)
return repo_path
def export_full_dataset() -> str:
import tempfile, zipfile, glob
tmp_dir = tempfile.mkdtemp()
repo_dir = snapshot_download(
repo_id = HF_DATASET_NAME,
repo_type = "dataset",
token = HF_TOKEN,
local_dir = tmp_dir
)
#Fusionner tous les JSONs annotation en un seul CSV
json_files = glob.glob(os.path.join(repo_dir, "annotations", "*.json"))
records = []
for path in json_files:
with open(path, "r", encoding = "utf-8") as f:
records.append(json.load(f))
df = pd.DataFrame(records)
csv_path = os.path.join(tmp_dir, "annotations.csv")
df.to_csv(csv_path, index = False)
#Emballer CSV + les fichiers audio dans un ZIP
zip_path = os.path.join(tmp_dir, "dataset_export.zip")
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
zf.write(csv_path, "annotations.csv")
for audio in glob.glob(os.path.join(repo_dir, "data", "*.wav")):
zf.write(audio, os.path.join("audio", os.path.basename(audio)))
return zip_path
def export_admin_audio_annotations() -> str:
"""Fusionne uniquement les annotations issues du mode audio_file en un CSV"""
import tempfile, glob
tmp_dir = tempfile.mkdtemp()
repo_dir = snapshot_download(
repo_id = HF_DATASET_NAME,
repo_type = "dataset",
token = HF_TOKEN,
local_dir = tmp_dir,
allow_patterns = ["annotations_*/*.json"]
)
json_files = glob.glob(os.path.join(repo_dir, "annotations_*", "*.json"))
records = []
for path in json_files:
with open(path, "r", encoding = "utf-8") as f:
records.append(json.load(f))
df = pd.DataFrame(records)
csv_path = os.path.join(tmp_dir, "admin_audio_annotations.csv")
df.to_csv(csv_path, index = False)
return csv_path
|