Spaces:
Running on Zero
Running on Zero
File size: 8,977 Bytes
1aa20d3 d368214 1aa20d3 d368214 1aa20d3 d368214 1aa20d3 d368214 1aa20d3 d368214 1aa20d3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 | from __future__ import annotations
import hashlib
import hmac
import json
import os
import re
import unicodedata
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Any
from huggingface_hub import CommitOperationDelete, HfApi, hf_hub_download
MAX_DELETE_OPERATIONS = 10
def _normalize_name(value: str) -> str:
normalized = unicodedata.normalize("NFKC", value or "")
return re.sub(r"\s+", " ", normalized).strip().casefold()
def _anonymous_label(name: str, report_key: str) -> str:
return hmac.new(
report_key.encode("utf-8"),
_normalize_name(name).encode("utf-8"),
hashlib.sha256,
).hexdigest()[:12]
def _read_repo_json(
repo_id: str,
path: str,
token: str,
) -> tuple[str, dict[str, Any]]:
local_path = hf_hub_download(
repo_id=repo_id,
filename=path,
repo_type="dataset",
token=token,
)
return path, json.loads(Path(local_path).read_text(encoding="utf-8"))
def run_data_cleanup(
request_key: str,
*,
repo_id: str,
token: str,
) -> str:
"""Apply one secret-scoped maintenance request to the private Dataset.
The response contains anonymous labels only. The temporary request secret
is installed and removed by the GitHub maintenance workflow.
"""
raw_payload = os.environ.get("DATA_CLEANUP_PAYLOAD", "")
if not raw_payload or not repo_id or not token:
return json.dumps({"ok": False, "error": "maintenance_disabled"})
payload = json.loads(raw_payload)
expected_key = str(payload.get("request_key", ""))
if not expected_key or not hmac.compare_digest(request_key or "", expected_key):
return json.dumps({"ok": False, "error": "not_authorized"})
if payload.get("version") != 1 or not payload.get("report_key"):
return json.dumps({"ok": False, "error": "invalid_payload"})
report_key = str(payload["report_key"])
apply_changes = payload.get("apply") is True
resume_changes = apply_changes and payload.get("resume") is True
full_names = [str(value) for value in payload.get("full_profiles", [])]
questionnaire_names = [
str(value) for value in payload.get("questionnaire_only", [])
]
normalized_full = {_normalize_name(value): value for value in full_names}
normalized_questionnaire = {
_normalize_name(value): value for value in questionnaire_names
}
if (
len(normalized_full) != len(full_names)
or len(normalized_questionnaire) != len(questionnaire_names)
or normalized_full.keys() & normalized_questionnaire.keys()
):
return json.dumps({"ok": False, "error": "invalid_targets"})
api = HfApi(token=token)
repo_info = api.repo_info(repo_id=repo_id, repo_type="dataset")
if getattr(repo_info, "private", None) is not True:
return json.dumps({"ok": False, "error": "dataset_not_private"})
repo_files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
participant_paths = sorted(
path
for path in repo_files
if path.startswith("participants/") and path.endswith(".json")
)
event_paths = sorted(
path
for path in repo_files
if path.startswith("events/") and path.endswith(".json")
)
with ThreadPoolExecutor(max_workers=min(16, max(1, len(participant_paths)))) as pool:
participant_pairs = list(
pool.map(
lambda path: _read_repo_json(repo_id, path, token),
participant_paths,
)
)
participants_by_name: dict[str, list[tuple[str, dict[str, Any]]]] = defaultdict(list)
for path, participant in participant_pairs:
normalized = _normalize_name(str(participant.get("display_name", "")))
participants_by_name[normalized].append((path, participant))
target_names = list(normalized_full) + list(normalized_questionnaire)
target_report = []
for normalized in target_names:
source_name = normalized_full.get(normalized) or normalized_questionnaire[normalized]
target_report.append(
{
"label": _anonymous_label(source_name, report_key),
"participant_records": len(participants_by_name.get(normalized, [])),
}
)
ambiguous = []
for normalized in normalized_full:
count = len(participants_by_name.get(normalized, []))
if count > 1 or (count == 0 and not resume_changes):
ambiguous.append(normalized)
for normalized in normalized_questionnaire:
if len(participants_by_name.get(normalized, [])) != 1:
ambiguous.append(normalized)
if ambiguous:
return json.dumps(
{
"ok": False,
"error": "ambiguous_or_missing",
"mode": "apply" if apply_changes else "dry_run",
"targets": target_report,
},
sort_keys=True,
)
full_ids = {
str(records[0][1]["participant_id"])
for name in normalized_full
if (records := participants_by_name.get(name))
}
questionnaire_ids = {
str(participants_by_name[name][0][1]["participant_id"])
for name in normalized_questionnaire
}
all_target_ids = full_ids | questionnaire_ids
full_event_paths: dict[str, list[str]] = defaultdict(list)
questionnaire_event_paths: dict[str, list[str]] = defaultdict(list)
def inspect_event(path: str) -> tuple[str, str, str]:
event_path, event = _read_repo_json(repo_id, path, token)
return (
event_path,
str(event.get("participant_id_hash", "")),
str(event.get("event_type", "")),
)
with ThreadPoolExecutor(max_workers=min(20, max(1, len(event_paths)))) as pool:
for path, participant_id, event_type in pool.map(inspect_event, event_paths):
if participant_id not in all_target_ids:
continue
if participant_id in full_ids:
full_event_paths[participant_id].append(path)
elif event_type == "questionnaire_submitted":
questionnaire_event_paths[participant_id].append(path)
delete_paths: set[str] = set()
details = []
for normalized, source_name in normalized_full.items():
participant_records = participants_by_name.get(normalized, [])
if not participant_records:
details.append(
{
"label": _anonymous_label(source_name, report_key),
"operation": "full",
"events": 0,
"participant_files": 0,
}
)
continue
participant_path, participant = participant_records[0]
participant_id = str(participant["participant_id"])
paths = full_event_paths[participant_id]
delete_paths.add(participant_path)
delete_paths.update(paths)
details.append(
{
"label": _anonymous_label(source_name, report_key),
"operation": "full",
"events": len(paths),
"participant_files": 1,
}
)
for normalized, source_name in normalized_questionnaire.items():
_, participant = participants_by_name[normalized][0]
participant_id = str(participant["participant_id"])
paths = questionnaire_event_paths[participant_id]
delete_paths.update(paths)
details.append(
{
"label": _anonymous_label(source_name, report_key),
"operation": "questionnaire",
"events": len(paths),
"participant_files": 0,
}
)
result = {
"ok": True,
"mode": "apply" if apply_changes else "dry_run",
"targets": target_report,
"details": details,
"total_delete_paths": len(delete_paths),
}
if not apply_changes:
return json.dumps(result, sort_keys=True)
paths = sorted(delete_paths)[:MAX_DELETE_OPERATIONS]
if paths:
api.create_commit(
repo_id=repo_id,
repo_type="dataset",
operations=[CommitOperationDelete(path_in_repo=path) for path in paths],
commit_message="Rimuove un gruppo di dati di pre-test selezionati",
)
remaining_files = set(api.list_repo_files(repo_id=repo_id, repo_type="dataset"))
result["remaining_delete_paths"] = len(delete_paths & remaining_files)
result["deleted_in_batch"] = len(paths)
result["complete"] = result["remaining_delete_paths"] == 0
return json.dumps(result, sort_keys=True)
|