| """Offline adapters for the selected native records.
|
|
|
| Media bindings are prepared separately with real codecs and audited rights.
|
| An adapter never downloads an image URL or assumes a dataset card licenses it.
|
| All reversals of a pair keep the same source group and media assets.
|
| """
|
| from __future__ import annotations
|
|
|
| import json
|
| import ast
|
| from .curated import digest, validate_calls
|
|
|
|
|
| def _json(x):
|
| return json.loads(x) if isinstance(x, str) else x
|
|
|
|
|
| def normalize(dataset: str, row: dict, provenance: dict, binding: dict | None = None) -> list[dict]:
|
| """`provenance` supplies id/revision/license/attribution/source_url/split.
|
|
|
| binding: {packet: encoded modality packet, assets: [...rights/checksums...],
|
| group: source object/video/speaker/uploader ID, caption: aligned caption}
|
| For media, `group` is mandatory: no clip-level or view-level random split.
|
| """
|
| def record(task, ins, outs, group):
|
| return {**provenance, "dataset": dataset, "task": task, "group": group, "inputs": ins, "outputs": outs,
|
| **({"assets": binding["assets"], "codec": binding.get("codec", {})} if binding else {})}
|
| text = lambda s: {"type": "text", "text": str(s).strip()}
|
| if dataset == "mbpp":
|
| source_split = provenance.get("source_split")
|
| expected = {"train": "train", "validation": "val", "test": "test"}
|
| if source_split not in expected or provenance.get("split") != expected[source_split]:
|
| raise ValueError("MBPP official train/validation/test provenance must be preserved")
|
| code = row["code"].strip()
|
| ast.parse(code)
|
| examples = "\n".join(row.get("test_list", []))
|
| prompt = row["text"].strip() + ("\n\nExamples:\n" + examples if examples else "")
|
| return [record("coding", [text(prompt)], [text("```python\n" + code + "\n```")], str(row["task_id"]))]
|
| if dataset in ("builtin", "user"):
|
| if not row.get("group"): raise ValueError("local records require a stable source group")
|
| return [record(row["task"], row["inputs"], row["outputs"], row["group"])]
|
| if dataset == "fineweb_edu":
|
| return [record("text", [], [text(row["text"])], row.get("url") or digest(row["text"]))]
|
| if dataset == "smoltalk":
|
| messages = row["messages"]
|
| if not messages or messages[-1]["role"] != "assistant":
|
| raise ValueError("SmolTalk row must end with a complete assistant response")
|
| context = "\n".join(f"{m['role']}: {m['content']}" for m in messages[:-1])
|
| if not context.strip(): raise ValueError("conversation has no conditioning")
|
| return [record("chat_text", [text(context)], [text(messages[-1]["content"])], digest(messages))]
|
| if dataset == "xlam":
|
| tools = _json(row["tools"])
|
| calls = _json(row["answers"])
|
| validate_calls(tools, calls)
|
|
|
| group = digest(sorted(tools, key=lambda x: x.get("function", x)["name"]))
|
| return [record("tools", [{"type": "tools", "value": tools}, text(row["query"])], [{"type": "calls", "value": calls}], group)]
|
| if binding is None or not binding.get("group") or not binding.get("assets"):
|
| raise ValueError("media adapter requires an encoded packet, group ID and audited asset binding")
|
| packet, group = binding["packet"], binding["group"]
|
| if dataset == "openimages_narratives":
|
| if not str(row["dataset_id"]).startswith("open_images"):
|
| raise ValueError("only the Open Images subset is selected; COCO/Flickr/ADE rights differ")
|
| cap, a, b, kind = row["caption"], "t2i", "i2t", "image"
|
| elif dataset == "finevideo":
|
|
|
| if not binding.get("caption_aligned") or not binding.get("window_seconds"):
|
| raise ValueError("FineVideo needs scene-aligned window caption and timestamps")
|
| cap, a, b, kind = binding["caption"], "t2v", "v2t", "video"
|
| elif dataset == "libritts_r":
|
| if not binding.get("complete_utterance"):
|
| raise ValueError("speech pair must retain the complete aligned utterance")
|
| cap, a, b, kind = row["text_normalized"], "t2a", "a2t", "audio"
|
| elif dataset == "fsd50k":
|
|
|
| cap, a, b, kind = "sound of " + ", ".join(row["labels"]), "t2a", "a2t", "audio"
|
| elif dataset == "objaverse_cap3d":
|
| cap, a, b, kind = row["caption"], "t2m", "m2t", "mesh"
|
| else:
|
| raise ValueError(f"no enabled adapter for {dataset}")
|
| if packet["type"] != kind or not str(cap).strip(): raise ValueError("packet kind or caption mismatch")
|
| out = [record(a, [text(cap)], [packet], group), record(b, [packet], [text(cap)], group)]
|
| if dataset == "finevideo" and len(packet["codes"]) >= 4:
|
| mid = len(packet["codes"]) // 2
|
| out.append(record("vcont", [{"type":"video", "codes":packet["codes"][:mid]}],
|
| [{"type":"video", "codes":packet["codes"][mid:]}], group))
|
| return out
|
|
|
