import os
import io
import glob
import html
import math
import json
import ast
import random
from typing import Any
import re
import numpy as np
import regex
from PIL import Image
import decord
import datasets
from datasets.distributed import split_dataset_by_node
from einops import rearrange
import torch
from torchdata.stateful_dataloader import StatefulDataLoader
from torch.utils.data import IterableDataset
from diffusers.utils import is_ftfy_available
from src.dataset.bucket_util import BucketGroup, reference_size, resize_pixels
from src.utils import get_from_dict
from src.utils.logging import get_logger
if is_ftfy_available():
import ftfy
datasets.disable_caching()
def prompt_clean(text: str) -> str:
if is_ftfy_available():
text = ftfy.fix_text(text)
text = html.unescape(html.unescape(text)).strip()
return regex.sub(r"\s+", " ", text).strip()
def parse_json_field(json_data: Any) -> Any:
"""Decode JSON metadata supplied as a mapping, string, or bytes."""
if not isinstance(json_data, (str, bytes, bytearray)):
return json_data
try:
return json.loads(json_data)
except (json.JSONDecodeError, UnicodeDecodeError):
text = json_data.decode("utf-8", "replace") if isinstance(
json_data, (bytes, bytearray)) else json_data
return ast.literal_eval(text)
def conversation_to_prompt(
conversation: list[dict[str, Any]],
drop_last_assistant: bool = False,
) -> str:
"""Convert conversation dict to prompt string."""
prompt = ""
if drop_last_assistant and len(conversation) > 0:
message = conversation[-1]
if message.get('role', '') == 'assistant' or message.get('from', '') == 'gpt':
conversation = conversation[:-1]
for message in conversation:
if 'role' in message:
role = message["role"]
content = message["content"]
if role == "system":
prompt += f"<|im_start|>system\n{content}<|im_end|>\n"
elif role == "user":
prompt += f"<|im_start|>user\n{content}<|im_end|>\n"
elif role == "assistant":
prompt += f"<|im_start|>assistant\n{content}<|im_end|>\n"
elif 'from' in message:
role = message["from"]
content = message["value"]
if role == "system":
prompt += f"<|im_start|>system\n{content}<|im_end|>\n"
elif role == "human":
prompt += f"<|im_start|>user\n{content}<|im_end|>\n"
elif role == "gpt":
prompt += f"<|im_start|>assistant\n{content}<|im_end|>\n"
return prompt
def to_rgb_on_white(image: Image.Image) -> Image.Image:
"""Convert to RGB, compositing transparency over white."""
has_alpha = (
image.mode in ("RGBA", "LA")
or (image.mode == "P" and "transparency" in image.info)
)
if not has_alpha:
return image.convert("RGB")
rgba = image.convert("RGBA")
background = Image.new("RGBA", rgba.size, (255, 255, 255, 255))
return Image.alpha_composite(background, rgba).convert("RGB")
def resize_crop_normalize(x: torch.Tensor, target_size: tuple[int, int]) -> torch.Tensor:
return resize_pixels(x, *target_size) / 127.5 - 1.0
class ImageVideoWebDataset(IterableDataset):
def __init__(
self,
image_data_files: str | list[str] | None,
multiple_images_data_files: str | list[str] | None,
multiple_videos_data_files: str | list[str] | None,
video_data_files: str | list[str] | None,
bucket_configs: list[tuple[int, int, int, int, int]] | None = None,
image_caption_keys: list[str] | None = None,
image_caption_sampling_prob: list[float] | None = None,
multiple_images_caption_keys: list[str] | None = None,
multiple_images_caption_sampling_prob: list[float] | None = None,
multiple_videos_caption_keys: list[str] | None = None,
multiple_videos_caption_sampling_prob: list[float] | None = None,
video_caption_keys: list[str] | None = None,
video_caption_sampling_prob: list[float] | None = None,
image_sampling_prob: float = 0.3,
multiple_images_sampling_prob: float = 0.3,
multiple_videos_sampling_prob: float = 0.0,
video_sampling_prob: float = 0.4,
ensure_divisible_shards: bool = True,
shuffle: bool = True,
fps: int = -1,
seed: int = 42,
tar_files_shuffle_seed: int | None = None,
buffer_size: int = 1000,
rec_aug_rate: float = 0.0,
):
task_configs = (
("image", image_data_files, image_sampling_prob, image_caption_keys, image_caption_sampling_prob),
("multiple_images", multiple_images_data_files, multiple_images_sampling_prob, multiple_images_caption_keys, multiple_images_caption_sampling_prob),
("multiple_videos", multiple_videos_data_files, multiple_videos_sampling_prob, multiple_videos_caption_keys, multiple_videos_caption_sampling_prob),
("video", video_data_files, video_sampling_prob, video_caption_keys, video_caption_sampling_prob),
)
for mode, files, weight, caption_keys, caption_weights in task_configs:
if not math.isfinite(weight) or weight < 0:
raise ValueError(f"{mode} sampling weight must be finite and non-negative.")
if weight == 0:
continue
if not files:
raise ValueError(f"{mode} data files are required when its sampling weight is positive.")
if not caption_keys or not caption_weights or len(caption_keys) != len(caption_weights):
raise ValueError(f"{mode} caption keys and weights must be non-empty and have matching lengths.")
if any(not math.isfinite(value) or value < 0 for value in caption_weights) or sum(caption_weights) <= 0:
raise ValueError(f"{mode} caption weights must be finite, non-negative, and have a positive sum.")
if sum(config[2] for config in task_configs) <= 0:
raise ValueError("At least one task sampling weight must be positive.")
self.image_caption_keys = image_caption_keys
self.image_caption_sampling_prob = image_caption_sampling_prob
self.multiple_images_caption_keys = multiple_images_caption_keys
self.multiple_images_caption_sampling_prob = multiple_images_caption_sampling_prob
self.multiple_videos_caption_keys = multiple_videos_caption_keys
self.multiple_videos_caption_sampling_prob = multiple_videos_caption_sampling_prob
self.video_caption_keys = video_caption_keys
self.video_caption_sampling_prob = video_caption_sampling_prob
self.image_sampling_prob = image_sampling_prob
self.multiple_images_sampling_prob = multiple_images_sampling_prob
self.multiple_videos_sampling_prob = multiple_videos_sampling_prob
self.video_sampling_prob = video_sampling_prob
self.ensure_divisible_shards = ensure_divisible_shards
self.shuffle = shuffle
self.seed = seed
self.tar_files_shuffle_seed = tar_files_shuffle_seed
self.buffer_size = buffer_size
self.fps = fps
self.rec_aug_rate = rec_aug_rate
_, rank = self._get_distributed_info()
self.rng = random.Random(seed + rank)
# Share task choices across ranks; advance only after yielding.
self.task_rng = random.Random(seed)
self.bucket_group = BucketGroup(bucket_configs)
self.current_buckets = {b: []
for b in self.bucket_group.bucket_configs}
self._image_dataset, self._image_iterator = None, None
if image_sampling_prob > 0:
self._image_dataset, self._image_iterator = self._init_dataset(
image_data_files)
self._multiple_images_dataset, self._multiple_images_iterator = None, None
if multiple_images_sampling_prob > 0:
# Declare both image extensions so shard schemas retain their images.
multiple_images_features = datasets.Features({
"json": datasets.Value("string"),
"__key__": datasets.Value("string"),
"__url__": datasets.Value("string"),
"0.jpg": datasets.Image(),
"1.jpg": datasets.Image(),
"2.jpg": datasets.Image(),
"0.png": datasets.Image(),
"1.png": datasets.Image(),
"2.png": datasets.Image(),
})
self._multiple_images_dataset, self._multiple_images_iterator = self._init_dataset(
multiple_images_data_files, features=multiple_images_features)
self._multiple_videos_dataset, self._multiple_videos_iterator = None, None
if multiple_videos_sampling_prob > 0:
multiple_videos_features = datasets.Features({
"json": datasets.Value("string"),
"__key__": datasets.Value("string"),
"__url__": datasets.Value("string"),
"0.mp4": datasets.Video(decode=False),
"1.mp4": datasets.Video(decode=False),
"0.jpg": datasets.Image(),
"0.png": datasets.Image(),
})
self._multiple_videos_dataset, self._multiple_videos_iterator = self._init_dataset(
multiple_videos_data_files, features=multiple_videos_features)
self._video_dataset, self._video_iterator = None, None
if video_sampling_prob > 0:
self._video_dataset, self._video_iterator = self._init_dataset(
video_data_files)
def _init_dataset(self, data_files: str | list[str], features=None):
world_size, rank = self._get_distributed_info()
data_files = [data_files] if isinstance(data_files, str) else list(data_files)
flags = [f.lower().endswith(".json") for f in data_files]
if all(flags):
all_json = True
elif not any(flags):
all_json = False
else:
raise ValueError(
"All files must be JSON or none of them should be JSON")
if all_json:
output_data_files = []
for data_file in data_files:
with open(data_file, "r") as f:
data_file = json.load(f)["resolved_files"]
output_data_files.extend(data_file)
data_files = output_data_files
if self.tar_files_shuffle_seed is not None:
tar_rng = random.Random(self.tar_files_shuffle_seed)
tar_rng.shuffle(data_files)
else:
data_files = [
path for pattern in data_files
for path in sorted(glob.glob(pattern, recursive="**" in pattern))
]
if not data_files:
raise FileNotFoundError("No training shards found.")
if self.ensure_divisible_shards:
data_files = data_files[:len(data_files) // world_size * world_size]
if not data_files:
raise ValueError(f"At least {world_size} shards are required for equal distribution.")
dataset = datasets.load_dataset(
"webdataset", data_files=data_files, streaming=True, split="train",
features=features)
dataset = split_dataset_by_node(
dataset, rank=rank, world_size=world_size)
if self.shuffle:
seed = self.seed + dataset.epoch
dataset = dataset.shuffle(seed=seed, buffer_size=self.buffer_size)
iterator = iter(dataset)
return dataset, iterator
def _get_distributed_info(self):
return int(os.getenv("WORLD_SIZE", "1")), int(os.getenv("RANK", "0"))
def _open_video_reader(self, video: Any):
decord.bridge.set_bridge("torch")
if isinstance(video, dict):
path = video.get("path")
video_bytes = video.get("bytes")
if video_bytes is not None:
video = video_bytes
elif path is not None:
return decord.VideoReader(path)
else:
raise ValueError("Invalid video dict: both 'path' and 'bytes' are None.")
if isinstance(video, (bytes, bytearray)):
return decord.VideoReader(io.BytesIO(video))
raise TypeError(f"Unsupported video type: {type(video)}")
def _process_video(self, video: Any):
video_reader = self._open_video_reader(video)
n_frame = len(video_reader)
if n_frame <= 0:
raise ValueError("Empty video encountered.")
height, width = video_reader[0].shape[:2]
fps_ratio = self._get_video_fps_ratio(video_reader)
bucket = self.bucket_group.find_best_bucket((1, int(n_frame // fps_ratio), height, width))
pixel = self._sample_video_with_bucket(video_reader, bucket)
pixel = pixel.unsqueeze(0)
return pixel, bucket
def _get_video_fps_ratio(self, video_reader: Any) -> float:
cur_fps = video_reader.get_avg_fps()
if self.fps > 0 and cur_fps > self.fps:
return cur_fps / self.fps
return 1.0
def _sample_video_with_bucket(
self,
video_reader: Any,
bucket: tuple[int, int, int, int, int],
start_ratio: float | None = None,
frame_rand: np.ndarray | None = None,
) -> torch.Tensor:
n_frame = len(video_reader)
if n_frame <= 0:
raise ValueError("Empty video encountered.")
_, _, sampling_n_frame, target_height, target_width = bucket
fps_ratio = self._get_video_fps_ratio(video_reader)
scaled_n_frame = math.ceil(sampling_n_frame * fps_ratio)
clip_len = min(n_frame, max(scaled_n_frame, sampling_n_frame))
max_start = n_frame - clip_len
if start_ratio is None:
start_idx = self.rng.randint(0, max_start) if max_start > 0 else 0
else:
start_idx = int(round(max_start * start_ratio)) if max_start > 0 else 0
bin_edges = np.linspace(0, clip_len, num=sampling_n_frame + 1, dtype=np.int64)
bin_starts = bin_edges[:-1]
bin_ends = np.maximum(bin_edges[1:], bin_starts + 1)
if frame_rand is None:
frame_rand = np.fromiter((self.rng.random() for _ in range(sampling_n_frame)), dtype=np.float32)
sample_idx = bin_starts + ((bin_ends - bin_starts) * frame_rand).astype(np.int64)
sample_idx = np.clip(sample_idx, 0, clip_len - 1)
frame_idx = (start_idx + sample_idx).tolist()
pixel = video_reader.get_batch(frame_idx).permute(0, 3, 1, 2)
pixel = resize_crop_normalize(pixel, (target_height, target_width))
return rearrange(pixel, "t c h w -> c t h w")
def _process_multiple_videos(self, videos: list[Any]):
if len(videos) == 0:
raise ValueError("No videos provided for processing.")
video_readers = [self._open_video_reader(video) for video in videos]
target_reader = video_readers[-1]
if len(target_reader) <= 0:
raise ValueError("Target video is empty.")
target_height, target_width = target_reader[0].shape[:2]
target_fps_ratio = self._get_video_fps_ratio(target_reader)
bucket = self.bucket_group.find_best_bucket(
(len(videos), int(len(target_reader) // target_fps_ratio), target_height, target_width)
)
start_ratio = self.rng.random()
_, _, sampling_n_frame, _, _ = bucket
frame_rand = np.fromiter((self.rng.random() for _ in range(sampling_n_frame)), dtype=np.float32)
pixel = [
self._sample_video_with_bucket(
video_reader,
bucket,
start_ratio=start_ratio,
frame_rand=frame_rand,
)
for video_reader in video_readers
]
pixel = torch.stack(pixel, dim=0)
return pixel, bucket
def _process_ref_image(self, image: Image.Image):
pixel = torch.from_numpy(np.array(to_rgb_on_white(image)))
h, w = pixel.shape[:2]
target_height, target_width = reference_size(h, w)
pixel = rearrange(pixel, "h w c -> 1 c h w")
pixel = resize_crop_normalize(pixel, (target_height, target_width))
return rearrange(pixel, "1 c h w -> c 1 h w")
def _detect_multi_image_ext(self, item) -> str | None:
"""Find the image extension used by a multi-image sample."""
for ext in ("jpg", "png"):
if item.get(f"0.{ext}") is not None and item.get(f"1.{ext}") is not None:
return ext
return None
def _collect_sequential_media(self, item, suffix: str):
media = []
index = 0
# Explicit schemas retain unused extension keys as None.
while item.get(f"{index}.{suffix}") is not None:
media.append(item[f"{index}.{suffix}"])
index += 1
return media
def _process_multiple_images(self, images: list[Image.Image], use_rec_aug: bool):
if len(images) == 0:
raise ValueError("No images provided for processing.")
bucket_source = images[0] if use_rec_aug else images[-1]
w, h = bucket_source.size
num_items = 2 if use_rec_aug else len(images)
bucket = self.bucket_group.find_best_bucket((num_items, 1, h, w))
_, _, _, target_height, target_width = bucket
pixel = []
for img in images:
img = torch.from_numpy(np.array(to_rgb_on_white(img)))
img = rearrange(img, "h w c -> 1 c h w")
img = resize_crop_normalize(img, (target_height, target_width))
img = rearrange(img, "1 c h w -> c 1 h w")
pixel.append(img)
if use_rec_aug:
recon_image = pixel[0]
pixel = [recon_image, recon_image]
pixel = torch.stack(pixel, dim=0)
return pixel, bucket
def _process_image(self, image: Image.Image):
pixel = torch.from_numpy(np.array(to_rgb_on_white(image)))
h, w = pixel.shape[:2]
pixel = rearrange(pixel, "h w c -> 1 c h w")
bucket = self.bucket_group.find_best_bucket((1, 1, h, w))
_, _, _, target_height, target_width = bucket
pixel = resize_crop_normalize(pixel, (target_height, target_width))
pixel = rearrange(pixel, "1 c h w -> c 1 h w")
pixel = pixel.unsqueeze(0)
return pixel, bucket
def _get_caption(self, json_data, mode: str):
logger = get_logger()
available_captions = []
weights = []
caption_keys = getattr(self, f"{mode}_caption_keys")
caption_sampling_prob = getattr(self, f"{mode}_caption_sampling_prob")
for key, prob in zip(caption_keys, caption_sampling_prob):
if prob == 0:
continue
try:
value = get_from_dict(json_data, key)
except (KeyError, TypeError):
continue
if value is None:
continue
try:
if isinstance(value, list):
caption = conversation_to_prompt(
value, drop_last_assistant=True)
else:
caption = prompt_clean(str(value))
except Exception:
logger.warning("Skipped an invalid caption.")
continue
if not caption.strip():
continue
available_captions.append(caption)
weights.append(prob)
if not available_captions:
msg = "No valid captions found."
logger.warning(msg)
raise ValueError(msg)
caption = self.rng.choices(available_captions, weights=weights, k=1)[0]
return caption
def _process_item(self, item):
json_data = item.get("json")
if json_data is None:
raise ValueError("Missing 'json' field in item")
try:
json_data = parse_json_field(json_data)
except (json.JSONDecodeError, UnicodeDecodeError, ValueError, SyntaxError) as error:
raise ValueError("Invalid JSON format in item['json']") from error
image_ext = self._detect_multi_image_ext(item)
single_image = next((item[ext] for ext in ("jpg", "png") if item.get(ext) is not None), None)
if item.get("0.mp4") is not None:
mode = "multiple_videos"
elif image_ext is not None:
mode = "multiple_images"
elif single_image is not None:
mode = "image"
elif item.get("mp4") is not None:
mode = "video"
else:
raise ValueError(f"Invalid item: {item.keys()}")
caption = self._get_caption(json_data, mode)
ref_image_pixel = None
if mode == "multiple_videos":
pixel, bucket_id = self._process_multiple_videos(
self._collect_sequential_media(item, "mp4")
)
ref_image = item.get("0.jpg")
if ref_image is None:
ref_image = item.get("0.png")
caption_only = isinstance(json_data, dict) and set(json_data) == {"caption"}
metadata = json_data.get("metadata") if isinstance(json_data, dict) else None
if not caption_only and not (isinstance(metadata, dict) and metadata.get("ref_image", None) is not None):
ref_image = None
if ref_image is not None:
ref_image_pixel = self._process_ref_image(ref_image)
elif mode == "multiple_images":
images = self._collect_sequential_media(item, image_ext)
# Three-item samples contain reference, source, and target images.
if len(images) == 3:
ref_image_pixel = self._process_ref_image(images[0])
pixel, bucket_id = self._process_multiple_images(
images[1:], use_rec_aug=False)
else:
if self.rng.random() < self.rec_aug_rate:
use_rec_aug = True
def replace_caption_text(caption: np.str_, new_text: str):
caption_str = str(caption)
replaced = re.sub(
r"(\n)(.*?)(<\|im_end\|>)",
rf"\1{new_text}\3",
caption_str,
flags=re.DOTALL
)
return np.str_(replaced)
caption = replace_caption_text(
caption=caption, new_text="Reconstruct this image.")
else:
use_rec_aug = False
pixel, bucket_id = self._process_multiple_images(images, use_rec_aug)
elif mode == "image":
pixel, bucket_id = self._process_image(single_image)
else:
pixel, bucket_id = self._process_video(item["mp4"])
return pixel, caption, bucket_id, ref_image_pixel
def _sample_task_mode(self):
"""Sample a task mode using the cross-rank synchronized RNG."""
return self.task_rng.choices(
['image', 'multiple_images', 'multiple_videos', 'video'],
weights=[
self.image_sampling_prob,
self.multiple_images_sampling_prob,
self.multiple_videos_sampling_prob,
self.video_sampling_prob],
k=1
)[0]
def _get_next_item(self, mode: str):
dataset = getattr(self, f"_{mode}_dataset")
iterator = getattr(self, f"_{mode}_iterator")
item = next(iterator, None)
if item is None:
dataset.set_epoch(dataset.epoch + 1)
new_iterator = iter(dataset)
setattr(self, f"_{mode}_iterator", new_iterator)
item = next(new_iterator)
return item
def __iter__(self):
logger = get_logger()
consecutive_read_failures = 0
consecutive_process_failures = 0
current_mode = self._sample_task_mode()
while True:
try:
item = self._get_next_item(current_mode)
consecutive_read_failures = 0
except Exception as error:
logger.warning(f"Failed to read a {current_mode} sample: {error}")
consecutive_read_failures += 1
if consecutive_read_failures >= 50:
logger.error(
"Exceeded dataset read failure limit. "
f"consecutive_read_failures={consecutive_read_failures}"
)
raise RuntimeError(
f"Dataset reader entered a failure loop for {consecutive_read_failures} consecutive attempts."
)
continue
url = item.get("__url__", "N/A")
key = item.get("__key__", "N/A")
try:
pixel, caption, bucket_id, ref_image_pixel = self._process_item(item)
consecutive_process_failures = 0
except Exception as e:
consecutive_process_failures += 1
logger.warning(f"Skipped sample {key}: {e}")
if consecutive_process_failures >= 50:
logger.error(
"Exceeded item processing failure limit. "
f"consecutive_process_failures={consecutive_process_failures}"
)
raise RuntimeError(
"Dataset processor entered a failure loop for "
f"{consecutive_process_failures} consecutive items."
) from e
continue
bucket = self.current_buckets[bucket_id]
bucket.append({"pixel": pixel, "caption": caption, "ref_image_pixel": ref_image_pixel,
"key": key, "url": url, "bucket_id": bucket_id})
if len(bucket) == bucket_id[0]:
batch = bucket[:]
bucket.clear()
yield batch
current_mode = self._sample_task_mode()
def state_dict(self):
return {
"image": self._image_dataset.state_dict() if self._image_dataset is not None else None,
"multiple_images": self._multiple_images_dataset.state_dict() if self._multiple_images_dataset is not None else None,
"multiple_videos": self._multiple_videos_dataset.state_dict() if self._multiple_videos_dataset is not None else None,
"video": self._video_dataset.state_dict() if self._video_dataset is not None else None,
"rng_state": self.rng.getstate(),
"task_rng_state": self.task_rng.getstate(),
"current_buckets": {bucket: items[:] for bucket, items in self.current_buckets.items()},
}
def load_state_dict(self, state_dict):
for mode in ("image", "multiple_images", "multiple_videos", "video"):
dataset = getattr(self, f"_{mode}_dataset")
if dataset is not None and state_dict[mode] is not None:
dataset.load_state_dict(state_dict[mode])
setattr(self, f"_{mode}_iterator", iter(dataset))
if "rng_state" in state_dict and state_dict["rng_state"] is not None:
self.rng.setstate(state_dict["rng_state"])
if "task_rng_state" in state_dict and state_dict["task_rng_state"] is not None:
self.task_rng.setstate(state_dict["task_rng_state"])
if "current_buckets" in state_dict and state_dict["current_buckets"] is not None:
saved_buckets = state_dict["current_buckets"]
if saved_buckets.keys() != self.current_buckets.keys():
raise ValueError("Saved dataloader buckets do not match the configured buckets.")
# A full bucket in an older snapshot has already been delivered.
self.current_buckets = {
bucket: items[:] if len(items) < bucket[0] else []
for bucket, items in saved_buckets.items()
}
def collate_batch(batch):
pixels = torch.stack([item["pixel"] for item in batch])
captions = [item["caption"] for item in batch]
ref_image_pixels = [item.get("ref_image_pixel") for item in batch]
keys = [item["key"] for item in batch]
urls = [item["url"] for item in batch]
bucket_ids = [item["bucket_id"] for item in batch]
return {
"pixel": pixels,
"caption": captions,
"ref_image_pixel": ref_image_pixels,
"key": keys,
"url": urls,
"bucket_id": bucket_ids,
}
def build_webdataset_dataloader(
image_data_files: str | list[str] | None = None,
multiple_images_data_files: str | list[str] | None = None,
multiple_videos_data_files: str | list[str] | None = None,
video_data_files: str | list[str] | None = None,
bucket_configs: list[tuple[int, int, int, int, int]] | None = None,
image_caption_keys: list[str] | None = None,
image_caption_sampling_prob: list[float] | None = None,
multiple_images_caption_keys: list[str] | None = None,
multiple_images_caption_sampling_prob: list[float] | None = None,
multiple_videos_caption_keys: list[str] | None = None,
multiple_videos_caption_sampling_prob: list[float] | None = None,
video_caption_keys: list[str] | None = None,
video_caption_sampling_prob: list[float] | None = None,
image_sampling_prob: float = 0.3,
multiple_images_sampling_prob: float = 0.3,
multiple_videos_sampling_prob: float = 0.0,
video_sampling_prob: float = 0.4,
shuffle: bool = True,
ensure_divisible_shards: bool = True,
num_workers: int = 1,
fps: int = -1,
seed: int = 42,
tar_files_shuffle_seed: int | None = None,
buffer_size: int = 1000,
rec_aug_rate: float = 0.0,
):
dataset = ImageVideoWebDataset(
image_data_files=image_data_files,
multiple_images_data_files=multiple_images_data_files,
multiple_videos_data_files=multiple_videos_data_files,
video_data_files=video_data_files,
bucket_configs=bucket_configs,
image_caption_keys=image_caption_keys,
image_caption_sampling_prob=image_caption_sampling_prob,
multiple_images_caption_keys=multiple_images_caption_keys,
multiple_images_caption_sampling_prob=multiple_images_caption_sampling_prob,
multiple_videos_caption_keys=multiple_videos_caption_keys,
multiple_videos_caption_sampling_prob=multiple_videos_caption_sampling_prob,
video_caption_keys=video_caption_keys,
video_caption_sampling_prob=video_caption_sampling_prob,
image_sampling_prob=image_sampling_prob,
multiple_images_sampling_prob=multiple_images_sampling_prob,
multiple_videos_sampling_prob=multiple_videos_sampling_prob,
video_sampling_prob=video_sampling_prob,
ensure_divisible_shards=ensure_divisible_shards,
shuffle=shuffle,
fps=fps,
seed=seed,
tar_files_shuffle_seed=tar_files_shuffle_seed,
buffer_size=buffer_size,
rec_aug_rate=rec_aug_rate,
)
return StatefulDataLoader(dataset, batch_size=None, num_workers=num_workers, collate_fn=collate_batch)