split.py

import os
import torch

def split_tensor_file(input_file_path: str, num_splits: int, output_dir: str = "./splits") -> list:
    os.makedirs(output_dir, exist_ok=True)
    
    # Load the original tensor
    print(f"Loading {input_file_path}...")
    tensor = torch.load(input_file_path)

    B, C, H, W = tensor.shape
    if num_splits > B:
        raise ValueError(f"Cannot split into {num_splits} files; batch size is only {B}.")
        
    print(f"Original tensor shape: {tensor.shape}")
    
    # Calculate chunk sizes
    base_chunk_size = B // num_splits
    remainder = B % num_splits
    
    split_sizes = [base_chunk_size + (1 if i < remainder else 0) for i in range(num_splits)]
    
    # Split the tensor along the batch dimension
    tensor_splits = torch.split(tensor, split_sizes, dim=0)
    
    saved_files = []
    base_name = os.path.splitext(os.path.basename(input_file_path))[0]
    
    for i, split_tensor in enumerate(tensor_splits):
        output_filename = f"{base_name}_split_{i+1:02d}.pt"
        output_path = os.path.join(output_dir, output_filename)
        
        cloned_split = split_tensor.clone()
        
        torch.save(cloned_split, output_path)
        saved_files.append(output_path)
        print(f"Saved: {output_path} (Shape: {list(cloned_split.shape)})")
        
    print(f"\nSuccessfully split tensor into {num_splits} files.")
    return saved_files


if __name__ == "__main__":
    split_files = split_tensor_file(r"C:\Users\Mahe\Development\GitHub\RMGD\data\cached_tensors\taesd_encoded_dataset\fully_encoded_celebahq_dataset.pt", num_splits=3, output_dir="./my_splits")