{ "cells": [ { "cell_type": "markdown", "source": [ "# PRE-ENCODE" ], "metadata": { "id": "-tW2VB9njCOX" }, "id": "-tW2VB9njCOX" }, { "cell_type": "code", "source": [ "# ============================================================\n", "# GEMMA 4 E2B — DOWNLOAD, DECRYPT, PRE-ENCODE + SINGLE SAFETENSORS\n", "# ============================================================\n", "#\n", "# SOURCE:\n", "# codeShare/kromoss-lora-exports/gemma_train_set.7z\n", "#\n", "# OUTER ARCHIVE:\n", "# Automatically detects:\n", "# - 7z\n", "# - AES-encrypted ZIP\n", "# - Standard ZIP\n", "#\n", "# Outer password:\n", "# blueberry\n", "#\n", "# INDIVIDUAL FILES:\n", "# Actual archive filenames:\n", "# N.jpg\n", "# N_input.txt\n", "# N_output.txt\n", "#\n", "# Contents are automatically detected:\n", "# - PCENC1 AES-256-GCM -> decrypted\n", "# - Otherwise -> treated as plaintext\n", "#\n", "# MODEL:\n", "# coder3101/gemma-4-E2B-it-heretic\n", "#\n", "# PROCESSING:\n", "# - Downloads and extracts the dataset.\n", "# - Matches files using their shared numeric ID.\n", "# - Loads AutoProcessor only; no model weights.\n", "# - Encodes one example at a time.\n", "# - Creates completion-only labels.\n", "# - Keeps processor tensors on CPU.\n", "# - Stores every example in ONE examples.safetensors file.\n", "# - Supports variable tensor shapes between examples.\n", "# - Does not store original images or source text in the upload.\n", "#\n", "# UPLOAD:\n", "# - examples.safetensors\n", "# - processor/ configuration\n", "# - manifest.json\n", "# - No original images or source text.\n", "# - Does not upload or replace gemma_train_set.7z.\n", "#\n", "# NOTE:\n", "# These are processor-encoded tensors, not transformer\n", "# hidden-state embeddings.\n", "#\n", "# ============================================================\n", "\n", "\n", "# ============================================================\n", "# INSTALL DEPENDENCIES\n", "# ============================================================\n", "\n", "!pip -q install \\\n", " \"transformers>=5.10.1\" \\\n", " \"huggingface_hub>=0.30.0\" \\\n", " \"pillow>=8.0,<13.0\" \\\n", " safetensors \\\n", " sentencepiece \\\n", " packaging \\\n", " pyzipper \\\n", " py7zr \\\n", " cryptography\n", "\n", "\n", "# ============================================================\n", "# IMPORTS\n", "# ============================================================\n", "\n", "import os\n", "import gc\n", "import re\n", "import json\n", "import shutil\n", "import warnings\n", "import hashlib\n", "import zipfile\n", "\n", "import torch\n", "import numpy as np\n", "import pyzipper\n", "import py7zr\n", "\n", "from pathlib import Path\n", "from io import BytesIO\n", "from PIL import Image, ImageOps\n", "\n", "from transformers import AutoProcessor\n", "\n", "from safetensors.torch import save_file, load_file\n", "\n", "from huggingface_hub import (\n", " HfApi,\n", " hf_hub_download,\n", ")\n", "\n", "from cryptography.hazmat.primitives.ciphers.aead import AESGCM\n", "from cryptography.hazmat.primitives.kdf.scrypt import Scrypt\n", "\n", "from google.colab import userdata\n", "\n", "warnings.filterwarnings(\"ignore\")\n", "\n", "\n", "# ============================================================\n", "# COLAB PARAMETERS\n", "# ============================================================\n", "\n", "#@markdown ## Model\n", "\n", "MODEL_ID = \"coder3101/gemma-4-E2B-it-heretic\" #@param {type:\"string\"}\n", "\n", "\n", "#@markdown ## Encrypted source archive\n", "\n", "SOURCE_REPO_ID = \"codeShare/kromoss-lora-exports\" #@param {type:\"string\"}\n", "\n", "SOURCE_REPO_TYPE = \"model\" #@param [\"model\", \"dataset\"]\n", "\n", "SOURCE_FILENAME = \"gemma_train_set.7z\" #@param {type:\"string\"}\n", "\n", "SOURCE_REVISION = \"main\" #@param {type:\"string\"}\n", "\n", "\n", "#@markdown ## Decryption passwords\n", "\n", "OUTER_ZIP_PASSWORD = \"blueberry\" #@param {type:\"string\"}\n", "\n", "INDIVIDUAL_FILE_PASSWORD = \"blueberry\" #@param {type:\"string\"}\n", "\n", "\n", "#@markdown ## Local source dataset\n", "\n", "DATASET_DIR = \"/content/gemma_training_data\" #@param {type:\"string\"}\n", "\n", "REPLACE_SOURCE_DATASET = True #@param {type:\"boolean\"}\n", "\n", "\n", "#@markdown ## Dataset filtering\n", "\n", "MIN_OUTPUT_CHARACTERS = 1 #@param {type:\"integer\"}\n", "\n", "MAX_TRAINING_SAMPLES = 0 #@param {type:\"integer\"}\n", "# 0 = use every valid example.\n", "\n", "\n", "#@markdown ## Hugging Face upload\n", "\n", "HF_REPO_ID = \"codeShare/kromoss-lora-exports\" #@param {type:\"string\"}\n", "\n", "HF_REPO_TYPE = \"model\" #@param [\"model\", \"dataset\"]\n", "\n", "HF_PATH_IN_REPO = \"gemma4_preencoded_dataset\" #@param {type:\"string\"}\n", "\n", "\n", "#@markdown ## Local output\n", "\n", "OUTPUT_DIR = \"/content/gemma4_preencoded\" #@param {type:\"string\"}\n", "\n", "REPLACE_LOCAL_OUTPUT = True #@param {type:\"boolean\"}\n", "\n", "\n", "#@markdown ## Upload verification\n", "\n", "VERIFY_REMOTE_UPLOAD = True #@param {type:\"boolean\"}\n", "\n", "\n", "# ============================================================\n", "# CONFIGURATION\n", "# ============================================================\n", "\n", "PREPROCESSING_VERSION = 7\n", "\n", "RANDOM_SEED = 1234\n", "\n", "torch.manual_seed(RANDOM_SEED)\n", "np.random.seed(RANDOM_SEED)\n", "\n", "IMAGE_EXTENSIONS = {\n", " \".jpg\", \".jpeg\", \".png\", \".webp\",\n", " \".bmp\", \".tif\", \".tiff\",\n", "}\n", "\n", "PCENC_MAGIC = b\"PCENC1\"\n", "\n", "PCENC_SCRYPT_N = 2**14\n", "PCENC_SCRYPT_R = 8\n", "PCENC_SCRYPT_P = 1\n", "\n", "PCENC_SALT_LENGTH = 16\n", "PCENC_NONCE_LENGTH = 12\n", "\n", "SEVEN_ZIP_MAGIC = b\"\\x37\\x7A\\xBC\\xAF\\x27\\x1C\"\n", "\n", "ZIP_MAGIC_PREFIXES = (\n", " b\"PK\\x03\\x04\",\n", " b\"PK\\x05\\x06\",\n", " b\"PK\\x07\\x08\",\n", ")\n", "\n", "\n", "# ============================================================\n", "# PATHS\n", "# ============================================================\n", "\n", "DATASET_DIR = Path(DATASET_DIR).expanduser().resolve()\n", "OUTPUT_DIR = Path(OUTPUT_DIR).expanduser().resolve()\n", "\n", "SOURCE_DOWNLOAD_DIR = Path(\"/content/gemma_encrypted_source\")\n", "SOURCE_ARCHIVE_PATH = SOURCE_DOWNLOAD_DIR / SOURCE_FILENAME\n", "\n", "OUTER_EXTRACT_DIR = Path(\"/content/gemma_outer_extracted\")\n", "\n", "PROCESSOR_DIR = OUTPUT_DIR / \"processor\"\n", "UPLOAD_DIR = OUTPUT_DIR / \"_hf_upload_payload\"\n", "\n", "SAFETENSORS_PATH = OUTPUT_DIR / \"examples.safetensors\"\n", "MANIFEST_PATH = OUTPUT_DIR / \"manifest.json\"\n", "\n", "\n", "if OUTPUT_DIR == DATASET_DIR or DATASET_DIR in OUTPUT_DIR.parents:\n", " raise RuntimeError(\n", " \"OUTPUT_DIR must not be inside DATASET_DIR.\"\n", " )\n", "\n", "if DATASET_DIR == SOURCE_DOWNLOAD_DIR or SOURCE_DOWNLOAD_DIR in DATASET_DIR.parents:\n", " raise RuntimeError(\n", " \"DATASET_DIR must not be inside SOURCE_DOWNLOAD_DIR.\"\n", " )\n", "\n", "if OUTPUT_DIR == SOURCE_DOWNLOAD_DIR or SOURCE_DOWNLOAD_DIR in OUTPUT_DIR.parents:\n", " raise RuntimeError(\n", " \"OUTPUT_DIR must not be inside SOURCE_DOWNLOAD_DIR.\"\n", " )\n", "\n", "\n", "# ============================================================\n", "# HUGGING FACE AUTHENTICATION\n", "# ============================================================\n", "\n", "print(\"=\" * 75)\n", "print(\"VERIFYING HUGGING FACE AUTHENTICATION\")\n", "print(\"=\" * 75)\n", "\n", "HF_TOKEN = None\n", "\n", "try:\n", " HF_TOKEN = userdata.get(\"HF_TOKEN\")\n", "except Exception:\n", " pass\n", "\n", "if not HF_TOKEN:\n", " HF_TOKEN = os.environ.get(\"HF_TOKEN\")\n", "\n", "if not HF_TOKEN:\n", " raise RuntimeError(\n", " \"HF_TOKEN was not found. Add it to Google Colab Secrets \"\n", " \"and enable notebook access.\"\n", " )\n", "\n", "api = HfApi(token=HF_TOKEN)\n", "\n", "repo_info = api.repo_info(\n", " repo_id=HF_REPO_ID,\n", " repo_type=HF_REPO_TYPE,\n", " token=HF_TOKEN,\n", ")\n", "\n", "if not getattr(repo_info, \"private\", False):\n", " raise RuntimeError(\n", " f\"{HF_REPO_ID} is not private. \"\n", " \"This workflow expects a private repository.\"\n", " )\n", "\n", "api.repo_info(\n", " repo_id=SOURCE_REPO_ID,\n", " repo_type=SOURCE_REPO_TYPE,\n", " token=HF_TOKEN,\n", ")\n", "\n", "print(\"HF token found.\")\n", "print(\"Source repository:\", SOURCE_REPO_ID)\n", "print(\"Destination repository:\", HF_REPO_ID)\n", "print(\"Destination repository private: Yes\")\n", "\n", "\n", "# ============================================================\n", "# DOWNLOAD SOURCE ARCHIVE\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"DOWNLOADING SOURCE ARCHIVE\")\n", "print(\"=\" * 75)\n", "\n", "SOURCE_DOWNLOAD_DIR.mkdir(\n", " parents=True,\n", " exist_ok=True,\n", ")\n", "\n", "downloaded_archive = hf_hub_download(\n", " repo_id=SOURCE_REPO_ID,\n", " filename=SOURCE_FILENAME,\n", " repo_type=SOURCE_REPO_TYPE,\n", " revision=SOURCE_REVISION,\n", " token=HF_TOKEN,\n", " local_dir=str(SOURCE_DOWNLOAD_DIR),\n", ")\n", "\n", "SOURCE_ARCHIVE_PATH = Path(downloaded_archive)\n", "\n", "if not SOURCE_ARCHIVE_PATH.is_file():\n", " raise FileNotFoundError(SOURCE_ARCHIVE_PATH)\n", "\n", "if SOURCE_ARCHIVE_PATH.stat().st_size == 0:\n", " raise RuntimeError(\"Downloaded archive is empty.\")\n", "\n", "archive_size = SOURCE_ARCHIVE_PATH.stat().st_size\n", "\n", "print(\"Downloaded:\", SOURCE_ARCHIVE_PATH)\n", "print(\n", " \"Archive size:\",\n", " f\"{archive_size / (1024**2):.2f} MB\",\n", ")\n", "\n", "\n", "# ============================================================\n", "# DETECT OUTER ARCHIVE FORMAT\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"DETECTING OUTER ARCHIVE FORMAT\")\n", "print(\"=\" * 75)\n", "\n", "with open(SOURCE_ARCHIVE_PATH, \"rb\") as file:\n", " archive_header = file.read(64)\n", "\n", "print(\"First 16 bytes:\", archive_header[:16].hex(\" \"))\n", "\n", "if archive_header.startswith(SEVEN_ZIP_MAGIC):\n", "\n", " OUTER_ARCHIVE_FORMAT = \"7z\"\n", "\n", "elif any(\n", " archive_header.startswith(prefix)\n", " for prefix in ZIP_MAGIC_PREFIXES\n", "):\n", "\n", " OUTER_ARCHIVE_FORMAT = \"zip\"\n", "\n", "else:\n", "\n", " raise RuntimeError(\n", " \"Unrecognized archive format.\\n\"\n", " f\"Header bytes: {archive_header!r}\"\n", " )\n", "\n", "print(\"Detected archive format:\", OUTER_ARCHIVE_FORMAT)\n", "\n", "\n", "# ============================================================\n", "# PCENC1 DECRYPTION\n", "# ============================================================\n", "\n", "def decrypt_pcenc1(encrypted_data, password):\n", "\n", " \"\"\"\n", " PCENC1 format:\n", "\n", " 6-byte magic\n", " 16-byte salt\n", " 12-byte nonce\n", " AES-256-GCM ciphertext and authentication tag\n", " \"\"\"\n", "\n", " if not encrypted_data.startswith(PCENC_MAGIC):\n", " raise ValueError(\"Invalid PCENC1 file header.\")\n", "\n", " salt_start = len(PCENC_MAGIC)\n", " salt_end = salt_start + PCENC_SALT_LENGTH\n", "\n", " nonce_start = salt_end\n", " nonce_end = nonce_start + PCENC_NONCE_LENGTH\n", "\n", " if len(encrypted_data) < nonce_end + 16:\n", " raise ValueError(\"PCENC1 file is too short.\")\n", "\n", " salt = encrypted_data[salt_start:salt_end]\n", " nonce = encrypted_data[nonce_start:nonce_end]\n", " ciphertext = encrypted_data[nonce_end:]\n", "\n", " password_bytes = (\n", " password.encode(\"utf-8\")\n", " if isinstance(password, str)\n", " else password\n", " )\n", "\n", " key = Scrypt(\n", " salt=salt,\n", " length=32,\n", " n=PCENC_SCRYPT_N,\n", " r=PCENC_SCRYPT_R,\n", " p=PCENC_SCRYPT_P,\n", " ).derive(password_bytes)\n", "\n", " return AESGCM(key).decrypt(\n", " nonce,\n", " ciphertext,\n", " PCENC_MAGIC,\n", " )\n", "\n", "\n", "def decrypt_if_encrypted(data, password):\n", "\n", " \"\"\"\n", " Detect encryption from the file contents, not the filename.\n", "\n", " Returns:\n", " decrypted_data, encryption_status\n", " \"\"\"\n", "\n", " if data.startswith(PCENC_MAGIC):\n", "\n", " return (\n", " decrypt_pcenc1(data, password),\n", " \"PCENC1 AES-256-GCM\",\n", " )\n", "\n", " return data, \"plaintext\"\n", "\n", "\n", "# ============================================================\n", "# SAFE ZIP EXTRACTION\n", "# ============================================================\n", "\n", "def safe_extract_zip(archive, destination):\n", "\n", " \"\"\"\n", " Extract ZIP files while preserving directory structure.\n", " Reject paths that escape the extraction directory.\n", " \"\"\"\n", "\n", " destination = Path(destination).resolve()\n", "\n", " archive_entries = archive.namelist()\n", "\n", " for archive_name in archive_entries:\n", "\n", " if archive_name.endswith(\"/\"):\n", " continue\n", "\n", " relative_path = Path(archive_name)\n", "\n", " if relative_path.is_absolute():\n", " raise RuntimeError(\n", " f\"Absolute ZIP path rejected: {archive_name}\"\n", " )\n", "\n", " if \"..\" in relative_path.parts:\n", " raise RuntimeError(\n", " f\"Unsafe ZIP path rejected: {archive_name}\"\n", " )\n", "\n", " target = (destination / relative_path).resolve()\n", "\n", " if destination not in target.parents and target != destination:\n", " raise RuntimeError(\n", " f\"ZIP path escapes extraction directory: {archive_name}\"\n", " )\n", "\n", " target.parent.mkdir(\n", " parents=True,\n", " exist_ok=True,\n", " )\n", "\n", " if target.exists():\n", " raise RuntimeError(\n", " f\"Duplicate ZIP extraction path: {target}\"\n", " )\n", "\n", " target.write_bytes(\n", " archive.read(archive_name)\n", " )\n", "\n", " return len(archive_entries)\n", "\n", "\n", "# ============================================================\n", "# EXTRACT OUTER ARCHIVE\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"EXTRACTING OUTER ARCHIVE\")\n", "print(\"=\" * 75)\n", "\n", "if OUTER_EXTRACT_DIR.exists():\n", " shutil.rmtree(OUTER_EXTRACT_DIR)\n", "\n", "OUTER_EXTRACT_DIR.mkdir(\n", " parents=True,\n", " exist_ok=True,\n", ")\n", "\n", "if OUTER_ARCHIVE_FORMAT == \"7z\":\n", "\n", " print(\"Opening 7z archive...\")\n", "\n", " try:\n", "\n", " with py7zr.SevenZipFile(\n", " SOURCE_ARCHIVE_PATH,\n", " mode=\"r\",\n", " password=OUTER_ZIP_PASSWORD,\n", " ) as archive:\n", "\n", " archive_entries = archive.getnames()\n", "\n", " if not archive_entries:\n", " raise RuntimeError(\n", " \"The 7z archive contains no entries.\"\n", " )\n", "\n", " print(\"Archive entries:\", len(archive_entries))\n", "\n", " archive.extractall(\n", " path=OUTER_EXTRACT_DIR,\n", " )\n", "\n", " except Exception as exc:\n", "\n", " raise RuntimeError(\n", " \"Failed to extract the 7z archive. \"\n", " \"Check the outer password and archive integrity.\\n\"\n", " f\"{type(exc).__name__}: {exc}\"\n", " ) from exc\n", "\n", "\n", "elif OUTER_ARCHIVE_FORMAT == \"zip\":\n", "\n", " print(\"Opening ZIP archive...\")\n", "\n", " try:\n", "\n", " with pyzipper.AESZipFile(\n", " SOURCE_ARCHIVE_PATH,\n", " mode=\"r\",\n", " ) as archive:\n", "\n", " archive.setpassword(\n", " OUTER_ZIP_PASSWORD.encode(\"utf-8\")\n", " )\n", "\n", " archive_entries = safe_extract_zip(\n", " archive,\n", " OUTER_EXTRACT_DIR,\n", " )\n", "\n", " if not archive_entries:\n", " raise RuntimeError(\n", " \"The ZIP archive contains no entries.\"\n", " )\n", "\n", " print(\"Archive entries:\", archive_entries)\n", "\n", " except Exception as exc:\n", "\n", " raise RuntimeError(\n", " \"Failed to extract the ZIP archive. \"\n", " \"Check the outer password and archive integrity.\\n\"\n", " f\"{type(exc).__name__}: {exc}\"\n", " ) from exc\n", "\n", "\n", "print(\"Outer archive extraction complete.\")\n", "\n", "\n", "# ============================================================\n", "# INDEX FILES USING ACTUAL ARCHIVE NAMING\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"INDEXING EXTRACTED DATASET FILES\")\n", "print(\"=\" * 75)\n", "\n", "extracted_files = [\n", " path\n", " for path in OUTER_EXTRACT_DIR.rglob(\"*\")\n", " if path.is_file()\n", "]\n", "\n", "if not extracted_files:\n", " raise RuntimeError(\n", " \"No files were extracted from the outer archive.\"\n", " )\n", "\n", "images_by_id = {}\n", "inputs_by_id = {}\n", "outputs_by_id = {}\n", "other_files = []\n", "\n", "\n", "def add_unique(mapping, sample_id, path, file_type):\n", "\n", " if sample_id in mapping:\n", "\n", " raise RuntimeError(\n", " f\"Duplicate {file_type} ID {sample_id}:\\n\"\n", " f\" {mapping[sample_id]}\\n\"\n", " f\" {path}\"\n", " )\n", "\n", " mapping[sample_id] = path\n", "\n", "\n", "for path in extracted_files:\n", "\n", " name = path.name\n", "\n", " # --------------------------------------------------------\n", " # Image: N.jpg, N.jpeg, etc.\n", " # --------------------------------------------------------\n", "\n", " if path.suffix.lower() in IMAGE_EXTENSIONS:\n", "\n", " sample_id = path.stem\n", "\n", " if sample_id.isdigit():\n", "\n", " add_unique(\n", " images_by_id,\n", " sample_id,\n", " path,\n", " \"image\",\n", " )\n", "\n", " continue\n", "\n", " # --------------------------------------------------------\n", " # Input: N_input.txt\n", " # --------------------------------------------------------\n", "\n", " if name.endswith(\"_input.txt\"):\n", "\n", " sample_id = name[:-len(\"_input.txt\")]\n", "\n", " if sample_id.isdigit():\n", "\n", " add_unique(\n", " inputs_by_id,\n", " sample_id,\n", " path,\n", " \"input\",\n", " )\n", "\n", " continue\n", "\n", " # --------------------------------------------------------\n", " # Output: N_output.txt\n", " # --------------------------------------------------------\n", "\n", " if name.endswith(\"_output.txt\"):\n", "\n", " sample_id = name[:-len(\"_output.txt\")]\n", "\n", " if sample_id.isdigit():\n", "\n", " add_unique(\n", " outputs_by_id,\n", " sample_id,\n", " path,\n", " \"output\",\n", " )\n", "\n", " continue\n", "\n", " other_files.append(path)\n", "\n", "\n", "image_ids = set(images_by_id)\n", "input_ids = set(inputs_by_id)\n", "output_ids = set(outputs_by_id)\n", "\n", "complete_ids = sorted(\n", " image_ids & input_ids & output_ids,\n", " key=int,\n", ")\n", "\n", "missing_images = sorted(\n", " (input_ids & output_ids) - image_ids,\n", " key=int,\n", ")\n", "\n", "missing_inputs = sorted(\n", " (image_ids & output_ids) - input_ids,\n", " key=int,\n", ")\n", "\n", "missing_outputs = sorted(\n", " (image_ids & input_ids) - output_ids,\n", " key=int,\n", ")\n", "\n", "print(\"Extracted files:\", len(extracted_files))\n", "print(\"Images:\", len(images_by_id))\n", "print(\"Inputs:\", len(inputs_by_id))\n", "print(\"Outputs:\", len(outputs_by_id))\n", "print(\"Complete triplets:\", len(complete_ids))\n", "\n", "print(\"Missing images:\", len(missing_images))\n", "print(\"Missing inputs:\", len(missing_inputs))\n", "print(\"Missing outputs:\", len(missing_outputs))\n", "\n", "print(\"\\nOther files:\")\n", "\n", "for path in other_files[:20]:\n", " print(\" \", path.relative_to(OUTER_EXTRACT_DIR))\n", "\n", "if not complete_ids:\n", " raise RuntimeError(\n", " \"No complete image/input/output triplets were found.\"\n", " )\n", "\n", "if missing_images or missing_inputs or missing_outputs:\n", "\n", " raise RuntimeError(\n", " \"The extracted dataset contains incomplete triplets. \"\n", " \"Processing has been stopped to avoid silently \"\n", " \"discarding dataset records.\"\n", " )\n", "\n", "print(\"\\nFirst three matched triplets:\")\n", "\n", "for sample_id in complete_ids[:3]:\n", "\n", " print(\n", " sample_id,\n", " images_by_id[sample_id].name,\n", " inputs_by_id[sample_id].name,\n", " outputs_by_id[sample_id].name,\n", " )\n", "\n", "\n", "# ============================================================\n", "# PREPARE SOURCE DATASET DIRECTORY\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"PREPARING SOURCE DATASET DIRECTORY\")\n", "print(\"=\" * 75)\n", "\n", "if DATASET_DIR.exists():\n", "\n", " if REPLACE_SOURCE_DATASET:\n", " shutil.rmtree(DATASET_DIR)\n", "\n", " else:\n", " raise FileExistsError(\n", " f\"Source dataset directory already exists: {DATASET_DIR}\"\n", " )\n", "\n", "DATASET_DIR.mkdir(\n", " parents=True,\n", " exist_ok=True,\n", ")\n", "\n", "print(\"Dataset directory:\", DATASET_DIR)\n", "\n", "\n", "# ============================================================\n", "# DECRYPT / COPY INDIVIDUAL FILES\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"PROCESSING INDIVIDUAL DATASET FILES\")\n", "print(\"=\" * 75)\n", "\n", "decrypted_count = 0\n", "plaintext_count = 0\n", "decryption_errors = []\n", "\n", "for index, sample_id in enumerate(complete_ids, start=1):\n", "\n", " source_paths = {\n", " \"image\": images_by_id[sample_id],\n", " \"input\": inputs_by_id[sample_id],\n", " \"output\": outputs_by_id[sample_id],\n", " }\n", "\n", " written_paths = []\n", "\n", " try:\n", "\n", " # ----------------------------------------------------\n", " # Read and detect encryption independently per file.\n", " # ----------------------------------------------------\n", "\n", " processed_data = {}\n", " encryption_status = {}\n", "\n", " for file_type, source_path in source_paths.items():\n", "\n", " raw_data = source_path.read_bytes()\n", "\n", " processed, status = decrypt_if_encrypted(\n", " raw_data,\n", " INDIVIDUAL_FILE_PASSWORD,\n", " )\n", "\n", " processed_data[file_type] = processed\n", " encryption_status[file_type] = status\n", "\n", " # ----------------------------------------------------\n", " # Validate image.\n", " # ----------------------------------------------------\n", "\n", " with Image.open(\n", " BytesIO(processed_data[\"image\"])\n", " ) as test_image:\n", "\n", " test_image.verify()\n", "\n", " # ----------------------------------------------------\n", " # Validate text encoding.\n", " # ----------------------------------------------------\n", "\n", " input_text = processed_data[\"input\"].decode(\n", " \"utf-8-sig\"\n", " )\n", "\n", " output_text = processed_data[\"output\"].decode(\n", " \"utf-8-sig\"\n", " )\n", "\n", " # ----------------------------------------------------\n", " # Write normalized dataset files.\n", " # ----------------------------------------------------\n", "\n", " image_path = DATASET_DIR / f\"{sample_id}.jpg\"\n", " input_path = DATASET_DIR / f\"{sample_id}_input.txt\"\n", " output_path = DATASET_DIR / f\"{sample_id}_output.txt\"\n", "\n", " image_path.write_bytes(\n", " processed_data[\"image\"]\n", " )\n", " written_paths.append(image_path)\n", "\n", " input_path.write_text(\n", " input_text,\n", " encoding=\"utf-8\",\n", " )\n", " written_paths.append(input_path)\n", "\n", " output_path.write_text(\n", " output_text,\n", " encoding=\"utf-8\",\n", " )\n", " written_paths.append(output_path)\n", "\n", " if any(\n", " status.startswith(\"PCENC1\")\n", " for status in encryption_status.values()\n", " ):\n", " decrypted_count += 1\n", " else:\n", " plaintext_count += 1\n", "\n", " if index <= 10 or index % 50 == 0:\n", "\n", " print(\n", " f\"[{index}/{len(complete_ids)}] \"\n", " f\"Processed {sample_id} | \"\n", " f\"image={encryption_status['image']} | \"\n", " f\"input={encryption_status['input']} | \"\n", " f\"output={encryption_status['output']}\"\n", " )\n", "\n", " except Exception as exc:\n", "\n", " for path in written_paths:\n", " path.unlink(missing_ok=True)\n", "\n", " decryption_errors.append({\n", " \"image_id\": sample_id,\n", " \"error\": f\"{type(exc).__name__}: {exc}\",\n", " })\n", "\n", " print(\n", " f\"ERROR processing {sample_id}: \"\n", " f\"{type(exc).__name__}: {exc}\"\n", " )\n", "\n", " finally:\n", "\n", " gc.collect()\n", "\n", "\n", "print(\"\\nIndividual file processing summary:\")\n", "print(\"Complete source triplets:\", len(complete_ids))\n", "print(\"Triplets containing PCENC1 files:\", decrypted_count)\n", "print(\"Triplets containing only plaintext files:\", plaintext_count)\n", "print(\"Processing failures:\", len(decryption_errors))\n", "\n", "if decrypted_count + plaintext_count == 0:\n", " raise RuntimeError(\n", " \"No source triplets were successfully processed.\"\n", " )\n", "\n", "\n", "# ============================================================\n", "# DISCOVER PROCESSED IMAGES\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"DISCOVERING PROCESSED IMAGES\")\n", "print(\"=\" * 75)\n", "\n", "def natural_sort_key(value):\n", "\n", " return [\n", " int(part) if part.isdigit() else part.lower()\n", " for part in re.split(r\"(\\d+)\", str(value))\n", " ]\n", "\n", "\n", "image_lookup = {}\n", "\n", "for image_path in DATASET_DIR.iterdir():\n", "\n", " if not image_path.is_file():\n", " continue\n", "\n", " if image_path.suffix.lower() not in IMAGE_EXTENSIONS:\n", " continue\n", "\n", " if image_path.stem in image_lookup:\n", "\n", " raise RuntimeError(\n", " f\"Duplicate image ID detected: {image_path.stem}\"\n", " )\n", "\n", " image_lookup[image_path.stem] = image_path\n", "\n", "\n", "image_ids = sorted(\n", " image_lookup,\n", " key=natural_sort_key,\n", ")\n", "\n", "print(\"Images discovered:\", len(image_ids))\n", "\n", "if not image_ids:\n", " raise RuntimeError(\n", " \"No supported images were found.\"\n", " )\n", "\n", "\n", "# ============================================================\n", "# VERIFY IMAGE / INPUT / OUTPUT TRIPLETS\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"VERIFYING SOURCE TRIPLETS\")\n", "print(\"=\" * 75)\n", "\n", "valid_examples = []\n", "\n", "missing_input = []\n", "missing_output = []\n", "invalid_examples = []\n", "\n", "for image_id in image_ids:\n", "\n", " image_path = image_lookup[image_id]\n", "\n", " input_path = DATASET_DIR / f\"{image_id}_input.txt\"\n", " output_path = DATASET_DIR / f\"{image_id}_output.txt\"\n", "\n", " if not input_path.is_file():\n", " missing_input.append(image_id)\n", " continue\n", "\n", " if not output_path.is_file():\n", " missing_output.append(image_id)\n", " continue\n", "\n", " try:\n", "\n", " input_text = input_path.read_text(\n", " encoding=\"utf-8-sig\"\n", " ).strip()\n", "\n", " output_text = output_path.read_text(\n", " encoding=\"utf-8-sig\"\n", " ).strip()\n", "\n", " if not input_text:\n", " raise ValueError(\"Empty input prompt.\")\n", "\n", " if len(output_text) < MIN_OUTPUT_CHARACTERS:\n", " raise ValueError(\n", " \"Output below minimum character count.\"\n", " )\n", "\n", " with Image.open(image_path) as source:\n", "\n", " source.load()\n", "\n", " image = ImageOps.exif_transpose(\n", " source\n", " ).convert(\"RGB\")\n", "\n", " image_size = list(image.size)\n", "\n", " image.close()\n", "\n", " valid_examples.append({\n", " \"image_id\": image_id,\n", " \"image_path\": str(image_path),\n", " \"input_text\": input_text,\n", " \"output_text\": output_text,\n", " \"image_size\": image_size,\n", " })\n", "\n", " except Exception as exc:\n", "\n", " invalid_examples.append({\n", " \"image_id\": image_id,\n", " \"error\": f\"{type(exc).__name__}: {exc}\",\n", " })\n", "\n", " print(\n", " f\"Invalid {image_id}: {exc}\"\n", " )\n", "\n", "\n", "if MAX_TRAINING_SAMPLES > 0:\n", " valid_examples = valid_examples[:MAX_TRAINING_SAMPLES]\n", "\n", "\n", "print(\"Valid examples:\", len(valid_examples))\n", "print(\"Missing input files:\", len(missing_input))\n", "print(\"Missing output files:\", len(missing_output))\n", "print(\"Invalid examples:\", len(invalid_examples))\n", "\n", "if not valid_examples:\n", " raise RuntimeError(\n", " \"No valid training examples were found.\"\n", " )\n", "\n", "\n", "# ============================================================\n", "# PREPARE OUTPUT DIRECTORY\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"PREPARING OUTPUT DIRECTORY\")\n", "print(\"=\" * 75)\n", "\n", "if OUTPUT_DIR.exists():\n", "\n", " if REPLACE_LOCAL_OUTPUT:\n", " shutil.rmtree(OUTPUT_DIR)\n", "\n", " else:\n", " raise FileExistsError(OUTPUT_DIR)\n", "\n", "PROCESSOR_DIR.mkdir(\n", " parents=True,\n", " exist_ok=True,\n", ")\n", "\n", "print(\"Output:\", OUTPUT_DIR)\n", "\n", "\n", "# ============================================================\n", "# LOAD PROCESSOR ONLY\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"LOADING GEMMA PROCESSOR\")\n", "print(\"=\" * 75)\n", "\n", "processor = AutoProcessor.from_pretrained(\n", " MODEL_ID,\n", " token=HF_TOKEN,\n", ")\n", "\n", "tokenizer = processor.tokenizer\n", "\n", "tokenizer.padding_side = \"right\"\n", "\n", "print(\"Processor loaded.\")\n", "print(\"Tokenizer vocabulary:\", len(tokenizer))\n", "print(\"No transformer model weights were loaded.\")\n", "\n", "\n", "# ============================================================\n", "# CONVERSATION BUILDER\n", "# ============================================================\n", "\n", "def get_example_messages(\n", " image,\n", " input_text,\n", " output_text,\n", "):\n", "\n", " return [\n", " {\n", " \"role\": \"user\",\n", " \"content\": [\n", " {\n", " \"type\": \"image\",\n", " \"image\": image,\n", " },\n", " {\n", " \"type\": \"text\",\n", " \"text\": input_text,\n", " },\n", " ],\n", " },\n", " {\n", " \"role\": \"assistant\",\n", " \"content\": [\n", " {\n", " \"type\": \"text\",\n", " \"text\": output_text,\n", " },\n", " ],\n", " },\n", " ]\n", "\n", "\n", "# ============================================================\n", "# TENSOR CONVERSION\n", "# ============================================================\n", "\n", "def tensorize_processor_output(\n", " value,\n", " prefix=\"\",\n", "):\n", "\n", " \"\"\"\n", " Convert processor outputs into a flat tensor dictionary.\n", "\n", " Safetensors stores tensors only. Non-tensor values are\n", " rejected instead of silently discarded.\n", " \"\"\"\n", "\n", " output = {}\n", "\n", " if isinstance(value, torch.Tensor):\n", "\n", " if not prefix:\n", " raise ValueError(\n", " \"Tensor has no assigned name.\"\n", " )\n", "\n", " output[prefix] = (\n", " value.detach()\n", " .cpu()\n", " .contiguous()\n", " .clone()\n", " )\n", "\n", " return output\n", "\n", " if isinstance(value, np.ndarray):\n", "\n", " if not prefix:\n", " raise ValueError(\n", " \"Array has no assigned name.\"\n", " )\n", "\n", " output[prefix] = torch.from_numpy(\n", " value.copy()\n", " ).contiguous()\n", "\n", " return output\n", "\n", " if isinstance(value, dict):\n", "\n", " for key, item in value.items():\n", "\n", " child_prefix = (\n", " f\"{prefix}.{key}\"\n", " if prefix\n", " else str(key)\n", " )\n", "\n", " output.update(\n", " tensorize_processor_output(\n", " item,\n", " child_prefix,\n", " )\n", " )\n", "\n", " return output\n", "\n", " if isinstance(value, (list, tuple)):\n", "\n", " for index, item in enumerate(value):\n", "\n", " child_prefix = (\n", " f\"{prefix}.{index}\"\n", " if prefix\n", " else str(index)\n", " )\n", "\n", " output.update(\n", " tensorize_processor_output(\n", " item,\n", " child_prefix,\n", " )\n", " )\n", "\n", " return output\n", "\n", " raise TypeError(\n", " f\"Unsupported non-tensor processor output at \"\n", " f\"{prefix!r}: {type(value).__name__}\"\n", " )\n", "\n", "\n", "# ============================================================\n", "# IMAGE TOKEN ID DISCOVERY\n", "# ============================================================\n", "\n", "def get_image_token_ids(tokenizer):\n", "\n", " \"\"\"\n", " Collect image-related token IDs exposed by the tokenizer.\n", "\n", " These tokens are masked in labels so the language modeling\n", " loss is applied to assistant text rather than image markers.\n", " \"\"\"\n", "\n", " token_ids = set()\n", "\n", " possible_attributes = [\n", " \"boi_token_id\",\n", " \"image_token_id\",\n", " \"eoi_token_id\",\n", " \"image_start_token_id\",\n", " \"image_end_token_id\",\n", " ]\n", "\n", " for attribute in possible_attributes:\n", "\n", " value = getattr(\n", " tokenizer,\n", " attribute,\n", " None,\n", " )\n", "\n", " if isinstance(value, int) and value >= 0:\n", " token_ids.add(value)\n", "\n", " possible_tokens = [\n", " \"\",\n", " \"<|image|>\",\n", " \"<|image_start|>\",\n", " \"<|image_end|>\",\n", " \"<|image_soft_token|>\",\n", " \"<|image_patch|>\",\n", " \"<|boi|>\",\n", " \"<|eoi|>\",\n", " ]\n", "\n", " for token in possible_tokens:\n", "\n", " token_id = tokenizer.convert_tokens_to_ids(\n", " token\n", " )\n", "\n", " if not isinstance(token_id, int):\n", " continue\n", "\n", " if token_id < 0:\n", " continue\n", "\n", " unknown_id = getattr(\n", " tokenizer,\n", " \"unk_token_id\",\n", " None,\n", " )\n", "\n", " if unknown_id is not None and token_id == unknown_id:\n", " continue\n", "\n", " if tokenizer.convert_ids_to_tokens(token_id) == token:\n", " token_ids.add(token_id)\n", "\n", " return sorted(token_ids)\n", "\n", "\n", "IMAGE_TOKEN_IDS = get_image_token_ids(tokenizer)\n", "\n", "print(\n", " \"Image-related token IDs:\",\n", " IMAGE_TOKEN_IDS,\n", ")\n", "\n", "\n", "# ============================================================\n", "# PRE-ENCODE ONE EXAMPLE\n", "# ============================================================\n", "\n", "def encode_example(\n", " example,\n", " example_index,\n", "):\n", "\n", " image_id = example[\"image_id\"]\n", "\n", " # --------------------------------------------------------\n", " # Load image\n", " # --------------------------------------------------------\n", "\n", " with Image.open(example[\"image_path\"]) as source:\n", "\n", " source.load()\n", "\n", " image = ImageOps.exif_transpose(\n", " source\n", " ).convert(\"RGB\")\n", "\n", " # --------------------------------------------------------\n", " # Build full conversation\n", " # --------------------------------------------------------\n", "\n", " messages = get_example_messages(\n", " image=image,\n", " input_text=example[\"input_text\"],\n", " output_text=example[\"output_text\"],\n", " )\n", "\n", " full_text = processor.apply_chat_template(\n", " messages,\n", " tokenize=False,\n", " add_generation_prompt=False,\n", " )\n", "\n", " # --------------------------------------------------------\n", " # Build user prefix\n", " # --------------------------------------------------------\n", "\n", " prefix_text = processor.apply_chat_template(\n", " messages[:1],\n", " tokenize=False,\n", " add_generation_prompt=True,\n", " )\n", "\n", " # --------------------------------------------------------\n", " # Encode full multimodal conversation\n", " # --------------------------------------------------------\n", "\n", " encoded = processor(\n", " text=[full_text],\n", " images=[image],\n", " padding=False,\n", " return_tensors=\"pt\",\n", " )\n", "\n", " # --------------------------------------------------------\n", " # Encode prefix using the same image\n", " # --------------------------------------------------------\n", "\n", " prefix_encoded = processor(\n", " text=[prefix_text],\n", " images=[image],\n", " padding=False,\n", " return_tensors=\"pt\",\n", " )\n", "\n", " required_keys = {\n", " \"input_ids\",\n", " \"attention_mask\",\n", " }\n", "\n", " missing_full = required_keys - set(encoded.keys())\n", "\n", " missing_prefix = (\n", " required_keys - set(prefix_encoded.keys())\n", " )\n", "\n", " if missing_full:\n", " raise RuntimeError(\n", " f\"Full processor output missing keys: {missing_full}\"\n", " )\n", "\n", " if missing_prefix:\n", " raise RuntimeError(\n", " f\"Prefix processor output missing keys: {missing_prefix}\"\n", " )\n", "\n", " input_ids = encoded[\"input_ids\"]\n", " attention_mask = encoded[\"attention_mask\"]\n", "\n", " prefix_ids = prefix_encoded[\"input_ids\"]\n", " prefix_attention_mask = prefix_encoded[\"attention_mask\"]\n", "\n", " prefix_length = int(\n", " prefix_attention_mask.sum().item()\n", " )\n", "\n", " sequence_length = int(\n", " input_ids.shape[1]\n", " )\n", "\n", " if prefix_length <= 0:\n", " raise RuntimeError(\n", " f\"Empty prefix for {image_id}.\"\n", " )\n", "\n", " if prefix_length >= sequence_length:\n", " raise RuntimeError(\n", " f\"Invalid prefix length for {image_id}: \"\n", " f\"{prefix_length} >= {sequence_length}\"\n", " )\n", "\n", " # --------------------------------------------------------\n", " # Verify prefix tokenization\n", " # --------------------------------------------------------\n", "\n", " if not torch.equal(\n", " input_ids[0, :prefix_length],\n", " prefix_ids[0, :prefix_length],\n", " ):\n", "\n", " raise RuntimeError(\n", " f\"Prefix token mismatch for {image_id}. \"\n", " \"Cannot safely construct completion-only labels.\"\n", " )\n", "\n", " # --------------------------------------------------------\n", " # Completion-only labels\n", " # --------------------------------------------------------\n", "\n", " labels = input_ids.clone()\n", "\n", " # Mask padding.\n", " labels[attention_mask == 0] = -100\n", "\n", " # Mask user prompt and generation prefix.\n", " labels[:, :prefix_length] = -100\n", "\n", " # Mask image marker tokens.\n", " for token_id in IMAGE_TOKEN_IDS:\n", " labels[labels == token_id] = -100\n", "\n", " target_tokens = int(\n", " (labels != -100).sum().item()\n", " )\n", "\n", " if target_tokens <= 0:\n", " raise RuntimeError(\n", " f\"No assistant target tokens for {image_id}.\"\n", " )\n", "\n", " # --------------------------------------------------------\n", " # Convert processor output to CPU tensors\n", " # --------------------------------------------------------\n", "\n", " tensor_dict = tensorize_processor_output(\n", " dict(encoded)\n", " )\n", "\n", " tensor_dict[\"labels\"] = (\n", " labels.detach()\n", " .cpu()\n", " .contiguous()\n", " .clone()\n", " )\n", "\n", " # --------------------------------------------------------\n", " # Namespace tensors by example index\n", " # --------------------------------------------------------\n", "\n", " example_prefix = f\"{example_index:06d}\"\n", "\n", " namespaced_tensors = {}\n", "\n", " for key, tensor in tensor_dict.items():\n", "\n", " tensor_name = f\"{example_prefix}.{key}\"\n", "\n", " if tensor_name in namespaced_tensors:\n", " raise RuntimeError(\n", " f\"Duplicate tensor key: {tensor_name}\"\n", " )\n", "\n", " namespaced_tensors[tensor_name] = tensor\n", "\n", " # --------------------------------------------------------\n", " # Metadata for manifest\n", " # --------------------------------------------------------\n", "\n", " tensor_metadata = {}\n", "\n", " for key, tensor in tensor_dict.items():\n", "\n", " tensor_metadata[key] = {\n", " \"safetensors_key\": (\n", " f\"{example_prefix}.{key}\"\n", " ),\n", " \"shape\": list(tensor.shape),\n", " \"dtype\": str(tensor.dtype),\n", " }\n", "\n", " record_metadata = {\n", " \"example_index\": example_index,\n", " \"image_id\": image_id,\n", " \"image_size\": example[\"image_size\"],\n", " \"sequence_length\": sequence_length,\n", " \"prefix_length\": prefix_length,\n", " \"target_tokens\": target_tokens,\n", " \"input_characters\": len(example[\"input_text\"]),\n", " \"output_characters\": len(example[\"output_text\"]),\n", " \"tensors\": tensor_metadata,\n", " }\n", "\n", " # --------------------------------------------------------\n", " # Release temporary objects\n", " # --------------------------------------------------------\n", "\n", " del image\n", " del messages\n", " del encoded\n", " del prefix_encoded\n", " del input_ids\n", " del attention_mask\n", " del prefix_ids\n", " del prefix_attention_mask\n", " del labels\n", " del tensor_dict\n", "\n", " return namespaced_tensors, record_metadata\n", "\n", "\n", "# ============================================================\n", "# PRE-ENCODE DATASET INTO ONE TENSOR COLLECTION\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"PRE-ENCODING MULTIMODAL DATASET\")\n", "print(\"=\" * 75)\n", "\n", "all_tensors = {}\n", "encoded_records = []\n", "encoding_errors = []\n", "\n", "for index, example in enumerate(valid_examples):\n", "\n", " print(\n", " f\"\\n[{index + 1}/{len(valid_examples)}] \"\n", " f\"Encoding example {example['image_id']}\"\n", " )\n", "\n", " try:\n", "\n", " example_tensors, metadata = encode_example(\n", " example,\n", " index,\n", " )\n", "\n", " collisions = (\n", " set(all_tensors) & set(example_tensors)\n", " )\n", "\n", " if collisions:\n", " raise RuntimeError(\n", " f\"Duplicate tensor names: {sorted(collisions)}\"\n", " )\n", "\n", " all_tensors.update(example_tensors)\n", "\n", " encoded_records.append(metadata)\n", "\n", " print(\n", " \"Sequence tokens:\",\n", " metadata[\"sequence_length\"],\n", " )\n", "\n", " print(\n", " \"Target tokens:\",\n", " metadata[\"target_tokens\"],\n", " )\n", "\n", " print(\n", " \"Tensor count:\",\n", " len(example_tensors),\n", " )\n", "\n", " example_storage = sum(\n", " tensor.numel() * tensor.element_size()\n", " for tensor in example_tensors.values()\n", " )\n", "\n", " print(\n", " \"Tensor storage:\",\n", " f\"{example_storage / (1024**2):.2f} MB\",\n", " )\n", "\n", " del example_tensors\n", "\n", " except Exception as exc:\n", "\n", " encoding_errors.append({\n", " \"image_id\": example[\"image_id\"],\n", " \"error\": f\"{type(exc).__name__}: {exc}\",\n", " })\n", "\n", " print(\n", " f\"ERROR encoding {example['image_id']}: {exc}\"\n", " )\n", "\n", " finally:\n", " gc.collect()\n", "\n", "\n", "if not encoded_records:\n", " raise RuntimeError(\n", " \"No examples were successfully pre-encoded.\"\n", " )\n", "\n", "if not all_tensors:\n", " raise RuntimeError(\n", " \"No tensors were generated.\"\n", " )\n", "\n", "\n", "# ============================================================\n", "# SAVE ONE SAFETENSORS FILE\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"SAVING SINGLE SAFETENSORS DATASET\")\n", "print(\"=\" * 75)\n", "\n", "safetensors_metadata = {\n", " \"format\": \"gemma4_multimodal_processor_tensors\",\n", " \"base_model\": MODEL_ID,\n", " \"preprocessing_version\": str(PREPROCESSING_VERSION),\n", " \"tensor_layout\": \"example_index.tensor_name\",\n", " \"loss_masking\": \"completion_only\",\n", " \"tensor_device\": \"cpu\",\n", "}\n", "\n", "print(\n", " \"Total examples:\",\n", " len(encoded_records),\n", ")\n", "\n", "print(\n", " \"Total tensors:\",\n", " len(all_tensors),\n", ")\n", "\n", "tensor_storage_bytes = sum(\n", " tensor.numel() * tensor.element_size()\n", " for tensor in all_tensors.values()\n", ")\n", "\n", "print(\n", " \"Uncompressed tensor storage:\",\n", " f\"{tensor_storage_bytes / (1024**3):.3f} GB\",\n", ")\n", "\n", "save_file(\n", " all_tensors,\n", " str(SAFETENSORS_PATH),\n", " metadata=safetensors_metadata,\n", ")\n", "\n", "if not SAFETENSORS_PATH.is_file():\n", " raise RuntimeError(\n", " \"Safetensors file was not created.\"\n", " )\n", "\n", "if SAFETENSORS_PATH.stat().st_size == 0:\n", " raise RuntimeError(\n", " \"Safetensors file is empty.\"\n", " )\n", "\n", "print(\"Saved:\", SAFETENSORS_PATH)\n", "\n", "print(\n", " \"File size:\",\n", " f\"{SAFETENSORS_PATH.stat().st_size / (1024**2):.2f} MB\",\n", ")\n", "\n", "del all_tensors\n", "gc.collect()\n", "\n", "\n", "# ============================================================\n", "# SAVE PROCESSOR CONFIGURATION\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"SAVING PROCESSOR CONFIGURATION\")\n", "print(\"=\" * 75)\n", "\n", "processor.save_pretrained(\n", " PROCESSOR_DIR\n", ")\n", "\n", "processor_files = [\n", " str(path.relative_to(PROCESSOR_DIR))\n", " for path in PROCESSOR_DIR.rglob(\"*\")\n", " if path.is_file()\n", "]\n", "\n", "print(\n", " \"Processor files saved:\",\n", " len(processor_files),\n", ")\n", "\n", "\n", "# ============================================================\n", "# CREATE MANIFEST\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"CREATING DATASET MANIFEST\")\n", "print(\"=\" * 75)\n", "\n", "manifest = {\n", " \"format\": \"gemma4_preencoded_multimodal_safetensors\",\n", " \"preprocessing_version\": PREPROCESSING_VERSION,\n", " \"base_model\": MODEL_ID,\n", "\n", " \"source_archive\": {\n", " \"repo_id\": SOURCE_REPO_ID,\n", " \"filename\": SOURCE_FILENAME,\n", " \"revision\": SOURCE_REVISION,\n", " \"detected_outer_format\": OUTER_ARCHIVE_FORMAT,\n", " \"individual_file_encryption\": (\n", " \"Automatically detected per file: PCENC1 AES-256-GCM \"\n", " \"or plaintext\"\n", " ),\n", " \"file_naming\": {\n", " \"image\": \"N.jpg\",\n", " \"input\": \"N_input.txt\",\n", " \"output\": \"N_output.txt\",\n", " \"matching\": \"shared numeric ID\",\n", " },\n", " },\n", "\n", " \"storage\": {\n", " \"format\": \"safetensors\",\n", " \"filename\": \"examples.safetensors\",\n", " \"one_file_for_all_examples\": True,\n", " \"tensor_device\": \"CPU\",\n", " \"tensor_key_format\": \"example_index.tensor_name\",\n", " \"variable_tensor_shapes_supported\": True,\n", " \"tensor_storage_bytes\": tensor_storage_bytes,\n", " \"tensor_file_bytes\": SAFETENSORS_PATH.stat().st_size,\n", " },\n", "\n", " \"encoding\": {\n", " \"one_example_encoded_at_a_time\": True,\n", " \"processor_only\": True,\n", " \"transformer_weights_loaded\": False,\n", " \"completion_only_loss\": True,\n", " \"user_prompt_masked\": True,\n", " \"image_tokens_masked\": True,\n", " \"assistant_output_trainable\": True,\n", " \"padding_masked\": True,\n", " \"image_token_ids_masked\": IMAGE_TOKEN_IDS,\n", " \"source_images_stored_in_tensor_file\": False,\n", " \"source_text_stored_in_tensor_file\": False,\n", " \"processor_configuration_path\": \"processor/\",\n", " },\n", "\n", " \"dataset\": {\n", " \"extracted_triplets\": len(complete_ids),\n", " \"triplets_processed_with_pcenc1\": decrypted_count,\n", " \"triplets_processed_as_plaintext\": plaintext_count,\n", " \"failed_decryptions_or_file_processing\": len(decryption_errors),\n", " \"source_examples\": len(valid_examples),\n", " \"successfully_encoded\": len(encoded_records),\n", " \"failed_encodings\": len(encoding_errors),\n", " \"examples\": encoded_records,\n", " \"encoding_errors\": encoding_errors,\n", " \"processing_errors\": decryption_errors,\n", " },\n", "}\n", "\n", "with open(\n", " MANIFEST_PATH,\n", " \"w\",\n", " encoding=\"utf-8\",\n", ") as file:\n", "\n", " json.dump(\n", " manifest,\n", " file,\n", " indent=2,\n", " ensure_ascii=False,\n", " )\n", "\n", "print(\n", " \"Manifest saved:\",\n", " MANIFEST_PATH,\n", ")\n", "\n", "\n", "# ============================================================\n", "# CREATE LOCAL README\n", "# ============================================================\n", "\n", "readme = f\"\"\"# Gemma 4 E2B Pre-encoded Multimodal Dataset\n", "\n", "Base model: `{MODEL_ID}`\n", "\n", "## Files\n", "\n", "- `examples.safetensors`: all pre-encoded examples in one file\n", "- `processor/`: processor and tokenizer configuration\n", "- `manifest.json`: example metadata and tensor index\n", "\n", "## Safetensors key layout\n", "\n", "Each example has a numeric namespace.\n", "\n", "For example:\n", "\n", "- `000000.input_ids`\n", "- `000000.attention_mask`\n", "- `000000.pixel_values`\n", "- `000000.labels`\n", "- `000001.input_ids`\n", "- `000001.attention_mask`\n", "- `000001.pixel_values`\n", "- `000001.labels`\n", "\n", "The exact tensor keys depend on the processor output.\n", "\n", "Tensor shapes may differ between examples. The manifest records\n", "the tensor names, shapes, and dtypes for every example.\n", "\n", "## Loss masking\n", "\n", "- User prompt tokens: -100\n", "- Image marker tokens: -100\n", "- Assistant target tokens: normal token IDs\n", "- Padding tokens: -100\n", "\n", "The original source images and source prompt/response text are\n", "not stored in the safetensors file.\n", "\n", "These are processor-encoded tensors, not transformer\n", "hidden-state embeddings. The base model weights were not loaded.\n", "\"\"\"\n", "\n", "with open(\n", " OUTPUT_DIR / \"README.md\",\n", " \"w\",\n", " encoding=\"utf-8\",\n", ") as file:\n", "\n", " file.write(readme)\n", "\n", "print(\n", " \"Local README saved. It will not be uploaded.\"\n", ")\n", "\n", "\n", "# ============================================================\n", "# VERIFY SINGLE SAFETENSORS FILE\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"VERIFYING SINGLE SAFETENSORS FILE\")\n", "print(\"=\" * 75)\n", "\n", "verified_tensors = load_file(\n", " str(SAFETENSORS_PATH),\n", " device=\"cpu\",\n", ")\n", "\n", "expected_tensor_count = sum(\n", " len(record[\"tensors\"])\n", " for record in encoded_records\n", ")\n", "\n", "if len(verified_tensors) != expected_tensor_count:\n", " raise RuntimeError(\n", " \"Safetensors tensor count mismatch. \"\n", " f\"Expected {expected_tensor_count}, \"\n", " f\"found {len(verified_tensors)}.\"\n", " )\n", "\n", "\n", "# ------------------------------------------------------------\n", "# Verify every tensor against manifest\n", "# ------------------------------------------------------------\n", "\n", "for record in encoded_records:\n", "\n", " for tensor_name, tensor_info in record[\"tensors\"].items():\n", "\n", " safetensors_key = tensor_info[\"safetensors_key\"]\n", "\n", " if safetensors_key not in verified_tensors:\n", " raise RuntimeError(\n", " f\"Missing tensor: {safetensors_key}\"\n", " )\n", "\n", " tensor = verified_tensors[safetensors_key]\n", "\n", " if list(tensor.shape) != tensor_info[\"shape\"]:\n", " raise RuntimeError(\n", " f\"Shape mismatch for {safetensors_key}\"\n", " )\n", "\n", " if str(tensor.dtype) != tensor_info[\"dtype\"]:\n", " raise RuntimeError(\n", " f\"Dtype mismatch for {safetensors_key}\"\n", " )\n", "\n", " if tensor.device.type != \"cpu\":\n", " raise RuntimeError(\n", " f\"Tensor is not on CPU: {safetensors_key}\"\n", " )\n", "\n", "\n", "# ------------------------------------------------------------\n", "# Verify required training tensors and labels\n", "# ------------------------------------------------------------\n", "\n", "for record in encoded_records:\n", "\n", " example_prefix = f\"{record['example_index']:06d}\"\n", "\n", " required = [\n", " f\"{example_prefix}.input_ids\",\n", " f\"{example_prefix}.attention_mask\",\n", " f\"{example_prefix}.labels\",\n", " ]\n", "\n", " for key in required:\n", "\n", " if key not in verified_tensors:\n", " raise RuntimeError(\n", " f\"Missing required tensor: {key}\"\n", " )\n", "\n", " input_ids = verified_tensors[\n", " f\"{example_prefix}.input_ids\"\n", " ]\n", "\n", " labels = verified_tensors[\n", " f\"{example_prefix}.labels\"\n", " ]\n", "\n", " attention_mask = verified_tensors[\n", " f\"{example_prefix}.attention_mask\"\n", " ]\n", "\n", " if input_ids.shape != labels.shape:\n", " raise RuntimeError(\n", " f\"Input/label shape mismatch for example \"\n", " f\"{record['image_id']}\"\n", " )\n", "\n", " if input_ids.shape != attention_mask.shape:\n", " raise RuntimeError(\n", " f\"Input/attention mask shape mismatch for \"\n", " f\"{record['image_id']}\"\n", " )\n", "\n", " if int(\n", " (labels != -100).sum().item()\n", " ) <= 0:\n", "\n", " raise RuntimeError(\n", " f\"No trainable target tokens for \"\n", " f\"{record['image_id']}\"\n", " )\n", "\n", " if torch.any(labels[attention_mask == 0] != -100):\n", " raise RuntimeError(\n", " f\"Padding labels are not masked for \"\n", " f\"{record['image_id']}\"\n", " )\n", "\n", "\n", "verified_tensor_count = len(verified_tensors)\n", "\n", "del verified_tensors\n", "gc.collect()\n", "\n", "print(\"Safetensors verification passed.\")\n", "print(\"Verified examples:\", len(encoded_records))\n", "print(\"Verified tensors:\", verified_tensor_count)\n", "print(\"Single safetensors file:\", SAFETENSORS_PATH.name)\n", "\n", "\n", "# ============================================================\n", "# BUILD STRICT UPLOAD STAGING DIRECTORY\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"BUILDING HUGGING FACE UPLOAD PAYLOAD\")\n", "print(\"=\" * 75)\n", "\n", "if UPLOAD_DIR.exists():\n", " shutil.rmtree(UPLOAD_DIR)\n", "\n", "UPLOAD_PROCESSOR_DIR = UPLOAD_DIR / \"processor\"\n", "\n", "UPLOAD_PROCESSOR_DIR.mkdir(\n", " parents=True,\n", " exist_ok=True,\n", ")\n", "\n", "# Copy the one safetensors file.\n", "shutil.copy2(\n", " SAFETENSORS_PATH,\n", " UPLOAD_DIR / \"examples.safetensors\",\n", ")\n", "\n", "# Copy processor assets.\n", "for source in PROCESSOR_DIR.rglob(\"*\"):\n", "\n", " if not source.is_file():\n", " continue\n", "\n", " relative_path = source.relative_to(\n", " PROCESSOR_DIR\n", " )\n", "\n", " destination = (\n", " UPLOAD_PROCESSOR_DIR / relative_path\n", " )\n", "\n", " destination.parent.mkdir(\n", " parents=True,\n", " exist_ok=True,\n", " )\n", "\n", " shutil.copy2(\n", " source,\n", " destination,\n", " )\n", "\n", "# Copy manifest.\n", "shutil.copy2(\n", " MANIFEST_PATH,\n", " UPLOAD_DIR / \"manifest.json\",\n", ")\n", "\n", "\n", "# ============================================================\n", "# VALIDATE UPLOAD PAYLOAD\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"VALIDATING UPLOAD PAYLOAD\")\n", "print(\"=\" * 75)\n", "\n", "payload_files = sorted(\n", " path\n", " for path in UPLOAD_DIR.rglob(\"*\")\n", " if path.is_file()\n", ")\n", "\n", "if not payload_files:\n", " raise RuntimeError(\n", " \"Upload payload is empty.\"\n", " )\n", "\n", "payload_relative_paths = [\n", " str(path.relative_to(UPLOAD_DIR))\n", " for path in payload_files\n", "]\n", "\n", "if \"examples.safetensors\" not in payload_relative_paths:\n", " raise RuntimeError(\n", " \"The single safetensors file is missing from upload payload.\"\n", " )\n", "\n", "if \"manifest.json\" not in payload_relative_paths:\n", " raise RuntimeError(\n", " \"Manifest is missing from upload payload.\"\n", " )\n", "\n", "safetensors_files = [\n", " path\n", " for path in payload_files\n", " if path.suffix.lower() == \".safetensors\"\n", "]\n", "\n", "if len(safetensors_files) != 1:\n", " raise RuntimeError(\n", " \"Expected exactly one safetensors file in upload payload. \"\n", " f\"Found {len(safetensors_files)}.\"\n", " )\n", "\n", "unexpected_pt_files = [\n", " path\n", " for path in payload_files\n", " if path.suffix.lower() == \".pt\"\n", "]\n", "\n", "if unexpected_pt_files:\n", " raise RuntimeError(\n", " \"Unexpected .pt files in upload payload: \"\n", " f\"{unexpected_pt_files}\"\n", " )\n", "\n", "if any(\n", " path.name == \"README.md\"\n", " for path in payload_files\n", "):\n", " raise RuntimeError(\n", " \"README.md must not be uploaded.\"\n", " )\n", "\n", "payload_size_bytes = sum(\n", " path.stat().st_size\n", " for path in payload_files\n", ")\n", "\n", "payload_size_mb = payload_size_bytes / (1024**2)\n", "\n", "print(\"Upload payload validated.\")\n", "print(\"Safetensors files:\", len(safetensors_files))\n", "print(\"Processor files:\", len(processor_files))\n", "print(\"Manifest included: Yes\")\n", "print(\"README included: No\")\n", "print(\"Source images included: No\")\n", "print(\"Source text files included: No\")\n", "print(\"Separate .pt files included: No\")\n", "print(\"Payload files:\", len(payload_files))\n", "print(\"Payload size:\", f\"{payload_size_mb:.2f} MB\")\n", "\n", "print(\"\\nPayload contents:\")\n", "\n", "for relative_path in payload_relative_paths:\n", " print(\" \", relative_path)\n", "\n", "\n", "# ============================================================\n", "# UPLOAD TO HUGGING FACE\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"UPLOADING SINGLE-SAFETENSORS DATASET\")\n", "print(\"=\" * 75)\n", "\n", "HF_PATH_IN_REPO = HF_PATH_IN_REPO.strip(\"/\")\n", "\n", "if not HF_PATH_IN_REPO:\n", " raise ValueError(\n", " \"HF_PATH_IN_REPO cannot be empty.\"\n", " )\n", "\n", "print(\"Repository:\", HF_REPO_ID)\n", "print(\"Repository type:\", HF_REPO_TYPE)\n", "print(\"Destination:\", HF_PATH_IN_REPO)\n", "print(\"Files to upload:\", len(payload_files))\n", "print(\"Upload size:\", f\"{payload_size_mb:.2f} MB\")\n", "\n", "# This operation uploads only the staging directory.\n", "# The encrypted source archive is never included.\n", "upload_result = api.upload_folder(\n", " folder_path=str(UPLOAD_DIR),\n", " repo_id=HF_REPO_ID,\n", " repo_type=HF_REPO_TYPE,\n", " path_in_repo=HF_PATH_IN_REPO,\n", " commit_message=(\n", " \"Upload Gemma 4 pre-encoded multimodal safetensors dataset\"\n", " ),\n", " token=HF_TOKEN,\n", ")\n", "\n", "print(\"\\nUpload operation returned successfully.\")\n", "print(\"Commit:\", upload_result)\n", "\n", "\n", "# ============================================================\n", "# VERIFY REMOTE UPLOAD\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"VERIFYING REMOTE HUGGING FACE FILES\")\n", "print(\"=\" * 75)\n", "\n", "remote_verified_count = None\n", "\n", "if VERIFY_REMOTE_UPLOAD:\n", "\n", " remote_files = api.list_repo_files(\n", " repo_id=HF_REPO_ID,\n", " repo_type=HF_REPO_TYPE,\n", " token=HF_TOKEN,\n", " )\n", "\n", " remote_prefix = (\n", " HF_PATH_IN_REPO.rstrip(\"/\") + \"/\"\n", " )\n", "\n", " remote_target_files = [\n", " path\n", " for path in remote_files\n", " if path.startswith(remote_prefix)\n", " ]\n", "\n", " remote_relative_files = {\n", " path[len(remote_prefix):]\n", " for path in remote_target_files\n", " }\n", "\n", " expected_relative_files = set(\n", " payload_relative_paths\n", " )\n", "\n", " missing_remote_files = (\n", " expected_relative_files - remote_relative_files\n", " )\n", "\n", " if missing_remote_files:\n", " raise RuntimeError(\n", " \"Remote verification found missing files:\\n\"\n", " + \"\\n\".join(sorted(missing_remote_files))\n", " )\n", "\n", " remote_safetensors = [\n", " path\n", " for path in remote_target_files\n", " if path.endswith(\".safetensors\")\n", " ]\n", "\n", " if len(remote_safetensors) != 1:\n", " raise RuntimeError(\n", " \"Expected exactly one remote safetensors file. \"\n", " f\"Found: {remote_safetensors}\"\n", " )\n", "\n", " remote_pt_files = [\n", " path\n", " for path in remote_target_files\n", " if path.endswith(\".pt\")\n", " ]\n", "\n", " if remote_pt_files:\n", " raise RuntimeError(\n", " \"Unexpected .pt files found in remote target directory:\\n\"\n", " + \"\\n\".join(remote_pt_files)\n", " )\n", "\n", " # Verify remote file sizes against local payload.\n", " remote_info = api.get_paths_info(\n", " repo_id=HF_REPO_ID,\n", " paths=[\n", " f\"{HF_PATH_IN_REPO}/{relative}\"\n", " for relative in payload_relative_paths\n", " ],\n", " repo_type=HF_REPO_TYPE,\n", " revision=\"main\",\n", " token=HF_TOKEN,\n", " )\n", "\n", " remote_size_by_path = {\n", " item.path: item.size\n", " for item in remote_info\n", " }\n", "\n", " for relative in payload_relative_paths:\n", "\n", " remote_path = (\n", " f\"{HF_PATH_IN_REPO}/{relative}\"\n", " )\n", "\n", " local_path = UPLOAD_DIR / relative\n", "\n", " expected_size = local_path.stat().st_size\n", "\n", " actual_size = remote_size_by_path.get(\n", " remote_path\n", " )\n", "\n", " if actual_size is None:\n", " raise RuntimeError(\n", " f\"Remote metadata missing for {remote_path}\"\n", " )\n", "\n", " if actual_size != expected_size:\n", " raise RuntimeError(\n", " f\"Remote size mismatch for {remote_path}: \"\n", " f\"local={expected_size}, remote={actual_size}\"\n", " )\n", "\n", " remote_verified_count = len(remote_safetensors)\n", "\n", " print(\"Remote verification passed.\")\n", " print(\"Expected files:\", len(expected_relative_files))\n", " print(\"Remote files in target:\", len(remote_target_files))\n", " print(\"Remote safetensors files:\", remote_verified_count)\n", " print(\"Remote .pt files: None\")\n", " print(\"Remote file sizes match local files.\")\n", "\n", "\n", "# ============================================================\n", "# FINAL SUMMARY\n", "# ============================================================\n", "\n", "print(\"\\n\" + \"=\" * 75)\n", "print(\"GEMMA 4 PRE-ENCODING AND UPLOAD COMPLETE\")\n", "print(\"=\" * 75)\n", "\n", "print(\"Base model:\", MODEL_ID)\n", "print(\"Source archive:\", SOURCE_FILENAME)\n", "print(\"Detected archive format:\", OUTER_ARCHIVE_FORMAT)\n", "\n", "print(\"Complete source triplets:\", len(complete_ids))\n", "print(\"Triplets processed with PCENC1:\", decrypted_count)\n", "print(\"Triplets processed as plaintext:\", plaintext_count)\n", "print(\"File processing failures:\", len(decryption_errors))\n", "\n", "print(\"Valid source examples:\", len(valid_examples))\n", "print(\"Successfully encoded:\", len(encoded_records))\n", "print(\"Encoding failures:\", len(encoding_errors))\n", "\n", "print(\"Verified examples:\", len(encoded_records))\n", "print(\"Verified tensors:\", verified_tensor_count)\n", "\n", "print(\"Local safetensors file:\", SAFETENSORS_PATH)\n", "\n", "print(\n", " \"Safetensors size:\",\n", " f\"{SAFETENSORS_PATH.stat().st_size / (1024**2):.2f} MB\",\n", ")\n", "\n", "print(\"Local output:\", OUTPUT_DIR)\n", "print(\"Hugging Face repository:\", HF_REPO_ID)\n", "print(\"Repository path:\", HF_PATH_IN_REPO)\n", "\n", "print(\"\\nUploaded content:\")\n", "print(\" - examples.safetensors\")\n", "print(\" - processor/ configuration and tokenizer assets\")\n", "print(\" - manifest.json\")\n", "\n", "print(\"\\nNot uploaded:\")\n", "print(\" - Original images\")\n", "print(\" - Original *_input.txt files\")\n", "print(\" - Original *_output.txt files\")\n", "print(\" - README.md\")\n", "print(\" - Encrypted source archive\")\n", "print(\" - Separate .pt files\")\n", "\n", "print(\"\\nRemote dataset location:\")\n", "\n", "print(\n", " f\"https://huggingface.co/{HF_REPO_ID}/tree/main/{HF_PATH_IN_REPO}\"\n", ")\n", "\n", "print(\"\\nAll successfully encoded examples are stored in one safetensors file.\")" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000, "referenced_widgets": [ "cdce483d10ed475c91691d0a223d9292", "3f88cec3260a4f10a7035f23f062e402", "82ded1b324dd428c8b61adcac5caafba", "ba1210281f9b451bad0ff2cedb345e91", "59baf10048e640d49bb383d245617536", "f9607b8a0c5d4e2d8ad91722137acc70", "62d4cf0a680f4bd18ba258743df2e8d0", "7a1100ec03014a08bad667af422f94ba", "24c083539f4b4b48bb46b0b3b6748e68", "cfbf37d1d40d4892819f7775c54eff29", "4baf986074084d8b879ca1548ab8a724", "6ff469f3f9be456c9a919607a6247068", "a6c8d0b608c3476db53f1ac328f1795c", "a05a6f32cc4b438a913a3d87a5c96d71", "4459e22d84da44638315f799e3fe2b80", "3aac8b491019422386b000e816ae8e65", "76a241c99c984162be9c8550e59bcc65", "6d1561f121c54231a79ed41d12ed4597", "dc99ae309a254505b579eee8d0ac5f4b", "d787e54676f84e77bcd573b46d856ea1", "28b50c5992f4467481dd20276f278185", "873e4ca0ba79432ba9640b86195bedff" ] }, "id": "dOa6n3NYjEIY", "outputId": "c9b331ef-cb98-4e8c-ab66-aed9d213553c" }, "id": "dOa6n3NYjEIY", "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "===========================================================================\n", "VERIFYING HUGGING FACE AUTHENTICATION\n", "===========================================================================\n", "HF token found.\n", "Source repository: codeShare/kromoss-lora-exports\n", "Destination repository: codeShare/kromoss-lora-exports\n", "Destination repository private: Yes\n", "\n", "===========================================================================\n", "DOWNLOADING SOURCE ARCHIVE\n", "===========================================================================\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "gemma_train_set.7z: reconstructing file: 0%| | 0.00B / 426MB " ], "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "cdce483d10ed475c91691d0a223d9292" } }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": [ "gemma_train_set.7z: downloading bytes: | 0.00B " ], "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "6ff469f3f9be456c9a919607a6247068" } }, "metadata": {} }, { "output_type": "stream", "name": "stdout", "text": [ "Downloaded: /content/gemma_encrypted_source/gemma_train_set.7z\n", "Archive size: 406.31 MB\n", "\n", "===========================================================================\n", "DETECTING OUTER ARCHIVE FORMAT\n", "===========================================================================\n", "First 16 bytes: 37 7a bc af 27 1c 00 04 e1 20 28 a6 e0 02 65 19\n", "Detected archive format: 7z\n", "\n", "===========================================================================\n", "EXTRACTING OUTER ARCHIVE\n", "===========================================================================\n", "Opening 7z archive...\n", "Archive entries: 8551\n", "Outer archive extraction complete.\n", "\n", "===========================================================================\n", "INDEXING EXTRACTED DATASET FILES\n", "===========================================================================\n", "Extracted files: 8551\n", "Images: 2850\n", "Inputs: 2850\n", "Outputs: 2850\n", "Complete triplets: 2850\n", "Missing images: 0\n", "Missing inputs: 0\n", "Missing outputs: 0\n", "\n", "Other files:\n", " processing_report.txt\n", "\n", "First three matched triplets:\n", "1 1.jpg 1_input.txt 1_output.txt\n", "2 2.jpg 2_input.txt 2_output.txt\n", "3 3.jpg 3_input.txt 3_output.txt\n", "\n", "===========================================================================\n", "PREPARING SOURCE DATASET DIRECTORY\n", "===========================================================================\n", "Dataset directory: /content/gemma_training_data\n", "\n", "===========================================================================\n", "PROCESSING INDIVIDUAL DATASET FILES\n", "===========================================================================\n", "[1/2850] Processed 1 | image=plaintext | input=plaintext | output=plaintext\n", "[2/2850] Processed 2 | image=plaintext | input=plaintext | output=plaintext\n", "[3/2850] Processed 3 | image=plaintext | input=plaintext | output=plaintext\n", "[4/2850] Processed 4 | image=plaintext | input=plaintext | output=plaintext\n", "[5/2850] Processed 5 | image=plaintext | input=plaintext | output=plaintext\n", "[6/2850] Processed 6 | image=plaintext | input=plaintext | output=plaintext\n", "[7/2850] Processed 7 | image=plaintext | input=plaintext | output=plaintext\n", "[8/2850] Processed 8 | image=plaintext | input=plaintext | output=plaintext\n", "[9/2850] Processed 9 | image=plaintext | input=plaintext | output=plaintext\n", "[10/2850] Processed 10 | image=plaintext | input=plaintext | output=plaintext\n", "[50/2850] Processed 50 | image=plaintext | input=plaintext | output=plaintext\n", "[100/2850] Processed 100 | image=plaintext | input=plaintext | output=plaintext\n", "[150/2850] Processed 150 | image=plaintext | input=plaintext | output=plaintext\n", "[200/2850] Processed 200 | image=plaintext | input=plaintext | output=plaintext\n", "[250/2850] Processed 250 | image=plaintext | input=plaintext | output=plaintext\n", "[300/2850] Processed 300 | image=plaintext | input=plaintext | output=plaintext\n" ] } ] } ], "metadata": { "colab": { "provenance": [] }, "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "name": "python" }, "widgets": { "application/vnd.jupyter.widget-state+json": { "version_major": 2, "version_minor": 0, "state": { "cdce483d10ed475c91691d0a223d9292": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_3f88cec3260a4f10a7035f23f062e402", "IPY_MODEL_82ded1b324dd428c8b61adcac5caafba", "IPY_MODEL_ba1210281f9b451bad0ff2cedb345e91" ], "layout": "IPY_MODEL_59baf10048e640d49bb383d245617536" } }, "3f88cec3260a4f10a7035f23f062e402": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_f9607b8a0c5d4e2d8ad91722137acc70", "placeholder": "​", "style": "IPY_MODEL_62d4cf0a680f4bd18ba258743df2e8d0", "value": "gemma_train_set.7z: reconstructing file: 100%" } }, "82ded1b324dd428c8b61adcac5caafba": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_7a1100ec03014a08bad667af422f94ba", "max": 426050352, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_24c083539f4b4b48bb46b0b3b6748e68", "value": 426050352 } }, "ba1210281f9b451bad0ff2cedb345e91": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_cfbf37d1d40d4892819f7775c54eff29", "placeholder": "​", "style": "IPY_MODEL_4baf986074084d8b879ca1548ab8a724", "value": "  426MB /  426MB, 37.0MB/s  " } }, "59baf10048e640d49bb383d245617536": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "f9607b8a0c5d4e2d8ad91722137acc70": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "62d4cf0a680f4bd18ba258743df2e8d0": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "7a1100ec03014a08bad667af422f94ba": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "24c083539f4b4b48bb46b0b3b6748e68": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "cfbf37d1d40d4892819f7775c54eff29": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "4baf986074084d8b879ca1548ab8a724": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "6ff469f3f9be456c9a919607a6247068": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_a6c8d0b608c3476db53f1ac328f1795c", "IPY_MODEL_a05a6f32cc4b438a913a3d87a5c96d71", "IPY_MODEL_4459e22d84da44638315f799e3fe2b80" ], "layout": "IPY_MODEL_3aac8b491019422386b000e816ae8e65" } }, "a6c8d0b608c3476db53f1ac328f1795c": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_76a241c99c984162be9c8550e59bcc65", "placeholder": "​", "style": "IPY_MODEL_6d1561f121c54231a79ed41d12ed4597", "value": "gemma_train_set.7z: downloading bytes: " } }, "a05a6f32cc4b438a913a3d87a5c96d71": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_dc99ae309a254505b579eee8d0ac5f4b", "max": 426050352, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_d787e54676f84e77bcd573b46d856ea1", "value": 426050352 } }, "4459e22d84da44638315f799e3fe2b80": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_28b50c5992f4467481dd20276f278185", "placeholder": "​", "style": "IPY_MODEL_873e4ca0ba79432ba9640b86195bedff", "value": "  426MB, 37.6MB/s  " } }, "3aac8b491019422386b000e816ae8e65": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "76a241c99c984162be9c8550e59bcc65": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "6d1561f121c54231a79ed41d12ed4597": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "dc99ae309a254505b579eee8d0ac5f4b": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d787e54676f84e77bcd573b46d856ea1": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "28b50c5992f4467481dd20276f278185": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "873e4ca0ba79432ba9640b86195bedff": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } } }, "nbformat": 4, "nbformat_minor": 5 }