{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [] }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" } }, "cells": [ { "cell_type": "markdown", "source": [ "# Sample Encrypted Dataset" ], "metadata": { "id": "DkphdgXC2gqu" } }, { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "c67a868d", "outputId": "275fe56f-3b31-451e-bcb7-da39dcd139d1" }, "source": [ "import os\n", "from google.colab import userdata\n", "import requests\n", "\n", "# Install py7zr and pyAesCrypt for extracting and decrypting the files\n", "!pip install -q py7zr pyAesCrypt\n", "\n", "# Retrieve HF_TOKEN securely from Colab secrets\n", "try:\n", " hf_token = userdata.get('HF_TOKEN')\n", " print(\"HF_TOKEN retrieved successfully.\")\n", "except Exception as e:\n", " print(f\"Error retrieving HF_TOKEN: {e}\")\n", " hf_token = None\n", "\n", "# Define paths and URLs\n", "url = 'https://huggingface.co/codeShare/kromoss-lora-exports/resolve/main/gemma_train_set.7z'\n", "local_7z_path = 'gemma_train_set.7z'\n", "extract_dir = 'extracted_dataset'\n", "\n", "# Download the file using requests with HF authentication\n", "headers = {'Authorization': f'Bearer {hf_token}'} if hf_token else {}\n", "print(\"Downloading file...\")\n", "r = requests.get(url, headers=headers, stream=True)\n", "if r.status_code == 200:\n", " with open(local_7z_path, 'wb') as f:\n", " for chunk in r.iter_content(chunk_size=8192):\n", " f.write(chunk)\n", " print(\"Download completed.\")\n", "else:\n", " print(f\"Failed to download. Status code: {r.status_code}. Response: {r.text[:500]}\")" ], "execution_count": 1, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "HF_TOKEN retrieved successfully.\n", "Downloading file...\n", "Download completed.\n" ] } ] }, { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "856003c2", "outputId": "60dcd7ae-b0b4-415a-e60f-23877b4fc716" }, "source": [ "import py7zr\n", "\n", "print(\"Extracting 7z archive with password...\")\n", "try:\n", " with py7zr.SevenZipFile(local_7z_path, mode='r', password='blueberry') as z:\n", " z.extractall(path=extract_dir)\n", " print(f\"Extraction completed to {extract_dir}/\")\n", "except Exception as e:\n", " print(f\"Extraction failed: {e}\")" ], "execution_count": 3, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Extracting 7z archive with password...\n", "Extraction completed to extracted_dataset/\n" ] } ] }, { "cell_type": "code", "source": [ "\n", "import os\n", "import re\n", "import glob\n", "import io\n", "\n", "from pathlib import Path\n", "from PIL import Image\n", "from IPython.display import display\n", "from cryptography.hazmat.primitives.ciphers.aead import AESGCM\n", "from cryptography.hazmat.primitives.kdf.scrypt import Scrypt\n", "\n", "# ============================================================\n", "# CONFIGURATION\n", "# ============================================================\n", "\n", "PASSWORD = \"blueberry\"\n", "PCENC_MAGIC = b\"PCENC1\"\n", "NUM_EXAMPLES = 20\n", "\n", "# ============================================================\n", "# DECRYPTION\n", "# ============================================================\n", "\n", "def decrypt_file_bytes(data, password=PASSWORD):\n", "\n", " if not data.startswith(PCENC_MAGIC):\n", " return data\n", "\n", " if len(data) < 50:\n", " raise ValueError(\"Invalid or truncated PCENC1 file.\")\n", "\n", " salt = data[6:22]\n", " nonce = data[22:34]\n", " ciphertext = data[34:]\n", "\n", " key = Scrypt(\n", " salt=salt,\n", " length=32,\n", " n=2**14,\n", " r=8,\n", " p=1,\n", " ).derive(password.encode(\"utf-8\"))\n", "\n", " return AESGCM(key).decrypt(\n", " nonce,\n", " ciphertext,\n", " PCENC_MAGIC\n", " )\n", "\n", "\n", "# ============================================================\n", "# FILE READERS\n", "# ============================================================\n", "\n", "def read_bytes(path):\n", "\n", " with open(path, \"rb\") as f:\n", " data = f.read()\n", "\n", " return decrypt_file_bytes(data)\n", "\n", "\n", "def read_text(path):\n", "\n", " return read_bytes(path).decode(\n", " \"utf-8-sig\"\n", " ).strip()\n", "\n", "\n", "# ============================================================\n", "# FIND NUMERIC IMAGE FILES\n", "# ============================================================\n", "\n", "image_files = glob.glob(\n", " os.path.join(extract_dir, \"*.jpg\")\n", ")\n", "\n", "image_files = sorted(\n", " image_files,\n", " key=lambda p: int(\n", " Path(p).stem\n", " ) if Path(p).stem.isdigit() else float(\"inf\")\n", ")\n", "\n", "print(\"Total images:\", len(image_files))\n", "print(\"Displaying first\", NUM_EXAMPLES, \"examples\")\n", "\n", "\n", "# ============================================================\n", "# DISPLAY FIRST 20 EXAMPLES\n", "# ============================================================\n", "\n", "display_count = min(\n", " NUM_EXAMPLES,\n", " len(image_files)\n", ")\n", "\n", "for index, image_path in enumerate(\n", " image_files[:display_count],\n", " start=1\n", "):\n", "\n", " item_id = Path(image_path).stem\n", "\n", " input_path = os.path.join(\n", " extract_dir,\n", " f\"{item_id}_input.txt\"\n", " )\n", "\n", " output_path = os.path.join(\n", " extract_dir,\n", " f\"{item_id}_output.txt\"\n", " )\n", "\n", " print(\"\\n\" + \"=\" * 90)\n", " print(f\"DATASET EXAMPLE {index}/{display_count} — ID {item_id}\")\n", " print(\"=\" * 90)\n", "\n", " # --------------------------------------------------------\n", " # INPUT\n", " # --------------------------------------------------------\n", "\n", " if not os.path.isfile(input_path):\n", " raise FileNotFoundError(input_path)\n", "\n", " input_text = read_text(input_path)\n", "\n", " print(\"\\nINPUT:\")\n", " print(input_text)\n", "\n", " # --------------------------------------------------------\n", " # OUTPUT\n", " # --------------------------------------------------------\n", "\n", " if not os.path.isfile(output_path):\n", " raise FileNotFoundError(output_path)\n", "\n", " output_text = read_text(output_path)\n", "\n", " print(\"\\nOUTPUT:\")\n", " print(output_text)\n", "\n", " # --------------------------------------------------------\n", " # IMAGE\n", " # --------------------------------------------------------\n", "\n", " image_bytes = read_bytes(image_path)\n", "\n", " image = Image.open(\n", " io.BytesIO(image_bytes)\n", " ).convert(\"RGB\")\n", "\n", " print(\"\\nIMAGE:\")\n", " display(image.resize((350, 350)))\n", "\n", "print(\"\\nSuccessfully displayed\", display_count, \"examples.\")\n" ], "metadata": { "id": "ASorxaSJ2A41" }, "execution_count": null, "outputs": [] } ] }