diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..e28bcb7d2a975836409ca8d7e8c975ba64e20af7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +images/augmentoolkit-logo.png filter=lfs diff=lfs merge=lfs -text diff --git a/huey_config.py b/huey_config.py new file mode 100644 index 0000000000000000000000000000000000000000..a609b42b569e0bea701a60a98aef0fefddd5ee95 --- /dev/null +++ b/huey_config.py @@ -0,0 +1,21 @@ +import os + +# Import RedisHuey and the necessary connection details from redis_config +from huey import RedisHuey +from redis_config import REDIS_HOST, REDIS_PORT, REDIS_DB + +# Define the Huey instance using Redis +# The 'name' parameter is optional but good practice, often set to the app name. +# It prefixes keys used by Huey in Redis to avoid collisions if Redis is shared. +huey = RedisHuey( + name="augmentoolkit_tasks", # Optional, helps namespace Huey keys in Redis + host=REDIS_HOST, + port=REDIS_PORT, + db=REDIS_DB, + # Add other RedisHuey parameters if needed (e.g., password, connection_pool) +) + +# Remove SQLite specific code +# os.makedirs("huey") +# _db_path = os.path.join(os.path.dirname(__file__), 'huey', 'huey_queue.db') +# huey = SqliteHuey(filename=_db_path) diff --git a/images/augmentoolkit-logo.png b/images/augmentoolkit-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..a6a03485d2dac348ef9269a2434d6745b849a6a8 --- /dev/null +++ b/images/augmentoolkit-logo.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a744037cb082826f037ee4c0066435c8ab91d62e2ce00026098e72ca7b95361 +size 237274 diff --git a/linux.sh b/linux.sh new file mode 100644 index 0000000000000000000000000000000000000000..bff47f52817a8d1e5017530af0fa3a702b05bf2d --- /dev/null +++ b/linux.sh @@ -0,0 +1,614 @@ +#!/bin/bash + +# Exit immediately if a command exits with a non-zero status. +# set -e # Disabled for manual error checking below + +echo "Starting Augmentoolkit services..." + +# --- Get script directory and change to it --- +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd "$SCRIPT_DIR" +if [ $? -ne 0 ]; then + echo "ERROR: Failed to change directory to script location '$SCRIPT_DIR'." + exit 1 +fi +echo "Running in directory: $(pwd)" + +# --- Cleanup Function --- +# Stores PIDs of background processes +HUEY_PID="" +UVICORN_PID="" +SERVE_PID="" # Though serve runs in foreground, trap might catch signals before it starts +VALKEY_PID="" # PID for valkey server + +cleanup() { + echo "" # Newline after ^C + echo "----------------------------------------" + echo "Signal received, stopping background services..." + + # Function to kill a process and wait for it to terminate + kill_and_wait() { + local pid=$1 + local name=$2 + local timeout=9 + + if [ -n "$pid" ] && kill -0 $pid &> /dev/null; then + echo "Stopping $name (PID $pid)..." + + # First try SIGTERM (graceful shutdown) + kill -TERM $pid 2>/dev/null + + # Wait for process to terminate gracefully + local count=0 + while [ $count -lt $timeout ] && kill -0 $pid &> /dev/null; do + sleep 1 + count=$((count + 1)) + done + + # If still running, force kill with SIGKILL + if kill -0 $pid &> /dev/null; then + echo "Process $pid ($name) didn't terminate gracefully, force killing..." + kill -KILL $pid 2>/dev/null + sleep 1 + fi + + # Final check + if kill -0 $pid &> /dev/null; then + echo "WARNING: Failed to stop $name (PID $pid)" + else + echo "$name stopped successfully." + fi + fi + } + + # Stop processes in reverse order of startup + kill_and_wait "$SERVE_PID" "frontend serve" + kill_and_wait "$UVICORN_PID" "Uvicorn API server" + kill_and_wait "$HUEY_PID" "Huey worker" + kill_and_wait "$VALKEY_PID" "Valkey server" + + # Find and kill any remaining huey consumers specific to this Augmentoolkit instance + echo "Searching for huey consumer processes in current directory ($SCRIPT_DIR)..." + + # Use ps with full command line and grep for huey_consumer processes in our specific directory + # This avoids killing huey consumers from other Augmentoolkit instances or projects + HUEY_PIDS=$(ps aux | grep -E "huey_consumer.*tasks\.huey" | grep "$SCRIPT_DIR" | grep -v grep | awk '{print $2}') + + if [ -n "$HUEY_PIDS" ]; then + echo "Found huey consumer processes in current directory: $HUEY_PIDS" + for pid in $HUEY_PIDS; do + if kill -0 $pid &> /dev/null; then + echo "Stopping huey consumer from current directory (PID $pid)..." + kill -TERM $pid 2>/dev/null + sleep 2 + if kill -0 $pid &> /dev/null; then + echo "Force killing huey consumer (PID $pid)..." + kill -KILL $pid 2>/dev/null + fi + fi + done + sleep 1 + # Final check for any remaining huey processes in our directory + REMAINING_HUEY=$(ps aux | grep -E "huey_consumer.*tasks\.huey" | grep "$SCRIPT_DIR" | grep -v grep | awk '{print $2}') + if [ -n "$REMAINING_HUEY" ]; then + echo "WARNING: Some huey consumer processes in current directory may still be running: $REMAINING_HUEY" + else + echo "All huey consumer processes from current directory stopped successfully." + fi + else + echo "No additional huey consumer processes found in current directory." + fi + + # Deactivate virtual environment if active + if command -v deactivate &> /dev/null; then + echo "Deactivating virtual environment..." + deactivate + fi + + # Give a moment for any remaining output to flush + sleep 1 + + # Clear any remaining output and reset cursor + echo "Cleanup complete." + echo "----------------------------------------" + + # Ensure cursor is visible and terminal is in normal state + printf "\033[?25h" # Show cursor + printf "\033[0m" # Reset all attributes + + exit 0 # Exit script after cleanup +} + +# Trap signals (INT: Ctrl+C, TERM: standard kill) and run cleanup function +trap cleanup SIGINT SIGTERM + +# --- 1. Python Virtual Environment & Dependencies --- +echo "----------------------------------------" +echo "1. Setting up Python virtual environment and installing dependencies..." +VENV_DIR=".venv" + +# Check if python3.11 command exists (preferred) +if command -v python3.11 &> /dev/null; then + PYTHON_CMD="python3.11" + echo "Using Python 3.11: $(python3.11 --version)" +elif command -v python3 &> /dev/null; then + PYTHON_CMD="python3" + PYTHON_VERSION=$(python3 --version 2>&1 | grep -oE '[0-9]+\.[0-9]+') + echo "Using Python: $(python3 --version)" + if [[ "$PYTHON_VERSION" != "3.11" ]]; then + echo "WARNING: Python 3.11 is recommended but not found. Using $PYTHON_VERSION instead." + echo "If you encounter issues, please install Python 3.11 specifically." + fi +else + echo "ERROR: Neither python3.11 nor python3 command found." + echo "Please install Python 3.11 (recommended) or Python 3.x and rerun the script." + echo "On Ubuntu/Debian, you can install Python 3.11 using:" + echo " sudo apt update && sudo apt install python3.11 python3.11-venv python3.11-dev" + echo "On RHEL/CentOS/Fedora, you can install Python 3.11 using:" + echo " sudo dnf install python3.11 python3.11-devel" + echo "Or download from: https://www.python.org/downloads/" + exit 1 +fi + +# Create virtual environment if it doesn't exist +if [ ! -d "$VENV_DIR" ]; then + echo "Creating virtual environment in '$VENV_DIR' using $PYTHON_CMD..." + $PYTHON_CMD -m venv "$VENV_DIR" + if [ $? -ne 0 ]; then + echo "ERROR: Failed to create Python virtual environment using $PYTHON_CMD." + exit 1 + fi +else + echo "Virtual environment '$VENV_DIR' already exists." +fi + +# Activate virtual environment +echo "Activating virtual environment..." +source "$VENV_DIR/bin/activate" +if [ $? -ne 0 ]; then + echo "ERROR: Failed to activate Python virtual environment." + exit 1 +fi +echo "Virtual environment activated. Python executable: $(which python)" + +# --- Check/Install uv --- +echo "Checking for uv..." +if ! command -v uv &> /dev/null; then + echo "'uv' command not found. Attempting to install uv using pip..." + # Ensure pip is available (it should be with the selected Python version) + if ! python -m pip --version &> /dev/null; then + echo "ERROR: python -m pip command failed. Cannot install uv." + exit 1 + fi + python -m pip install uv + if [ $? -ne 0 ]; then + echo "ERROR: Failed to install uv using pip. Please install uv manually (e.g., 'pip install uv') and rerun the script." + exit 1 + fi + echo "uv installed successfully." +else + echo "uv found." +fi +# --- End Check/Install uv --- + + +# Install dependencies using uv +echo "Installing Python dependencies from requirements.txt using uv..." +# Assuming uv is in PATH after the check above and venv is activated +uv pip install -r requirements.txt +if [ $? -ne 0 ]; then + echo "ERROR: Failed to install Python dependencies using uv. Please check requirements.txt and uv logs." + exit 1 # Cleanup trap will handle exit +fi +echo "Python dependencies installed successfully." + +# Download spaCy model for anonymization +echo "Downloading spaCy model (en_core_web_lg) for anonymization..." +python -m spacy download en_core_web_lg +if [ $? -ne 0 ]; then + echo "WARNING: Failed to download spaCy model. Anonymization might fall back to NLTK." +fi + + +# --- Helper Process Logging Setup --- +echo "----------------------------------------" +echo "Setting up helper process logging..." + +# Create helper_process_logs directory if it doesn't exist +HELPER_LOGS_DIR="helper_process_logs" +if [ ! -d "$HELPER_LOGS_DIR" ]; then + echo "Creating helper process logs directory: $HELPER_LOGS_DIR" + mkdir -p "$HELPER_LOGS_DIR" + if [ $? -ne 0 ]; then + echo "ERROR: Failed to create helper process logs directory." + exit 1 + fi +else + echo "Helper process logs directory already exists: $HELPER_LOGS_DIR" +fi + +# Generate timestamp for log files +TIMESTAMP=$(date +"%Y%m%d_%H%M%S") +VALKEY_LOG_FILE="$HELPER_LOGS_DIR/valkey_server_${TIMESTAMP}.log" +HUEY_LOG_FILE="$HELPER_LOGS_DIR/huey_worker_${TIMESTAMP}.log" +UVICORN_LOG_FILE="$HELPER_LOGS_DIR/uvicorn_api_${TIMESTAMP}.log" +FRONTEND_LOG_FILE="$HELPER_LOGS_DIR/frontend_serve_${TIMESTAMP}.log" + +echo "Helper process logs will be saved to:" +echo " Valkey server: $VALKEY_LOG_FILE" +echo " Huey worker: $HUEY_LOG_FILE" +echo " Uvicorn API: $UVICORN_LOG_FILE" +echo " Frontend serve: $FRONTEND_LOG_FILE" +echo "Helper process logging setup complete." + +#### if REDIS_HOST skip +if [ -z "$REDIS_HOST" ]; then + # --- 1.5 Valkey/Redis Check --- + echo "----------------------------------------" + echo "1.5. Checking for running Valkey/Redis service..." + + SERVER_CMD="" + SERVICE_NAME="" + + # Check for valkey-server first + if command -v valkey-server &> /dev/null; then + SERVER_CMD="valkey-server" + SERVICE_NAME="valkey-server" + # Fallback to redis-server + elif command -v redis-server &> /dev/null; then + SERVER_CMD="redis-server" + SERVICE_NAME="redis-server" + else + # No background services started yet, just exit + # Check if something is listening on the default Valkey/Redis port (6379) + if command -v ss &> /dev/null; then + PORT_CHECK_CMD="ss -ltn" + else + PORT_CHECK_CMD="netstat -ltn" + fi + + if $PORT_CHECK_CMD | grep -q ':6379'; then + echo "WARNING: Neither 'valkey-server' nor 'redis-server' found in PATH, but something is listening on port 6379." + echo "Assuming a compatible Valkey/Redis server is running (possibly built from source or custom install)." + else + echo "Neither 'valkey-server' nor 'redis-server' found in PATH." + echo "Building Valkey from source..." + + # Check if valkey directory already exists + if [ ! -d "valkey" ]; then + echo "Cloning Valkey repository..." + git clone https://github.com/valkey-io/valkey.git + if [ $? -ne 0 ]; then + echo "ERROR: Failed to clone Valkey repository. Please check your internet connection and git installation." + exit 1 + fi + else + echo "Valkey directory already exists, skipping clone." + fi + + echo "Building Valkey..." + cd valkey + make + if [ $? -ne 0 ]; then + echo "ERROR: Failed to build Valkey. Please check build dependencies (gcc, make, etc.)." + cd "$SCRIPT_DIR" + exit 1 + fi + + echo "Starting Valkey server in background..." + echo "Output will be logged to: $VALKEY_LOG_FILE" + cd src + ./valkey-server > "../../$VALKEY_LOG_FILE" 2>&1 & + VALKEY_PID=$! + cd "$SCRIPT_DIR" + + # Give it a moment to start/fail + sleep 4 + if ! kill -0 $VALKEY_PID &> /dev/null; then + echo "ERROR: Valkey server (PID $VALKEY_PID) failed to start or crashed immediately." + echo "Check the log file for details: $VALKEY_LOG_FILE" + exit 1 + fi + echo "Valkey server started successfully with PID $VALKEY_PID." + + # Set SERVER_CMD and SERVICE_NAME for consistency with the rest of the script + SERVER_CMD="./valkey/src/valkey-server" + SERVICE_NAME="valkey-server" + fi + fi + echo "Found $SERVER_CMD." + + # If we didn't just start Valkey server locally, perform service checks + if [ -z "$VALKEY_PID" ]; then + # Check if systemctl exists and is usable + if command -v systemctl &> /dev/null && systemctl status &> /dev/null; then + # systemctl seems to be working, so let's use it + if ! systemctl is-active --quiet $SERVICE_NAME; then + echo "WARNING: The $SERVICE_NAME service is not active according to systemctl." + # Check if something is listening on port 6379 as fallback + if ss -ltn | grep -q ':6379' 2>/dev/null; then + echo "However, something is listening on port 6379. Assuming a compatible Valkey/Redis server is running." + else + echo "ERROR: The $SERVICE_NAME service is not active and nothing is listening on port 6379." + echo "Please start the service, e.g., 'sudo systemctl start $SERVICE_NAME' and ensure it's enabled to start on boot, e.g., 'sudo systemctl enable $SERVICE_NAME'." + exit 1 + fi + else + echo "The $SERVICE_NAME service is active." + fi + else + # systemctl not available or not working, fall back to simple port check + echo "WARNING: 'systemctl' not available/running. Checking for service via port." + if ss -ltn | grep -q ':6379' 2>/dev/null; then + echo "A service is listening on port 6379. Assuming a compatible Valkey/Redis server is running." + else + echo "ERROR: No running Valkey/Redis service detected on port 6379, and could not start one automatically as it was expected to be running." + echo "Please start valkey-server or redis-server manually in another terminal." + exit 1 + fi + fi + else + echo "Valkey server was started by this script (PID $VALKEY_PID), skipping system service check." + fi + # --- End Valkey/Redis Check --- +fi + +# --- 2. Huey Worker --- +echo "----------------------------------------" +echo "2. Starting Huey worker in the background (using venv)..." +echo "Output will be logged to: $HUEY_LOG_FILE" +# Call the huey_consumer script directly (should be in PATH from venv) +# Redirect both stdout and stderr to the log file +huey_consumer tasks.huey > "$HUEY_LOG_FILE" 2>&1 & +HUEY_PID=$! +# Give it a moment to start/fail +sleep 2 +if ! kill -0 $HUEY_PID &> /dev/null; then + echo "ERROR: Huey worker (PID $HUEY_PID) failed to start or crashed immediately." + echo "Check the log file for details: $HUEY_LOG_FILE" + exit 1 # Cleanup trap will handle exit +fi +echo "Huey worker started successfully with PID $HUEY_PID." + + +# --- 3. API Server --- +echo "----------------------------------------" +echo "3. Starting FastAPI server (Uvicorn) in the background (using venv)..." +echo "Output will be logged to: $UVICORN_LOG_FILE" +# Host 0.0.0.0 makes it accessible on the network +# Use python -m uvicorn to ensure using the venv uvicorn +# Redirect both stdout and stderr to the log file +python -m uvicorn api:app --host 0.0.0.0 --port 8000 > "$UVICORN_LOG_FILE" 2>&1 & +UVICORN_PID=$! +# Give it a moment to start/fail +sleep 2 +if ! kill -0 $UVICORN_PID &> /dev/null; then + echo "ERROR: Uvicorn server (PID $UVICORN_PID) failed to start or crashed immediately." + echo "Check the log file for details: $UVICORN_LOG_FILE" + # Cleanup trap will handle stopping Huey + exit 1 # Cleanup trap will handle exit +fi +echo "Uvicorn server started successfully with PID $UVICORN_PID. Access API at http://localhost:8000" + + +# --- 4. NPM Check --- +echo "----------------------------------------" +echo "4. Checking for npm..." +if ! command -v npm &> /dev/null; then + echo "ERROR: npm command not found." + echo "Please install Node.js and npm (e.g., via your system package manager like 'sudo apt install nodejs npm' or better still, install nvm and use that to get node (ask an AI how to get nodejs via nvm)) and run this script again." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + exit 1 +fi +echo "npm found." + +# Check Node.js version +NODE_VERSION=$(node --version 2>/dev/null | sed 's/v//') +if [ -z "$NODE_VERSION" ]; then + echo "ERROR: Could not determine Node.js version." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + exit 1 +fi + +# Extract major version number +NODE_MAJOR_VERSION=$(echo "$NODE_VERSION" | cut -d. -f1) + +echo "Node.js version: v$NODE_VERSION" + +# Check if Node.js version is at least 18 +if [ "$NODE_MAJOR_VERSION" -lt 18 ]; then + echo "ERROR: Node.js version $NODE_VERSION is too old." + echo "This application requires Node.js 18.0.0 or higher (20+ recommended)." + echo "" + echo "To upgrade Node.js, run the following commands:" + echo "" + echo "1. Add NodeSource repository:" + echo " curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash -" + echo "" + echo "2. Remove conflicting packages (if any):" + echo " sudo apt-get remove -y libnode-dev libnode72" + echo "" + echo "3. Install Node.js 20.x:" + echo " sudo apt-get install -y nodejs" + echo "" + echo " OR run all steps at once:" + echo " curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - && sudo apt-get remove -y libnode-dev libnode72 2>/dev/null || true && sudo apt-get install -y nodejs" + echo "" + echo "Alternative method using Node Version Manager (nvm):" + echo " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash" + echo " source ~/.bashrc" + echo " nvm install 20" + echo " nvm use 20" + echo "" + echo "After upgrading, please run this script again." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + exit 1 +fi + + +# --- 5. Frontend Dependencies --- +echo "----------------------------------------" +echo "5. Installing frontend dependencies in atk-interface..." +if [ ! -d "atk-interface" ]; then + echo "ERROR: Directory 'atk-interface' not found in $(pwd)." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + exit 1 +fi +cd atk-interface +npm install +if [ $? -ne 0 ]; then + echo "ERROR: Failed to install npm dependencies in atk-interface. Check package.json and npm logs." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + cd "$SCRIPT_DIR" # Go back to root + exit 1 +fi +echo "Frontend dependencies installed." + + +# --- 6. Frontend Build --- +echo "----------------------------------------" +echo "6. Building frontend application in atk-interface..." +npm run build +if [ $? -ne 0 ]; then + echo "ERROR: Failed to build frontend application. Check build script in package.json and logs." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + cd "$SCRIPT_DIR" # Go back to root + exit 1 +fi +echo "Frontend built successfully." + + +# --- 7. Frontend Serve --- +echo "----------------------------------------" +echo "7. Serving frontend application from atk-interface/dist/ in the FOREGROUND..." +echo "Output will be logged to: $FRONTEND_LOG_FILE" +# Ensure dist directory exists after build +if [ ! -d "dist" ]; then + echo "ERROR: 'dist' directory not found in atk-interface after build." + # Cleanup trap will handle stopping background services + exit 1 +fi + + +# Show summary of all log files +echo "" +echo "═══════════════════════════════════════════════════════════════════════════════" +echo "All helper processes are now running with output redirected to log files:" +if [ -n "$VALKEY_PID" ]; then +echo " • Valkey server logs: $VALKEY_LOG_FILE" +fi +echo " • Huey worker logs: $HUEY_LOG_FILE" +echo " • Uvicorn API logs: $UVICORN_LOG_FILE" +echo " • Frontend serve logs: $FRONTEND_LOG_FILE" +echo "" +echo "You can monitor these logs in real-time with:" +if [ -n "$VALKEY_PID" ]; then +echo " tail -f $VALKEY_LOG_FILE" +fi +echo " tail -f $HUEY_LOG_FILE" +echo " tail -f $UVICORN_LOG_FILE" +echo " tail -f $FRONTEND_LOG_FILE" +echo "═══════════════════════════════════════════════════════════════════════════════" +echo "" + +# Serve on port 5173 (matches CORS config) +# Run npx serve in the background first, then open browser once it's ready +echo "Starting serve process (output redirected to log file)..." +# Redirect both stdout and stderr to the log file - use ../ to go back to script directory +npx serve -s dist --listen 5173 > "../$FRONTEND_LOG_FILE" 2>&1 & +SERVE_PID=$! + +# Wait for the serve process to be ready +echo "Waiting for frontend server to start..." +sleep 3 + +# Check if serve process is still running +if ! kill -0 $SERVE_PID &> /dev/null; then + echo "ERROR: Frontend serve process failed to start or crashed immediately." + echo "Check the log file for details: $FRONTEND_LOG_FILE" + cleanup + exit 1 +fi + +# Wait for the server to actually be listening on port 5173 +echo "Checking if server is responding..." +for i in {1..10}; do + if command -v curl &> /dev/null; then + if curl -s http://localhost:5173 > /dev/null 2>&1; then + break + fi + elif command -v wget &> /dev/null; then + if wget -q --spider http://localhost:5173 > /dev/null 2>&1; then + break + fi + else + # Fallback: just check if something is listening on port 5173 + if command -v ss &> /dev/null; then + if ss -ltn | grep -q ':5173'; then + break + fi + elif command -v netstat &> /dev/null; then + if netstat -ltn | grep -q ':5173'; then + break + fi + else + # No way to check, just wait a bit more + sleep 1 + break + fi + fi + sleep 1 +done + +echo "Opening interface..." + +# --- Check for xdg-open --- +echo "Checking for xdg-open..." +if ! command -v xdg-open &> /dev/null; then + echo "WARNING: 'xdg-open' command not found. Cannot automatically open browser." + echo "Please install xdg-utils (e.g., 'sudo apt install xdg-utils' or 'sudo dnf install xdg-utils') or open http://localhost:5173 manually." + # Not exiting, just warning +else + echo "xdg-open found. Opening http://localhost:5173 in default browser... (if this fails, open http://localhost:5173 in a web browser of your choice)" + xdg-open http://localhost:5173 +fi + +echo "Frontend server is now running. Press Ctrl+C to stop all services." + +# Wait for the serve process to finish (it's now running in the background) +wait $SERVE_PID +SERVE_EXIT_CODE=$? +SERVE_PID="" # Clear PID as it has exited + +if [ $SERVE_EXIT_CODE -ne 0 ]; then + echo "WARNING: Frontend serve process exited with code $SERVE_EXIT_CODE." + echo "Check the log file for details: $FRONTEND_LOG_FILE" + # Consider if we should stop background services here if serve crashes + # Calling cleanup manually if serve fails + cleanup + exit 1 # Exit with error code +fi + + +# --- 8. Done (Only reached if npx serve exits gracefully, e.g., if killed separately) --- +cd "$SCRIPT_DIR" # Go back to root directory +echo "----------------------------------------" +echo "Frontend serve process finished." +echo "" +echo "Helper process log files were saved to:" +if [ -n "$VALKEY_PID" ]; then +echo " • Valkey server logs: $VALKEY_LOG_FILE" +fi +echo " • Huey worker logs: $HUEY_LOG_FILE" +echo " • Uvicorn API logs: $UVICORN_LOG_FILE" +echo " • Frontend serve logs: $FRONTEND_LOG_FILE" +# Call cleanup to stop background services if serve finishes normally +cleanup + +exit 0 diff --git a/local_linux.sh b/local_linux.sh new file mode 100644 index 0000000000000000000000000000000000000000..4e3808176da1124ef135ec836c82dca3935b77d0 --- /dev/null +++ b/local_linux.sh @@ -0,0 +1,738 @@ +#!/bin/bash + +# Exit immediately if a command exits with a non-zero status. +# set -e # Disabled for manual error checking below + +# Parse command line arguments +MODEL_TYPE="normal" +TENSOR_PARALLELISM=1 + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --tensor-parallelism) + TENSOR_PARALLELISM="$2" + shift 2 + ;; + -h|--help) + echo "Usage: $0 [MODEL_TYPE] [OPTIONS]" + echo "" + echo "MODEL_TYPE options:" + echo " normal - Use default model (Heralax/Augmentoolkit-DataSpecialist-v0.1)" + echo " small - Use quantized model (Heralax/Augmentoolkit-DataSpecialist-gptqmodel-4bit)" + echo " path/to/custom/model - Use custom model path" + echo "" + echo "OPTIONS:" + echo " --tensor-parallelism INT - Number of tensor parallel processes for vLLM (default: 1)" + echo "" + echo "Examples:" + echo " $0 # Use normal model (default)" + echo " $0 small # Use quantized model" + echo " $0 deepseek-ai/DeepSeek-R1 # Use custom model" + echo " $0 normal --tensor-parallelism 2 # Use normal model with 2 tensor parallel processes" + echo " $0 --tensor-parallelism 4 # Use default model with 4 tensor parallel processes" + echo "" + exit 0 + ;; + -*) + echo "Unknown option: $1" + echo "Use --help for usage information." + exit 1 + ;; + *) + # This is the MODEL_TYPE (positional argument) + if [ "$MODEL_TYPE" = "normal" ]; then + MODEL_TYPE="$1" + else + echo "Error: Multiple model types specified" + echo "Use --help for usage information." + exit 1 + fi + shift + ;; + esac +done + +# Validate tensor parallelism value +if ! [[ "$TENSOR_PARALLELISM" =~ ^[0-9]+$ ]] || [ "$TENSOR_PARALLELISM" -lt 1 ]; then + echo "Error: --tensor-parallelism must be a positive integer (got: $TENSOR_PARALLELISM)" + exit 1 +fi + +echo "Starting Augmentoolkit services with model type: $MODEL_TYPE" +echo "Tensor parallelism: $TENSOR_PARALLELISM" +echo "(Use '$0 --help' to see available model options)" + +# --- Get script directory and change to it --- +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd "$SCRIPT_DIR" +if [ $? -ne 0 ]; then + echo "ERROR: Failed to change directory to script location '$SCRIPT_DIR'." + exit 1 +fi +echo "Running in directory: $(pwd)" + +# --- Cleanup Function --- +# Stores PIDs of background processes +HUEY_PID="" +UVICORN_PID="" +SERVE_PID="" # Though serve runs in foreground, trap might catch signals before it starts +LLAMA_SERVER_PID="" # PID for llama.cpp server +VALKEY_PID="" # PID for valkey server + +cleanup() { + echo "" # Newline after ^C + echo "----------------------------------------" + echo "Signal received, stopping background services..." + + # Function to kill a process and wait for it to terminate + kill_and_wait() { + local pid=$1 + local name=$2 + local timeout=9 + + if [ -n "$pid" ] && kill -0 $pid &> /dev/null; then + echo "Stopping $name (PID $pid)..." + + # First try SIGTERM (graceful shutdown) + kill -TERM $pid 2>/dev/null + + # Wait for process to terminate gracefully + local count=0 + while [ $count -lt $timeout ] && kill -0 $pid &> /dev/null; do + sleep 1 + count=$((count + 1)) + done + + # If still running, force kill with SIGKILL + if kill -0 $pid &> /dev/null; then + echo "Process $pid ($name) didn't terminate gracefully, force killing..." + kill -KILL $pid 2>/dev/null + sleep 1 + fi + + # Final check + if kill -0 $pid &> /dev/null; then + echo "WARNING: Failed to stop $name (PID $pid)" + else + echo "$name stopped successfully." + fi + fi + } + + # Stop processes in reverse order of startup + kill_and_wait "$SERVE_PID" "frontend serve" + kill_and_wait "$UVICORN_PID" "Uvicorn API server" + kill_and_wait "$HUEY_PID" "Huey worker" + kill_and_wait "$LLAMA_SERVER_PID" "VLLM server" + kill_and_wait "$VALKEY_PID" "Valkey server" + + # Find and kill any remaining huey consumers specific to this Augmentoolkit instance + echo "Searching for huey consumer processes in current directory ($SCRIPT_DIR)..." + + # Use ps with full command line and grep for huey_consumer processes in our specific directory + # This avoids killing huey consumers from other Augmentoolkit instances or projects + HUEY_PIDS=$(ps aux | grep -E "huey_consumer.*tasks\.huey" | grep "$SCRIPT_DIR" | grep -v grep | awk '{print $2}') + + if [ -n "$HUEY_PIDS" ]; then + echo "Found huey consumer processes in current directory: $HUEY_PIDS" + for pid in $HUEY_PIDS; do + if kill -0 $pid &> /dev/null; then + echo "Stopping huey consumer from current directory (PID $pid)..." + kill -TERM $pid 2>/dev/null + sleep 2 + if kill -0 $pid &> /dev/null; then + echo "Force killing huey consumer (PID $pid)..." + kill -KILL $pid 2>/dev/null + fi + fi + done + sleep 1 + # Final check for any remaining huey processes in our directory + REMAINING_HUEY=$(ps aux | grep -E "huey_consumer.*tasks\.huey" | grep "$SCRIPT_DIR" | grep -v grep | awk '{print $2}') + if [ -n "$REMAINING_HUEY" ]; then + echo "WARNING: Some huey consumer processes in current directory may still be running: $REMAINING_HUEY" + else + echo "All huey consumer processes from current directory stopped successfully." + fi + else + echo "No additional huey consumer processes found in current directory." + fi + + # Deactivate virtual environment if active + if command -v deactivate &> /dev/null; then + echo "Deactivating virtual environment..." + deactivate + fi + + # Give a moment for any remaining output to flush + sleep 1 + + # Clear any remaining output and reset cursor + echo "Cleanup complete." + echo "----------------------------------------" + + # Ensure cursor is visible and terminal is in normal state + printf "\033[?25h" # Show cursor + printf "\033[0m" # Reset all attributes + + echo "" # Add extra newline for visual clarity before prompt returns +} + +# Trap signals (INT: Ctrl+C, TERM: standard kill) and run cleanup function +trap cleanup SIGINT SIGTERM + +# --- 1. Python Virtual Environment & Dependencies --- +echo "----------------------------------------" +echo "1 / 9. Setting up Python virtual environment and installing dependencies..." +VENV_DIR=".venv" + +# Check if python3.11 command exists (preferred) +if command -v python3.11 &> /dev/null; then + PYTHON_CMD="python3.11" + echo "Using Python 3.11: $(python3.11 --version)" +elif command -v python3 &> /dev/null; then + PYTHON_CMD="python3" + PYTHON_VERSION=$(python3 --version 2>&1 | grep -oE '[0-9]+\.[0-9]+') + echo "Using Python: $(python3 --version)" + if [[ "$PYTHON_VERSION" != "3.11" ]]; then + echo "WARNING: Python 3.11 is recommended but not found. Using $PYTHON_VERSION instead." + echo "If you encounter issues, please install Python 3.11 specifically." + fi +else + echo "ERROR: Neither python3.11 nor python3 command found." + echo "Please install Python 3.11 (recommended) or Python 3.x and rerun the script." + echo "On Ubuntu/Debian, you can install Python 3.11 using:" + echo " sudo apt update && sudo apt install python3.11 python3.11-venv python3.11-dev" + echo "On RHEL/CentOS/Fedora, you can install Python 3.11 using:" + echo " sudo dnf install python3.11 python3.11-devel" + echo "Or download from: https://www.python.org/downloads/" + exit 1 +fi + +# Create virtual environment if it doesn't exist +if [ ! -d "$VENV_DIR" ]; then + echo "Creating virtual environment in '$VENV_DIR' using $PYTHON_CMD..." + $PYTHON_CMD -m venv "$VENV_DIR" + if [ $? -ne 0 ]; then + echo "ERROR: Failed to create Python virtual environment using $PYTHON_CMD." + exit 1 + fi +else + echo "Virtual environment '$VENV_DIR' already exists." +fi + +# Activate virtual environment +echo "Activating virtual environment..." +source "$VENV_DIR/bin/activate" +if [ $? -ne 0 ]; then + echo "ERROR: Failed to activate Python virtual environment." + exit 1 +fi +echo "Virtual environment activated. Python executable: $(which python)" + +# --- Check/Install uv --- +echo "Checking for uv..." +if ! command -v uv &> /dev/null; then + echo "'uv' command not found. Attempting to install uv using pip..." + # Ensure pip is available (it should be with python3) + if ! python -m pip --version &> /dev/null; then + echo "ERROR: python -m pip command failed. Cannot install uv." + exit 1 + fi + python -m pip install uv + if [ $? -ne 0 ]; then + echo "ERROR: Failed to install uv using pip. Please install uv manually (e.g., 'pip install uv') and rerun the script." + exit 1 + fi + echo "uv installed successfully." +else + echo "uv found." +fi +# --- End Check/Install uv --- + + +# Install dependencies using uv +echo "Installing Python dependencies from requirements.txt using uv..." +# Assuming uv is in PATH after the check above and venv is activated +uv pip install -r requirements.txt +if [ $? -ne 0 ]; then + echo "ERROR: Failed to install Python dependencies using uv. Please check requirements.txt and uv logs." + exit 1 # Cleanup trap will handle exit +fi +echo "Installing vllm using uv..." +uv pip install vllm +if [ $? -ne 0 ]; then + echo "ERROR: Failed to install vllm using uv." + exit 1 +fi + +echo "Python dependencies installed successfully." + + +# --- Helper Process Logging Setup --- +echo "----------------------------------------" +echo "Setting up helper process logging..." + +# Create helper_process_logs directory if it doesn't exist +HELPER_LOGS_DIR="helper_process_logs" +if [ ! -d "$HELPER_LOGS_DIR" ]; then + echo "Creating helper process logs directory: $HELPER_LOGS_DIR" + mkdir -p "$HELPER_LOGS_DIR" + if [ $? -ne 0 ]; then + echo "ERROR: Failed to create helper process logs directory." + exit 1 + fi +else + echo "Helper process logs directory already exists: $HELPER_LOGS_DIR" +fi + +# Generate timestamp for log files +TIMESTAMP=$(date +"%Y%m%d_%H%M%S") +VALKEY_LOG_FILE="$HELPER_LOGS_DIR/valkey_server_${TIMESTAMP}.log" +LLAMA_SERVER_LOG_FILE="$HELPER_LOGS_DIR/llama_server_${TIMESTAMP}.log" +HUEY_LOG_FILE="$HELPER_LOGS_DIR/huey_worker_${TIMESTAMP}.log" +UVICORN_LOG_FILE="$HELPER_LOGS_DIR/uvicorn_api_${TIMESTAMP}.log" +FRONTEND_LOG_FILE="$HELPER_LOGS_DIR/frontend_serve_${TIMESTAMP}.log" + +echo "Helper process logs will be saved to:" +echo " Valkey server: $VALKEY_LOG_FILE" +echo " VLLM server: $LLAMA_SERVER_LOG_FILE" +echo " Huey worker: $HUEY_LOG_FILE" +echo " Uvicorn API: $UVICORN_LOG_FILE" +echo " Frontend serve: $FRONTEND_LOG_FILE" +echo "Helper process logging setup complete." + +#### if REDIS_HOST skip +if [ -z "$REDIS_HOST" ]; then + # --- 1.5 Valkey/Redis Check --- + echo "----------------------------------------" + echo "2 / 9. Checking for running Valkey/Redis service..." + + SERVER_CMD="" + SERVICE_NAME="" + + # Check for valkey-server first + if command -v valkey-server &> /dev/null; then + SERVER_CMD="valkey-server" + SERVICE_NAME="valkey-server" + # Fallback to redis-server + elif command -v redis-server &> /dev/null; then + SERVER_CMD="redis-server" + SERVICE_NAME="redis-server" + else + # Check if something is listening on the default Valkey/Redis port (6379) + if command -v ss &> /dev/null; then + PORT_CHECK_CMD="ss -ltn" + else + PORT_CHECK_CMD="netstat -ltn" + fi + + if $PORT_CHECK_CMD | grep -q ':6379'; then + echo "WARNING: Neither 'valkey-server' nor 'redis-server' found in PATH, but something is listening on port 6379." + echo "Assuming a compatible Valkey/Redis server is running (possibly built from source or custom install)." + else + echo "Neither 'valkey-server' nor 'redis-server' found in PATH." + echo "Building Valkey from source..." + + # Check if valkey directory already exists + if [ ! -d "valkey" ]; then + echo "Cloning Valkey repository..." + git clone https://github.com/valkey-io/valkey.git + if [ $? -ne 0 ]; then + echo "ERROR: Failed to clone Valkey repository. Please check your internet connection and git installation." + exit 1 + fi + else + echo "Valkey directory already exists, skipping clone." + fi + + # Only build if valkey-server doesn't exist + if [ ! -f "valkey/src/valkey-server" ]; then + echo "Building Valkey..." + cd valkey + make + if [ $? -ne 0 ]; then + echo "ERROR: Failed to build Valkey. Please check build dependencies (gcc, make, etc.)." + cd "$SCRIPT_DIR" + exit 1 + fi + cd "$SCRIPT_DIR" + else + echo "Valkey already built, skipping build." + fi + + echo "Starting Valkey server in background..." + echo "Output will be logged to: $VALKEY_LOG_FILE" + cd valkey/src + ./valkey-server > "../../$VALKEY_LOG_FILE" 2>&1 & + VALKEY_PID=$! + cd "$SCRIPT_DIR" + + # Give it a moment to start/fail + sleep 4 + if ! kill -0 $VALKEY_PID &> /dev/null; then + echo "ERROR: Valkey server (PID $VALKEY_PID) failed to start or crashed immediately." + echo "Check the log file for details: $VALKEY_LOG_FILE" + exit 1 + fi + echo "Valkey server started successfully with PID $VALKEY_PID." + + # Set SERVER_CMD and SERVICE_NAME for consistency with the rest of the script + SERVER_CMD="./valkey/src/valkey-server" + SERVICE_NAME="valkey-server" + fi + fi + echo "Found $SERVER_CMD." + + # If we didn't just start Valkey server locally, perform service checks + if [ -z "$VALKEY_PID" ]; then + # Check if systemctl exists and is usable + if command -v systemctl &> /dev/null && systemctl status &> /dev/null; then + # systemctl seems to be working, so let's use it + if ! systemctl is-active --quiet $SERVICE_NAME; then + echo "WARNING: The $SERVICE_NAME service is not active according to systemctl." + # Check if something is listening on port 6379 as fallback + if ss -ltn | grep -q ':6379' 2>/dev/null; then + echo "However, something is listening on port 6379. Assuming a compatible Valkey/Redis server is running." + else + echo "ERROR: The $SERVICE_NAME service is not active and nothing is listening on port 6379." + echo "Please start the service, e.g., 'sudo systemctl start $SERVICE_NAME' and ensure it's enabled to start on boot, e.g., 'sudo systemctl enable $SERVICE_NAME'." + exit 1 + fi + else + echo "The $SERVICE_NAME service is active." + fi + else + # systemctl not available or not working, fall back to simple port check + echo "WARNING: 'systemctl' not available/running. Checking for service via port." + if ss -ltn | grep -q ':6379' 2>/dev/null; then + echo "A service is listening on port 6379. Assuming a compatible Valkey/Redis server is running." + else + echo "ERROR: No running Valkey/Redis service detected on port 6379, and could not start one automatically as it was expected to be running." + echo "Please start valkey-server or redis-server manually in another terminal." + exit 1 + fi + fi + else + echo "Valkey server was started by this script (PID $VALKEY_PID), skipping system service check." + fi + # --- End Valkey/Redis Check (Step 2) --- +fi + +# --- 1.6. VLLM Setup and Server --- +echo "----------------------------------------" +echo "3 / 9. Starting VLLM server..." +echo "Output will be logged to: $LLAMA_SERVER_LOG_FILE" + +# Determine which model to use based on argument +if [ "$MODEL_TYPE" = "normal" ]; then + MODEL_NAME="Heralax/Augmentoolkit-DataSpecialist-v0.1" + echo "Using normal model: $MODEL_NAME" +elif [ "$MODEL_TYPE" = "small" ]; then + MODEL_NAME="Heralax/Augmentoolkit-DataSpecialist-gptqmodel-4bit" + echo "Using small (quantized) model: $MODEL_NAME" + echo "==============================================" + echo "!!! IMPORTANT NOTICE !!!" + echo "==============================================" + echo "You are using the quantized model." + echo "When doing local dataset generation, you MUST use:" + echo " 'Heralax/Augmentoolkit-DataSpecialist-gptqmodel-4bit'" + echo "in your config files instead of the default model." + echo "" + echo "Update your config files to use the quantized model:" + echo " - pdf_cleaning_small_model: Heralax/Augmentoolkit-DataSpecialist-gptqmodel-4bit" + echo " - pdf_cleaning_large_model: Heralax/Augmentoolkit-DataSpecialist-gptqmodel-4bit" + echo " - representation_variation_small_model: Heralax/Augmentoolkit-DataSpecialist-gptqmodel-4bit" + echo " - representation_variation_large_model: Heralax/Augmentoolkit-DataSpecialist-gptqmodel-4bit" + echo " - ...etc... for any model. Find replace all cases of 'Heralax/Augmentoolkit-DataSpecialist-v0.1' with the gptq-4bit one." + echo "==============================================" + echo "" +else + MODEL_NAME="$MODEL_TYPE" + echo "Using custom model: $MODEL_NAME" +fi + +# Start vllm server with output redirection +vllm serve "$MODEL_NAME" --port 8082 --max-model-len 20000 --tensor-parallel-size "$TENSOR_PARALLELISM" > "$LLAMA_SERVER_LOG_FILE" 2>&1 & +LLAMA_SERVER_PID=$! +sleep 5 # Give it a moment to start/fail + +echo "VLLM server started with PID $LLAMA_SERVER_PID." +echo "Access VLLM server API at http://localhost:8082" +# --- End VLLM Setup (Step 3) --- + + +# --- 2. Huey Worker --- +echo "----------------------------------------" +echo "4 / 9. Starting Huey worker in the background (using venv)..." +echo "Output will be logged to: $HUEY_LOG_FILE" +# Call the huey_consumer script directly (should be in PATH from venv) +# Redirect both stdout and stderr to the log file +huey_consumer tasks.huey > "$HUEY_LOG_FILE" 2>&1 & +HUEY_PID=$! +# Give it a moment to start/fail +sleep 2 +if ! kill -0 $HUEY_PID &> /dev/null; then + echo "ERROR: Huey worker (PID $HUEY_PID) failed to start or crashed immediately." + echo "Check the log file for details: $HUEY_LOG_FILE" + exit 1 # Cleanup trap will handle exit +fi +echo "Huey worker started successfully with PID $HUEY_PID." + + +# --- 3. API Server --- +echo "----------------------------------------" +echo "5 / 9. Starting FastAPI server (Uvicorn) in the background (using venv)..." +echo "Output will be logged to: $UVICORN_LOG_FILE" +# Host 0.0.0.0 makes it accessible on the network +# Use python -m uvicorn to ensure using the venv uvicorn +# Redirect both stdout and stderr to the log file +python -m uvicorn api:app --host 0.0.0.0 --port 8000 > "$UVICORN_LOG_FILE" 2>&1 & +UVICORN_PID=$! +# Give it a moment to start/fail +sleep 2 +if ! kill -0 $UVICORN_PID &> /dev/null; then + echo "ERROR: Uvicorn server (PID $UVICORN_PID) failed to start or crashed immediately." + echo "Check the log file for details: $UVICORN_LOG_FILE" + # Cleanup trap will handle stopping Huey + exit 1 # Cleanup trap will handle exit +fi +echo "Uvicorn server started successfully with PID $UVICORN_PID. Access API at http://localhost:8000" + + +# --- 4. NPM Check --- +echo "----------------------------------------" +echo "6 / 9. Checking for npm..." +if ! command -v npm &> /dev/null; then + echo "ERROR: npm command not found." + echo "Please install Node.js and npm (e.g., via your system package manager like 'sudo apt install nodejs npm') and run this script again." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + exit 1 +fi +echo "npm found." + +# Check Node.js version +NODE_VERSION=$(node --version 2>/dev/null | sed 's/v//') +if [ -z "$NODE_VERSION" ]; then + echo "ERROR: Could not determine Node.js version." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + exit 1 +fi + +# Extract major version number +NODE_MAJOR_VERSION=$(echo "$NODE_VERSION" | cut -d. -f1) + +echo "Node.js version: v$NODE_VERSION" + +# Check if Node.js version is at least 18 +if [ "$NODE_MAJOR_VERSION" -lt 18 ]; then + echo "ERROR: Node.js version $NODE_VERSION is too old." + echo "This application requires Node.js 18.0.0 or higher (20+ recommended)." + echo "" + echo "To upgrade Node.js, run the following commands:" + echo "" + echo "1. Add NodeSource repository:" + echo " curl -fsSL https://deb.nodesource.com/setup_20.x | bash -" + echo "" + echo "2. Remove conflicting packages (if any):" + echo " apt-get remove -y libnode-dev libnode72" + echo "" + echo "3. Install Node.js 20.x:" + echo " apt-get install -y nodejs" + echo "" + echo " OR run all steps at once:" + echo " curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && apt-get remove -y libnode-dev libnode72 2>/dev/null || true && apt-get install -y nodejs" + echo "" + echo "Alternative method using Node Version Manager (nvm):" + echo " curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash" + echo " source ~/.bashrc" + echo " nvm install 20" + echo " nvm use 20" + echo "" + echo "After upgrading, please run this script again." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + exit 1 +fi + + +# --- 5. Frontend Dependencies --- +echo "----------------------------------------" +echo "7 / 9. Installing frontend dependencies in atk-interface..." +if [ ! -d "atk-interface" ]; then + echo "ERROR: Directory 'atk-interface' not found in $(pwd)." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + exit 1 +fi +cd atk-interface +npm install +if [ $? -ne 0 ]; then + echo "ERROR: Failed to install npm dependencies in atk-interface. Check package.json and npm logs." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + cd "$SCRIPT_DIR" # Go back to root + exit 1 +fi +echo "Frontend dependencies installed." + + +# --- 6. Frontend Build --- +echo "----------------------------------------" +echo "8 / 9. Building frontend application in atk-interface..." +npm run build +if [ $? -ne 0 ]; then + echo "ERROR: Failed to build frontend application. Check build script in package.json and logs." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + cd "$SCRIPT_DIR" # Go back to root + exit 1 +fi +echo "Frontend built successfully." + + +# --- 7. Frontend Serve --- +echo "----------------------------------------" +echo "9 / 9. Serving frontend application from atk-interface/dist/ in the FOREGROUND..." +echo "Output will be logged to: $FRONTEND_LOG_FILE" +# Ensure dist directory exists after build +if [ ! -d "dist" ]; then + echo "ERROR: 'dist' directory not found in atk-interface after build." + # Cleanup trap will handle stopping background services + exit 1 +fi + + +# Show summary of all log files +echo "" +echo "═══════════════════════════════════════════════════════════════════════════════" +echo "All helper processes are now running with output redirected to log files:" +if [ -n "$VALKEY_PID" ]; then +echo " • Valkey server logs: $VALKEY_LOG_FILE" +fi +echo " • VLLM server logs: $LLAMA_SERVER_LOG_FILE" +echo " • Huey worker logs: $HUEY_LOG_FILE" +echo " • Uvicorn API logs: $UVICORN_LOG_FILE" +echo " • Frontend serve logs: $FRONTEND_LOG_FILE" +echo "" +echo "You can monitor these logs in real-time with:" +if [ -n "$VALKEY_PID" ]; then +echo " tail -f $VALKEY_LOG_FILE" +fi +echo " tail -f $LLAMA_SERVER_LOG_FILE" +echo " tail -f $HUEY_LOG_FILE" +echo " tail -f $UVICORN_LOG_FILE" +echo " tail -f $FRONTEND_LOG_FILE" +echo "═══════════════════════════════════════════════════════════════════════════════" +echo "" + +# Serve on port 5173 (matches CORS config) +# Run npx serve in the background first, then open browser once it's ready +echo "Starting serve process (output redirected to log file)..." +# Redirect both stdout and stderr to the log file - use ../ to go back to script directory +npx serve -s dist --listen 5173 > "../$FRONTEND_LOG_FILE" 2>&1 & +SERVE_PID=$! + +# Wait for the serve process to be ready +echo "Waiting for frontend server to start..." +sleep 3 + +# Check if serve process is still running +if ! kill -0 $SERVE_PID &> /dev/null; then + echo "ERROR: Frontend serve process failed to start or crashed immediately." + echo "Check the log file for details: $FRONTEND_LOG_FILE" + cleanup + exit 1 +fi + +# Wait for the server to actually be listening on port 5173 +echo "Checking if server is responding..." +for i in {1..10}; do + if command -v curl &> /dev/null; then + if curl -s http://localhost:5173 > /dev/null 2>&1; then + break + fi + elif command -v wget &> /dev/null; then + if wget -q --spider http://localhost:5173 > /dev/null 2>&1; then + break + fi + else + # Fallback: just check if something is listening on port 5173 + if command -v ss &> /dev/null; then + if ss -ltn | grep -q ':5173'; then + break + fi + elif command -v netstat &> /dev/null; then + if netstat -ltn | grep -q ':5173'; then + break + fi + else + # No way to check, just wait a bit more + sleep 1 + break + fi + fi + sleep 1 +done + +if ! kill -0 $LLAMA_SERVER_PID &> /dev/null; then + echo "ERROR: VLLM server (PID $LLAMA_SERVER_PID) failed to start or crashed." + echo "Check the log file for details: $LLAMA_SERVER_LOG_FILE" + echo "The model probably wants to take too much memory. You may need to:" + echo " - Use 'small' argument to run quantized model: ./local_linux.sh small" + echo " - Use a custom model path: ./local_linux.sh path/to/your/model" + echo " - Use an API instead of local inference" + cleanup + exit 1 +fi + +echo "Opening interface..." + +# --- Check for xdg-open --- +echo "Checking for xdg-open..." +if ! command -v xdg-open &> /dev/null; then + echo "WARNING: 'xdg-open' command not found. Cannot automatically open browser." + echo "Please install xdg-utils (e.g., 'sudo apt install xdg-utils' or 'sudo dnf install xdg-utils') or open http://localhost:5173 manually." + # Not exiting, just warning +else + echo "xdg-open found. Opening http://localhost:5173 in default browser..." + xdg-open http://localhost:5173 +fi + +echo "Frontend server is now running. Press Ctrl+C to stop all services." + +# Wait for the serve process to finish (it's now running in the background) +wait $SERVE_PID +SERVE_EXIT_CODE=$? +SERVE_PID="" # Clear PID as it has exited + +if [ $SERVE_EXIT_CODE -ne 0 ]; then + echo "WARNING: Frontend serve process exited with code $SERVE_EXIT_CODE." + echo "Check the log file for details: $FRONTEND_LOG_FILE" + # Consider if we should stop background services here if serve crashes + # Calling cleanup manually if serve fails + cleanup + exit 1 # Exit with error code +fi + + +# --- 8. Done (Only reached if npx serve exits gracefully, e.g., if killed separately) --- +cd "$SCRIPT_DIR" # Go back to root directory +echo "----------------------------------------" +echo "Frontend serve process finished." +echo "" +echo "Helper process log files were saved to:" +if [ -n "$VALKEY_PID" ]; then +echo " • Valkey server logs: $VALKEY_LOG_FILE" +fi +echo " • VLLM server logs: $LLAMA_SERVER_LOG_FILE" +echo " • Huey worker logs: $HUEY_LOG_FILE" +echo " • Uvicorn API logs: $UVICORN_LOG_FILE" +echo " • Frontend serve logs: $FRONTEND_LOG_FILE" +# Call cleanup to stop background services if serve finishes normally +cleanup + +exit 0 diff --git a/local_macos.sh b/local_macos.sh new file mode 100644 index 0000000000000000000000000000000000000000..2a5e0feb484964355f891ffd4fa84c2f156afc27 --- /dev/null +++ b/local_macos.sh @@ -0,0 +1,713 @@ +#!/bin/bash + +# Exit immediately if a command exits with a non-zero status. +# set -e # Disabled for manual error checking below + +echo "Starting Augmentoolkit services..." + +# --- Get script directory and change to it --- +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd "$SCRIPT_DIR" +if [ $? -ne 0 ]; then + echo "ERROR: Failed to change directory to script location '$SCRIPT_DIR'." + exit 1 +fi +echo "Running in directory: $(pwd)" + +# --- Cleanup Function --- +# Stores PIDs of background processes +HUEY_PID="" +UVICORN_PID="" +SERVE_PID="" # Though serve runs in foreground, trap might catch signals before it starts +LLAMA_SERVER_PID="" # PID for llama.cpp server + +cleanup() { + echo "" # Newline after ^C + echo "----------------------------------------" + echo "Signal received, stopping background services..." + + # Function to kill a process and wait for it to terminate + kill_and_wait() { + local pid=$1 + local name=$2 + local timeout=9 + + if [ -n "$pid" ] && kill -0 $pid &> /dev/null; then + echo "Stopping $name (PID $pid)..." + + # First try SIGTERM (graceful shutdown) + kill -TERM $pid 2>/dev/null + + # Wait for process to terminate gracefully + local count=0 + while [ $count -lt $timeout ] && kill -0 $pid &> /dev/null; do + sleep 1 + count=$((count + 1)) + done + + # If still running, force kill with SIGKILL + if kill -0 $pid &> /dev/null; then + echo "Process $pid ($name) didn't terminate gracefully, force killing..." + kill -KILL $pid 2>/dev/null + sleep 1 + fi + + # Final check + if kill -0 $pid &> /dev/null; then + echo "WARNING: Failed to stop $name (PID $pid)" + else + echo "$name stopped successfully." + fi + fi + } + + # Stop processes in reverse order of startup + kill_and_wait "$SERVE_PID" "frontend serve" + kill_and_wait "$UVICORN_PID" "Uvicorn API server" + kill_and_wait "$HUEY_PID" "Huey worker" + kill_and_wait "$LLAMA_SERVER_PID" "LLaMA.cpp server" + + # Find and kill any remaining huey consumers specific to this Augmentoolkit instance + echo "Searching for huey consumer processes in current directory ($SCRIPT_DIR)..." + + # Use ps with full command line and grep for huey_consumer processes in our specific directory + # This avoids killing huey consumers from other Augmentoolkit instances or projects + HUEY_PIDS=$(ps aux | grep -E "huey_consumer.*tasks\.huey" | grep "$SCRIPT_DIR" | grep -v grep | awk '{print $2}') + + if [ -n "$HUEY_PIDS" ]; then + echo "Found huey consumer processes in current directory: $HUEY_PIDS" + for pid in $HUEY_PIDS; do + if kill -0 $pid &> /dev/null; then + echo "Stopping huey consumer from current directory (PID $pid)..." + kill -TERM $pid 2>/dev/null + sleep 2 + if kill -0 $pid &> /dev/null; then + echo "Force killing huey consumer (PID $pid)..." + kill -KILL $pid 2>/dev/null + fi + fi + done + sleep 1 + # Final check for any remaining huey processes in our directory + REMAINING_HUEY=$(ps aux | grep -E "huey_consumer.*tasks\.huey" | grep "$SCRIPT_DIR" | grep -v grep | awk '{print $2}') + if [ -n "$REMAINING_HUEY" ]; then + echo "WARNING: Some huey consumer processes in current directory may still be running: $REMAINING_HUEY" + else + echo "All huey consumer processes from current directory stopped successfully." + fi + else + echo "No additional huey consumer processes found in current directory." + fi + + # Deactivate virtual environment if active + if command -v deactivate &> /dev/null; then + echo "Deactivating virtual environment..." + deactivate + fi + + # Give a moment for any remaining output to flush + sleep 1 + + # Clear any remaining output and reset cursor + echo "Cleanup complete." + echo "----------------------------------------" + + # Ensure cursor is visible and terminal is in normal state + printf "\033[?25h" # Show cursor + printf "\033[0m" # Reset all attributes + + # Do not exit from cleanup function itself, let the caller decide exit status + echo "" # Add extra newline for visual clarity before prompt returns +} + +# Trap signals (INT: Ctrl+C, TERM: standard kill) and run cleanup function +trap cleanup SIGINT SIGTERM + +# --- 1. Python Virtual Environment & Dependencies --- +echo "----------------------------------------" +echo "1 / 10. Setting up Python virtual environment and installing dependencies..." +VENV_DIR=".venv" + +# Check if python3.11 command exists (preferred) +if command -v python3.11 &> /dev/null; then + PYTHON_CMD="python3.11" + echo "Using Python 3.11: $(python3.11 --version)" +elif command -v python3 &> /dev/null; then + PYTHON_CMD="python3" + PYTHON_VERSION=$(python3 --version 2>&1 | grep -oE '[0-9]+\.[0-9]+') + echo "Using Python: $(python3 --version)" + if [[ "$PYTHON_VERSION" != "3.11" ]]; then + echo "WARNING: Python 3.11 is recommended but not found. Using $PYTHON_VERSION instead." + echo "If you encounter issues, please install Python 3.11 specifically." + fi +else + echo "ERROR: Neither python3.11 nor python3 command found." + echo "Please install Python 3.11 (recommended) or Python 3.x and rerun the script." + echo "On macOS, you can install Python 3.11 using:" + echo " brew install python@3.11" + echo "Or download from: https://www.python.org/downloads/" + exit 1 +fi + +# --- Check/Install uv --- +echo "Checking for uv..." +if ! command -v uv &> /dev/null; then + echo "'uv' command not found. Attempting to install uv using pip..." + # Ensure pip is available (it should be with the selected Python version) + if ! $PYTHON_CMD -m pip --version &> /dev/null; then + echo "ERROR: $PYTHON_CMD -m pip command failed. Cannot install uv." + exit 1 + fi + $PYTHON_CMD -m pip install uv + if [ $? -ne 0 ]; then + echo "ERROR: Failed to install uv using pip. Please install uv manually (e.g., 'pip install uv' or 'brew install uv') and rerun the script." + exit 1 + fi + echo "uv installed successfully." +else + echo "uv found." +fi +# --- End Check/Install uv --- + + +# Create virtual environment if it doesn't exist +if [ ! -d "$VENV_DIR" ]; then + echo "Creating virtual environment in '$VENV_DIR' using $PYTHON_CMD..." + $PYTHON_CMD -m venv "$VENV_DIR" + if [ $? -ne 0 ]; then + echo "ERROR: Failed to create Python virtual environment using $PYTHON_CMD." + exit 1 + fi +else + echo "Virtual environment '$VENV_DIR' already exists." +fi + +# Activate virtual environment +echo "Activating virtual environment..." +source "$VENV_DIR/bin/activate" +if [ $? -ne 0 ]; then + echo "ERROR: Failed to activate Python virtual environment." + exit 1 +fi +echo "Virtual environment activated. Python executable: $(which python)" + +# --- Attempt to install SSL certificates (macOS specific fix) --- +echo "Attempting to install SSL certificates for the active Python environment..." +# Get the full path to the python executable inside the venv +VENV_PYTHON_EXE=$(which python) +# Try to infer the Applications directory path (this is heuristic) +# Assumes Python was installed via the standard macOS installer +PYTHON_BASE_DIR=$(dirname "$(dirname "$VENV_PYTHON_EXE")") # Go up two levels from venv/bin/python +CERT_SCRIPT_PATH="" +# Check common installation patterns +if [ -d "$PYTHON_BASE_DIR/Install Certificates.command" ]; then + # Might be like /Library/Frameworks/Python.framework/Versions/3.11/Install Certificates.command + CERT_SCRIPT_PATH="$PYTHON_BASE_DIR/Install Certificates.command" +elif [ -d "/Applications/Python $(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')/Install Certificates.command" ]; then + # Check standard /Applications path using the version + PY_VERSION_SHORT=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') + CERT_SCRIPT_PATH="/Applications/Python ${PY_VERSION_SHORT}/Install Certificates.command" +fi + +if [ -n "$CERT_SCRIPT_PATH" ] && [ -f "$CERT_SCRIPT_PATH" ]; then + echo "Found certificate installation script: $CERT_SCRIPT_PATH" + echo "Running certificate installation script..." + # Execute the script using the venv's python + "$CERT_SCRIPT_PATH" + if [ $? -ne 0 ]; then + echo "WARNING: Certificate installation script finished with a non-zero status. SSL issues might persist." + # Continue anyway, maybe it partially worked or wasn't strictly needed + else + echo "Certificate installation script completed successfully." + fi +else + echo "WARNING: Could not find the 'Install Certificates.command' script automatically." + echo "If you encounter SSL errors (e.g., downloading NLTK data, used in chunking), you may need to run it manually." + echo "Look for it in your Python application folder (e.g., /Applications/Python 3.11/)." +fi +# --- End SSL Certificate Install (Step 2) --- + + +# Install dependencies using uv +echo "Installing Python dependencies from requirements.txt using uv..." +# Assuming uv is in PATH after the check above and venv is activated +uv pip install -r requirements.txt +if [ $? -ne 0 ]; then + echo "ERROR: Failed to install Python dependencies using uv. Please check requirements.txt and uv logs." + exit 1 # Cleanup trap will handle exit +fi +echo "Python dependencies installed successfully." + +# --- Configure SSL Certificate Path (Part of Step 2 Logic) --- +echo "----------------------------------------" +echo "2 / 10. Configuring SSL certificate path..." # Renumbered + +# Ensure certifi is installed (uv pip install should handle this if in requirements.txt, +# but explicit check adds robustness - though uv handles it well) +# It's likely already installed as a dependency of requests/huggingface_hub etc. +# If dependency issues arise, add 'certifi' explicitly to requirements.txt + +# Get the certificate path from certifi +CERT_PATH=$(python -c "import certifi; print(certifi.where())" 2>/dev/null) # Suppress stderr in case certifi isn't installed yet + +if [ -n "$CERT_PATH" ] && [ -f "$CERT_PATH" ]; then + echo "Setting SSL_CERT_FILE environment variable to: $CERT_PATH" + export SSL_CERT_FILE="$CERT_PATH" + echo "SSL_CERT_FILE set for this script session and its children." + # Optional: Add a connection test here if desired + # python -c "import requests; r = requests.get('https://huggingface.co', timeout=10); print(f'Connection test status: {r.status_code}')" + # if [ $? -ne 0 ]; then echo "WARNING: Connection test failed after setting SSL_CERT_FILE."; fi +else + echo "WARNING: Could not determine certificate path using certifi. Certifi might not be installed yet or there's an issue." + echo "Attempting to proceed, but SSL errors (like CERTIFICATE_VERIFY_FAILED) may occur." + echo "Ensure 'certifi' is listed in requirements.txt if SSL errors persist." +fi +# --- End SSL Configuration --- + + +# --- Helper Process Logging Setup --- +echo "----------------------------------------" +echo "Setting up helper process logging..." + +# Create helper_process_logs directory if it doesn't exist +HELPER_LOGS_DIR="helper_process_logs" +if [ ! -d "$HELPER_LOGS_DIR" ]; then + echo "Creating helper process logs directory: $HELPER_LOGS_DIR" + mkdir -p "$HELPER_LOGS_DIR" + if [ $? -ne 0 ]; then + echo "ERROR: Failed to create helper process logs directory." + exit 1 + fi +else + echo "Helper process logs directory already exists: $HELPER_LOGS_DIR" +fi + +# Generate timestamp for log files +TIMESTAMP=$(date +"%Y%m%d_%H%M%S") +LLAMA_SERVER_LOG_FILE="$HELPER_LOGS_DIR/llama_server_${TIMESTAMP}.log" +HUEY_LOG_FILE="$HELPER_LOGS_DIR/huey_worker_${TIMESTAMP}.log" +UVICORN_LOG_FILE="$HELPER_LOGS_DIR/uvicorn_api_${TIMESTAMP}.log" +FRONTEND_LOG_FILE="$HELPER_LOGS_DIR/frontend_serve_${TIMESTAMP}.log" + +echo "Helper process logs will be saved to:" +echo " LLaMA server: $LLAMA_SERVER_LOG_FILE" +echo " Huey worker: $HUEY_LOG_FILE" +echo " Uvicorn API: $UVICORN_LOG_FILE" +echo " Frontend serve: $FRONTEND_LOG_FILE" +echo "Helper process logging setup complete." + + +# --- 1.5 Valkey (Redis Fork) Check & Installation --- +echo "----------------------------------------" +echo "3 / 10. Checking for Valkey (Redis fork)..." # Renumbered + +# First, check if something is already listening on port 6379 +if command -v lsof &> /dev/null; then + PORT_CHECK_CMD="lsof -i :6379" +elif command -v netstat &> /dev/null; then + PORT_CHECK_CMD="netstat -an" +else + PORT_CHECK_CMD="" +fi + +if [ -n "$PORT_CHECK_CMD" ]; then + if [ "$PORT_CHECK_CMD" = "lsof -i :6379" ]; then + if $PORT_CHECK_CMD &> /dev/null; then + echo "Something is already listening on port 6379. Assuming a compatible Valkey/Redis server is running." + echo "Skipping Valkey installation and setup." + else + NEED_VALKEY_SETUP=true + fi + else + # netstat case + if $PORT_CHECK_CMD | grep -q ':6379'; then + echo "Something is already listening on port 6379. Assuming a compatible Valkey/Redis server is running." + echo "Skipping Valkey installation and setup." + else + NEED_VALKEY_SETUP=true + fi + fi +else + echo "WARNING: Cannot check port 6379 (lsof and netstat not available). Proceeding with Valkey setup." + NEED_VALKEY_SETUP=true +fi + +if [ "$NEED_VALKEY_SETUP" = true ]; then + # Check if brew command exists +if ! command -v brew &> /dev/null; then + echo "Homebrew (brew) command not found." + read -p "Homebrew is required to install Valkey. Install Homebrew now? (y/N) " -n 1 -r + echo # Move to a new line after input + if [[ $REPLY =~ ^[Yy]$ ]]; then + echo "Attempting to install Homebrew. This may take a few minutes and will likely ask for your administrator password..." + # Execute the official Homebrew installation script + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + # Check if installation succeeded by seeing if 'brew' command is now available + # Might require sourcing profile or opening new shell, but let's check command first + if ! command -v brew &> /dev/null; then + echo "ERROR: Homebrew installation appears to have failed or is not available in this shell session." + echo "Please try installing Homebrew manually from https://brew.sh/ and run this script again, possibly in a new terminal window." + exit 1 + else + # Add brew to PATH for the current script execution if possible + # This is complex as the path depends on architecture (Intel vs ARM) + # For simplicity, we'll rely on the installer potentially modifying the profile + # or the user running in a new shell later if 'brew' isn't found immediately after install. + # A more robust solution might involve detecting arch and adding the known path. + echo "Homebrew installed successfully. Proceeding..." + # Re-check explicitly here just in case the path isn't immediately active + if ! command -v brew &> /dev/null; then + echo "ERROR: Homebrew installed but 'brew' command still not found in PATH for this session." + echo "Please open a new terminal window and run this script again." + exit 1 + fi + fi + else + echo "Homebrew installation declined. Valkey cannot be installed automatically." + echo "Please install Homebrew (https://brew.sh/) and Valkey ('brew install valkey') manually and rerun the script." + exit 1 + fi +fi +echo "Homebrew found." + +# Check if valkey is installed +if ! brew list valkey &>/dev/null; then + echo "Valkey not found via Homebrew. Attempting to install..." + # Update brew minimally and install valkey + # Using 'brew install' directly avoids running 'brew update' if possible, + # but Homebrew might still update itself or core formulae. + # We avoid 'brew upgrade' to prevent upgrading all packages. + brew install valkey + if [ $? -ne 0 ]; then + echo "ERROR: Failed to install Valkey using Homebrew. Please install it manually ('brew install valkey') and rerun the script." + exit 1 + fi + echo "Valkey installed successfully." +else + echo "Valkey found via Homebrew." +fi + +# Check if Valkey service is running, start if not +if ! brew services list | grep -q "valkey.*started"; then + echo "Valkey service is not running. Starting it now..." + brew services start valkey + if [ $? -ne 0 ]; then + echo "ERROR: Failed to start Valkey service using Homebrew. Please check Homebrew services ('brew services list') and start Valkey manually if needed." + exit 1 + fi + # Give it a moment to start + sleep 2 + if ! brew services list | grep -q "valkey.*started"; then + echo "ERROR: Valkey service failed to stay running after starting. Please investigate ('brew services list', check logs)." + exit 1 + fi + echo "Valkey service started successfully." +else + echo "Valkey service is already running." +fi +fi # End NEED_VALKEY_SETUP conditional +# --- End Valkey Check --- + + +# --- 1.6. LLaMA.cpp Setup and Server --- +echo "----------------------------------------" +echo "4 / 10. Setting up LLaMA.cpp server..." # Renumbered +LLAMA_CPP_DIR_NAME="llama.cpp" # Relative to SCRIPT_DIR +LLAMA_CPP_FULL_PATH="$SCRIPT_DIR/$LLAMA_CPP_DIR_NAME" +LLAMA_MODEL_SUBDIR="models" # Subdirectory within LLAMA_CPP_DIR + +# Model details +MODEL_URL="https://huggingface.co/Heralax/Augmentoolkit-DataSpecialist-v0.1-GGUF/resolve/main/Augmentoolkit-DataSpecialist-7.2B-Q8_0.gguf?download=true" +MODEL_FILENAME="augmentoolkit-datagen-7.2B-Q8_0.gguf" +LLAMA_MODEL_PATH="$LLAMA_CPP_FULL_PATH/$LLAMA_MODEL_SUBDIR/$MODEL_FILENAME" + +# Check for prerequisites +if ! command -v git &> /dev/null; then + echo "ERROR: git command not found. Please install git and rerun the script." + cleanup + exit 1 +fi +if ! command -v cmake &> /dev/null; then + echo "ERROR: cmake command not found. Please install cmake (e.g., 'brew install cmake') and rerun the script." + cleanup + exit 1 +fi +DOWNLOAD_CMD="" +if command -v curl &> /dev/null; then # Prefer curl as it's more common on macOS by default + DOWNLOAD_CMD="curl" +elif command -v wget &> /dev/null; then + DOWNLOAD_CMD="wget" +else + echo "ERROR: Neither curl nor wget found. Please install one of them to download the model." + cleanup + exit 1 +fi + +# Clone llama.cpp if not already cloned +if [ ! -d "$LLAMA_CPP_FULL_PATH" ]; then + echo "Cloning llama.cpp repository into '$LLAMA_CPP_FULL_PATH'..." + git clone https://github.com/ggml-org/llama.cpp.git "$LLAMA_CPP_FULL_PATH" + if [ $? -ne 0 ]; then + echo "ERROR: Failed to clone llama.cpp repository." + cleanup + exit 1 + fi + echo "Checking out specific commit..." + cd "$LLAMA_CPP_FULL_PATH" + # git checkout 3cb203c89f60483e349f841684173446ed23c28f + git checkout 3cb203c89f60483e349f841684173446ed23c28f + if [ $? -ne 0 ]; then + echo "ERROR: Failed to checkout specific commit." + cleanup + exit 1 + fi + cd - > /dev/null +else + echo "llama.cpp directory '$LLAMA_CPP_FULL_PATH' already exists. Skipping clone." + # Optional: Add logic to pull latest changes if desired + # echo "Pulling latest changes for llama.cpp..." + # (cd "$LLAMA_CPP_FULL_PATH" && git pull) +fi + +# Create model directory inside llama.cpp if it doesn't exist +mkdir -p "$LLAMA_CPP_FULL_PATH/$LLAMA_MODEL_SUBDIR" +if [ $? -ne 0 ]; then + echo "ERROR: Failed to create model directory '$LLAMA_CPP_FULL_PATH/$LLAMA_MODEL_SUBDIR'." + cleanup + exit 1 +fi + +# Download model if it doesn't exist +if [ ! -f "$LLAMA_MODEL_PATH" ]; then + echo "LLaMA.cpp model file '$MODEL_FILENAME' not found at '$LLAMA_MODEL_PATH'." + read -p "This will download a ~7.7GB model file. This may take a while and obviously requires a bit of disk space. Proceed? (y/N) " -n 1 -r + echo # Newline + if [[ $REPLY =~ ^[Yy]$ ]]; then + echo "Downloading LLaMA.cpp model to '$LLAMA_MODEL_PATH'..." + if [ "$DOWNLOAD_CMD" = "curl" ]; then + curl -L "$MODEL_URL" -o "$LLAMA_MODEL_PATH" + else # wget + wget "$MODEL_URL" -O "$LLAMA_MODEL_PATH" + fi + if [ $? -ne 0 ]; then + echo "ERROR: Failed to download LLaMA.cpp model." + rm -f "$LLAMA_MODEL_PATH" # Remove partial download if any + cleanup + exit 1 + fi + echo "Model downloaded successfully." + else + echo "Model download declined. LLaMA.cpp server cannot start without the model." + cleanup + exit 1 + fi +else + echo "LLaMA.cpp model '$MODEL_FILENAME' already exists at '$LLAMA_MODEL_PATH'." +fi + +# Build llama.cpp +LLAMA_SERVER_BINARY_PATH="$LLAMA_CPP_FULL_PATH/build/bin/llama-server" +if [ ! -f "$LLAMA_SERVER_BINARY_PATH" ]; then + echo "Building llama.cpp (this may take several minutes)..." + cd "$LLAMA_CPP_FULL_PATH" # Change to llama.cpp directory for build + + NUM_CORES=$(sysctl -n hw.ncpu 2>/dev/null || echo 4) # Default to 4 if sysctl fails + + cmake -B build # Metal is enabled by default on macOS + if [ $? -ne 0 ]; then + echo "ERROR: cmake configuration for llama.cpp failed." + cd "$SCRIPT_DIR" # Return to original script directory + cleanup + exit 1 + fi + cmake --build build --config Release -j "$NUM_CORES" + if [ $? -ne 0 ]; then + echo "ERROR: cmake build for llama.cpp failed." + cd "$SCRIPT_DIR" # Return to original script directory + cleanup + exit 1 + fi + echo "llama.cpp built successfully. Server binary at '$LLAMA_SERVER_BINARY_PATH'." + cd "$SCRIPT_DIR" # Return to original script directory +else + echo "llama-server binary already exists at '$LLAMA_SERVER_BINARY_PATH'. Skipping build." +fi + +# Start llama-server +echo "Starting LLaMA.cpp server in the background..." +echo "Output will be logged to: $LLAMA_SERVER_LOG_FILE" +LLAMA_SERVER_PORT=8082 # Port for llama.cpp server +# The model path for llama-server is absolute: $LLAMA_MODEL_PATH +# The server binary path is absolute: $LLAMA_SERVER_BINARY_PATH + +# Redirect both stdout and stderr to the log file +"$LLAMA_SERVER_BINARY_PATH" -m "$LLAMA_MODEL_PATH" -ngl 100 -c 30000 --port "$LLAMA_SERVER_PORT" --host 0.0.0.0 > "$LLAMA_SERVER_LOG_FILE" 2>&1 & +LLAMA_SERVER_PID=$! +sleep 3 # Give it a moment to start/fail + +if ! kill -0 $LLAMA_SERVER_PID &> /dev/null; then + echo "ERROR: LLaMA.cpp server (PID $LLAMA_SERVER_PID) failed to start or crashed." + echo "Check the log file for details: $LLAMA_SERVER_LOG_FILE" + # No need to cd back as we are already in SCRIPT_DIR if build was skipped or after build + cleanup + exit 1 +fi +echo "LLaMA.cpp server started successfully with PID $LLAMA_SERVER_PID." +echo "Access LLaMA.cpp server API at http://localhost:$LLAMA_SERVER_PORT" +# --- End LLaMA.cpp Setup (Step 4) --- + + +# --- 2. Huey Worker --- +echo "----------------------------------------" +echo "5 / 10. Starting Huey worker in the background (using venv)..." # Renumbered +echo "Output will be logged to: $HUEY_LOG_FILE" +# Call the huey_consumer script directly (should be in PATH from venv) +# Redirect both stdout and stderr to the log file +huey_consumer tasks.huey > "$HUEY_LOG_FILE" 2>&1 & +HUEY_PID=$! +# Give it a moment to start/fail +sleep 2 +if ! kill -0 $HUEY_PID &> /dev/null; then + echo "ERROR: Huey worker (PID $HUEY_PID) failed to start or crashed immediately." + echo "Check the log file for details: $HUEY_LOG_FILE" + exit 1 # Cleanup trap will handle exit +fi +echo "Huey worker started successfully with PID $HUEY_PID." + + +# --- 3. API Server --- +echo "----------------------------------------" +echo "6 / 10. Starting FastAPI server (Uvicorn) in the background (using venv)..." # Renumbered +echo "Output will be logged to: $UVICORN_LOG_FILE" +# Host 0.0.0.0 makes it accessible on the network +# Use python -m uvicorn to ensure using the venv uvicorn +# Redirect both stdout and stderr to the log file +python -m uvicorn api:app --host 0.0.0.0 --port 8000 > "$UVICORN_LOG_FILE" 2>&1 & +UVICORN_PID=$! +# Give it a moment to start/fail +sleep 2 +if ! kill -0 $UVICORN_PID &> /dev/null; then + echo "ERROR: Uvicorn server (PID $UVICORN_PID) failed to start or crashed immediately." + echo "Check the log file for details: $UVICORN_LOG_FILE" + # Cleanup trap will handle stopping Huey + exit 1 # Cleanup trap will handle exit +fi +echo "Uvicorn server started successfully with PID $UVICORN_PID. Access API at http://localhost:8000" + + +# --- 4. NPM Check --- +echo "----------------------------------------" +echo "7 / 10. Checking for npm..." # Renumbered +if ! command -v npm &> /dev/null; then + echo "ERROR: npm command not found." + echo "Please install Node.js and npm (e.g., from https://nodejs.org/) and run this script again." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + exit 1 +fi +echo "npm found." + + +# --- 5. Frontend Dependencies --- +echo "----------------------------------------" +echo "8 / 10. Installing frontend dependencies in atk-interface..." # Renumbered +if [ ! -d "atk-interface" ]; then + echo "ERROR: Directory 'atk-interface' not found in $(pwd)." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + exit 1 +fi +cd atk-interface +npm install +if [ $? -ne 0 ]; then + echo "ERROR: Failed to install npm dependencies in atk-interface. Check package.json and npm logs." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + cd "$SCRIPT_DIR" # Go back to root + exit 1 +fi +echo "Frontend dependencies installed." + + +# --- 6. Frontend Build --- +echo "----------------------------------------" +echo "9 / 10. Building frontend application in atk-interface..." # Renumbered +npm run build +if [ $? -ne 0 ]; then + echo "ERROR: Failed to build frontend application. Check build script in package.json and logs." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + cd "$SCRIPT_DIR" # Go back to root + exit 1 +fi +echo "Frontend built successfully." + + +# --- 7. Frontend Serve --- +echo "----------------------------------------" +echo "10 / 10. Serving frontend application from atk-interface/dist/ in the FOREGROUND..." # Renumbered +echo "Output will be logged to: $FRONTEND_LOG_FILE" +# Ensure dist directory exists after build +if [ ! -d "dist" ]; then + echo "ERROR: 'dist' directory not found in atk-interface after build." + # Cleanup trap will handle stopping background services + exit 1 +fi + + +echo "Opening interface..." + + +# Open the URL in the default browser (macOS) +echo "Opening http://localhost:5173 in default browser..." +open http://localhost:5173 + +# Show summary of all log files +echo "" +echo "═══════════════════════════════════════════════════════════════════════════════" +echo "All helper processes are now running with output redirected to log files:" +echo " • LLaMA server logs: $LLAMA_SERVER_LOG_FILE" +echo " • Huey worker logs: $HUEY_LOG_FILE" +echo " • Uvicorn API logs: $UVICORN_LOG_FILE" +echo " • Frontend serve logs: $FRONTEND_LOG_FILE" +echo "" +echo "You can monitor these logs in real-time with:" +echo " tail -f $LLAMA_SERVER_LOG_FILE" +echo " tail -f $HUEY_LOG_FILE" +echo " tail -f $UVICORN_LOG_FILE" +echo " tail -f $FRONTEND_LOG_FILE" +echo "═══════════════════════════════════════════════════════════════════════════════" +echo "" + +# Serve on port 5173 (matches CORS config) +# Run npx serve in the foreground with output redirected to log file +echo "Starting serve process (output redirected to log file)..." +# Redirect both stdout and stderr to the log file - use ../ to go back to script directory +npx serve -s dist --listen 5173 > "../$FRONTEND_LOG_FILE" 2>&1 +SERVE_EXIT_CODE=$? +SERVE_PID="" # Clear PID as it has exited + +if [ $SERVE_EXIT_CODE -ne 0 ]; then + echo "WARNING: Frontend serve process exited with code $SERVE_EXIT_CODE." + echo "Check the log file for details: $FRONTEND_LOG_FILE" + # Consider if we should stop background services here if serve crashes + # Calling cleanup manually if serve fails + cleanup + exit 1 # Exit with error code +fi + + +# --- 8. Done (Only reached if npx serve exits gracefully, e.g., if killed separately) --- +cd "$SCRIPT_DIR" # Go back to root directory +echo "----------------------------------------" +echo "Frontend serve process finished." +echo "" +echo "Helper process log files were saved to:" +echo " • LLaMA server logs: $LLAMA_SERVER_LOG_FILE" +echo " • Huey worker logs: $HUEY_LOG_FILE" +echo " • Uvicorn API logs: $UVICORN_LOG_FILE" +echo " • Frontend serve logs: $FRONTEND_LOG_FILE" +# Call cleanup to stop background services if serve finishes normally +cleanup + +exit 0 diff --git a/logs/.gitkeep b/logs/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..b28b04f643122b019e912540f228c8ed20be9eeb --- /dev/null +++ b/logs/.gitkeep @@ -0,0 +1,3 @@ + + + diff --git a/macos.sh b/macos.sh new file mode 100644 index 0000000000000000000000000000000000000000..ef90fb18d6bdd08baa1cc3163e009288a70f85a6 --- /dev/null +++ b/macos.sh @@ -0,0 +1,612 @@ +#!/bin/bash + +# Exit immediately if a command exits with a non-zero status. +# set -e # Disabled for manual error checking below + +echo "Starting Augmentoolkit services..." + +# --- Get script directory and change to it --- +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd "$SCRIPT_DIR" +if [ $? -ne 0 ]; then + echo "ERROR: Failed to change directory to script location '$SCRIPT_DIR'." + exit 1 +fi +echo "Running in directory: $(pwd)" + +# --- Cleanup Function --- +# Stores PIDs of background processes +HUEY_PID="" +UVICORN_PID="" +SERVE_PID="" # Though serve runs in foreground, trap might catch signals before it starts + +cleanup() { + echo "" # Newline after ^C + echo "----------------------------------------" + echo "Signal received, stopping background services..." + + # Function to kill a process and wait for it to terminate + kill_and_wait() { + local pid=$1 + local name=$2 + local timeout=9 + + if [ -n "$pid" ] && kill -0 $pid &> /dev/null; then + echo "Stopping $name (PID $pid)..." + + # First try SIGTERM (graceful shutdown) + kill -TERM $pid 2>/dev/null + + # Wait for process to terminate gracefully + local count=0 + while [ $count -lt $timeout ] && kill -0 $pid &> /dev/null; do + sleep 1 + count=$((count + 1)) + done + + # If still running, force kill with SIGKILL + if kill -0 $pid &> /dev/null; then + echo "Process $pid ($name) didn't terminate gracefully, force killing..." + kill -KILL $pid 2>/dev/null + sleep 1 + fi + + # Final check + if kill -0 $pid &> /dev/null; then + echo "WARNING: Failed to stop $name (PID $pid)" + else + echo "$name stopped successfully." + fi + fi + } + + # Stop processes in reverse order of startup + kill_and_wait "$SERVE_PID" "frontend serve" + kill_and_wait "$UVICORN_PID" "Uvicorn API server" + kill_and_wait "$HUEY_PID" "Huey worker" + + # Find and kill any remaining huey consumers specific to this Augmentoolkit instance + echo "Searching for huey consumer processes in current directory ($SCRIPT_DIR)..." + + # Use ps with full command line and grep for huey_consumer processes in our specific directory + # This avoids killing huey consumers from other Augmentoolkit instances or projects + HUEY_PIDS=$(ps aux | grep -E "huey_consumer.*tasks\.huey" | grep "$SCRIPT_DIR" | grep -v grep | awk '{print $2}') + + if [ -n "$HUEY_PIDS" ]; then + echo "Found huey consumer processes in current directory: $HUEY_PIDS" + for pid in $HUEY_PIDS; do + if kill -0 $pid &> /dev/null; then + echo "Stopping huey consumer from current directory (PID $pid)..." + kill -TERM $pid 2>/dev/null + sleep 2 + if kill -0 $pid &> /dev/null; then + echo "Force killing huey consumer (PID $pid)..." + kill -KILL $pid 2>/dev/null + fi + fi + done + sleep 1 + # Final check for any remaining huey processes in our directory + REMAINING_HUEY=$(ps aux | grep -E "huey_consumer.*tasks\.huey" | grep "$SCRIPT_DIR" | grep -v grep | awk '{print $2}') + if [ -n "$REMAINING_HUEY" ]; then + echo "WARNING: Some huey consumer processes in current directory may still be running: $REMAINING_HUEY" + else + echo "All huey consumer processes from current directory stopped successfully." + fi + else + echo "No additional huey consumer processes found in current directory." + fi + + # Deactivate virtual environment if active + if command -v deactivate &> /dev/null; then + echo "Deactivating virtual environment..." + deactivate + fi + + # Give a moment for any remaining output to flush + sleep 1 + + # Clear any remaining output and reset cursor + echo "Cleanup complete." + echo "----------------------------------------" + + # Ensure cursor is visible and terminal is in normal state + printf "\033[?25h" # Show cursor + printf "\033[0m" # Reset all attributes + + exit 0 # Exit script after cleanup +} + +# Trap signals (INT: Ctrl+C, TERM: standard kill) and run cleanup function +trap cleanup SIGINT SIGTERM + +# --- 1. Python Virtual Environment & Dependencies --- +echo "----------------------------------------" +echo "1. Setting up Python virtual environment and installing dependencies..." +VENV_DIR=".venv" + +# Check if python3.11 command exists (preferred) +if command -v python3.11 &> /dev/null; then + PYTHON_CMD="python3.11" + echo "Using Python 3.11: $(python3.11 --version)" +elif command -v python3 &> /dev/null; then + PYTHON_CMD="python3" + PYTHON_VERSION=$(python3 --version 2>&1 | grep -oE '[0-9]+\.[0-9]+') + echo "Using Python: $(python3 --version)" + if [[ "$PYTHON_VERSION" != "3.11" ]]; then + echo "WARNING: Python 3.11 is recommended but not found. Using $PYTHON_VERSION instead." + echo "If you encounter issues, please install Python 3.11 specifically." + fi +else + echo "ERROR: Neither python3.11 nor python3 command found." + echo "Please install Python 3.11 (recommended) or Python 3.x and rerun the script." + echo "On macOS, you can install Python 3.11 using:" + echo " brew install python@3.11" + echo "Or download from: https://www.python.org/downloads/" + exit 1 +fi + +# --- Check/Install uv --- +echo "Checking for uv..." +if ! command -v uv &> /dev/null; then + echo "'uv' command not found. Attempting to install uv using pip..." + # Ensure pip is available (it should be with the selected Python version) + if ! $PYTHON_CMD -m pip --version &> /dev/null; then + echo "ERROR: $PYTHON_CMD -m pip command failed. Cannot install uv." + exit 1 + fi + $PYTHON_CMD -m pip install uv + if [ $? -ne 0 ]; then + echo "ERROR: Failed to install uv using pip. Please install uv manually (e.g., 'pip install uv' or 'brew install uv') and rerun the script." + exit 1 + fi + echo "uv installed successfully." +else + echo "uv found." +fi +# --- End Check/Install uv --- + + +# Create virtual environment if it doesn't exist +if [ ! -d "$VENV_DIR" ]; then + echo "Creating virtual environment in '$VENV_DIR' using $PYTHON_CMD..." + $PYTHON_CMD -m venv "$VENV_DIR" + if [ $? -ne 0 ]; then + echo "ERROR: Failed to create Python virtual environment using $PYTHON_CMD." + exit 1 + fi +else + echo "Virtual environment '$VENV_DIR' already exists." +fi + +# Activate virtual environment +echo "Activating virtual environment..." +source "$VENV_DIR/bin/activate" +if [ $? -ne 0 ]; then + echo "ERROR: Failed to activate Python virtual environment." + exit 1 +fi +echo "Virtual environment activated. Python executable: $(which python)" + +# --- Attempt to install SSL certificates (macOS specific fix) --- +echo "Attempting to install SSL certificates for the active Python environment..." +# Get the full path to the python executable inside the venv +VENV_PYTHON_EXE=$(which python) +# Try to infer the Applications directory path (this is heuristic) +# Assumes Python was installed via the standard macOS installer +PYTHON_BASE_DIR=$(dirname "$(dirname "$VENV_PYTHON_EXE")") # Go up two levels from venv/bin/python +CERT_SCRIPT_PATH="" +# Check common installation patterns +if [ -d "$PYTHON_BASE_DIR/Install Certificates.command" ]; then + # Might be like /Library/Frameworks/Python.framework/Versions/3.11/Install Certificates.command + CERT_SCRIPT_PATH="$PYTHON_BASE_DIR/Install Certificates.command" +elif [ -d "/Applications/Python $(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')/Install Certificates.command" ]; then + # Check standard /Applications path using the version + PY_VERSION_SHORT=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') + CERT_SCRIPT_PATH="/Applications/Python ${PY_VERSION_SHORT}/Install Certificates.command" +fi + +if [ -n "$CERT_SCRIPT_PATH" ] && [ -f "$CERT_SCRIPT_PATH" ]; then + echo "Found certificate installation script: $CERT_SCRIPT_PATH" + echo "Running certificate installation script..." + # Execute the script using the venv's python + "$CERT_SCRIPT_PATH" + if [ $? -ne 0 ]; then + echo "WARNING: Certificate installation script finished with a non-zero status. SSL issues might persist." + # Continue anyway, maybe it partially worked or wasn't strictly needed + else + echo "Certificate installation script completed successfully." + fi +else + echo "WARNING: Could not find the 'Install Certificates.command' script automatically." + echo "If you encounter SSL errors (e.g., downloading NLTK data, used in chunking), you may need to run it manually." + echo "Look for it in your Python application folder (e.g., /Applications/Python 3.11/)." +fi +# --- End SSL Certificate Install --- + + +# Install dependencies using uv +echo "Installing Python dependencies from requirements.txt using uv..." +# Assuming uv is in PATH after the check above and venv is activated +uv pip install -r requirements.txt +if [ $? -ne 0 ]; then + echo "ERROR: Failed to install Python dependencies using uv. Please check requirements.txt and uv logs." + exit 1 # Cleanup trap will handle exit +fi +echo "Python dependencies installed successfully." + +# Download spaCy model for anonymization +echo "Downloading spaCy model (en_core_web_lg) for anonymization..." +python3 -m spacy download en_core_web_lg +if [ $? -ne 0 ]; then + echo "WARNING: Failed to download spaCy model. Anonymization might fall back to NLTK." +fi + +# --- Configure SSL Certificate Path --- +echo "----------------------------------------" +echo "Configuring SSL certificate path..." + +# Ensure certifi is installed (uv pip install should handle this if in requirements.txt, +# but explicit check adds robustness - though uv handles it well) +# It's likely already installed as a dependency of requests/huggingface_hub etc. +# If dependency issues arise, add 'certifi' explicitly to requirements.txt + +# Get the certificate path from certifi +CERT_PATH=$(python -c "import certifi; print(certifi.where())" 2>/dev/null) # Suppress stderr in case certifi isn't installed yet + +if [ -n "$CERT_PATH" ] && [ -f "$CERT_PATH" ]; then + echo "Setting SSL_CERT_FILE environment variable to: $CERT_PATH" + export SSL_CERT_FILE="$CERT_PATH" + echo "SSL_CERT_FILE set for this script session and its children." + # Optional: Add a connection test here if desired + # python -c "import requests; r = requests.get('https://huggingface.co', timeout=10); print(f'Connection test status: {r.status_code}')" + # if [ $? -ne 0 ]; then echo "WARNING: Connection test failed after setting SSL_CERT_FILE."; fi +else + echo "WARNING: Could not determine certificate path using certifi. Certifi might not be installed yet or there's an issue." + echo "Attempting to proceed, but SSL errors (like CERTIFICATE_VERIFY_FAILED) may occur." + echo "Ensure 'certifi' is listed in requirements.txt if SSL errors persist." +fi +# --- End SSL Configuration --- + + +# --- Helper Process Logging Setup --- +echo "----------------------------------------" +echo "Setting up helper process logging..." + +# Create helper_process_logs directory if it doesn't exist +HELPER_LOGS_DIR="helper_process_logs" +if [ ! -d "$HELPER_LOGS_DIR" ]; then + echo "Creating helper process logs directory: $HELPER_LOGS_DIR" + mkdir -p "$HELPER_LOGS_DIR" + if [ $? -ne 0 ]; then + echo "ERROR: Failed to create helper process logs directory." + exit 1 + fi +else + echo "Helper process logs directory already exists: $HELPER_LOGS_DIR" +fi + +# Generate timestamp for log files +TIMESTAMP=$(date +"%Y%m%d_%H%M%S") +HUEY_LOG_FILE="$HELPER_LOGS_DIR/huey_worker_${TIMESTAMP}.log" +UVICORN_LOG_FILE="$HELPER_LOGS_DIR/uvicorn_api_${TIMESTAMP}.log" +FRONTEND_LOG_FILE="$HELPER_LOGS_DIR/frontend_serve_${TIMESTAMP}.log" + +echo "Helper process logs will be saved to:" +echo " Huey worker: $HUEY_LOG_FILE" +echo " Uvicorn API: $UVICORN_LOG_FILE" +echo " Frontend serve: $FRONTEND_LOG_FILE" +echo "Helper process logging setup complete." + + +# --- 1.5 Valkey (Redis Fork) Check & Installation --- +echo "----------------------------------------" +echo "1.5. Checking for Valkey (Redis fork)..." + +# First, check if something is already listening on port 6379 +if command -v lsof &> /dev/null; then + PORT_CHECK_CMD="lsof -i :6379" +elif command -v netstat &> /dev/null; then + PORT_CHECK_CMD="netstat -an" +else + PORT_CHECK_CMD="" +fi + +if [ -n "$PORT_CHECK_CMD" ]; then + if [ "$PORT_CHECK_CMD" = "lsof -i :6379" ]; then + if $PORT_CHECK_CMD &> /dev/null; then + echo "Something is already listening on port 6379. Assuming a compatible Valkey/Redis server is running." + echo "Skipping Valkey installation and setup." + else + NEED_VALKEY_SETUP=true + fi + else + # netstat case + if $PORT_CHECK_CMD | grep -q ':6379'; then + echo "Something is already listening on port 6379. Assuming a compatible Valkey/Redis server is running." + echo "Skipping Valkey installation and setup." + else + NEED_VALKEY_SETUP=true + fi + fi +else + echo "WARNING: Cannot check port 6379 (lsof and netstat not available). Proceeding with Valkey setup." + NEED_VALKEY_SETUP=true +fi + +if [ "$NEED_VALKEY_SETUP" = true ]; then + # Check if brew command exists + +# Check if brew command exists +if ! command -v brew &> /dev/null; then + echo "Homebrew (brew) command not found." + read -p "Homebrew is required to install Valkey. Install Homebrew now? (y/N) " -n 1 -r + echo # Move to a new line after input + if [[ $REPLY =~ ^[Yy]$ ]]; then + echo "Attempting to install Homebrew. This may take a few minutes and will likely ask for your administrator password..." + # Execute the official Homebrew installation script + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + # Check if installation succeeded by seeing if 'brew' command is now available + # Might require sourcing profile or opening new shell, but let's check command first + if ! command -v brew &> /dev/null; then + echo "ERROR: Homebrew installation appears to have failed or is not available in this shell session." + echo "Please try installing Homebrew manually from https://brew.sh/ and run this script again, possibly in a new terminal window." + exit 1 + else + # Add brew to PATH for the current script execution if possible + # This is complex as the path depends on architecture (Intel vs ARM) + # For simplicity, we'll rely on the installer potentially modifying the profile + # or the user running in a new shell later if 'brew' isn't found immediately after install. + # A more robust solution might involve detecting arch and adding the known path. + echo "Homebrew installed successfully. Proceeding..." + # Re-check explicitly here just in case the path isn't immediately active + if ! command -v brew &> /dev/null; then + echo "ERROR: Homebrew installed but 'brew' command still not found in PATH for this session." + echo "Please open a new terminal window and run this script again." + exit 1 + fi + fi + else + echo "Homebrew installation declined. Valkey cannot be installed automatically." + echo "Please install Homebrew (https://brew.sh/) and Valkey ('brew install valkey') manually and rerun the script." + exit 1 + fi +fi +echo "Homebrew found." + +# Check if valkey is installed +if ! brew list valkey &>/dev/null; then + echo "Valkey not found via Homebrew. Attempting to install..." + # Update brew minimally and install valkey + # Using 'brew install' directly avoids running 'brew update' if possible, + # but Homebrew might still update itself or core formulae. + # We avoid 'brew upgrade' to prevent upgrading all packages. + brew install valkey + if [ $? -ne 0 ]; then + echo "ERROR: Failed to install Valkey using Homebrew. Please install it manually ('brew install valkey') and rerun the script." + exit 1 + fi + echo "Valkey installed successfully." +else + echo "Valkey found via Homebrew." +fi + +# Check if Valkey service is running, start if not +if ! brew services list | grep -q "valkey.*started"; then + echo "Valkey service is not running. Starting it now..." + brew services start valkey + if [ $? -ne 0 ]; then + echo "ERROR: Failed to start Valkey service using Homebrew. Please check Homebrew services ('brew services list') and start Valkey manually if needed." + exit 1 + fi + # Give it a moment to start + sleep 2 + if ! brew services list | grep -q "valkey.*started"; then + echo "ERROR: Valkey service failed to stay running after starting. Please investigate ('brew services list', check logs)." + exit 1 + fi + echo "Valkey service started successfully." +else + echo "Valkey service is already running." +fi +fi # End NEED_VALKEY_SETUP conditional +# --- End Valkey Check --- + + +# --- 2. Huey Worker --- +echo "----------------------------------------" +echo "2. Starting Huey worker in the background (using venv)..." +echo "Output will be logged to: $HUEY_LOG_FILE" +# Call the huey_consumer script directly (should be in PATH from venv) +# Redirect both stdout and stderr to the log file +huey_consumer tasks.huey > "$HUEY_LOG_FILE" 2>&1 & +HUEY_PID=$! +# Give it a moment to start/fail +sleep 2 +if ! kill -0 $HUEY_PID &> /dev/null; then + echo "ERROR: Huey worker (PID $HUEY_PID) failed to start or crashed immediately." + echo "Check the log file for details: $HUEY_LOG_FILE" + exit 1 # Cleanup trap will handle exit +fi +echo "Huey worker started successfully with PID $HUEY_PID." + + +# --- 3. API Server --- +echo "----------------------------------------" +echo "3. Starting FastAPI server (Uvicorn) in the background (using venv)..." +echo "Output will be logged to: $UVICORN_LOG_FILE" +# Host 0.0.0.0 makes it accessible on the network +# Use python -m uvicorn to ensure using the venv uvicorn +# Redirect both stdout and stderr to the log file +python -m uvicorn api:app --host 0.0.0.0 --port 8000 > "$UVICORN_LOG_FILE" 2>&1 & +UVICORN_PID=$! +# Give it a moment to start/fail +sleep 2 +if ! kill -0 $UVICORN_PID &> /dev/null; then + echo "ERROR: Uvicorn server (PID $UVICORN_PID) failed to start or crashed immediately." + echo "Check the log file for details: $UVICORN_LOG_FILE" + # Cleanup trap will handle stopping Huey + exit 1 # Cleanup trap will handle exit +fi +echo "Uvicorn server started successfully with PID $UVICORN_PID. Access API at http://localhost:8000" + + +# --- 4. NPM Check --- +echo "----------------------------------------" +echo "4. Checking for npm..." +if ! command -v npm &> /dev/null; then + echo "ERROR: npm command not found." + echo "Please install Node.js and npm (e.g., from https://nodejs.org/) and run this script again." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + exit 1 +fi +echo "npm found." + + +# --- 5. Frontend Dependencies --- +echo "----------------------------------------" +echo "5. Installing frontend dependencies in atk-interface..." +if [ ! -d "atk-interface" ]; then + echo "ERROR: Directory 'atk-interface' not found in $(pwd)." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + exit 1 +fi +cd atk-interface +npm install +if [ $? -ne 0 ]; then + echo "ERROR: Failed to install npm dependencies in atk-interface. Check package.json and npm logs." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + cd "$SCRIPT_DIR" # Go back to root + exit 1 +fi +echo "Frontend dependencies installed." + + +# --- 6. Frontend Build --- +echo "----------------------------------------" +echo "6. Building frontend application in atk-interface..." +npm run build +if [ $? -ne 0 ]; then + echo "ERROR: Failed to build frontend application. Check build script in package.json and logs." + echo "Stopping background services (PIDs $HUEY_PID, $UVICORN_PID)..." + kill $HUEY_PID $UVICORN_PID + cd "$SCRIPT_DIR" # Go back to root + exit 1 +fi +echo "Frontend built successfully." + + +# --- 7. Frontend Serve --- +echo "----------------------------------------" +echo "7. Serving frontend application from atk-interface/dist/ in the FOREGROUND..." +echo "Output will be logged to: $FRONTEND_LOG_FILE" +# Ensure dist directory exists after build +if [ ! -d "dist" ]; then + echo "ERROR: 'dist' directory not found in atk-interface after build." + # Cleanup trap will handle stopping background services + exit 1 +fi + +# Show summary of all log files +echo "" +echo "═══════════════════════════════════════════════════════════════════════════════" +echo "All helper processes are now running with output redirected to log files:" +echo " • Huey worker logs: $HUEY_LOG_FILE" +echo " • Uvicorn API logs: $UVICORN_LOG_FILE" +echo " • Frontend serve logs: $FRONTEND_LOG_FILE" +echo "" +echo "You can monitor these logs in real-time with:" +echo " tail -f $HUEY_LOG_FILE" +echo " tail -f $UVICORN_LOG_FILE" +echo " tail -f $FRONTEND_LOG_FILE" +echo "═══════════════════════════════════════════════════════════════════════════════" +echo "" + +# Serve on port 5173 (matches CORS config) +# Run npx serve in the background first, then open browser once it's ready +echo "Starting serve process (output redirected to log file)..." +# Redirect both stdout and stderr to the log file - use ../ to go back to script directory +npx serve -s dist --listen 5173 > "../$FRONTEND_LOG_FILE" 2>&1 & +SERVE_PID=$! + +# Wait for the serve process to be ready +echo "Waiting for frontend server to start..." +sleep 3 + +# Check if serve process is still running +if ! kill -0 $SERVE_PID &> /dev/null; then + echo "ERROR: Frontend serve process failed to start or crashed immediately." + echo "Check the log file for details: $FRONTEND_LOG_FILE" + cleanup + exit 1 +fi + +# Wait for the server to actually be listening on port 5173 +echo "Checking if server is responding..." +for i in {1..10}; do + if command -v curl &> /dev/null; then + if curl -s http://localhost:5173 > /dev/null 2>&1; then + break + fi + elif command -v nc &> /dev/null; then + if nc -z localhost 5173 > /dev/null 2>&1; then + break + fi + else + # Fallback: just check if something is listening on port 5173 + if command -v lsof &> /dev/null; then + if lsof -i :5173 > /dev/null 2>&1; then + break + fi + elif command -v netstat &> /dev/null; then + if netstat -an | grep -q ':5173'; then + break + fi + else + # No way to check, just wait a bit more + sleep 1 + break + fi + fi + sleep 1 +done + +echo "Opening interface..." + +# Open the URL in the default browser (macOS) +echo "Opening http://localhost:5173 in default browser..." +open http://localhost:5173 + +echo "Frontend server is now running. Press Ctrl+C to stop all services." + +# Wait for the serve process to finish (it's now running in the background) +wait $SERVE_PID +SERVE_EXIT_CODE=$? +SERVE_PID="" # Clear PID as it has exited + +if [ $SERVE_EXIT_CODE -ne 0 ]; then + echo "WARNING: Frontend serve process exited with code $SERVE_EXIT_CODE." + echo "Check the log file for details: $FRONTEND_LOG_FILE" + # Consider if we should stop background services here if serve crashes + # Calling cleanup manually if serve fails + cleanup + exit 1 # Exit with error code +fi + + +# --- 8. Done (Only reached if npx serve exits gracefully, e.g., if killed separately) --- +cd "$SCRIPT_DIR" # Go back to root directory +echo "----------------------------------------" +echo "Frontend serve process finished." +echo "" +echo "Helper process log files were saved to:" +echo " • Huey worker logs: $HUEY_LOG_FILE" +echo " • Uvicorn API logs: $UVICORN_LOG_FILE" +echo " • Frontend serve logs: $FRONTEND_LOG_FILE" +# Call cleanup to stop background services if serve finishes normally +cleanup + +exit 0 \ No newline at end of file diff --git a/ocr/.env.example b/ocr/.env.example new file mode 100644 index 0000000000000000000000000000000000000000..66cf8d8851d1120ee6efc9542c71c134f86330ba --- /dev/null +++ b/ocr/.env.example @@ -0,0 +1,23 @@ +# Securely provision values, do not commit .env or print the process environment. +OCR_DISPATCHER_TOKEN= +OCR_CPU_AUTH_TOKEN= +OCR_MODELS_DIR=/opt/ocr-models +# Empty means trusted local CPU. Remote URL must be HTTPS. +OCR_CPU_URL= +OCR_MIN_SENSITIVITY=D2 +OCR_ALLOW_EXTERNAL_CPU_D0=true +OCR_ALLOW_EXTERNAL_CPU_D1=false +OCR_ALLOW_EXTERNAL_CPU_D2=false +OCR_ENABLE_GPU=false +FREELLMAPI_BASE_URL= +FREELLMAPI_API_KEY= +OCR_GPU_MODEL=alias-ocr +OCR_CPU_CONCURRENCY=1 +OCR_GPU_CONCURRENCY=2 +OCR_ENABLE_GPU_OVERFLOW=false +OCR_OVERFLOW_BENCHMARK_QUALIFIED=false +OCR_QUALITY_DUAL_RUN=false +OCR_MAX_UPLOAD_MIB=25 +OCR_MAX_PAGES=100 +OCR_MAX_PIXELS=20000000 +OCR_JOB_TTL_MIN=30 diff --git a/ocr/LICENSE b/ocr/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..f5e33c961122915efb06c5a6314806f58cec6c3b --- /dev/null +++ b/ocr/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Evan Armstrong + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ocr/README.md b/ocr/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bd9fdb33dab2d5555e94be17148816c7fc601364 --- /dev/null +++ b/ocr/README.md @@ -0,0 +1,17 @@ +# Hybrid CPU-first OCR + +Separate private dispatcher and authenticated PP-OCRv6 Small CPU worker. +No production integration or startup occurs on import. See: + +- [API and schemas](docs/api.md) +- [Install, HF packaging, secrets and rollback](docs/deployment.md) +- [Routing, limits and cancellation](docs/routing.md) +- [Reproducible benchmarks](docs/benchmark.md) and [distribution notices](THIRD_PARTY.md) +- [Security boundaries](docs/threat-model.md) +- [Audit](docs/audit.md) and [decisions/deviations](docs/adr.md) +- [Release-readiness report](../docs/OCR_RELEASE_READINESS.md) + +`python -m ocr.cli run file.pdf --format markdown` runs trusted local CPU. +`python -m ocr.cli probe-alias` uses a synthetic public image and credentials from +runtime environment. `python -m ocr.benchmark` generates a CC0 synthetic DE/EN +corpus and measures real CPU OCR. Live alias and all D2 remote routing default off. diff --git a/ocr/THIRD_PARTY.md b/ocr/THIRD_PARTY.md new file mode 100644 index 0000000000000000000000000000000000000000..3c00ecdb9acd45596906ae059c10f76f402d6794 --- /dev/null +++ b/ocr/THIRD_PARTY.md @@ -0,0 +1,23 @@ +# Distribution notices + +The application code inherits the repository's MIT license, copied to `ocr/LICENSE` +for a standalone worker staging directory. The Space README's license field refers +to that application code, not to every bundled model/dependency. + +PP-OCRv6 Small detector/recognizer artifacts come from the RapidAI/RapidOCR +ModelScope repository, revision `v3.10.0`; its public metadata declares Apache +License 2.0. Exact HTTPS URLs and upstream SHA256s are in `models.json`. Preserve +Apache license/attribution requirements when distributing the baked model files. +The pinned RapidOCR distribution includes its Apache-2.0 full license in installed +distribution metadata. No additional model redistribution rights are asserted. + +Runtime dependencies retain their installed `.dist-info` license files in the +Docker image. Key metadata: RapidOCR Apache-2.0; ONNX Runtime MIT; Pillow MIT-CMU; +pypdf BSD-3-Clause; pypdfium2 BSD-3-Clause/Apache-2.0 plus bundled PDFium dependency +licenses; FastAPI MIT; HTTPX BSD-3-Clause. See those full installed notices rather +than treating this summary as a substitute. Debian's DejaVu font copyright notice +is retained under `/usr/share/doc/fonts-dejavu-core/copyright`. + +Review all transitive notices and operator/provider terms before publication. +Document ownership/processing rights and downstream provider compliance are the +operator's responsibility; the software makes no regulatory certification claim. diff --git a/ocr/__init__.py b/ocr/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c0b42b19c305df2cfd28e5d5fd70110e9cce4b29 --- /dev/null +++ b/ocr/__init__.py @@ -0,0 +1,3 @@ +"""CPU-first OCR. No backend or network is initialized on import.""" + +__version__ = "1.0.0" diff --git a/ocr/adapters/__init__.py b/ocr/adapters/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/ocr/adapters/cpu_http.py b/ocr/adapters/cpu_http.py new file mode 100644 index 0000000000000000000000000000000000000000..8de77e2cbd29af1163e6d83a820a5f7d6c5b61c4 --- /dev/null +++ b/ocr/adapters/cpu_http.py @@ -0,0 +1,61 @@ +import httpx + +from ..schemas import BackendCapabilities, BackendHealth, OCRPageResult, SafeError +from .http import request_json + + +class RemoteCPU: + name = "cpu" + external = True + + def __init__(self, url, token, timeout=90, transport=None): + if not token: + raise ValueError("worker_token_required") + self.url = url + self.destination = url + self.client = httpx.AsyncClient( + timeout=timeout, + follow_redirects=False, + transport=transport, + headers={"Authorization": "Bearer " + token}, + ) + + async def close(self): + await self.client.aclose() + + async def capabilities(self): + # Conservative local contract; don't inherit a remote service's sensitive-class claims. + return BackendCapabilities( + name="cpu", + bounding_boxes=True, + languages=["de", "en"], + supports_sensitive_classes=["D0"], + multimodal_verified=True, + ) + + async def health(self): + try: + r = await request_json(self.client, "GET", self.url + "/ready", limit=4096) + return BackendHealth(ready=r.get("ready") is True) + except SafeError: + return BackendHealth(ready=False, code="cpu_worker_unavailable") + + async def recognize(self, page, options): + r = await request_json( + self.client, + "POST", + self.url + "/internal/ocr/page", + files={"file": ("page.png", page.image, "image/png")}, + data={"request_id": page.page_id, "options": "{}"}, + ) + try: + result = OCRPageResult.model_validate(r) + except Exception: + raise SafeError("invalid_cpu_response") from None + if result.source != "cpu_ocr" or result.engine != "pp-ocrv6-small": + raise SafeError("unexpected_cpu_engine") + if result.image_width != page.width or result.image_height != page.height: + raise SafeError("cpu_geometry_mismatch") + result.page_index = page.index + result.provider_destination = self.destination + return result diff --git a/ocr/adapters/freellm_alias.py b/ocr/adapters/freellm_alias.py new file mode 100644 index 0000000000000000000000000000000000000000..3413881549758e9dd1280cbd0a2e0ac99d853291 --- /dev/null +++ b/ocr/adapters/freellm_alias.py @@ -0,0 +1,160 @@ +import base64 +import io +import time + +import httpx +from PIL import Image, ImageDraw + +from ..normalize import grounding +from ..schemas import BackendCapabilities, BackendHealth, OCRPageResult, SafeError +from .http import request_json + + +class FreeLLMAlias: + name = "gpu" + external = True + + def __init__(self, url, key, timeout=90, transport=None): + if not url or not key: + raise ValueError("alias_configuration_missing") + self.url = url + self.destination = None + self.qualified = False + self.qualified_at = 0 + self.actual_model = None + self.grounding_verified = False + self.client = httpx.AsyncClient( + timeout=timeout, + follow_redirects=False, + transport=transport, + headers={"Authorization": "Bearer " + key}, + ) + + async def close(self): + await self.client.aclose() + + async def capabilities(self): + return BackendCapabilities( + name="gpu", + layout_blocks=self.grounding_verified, + bounding_boxes=self.grounding_verified, + grounding_verified=self.grounding_verified, + multimodal_verified=self.qualified, + effective_destination_verifiable=False, + supports_sensitive_classes=["D0"] if self.qualified else [], + ) + + async def health(self): + return BackendHealth( + ready=self.qualified and time.monotonic() - self.qualified_at < 3600, + code="ok" if self.qualified else "alias_unqualified", + ) + + async def _completion(self, image, *, probe=False): + # Separate copy: CPU source raster and coordinate system remain untouched. + with Image.open(io.BytesIO(image)) as im: + im = im.convert("RGB") + im.thumbnail((2048, 2048)) + b = io.BytesIO() + im.save(b, format="PNG") + content = [ + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64," + base64.b64encode(b.getvalue()).decode() + }, + } + ] + # No unverified provider-specific options, tools or thinking flags. + content.append( + { + "type": "text", + "text": "Transcribe the document verbatim. Return only document text or Markdown.", + } + ) + data = await request_json( + self.client, + "POST", + self.url + "/chat/completions", + json={ + "model": "alias-ocr", + "messages": [{"role": "user", "content": content}], + "stream": False, + "temperature": 0.1, + }, + ) + try: + choice = data["choices"][0] + text = choice["message"]["content"] + if choice.get("finish_reason") not in ("stop",): + raise SafeError("alias_truncated") + if not isinstance(text, str) or not text.strip(): + raise SafeError("alias_empty") + if "" in text or "" in text: + raise SafeError("alias_reasoning_output") + if len(text) > 2_000_000: + raise SafeError("backend_response_limit") + model = data.get("model") + # Response metadata is recorded as an observation, NEVER proof of enforced routing. + observed = data.get("provider") if isinstance(data.get("provider"), str) else None + return ( + text, + model if isinstance(model, str) and model != "alias-ocr" else None, + observed, + ) + except SafeError: + raise + except (KeyError, IndexError, TypeError): + raise SafeError("alias_invalid_response") from None + + async def probe(self): + self.qualified = False + # Synthetic D0 challenge text is not included in the prompt; a text-only echo cannot pass. + import secrets + + challenge = "OCR " + secrets.token_hex(3).upper() + im = Image.new("RGB", (800, 160), "white") + d = ImageDraw.Draw(im) + from PIL import ImageFont + + font = ImageFont.truetype("DejaVuSans.ttf", 48) + d.text((25, 40), challenge, fill="black", font=font) + b = io.BytesIO() + im.save(b, format="PNG") + text, model, provider = await self._completion(b.getvalue(), probe=True) + if challenge.casefold() not in text.casefold(): + raise SafeError("alias_multimodal_probe_failed") + self.qualified = True + self.qualified_at = time.monotonic() + self.actual_model = model + # Observed metadata is not a verified effective destination/fallback guarantee. + return { + "multimodal_verified": True, + "observed_model": model, + "observed_provider": provider, + "effective_destination_verified": False, + "grounding_verified": False, + "d2_qualified": False, + } + + async def recognize(self, page, options): + if not (await self.health()).ready: + raise SafeError("alias_unqualified") + text, model, provider = await self._completion(page.image) + # Model changes revoke prior probe qualification, even for D0. + if model != self.actual_model: + self.qualified = False + raise SafeError("alias_model_changed") + blocks = grounding(text) if self.grounding_verified else [] + return OCRPageResult( + page_index=page.index, + source="alias_ocr", + engine=model, + provider_destination=None, + text=text, + markdown=text, + image_width=page.width, + image_height=page.height, + blocks=blocks, + flags=["destination_unverified"] + ([] if blocks else ["boxes_unavailable"]), + ) diff --git a/ocr/adapters/http.py b/ocr/adapters/http.py new file mode 100644 index 0000000000000000000000000000000000000000..1ceb09b25730decf7ec41398eedae594bd80857b --- /dev/null +++ b/ocr/adapters/http.py @@ -0,0 +1,30 @@ +import json + +import httpx + +from ..schemas import SafeError + + +async def request_json(client, method, url, *, limit=4 * 1024 * 1024, **kwargs): + try: + async with client.stream(method, url, **kwargs) as r: + if r.status_code >= 400: + raise SafeError( + f"backend_http_{r.status_code}" + if r.status_code in (429, 502, 503, 401, 403) + else "backend_http_error", + r.status_code in (429, 502, 503), + ) + data = bytearray() + async for part in r.aiter_bytes(): + data.extend(part) + if len(data) > limit: + raise SafeError("backend_response_limit") + try: + return json.loads(data) + except (ValueError, TypeError): + raise SafeError("backend_invalid_json") from None + except SafeError: + raise + except (httpx.TimeoutException, httpx.TransportError): + raise SafeError("backend_unavailable", True) from None diff --git a/ocr/adapters/rapidocr_cpu.py b/ocr/adapters/rapidocr_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..075f36be7412b62e3b23fdec63b02e7b06e4ac80 --- /dev/null +++ b/ocr/adapters/rapidocr_cpu.py @@ -0,0 +1,146 @@ +import asyncio +import hashlib +import importlib.metadata +import io +import json +from pathlib import Path + +import numpy as np +from PIL import Image + +from ..normalize import polygon +from ..schemas import ( + BackendCapabilities, + BackendHealth, + Block, + OCRPageResult, + SafeError, +) + + +class RapidCPU: + name = "cpu" + destination = "local-cpu" + external = False + + def __init__(self, models_dir): + self.models_dir = Path(models_dir) + self.engine = None + self.lock = asyncio.Lock() + + def _initialize(self): + import cv2 + from rapidocr import EngineType, ModelType, OCRVersion, RapidOCR + + cv2.setNumThreads(2) + + manifest = json.loads((Path(__file__).parents[1] / "models.json").read_text()) + params = { + "Global.use_cls": False, + "Global.log_level": "critical", + "EngineConfig.onnxruntime.intra_op_num_threads": 2, + "EngineConfig.onnxruntime.inter_op_num_threads": 1, + "EngineConfig.onnxruntime.use_cuda": False, + } + for kind in ("Det", "Rec"): + item = manifest[kind] + path = self.models_dir / item["filename"] + if ( + not path.is_file() + or hashlib.sha256(path.read_bytes()).hexdigest() != item["sha256"] + ): + raise SafeError("model_integrity_unavailable") + params.update( + { + f"{kind}.model_path": str(path), + f"{kind}.model_type": ModelType.SMALL, + f"{kind}.ocr_version": OCRVersion.PPOCRV6, + f"{kind}.engine_type": EngineType.ONNXRUNTIME, + } + ) + self.engine = RapidOCR(params=params) + # Force lazy models to initialize before readiness; never download implicitly. + self.engine(np.full((96, 256, 3), 255, dtype=np.uint8), use_cls=False) + self.engine( + np.full((48, 256, 3), 255, dtype=np.uint8), + use_det=False, + use_cls=False, + use_rec=True, + ) + self.engine.update_params(use_det=True, use_cls=False, use_rec=True) + for model in (self.engine.text_det, self.engine.text_rec): + if model is None or model.session.session.get_providers() != ["CPUExecutionProvider"]: + self.engine = None + raise SafeError("unexpected_cpu_execution_provider") + + async def initialize(self): + async with self.lock: + if self.engine is None: + try: + await asyncio.to_thread(self._initialize) + except SafeError: + raise + except Exception: + self.engine = None + raise SafeError("cpu_initialization_failed") from None + + async def capabilities(self): + return BackendCapabilities( + name="cpu", + bounding_boxes=True, + languages=["de", "en"], + supports_sensitive_classes=["D0", "D1", "D2"], + effective_destination_verifiable=True, + multimodal_verified=self.engine is not None, + ) + + async def health(self): + return BackendHealth( + ready=self.engine is not None, + code="ok" if self.engine else "model_not_ready", + ) + + def _recognize(self, page): + image = np.asarray(Image.open(io.BytesIO(page.image)).convert("RGB"))[:, :, ::-1].copy() + out = self.engine(image, use_cls=False) + blocks = [] + if out.txts is not None: + for text, score, box in zip(out.txts, out.scores, out.boxes, strict=True): + p, b = polygon(box, page.width, page.height) + blocks.append( + Block( + text=text, + confidence=float(score), + confidence_kind="rapidocr-recognition", + polygon_1000=p, + bbox_1000=b, + ) + ) + # Preserve engine reading order; multi-column order is explicitly flagged by quality checks. + text = "\n".join(b.text for b in blocks) + return OCRPageResult( + page_index=page.index, + source="cpu_ocr", + engine="pp-ocrv6-small", + backend_version=f"rapidocr/{importlib.metadata.version('rapidocr')};onnxruntime/{importlib.metadata.version('onnxruntime')}", + provider_destination=self.destination, + text=text, + markdown=text, + image_width=page.width, + image_height=page.height, + blocks=blocks, + ) + + async def recognize(self, page, options): + if self.engine is None: + raise SafeError("model_not_ready") + # Shield native work and retain the lock through cancellation: ORT cannot be force-cancelled. + async with self.lock: + task = asyncio.create_task(asyncio.to_thread(self._recognize, page)) + try: + return await asyncio.shield(task) + except asyncio.CancelledError: + await task + raise + except Exception: + raise SafeError("cpu_inference_failed") from None diff --git a/ocr/anonymization/__init__.py b/ocr/anonymization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..aed9160f820185d99ab5f9814e7315e63eac31bd --- /dev/null +++ b/ocr/anonymization/__init__.py @@ -0,0 +1,15 @@ +"""Explicit trusted-layer anonymization; no legacy fallback or network inference.""" + +from .candidate_io import parse_candidate +from .engine import AnonymizationEngine +from .release_policy import ReleaseAuthority +from .schemas import Entity, LegacyReleaseCandidate, ReleaseCandidate + +__all__ = [ + "AnonymizationEngine", + "Entity", + "ReleaseAuthority", + "ReleaseCandidate", + "LegacyReleaseCandidate", + "parse_candidate", +] diff --git a/ocr/anonymization/__main__.py b/ocr/anonymization/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..23a8697e92d1bd677c59f2e62a92e9f9b8f86691 --- /dev/null +++ b/ocr/anonymization/__main__.py @@ -0,0 +1,40 @@ +"""Trusted local workflow: OCRResult JSON -> candidate JSON. No release shortcut.""" + +import argparse +from pathlib import Path + +from ocr.handoff import anonymization_handoff +from ocr.schemas import OCRResult, SafeError + +from .atomic_output import write_candidate +from .engine import AnonymizationEngine +from .recognizer import GLiNERRecognizer + + +def main(): + parser = argparse.ArgumentParser( + description="Local OCR anonymization candidate (review required)" + ) + parser.add_argument("--model-cache", required=True) + parser.add_argument( + "--input", required=True, help="Classified OCRResult JSON; authorized local filesystem only" + ) + parser.add_argument( + "--output", required=True, help="New candidate JSON file (still classified)" + ) + args = parser.parse_args() + try: + path = Path(args.input) + if path.stat().st_size > 8 * 1024 * 1024: + raise SafeError("anonymization_input_limit") + evidence = OCRResult.model_validate_json(path.read_bytes()) + model = GLiNERRecognizer(args.model_cache) + candidate = anonymization_handoff(evidence, AnonymizationEngine(model)) + write_candidate(args.output, (candidate.model_dump_json(indent=2) + "\n").encode()) + except Exception: + raise SystemExit("anonymization_candidate_failed") from None + print("candidate_created_review_required") + + +if __name__ == "__main__": + main() diff --git a/ocr/anonymization/approval_ledger.py b/ocr/anonymization/approval_ledger.py new file mode 100644 index 0000000000000000000000000000000000000000..429f0eaca203bd982c716469559628f09d1a17b4 --- /dev/null +++ b/ocr/anonymization/approval_ledger.py @@ -0,0 +1,109 @@ +"""Explicitly provisioned, persistent single-host SQLite replay ledger.""" + +import hashlib +import os +import sqlite3 +import stat +import time +import uuid +from contextlib import closing +from pathlib import Path + +from ocr.schemas import SafeError + + +class SQLiteApprovalLedger: + """Provision once with initialize=True; ordinary startup NEVER creates storage. + + Persist ledger_id in private configuration and supply expected_id on restart. + Local ledgers must not share a signing key across hosts. A genuinely shared + durable backend is required for a shared key domain, not SQLite copies/NFS. + """ + + def __init__(self, path, *, initialize=False, expected_id=None): + self.path = Path(path).absolute() + try: + if initialize: + fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600) + os.close(fd) + self._metadata() + with closing(self._connect()) as connection, connection: + if initialize: + connection.execute("PRAGMA synchronous=FULL") + connection.execute( + "CREATE TABLE consumed (id TEXT PRIMARY KEY, expires INTEGER NOT NULL)" + ) + connection.execute("CREATE TABLE identity (id TEXT NOT NULL)") + connection.execute("INSERT INTO identity VALUES (?)", (uuid.uuid4().hex,)) + if connection.execute("PRAGMA quick_check").fetchone() != ("ok",): + raise ValueError() + self.ledger_id = connection.execute("SELECT id FROM identity").fetchone()[0] + if expected_id is not None and expected_id != self.ledger_id: + raise ValueError() + metadata = self.path.lstat() + self._inode = (metadata.st_dev, metadata.st_ino) + except Exception: + raise SafeError("approval_ledger_unavailable") from None + + def _metadata(self): + metadata, parent = self.path.lstat(), self.path.parent.lstat() + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != os.getuid() + or metadata.st_mode & 0o077 + or not stat.S_ISDIR(parent.st_mode) + or parent.st_uid != os.getuid() + or parent.st_mode & 0o077 + ): + raise ValueError() + return metadata + + def _connect(self): + return sqlite3.connect(self.path.as_uri() + "?mode=rw", uri=True, timeout=5) + + def consume(self, approval_id, expires_at): + try: + now = int(time.time()) + if type(expires_at) is not int or expires_at <= now: + return False + metadata = self._metadata() + if (metadata.st_dev, metadata.st_ino) != self._inode: + raise ValueError() + with closing(self._connect()) as connection, connection: + connection.execute("PRAGMA synchronous=FULL") + connection.execute("BEGIN IMMEDIATE") + # Lock acquisition may have waited beyond approval expiration. + # Recheck at the consumption transaction, not only before waiting. + if expires_at <= int(time.time()): + return False + if connection.execute("SELECT id FROM identity").fetchone() != (self.ledger_id,): + raise ValueError() + # Retain consumed IDs: clock rollback must not resurrect approvals + # after expiry-based garbage collection. Offline archive is operator work. + connection.execute("INSERT INTO consumed VALUES (?, ?)", (approval_id, expires_at)) + return True + except sqlite3.IntegrityError: + return False + except Exception: + raise SafeError("approval_ledger_unavailable") from None + + +def validate_key_topology(bindings, *, signing_keys): + """Compare actual key fingerprints, never caller-chosen key aliases. + + This implementation qualifies local per-host key domains only. A shared + backend declaration is NOT evidence of shared durability; that integration + remains blocked until a real backend is implemented and qualified separately. + """ + try: + groups = {} + for entry in bindings: + key = signing_keys[entry["key_id"]] + if not isinstance(key, bytes) or len(key) < 32: + raise ValueError() + fingerprint = hashlib.sha256(key).hexdigest() + groups.setdefault(fingerprint, set()).add(entry["host_id"]) + if any(len(hosts) > 1 for hosts in groups.values()): + raise ValueError() + except Exception: + raise SafeError("shared_key_multi_host_backend_not_qualified") from None diff --git a/ocr/anonymization/artifacts.json b/ocr/anonymization/artifacts.json new file mode 100644 index 0000000000000000000000000000000000000000..a5b05b2d98df0930d5e95e94c1063887e1c9bbe3 --- /dev/null +++ b/ocr/anonymization/artifacts.json @@ -0,0 +1,11 @@ +{ + "model": "VAGOsolutions/SauerkrautLM-GLiNER", + "revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", + "files": { + "gliner_config.json": "f0659e7440bff3dd8630c8c553c30e0244d3949b7e6fa6e02938fb4bb4e2e9da", + "pytorch_model.bin": "a4f25b51796426ca39bf5b59ea2d142902693b690ffc1458a792267e45e3c269", + "special_tokens_map.json": "6e3204c5a4004719c185007a745d1940471a6fb02521cfa0a00306dbb6bc5903", + "tokenizer.json": "4f65190517e4633347d84e0225b8c5c66a1a62321107ebc73c09d5cefa2b5cd5", + "tokenizer_config.json": "ae8ad41b7039b6f1121db8be5790eae14ca7226ec14110406d4068797f505705" + } +} diff --git a/ocr/anonymization/atomic_output.py b/ocr/anonymization/atomic_output.py new file mode 100644 index 0000000000000000000000000000000000000000..369dba1a511a9671da862d91f954d61d5d9727df --- /dev/null +++ b/ocr/anonymization/atomic_output.py @@ -0,0 +1,41 @@ +"""Publish a complete private file atomically, without overwriting any target.""" + +import fcntl +import os +import tempfile +from pathlib import Path + +from ocr.schemas import SafeError + + +def write_candidate(path, data): + temporary, directory = None, None + try: + path = Path(path) + directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + fcntl.flock(directory, fcntl.LOCK_SH) + # Same-directory staging guarantees a same-filesystem atomic hard link. + fd, temporary = tempfile.mkstemp(prefix=".anonymization-", dir=path.parent) + with os.fdopen(fd, "wb") as stream: + if stream.write(data) != len(data): + raise OSError("incomplete_write") + stream.flush() + os.fsync(stream.fileno()) + # link() fails if ANY target (including a symlink) already exists. + # Never use rename()/replace(), which can clobber another process's file. + os.link(temporary, path) + except Exception: + raise SafeError("anonymization_candidate_write_failed") from None + finally: + if temporary is not None: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + except OSError: + # Never print classified bytes or paths on cleanup failure. A + # leftover staging file is private (0600), never an approved export. + pass + + if directory is not None: + os.close(directory) diff --git a/ocr/anonymization/candidate_io.py b/ocr/anonymization/candidate_io.py new file mode 100644 index 0000000000000000000000000000000000000000..8dbf1dec3feada2b64ba7c2d8dbcbe34a73fb738 --- /dev/null +++ b/ocr/anonymization/candidate_io.py @@ -0,0 +1,22 @@ +"""Explicit candidate version dispatch. No rewriting, migration or approval copying.""" + +import json + +from ocr.schemas import SafeError + +from .schemas import CorrectedCandidate, LegacyReleaseCandidate, ReleaseCandidate + + +def parse_candidate(data): + try: + if not isinstance(data, (str, bytes)) or len(data) > 16 * 1024 * 1024: + raise ValueError() + version = json.loads(data)["schema_version"] + cls = { + "ocr-anonymization-candidate/v1": LegacyReleaseCandidate, + "ocr-anonymization-candidate/v2": ReleaseCandidate, + "ocr-anonymization-candidate/v3": CorrectedCandidate, + }[version] + return cls.model_validate_json(data) + except Exception: + raise SafeError("unsupported_or_invalid_candidate") from None diff --git a/ocr/anonymization/compare03d.py b/ocr/anonymization/compare03d.py new file mode 100644 index 0000000000000000000000000000000000000000..ac6db144e8da962ca20a2538cce7ab70fedaaa24 --- /dev/null +++ b/ocr/anonymization/compare03d.py @@ -0,0 +1,111 @@ +"""Development-only fixed experiments, never a claim of independent holdout quality. + +All input paths are fixed repository synthetic corpora. Per-case payload evidence +is intentionally synthetic. The three datasets are development data from Gate03d. +""" + +import hashlib +import json +import resource +import subprocess +import sys +import time +from pathlib import Path + +from .atomic_output import write_candidate +from .detectors import detect_contacts as detect +from .evaluate import score_cases +from .investigate import Recorded +from .recognizer import DEFAULT_PROMPTS, GLiNERRecognizer +from .replacement import replace, resolve_overlaps, validate_entities +from .terminology import DEFAULT_TERMS, ReligiousTerminology + + +def run(cache, output): + denied = [] + + def offline(event, args): + if event in ("socket.connect", "socket.bind", "socket.getaddrinfo"): + denied.append(event) + raise PermissionError("qualification_network_disabled") + + sys.addaudithook(offline) + start = time.perf_counter() + model = GLiNERRecognizer(cache) + load = (time.perf_counter() - start) * 1000 + model.recognize("Synthetic warmup text.") + root = Path(__file__).parent / "fixtures" + paths = { + "qualification": root / "qualification.json", + "validation": root / "validation.json", + "development03c": root / "holdout03c/cases.json", + } + suites = {n: json.loads(p.read_bytes()) for n, p in paths.items()} + terminology = ReligiousTerminology() + + def hybrid(text): + return detect(text) + terminology.detect(text) + + report = dict( + schema_version="gate03d-development-comparison/v1", + source_sha=subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), + model_revision=model.revision, + corpus_sha256={n: hashlib.sha256(p.read_bytes()).hexdigest() for n, p in paths.items()}, + terminology=list(DEFAULT_TERMS), + loading_ms=load, + experiments={}, + automated_release_eligible=False, + ) + baseline = {} + # Fixed before inference. No low-threshold sweep or fixture-specific patterns. + for name, prompts, detector in ( + ("control", DEFAULT_PROMPTS, detect), + ( + "semantic_status_prompt", + DEFAULT_PROMPTS | {"RELIGION": "religion or religious non-affiliation"}, + detect, + ), + ("hybrid_terminology", DEFAULT_PROMPTS, hybrid), + ): + model.prompts = dict(prompts) + recorder = Recorded(model) + result = dict(prompts=model.prompts, threshold=model.threshold, suites={}, changes=[]) + for suite, cases in suites.items(): + result["suites"][suite] = score_cases(recorder, cases, detector=detector) + for case in cases: + text = case["text"] + raw = recorder.predictions[text] + combined = resolve_overlaps(validate_entities(text, raw + detector(text))) + output_text, _ = replace(text, combined) + predicted = {(p.start, p.end, p.label) for p in combined} + gold = {(p["start"], p["end"], p["label"]) for p in case["spans"]} + item = dict( + suite=suite, + case_id=case["id"], + false_positives=sorted(predicted - gold), + false_negatives=sorted(gold - predicted), + exact_match=output_text == case["expected"], + predictions=[p.model_dump() for p in combined], + ) + key = (suite, case["id"]) + if name == "control": + baseline[key] = item + if item["false_positives"] or item["false_negatives"]: + result["changes"].append(item) + elif item["predictions"] != baseline[key]["predictions"]: + result["changes"].append(dict(before=baseline[key], after=item)) + report["experiments"][name] = result + report["peak_rss_mib"] = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 + report["denied_network_calls"] = denied + write_candidate(output, (json.dumps(report, ensure_ascii=False, indent=2) + "\n").encode()) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Compare fixed Gate03d development alternatives") + parser.add_argument("--cache", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run(args.cache, args.output) + print("comparison_recorded_check_metrics_before_selection") diff --git a/ocr/anonymization/corrections.py b/ocr/anonymization/corrections.py new file mode 100644 index 0000000000000000000000000000000000000000..021e1e5d065456eacd616a0f38ca3663f1b24d0b --- /dev/null +++ b/ocr/anonymization/corrections.py @@ -0,0 +1,754 @@ +"""Source-bound, authorized human corrections. No recognition optimization/export. + +Machine snapshots, proposals and published effective states are immutable private +records. The host must implement durable atomic CAS publication and current ACLs; +no production identity/store is supplied. Candidate metadata contains hashes only. +""" + +import hashlib +import json +import re +import time +import uuid +from typing import Literal, Protocol + +from pydantic import Field, model_validator + +from ocr.schemas import OCRResult, SafeError + +from .private_review import ( + PrivateReviewBridge, + PrivateReviewHost, + PrivateReviewMaterial, + ReviewBinding, + SourcePageReference, +) +from .provenance import provenance +from .release_policy import digest +from .replacement import replace, validate_entities +from .schemas import ( + LABELS, + CandidatePage, + CorrectedCandidate, + CorrectionProvenance, + Entity, + Label, + RecognitionProvenance, + ReleaseCandidate, + StrictModel, +) + +POLICY = "reviewed-corrections-v1" + + +def commitment(value): + if isinstance(value, StrictModel): + value = value.model_dump(mode="json") + return hashlib.sha256( + json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False + ).encode() + ).hexdigest() + + +def author_digest(subject): + return commitment(dict(domain="private-correction-author/v1", subject=subject)) + + +class Observation(StrictModel): + observation_id: str = Field(pattern=r"^[a-f0-9]{64}$") + page_index: int = Field(ge=0) + entity: Entity = Field(repr=False) + human_correction_id: str | None = Field(default=None, pattern=r"^[a-f0-9]{32}$") + + @model_validator(mode="after") + def explicit_human_provenance(self): + if self.human_correction_id is not None and ( + self.entity.source != "deterministic" + or self.entity.model_revision != POLICY + or self.entity.score is not None + or self.entity.normalization != "identity" + ): + raise ValueError("invalid_human_observation") + return self + + +class Placeholder(StrictModel): + category: Label + token: str = Field(pattern=r"^\[[A-Z_]{1,40}\]$") + + +class MachineSnapshot(StrictModel): + schema_version: Literal["private-machine-snapshot/v1"] = "private-machine-snapshot/v1" + evidence_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + sources: tuple[SourcePageReference, ...] = Field(min_length=1, max_length=100, repr=False) + observations: tuple[Observation, ...] = Field(max_length=100_000, repr=False) + provenance: RecognitionProvenance + placeholders: tuple[Placeholder, ...] + page_sources: tuple[Literal["embedded_text", "cpu_ocr", "alias_ocr"], ...] + machine_candidate: ReleaseCandidate | None = Field(repr=False) + + @model_validator(mode="after") + def machine_only(self): + if ( + any(o.human_correction_id is not None for o in self.observations) + or {p.category for p in self.placeholders} != set(LABELS) + or len(self.placeholders) != len(LABELS) + or len(self.sources) != len(self.page_sources) + or len({s.page_index for s in self.sources}) != len(self.sources) + or sum(s.characters for s in self.sources) > 2_000_000 + or len({s.sensitivity for s in self.sources}) != 1 + or ( + self.machine_candidate is not None + and type(self.machine_candidate) is not ReleaseCandidate + ) + ): + raise ValueError("invalid_machine_snapshot") + return self + + +class CorrectionContext(StrictModel): + schema_version: Literal["private-correction-context/v1"] = "private-correction-context/v1" + material: PrivateReviewMaterial = Field(repr=False) + machine: MachineSnapshot = Field(repr=False) + effective: tuple[Observation, ...] = Field(max_length=100_000, repr=False) + + @model_validator(mode="after") + def root_binding(self): + if ( + self.material.evidence_digest != self.machine.evidence_digest + or self.material.sources != self.machine.sources + or self.material.provenance != self.machine.provenance + ): + raise ValueError("correction_context_mismatch") + candidate = self.material.candidate + if type(candidate) is CorrectedCandidate: + c = candidate.correction + if ( + c.machine_snapshot_digest != commitment(self.machine) + or c.original_evidence_digest != self.machine.evidence_digest + or c.source_references_digest + != commitment([s.model_dump(mode="json") for s in self.machine.sources]) + or c.effective_state_digest != state_digest(self.machine, self.effective) + ): + raise ValueError("correction_context_mismatch") + elif ( + self.effective != self.machine.observations + or candidate != self.machine.machine_candidate + ): + raise ValueError("correction_context_mismatch") + if candidate is None and self.material.blocking_reason != "conflicting_spans": + raise ValueError("unsupported_failed_correction_context") + return self + + +def state_digest(machine, effective): + return commitment( + dict( + machine=commitment(machine), observations=[o.model_dump(mode="json") for o in effective] + ) + ) + + +class SpanBinding(StrictModel): + page_index: int = Field(ge=0) + start: int = Field(ge=0) + end: int = Field(gt=0) + source_span_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + + +class CorrectionEdit(StrictModel): + span: SpanBinding + category: Label + action: Literal["mask", "suppress"] + rationale: Literal[ + "missing_entity", "incomplete_boundary", "wrong_category", "span_conflict", "false_positive" + ] + target_observation_ids: tuple[str, ...] = Field(max_length=100_000) + + @model_validator(mode="after") + def valid_action(self): + if ( + any(not re.fullmatch(r"[a-f0-9]{64}", s) for s in self.target_observation_ids) + or len(set(self.target_observation_ids)) != len(self.target_observation_ids) + or (self.action == "suppress") != (self.rationale == "false_positive") + or (not self.target_observation_ids) != (self.rationale == "missing_entity") + ): + raise ValueError("invalid_correction_action") + return self + + +class CorrectionProposal(StrictModel): + schema_version: Literal["private-ocr-correction/v1"] = "private-ocr-correction/v1" + policy_version: Literal["reviewed-corrections-v1"] = "reviewed-corrections-v1" + correction_id: str = Field(pattern=r"^[a-f0-9]{32}$") + review_ref: str = Field(pattern=r"^[a-zA-Z0-9_-]{1,128}$") + author_subject: str = Field(pattern=r"^[a-zA-Z0-9:_-]{1,128}$", repr=False) + evidence_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + context_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + machine_snapshot_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + prior_candidate_digest: str | None = Field(pattern=r"^[a-f0-9]{64}$") + provenance: RecognitionProvenance + edits: tuple[CorrectionEdit, ...] = Field(min_length=1, max_length=1000) + created_at: int + expires_at: int + + @model_validator(mode="after") + def short_lived(self): + if not 0 <= self.created_at < self.expires_at <= self.created_at + 900: + raise ValueError("invalid_correction_expiration") + return self + + +class StoredProposal(StrictModel): + proposal: CorrectionProposal = Field(repr=False) + revoked: bool + published_candidate_digest: str | None = Field(pattern=r"^[a-f0-9]{64}$") + + published_review_ref: str | None = Field(default=None, pattern=r"^[a-zA-Z0-9_-]{1,128}$") + + @model_validator(mode="after") + def complete_publication_pointer(self): + if (self.published_review_ref is None) != (self.published_candidate_digest is None): + raise ValueError("invalid_publication_pointer") + return self + + +class PublicationReceipt(StrictModel): + correction_id: str = Field(pattern=r"^[a-f0-9]{32}$") + proposal_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + candidate_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + context_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + review_ref: str = Field(pattern=r"^[a-zA-Z0-9_-]{1,128}$") + automated_release_eligible: Literal[False] = False + release_approved: Literal[False] = False + + +class CorrectionReview(StrictModel): + context: CorrectionContext = Field(repr=False) + record: StoredProposal = Field(repr=False) + release_approved: Literal[False] = False + + +class CorrectionHost(PrivateReviewHost, Protocol): + """Required host contract, NOT a durable storage implementation. + + store/publish must durably commit reauthorization, compare-and-swap of the + active head, immutable records and single-use publication in one transaction. + An RLock fixture demonstrates only in-process semantics. Crash/restart and + multi-process durability remain unqualified until a real adapter is supplied. + """ + + def load_correction_context(self, review_ref: str) -> CorrectionContext: ... + def correction_binding(self, correction_id: str) -> str: ... + def load_correction(self, correction_id: str) -> StoredProposal: ... + def store_correction( + self, credential: object, proposal: CorrectionProposal, expected_context: str + ) -> bool: ... + def publish_correction( + self, + credential: object, + proposal: CorrectionProposal, + context: CorrectionContext, + expected_context: str, + ) -> PublicationReceipt: ... + + # Atomic operations must reauthorize and CAS the active context, retain all + # old snapshots/proposals and consume this correction once in DURABLE storage. + + +def span_binding(context, page_index, start, end, text): + source = next(s for s in context.machine.sources if s.page_index == page_index) + if ( + type(start) is not int + or type(end) is not int + or not 0 <= start < end <= len(text) + or len(text) != source.characters + or hashlib.sha256(text.encode()).hexdigest() != source.sha256 + ): + raise ValueError() + value = commitment( + dict( + domain="classified-original-span/v1", + evidence=context.machine.evidence_digest, + page=source.model_dump(mode="json"), + start=start, + end=end, + text_sha256=hashlib.sha256(text[start:end].encode()).hexdigest(), + ) + ) + return SpanBinding(page_index=page_index, start=start, end=end, source_span_digest=value) + + +def render(machine, effective, texts): + pages = [] + allowed_pages = {s.page_index for s in machine.sources} + if any(o.page_index not in allowed_pages for o in effective): + raise ValueError() + for source, page_source in zip(machine.sources, machine.page_sources, strict=True): + text = texts[source.page_index] + if ( + len(text) != source.characters + or hashlib.sha256(text.encode()).hexdigest() != source.sha256 + ): + raise ValueError() + observations = [o for o in effective if o.page_index == source.page_index] + entities = validate_entities(text, [o.entity for o in observations]) + output, tags = replace(text, entities, {p.category: p.token for p in machine.placeholders}) + human = { + (o.entity.start, o.entity.end, o.entity.label) + for o in observations + if o.human_correction_id is not None + } + tags = tuple( + t.model_copy(update={"source": "human_correction"}) + if (t.start, t.end, t.label) in human and t.model_revision == POLICY + else t + for t in tags + ) + pages.append( + CandidatePage( + page_index=source.page_index, + anonymized_text=output, + tags=tags, + source=page_source, + original_sensitivity=source.sensitivity, + ) + ) + return tuple(pages) + + +def prepare_review(engine, result, *, review_ref, environment_id, expires_at): + """Trusted capture. Runtime failure aborts; recoverable span conflicts are retained. + + The supplied review/env references are server-operator registration inputs, not + public request fields. Return objects must be stored in an immutable private host. + """ + try: + snapshot = OCRResult.model_validate_json(result.model_dump_json()) + engine.validate_document(snapshot) + original_digest = hashlib.sha256(snapshot.model_dump_json().encode()).hexdigest() + config = provenance( + engine.recognizer, engine.terminology, engine.status_policy, engine.placeholders + ) + sources, observations, texts = [], [], {} + for page in snapshot.pages: + text = page.text + texts[page.page_index] = text + sources.append( + SourcePageReference( + evidence_ref=commitment(dict(evidence=original_digest, page=page.page_index)), + page_index=page.page_index, + sensitivity=snapshot.sensitivity, + sha256=hashlib.sha256(text.encode()).hexdigest(), + characters=len(text), + ) + ) + for entity in engine.observe(text): + payload = dict( + evidence=original_digest, + config=config.configuration_digest, + page=page.page_index, + entity=entity.model_dump(mode="json"), + ) + observations.append( + Observation( + observation_id=commitment(payload), + page_index=page.page_index, + entity=entity, + ) + ) + if config != provenance( + engine.recognizer, engine.terminology, engine.status_policy, engine.placeholders + ): + raise ValueError() + placeholders = ( + engine.placeholders + if engine.placeholders is not None + else {k: f"[{k}]" for k in LABELS} + ) + machine = MachineSnapshot( + evidence_digest=original_digest, + sources=tuple(sources), + observations=tuple(observations), + provenance=config, + placeholders=tuple( + Placeholder(category=k, token=v) for k, v in sorted(placeholders.items()) + ), + page_sources=tuple(p.source for p in snapshot.pages), + machine_candidate=None, + ) + blocked = None + try: + pages = render(machine, machine.observations, texts) + reasons = ("recognition_incomplete_possible",) + ( + ("empty_prediction",) + if any( + not any( + o.page_index == source.page_index and o.entity.source == "multilingual_ner" + for o in observations + ) + for source in sources + ) + else () + ) + candidate = ReleaseCandidate( + candidate_id=uuid.uuid4().hex, + sensitivity=snapshot.sensitivity, + pages=pages, + model_revision=config.model_revision, + provenance=config, + review_reasons=reasons, + ) + machine = machine.model_copy(update={"machine_candidate": candidate}) + except SafeError as exc: + if exc.code != "ambiguous_entity_overlap": + raise + candidate, blocked = None, "conflicting_spans" + material = PrivateReviewMaterial( + binding=ReviewBinding( + review_ref=review_ref, + environment_id=environment_id, + sensitivity=snapshot.sensitivity, + expires_at=expires_at, + revoked=False, + ), + evidence_digest=original_digest, + sources=tuple(sources), + candidate=candidate, + candidate_digest=digest(candidate) if candidate else None, + provenance=config, + blocking_reason=blocked, + ) + return CorrectionContext(material=material, machine=machine, effective=machine.observations) + except Exception: + raise SafeError("correction_capture_failed") from None + + +def apply_edits(context, proposal, texts): + if ( + proposal.context_digest != commitment(context) + or proposal.evidence_digest != context.machine.evidence_digest + or proposal.provenance != context.machine.provenance + or proposal.machine_snapshot_digest != commitment(context.machine) + or proposal.prior_candidate_digest != context.material.candidate_digest + or proposal.review_ref != context.material.binding.review_ref + ): + raise ValueError() + effective = {o.observation_id: o for o in context.effective} + if any(effective[o.observation_id] != o for o in context.effective): + raise ValueError() + ordered = sorted(proposal.edits, key=lambda e: (e.span.page_index, e.span.start, e.span.end)) + used = set() + previous = None + for index, edit in enumerate(ordered): + s = edit.span + if previous is not None and previous.page_index == s.page_index and s.start < previous.end: + raise SafeError("correction_conflict") + previous = s + if span_binding(context, s.page_index, s.start, s.end, texts[s.page_index]) != s: + raise ValueError() + targets = set(edit.target_observation_ids) + if used & targets or not targets <= set(effective): + raise SafeError("correction_conflict") + used |= targets + target_obs = [effective[k] for k in targets] + overlap = { + k + for k, o in effective.items() + if o.page_index == s.page_index and o.entity.start < s.end and s.start < o.entity.end + } + if edit.action == "mask": + if targets != overlap: + raise SafeError("correction_conflict") + if edit.rationale == "wrong_category" and any( + (o.entity.start, o.entity.end) != (s.start, s.end) for o in target_obs + ): + raise ValueError() + elif any( + o.page_index != s.page_index + or o.entity.start != s.start + or o.entity.end != s.end + or o.entity.label != edit.category + for o in target_obs + ): + raise ValueError() + for key in targets: + del effective[key] + if edit.action == "mask": + entity = Entity( + start=s.start, + end=s.end, + label=edit.category, + text=texts[s.page_index][s.start : s.end], + source="deterministic", + model_revision=POLICY, + ) + key = commitment( + dict( + correction=proposal.correction_id, ordinal=index, span=s.model_dump(mode="json") + ) + ) + effective[key] = Observation( + observation_id=key, + page_index=s.page_index, + entity=entity, + human_correction_id=proposal.correction_id, + ) + result = tuple( + sorted( + effective.values(), + key=lambda o: (o.page_index, o.entity.start, o.entity.end, o.observation_id), + ) + ) + render( + context.machine, result, texts + ) # Unresolved machine conflicts still block the whole job. + return result + + +class CorrectionService: + def __init__(self, host: CorrectionHost): + self.host = host + self.review = PrivateReviewBridge(host) + + def _authorize(self, credential, review_ref, scope): + binding = self.review._binding(credential, review_ref) + identity = self.review.authorization.identity(credential) + if ( + self.host.current_acl( + identity, + "anonymization:correction:" + scope, + binding.sensitivity, + binding.environment_id, + ) + is not True + ): + raise ValueError() + return identity, binding + + def _context(self, credential, review_ref, scope): + identity, binding = self._authorize(credential, review_ref, scope) + material = self.review.open(credential, review_ref) + context = CorrectionContext.model_validate(self.host.load_correction_context(review_ref)) + if context.material != material or self._authorize(credential, review_ref, scope) != ( + identity, + binding, + ): + raise ValueError() + return identity, context + + def inspect(self, credential, review_ref): + try: + return self._context(credential, review_ref, "inspect")[1] + except Exception: + raise SafeError("private_correction_denied") from None + + def _texts(self, credential, context): + texts = { + s.page_index: self.review.source_page( + credential, context.material.binding.review_ref, s.page_index + ).text + for s in context.machine.sources + } + # Validate every captured/effective source slice, including observations + # the proposed correction intends to supersede. No corrupt evidence repair. + for observation in context.machine.observations: + expected = commitment( + dict( + evidence=context.machine.evidence_digest, + config=context.machine.provenance.configuration_digest, + page=observation.page_index, + entity=observation.entity.model_dump(mode="json"), + ) + ) + if observation.observation_id != expected: + raise ValueError() + for observations in (context.machine.observations, context.effective): + for source in context.machine.sources: + validate_entities( + texts[source.page_index], + [o.entity for o in observations if o.page_index == source.page_index], + ) + if any(o.page_index not in texts for o in observations): + raise ValueError() + try: + original = render(context.machine, context.machine.observations, texts) + except SafeError as exc: + if ( + exc.code != "ambiguous_entity_overlap" + or context.machine.machine_candidate is not None + ): + raise + else: + if ( + context.machine.machine_candidate is None + or original != context.machine.machine_candidate.pages + ): + raise ValueError() + candidate = context.material.candidate + if ( + candidate is not None + and render(context.machine, context.effective, texts) != candidate.pages + ): + raise ValueError() + return texts + + def bind(self, credential, review_ref, page_index, start, end): + try: + _, context = self._context(credential, review_ref, "submit") + text = self.review.source_page(credential, review_ref, page_index).text + value = span_binding(context, page_index, start, end, text) + self._authorize(credential, review_ref, "submit") + return value + except Exception: + raise SafeError("private_correction_denied") from None + + def propose(self, credential, review_ref, edits): + try: + identity, context = self._context(credential, review_ref, "submit") + if type(edits) not in (tuple, list) or not 1 <= len(edits) <= 1000: + raise ValueError() + edits = tuple(CorrectionEdit.model_validate(e) for e in edits) + now = int(time.time()) + proposal = CorrectionProposal( + correction_id=uuid.uuid4().hex, + review_ref=review_ref, + author_subject=identity.subject, + evidence_digest=context.machine.evidence_digest, + context_digest=commitment(context), + machine_snapshot_digest=commitment(context.machine), + prior_candidate_digest=context.material.candidate_digest, + provenance=context.machine.provenance, + edits=edits, + created_at=now, + expires_at=min(now + 900, context.material.binding.expires_at), + ) + texts = self._texts(credential, context) + apply_edits(context, proposal, texts) + if self._context(credential, review_ref, "submit") != (identity, context): + raise ValueError() + if self.host.store_correction(credential, proposal, commitment(context)) is not True: + raise ValueError() + stored = StoredProposal.model_validate( + self.host.load_correction(proposal.correction_id) + ) + if ( + stored.proposal != proposal + or stored.revoked + or stored.published_candidate_digest is not None + ): + raise ValueError() + return proposal + except Exception: + raise SafeError("private_correction_denied") from None + + def inspect_proposal(self, credential, correction_id): + try: + if type(correction_id) is not str or not re.fullmatch(r"[a-f0-9]{32}", correction_id): + raise ValueError() + # Authenticate first; the host must authorize this opaque-ID lookup + # against its record binding before returning any proposal data. + self.review.authorization.identity(credential) + reference = self.host.correction_binding(correction_id) + _, context = self._context(credential, reference, "inspect") + record = StoredProposal.model_validate(self.host.load_correction(correction_id)) + if ( + record.proposal.correction_id != correction_id + or record.proposal.review_ref != reference + ): + raise ValueError() + self._authorize(credential, reference, "inspect") + return CorrectionReview(context=context, record=record) + except Exception: + raise SafeError("private_correction_denied") from None + + def apply(self, credential, correction_id): + try: + view = self.inspect_proposal(credential, correction_id) + proposal = view.record.proposal + identity, context = self._context(credential, proposal.review_ref, "apply") + now = int(time.time()) + if ( + view.record.revoked + or view.record.published_candidate_digest is not None + or identity.subject != proposal.author_subject + or not proposal.created_at <= now < proposal.expires_at + ): + raise ValueError() + texts = self._texts(credential, context) + effective = apply_edits(context, proposal, texts) + pages = render(context.machine, effective, texts) + previous = context.material.candidate + authors = ( + previous.correction.author_subject_digests + if type(previous) is CorrectedCandidate + else () + ) + authors = tuple(sorted(set(authors + (author_digest(identity.subject),)))) + correction = CorrectionProvenance( + correction_id=proposal.correction_id, + proposal_digest=commitment(proposal), + original_evidence_digest=context.machine.evidence_digest, + machine_snapshot_digest=commitment(context.machine), + source_references_digest=commitment( + [s.model_dump(mode="json") for s in context.machine.sources] + ), + prior_candidate_digest=context.material.candidate_digest, + prior_state_digest=state_digest(context.machine, context.effective), + effective_state_digest=state_digest(context.machine, effective), + author_subject_digests=authors, + edit_count=len(proposal.edits), + ) + candidate = CorrectedCandidate( + candidate_id=uuid.uuid4().hex, + sensitivity=context.material.binding.sensitivity, + pages=pages, + model_revision=context.machine.provenance.model_revision, + provenance=context.machine.provenance, + correction=correction, + review_reasons=("recognition_incomplete_possible",) + + ( + ("empty_prediction",) + if any( + not any( + o.page_index == source.page_index + and o.entity.source == "multilingual_ner" + for o in context.machine.observations + ) + for source in context.machine.sources + ) + else () + ) + + ("human_correction_requires_review",), + ) + binding = context.material.binding.model_copy( + update={"review_ref": "correction-" + uuid.uuid4().hex} + ) + material = PrivateReviewMaterial( + binding=binding, + evidence_digest=context.machine.evidence_digest, + sources=context.machine.sources, + candidate=candidate, + candidate_digest=digest(candidate), + provenance=context.machine.provenance, + blocking_reason=None, + ) + new = CorrectionContext(material=material, machine=context.machine, effective=effective) + if self._context(credential, proposal.review_ref, "apply") != (identity, context): + raise ValueError() + expected = PublicationReceipt( + correction_id=proposal.correction_id, + proposal_digest=commitment(proposal), + candidate_digest=digest(candidate), + context_digest=commitment(new), + review_ref=binding.review_ref, + ) + actual = PublicationReceipt.model_validate( + self.host.publish_correction(credential, proposal, new, commitment(context)) + ) + if actual != expected: + raise ValueError() + return actual + except Exception: + raise SafeError("private_correction_denied") from None diff --git a/ocr/anonymization/demonstrate03f.py b/ocr/anonymization/demonstrate03f.py new file mode 100644 index 0000000000000000000000000000000000000000..c803d83c9a8cd1ba6ab621487de01f1c82f87cba --- /dev/null +++ b/ocr/anonymization/demonstrate03f.py @@ -0,0 +1,220 @@ +"""Pinned-model utility demo on EXISTING synthetic development cases only. + +The isolated host simulates a human choosing existing gold spans; these are not +model predictions or a new holdout. No completed release review is manufactured. +""" + +import argparse +import hashlib +import importlib.metadata +import json +import resource +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +from ocr.schemas import SafeError +from ocr.tests.fixtures.correction_host import CorrectionHostFixture +from ocr.tests.test_anonymization_security import evidence + +from .approval_ledger import SQLiteApprovalLedger +from .atomic_output import write_candidate +from .corrections import CorrectionEdit, CorrectionService, commitment, prepare_review +from .engine import AnonymizationEngine +from .host_adapter import HostAuthorizationBridge +from .recognizer import GLiNERRecognizer +from .release_policy import digest + + +def coverage(candidate, case): + if candidate is None: + return None + tags = candidate.pages[0].tags + return all( + any(t.start <= i < t.end for t in tags) + for span in case["spans"] + for i in range(span["start"], span["end"]) + if case["text"][i].isalnum() + ) + + +def run(cache, output, *, correct_controls_only=False): + if Path(output).exists(): + raise FileExistsError("correction_demo_exists") + denied = [] + + def offline(event, args): + if event in ("socket.connect", "socket.bind", "socket.getaddrinfo"): + denied.append(event) + raise PermissionError("correction_demo_network_disabled") + + sys.addaudithook(offline) + source = Path(__file__).parent / "fixtures/holdout03d/cases.json" + cases = json.loads(source.read_bytes()) + selected = {4, 5} if correct_controls_only else {1, 2, 3, 55, 63, 78, 125, 127, 137} + start = time.perf_counter() + model = GLiNERRecognizer(cache) + loading_ms = (time.perf_counter() - start) * 1000 + rows = [] + for case in cases: + number = int(case["id"].split("-")[-1]) + if number not in selected: + continue + start = time.perf_counter() + context = prepare_review( + AnonymizationEngine(model), + evidence("D2", (case["text"],)), + review_ref=case["id"], + environment_id="synthetic-room", + expires_at=int(time.time()) + 600, + ) + inference_ms = (time.perf_counter() - start) * 1000 + host = CorrectionHostFixture(context, {0: case["text"]}, separate_approver=True) + service = CorrectionService(host) + candidate = context.material.candidate + before = coverage(candidate, case) + original = ( + "blocked_conflicting_spans" + if candidate is None + else "complete_no_measured_leakage" + if before + else "candidate_partial_mask" + ) + edits = [] + if number not in (1, 2, 3, 4, 5): + for span in case["spans"]: + if not (number == 125 and span["label"] == "ORGANIZATION") and not ( + number != 125 and span["label"] == "RELIGION" + ): + continue + targets = tuple( + sorted( + { + o.observation_id + for o in context.effective + if o.page_index == 0 + and o.entity.start < span["end"] + and span["start"] < o.entity.end + } + ) + ) + edits.append( + CorrectionEdit( + span=service.bind( + host.author_credential, case["id"], 0, span["start"], span["end"] + ), + category=span["label"], + action="mask", + rationale="incomplete_boundary" if number == 125 else "span_conflict", + target_observation_ids=targets, + ) + ) + malicious_rejected = unauthorized_rejected = False + if edits: + forged = edits[0].model_copy( + update={"span": edits[0].span.model_copy(update={"source_span_digest": "f" * 64})} + ) + try: + service.propose(host.author_credential, case["id"], (forged,)) + except SafeError as exc: + malicious_rejected = exc.code == "private_correction_denied" + try: + service.propose(host.approver_credential, case["id"], tuple(edits)) + except SafeError as exc: + unauthorized_rejected = exc.code == "private_correction_denied" + proposal = service.propose(host.author_credential, case["id"], tuple(edits)) + receipt = service.apply(host.author_credential, proposal.correction_id) + candidate = host.candidates[receipt.candidate_digest] + assert host.contexts[case["id"]] == context + with tempfile.TemporaryDirectory() as directory: + authority = HostAuthorizationBridge(host).authority( + b"synthetic-demo-signing-key-not-production", + SQLiteApprovalLedger(Path(directory) / "approvals.sqlite", initialize=True), + ("synthetic-private",), + ) + blocked = False + try: + authority.export( + candidate, None, actor=host.author_credential, destination="synthetic-private" + ) + except SafeError as exc: + blocked = exc.code == "release_not_approved" + rows.append( + dict( + case_id=case["id"], + original_outcome=original, + machine_candidate_digest=context.material.candidate_digest, + machine_observations_digest=commitment(context.machine), + source_canaries_covered_before=before, + proposed_edits=[e.model_dump(mode="json") for e in edits], + new_candidate_produced=bool(edits), + final_candidate_digest=digest(candidate), + source_canaries_covered_after=coverage(candidate, case), + document_exact_match_after=candidate.pages[0].anonymized_text == case["expected"], + final_sensitivity=candidate.sensitivity, + release_blocked=blocked, + completed_release_reviews=len(host.completed), + malformed_correction_rejected=malicious_rejected if edits else None, + unauthorized_correction_rejected=unauthorized_rejected if edits else None, + capture_inference_ms=inference_ms, + ) + ) + report = dict( + schema_version="gate03f-synthetic-reviewed-correction-demo/v1", + source_sha=subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), + corpus_sha256=hashlib.sha256(source.read_bytes()).hexdigest(), + model_revision=model.revision, + prompts=model.prompts, + threshold=model.threshold, + config_digest=context.machine.provenance.configuration_digest, + actual_inference_calls=len(rows), + correct_controls_only=correct_controls_only, + loading_ms=loading_ms, + peak_rss_mib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, + runtime_versions={ + k: importlib.metadata.version(k) + for k in ("gliner", "torch", "transformers", "numpy", "pydantic") + }, + denied_network_calls=denied, + cases=rows, + correction_source="simulated authorized human using existing synthetic gold spans", + production_identity_qualified=False, + durable_correction_store_qualified=False, + independent_qualification=False, + model_qualification_passed=False, + corrected_outputs_count_as_model_true_positives=False, + automated_release_eligible=False, + ) + write_candidate(output, (json.dumps(report, indent=2) + "\n").encode()) + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cache", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--correct-controls-only", action="store_true") + args = parser.parse_args() + try: + report = run(args.cache, args.output, correct_controls_only=args.correct_controls_only) + except Exception: + raise SystemExit("private_correction_demo_failed") from None + # Emit only aggregate results; source text and paths are never printed. + print( + json.dumps( + dict( + cases=len(report["cases"]), + manual_corrections=sum(c["new_candidate_produced"] for c in report["cases"]), + all_seeded_canaries_covered=all( + c["source_canaries_covered_after"] for c in report["cases"] + ), + all_release_blocked=all(c["release_blocked"] for c in report["cases"]), + ) + ) + ) + + +if __name__ == "__main__": + main() diff --git a/ocr/anonymization/detectors.py b/ocr/anonymization/detectors.py new file mode 100644 index 0000000000000000000000000000000000000000..8d8afa152ecc73c81e2f8bebd96bc693cb3beefb --- /dev/null +++ b/ocr/anonymization/detectors.py @@ -0,0 +1,38 @@ +"""Deterministic contact patterns, not an exhaustive identifier detector.""" + +import re + +from .schemas import Entity +from .status import StatusPolicy +from .terminology import ReligiousTerminology + +DEFAULT_TERMINOLOGY = ReligiousTerminology() + +EMAIL = re.compile(r"(?= before["candidate_return_rate"] + okay &= ( + result["accuracy"]["document_exact_match_rate"] + >= before["accuracy"]["document_exact_match_rate"] + ) + for metric in ("candidate_partial_canaries", "candidate_fully_unmasked_canaries"): + okay &= all(result[metric][k] <= before[metric][k] for k in result[metric]) + improvements += sum(before[metric].values()) - sum(result[metric].values()) + okay &= all( + result["accuracy"]["combined_entity_metrics"][k]["fp"] <= v["fp"] + for k, v in before["accuracy"]["combined_entity_metrics"].items() + ) + report["selection_criteria"][name] = dict( + passed=bool(okay and improvements > 0), net_canary_improvements=improvements + ) + report["peak_rss_mib"] = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 + report["denied_network_calls"] = denied + write_candidate(output, (json.dumps(report, ensure_ascii=False, indent=2) + "\n").encode()) + return report + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Pinned development comparison, not a holdout") + parser.add_argument("--cache", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + result = run(args.cache, args.output) + print(json.dumps(result["selection_criteria"])) diff --git a/ocr/anonymization/engine.py b/ocr/anonymization/engine.py new file mode 100644 index 0000000000000000000000000000000000000000..d9d4e0e91956fed5de3c34a86e77065246c83923 --- /dev/null +++ b/ocr/anonymization/engine.py @@ -0,0 +1,149 @@ +"""Trusted local recognition and all-or-nothing OCR handoff.""" + +import uuid + +from ocr.schemas import OCRResult, SafeError + +from .detectors import detect +from .model import MODEL_REVISION +from .provenance import provenance +from .replacement import replace, validate_entities +from .schemas import LABELS, CandidatePage, ReleaseCandidate +from .status import StatusPolicy +from .terminology import DEFAULT_TERMS, ReligiousTerminology + + +class AnonymizationEngine: + def __init__( + self, + recognizer, + *, + placeholders=None, + religion_terms=DEFAULT_TERMS, + status_policy=StatusPolicy(), + ): + self.recognizer = recognizer + self.placeholders = placeholders + self.terminology = ReligiousTerminology(religion_terms) + if type(status_policy) is not StatusPolicy: + raise SafeError("invalid_status_configuration") + self.status_policy = status_policy + + def observe(self, text): + try: + if set(self.recognizer.categories) != set(LABELS): + raise SafeError("unsupported_required_categories") + # Only local implementations are accepted. No remote recognizer transport exists. + if self.recognizer.local is not True: + raise SafeError("no_authorized_destination") + if self.recognizer.revision != MODEL_REVISION: + raise SafeError("anonymization_model_unavailable") + if not isinstance(text, str) or len(text) > 2_000_000: + raise SafeError("anonymization_text_limit") + predictions = self.recognizer.recognize(text) + if not isinstance(predictions, (list, tuple)): + raise SafeError("invalid_entity_spans") + predictions = validate_entities(text, predictions) + if any( + p.source != "multilingual_ner" or p.model_revision != MODEL_REVISION + for p in predictions + ): + raise SafeError("invalid_entity_provenance") + return predictions + detect( + text, terminology=self.terminology, status=self.status_policy + ) + except SafeError as exc: + allowed = { + "unsupported_required_categories", + "no_authorized_destination", + "invalid_entity_spans", + "invalid_entity_provenance", + "ambiguous_entity_overlap", + "anonymization_model_unavailable", + "anonymization_text_limit", + "anonymization_token_limit", + "anonymization_inference_failed", + } + raise SafeError(exc.code if exc.code in allowed else "anonymization_failed") from None + except Exception: + raise SafeError("anonymization_failed") from None + + def transform(self, text): + predictions = self.observe(text) + try: + output, tags = replace(text, predictions, self.placeholders) + return output, tags, not any(p.source == "multilingual_ner" for p in predictions) + except SafeError as exc: + if exc.code not in ( + "unsupported_required_categories", + "invalid_entity_spans", + "ambiguous_entity_overlap", + ): + raise SafeError("anonymization_failed") from None + raise + except Exception: + raise SafeError("anonymization_failed") from None + + @staticmethod + def validate_document(result): + if not isinstance(result, OCRResult): + raise SafeError("invalid_ocr_evidence") + if ( + result.status != "completed" + or result.failures + or not result.pages + or result.page_count != len(result.pages) + or result.page_count > 100 + or sum(len(p.text) for p in result.pages) > 2_000_000 + ): + raise SafeError("incomplete_ocr_document") + indexes = [p.page_index for p in result.pages] + if indexes not in ( + list(range(result.page_count)), + list(range(1, result.page_count + 1)), + ): + raise SafeError("incomplete_ocr_document") + # Sensitivity mismatches are rejected, never silently downgraded. + if any(p.sensitivity != result.sensitivity for p in result.pages): + raise SafeError("classification_mismatch") + + def handoff(self, result: OCRResult) -> ReleaseCandidate: + try: + self.validate_document(result) + commitment = provenance( + self.recognizer, self.terminology, self.status_policy, self.placeholders + ) + pages, empty = [], False + for p in result.pages: + text, tags, no_predictions = self.transform(p.text) + empty |= no_predictions + pages.append( + CandidatePage( + page_index=p.page_index, + anonymized_text=text, + tags=tags, + source=p.source, + original_sensitivity=p.sensitivity, + ) + ) + if commitment != provenance( + self.recognizer, self.terminology, self.status_policy, self.placeholders + ): + raise SafeError("recognition_configuration_changed") + reasons = ("recognition_incomplete_possible",) + ( + ("empty_prediction",) if empty else () + ) + # Rebuild exclusively from allowlisted fields; Markdown, blocks, alternatives, + # engine/provider strings, page IDs and job IDs stay with classified evidence. + return ReleaseCandidate( + candidate_id=uuid.uuid4().hex, + sensitivity=result.sensitivity, + pages=tuple(pages), + model_revision=self.recognizer.revision, + review_reasons=reasons, + provenance=commitment, + ) + except SafeError: + raise + except Exception: + raise SafeError("anonymization_failed") from None diff --git a/ocr/anonymization/evaluate.py b/ocr/anonymization/evaluate.py new file mode 100644 index 0000000000000000000000000000000000000000..7707cfb5347a1236962a4a8b12bf44220b699c82 --- /dev/null +++ b/ocr/anonymization/evaluate.py @@ -0,0 +1,196 @@ +"""Opt-in synthetic evaluation; records aggregate metrics, never document payloads.""" + +import importlib.metadata +import json +import resource +import statistics +import time +from collections import Counter +from pathlib import Path + +from ocr.schemas import SafeError + +from .detectors import detect +from .model import MODEL_ID, MODEL_REVISION +from .replacement import replace, resolve_overlaps, validate_entities +from .schemas import LABELS + + +def metrics(counts): + output = {} + for label in LABELS: + tp, fp, fn = (counts[(label, name)] for name in ("tp", "fp", "fn")) + p = tp / (tp + fp) if tp + fp else 0.0 + r = tp / (tp + fn) if tp + fn else 0.0 + output[label] = dict( + tp=tp, fp=fp, fn=fn, precision=p, recall=r, f1=2 * p * r / (p + r) if p + r else 0.0 + ) + return output + + +def score_cases(recognizer, cases, *, detector=detect, case_reports=None): + ner_counts, combined_counts, leaks, failures = Counter(), Counter(), Counter(), Counter() + literal_leaks = Counter() + partial_canaries, uncovered_canaries, coverage = Counter(), Counter(), Counter() + durations, exact, blocked = [], 0, 0 + groups = {} + for case in cases: + text = case["text"] + gold = {(s["start"], s["end"], s["label"]) for s in case["spans"]} + for s in case["spans"]: + if text[s["start"] : s["end"]] != s["text"]: + raise SafeError("invalid_evaluation_fixture") + start = time.perf_counter() + actual, predicted, ner = None, set(), set() + raw, combined, error_code = [], [], None + try: + raw = validate_entities(text, recognizer.recognize(text)) + ner = {(s.start, s.end, s.label) for s in raw} + combined = resolve_overlaps(validate_entities(text, raw + detector(text))) + predicted = {(s.start, s.end, s.label) for s in combined} + actual, _ = replace(text, combined) + except SafeError as e: + error_code = e.code + failures[e.code] += 1 + blocked += 1 + durations.append((time.perf_counter() - start) * 1000) + for predicted_set, counts in ((ner, ner_counts), (predicted, combined_counts)): + for _, _, label in predicted_set & gold: + counts[(label, "tp")] += 1 + for _, _, label in predicted_set - gold: + counts[(label, "fp")] += 1 + for _, _, label in gold - predicted_set: + counts[(label, "fn")] += 1 + exact += actual == case["expected"] + for s in case["spans"]: + if actual is None or s["text"].casefold() in actual.casefold(): + literal_leaks[s["label"]] += 1 + sensitive_positions = [i for i in range(s["start"], s["end"]) if text[i].isalnum()] + covered = sum( + actual is not None and any(left <= i < right for left, right, _ in predicted) + for i in sensitive_positions + ) + coverage[(s["label"], "total")] += len(sensitive_positions) + coverage[(s["label"], "covered")] += covered + if covered < len(sensitive_positions): + leaks[s["label"]] += 1 + if covered: + partial_canaries[s["label"]] += 1 + else: + uncovered_canaries[s["label"]] += 1 + if case_reports is not None: + case_reports.append( + dict( + case_id=case["id"], + scenario=case.get("scenario"), + predictions=[s.model_dump() for s in raw], + combined_predictions=[s.model_dump() for s in combined], + false_positives=sorted(predicted - gold), + false_negatives=sorted(gold - predicted), + exact_match=actual == case["expected"], + error_code=error_code, + canaries=[ + dict( + **s, + uncovered_positions=[ + i + for i in range(s["start"], s["end"]) + if text[i].isalnum() + and ( + actual is None + or not any(left <= i < right for left, right, _ in predicted) + ) + ], + ) + for s in case["spans"] + ], + ) + ) + group = case.get("language", "unspecified") + stats = groups.setdefault(group, dict(documents=0, exact_matches=0, false_negatives=0)) + stats["documents"] += 1 + stats["exact_matches"] += actual == case["expected"] + stats["false_negatives"] += len(gold - predicted) + if case.get("ocr_noise"): + stats = groups.setdefault( + "ocr_noise", dict(documents=0, exact_matches=0, false_negatives=0) + ) + stats["documents"] += 1 + stats["exact_matches"] += actual == case["expected"] + stats["false_negatives"] += len(gold - predicted) + return dict( + documents=len(cases), + ner_entity_metrics=metrics(ner_counts), + combined_entity_metrics=metrics(combined_counts), + unredacted_canaries={k: leaks[k] for k in LABELS}, + literal_unredacted_canaries={k: literal_leaks[k] for k in LABELS}, + partial_canaries={k: partial_canaries[k] for k in LABELS}, + fully_uncovered_canaries={k: uncovered_canaries[k] for k in LABELS}, + alphanumeric_canary_coverage={ + k: dict(covered=coverage[(k, "covered")], total=coverage[(k, "total")]) for k in LABELS + }, + document_exact_match_rate=exact / len(cases), + groups=groups, + blocked_documents=blocked, + failure_codes=dict(failures), + warm_latency_ms=dict(median=statistics.median(durations), max=max(durations)), + automated_release_eligible=False, + qualification_passed=not any(leaks.values()) and blocked == 0, + ) + + +def run(cache, output): + from .recognizer import GLiNERRecognizer + + start = time.perf_counter() + recognizer = GLiNERRecognizer(cache) + load = (time.perf_counter() - start) * 1000 + # Explicit warmup is synthetic public text, independently reported. + recognizer.recognize("The cat sat on the mat.") + report = dict( + model=MODEL_ID, + revision=MODEL_REVISION, + threshold=recognizer.threshold, + prompts=recognizer.prompts, + runtime_versions={ + k: importlib.metadata.version(k) + for k in ("gliner", "torch", "transformers", "numpy", "pydantic") + }, + device="cpu", + threads=2, + loading_ms=load, + suites={}, + ) + root = Path(__file__).parent / "fixtures" + for name in ("qualification", "validation"): + report["suites"][name] = score_cases( + recognizer, json.loads((root / f"{name}.json").read_text()) + ) + report["peak_rss_mib"] = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 + report["automated_release_eligible"] = False + report["recognition_qualification_passed"] = all( + s["qualification_passed"] for s in report["suites"].values() + ) + Path(output).write_text(json.dumps(report, indent=2) + "\n") + return report + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Pinned local synthetic model evaluation") + parser.add_argument("--cache", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + try: + report = run(args.cache, args.output) + except Exception: + raise SystemExit("anonymization_evaluation_failed") from None + print( + json.dumps( + { + "recognition_qualification_passed": report["recognition_qualification_passed"], + "automated_release_eligible": False, + } + ) + ) diff --git a/ocr/anonymization/fixtures/holdout03c/cases.json b/ocr/anonymization/fixtures/holdout03c/cases.json new file mode 100644 index 0000000000000000000000000000000000000000..c86609d9dbf19642be3fefc55db582df33fa5982 --- /dev/null +++ b/ocr/anonymization/fixtures/holdout03c/cases.json @@ -0,0 +1,3854 @@ +[ + { + "id": "holdout03c-000", + "language": "de", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "Auskunft: Saskia Dornfeld arbeitet für Aster Research Institute in Osnabrück.", + "spans": [ + { + "start": 10, + "end": 25, + "label": "PERSON", + "text": "Saskia Dornfeld" + }, + { + "start": 39, + "end": 63, + "label": "ORGANIZATION", + "text": "Aster Research Institute" + }, + { + "start": 67, + "end": 76, + "label": "CITY", + "text": "Osnabrück" + } + ], + "expected": "Auskunft: [PERSON] arbeitet für [ORGANIZATION] in [CITY]." + }, + { + "id": "holdout03c-001", + "language": "en", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "The contact Saskia Dornfeld represents Aster Research Institute near Osnabrück.", + "spans": [ + { + "start": 12, + "end": 27, + "label": "PERSON", + "text": "Saskia Dornfeld" + }, + { + "start": 39, + "end": 63, + "label": "ORGANIZATION", + "text": "Aster Research Institute" + }, + { + "start": 69, + "end": 78, + "label": "CITY", + "text": "Osnabrück" + } + ], + "expected": "The contact [PERSON] represents [ORGANIZATION] near [CITY]." + }, + { + "id": "holdout03c-002", + "language": "de", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Saskia Dornfeld nennt sikhistisch als Zugehörigkeit. E-Mail: case101.review@example.invalid; Telefon: +49 541 730101.", + "spans": [ + { + "start": 0, + "end": 15, + "label": "PERSON", + "text": "Saskia Dornfeld" + }, + { + "start": 22, + "end": 33, + "label": "RELIGION", + "text": "sikhistisch" + }, + { + "start": 61, + "end": 91, + "label": "EMAIL", + "text": "case101.review@example.invalid" + }, + { + "start": 102, + "end": 116, + "label": "PHONE", + "text": "+49 541 730101" + } + ], + "expected": "[PERSON] nennt [RELIGION] als Zugehörigkeit. E-Mail: [EMAIL]; Telefon: [PHONE]." + }, + { + "id": "holdout03c-003", + "language": "en", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Saskia Dornfeld explicitly names Sikhism as their faith. Write to case101.review@example.invalid or call +49 541 730101.", + "spans": [ + { + "start": 0, + "end": 15, + "label": "PERSON", + "text": "Saskia Dornfeld" + }, + { + "start": 33, + "end": 40, + "label": "RELIGION", + "text": "Sikhism" + }, + { + "start": 66, + "end": 96, + "label": "EMAIL", + "text": "case101.review@example.invalid" + }, + { + "start": 105, + "end": 119, + "label": "PHONE", + "text": "+49 541 730101" + } + ], + "expected": "[PERSON] explicitly names [RELIGION] as their faith. Write to [EMAIL] or call [PHONE]." + }, + { + "id": "holdout03c-004", + "language": "de", + "ocr_noise": false, + "features": [ + "repeats", + "paragraphs" + ], + "text": "Saskia Dornfeld besucht Osnabrück.\n\nSaskia Dornfeld bestätigt den Termin bei Aster Research Institute.", + "spans": [ + { + "start": 0, + "end": 15, + "label": "PERSON", + "text": "Saskia Dornfeld" + }, + { + "start": 24, + "end": 33, + "label": "CITY", + "text": "Osnabrück" + }, + { + "start": 36, + "end": 51, + "label": "PERSON", + "text": "Saskia Dornfeld" + }, + { + "start": 77, + "end": 101, + "label": "ORGANIZATION", + "text": "Aster Research Institute" + } + ], + "expected": "[PERSON] besucht [CITY].\n\n[PERSON] bestätigt den Termin bei [ORGANIZATION]." + }, + { + "id": "holdout03c-005", + "language": "en", + "ocr_noise": false, + "features": [ + "repeats", + "ambiguous_geography" + ], + "text": "In Osnabrück, Saskia Dornfeld signed for Aster Research Institute. The representative Saskia Dornfeld returned to Osnabrück.", + "spans": [ + { + "start": 3, + "end": 12, + "label": "CITY", + "text": "Osnabrück" + }, + { + "start": 14, + "end": 29, + "label": "PERSON", + "text": "Saskia Dornfeld" + }, + { + "start": 41, + "end": 65, + "label": "ORGANIZATION", + "text": "Aster Research Institute" + }, + { + "start": 86, + "end": 101, + "label": "PERSON", + "text": "Saskia Dornfeld" + }, + { + "start": 114, + "end": 123, + "label": "CITY", + "text": "Osnabrück" + } + ], + "expected": "In [CITY], [PERSON] signed for [ORGANIZATION]. The representative [PERSON] returned to [CITY]." + }, + { + "id": "holdout03c-006", + "language": "de", + "ocr_noise": true, + "features": [ + "line_wrap", + "organization" + ], + "text": "Name: Saskia-\nDornfeld\nOrt: Osna-\nbrück\nArbeitgeber: Aster Research Institute.", + "spans": [ + { + "start": 6, + "end": 22, + "label": "PERSON", + "text": "Saskia-\nDornfeld" + }, + { + "start": 28, + "end": 39, + "label": "CITY", + "text": "Osna-\nbrück" + }, + { + "start": 53, + "end": 77, + "label": "ORGANIZATION", + "text": "Aster Research Institute" + } + ], + "expected": "Name: [PERSON]\nOrt: [CITY]\nArbeitgeber: [ORGANIZATION]." + }, + { + "id": "holdout03c-007", + "language": "en", + "ocr_noise": true, + "features": [ + "ocr_spelling", + "organization" + ], + "text": "Scanned record: Sask1a D0rnfeld resides in Osn4brück and works at Aster Research Institute.", + "spans": [ + { + "start": 16, + "end": 31, + "label": "PERSON", + "text": "Sask1a D0rnfeld" + }, + { + "start": 43, + "end": 52, + "label": "CITY", + "text": "Osn4brück" + }, + { + "start": 66, + "end": 90, + "label": "ORGANIZATION", + "text": "Aster Research Institute" + } + ], + "expected": "Scanned record: [PERSON] resides in [CITY] and works at [ORGANIZATION]." + }, + { + "id": "holdout03c-008", + "language": "de", + "ocr_noise": false, + "features": [ + "religious_reference", + "no_inferred_affiliation" + ], + "text": "Das Thema sikhistisch wird in einem Bericht von Aster Research Institute erwähnt. Für Saskia Dornfeld wird keine Religionszugehörigkeit behauptet.", + "spans": [ + { + "start": 10, + "end": 21, + "label": "RELIGION", + "text": "sikhistisch" + }, + { + "start": 48, + "end": 72, + "label": "ORGANIZATION", + "text": "Aster Research Institute" + }, + { + "start": 86, + "end": 101, + "label": "PERSON", + "text": "Saskia Dornfeld" + } + ], + "expected": "Das Thema [RELIGION] wird in einem Bericht von [ORGANIZATION] erwähnt. Für [PERSON] wird keine Religionszugehörigkeit behauptet." + }, + { + "id": "holdout03c-009", + "language": "en", + "ocr_noise": false, + "features": [ + "negative", + "religious_reference" + ], + "text": "Worksheet 101: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details.", + "spans": [], + "expected": "Worksheet 101: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details." + }, + { + "id": "holdout03c-010", + "language": "de", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "Auskunft: Emil Farber arbeitet für Northbridge Analytics in Potsdam.", + "spans": [ + { + "start": 10, + "end": 21, + "label": "PERSON", + "text": "Emil Farber" + }, + { + "start": 35, + "end": 56, + "label": "ORGANIZATION", + "text": "Northbridge Analytics" + }, + { + "start": 60, + "end": 67, + "label": "CITY", + "text": "Potsdam" + } + ], + "expected": "Auskunft: [PERSON] arbeitet für [ORGANIZATION] in [CITY]." + }, + { + "id": "holdout03c-011", + "language": "en", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "The contact Emil Farber represents Northbridge Analytics near Potsdam.", + "spans": [ + { + "start": 12, + "end": 23, + "label": "PERSON", + "text": "Emil Farber" + }, + { + "start": 35, + "end": 56, + "label": "ORGANIZATION", + "text": "Northbridge Analytics" + }, + { + "start": 62, + "end": 69, + "label": "CITY", + "text": "Potsdam" + } + ], + "expected": "The contact [PERSON] represents [ORGANIZATION] near [CITY]." + }, + { + "id": "holdout03c-012", + "language": "de", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Emil Farber nennt orthodox als Zugehörigkeit. E-Mail: case102.review@example.invalid; Telefon: +49 541 730102.", + "spans": [ + { + "start": 0, + "end": 11, + "label": "PERSON", + "text": "Emil Farber" + }, + { + "start": 18, + "end": 26, + "label": "RELIGION", + "text": "orthodox" + }, + { + "start": 54, + "end": 84, + "label": "EMAIL", + "text": "case102.review@example.invalid" + }, + { + "start": 95, + "end": 109, + "label": "PHONE", + "text": "+49 541 730102" + } + ], + "expected": "[PERSON] nennt [RELIGION] als Zugehörigkeit. E-Mail: [EMAIL]; Telefon: [PHONE]." + }, + { + "id": "holdout03c-013", + "language": "en", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Emil Farber explicitly names Orthodox Christianity as their faith. Write to case102.review@example.invalid or call +49 541 730102.", + "spans": [ + { + "start": 0, + "end": 11, + "label": "PERSON", + "text": "Emil Farber" + }, + { + "start": 29, + "end": 50, + "label": "RELIGION", + "text": "Orthodox Christianity" + }, + { + "start": 76, + "end": 106, + "label": "EMAIL", + "text": "case102.review@example.invalid" + }, + { + "start": 115, + "end": 129, + "label": "PHONE", + "text": "+49 541 730102" + } + ], + "expected": "[PERSON] explicitly names [RELIGION] as their faith. Write to [EMAIL] or call [PHONE]." + }, + { + "id": "holdout03c-014", + "language": "de", + "ocr_noise": false, + "features": [ + "repeats", + "paragraphs" + ], + "text": "Emil Farber besucht Potsdam.\n\nEmil Farber bestätigt den Termin bei Northbridge Analytics.", + "spans": [ + { + "start": 0, + "end": 11, + "label": "PERSON", + "text": "Emil Farber" + }, + { + "start": 20, + "end": 27, + "label": "CITY", + "text": "Potsdam" + }, + { + "start": 30, + "end": 41, + "label": "PERSON", + "text": "Emil Farber" + }, + { + "start": 67, + "end": 88, + "label": "ORGANIZATION", + "text": "Northbridge Analytics" + } + ], + "expected": "[PERSON] besucht [CITY].\n\n[PERSON] bestätigt den Termin bei [ORGANIZATION]." + }, + { + "id": "holdout03c-015", + "language": "en", + "ocr_noise": false, + "features": [ + "repeats", + "ambiguous_geography" + ], + "text": "In Potsdam, Emil Farber signed for Northbridge Analytics. The representative Emil Farber returned to Potsdam.", + "spans": [ + { + "start": 3, + "end": 10, + "label": "CITY", + "text": "Potsdam" + }, + { + "start": 12, + "end": 23, + "label": "PERSON", + "text": "Emil Farber" + }, + { + "start": 35, + "end": 56, + "label": "ORGANIZATION", + "text": "Northbridge Analytics" + }, + { + "start": 77, + "end": 88, + "label": "PERSON", + "text": "Emil Farber" + }, + { + "start": 101, + "end": 108, + "label": "CITY", + "text": "Potsdam" + } + ], + "expected": "In [CITY], [PERSON] signed for [ORGANIZATION]. The representative [PERSON] returned to [CITY]." + }, + { + "id": "holdout03c-016", + "language": "de", + "ocr_noise": true, + "features": [ + "line_wrap", + "organization" + ], + "text": "Name: Emil-\nFarber\nOrt: Pot-\nsdam\nArbeitgeber: Northbridge Analytics.", + "spans": [ + { + "start": 6, + "end": 18, + "label": "PERSON", + "text": "Emil-\nFarber" + }, + { + "start": 24, + "end": 33, + "label": "CITY", + "text": "Pot-\nsdam" + }, + { + "start": 47, + "end": 68, + "label": "ORGANIZATION", + "text": "Northbridge Analytics" + } + ], + "expected": "Name: [PERSON]\nOrt: [CITY]\nArbeitgeber: [ORGANIZATION]." + }, + { + "id": "holdout03c-017", + "language": "en", + "ocr_noise": true, + "features": [ + "ocr_spelling", + "organization" + ], + "text": "Scanned record: Em1l Farber resides in P0tsd4m and works at Northbridge Analytics.", + "spans": [ + { + "start": 16, + "end": 27, + "label": "PERSON", + "text": "Em1l Farber" + }, + { + "start": 39, + "end": 46, + "label": "CITY", + "text": "P0tsd4m" + }, + { + "start": 60, + "end": 81, + "label": "ORGANIZATION", + "text": "Northbridge Analytics" + } + ], + "expected": "Scanned record: [PERSON] resides in [CITY] and works at [ORGANIZATION]." + }, + { + "id": "holdout03c-018", + "language": "de", + "ocr_noise": false, + "features": [ + "religious_reference", + "no_inferred_affiliation" + ], + "text": "Das Thema orthodox wird in einem Bericht von Northbridge Analytics erwähnt. Für Emil Farber wird keine Religionszugehörigkeit behauptet.", + "spans": [ + { + "start": 10, + "end": 18, + "label": "RELIGION", + "text": "orthodox" + }, + { + "start": 45, + "end": 66, + "label": "ORGANIZATION", + "text": "Northbridge Analytics" + }, + { + "start": 80, + "end": 91, + "label": "PERSON", + "text": "Emil Farber" + } + ], + "expected": "Das Thema [RELIGION] wird in einem Bericht von [ORGANIZATION] erwähnt. Für [PERSON] wird keine Religionszugehörigkeit behauptet." + }, + { + "id": "holdout03c-019", + "language": "en", + "ocr_noise": false, + "features": [ + "negative", + "religious_reference" + ], + "text": "Worksheet 102: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details.", + "spans": [], + "expected": "Worksheet 102: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details." + }, + { + "id": "holdout03c-020", + "language": "de", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "Auskunft: Cora Lindenfels arbeitet für Amber Health Foundation in Dundee.", + "spans": [ + { + "start": 10, + "end": 25, + "label": "PERSON", + "text": "Cora Lindenfels" + }, + { + "start": 39, + "end": 62, + "label": "ORGANIZATION", + "text": "Amber Health Foundation" + }, + { + "start": 66, + "end": 72, + "label": "CITY", + "text": "Dundee" + } + ], + "expected": "Auskunft: [PERSON] arbeitet für [ORGANIZATION] in [CITY]." + }, + { + "id": "holdout03c-021", + "language": "en", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "The contact Cora Lindenfels represents Amber Health Foundation near Dundee.", + "spans": [ + { + "start": 12, + "end": 27, + "label": "PERSON", + "text": "Cora Lindenfels" + }, + { + "start": 39, + "end": 62, + "label": "ORGANIZATION", + "text": "Amber Health Foundation" + }, + { + "start": 68, + "end": 74, + "label": "CITY", + "text": "Dundee" + } + ], + "expected": "The contact [PERSON] represents [ORGANIZATION] near [CITY]." + }, + { + "id": "holdout03c-022", + "language": "de", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Cora Lindenfels nennt jainistisch als Zugehörigkeit. E-Mail: case103.review@example.invalid; Telefon: +49 541 730103.", + "spans": [ + { + "start": 0, + "end": 15, + "label": "PERSON", + "text": "Cora Lindenfels" + }, + { + "start": 22, + "end": 33, + "label": "RELIGION", + "text": "jainistisch" + }, + { + "start": 61, + "end": 91, + "label": "EMAIL", + "text": "case103.review@example.invalid" + }, + { + "start": 102, + "end": 116, + "label": "PHONE", + "text": "+49 541 730103" + } + ], + "expected": "[PERSON] nennt [RELIGION] als Zugehörigkeit. E-Mail: [EMAIL]; Telefon: [PHONE]." + }, + { + "id": "holdout03c-023", + "language": "en", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Cora Lindenfels explicitly names Jainism as their faith. Write to case103.review@example.invalid or call +49 541 730103.", + "spans": [ + { + "start": 0, + "end": 15, + "label": "PERSON", + "text": "Cora Lindenfels" + }, + { + "start": 33, + "end": 40, + "label": "RELIGION", + "text": "Jainism" + }, + { + "start": 66, + "end": 96, + "label": "EMAIL", + "text": "case103.review@example.invalid" + }, + { + "start": 105, + "end": 119, + "label": "PHONE", + "text": "+49 541 730103" + } + ], + "expected": "[PERSON] explicitly names [RELIGION] as their faith. Write to [EMAIL] or call [PHONE]." + }, + { + "id": "holdout03c-024", + "language": "de", + "ocr_noise": false, + "features": [ + "repeats", + "paragraphs" + ], + "text": "Cora Lindenfels besucht Dundee.\n\nCora Lindenfels bestätigt den Termin bei Amber Health Foundation.", + "spans": [ + { + "start": 0, + "end": 15, + "label": "PERSON", + "text": "Cora Lindenfels" + }, + { + "start": 24, + "end": 30, + "label": "CITY", + "text": "Dundee" + }, + { + "start": 33, + "end": 48, + "label": "PERSON", + "text": "Cora Lindenfels" + }, + { + "start": 74, + "end": 97, + "label": "ORGANIZATION", + "text": "Amber Health Foundation" + } + ], + "expected": "[PERSON] besucht [CITY].\n\n[PERSON] bestätigt den Termin bei [ORGANIZATION]." + }, + { + "id": "holdout03c-025", + "language": "en", + "ocr_noise": false, + "features": [ + "repeats", + "ambiguous_geography" + ], + "text": "In Dundee, Cora Lindenfels signed for Amber Health Foundation. The representative Cora Lindenfels returned to Dundee.", + "spans": [ + { + "start": 3, + "end": 9, + "label": "CITY", + "text": "Dundee" + }, + { + "start": 11, + "end": 26, + "label": "PERSON", + "text": "Cora Lindenfels" + }, + { + "start": 38, + "end": 61, + "label": "ORGANIZATION", + "text": "Amber Health Foundation" + }, + { + "start": 82, + "end": 97, + "label": "PERSON", + "text": "Cora Lindenfels" + }, + { + "start": 110, + "end": 116, + "label": "CITY", + "text": "Dundee" + } + ], + "expected": "In [CITY], [PERSON] signed for [ORGANIZATION]. The representative [PERSON] returned to [CITY]." + }, + { + "id": "holdout03c-026", + "language": "de", + "ocr_noise": true, + "features": [ + "line_wrap", + "organization" + ], + "text": "Name: Cora-\nLindenfels\nOrt: Dun-\ndee\nArbeitgeber: Amber Health Foundation.", + "spans": [ + { + "start": 6, + "end": 22, + "label": "PERSON", + "text": "Cora-\nLindenfels" + }, + { + "start": 28, + "end": 36, + "label": "CITY", + "text": "Dun-\ndee" + }, + { + "start": 50, + "end": 73, + "label": "ORGANIZATION", + "text": "Amber Health Foundation" + } + ], + "expected": "Name: [PERSON]\nOrt: [CITY]\nArbeitgeber: [ORGANIZATION]." + }, + { + "id": "holdout03c-027", + "language": "en", + "ocr_noise": true, + "features": [ + "ocr_spelling", + "organization" + ], + "text": "Scanned record: C0ra L1ndenfels resides in Dundee and works at Amber Health Foundation.", + "spans": [ + { + "start": 16, + "end": 31, + "label": "PERSON", + "text": "C0ra L1ndenfels" + }, + { + "start": 43, + "end": 49, + "label": "CITY", + "text": "Dundee" + }, + { + "start": 63, + "end": 86, + "label": "ORGANIZATION", + "text": "Amber Health Foundation" + } + ], + "expected": "Scanned record: [PERSON] resides in [CITY] and works at [ORGANIZATION]." + }, + { + "id": "holdout03c-028", + "language": "de", + "ocr_noise": false, + "features": [ + "religious_reference", + "no_inferred_affiliation" + ], + "text": "Das Thema jainistisch wird in einem Bericht von Amber Health Foundation erwähnt. Für Cora Lindenfels wird keine Religionszugehörigkeit behauptet.", + "spans": [ + { + "start": 10, + "end": 21, + "label": "RELIGION", + "text": "jainistisch" + }, + { + "start": 48, + "end": 71, + "label": "ORGANIZATION", + "text": "Amber Health Foundation" + }, + { + "start": 85, + "end": 100, + "label": "PERSON", + "text": "Cora Lindenfels" + } + ], + "expected": "Das Thema [RELIGION] wird in einem Bericht von [ORGANIZATION] erwähnt. Für [PERSON] wird keine Religionszugehörigkeit behauptet." + }, + { + "id": "holdout03c-029", + "language": "en", + "ocr_noise": false, + "features": [ + "negative", + "religious_reference" + ], + "text": "Worksheet 103: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details.", + "spans": [], + "expected": "Worksheet 103: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details." + }, + { + "id": "holdout03c-030", + "language": "de", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "Auskunft: Jasper Keene arbeitet für Silver Finch Robotics in Aberdeen.", + "spans": [ + { + "start": 10, + "end": 22, + "label": "PERSON", + "text": "Jasper Keene" + }, + { + "start": 36, + "end": 57, + "label": "ORGANIZATION", + "text": "Silver Finch Robotics" + }, + { + "start": 61, + "end": 69, + "label": "CITY", + "text": "Aberdeen" + } + ], + "expected": "Auskunft: [PERSON] arbeitet für [ORGANIZATION] in [CITY]." + }, + { + "id": "holdout03c-031", + "language": "en", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "The contact Jasper Keene represents Silver Finch Robotics near Aberdeen.", + "spans": [ + { + "start": 12, + "end": 24, + "label": "PERSON", + "text": "Jasper Keene" + }, + { + "start": 36, + "end": 57, + "label": "ORGANIZATION", + "text": "Silver Finch Robotics" + }, + { + "start": 63, + "end": 71, + "label": "CITY", + "text": "Aberdeen" + } + ], + "expected": "The contact [PERSON] represents [ORGANIZATION] near [CITY]." + }, + { + "id": "holdout03c-032", + "language": "de", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Jasper Keene nennt hinduistisch als Zugehörigkeit. E-Mail: case104.review@example.invalid; Telefon: +49 541 730104.", + "spans": [ + { + "start": 0, + "end": 12, + "label": "PERSON", + "text": "Jasper Keene" + }, + { + "start": 19, + "end": 31, + "label": "RELIGION", + "text": "hinduistisch" + }, + { + "start": 59, + "end": 89, + "label": "EMAIL", + "text": "case104.review@example.invalid" + }, + { + "start": 100, + "end": 114, + "label": "PHONE", + "text": "+49 541 730104" + } + ], + "expected": "[PERSON] nennt [RELIGION] als Zugehörigkeit. E-Mail: [EMAIL]; Telefon: [PHONE]." + }, + { + "id": "holdout03c-033", + "language": "en", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Jasper Keene explicitly names Hinduism as their faith. Write to case104.review@example.invalid or call +49 541 730104.", + "spans": [ + { + "start": 0, + "end": 12, + "label": "PERSON", + "text": "Jasper Keene" + }, + { + "start": 30, + "end": 38, + "label": "RELIGION", + "text": "Hinduism" + }, + { + "start": 64, + "end": 94, + "label": "EMAIL", + "text": "case104.review@example.invalid" + }, + { + "start": 103, + "end": 117, + "label": "PHONE", + "text": "+49 541 730104" + } + ], + "expected": "[PERSON] explicitly names [RELIGION] as their faith. Write to [EMAIL] or call [PHONE]." + }, + { + "id": "holdout03c-034", + "language": "de", + "ocr_noise": false, + "features": [ + "repeats", + "paragraphs" + ], + "text": "Jasper Keene besucht Aberdeen.\n\nJasper Keene bestätigt den Termin bei Silver Finch Robotics.", + "spans": [ + { + "start": 0, + "end": 12, + "label": "PERSON", + "text": "Jasper Keene" + }, + { + "start": 21, + "end": 29, + "label": "CITY", + "text": "Aberdeen" + }, + { + "start": 32, + "end": 44, + "label": "PERSON", + "text": "Jasper Keene" + }, + { + "start": 70, + "end": 91, + "label": "ORGANIZATION", + "text": "Silver Finch Robotics" + } + ], + "expected": "[PERSON] besucht [CITY].\n\n[PERSON] bestätigt den Termin bei [ORGANIZATION]." + }, + { + "id": "holdout03c-035", + "language": "en", + "ocr_noise": false, + "features": [ + "repeats", + "ambiguous_geography" + ], + "text": "In Aberdeen, Jasper Keene signed for Silver Finch Robotics. The representative Jasper Keene returned to Aberdeen.", + "spans": [ + { + "start": 3, + "end": 11, + "label": "CITY", + "text": "Aberdeen" + }, + { + "start": 13, + "end": 25, + "label": "PERSON", + "text": "Jasper Keene" + }, + { + "start": 37, + "end": 58, + "label": "ORGANIZATION", + "text": "Silver Finch Robotics" + }, + { + "start": 79, + "end": 91, + "label": "PERSON", + "text": "Jasper Keene" + }, + { + "start": 104, + "end": 112, + "label": "CITY", + "text": "Aberdeen" + } + ], + "expected": "In [CITY], [PERSON] signed for [ORGANIZATION]. The representative [PERSON] returned to [CITY]." + }, + { + "id": "holdout03c-036", + "language": "de", + "ocr_noise": true, + "features": [ + "line_wrap", + "organization" + ], + "text": "Name: Jasper-\nKeene\nOrt: Aber-\ndeen\nArbeitgeber: Silver Finch Robotics.", + "spans": [ + { + "start": 6, + "end": 19, + "label": "PERSON", + "text": "Jasper-\nKeene" + }, + { + "start": 25, + "end": 35, + "label": "CITY", + "text": "Aber-\ndeen" + }, + { + "start": 49, + "end": 70, + "label": "ORGANIZATION", + "text": "Silver Finch Robotics" + } + ], + "expected": "Name: [PERSON]\nOrt: [CITY]\nArbeitgeber: [ORGANIZATION]." + }, + { + "id": "holdout03c-037", + "language": "en", + "ocr_noise": true, + "features": [ + "ocr_spelling", + "organization" + ], + "text": "Scanned record: Jasper Keene resides in Aberdeen and works at Silver Finch Robotics.", + "spans": [ + { + "start": 16, + "end": 28, + "label": "PERSON", + "text": "Jasper Keene" + }, + { + "start": 40, + "end": 48, + "label": "CITY", + "text": "Aberdeen" + }, + { + "start": 62, + "end": 83, + "label": "ORGANIZATION", + "text": "Silver Finch Robotics" + } + ], + "expected": "Scanned record: [PERSON] resides in [CITY] and works at [ORGANIZATION]." + }, + { + "id": "holdout03c-038", + "language": "de", + "ocr_noise": false, + "features": [ + "religious_reference", + "no_inferred_affiliation" + ], + "text": "Das Thema hinduistisch wird in einem Bericht von Silver Finch Robotics erwähnt. Für Jasper Keene wird keine Religionszugehörigkeit behauptet.", + "spans": [ + { + "start": 10, + "end": 22, + "label": "RELIGION", + "text": "hinduistisch" + }, + { + "start": 49, + "end": 70, + "label": "ORGANIZATION", + "text": "Silver Finch Robotics" + }, + { + "start": 84, + "end": 96, + "label": "PERSON", + "text": "Jasper Keene" + } + ], + "expected": "Das Thema [RELIGION] wird in einem Bericht von [ORGANIZATION] erwähnt. Für [PERSON] wird keine Religionszugehörigkeit behauptet." + }, + { + "id": "holdout03c-039", + "language": "en", + "ocr_noise": false, + "features": [ + "negative", + "religious_reference" + ], + "text": "Worksheet 104: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details.", + "spans": [], + "expected": "Worksheet 104: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details." + }, + { + "id": "holdout03c-040", + "language": "de", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "Auskunft: Frida Östberg arbeitet für Blue Cedar Cooperative in Uppsala.", + "spans": [ + { + "start": 10, + "end": 23, + "label": "PERSON", + "text": "Frida Östberg" + }, + { + "start": 37, + "end": 59, + "label": "ORGANIZATION", + "text": "Blue Cedar Cooperative" + }, + { + "start": 63, + "end": 70, + "label": "CITY", + "text": "Uppsala" + } + ], + "expected": "Auskunft: [PERSON] arbeitet für [ORGANIZATION] in [CITY]." + }, + { + "id": "holdout03c-041", + "language": "en", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "The contact Frida Östberg represents Blue Cedar Cooperative near Uppsala.", + "spans": [ + { + "start": 12, + "end": 25, + "label": "PERSON", + "text": "Frida Östberg" + }, + { + "start": 37, + "end": 59, + "label": "ORGANIZATION", + "text": "Blue Cedar Cooperative" + }, + { + "start": 65, + "end": 72, + "label": "CITY", + "text": "Uppsala" + } + ], + "expected": "The contact [PERSON] represents [ORGANIZATION] near [CITY]." + }, + { + "id": "holdout03c-042", + "language": "de", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Frida Östberg nennt bahai als Zugehörigkeit. E-Mail: case105.review@example.invalid; Telefon: +49 541 730105.", + "spans": [ + { + "start": 0, + "end": 13, + "label": "PERSON", + "text": "Frida Östberg" + }, + { + "start": 20, + "end": 25, + "label": "RELIGION", + "text": "bahai" + }, + { + "start": 53, + "end": 83, + "label": "EMAIL", + "text": "case105.review@example.invalid" + }, + { + "start": 94, + "end": 108, + "label": "PHONE", + "text": "+49 541 730105" + } + ], + "expected": "[PERSON] nennt [RELIGION] als Zugehörigkeit. E-Mail: [EMAIL]; Telefon: [PHONE]." + }, + { + "id": "holdout03c-043", + "language": "en", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Frida Östberg explicitly names Baháʼí Faith as their faith. Write to case105.review@example.invalid or call +49 541 730105.", + "spans": [ + { + "start": 0, + "end": 13, + "label": "PERSON", + "text": "Frida Östberg" + }, + { + "start": 31, + "end": 43, + "label": "RELIGION", + "text": "Baháʼí Faith" + }, + { + "start": 69, + "end": 99, + "label": "EMAIL", + "text": "case105.review@example.invalid" + }, + { + "start": 108, + "end": 122, + "label": "PHONE", + "text": "+49 541 730105" + } + ], + "expected": "[PERSON] explicitly names [RELIGION] as their faith. Write to [EMAIL] or call [PHONE]." + }, + { + "id": "holdout03c-044", + "language": "de", + "ocr_noise": false, + "features": [ + "repeats", + "paragraphs" + ], + "text": "Frida Östberg besucht Uppsala.\n\nFrida Östberg bestätigt den Termin bei Blue Cedar Cooperative.", + "spans": [ + { + "start": 0, + "end": 13, + "label": "PERSON", + "text": "Frida Östberg" + }, + { + "start": 22, + "end": 29, + "label": "CITY", + "text": "Uppsala" + }, + { + "start": 32, + "end": 45, + "label": "PERSON", + "text": "Frida Östberg" + }, + { + "start": 71, + "end": 93, + "label": "ORGANIZATION", + "text": "Blue Cedar Cooperative" + } + ], + "expected": "[PERSON] besucht [CITY].\n\n[PERSON] bestätigt den Termin bei [ORGANIZATION]." + }, + { + "id": "holdout03c-045", + "language": "en", + "ocr_noise": false, + "features": [ + "repeats", + "ambiguous_geography" + ], + "text": "In Uppsala, Frida Östberg signed for Blue Cedar Cooperative. The representative Frida Östberg returned to Uppsala.", + "spans": [ + { + "start": 3, + "end": 10, + "label": "CITY", + "text": "Uppsala" + }, + { + "start": 12, + "end": 25, + "label": "PERSON", + "text": "Frida Östberg" + }, + { + "start": 37, + "end": 59, + "label": "ORGANIZATION", + "text": "Blue Cedar Cooperative" + }, + { + "start": 80, + "end": 93, + "label": "PERSON", + "text": "Frida Östberg" + }, + { + "start": 106, + "end": 113, + "label": "CITY", + "text": "Uppsala" + } + ], + "expected": "In [CITY], [PERSON] signed for [ORGANIZATION]. The representative [PERSON] returned to [CITY]." + }, + { + "id": "holdout03c-046", + "language": "de", + "ocr_noise": true, + "features": [ + "line_wrap", + "organization" + ], + "text": "Name: Frida-\nÖstberg\nOrt: Upp-\nsala\nArbeitgeber: Blue Cedar Cooperative.", + "spans": [ + { + "start": 6, + "end": 20, + "label": "PERSON", + "text": "Frida-\nÖstberg" + }, + { + "start": 26, + "end": 35, + "label": "CITY", + "text": "Upp-\nsala" + }, + { + "start": 49, + "end": 71, + "label": "ORGANIZATION", + "text": "Blue Cedar Cooperative" + } + ], + "expected": "Name: [PERSON]\nOrt: [CITY]\nArbeitgeber: [ORGANIZATION]." + }, + { + "id": "holdout03c-047", + "language": "en", + "ocr_noise": true, + "features": [ + "ocr_spelling", + "organization" + ], + "text": "Scanned record: Fr1da Östberg resides in Upps4l4 and works at Blue Cedar Cooperative.", + "spans": [ + { + "start": 16, + "end": 29, + "label": "PERSON", + "text": "Fr1da Östberg" + }, + { + "start": 41, + "end": 48, + "label": "CITY", + "text": "Upps4l4" + }, + { + "start": 62, + "end": 84, + "label": "ORGANIZATION", + "text": "Blue Cedar Cooperative" + } + ], + "expected": "Scanned record: [PERSON] resides in [CITY] and works at [ORGANIZATION]." + }, + { + "id": "holdout03c-048", + "language": "de", + "ocr_noise": false, + "features": [ + "religious_reference", + "no_inferred_affiliation" + ], + "text": "Das Thema bahai wird in einem Bericht von Blue Cedar Cooperative erwähnt. Für Frida Östberg wird keine Religionszugehörigkeit behauptet.", + "spans": [ + { + "start": 10, + "end": 15, + "label": "RELIGION", + "text": "bahai" + }, + { + "start": 42, + "end": 64, + "label": "ORGANIZATION", + "text": "Blue Cedar Cooperative" + }, + { + "start": 78, + "end": 91, + "label": "PERSON", + "text": "Frida Östberg" + } + ], + "expected": "Das Thema [RELIGION] wird in einem Bericht von [ORGANIZATION] erwähnt. Für [PERSON] wird keine Religionszugehörigkeit behauptet." + }, + { + "id": "holdout03c-049", + "language": "en", + "ocr_noise": false, + "features": [ + "negative", + "religious_reference" + ], + "text": "Worksheet 105: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details.", + "spans": [], + "expected": "Worksheet 105: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details." + }, + { + "id": "holdout03c-050", + "language": "de", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "Auskunft: Owen Fairchild arbeitet für Oakwell Public Library in Leicester.", + "spans": [ + { + "start": 10, + "end": 24, + "label": "PERSON", + "text": "Owen Fairchild" + }, + { + "start": 38, + "end": 60, + "label": "ORGANIZATION", + "text": "Oakwell Public Library" + }, + { + "start": 64, + "end": 73, + "label": "CITY", + "text": "Leicester" + } + ], + "expected": "Auskunft: [PERSON] arbeitet für [ORGANIZATION] in [CITY]." + }, + { + "id": "holdout03c-051", + "language": "en", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "The contact Owen Fairchild represents Oakwell Public Library near Leicester.", + "spans": [ + { + "start": 12, + "end": 26, + "label": "PERSON", + "text": "Owen Fairchild" + }, + { + "start": 38, + "end": 60, + "label": "ORGANIZATION", + "text": "Oakwell Public Library" + }, + { + "start": 66, + "end": 75, + "label": "CITY", + "text": "Leicester" + } + ], + "expected": "The contact [PERSON] represents [ORGANIZATION] near [CITY]." + }, + { + "id": "holdout03c-052", + "language": "de", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Owen Fairchild nennt buddhistisch als Zugehörigkeit. E-Mail: case106.review@example.invalid; Telefon: +49 541 730106.", + "spans": [ + { + "start": 0, + "end": 14, + "label": "PERSON", + "text": "Owen Fairchild" + }, + { + "start": 21, + "end": 33, + "label": "RELIGION", + "text": "buddhistisch" + }, + { + "start": 61, + "end": 91, + "label": "EMAIL", + "text": "case106.review@example.invalid" + }, + { + "start": 102, + "end": 116, + "label": "PHONE", + "text": "+49 541 730106" + } + ], + "expected": "[PERSON] nennt [RELIGION] als Zugehörigkeit. E-Mail: [EMAIL]; Telefon: [PHONE]." + }, + { + "id": "holdout03c-053", + "language": "en", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Owen Fairchild explicitly names Buddhism as their faith. Write to case106.review@example.invalid or call +49 541 730106.", + "spans": [ + { + "start": 0, + "end": 14, + "label": "PERSON", + "text": "Owen Fairchild" + }, + { + "start": 32, + "end": 40, + "label": "RELIGION", + "text": "Buddhism" + }, + { + "start": 66, + "end": 96, + "label": "EMAIL", + "text": "case106.review@example.invalid" + }, + { + "start": 105, + "end": 119, + "label": "PHONE", + "text": "+49 541 730106" + } + ], + "expected": "[PERSON] explicitly names [RELIGION] as their faith. Write to [EMAIL] or call [PHONE]." + }, + { + "id": "holdout03c-054", + "language": "de", + "ocr_noise": false, + "features": [ + "repeats", + "paragraphs" + ], + "text": "Owen Fairchild besucht Leicester.\n\nOwen Fairchild bestätigt den Termin bei Oakwell Public Library.", + "spans": [ + { + "start": 0, + "end": 14, + "label": "PERSON", + "text": "Owen Fairchild" + }, + { + "start": 23, + "end": 32, + "label": "CITY", + "text": "Leicester" + }, + { + "start": 35, + "end": 49, + "label": "PERSON", + "text": "Owen Fairchild" + }, + { + "start": 75, + "end": 97, + "label": "ORGANIZATION", + "text": "Oakwell Public Library" + } + ], + "expected": "[PERSON] besucht [CITY].\n\n[PERSON] bestätigt den Termin bei [ORGANIZATION]." + }, + { + "id": "holdout03c-055", + "language": "en", + "ocr_noise": false, + "features": [ + "repeats", + "ambiguous_geography" + ], + "text": "In Leicester, Owen Fairchild signed for Oakwell Public Library. The representative Owen Fairchild returned to Leicester.", + "spans": [ + { + "start": 3, + "end": 12, + "label": "CITY", + "text": "Leicester" + }, + { + "start": 14, + "end": 28, + "label": "PERSON", + "text": "Owen Fairchild" + }, + { + "start": 40, + "end": 62, + "label": "ORGANIZATION", + "text": "Oakwell Public Library" + }, + { + "start": 83, + "end": 97, + "label": "PERSON", + "text": "Owen Fairchild" + }, + { + "start": 110, + "end": 119, + "label": "CITY", + "text": "Leicester" + } + ], + "expected": "In [CITY], [PERSON] signed for [ORGANIZATION]. The representative [PERSON] returned to [CITY]." + }, + { + "id": "holdout03c-056", + "language": "de", + "ocr_noise": true, + "features": [ + "line_wrap", + "organization" + ], + "text": "Name: Owen-\nFairchild\nOrt: Leic-\nester\nArbeitgeber: Oakwell Public Library.", + "spans": [ + { + "start": 6, + "end": 21, + "label": "PERSON", + "text": "Owen-\nFairchild" + }, + { + "start": 27, + "end": 38, + "label": "CITY", + "text": "Leic-\nester" + }, + { + "start": 52, + "end": 74, + "label": "ORGANIZATION", + "text": "Oakwell Public Library" + } + ], + "expected": "Name: [PERSON]\nOrt: [CITY]\nArbeitgeber: [ORGANIZATION]." + }, + { + "id": "holdout03c-057", + "language": "en", + "ocr_noise": true, + "features": [ + "ocr_spelling", + "organization" + ], + "text": "Scanned record: Owen Fa1rch1ld resides in Leicester and works at Oakwell Public Library.", + "spans": [ + { + "start": 16, + "end": 30, + "label": "PERSON", + "text": "Owen Fa1rch1ld" + }, + { + "start": 42, + "end": 51, + "label": "CITY", + "text": "Leicester" + }, + { + "start": 65, + "end": 87, + "label": "ORGANIZATION", + "text": "Oakwell Public Library" + } + ], + "expected": "Scanned record: [PERSON] resides in [CITY] and works at [ORGANIZATION]." + }, + { + "id": "holdout03c-058", + "language": "de", + "ocr_noise": false, + "features": [ + "religious_reference", + "no_inferred_affiliation" + ], + "text": "Das Thema buddhistisch wird in einem Bericht von Oakwell Public Library erwähnt. Für Owen Fairchild wird keine Religionszugehörigkeit behauptet.", + "spans": [ + { + "start": 10, + "end": 22, + "label": "RELIGION", + "text": "buddhistisch" + }, + { + "start": 49, + "end": 71, + "label": "ORGANIZATION", + "text": "Oakwell Public Library" + }, + { + "start": 85, + "end": 99, + "label": "PERSON", + "text": "Owen Fairchild" + } + ], + "expected": "Das Thema [RELIGION] wird in einem Bericht von [ORGANIZATION] erwähnt. Für [PERSON] wird keine Religionszugehörigkeit behauptet." + }, + { + "id": "holdout03c-059", + "language": "en", + "ocr_noise": false, + "features": [ + "negative", + "religious_reference" + ], + "text": "Worksheet 106: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details.", + "spans": [], + "expected": "Worksheet 106: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details." + }, + { + "id": "holdout03c-060", + "language": "de", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "Auskunft: Nele Brückner arbeitet für Westhaven Water Authority in Würzburg.", + "spans": [ + { + "start": 10, + "end": 23, + "label": "PERSON", + "text": "Nele Brückner" + }, + { + "start": 37, + "end": 62, + "label": "ORGANIZATION", + "text": "Westhaven Water Authority" + }, + { + "start": 66, + "end": 74, + "label": "CITY", + "text": "Würzburg" + } + ], + "expected": "Auskunft: [PERSON] arbeitet für [ORGANIZATION] in [CITY]." + }, + { + "id": "holdout03c-061", + "language": "en", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "The contact Nele Brückner represents Westhaven Water Authority near Würzburg.", + "spans": [ + { + "start": 12, + "end": 25, + "label": "PERSON", + "text": "Nele Brückner" + }, + { + "start": 37, + "end": 62, + "label": "ORGANIZATION", + "text": "Westhaven Water Authority" + }, + { + "start": 68, + "end": 76, + "label": "CITY", + "text": "Würzburg" + } + ], + "expected": "The contact [PERSON] represents [ORGANIZATION] near [CITY]." + }, + { + "id": "holdout03c-062", + "language": "de", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Nele Brückner nennt katholisch als Zugehörigkeit. E-Mail: case107.review@example.invalid; Telefon: +49 541 730107.", + "spans": [ + { + "start": 0, + "end": 13, + "label": "PERSON", + "text": "Nele Brückner" + }, + { + "start": 20, + "end": 30, + "label": "RELIGION", + "text": "katholisch" + }, + { + "start": 58, + "end": 88, + "label": "EMAIL", + "text": "case107.review@example.invalid" + }, + { + "start": 99, + "end": 113, + "label": "PHONE", + "text": "+49 541 730107" + } + ], + "expected": "[PERSON] nennt [RELIGION] als Zugehörigkeit. E-Mail: [EMAIL]; Telefon: [PHONE]." + }, + { + "id": "holdout03c-063", + "language": "en", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Nele Brückner explicitly names Catholicism as their faith. Write to case107.review@example.invalid or call +49 541 730107.", + "spans": [ + { + "start": 0, + "end": 13, + "label": "PERSON", + "text": "Nele Brückner" + }, + { + "start": 31, + "end": 42, + "label": "RELIGION", + "text": "Catholicism" + }, + { + "start": 68, + "end": 98, + "label": "EMAIL", + "text": "case107.review@example.invalid" + }, + { + "start": 107, + "end": 121, + "label": "PHONE", + "text": "+49 541 730107" + } + ], + "expected": "[PERSON] explicitly names [RELIGION] as their faith. Write to [EMAIL] or call [PHONE]." + }, + { + "id": "holdout03c-064", + "language": "de", + "ocr_noise": false, + "features": [ + "repeats", + "paragraphs" + ], + "text": "Nele Brückner besucht Würzburg.\n\nNele Brückner bestätigt den Termin bei Westhaven Water Authority.", + "spans": [ + { + "start": 0, + "end": 13, + "label": "PERSON", + "text": "Nele Brückner" + }, + { + "start": 22, + "end": 30, + "label": "CITY", + "text": "Würzburg" + }, + { + "start": 33, + "end": 46, + "label": "PERSON", + "text": "Nele Brückner" + }, + { + "start": 72, + "end": 97, + "label": "ORGANIZATION", + "text": "Westhaven Water Authority" + } + ], + "expected": "[PERSON] besucht [CITY].\n\n[PERSON] bestätigt den Termin bei [ORGANIZATION]." + }, + { + "id": "holdout03c-065", + "language": "en", + "ocr_noise": false, + "features": [ + "repeats", + "ambiguous_geography" + ], + "text": "In Würzburg, Nele Brückner signed for Westhaven Water Authority. The representative Nele Brückner returned to Würzburg.", + "spans": [ + { + "start": 3, + "end": 11, + "label": "CITY", + "text": "Würzburg" + }, + { + "start": 13, + "end": 26, + "label": "PERSON", + "text": "Nele Brückner" + }, + { + "start": 38, + "end": 63, + "label": "ORGANIZATION", + "text": "Westhaven Water Authority" + }, + { + "start": 84, + "end": 97, + "label": "PERSON", + "text": "Nele Brückner" + }, + { + "start": 110, + "end": 118, + "label": "CITY", + "text": "Würzburg" + } + ], + "expected": "In [CITY], [PERSON] signed for [ORGANIZATION]. The representative [PERSON] returned to [CITY]." + }, + { + "id": "holdout03c-066", + "language": "de", + "ocr_noise": true, + "features": [ + "line_wrap", + "organization" + ], + "text": "Name: Nele-\nBrückner\nOrt: Würz-\nburg\nArbeitgeber: Westhaven Water Authority.", + "spans": [ + { + "start": 6, + "end": 20, + "label": "PERSON", + "text": "Nele-\nBrückner" + }, + { + "start": 26, + "end": 36, + "label": "CITY", + "text": "Würz-\nburg" + }, + { + "start": 50, + "end": 75, + "label": "ORGANIZATION", + "text": "Westhaven Water Authority" + } + ], + "expected": "Name: [PERSON]\nOrt: [CITY]\nArbeitgeber: [ORGANIZATION]." + }, + { + "id": "holdout03c-067", + "language": "en", + "ocr_noise": true, + "features": [ + "ocr_spelling", + "organization" + ], + "text": "Scanned record: Nele Brückner resides in Würzburg and works at Westhaven Water Authority.", + "spans": [ + { + "start": 16, + "end": 29, + "label": "PERSON", + "text": "Nele Brückner" + }, + { + "start": 41, + "end": 49, + "label": "CITY", + "text": "Würzburg" + }, + { + "start": 63, + "end": 88, + "label": "ORGANIZATION", + "text": "Westhaven Water Authority" + } + ], + "expected": "Scanned record: [PERSON] resides in [CITY] and works at [ORGANIZATION]." + }, + { + "id": "holdout03c-068", + "language": "de", + "ocr_noise": false, + "features": [ + "religious_reference", + "no_inferred_affiliation" + ], + "text": "Das Thema katholisch wird in einem Bericht von Westhaven Water Authority erwähnt. Für Nele Brückner wird keine Religionszugehörigkeit behauptet.", + "spans": [ + { + "start": 10, + "end": 20, + "label": "RELIGION", + "text": "katholisch" + }, + { + "start": 47, + "end": 72, + "label": "ORGANIZATION", + "text": "Westhaven Water Authority" + }, + { + "start": 86, + "end": 99, + "label": "PERSON", + "text": "Nele Brückner" + } + ], + "expected": "Das Thema [RELIGION] wird in einem Bericht von [ORGANIZATION] erwähnt. Für [PERSON] wird keine Religionszugehörigkeit behauptet." + }, + { + "id": "holdout03c-069", + "language": "en", + "ocr_noise": false, + "features": [ + "negative", + "religious_reference" + ], + "text": "Worksheet 107: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details.", + "spans": [], + "expected": "Worksheet 107: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details." + }, + { + "id": "holdout03c-070", + "language": "de", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "Auskunft: Silas Hawthorne arbeitet für Juniper Legal Services in Portland.", + "spans": [ + { + "start": 10, + "end": 25, + "label": "PERSON", + "text": "Silas Hawthorne" + }, + { + "start": 39, + "end": 61, + "label": "ORGANIZATION", + "text": "Juniper Legal Services" + }, + { + "start": 65, + "end": 73, + "label": "CITY", + "text": "Portland" + } + ], + "expected": "Auskunft: [PERSON] arbeitet für [ORGANIZATION] in [CITY]." + }, + { + "id": "holdout03c-071", + "language": "en", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "The contact Silas Hawthorne represents Juniper Legal Services near Portland.", + "spans": [ + { + "start": 12, + "end": 27, + "label": "PERSON", + "text": "Silas Hawthorne" + }, + { + "start": 39, + "end": 61, + "label": "ORGANIZATION", + "text": "Juniper Legal Services" + }, + { + "start": 67, + "end": 75, + "label": "CITY", + "text": "Portland" + } + ], + "expected": "The contact [PERSON] represents [ORGANIZATION] near [CITY]." + }, + { + "id": "holdout03c-072", + "language": "de", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Silas Hawthorne nennt evangelisch als Zugehörigkeit. E-Mail: case108.review@example.invalid; Telefon: +49 541 730108.", + "spans": [ + { + "start": 0, + "end": 15, + "label": "PERSON", + "text": "Silas Hawthorne" + }, + { + "start": 22, + "end": 33, + "label": "RELIGION", + "text": "evangelisch" + }, + { + "start": 61, + "end": 91, + "label": "EMAIL", + "text": "case108.review@example.invalid" + }, + { + "start": 102, + "end": 116, + "label": "PHONE", + "text": "+49 541 730108" + } + ], + "expected": "[PERSON] nennt [RELIGION] als Zugehörigkeit. E-Mail: [EMAIL]; Telefon: [PHONE]." + }, + { + "id": "holdout03c-073", + "language": "en", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Silas Hawthorne explicitly names Protestantism as their faith. Write to case108.review@example.invalid or call +49 541 730108.", + "spans": [ + { + "start": 0, + "end": 15, + "label": "PERSON", + "text": "Silas Hawthorne" + }, + { + "start": 33, + "end": 46, + "label": "RELIGION", + "text": "Protestantism" + }, + { + "start": 72, + "end": 102, + "label": "EMAIL", + "text": "case108.review@example.invalid" + }, + { + "start": 111, + "end": 125, + "label": "PHONE", + "text": "+49 541 730108" + } + ], + "expected": "[PERSON] explicitly names [RELIGION] as their faith. Write to [EMAIL] or call [PHONE]." + }, + { + "id": "holdout03c-074", + "language": "de", + "ocr_noise": false, + "features": [ + "repeats", + "paragraphs" + ], + "text": "Silas Hawthorne besucht Portland.\n\nSilas Hawthorne bestätigt den Termin bei Juniper Legal Services.", + "spans": [ + { + "start": 0, + "end": 15, + "label": "PERSON", + "text": "Silas Hawthorne" + }, + { + "start": 24, + "end": 32, + "label": "CITY", + "text": "Portland" + }, + { + "start": 35, + "end": 50, + "label": "PERSON", + "text": "Silas Hawthorne" + }, + { + "start": 76, + "end": 98, + "label": "ORGANIZATION", + "text": "Juniper Legal Services" + } + ], + "expected": "[PERSON] besucht [CITY].\n\n[PERSON] bestätigt den Termin bei [ORGANIZATION]." + }, + { + "id": "holdout03c-075", + "language": "en", + "ocr_noise": false, + "features": [ + "repeats", + "ambiguous_geography" + ], + "text": "In Portland, Silas Hawthorne signed for Juniper Legal Services. The representative Silas Hawthorne returned to Portland.", + "spans": [ + { + "start": 3, + "end": 11, + "label": "CITY", + "text": "Portland" + }, + { + "start": 13, + "end": 28, + "label": "PERSON", + "text": "Silas Hawthorne" + }, + { + "start": 40, + "end": 62, + "label": "ORGANIZATION", + "text": "Juniper Legal Services" + }, + { + "start": 83, + "end": 98, + "label": "PERSON", + "text": "Silas Hawthorne" + }, + { + "start": 111, + "end": 119, + "label": "CITY", + "text": "Portland" + } + ], + "expected": "In [CITY], [PERSON] signed for [ORGANIZATION]. The representative [PERSON] returned to [CITY]." + }, + { + "id": "holdout03c-076", + "language": "de", + "ocr_noise": true, + "features": [ + "line_wrap", + "organization" + ], + "text": "Name: Silas-\nHawthorne\nOrt: Port-\nland\nArbeitgeber: Juniper Legal Services.", + "spans": [ + { + "start": 6, + "end": 22, + "label": "PERSON", + "text": "Silas-\nHawthorne" + }, + { + "start": 28, + "end": 38, + "label": "CITY", + "text": "Port-\nland" + }, + { + "start": 52, + "end": 74, + "label": "ORGANIZATION", + "text": "Juniper Legal Services" + } + ], + "expected": "Name: [PERSON]\nOrt: [CITY]\nArbeitgeber: [ORGANIZATION]." + }, + { + "id": "holdout03c-077", + "language": "en", + "ocr_noise": true, + "features": [ + "ocr_spelling", + "organization" + ], + "text": "Scanned record: S1las Hawth0rne resides in P0rtl4nd and works at Juniper Legal Services.", + "spans": [ + { + "start": 16, + "end": 31, + "label": "PERSON", + "text": "S1las Hawth0rne" + }, + { + "start": 43, + "end": 51, + "label": "CITY", + "text": "P0rtl4nd" + }, + { + "start": 65, + "end": 87, + "label": "ORGANIZATION", + "text": "Juniper Legal Services" + } + ], + "expected": "Scanned record: [PERSON] resides in [CITY] and works at [ORGANIZATION]." + }, + { + "id": "holdout03c-078", + "language": "de", + "ocr_noise": false, + "features": [ + "religious_reference", + "no_inferred_affiliation" + ], + "text": "Das Thema evangelisch wird in einem Bericht von Juniper Legal Services erwähnt. Für Silas Hawthorne wird keine Religionszugehörigkeit behauptet.", + "spans": [ + { + "start": 10, + "end": 21, + "label": "RELIGION", + "text": "evangelisch" + }, + { + "start": 48, + "end": 70, + "label": "ORGANIZATION", + "text": "Juniper Legal Services" + }, + { + "start": 84, + "end": 99, + "label": "PERSON", + "text": "Silas Hawthorne" + } + ], + "expected": "Das Thema [RELIGION] wird in einem Bericht von [ORGANIZATION] erwähnt. Für [PERSON] wird keine Religionszugehörigkeit behauptet." + }, + { + "id": "holdout03c-079", + "language": "en", + "ocr_noise": false, + "features": [ + "negative", + "religious_reference" + ], + "text": "Worksheet 108: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details.", + "spans": [], + "expected": "Worksheet 108: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details." + }, + { + "id": "holdout03c-080", + "language": "de", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "Auskunft: Alva Rosenqvist arbeitet für Maple Ridge University in Bergen.", + "spans": [ + { + "start": 10, + "end": 25, + "label": "PERSON", + "text": "Alva Rosenqvist" + }, + { + "start": 39, + "end": 61, + "label": "ORGANIZATION", + "text": "Maple Ridge University" + }, + { + "start": 65, + "end": 71, + "label": "CITY", + "text": "Bergen" + } + ], + "expected": "Auskunft: [PERSON] arbeitet für [ORGANIZATION] in [CITY]." + }, + { + "id": "holdout03c-081", + "language": "en", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "The contact Alva Rosenqvist represents Maple Ridge University near Bergen.", + "spans": [ + { + "start": 12, + "end": 27, + "label": "PERSON", + "text": "Alva Rosenqvist" + }, + { + "start": 39, + "end": 61, + "label": "ORGANIZATION", + "text": "Maple Ridge University" + }, + { + "start": 67, + "end": 73, + "label": "CITY", + "text": "Bergen" + } + ], + "expected": "The contact [PERSON] represents [ORGANIZATION] near [CITY]." + }, + { + "id": "holdout03c-082", + "language": "de", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Alva Rosenqvist nennt jüdisch als Zugehörigkeit. E-Mail: case109.review@example.invalid; Telefon: +49 541 730109.", + "spans": [ + { + "start": 0, + "end": 15, + "label": "PERSON", + "text": "Alva Rosenqvist" + }, + { + "start": 22, + "end": 29, + "label": "RELIGION", + "text": "jüdisch" + }, + { + "start": 57, + "end": 87, + "label": "EMAIL", + "text": "case109.review@example.invalid" + }, + { + "start": 98, + "end": 112, + "label": "PHONE", + "text": "+49 541 730109" + } + ], + "expected": "[PERSON] nennt [RELIGION] als Zugehörigkeit. E-Mail: [EMAIL]; Telefon: [PHONE]." + }, + { + "id": "holdout03c-083", + "language": "en", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Alva Rosenqvist explicitly names Judaism as their faith. Write to case109.review@example.invalid or call +49 541 730109.", + "spans": [ + { + "start": 0, + "end": 15, + "label": "PERSON", + "text": "Alva Rosenqvist" + }, + { + "start": 33, + "end": 40, + "label": "RELIGION", + "text": "Judaism" + }, + { + "start": 66, + "end": 96, + "label": "EMAIL", + "text": "case109.review@example.invalid" + }, + { + "start": 105, + "end": 119, + "label": "PHONE", + "text": "+49 541 730109" + } + ], + "expected": "[PERSON] explicitly names [RELIGION] as their faith. Write to [EMAIL] or call [PHONE]." + }, + { + "id": "holdout03c-084", + "language": "de", + "ocr_noise": false, + "features": [ + "repeats", + "paragraphs" + ], + "text": "Alva Rosenqvist besucht Bergen.\n\nAlva Rosenqvist bestätigt den Termin bei Maple Ridge University.", + "spans": [ + { + "start": 0, + "end": 15, + "label": "PERSON", + "text": "Alva Rosenqvist" + }, + { + "start": 24, + "end": 30, + "label": "CITY", + "text": "Bergen" + }, + { + "start": 33, + "end": 48, + "label": "PERSON", + "text": "Alva Rosenqvist" + }, + { + "start": 74, + "end": 96, + "label": "ORGANIZATION", + "text": "Maple Ridge University" + } + ], + "expected": "[PERSON] besucht [CITY].\n\n[PERSON] bestätigt den Termin bei [ORGANIZATION]." + }, + { + "id": "holdout03c-085", + "language": "en", + "ocr_noise": false, + "features": [ + "repeats", + "ambiguous_geography" + ], + "text": "In Bergen, Alva Rosenqvist signed for Maple Ridge University. The representative Alva Rosenqvist returned to Bergen.", + "spans": [ + { + "start": 3, + "end": 9, + "label": "CITY", + "text": "Bergen" + }, + { + "start": 11, + "end": 26, + "label": "PERSON", + "text": "Alva Rosenqvist" + }, + { + "start": 38, + "end": 60, + "label": "ORGANIZATION", + "text": "Maple Ridge University" + }, + { + "start": 81, + "end": 96, + "label": "PERSON", + "text": "Alva Rosenqvist" + }, + { + "start": 109, + "end": 115, + "label": "CITY", + "text": "Bergen" + } + ], + "expected": "In [CITY], [PERSON] signed for [ORGANIZATION]. The representative [PERSON] returned to [CITY]." + }, + { + "id": "holdout03c-086", + "language": "de", + "ocr_noise": true, + "features": [ + "line_wrap", + "organization" + ], + "text": "Name: Alva-\nRosenqvist\nOrt: Ber-\ngen\nArbeitgeber: Maple Ridge University.", + "spans": [ + { + "start": 6, + "end": 22, + "label": "PERSON", + "text": "Alva-\nRosenqvist" + }, + { + "start": 28, + "end": 36, + "label": "CITY", + "text": "Ber-\ngen" + }, + { + "start": 50, + "end": 72, + "label": "ORGANIZATION", + "text": "Maple Ridge University" + } + ], + "expected": "Name: [PERSON]\nOrt: [CITY]\nArbeitgeber: [ORGANIZATION]." + }, + { + "id": "holdout03c-087", + "language": "en", + "ocr_noise": true, + "features": [ + "ocr_spelling", + "organization" + ], + "text": "Scanned record: Alva R0senqv1st resides in Bergen and works at Maple Ridge University.", + "spans": [ + { + "start": 16, + "end": 31, + "label": "PERSON", + "text": "Alva R0senqv1st" + }, + { + "start": 43, + "end": 49, + "label": "CITY", + "text": "Bergen" + }, + { + "start": 63, + "end": 85, + "label": "ORGANIZATION", + "text": "Maple Ridge University" + } + ], + "expected": "Scanned record: [PERSON] resides in [CITY] and works at [ORGANIZATION]." + }, + { + "id": "holdout03c-088", + "language": "de", + "ocr_noise": false, + "features": [ + "religious_reference", + "no_inferred_affiliation" + ], + "text": "Das Thema jüdisch wird in einem Bericht von Maple Ridge University erwähnt. Für Alva Rosenqvist wird keine Religionszugehörigkeit behauptet.", + "spans": [ + { + "start": 10, + "end": 17, + "label": "RELIGION", + "text": "jüdisch" + }, + { + "start": 44, + "end": 66, + "label": "ORGANIZATION", + "text": "Maple Ridge University" + }, + { + "start": 80, + "end": 95, + "label": "PERSON", + "text": "Alva Rosenqvist" + } + ], + "expected": "Das Thema [RELIGION] wird in einem Bericht von [ORGANIZATION] erwähnt. Für [PERSON] wird keine Religionszugehörigkeit behauptet." + }, + { + "id": "holdout03c-089", + "language": "en", + "ocr_noise": false, + "features": [ + "negative", + "religious_reference" + ], + "text": "Worksheet 109: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details.", + "spans": [], + "expected": "Worksheet 109: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details." + }, + { + "id": "holdout03c-090", + "language": "de", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "Auskunft: Theo Merriweather arbeitet für Polar Systems Ltd. in York.", + "spans": [ + { + "start": 10, + "end": 27, + "label": "PERSON", + "text": "Theo Merriweather" + }, + { + "start": 41, + "end": 59, + "label": "ORGANIZATION", + "text": "Polar Systems Ltd." + }, + { + "start": 63, + "end": 67, + "label": "CITY", + "text": "York" + } + ], + "expected": "Auskunft: [PERSON] arbeitet für [ORGANIZATION] in [CITY]." + }, + { + "id": "holdout03c-091", + "language": "en", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "The contact Theo Merriweather represents Polar Systems Ltd. near York.", + "spans": [ + { + "start": 12, + "end": 29, + "label": "PERSON", + "text": "Theo Merriweather" + }, + { + "start": 41, + "end": 59, + "label": "ORGANIZATION", + "text": "Polar Systems Ltd." + }, + { + "start": 65, + "end": 69, + "label": "CITY", + "text": "York" + } + ], + "expected": "The contact [PERSON] represents [ORGANIZATION] near [CITY]." + }, + { + "id": "holdout03c-092", + "language": "de", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Theo Merriweather nennt muslimisch als Zugehörigkeit. E-Mail: case110.review@example.invalid; Telefon: +49 541 730110.", + "spans": [ + { + "start": 0, + "end": 17, + "label": "PERSON", + "text": "Theo Merriweather" + }, + { + "start": 24, + "end": 34, + "label": "RELIGION", + "text": "muslimisch" + }, + { + "start": 62, + "end": 92, + "label": "EMAIL", + "text": "case110.review@example.invalid" + }, + { + "start": 103, + "end": 117, + "label": "PHONE", + "text": "+49 541 730110" + } + ], + "expected": "[PERSON] nennt [RELIGION] als Zugehörigkeit. E-Mail: [EMAIL]; Telefon: [PHONE]." + }, + { + "id": "holdout03c-093", + "language": "en", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Theo Merriweather explicitly names Islam as their faith. Write to case110.review@example.invalid or call +49 541 730110.", + "spans": [ + { + "start": 0, + "end": 17, + "label": "PERSON", + "text": "Theo Merriweather" + }, + { + "start": 35, + "end": 40, + "label": "RELIGION", + "text": "Islam" + }, + { + "start": 66, + "end": 96, + "label": "EMAIL", + "text": "case110.review@example.invalid" + }, + { + "start": 105, + "end": 119, + "label": "PHONE", + "text": "+49 541 730110" + } + ], + "expected": "[PERSON] explicitly names [RELIGION] as their faith. Write to [EMAIL] or call [PHONE]." + }, + { + "id": "holdout03c-094", + "language": "de", + "ocr_noise": false, + "features": [ + "repeats", + "paragraphs" + ], + "text": "Theo Merriweather besucht York.\n\nTheo Merriweather bestätigt den Termin bei Polar Systems Ltd..", + "spans": [ + { + "start": 0, + "end": 17, + "label": "PERSON", + "text": "Theo Merriweather" + }, + { + "start": 26, + "end": 30, + "label": "CITY", + "text": "York" + }, + { + "start": 33, + "end": 50, + "label": "PERSON", + "text": "Theo Merriweather" + }, + { + "start": 76, + "end": 94, + "label": "ORGANIZATION", + "text": "Polar Systems Ltd." + } + ], + "expected": "[PERSON] besucht [CITY].\n\n[PERSON] bestätigt den Termin bei [ORGANIZATION]." + }, + { + "id": "holdout03c-095", + "language": "en", + "ocr_noise": false, + "features": [ + "repeats", + "ambiguous_geography" + ], + "text": "In York, Theo Merriweather signed for Polar Systems Ltd.. The representative Theo Merriweather returned to York.", + "spans": [ + { + "start": 3, + "end": 7, + "label": "CITY", + "text": "York" + }, + { + "start": 9, + "end": 26, + "label": "PERSON", + "text": "Theo Merriweather" + }, + { + "start": 38, + "end": 56, + "label": "ORGANIZATION", + "text": "Polar Systems Ltd." + }, + { + "start": 77, + "end": 94, + "label": "PERSON", + "text": "Theo Merriweather" + }, + { + "start": 107, + "end": 111, + "label": "CITY", + "text": "York" + } + ], + "expected": "In [CITY], [PERSON] signed for [ORGANIZATION]. The representative [PERSON] returned to [CITY]." + }, + { + "id": "holdout03c-096", + "language": "de", + "ocr_noise": true, + "features": [ + "line_wrap", + "organization" + ], + "text": "Name: Theo-\nMerriweather\nOrt: Yo-\nrk\nArbeitgeber: Polar Systems Ltd..", + "spans": [ + { + "start": 6, + "end": 24, + "label": "PERSON", + "text": "Theo-\nMerriweather" + }, + { + "start": 30, + "end": 36, + "label": "CITY", + "text": "Yo-\nrk" + }, + { + "start": 50, + "end": 68, + "label": "ORGANIZATION", + "text": "Polar Systems Ltd." + } + ], + "expected": "Name: [PERSON]\nOrt: [CITY]\nArbeitgeber: [ORGANIZATION]." + }, + { + "id": "holdout03c-097", + "language": "en", + "ocr_noise": true, + "features": [ + "ocr_spelling", + "organization" + ], + "text": "Scanned record: The0 Merr1weather resides in Y0rk and works at Polar Systems Ltd..", + "spans": [ + { + "start": 16, + "end": 33, + "label": "PERSON", + "text": "The0 Merr1weather" + }, + { + "start": 45, + "end": 49, + "label": "CITY", + "text": "Y0rk" + }, + { + "start": 63, + "end": 81, + "label": "ORGANIZATION", + "text": "Polar Systems Ltd." + } + ], + "expected": "Scanned record: [PERSON] resides in [CITY] and works at [ORGANIZATION]." + }, + { + "id": "holdout03c-098", + "language": "de", + "ocr_noise": false, + "features": [ + "religious_reference", + "no_inferred_affiliation" + ], + "text": "Das Thema muslimisch wird in einem Bericht von Polar Systems Ltd. erwähnt. Für Theo Merriweather wird keine Religionszugehörigkeit behauptet.", + "spans": [ + { + "start": 10, + "end": 20, + "label": "RELIGION", + "text": "muslimisch" + }, + { + "start": 47, + "end": 65, + "label": "ORGANIZATION", + "text": "Polar Systems Ltd." + }, + { + "start": 79, + "end": 96, + "label": "PERSON", + "text": "Theo Merriweather" + } + ], + "expected": "Das Thema [RELIGION] wird in einem Bericht von [ORGANIZATION] erwähnt. Für [PERSON] wird keine Religionszugehörigkeit behauptet." + }, + { + "id": "holdout03c-099", + "language": "en", + "ocr_noise": false, + "features": [ + "negative", + "religious_reference" + ], + "text": "Worksheet 110: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details.", + "spans": [], + "expected": "Worksheet 110: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details." + }, + { + "id": "holdout03c-100", + "language": "de", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "Auskunft: Maja Kirschbaum arbeitet für Sonnenbogen Verein in Lüneburg.", + "spans": [ + { + "start": 10, + "end": 25, + "label": "PERSON", + "text": "Maja Kirschbaum" + }, + { + "start": 39, + "end": 57, + "label": "ORGANIZATION", + "text": "Sonnenbogen Verein" + }, + { + "start": 61, + "end": 69, + "label": "CITY", + "text": "Lüneburg" + } + ], + "expected": "Auskunft: [PERSON] arbeitet für [ORGANIZATION] in [CITY]." + }, + { + "id": "holdout03c-101", + "language": "en", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "The contact Maja Kirschbaum represents Sonnenbogen Verein near Lüneburg.", + "spans": [ + { + "start": 12, + "end": 27, + "label": "PERSON", + "text": "Maja Kirschbaum" + }, + { + "start": 39, + "end": 57, + "label": "ORGANIZATION", + "text": "Sonnenbogen Verein" + }, + { + "start": 63, + "end": 71, + "label": "CITY", + "text": "Lüneburg" + } + ], + "expected": "The contact [PERSON] represents [ORGANIZATION] near [CITY]." + }, + { + "id": "holdout03c-102", + "language": "de", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Maja Kirschbaum nennt konfessionslos als Zugehörigkeit. E-Mail: case111.review@example.invalid; Telefon: +49 541 730111.", + "spans": [ + { + "start": 0, + "end": 15, + "label": "PERSON", + "text": "Maja Kirschbaum" + }, + { + "start": 22, + "end": 36, + "label": "RELIGION", + "text": "konfessionslos" + }, + { + "start": 64, + "end": 94, + "label": "EMAIL", + "text": "case111.review@example.invalid" + }, + { + "start": 105, + "end": 119, + "label": "PHONE", + "text": "+49 541 730111" + } + ], + "expected": "[PERSON] nennt [RELIGION] als Zugehörigkeit. E-Mail: [EMAIL]; Telefon: [PHONE]." + }, + { + "id": "holdout03c-103", + "language": "en", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Maja Kirschbaum explicitly names atheism as their faith. Write to case111.review@example.invalid or call +49 541 730111.", + "spans": [ + { + "start": 0, + "end": 15, + "label": "PERSON", + "text": "Maja Kirschbaum" + }, + { + "start": 33, + "end": 40, + "label": "RELIGION", + "text": "atheism" + }, + { + "start": 66, + "end": 96, + "label": "EMAIL", + "text": "case111.review@example.invalid" + }, + { + "start": 105, + "end": 119, + "label": "PHONE", + "text": "+49 541 730111" + } + ], + "expected": "[PERSON] explicitly names [RELIGION] as their faith. Write to [EMAIL] or call [PHONE]." + }, + { + "id": "holdout03c-104", + "language": "de", + "ocr_noise": false, + "features": [ + "repeats", + "paragraphs" + ], + "text": "Maja Kirschbaum besucht Lüneburg.\n\nMaja Kirschbaum bestätigt den Termin bei Sonnenbogen Verein.", + "spans": [ + { + "start": 0, + "end": 15, + "label": "PERSON", + "text": "Maja Kirschbaum" + }, + { + "start": 24, + "end": 32, + "label": "CITY", + "text": "Lüneburg" + }, + { + "start": 35, + "end": 50, + "label": "PERSON", + "text": "Maja Kirschbaum" + }, + { + "start": 76, + "end": 94, + "label": "ORGANIZATION", + "text": "Sonnenbogen Verein" + } + ], + "expected": "[PERSON] besucht [CITY].\n\n[PERSON] bestätigt den Termin bei [ORGANIZATION]." + }, + { + "id": "holdout03c-105", + "language": "en", + "ocr_noise": false, + "features": [ + "repeats", + "ambiguous_geography" + ], + "text": "In Lüneburg, Maja Kirschbaum signed for Sonnenbogen Verein. The representative Maja Kirschbaum returned to Lüneburg.", + "spans": [ + { + "start": 3, + "end": 11, + "label": "CITY", + "text": "Lüneburg" + }, + { + "start": 13, + "end": 28, + "label": "PERSON", + "text": "Maja Kirschbaum" + }, + { + "start": 40, + "end": 58, + "label": "ORGANIZATION", + "text": "Sonnenbogen Verein" + }, + { + "start": 79, + "end": 94, + "label": "PERSON", + "text": "Maja Kirschbaum" + }, + { + "start": 107, + "end": 115, + "label": "CITY", + "text": "Lüneburg" + } + ], + "expected": "In [CITY], [PERSON] signed for [ORGANIZATION]. The representative [PERSON] returned to [CITY]." + }, + { + "id": "holdout03c-106", + "language": "de", + "ocr_noise": true, + "features": [ + "line_wrap", + "organization" + ], + "text": "Name: Maja-\nKirschbaum\nOrt: Lüne-\nburg\nArbeitgeber: Sonnenbogen Verein.", + "spans": [ + { + "start": 6, + "end": 22, + "label": "PERSON", + "text": "Maja-\nKirschbaum" + }, + { + "start": 28, + "end": 38, + "label": "CITY", + "text": "Lüne-\nburg" + }, + { + "start": 52, + "end": 70, + "label": "ORGANIZATION", + "text": "Sonnenbogen Verein" + } + ], + "expected": "Name: [PERSON]\nOrt: [CITY]\nArbeitgeber: [ORGANIZATION]." + }, + { + "id": "holdout03c-107", + "language": "en", + "ocr_noise": true, + "features": [ + "ocr_spelling", + "organization" + ], + "text": "Scanned record: Maja K1rschbaum resides in Lüneburg and works at Sonnenbogen Verein.", + "spans": [ + { + "start": 16, + "end": 31, + "label": "PERSON", + "text": "Maja K1rschbaum" + }, + { + "start": 43, + "end": 51, + "label": "CITY", + "text": "Lüneburg" + }, + { + "start": 65, + "end": 83, + "label": "ORGANIZATION", + "text": "Sonnenbogen Verein" + } + ], + "expected": "Scanned record: [PERSON] resides in [CITY] and works at [ORGANIZATION]." + }, + { + "id": "holdout03c-108", + "language": "de", + "ocr_noise": false, + "features": [ + "religious_reference", + "no_inferred_affiliation" + ], + "text": "Das Thema konfessionslos wird in einem Bericht von Sonnenbogen Verein erwähnt. Für Maja Kirschbaum wird keine Religionszugehörigkeit behauptet.", + "spans": [ + { + "start": 10, + "end": 24, + "label": "RELIGION", + "text": "konfessionslos" + }, + { + "start": 51, + "end": 69, + "label": "ORGANIZATION", + "text": "Sonnenbogen Verein" + }, + { + "start": 83, + "end": 98, + "label": "PERSON", + "text": "Maja Kirschbaum" + } + ], + "expected": "Das Thema [RELIGION] wird in einem Bericht von [ORGANIZATION] erwähnt. Für [PERSON] wird keine Religionszugehörigkeit behauptet." + }, + { + "id": "holdout03c-109", + "language": "en", + "ocr_noise": false, + "features": [ + "negative", + "religious_reference" + ], + "text": "Worksheet 111: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details.", + "spans": [], + "expected": "Worksheet 111: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details." + }, + { + "id": "holdout03c-110", + "language": "de", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "Auskunft: Ronan Whitmore arbeitet für Beacon Valley Labs in Bath.", + "spans": [ + { + "start": 10, + "end": 24, + "label": "PERSON", + "text": "Ronan Whitmore" + }, + { + "start": 38, + "end": 56, + "label": "ORGANIZATION", + "text": "Beacon Valley Labs" + }, + { + "start": 60, + "end": 64, + "label": "CITY", + "text": "Bath" + } + ], + "expected": "Auskunft: [PERSON] arbeitet für [ORGANIZATION] in [CITY]." + }, + { + "id": "holdout03c-111", + "language": "en", + "ocr_noise": false, + "features": [ + "organization", + "geography" + ], + "text": "The contact Ronan Whitmore represents Beacon Valley Labs near Bath.", + "spans": [ + { + "start": 12, + "end": 26, + "label": "PERSON", + "text": "Ronan Whitmore" + }, + { + "start": 38, + "end": 56, + "label": "ORGANIZATION", + "text": "Beacon Valley Labs" + }, + { + "start": 62, + "end": 66, + "label": "CITY", + "text": "Bath" + } + ], + "expected": "The contact [PERSON] represents [ORGANIZATION] near [CITY]." + }, + { + "id": "holdout03c-112", + "language": "de", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Ronan Whitmore nennt christlich als Zugehörigkeit. E-Mail: case112.review@example.invalid; Telefon: +49 541 730112.", + "spans": [ + { + "start": 0, + "end": 14, + "label": "PERSON", + "text": "Ronan Whitmore" + }, + { + "start": 21, + "end": 31, + "label": "RELIGION", + "text": "christlich" + }, + { + "start": 59, + "end": 89, + "label": "EMAIL", + "text": "case112.review@example.invalid" + }, + { + "start": 100, + "end": 114, + "label": "PHONE", + "text": "+49 541 730112" + } + ], + "expected": "[PERSON] nennt [RELIGION] als Zugehörigkeit. E-Mail: [EMAIL]; Telefon: [PHONE]." + }, + { + "id": "holdout03c-113", + "language": "en", + "ocr_noise": false, + "features": [ + "affiliation", + "contacts" + ], + "text": "Ronan Whitmore explicitly names Christianity as their faith. Write to case112.review@example.invalid or call +49 541 730112.", + "spans": [ + { + "start": 0, + "end": 14, + "label": "PERSON", + "text": "Ronan Whitmore" + }, + { + "start": 32, + "end": 44, + "label": "RELIGION", + "text": "Christianity" + }, + { + "start": 70, + "end": 100, + "label": "EMAIL", + "text": "case112.review@example.invalid" + }, + { + "start": 109, + "end": 123, + "label": "PHONE", + "text": "+49 541 730112" + } + ], + "expected": "[PERSON] explicitly names [RELIGION] as their faith. Write to [EMAIL] or call [PHONE]." + }, + { + "id": "holdout03c-114", + "language": "de", + "ocr_noise": false, + "features": [ + "repeats", + "paragraphs" + ], + "text": "Ronan Whitmore besucht Bath.\n\nRonan Whitmore bestätigt den Termin bei Beacon Valley Labs.", + "spans": [ + { + "start": 0, + "end": 14, + "label": "PERSON", + "text": "Ronan Whitmore" + }, + { + "start": 23, + "end": 27, + "label": "CITY", + "text": "Bath" + }, + { + "start": 30, + "end": 44, + "label": "PERSON", + "text": "Ronan Whitmore" + }, + { + "start": 70, + "end": 88, + "label": "ORGANIZATION", + "text": "Beacon Valley Labs" + } + ], + "expected": "[PERSON] besucht [CITY].\n\n[PERSON] bestätigt den Termin bei [ORGANIZATION]." + }, + { + "id": "holdout03c-115", + "language": "en", + "ocr_noise": false, + "features": [ + "repeats", + "ambiguous_geography" + ], + "text": "In Bath, Ronan Whitmore signed for Beacon Valley Labs. The representative Ronan Whitmore returned to Bath.", + "spans": [ + { + "start": 3, + "end": 7, + "label": "CITY", + "text": "Bath" + }, + { + "start": 9, + "end": 23, + "label": "PERSON", + "text": "Ronan Whitmore" + }, + { + "start": 35, + "end": 53, + "label": "ORGANIZATION", + "text": "Beacon Valley Labs" + }, + { + "start": 74, + "end": 88, + "label": "PERSON", + "text": "Ronan Whitmore" + }, + { + "start": 101, + "end": 105, + "label": "CITY", + "text": "Bath" + } + ], + "expected": "In [CITY], [PERSON] signed for [ORGANIZATION]. The representative [PERSON] returned to [CITY]." + }, + { + "id": "holdout03c-116", + "language": "de", + "ocr_noise": true, + "features": [ + "line_wrap", + "organization" + ], + "text": "Name: Ronan-\nWhitmore\nOrt: Ba-\nth\nArbeitgeber: Beacon Valley Labs.", + "spans": [ + { + "start": 6, + "end": 21, + "label": "PERSON", + "text": "Ronan-\nWhitmore" + }, + { + "start": 27, + "end": 33, + "label": "CITY", + "text": "Ba-\nth" + }, + { + "start": 47, + "end": 65, + "label": "ORGANIZATION", + "text": "Beacon Valley Labs" + } + ], + "expected": "Name: [PERSON]\nOrt: [CITY]\nArbeitgeber: [ORGANIZATION]." + }, + { + "id": "holdout03c-117", + "language": "en", + "ocr_noise": true, + "features": [ + "ocr_spelling", + "organization" + ], + "text": "Scanned record: R0nan Wh1tm0re resides in B4th and works at Beacon Valley Labs.", + "spans": [ + { + "start": 16, + "end": 30, + "label": "PERSON", + "text": "R0nan Wh1tm0re" + }, + { + "start": 42, + "end": 46, + "label": "CITY", + "text": "B4th" + }, + { + "start": 60, + "end": 78, + "label": "ORGANIZATION", + "text": "Beacon Valley Labs" + } + ], + "expected": "Scanned record: [PERSON] resides in [CITY] and works at [ORGANIZATION]." + }, + { + "id": "holdout03c-118", + "language": "de", + "ocr_noise": false, + "features": [ + "religious_reference", + "no_inferred_affiliation" + ], + "text": "Das Thema christlich wird in einem Bericht von Beacon Valley Labs erwähnt. Für Ronan Whitmore wird keine Religionszugehörigkeit behauptet.", + "spans": [ + { + "start": 10, + "end": 20, + "label": "RELIGION", + "text": "christlich" + }, + { + "start": 47, + "end": 65, + "label": "ORGANIZATION", + "text": "Beacon Valley Labs" + }, + { + "start": 79, + "end": 93, + "label": "PERSON", + "text": "Ronan Whitmore" + } + ], + "expected": "Das Thema [RELIGION] wird in einem Bericht von [ORGANIZATION] erwähnt. Für [PERSON] wird keine Religionszugehörigkeit behauptet." + }, + { + "id": "holdout03c-119", + "language": "en", + "ocr_noise": false, + "features": [ + "negative", + "religious_reference" + ], + "text": "Worksheet 112: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details.", + "spans": [], + "expected": "Worksheet 112: a name alone establishes no religious affiliation. The plain record contains no named person, place, organization or contact details." + } +] diff --git a/ocr/anonymization/fixtures/holdout03c/freeze.json b/ocr/anonymization/fixtures/holdout03c/freeze.json new file mode 100644 index 0000000000000000000000000000000000000000..bf80ce1464025abcdc2091deacb94122c28e92c4 --- /dev/null +++ b/ocr/anonymization/fixtures/holdout03c/freeze.json @@ -0,0 +1,18 @@ +{ + "schema_version": "frozen-holdout/v1", + "cases": 120, + "sha256": "0174fb41d34d0a4db18877c68fb70149eaa67c05da86608d5da4ff8765a52c15", + "source_head": "176176a6e13dfb1deb1a6f197b7fa46515b0e1f6", + "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", + "control_prompts": { + "PERSON": "person", + "CITY": "location", + "RELIGION": "religion", + "EMAIL": "email address", + "PHONE": "phone number", + "ORGANIZATION": "organization" + }, + "threshold": 0.5, + "evaluation_run_before_freeze": false, + "design": "12 novel entity sets x 10 bilingual contexts; 60 de / 60 en; 24 OCR-noise cases. Synthetic/template-based, not a population sample. No holdout-driven tuning permitted before first recorded result." +} diff --git a/ocr/anonymization/fixtures/holdout03d/README.md b/ocr/anonymization/fixtures/holdout03d/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e997ba27c92b8f533c90731395a57c625e7f1480 --- /dev/null +++ b/ocr/anonymization/fixtures/holdout03d/README.md @@ -0,0 +1,35 @@ +# Gate 03d independent qualification corpus + +Frozen before first model evaluation. 160 individually authored synthetic cases, +80 German and 80 English, in ten strata of sixteen: explicit affiliation, +non-affiliation, belief names, general topics, negation/quotation, contextual +ambiguity, OCR corruption, multiple entities, varied formats/repetition and +negative controls. No real personal data. These are authored by the implementing +agent, not an externally blinded or population-representative sample. + +Gold spans protect religious terminology and explicit status phrases as text; +they do not assert any person's religion. Religion names in general topics, +quotations and negations are protected. Status phrases include their negation +where the phrase expresses non-affiliation. Ambiguous adjectives in a clearly +religious context are protected; mathematical/marketing metaphors are negatives. +Deliberately corrupted OCR spellings retain their exact source spans. Person, +geography, organization and contact spans use the existing category meanings. + +`cases.json` and `freeze.json` are the immutable first-evaluation inputs. No model +inference was used to create or annotate them. The new configuration was selected +using only the previous 40+10+120 development cases. The first result must be kept +regardless of leakage, false positives or blocked documents. Once inspected, +this corpus is development evidence for any future tuning. A zero-canary finite +sample cannot establish universal privacy safety. Automatic release stays off. + +Reproduce in a checkout of the recorded freeze commit, using the pinned locked +CPU dependencies and independently integrity-verified local model cache: + +```sh +HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_HUB_DISABLE_TELEMETRY=1 \ + python -m ocr.anonymization.qualify03d \ + --cache /private/model-cache --output /private/new-qualification-result.json +``` + +The command refuses to overwrite an existing result and returns nonzero on a +failed recognition gate, after saving the full first result and per-case evidence. diff --git a/ocr/anonymization/fixtures/holdout03d/cases.json b/ocr/anonymization/fixtures/holdout03d/cases.json new file mode 100644 index 0000000000000000000000000000000000000000..64187abbc5108fafe81959a5ba56b212531e3305 --- /dev/null +++ b/ocr/anonymization/fixtures/holdout03d/cases.json @@ -0,0 +1,2730 @@ +[ + { + "id": "holdout03d-001", + "language": "de", + "scenario": "affiliation", + "ocr_noise": false, + "text": "Im Aufnahmebogen steht: Bekenntnis römisch-katholisch.", + "spans": [ + { + "start": 35, + "end": 53, + "label": "RELIGION", + "text": "römisch-katholisch" + } + ], + "expected": "Im Aufnahmebogen steht: Bekenntnis [RELIGION]." + }, + { + "id": "holdout03d-002", + "language": "en", + "scenario": "affiliation", + "ocr_noise": false, + "text": "Faith field on the admission card: Roman Catholic.", + "spans": [ + { + "start": 35, + "end": 49, + "label": "RELIGION", + "text": "Roman Catholic" + } + ], + "expected": "Faith field on the admission card: [RELIGION]." + }, + { + "id": "holdout03d-003", + "language": "de", + "scenario": "affiliation", + "ocr_noise": false, + "text": "Tabea Hornfels beschreibt ihre Tradition als sunnitisch.", + "spans": [ + { + "start": 0, + "end": 14, + "label": "PERSON", + "text": "Tabea Hornfels" + }, + { + "start": 45, + "end": 55, + "label": "RELIGION", + "text": "sunnitisch" + } + ], + "expected": "[PERSON] beschreibt ihre Tradition als [RELIGION]." + }, + { + "id": "holdout03d-004", + "language": "en", + "scenario": "affiliation", + "ocr_noise": false, + "text": "Ewan Fairbrook identifies as Shia Muslim in the consent note.", + "spans": [ + { + "start": 0, + "end": 14, + "label": "PERSON", + "text": "Ewan Fairbrook" + }, + { + "start": 29, + "end": 40, + "label": "RELIGION", + "text": "Shia Muslim" + } + ], + "expected": "[PERSON] identifies as [RELIGION] in the consent note." + }, + { + "id": "holdout03d-005", + "language": "de", + "scenario": "affiliation", + "ocr_noise": false, + "text": "Als Glaubensrichtung wurde Jesidentum eingetragen.", + "spans": [ + { + "start": 27, + "end": 37, + "label": "RELIGION", + "text": "Jesidentum" + } + ], + "expected": "Als Glaubensrichtung wurde [RELIGION] eingetragen." + }, + { + "id": "holdout03d-006", + "language": "en", + "scenario": "affiliation", + "ocr_noise": false, + "text": "The registrant selected Quaker under faith community.", + "spans": [ + { + "start": 24, + "end": 30, + "label": "RELIGION", + "text": "Quaker" + } + ], + "expected": "The registrant selected [RELIGION] under faith community." + }, + { + "id": "holdout03d-007", + "language": "de", + "scenario": "affiliation", + "ocr_noise": false, + "text": "Religiöse Selbstauskunft: Sikhismus; Veröffentlichung gesperrt.", + "spans": [ + { + "start": 26, + "end": 35, + "label": "RELIGION", + "text": "Sikhismus" + } + ], + "expected": "Religiöse Selbstauskunft: [RELIGION]; Veröffentlichung gesperrt." + }, + { + "id": "holdout03d-008", + "language": "en", + "scenario": "affiliation", + "ocr_noise": false, + "text": "The private survey records Hinduism as the participant's belief.", + "spans": [ + { + "start": 27, + "end": 35, + "label": "RELIGION", + "text": "Hinduism" + } + ], + "expected": "The private survey records [RELIGION] as the participant's belief." + }, + { + "id": "holdout03d-009", + "language": "de", + "scenario": "affiliation", + "ocr_noise": false, + "text": "Die Person gehört nach eigener Auskunft dem Judentum an.", + "spans": [ + { + "start": 44, + "end": 52, + "label": "RELIGION", + "text": "Judentum" + } + ], + "expected": "Die Person gehört nach eigener Auskunft dem [RELIGION] an." + }, + { + "id": "holdout03d-010", + "language": "en", + "scenario": "affiliation", + "ocr_noise": false, + "text": "The speaker described her religion as Buddhism.", + "spans": [ + { + "start": 38, + "end": 46, + "label": "RELIGION", + "text": "Buddhism" + } + ], + "expected": "The speaker described her religion as [RELIGION]." + }, + { + "id": "holdout03d-011", + "language": "de", + "scenario": "affiliation", + "ocr_noise": false, + "text": "Auf Seite zwei lautet die Angabe zur Religion Bahai.", + "spans": [ + { + "start": 46, + "end": 51, + "label": "RELIGION", + "text": "Bahai" + } + ], + "expected": "Auf Seite zwei lautet die Angabe zur Religion [RELIGION]." + }, + { + "id": "holdout03d-012", + "language": "en", + "scenario": "affiliation", + "ocr_noise": false, + "text": "In a handwritten footnote the applicant wrote Jainism.", + "spans": [ + { + "start": 46, + "end": 53, + "label": "RELIGION", + "text": "Jainism" + } + ], + "expected": "In a handwritten footnote the applicant wrote [RELIGION]." + }, + { + "id": "holdout03d-013", + "language": "de", + "scenario": "affiliation", + "ocr_noise": false, + "text": "Konfession des Antragstellers: evangelisch-lutherisch.", + "spans": [ + { + "start": 31, + "end": 53, + "label": "RELIGION", + "text": "evangelisch-lutherisch" + } + ], + "expected": "Konfession des Antragstellers: [RELIGION]." + }, + { + "id": "holdout03d-014", + "language": "en", + "scenario": "affiliation", + "ocr_noise": false, + "text": "The confidential intake identifies the resident as Seventh-day Adventist.", + "spans": [ + { + "start": 51, + "end": 72, + "label": "RELIGION", + "text": "Seventh-day Adventist" + } + ], + "expected": "The confidential intake identifies the resident as [RELIGION]." + }, + { + "id": "holdout03d-015", + "language": "de", + "scenario": "affiliation", + "ocr_noise": false, + "text": "Raban Moosfeld bekennt sich zum Christentum.", + "spans": [ + { + "start": 0, + "end": 14, + "label": "PERSON", + "text": "Raban Moosfeld" + }, + { + "start": 32, + "end": 43, + "label": "RELIGION", + "text": "Christentum" + } + ], + "expected": "[PERSON] bekennt sich zum [RELIGION]." + }, + { + "id": "holdout03d-016", + "language": "en", + "scenario": "affiliation", + "ocr_noise": false, + "text": "Iona Crestwell states that she practices Zoroastrianism.", + "spans": [ + { + "start": 0, + "end": 14, + "label": "PERSON", + "text": "Iona Crestwell" + }, + { + "start": 41, + "end": 55, + "label": "RELIGION", + "text": "Zoroastrianism" + } + ], + "expected": "[PERSON] states that she practices [RELIGION]." + }, + { + "id": "holdout03d-017", + "language": "de", + "scenario": "nonaffiliation", + "ocr_noise": false, + "text": "Die Antwort im Konfessionsfeld lautet konfessionsfrei.", + "spans": [ + { + "start": 38, + "end": 53, + "label": "RELIGION", + "text": "konfessionsfrei" + } + ], + "expected": "Die Antwort im Konfessionsfeld lautet [RELIGION]." + }, + { + "id": "holdout03d-018", + "language": "en", + "scenario": "nonaffiliation", + "ocr_noise": false, + "text": "Religious status: no religious affiliation.", + "spans": [ + { + "start": 18, + "end": 42, + "label": "RELIGION", + "text": "no religious affiliation" + } + ], + "expected": "Religious status: [RELIGION]." + }, + { + "id": "holdout03d-019", + "language": "de", + "scenario": "nonaffiliation", + "ocr_noise": false, + "text": "Antrag: Die Bewohnerin ist religionslos.", + "spans": [ + { + "start": 27, + "end": 39, + "label": "RELIGION", + "text": "religionslos" + } + ], + "expected": "Antrag: Die Bewohnerin ist [RELIGION]." + }, + { + "id": "holdout03d-020", + "language": "en", + "scenario": "nonaffiliation", + "ocr_noise": false, + "text": "On the registration slip the visitor wrote atheist.", + "spans": [ + { + "start": 43, + "end": 50, + "label": "RELIGION", + "text": "atheist" + } + ], + "expected": "On the registration slip the visitor wrote [RELIGION]." + }, + { + "id": "holdout03d-021", + "language": "de", + "scenario": "nonaffiliation", + "ocr_noise": false, + "text": "Selbstbeschreibung im versiegelten Fragebogen: agnostisch.", + "spans": [ + { + "start": 47, + "end": 57, + "label": "RELIGION", + "text": "agnostisch" + } + ], + "expected": "Selbstbeschreibung im versiegelten Fragebogen: [RELIGION]." + }, + { + "id": "holdout03d-022", + "language": "en", + "scenario": "nonaffiliation", + "ocr_noise": false, + "text": "The individual describes himself as agnostic.", + "spans": [ + { + "start": 36, + "end": 44, + "label": "RELIGION", + "text": "agnostic" + } + ], + "expected": "The individual describes himself as [RELIGION]." + }, + { + "id": "holdout03d-023", + "language": "de", + "scenario": "nonaffiliation", + "ocr_noise": false, + "text": "Zur Frage nach der Religion: ohne Konfession.", + "spans": [ + { + "start": 29, + "end": 44, + "label": "RELIGION", + "text": "ohne Konfession" + } + ], + "expected": "Zur Frage nach der Religion: [RELIGION]." + }, + { + "id": "holdout03d-024", + "language": "en", + "scenario": "nonaffiliation", + "ocr_noise": false, + "text": "The recorded answer was nonreligious, with no additional details.", + "spans": [ + { + "start": 24, + "end": 36, + "label": "RELIGION", + "text": "nonreligious" + } + ], + "expected": "The recorded answer was [RELIGION], with no additional details." + }, + { + "id": "holdout03d-025", + "language": "de", + "scenario": "nonaffiliation", + "ocr_noise": false, + "text": "Für den Eintrag gilt keine Religion als wörtliche Antwort.", + "spans": [ + { + "start": 21, + "end": 35, + "label": "RELIGION", + "text": "keine Religion" + } + ], + "expected": "Für den Eintrag gilt [RELIGION] als wörtliche Antwort." + }, + { + "id": "holdout03d-026", + "language": "en", + "scenario": "nonaffiliation", + "ocr_noise": false, + "text": "Affiliation was entered as religiously unaffiliated.", + "spans": [ + { + "start": 27, + "end": 51, + "label": "RELIGION", + "text": "religiously unaffiliated" + } + ], + "expected": "Affiliation was entered as [RELIGION]." + }, + { + "id": "holdout03d-027", + "language": "de", + "scenario": "nonaffiliation", + "ocr_noise": false, + "text": "Die vertrauliche Karte vermerkt konfessionslos neben dem Geburtsjahr.", + "spans": [ + { + "start": 32, + "end": 46, + "label": "RELIGION", + "text": "konfessionslos" + } + ], + "expected": "Die vertrauliche Karte vermerkt [RELIGION] neben dem Geburtsjahr." + }, + { + "id": "holdout03d-028", + "language": "en", + "scenario": "nonaffiliation", + "ocr_noise": false, + "text": "Personal belief: atheism; this field must remain private.", + "spans": [ + { + "start": 17, + "end": 24, + "label": "RELIGION", + "text": "atheism" + } + ], + "expected": "Personal belief: [RELIGION]; this field must remain private." + }, + { + "id": "holdout03d-029", + "language": "de", + "scenario": "nonaffiliation", + "ocr_noise": false, + "text": "Beim Interview wurde Atheismus als eigene Haltung genannt.", + "spans": [ + { + "start": 21, + "end": 30, + "label": "RELIGION", + "text": "Atheismus" + } + ], + "expected": "Beim Interview wurde [RELIGION] als eigene Haltung genannt." + }, + { + "id": "holdout03d-030", + "language": "en", + "scenario": "nonaffiliation", + "ocr_noise": false, + "text": "The resident's own answer is no religion.", + "spans": [ + { + "start": 29, + "end": 40, + "label": "RELIGION", + "text": "no religion" + } + ], + "expected": "The resident's own answer is [RELIGION]." + }, + { + "id": "holdout03d-031", + "language": "de", + "scenario": "nonaffiliation", + "ocr_noise": false, + "text": "Als Weltanschauung wird Agnostizismus angegeben.", + "spans": [ + { + "start": 24, + "end": 37, + "label": "RELIGION", + "text": "Agnostizismus" + } + ], + "expected": "Als Weltanschauung wird [RELIGION] angegeben." + }, + { + "id": "holdout03d-032", + "language": "en", + "scenario": "nonaffiliation", + "ocr_noise": false, + "text": "The form marks the applicant non-religious.", + "spans": [ + { + "start": 29, + "end": 42, + "label": "RELIGION", + "text": "non-religious" + } + ], + "expected": "The form marks the applicant [RELIGION]." + }, + { + "id": "holdout03d-033", + "language": "de", + "scenario": "belief_names", + "ocr_noise": false, + "text": "Ein Glossar definiert Daoismus auf der ersten Seite.", + "spans": [ + { + "start": 22, + "end": 30, + "label": "RELIGION", + "text": "Daoismus" + } + ], + "expected": "Ein Glossar definiert [RELIGION] auf der ersten Seite." + }, + { + "id": "holdout03d-034", + "language": "en", + "scenario": "belief_names", + "ocr_noise": false, + "text": "The index contains an entry for Shintoism.", + "spans": [ + { + "start": 32, + "end": 41, + "label": "RELIGION", + "text": "Shintoism" + } + ], + "expected": "The index contains an entry for [RELIGION]." + }, + { + "id": "holdout03d-035", + "language": "de", + "scenario": "belief_names", + "ocr_noise": false, + "text": "Die Karteikarte trägt nur das Stichwort Islam.", + "spans": [ + { + "start": 40, + "end": 45, + "label": "RELIGION", + "text": "Islam" + } + ], + "expected": "Die Karteikarte trägt nur das Stichwort [RELIGION]." + }, + { + "id": "holdout03d-036", + "language": "en", + "scenario": "belief_names", + "ocr_noise": false, + "text": "A dictionary heading reads Judaism.", + "spans": [ + { + "start": 27, + "end": 34, + "label": "RELIGION", + "text": "Judaism" + } + ], + "expected": "A dictionary heading reads [RELIGION]." + }, + { + "id": "holdout03d-037", + "language": "de", + "scenario": "belief_names", + "ocr_noise": false, + "text": "Im Inhaltsverzeichnis stehen Hinduismus und Buddhismus.", + "spans": [ + { + "start": 29, + "end": 39, + "label": "RELIGION", + "text": "Hinduismus" + }, + { + "start": 44, + "end": 54, + "label": "RELIGION", + "text": "Buddhismus" + } + ], + "expected": "Im Inhaltsverzeichnis stehen [RELIGION] und [RELIGION]." + }, + { + "id": "holdout03d-038", + "language": "en", + "scenario": "belief_names", + "ocr_noise": false, + "text": "The glossary compares Christianity with Taoism.", + "spans": [ + { + "start": 22, + "end": 34, + "label": "RELIGION", + "text": "Christianity" + }, + { + "start": 40, + "end": 46, + "label": "RELIGION", + "text": "Taoism" + } + ], + "expected": "The glossary compares [RELIGION] with [RELIGION]." + }, + { + "id": "holdout03d-039", + "language": "de", + "scenario": "belief_names", + "ocr_noise": false, + "text": "Das Museum beschriftet einen Abschnitt mit Jainismus.", + "spans": [ + { + "start": 43, + "end": 52, + "label": "RELIGION", + "text": "Jainismus" + } + ], + "expected": "Das Museum beschriftet einen Abschnitt mit [RELIGION]." + }, + { + "id": "holdout03d-040", + "language": "en", + "scenario": "belief_names", + "ocr_noise": false, + "text": "The catalog classifies the manuscript under Sikhism.", + "spans": [ + { + "start": 44, + "end": 51, + "label": "RELIGION", + "text": "Sikhism" + } + ], + "expected": "The catalog classifies the manuscript under [RELIGION]." + }, + { + "id": "holdout03d-041", + "language": "de", + "scenario": "belief_names", + "ocr_noise": false, + "text": "Der Buchtitel erwähnt Zoroastrismus, ohne Personen zu benennen.", + "spans": [ + { + "start": 22, + "end": 35, + "label": "RELIGION", + "text": "Zoroastrismus" + } + ], + "expected": "Der Buchtitel erwähnt [RELIGION], ohne Personen zu benennen." + }, + { + "id": "holdout03d-042", + "language": "en", + "scenario": "belief_names", + "ocr_noise": false, + "text": "A subject heading names Baháʼí Faith.", + "spans": [ + { + "start": 24, + "end": 36, + "label": "RELIGION", + "text": "Baháʼí Faith" + } + ], + "expected": "A subject heading names [RELIGION]." + }, + { + "id": "holdout03d-043", + "language": "de", + "scenario": "belief_names", + "ocr_noise": false, + "text": "Die Übersicht behandelt Katholizismus in einem historischen Kapitel.", + "spans": [ + { + "start": 24, + "end": 37, + "label": "RELIGION", + "text": "Katholizismus" + } + ], + "expected": "Die Übersicht behandelt [RELIGION] in einem historischen Kapitel." + }, + { + "id": "holdout03d-044", + "language": "en", + "scenario": "belief_names", + "ocr_noise": false, + "text": "The bibliography has a subsection titled Protestantism.", + "spans": [ + { + "start": 41, + "end": 54, + "label": "RELIGION", + "text": "Protestantism" + } + ], + "expected": "The bibliography has a subsection titled [RELIGION]." + }, + { + "id": "holdout03d-045", + "language": "de", + "scenario": "belief_names", + "ocr_noise": false, + "text": "Auf dem Deckblatt steht Orthodoxie.", + "spans": [ + { + "start": 24, + "end": 34, + "label": "RELIGION", + "text": "Orthodoxie" + } + ], + "expected": "Auf dem Deckblatt steht [RELIGION]." + }, + { + "id": "holdout03d-046", + "language": "en", + "scenario": "belief_names", + "ocr_noise": false, + "text": "A lecture outline begins with Orthodox Christianity.", + "spans": [ + { + "start": 30, + "end": 51, + "label": "RELIGION", + "text": "Orthodox Christianity" + } + ], + "expected": "A lecture outline begins with [RELIGION]." + }, + { + "id": "holdout03d-047", + "language": "de", + "scenario": "belief_names", + "ocr_noise": false, + "text": "Das Handbuch unterscheidet Sikhismus von Jainismus.", + "spans": [ + { + "start": 27, + "end": 36, + "label": "RELIGION", + "text": "Sikhismus" + }, + { + "start": 41, + "end": 50, + "label": "RELIGION", + "text": "Jainismus" + } + ], + "expected": "Das Handbuch unterscheidet [RELIGION] von [RELIGION]." + }, + { + "id": "holdout03d-048", + "language": "en", + "scenario": "belief_names", + "ocr_noise": false, + "text": "The encyclopedia provides a chapter about Confucianism.", + "spans": [ + { + "start": 42, + "end": 54, + "label": "RELIGION", + "text": "Confucianism" + } + ], + "expected": "The encyclopedia provides a chapter about [RELIGION]." + }, + { + "id": "holdout03d-049", + "language": "de", + "scenario": "general_topics", + "ocr_noise": false, + "text": "Die Ausstellung erklärt das Fasten im Islam; Besucherangaben fehlen.", + "spans": [ + { + "start": 38, + "end": 43, + "label": "RELIGION", + "text": "Islam" + } + ], + "expected": "Die Ausstellung erklärt das Fasten im [RELIGION]; Besucherangaben fehlen." + }, + { + "id": "holdout03d-050", + "language": "en", + "scenario": "general_topics", + "ocr_noise": false, + "text": "The essay examines pilgrimage in Hinduism without naming a follower.", + "spans": [ + { + "start": 33, + "end": 41, + "label": "RELIGION", + "text": "Hinduism" + } + ], + "expected": "The essay examines pilgrimage in [RELIGION] without naming a follower." + }, + { + "id": "holdout03d-051", + "language": "de", + "scenario": "general_topics", + "ocr_noise": false, + "text": "Historische Randnotiz: Christentum im frühen Mittelalter.", + "spans": [ + { + "start": 23, + "end": 34, + "label": "RELIGION", + "text": "Christentum" + } + ], + "expected": "Historische Randnotiz: [RELIGION] im frühen Mittelalter." + }, + { + "id": "holdout03d-052", + "language": "en", + "scenario": "general_topics", + "ocr_noise": false, + "text": "The radio program discusses Buddhism as a topic, not the presenter's identity.", + "spans": [ + { + "start": 28, + "end": 36, + "label": "RELIGION", + "text": "Buddhism" + } + ], + "expected": "The radio program discusses [RELIGION] as a topic, not the presenter's identity." + }, + { + "id": "holdout03d-053", + "language": "de", + "scenario": "general_topics", + "ocr_noise": false, + "text": "Auf der Leseliste steht ein Werk zum Judentum.", + "spans": [ + { + "start": 37, + "end": 45, + "label": "RELIGION", + "text": "Judentum" + } + ], + "expected": "Auf der Leseliste steht ein Werk zum [RELIGION]." + }, + { + "id": "holdout03d-054", + "language": "en", + "scenario": "general_topics", + "ocr_noise": false, + "text": "A public seminar on Jainism does not establish anyone's affiliation.", + "spans": [ + { + "start": 20, + "end": 27, + "label": "RELIGION", + "text": "Jainism" + } + ], + "expected": "A public seminar on [RELIGION] does not establish anyone's affiliation." + }, + { + "id": "holdout03d-055", + "language": "de", + "scenario": "general_topics", + "ocr_noise": false, + "text": "Die Wandtafel erläutert Bahai als historischen Gegenstand.", + "spans": [ + { + "start": 24, + "end": 29, + "label": "RELIGION", + "text": "Bahai" + } + ], + "expected": "Die Wandtafel erläutert [RELIGION] als historischen Gegenstand." + }, + { + "id": "holdout03d-056", + "language": "en", + "scenario": "general_topics", + "ocr_noise": false, + "text": "The podcast's subject is Sikhism, with no personal disclosures.", + "spans": [ + { + "start": 25, + "end": 32, + "label": "RELIGION", + "text": "Sikhism" + } + ], + "expected": "The podcast's subject is [RELIGION], with no personal disclosures." + }, + { + "id": "holdout03d-057", + "language": "de", + "scenario": "general_topics", + "ocr_noise": false, + "text": "Kapitelüberschrift: Atheismus und gesellschaftlicher Wandel.", + "spans": [ + { + "start": 20, + "end": 29, + "label": "RELIGION", + "text": "Atheismus" + } + ], + "expected": "Kapitelüberschrift: [RELIGION] und gesellschaftlicher Wandel." + }, + { + "id": "holdout03d-058", + "language": "en", + "scenario": "general_topics", + "ocr_noise": false, + "text": "The chart compares atheism and agnosticism at a conceptual level.", + "spans": [ + { + "start": 19, + "end": 26, + "label": "RELIGION", + "text": "atheism" + }, + { + "start": 31, + "end": 42, + "label": "RELIGION", + "text": "agnosticism" + } + ], + "expected": "The chart compares [RELIGION] and [RELIGION] at a conceptual level." + }, + { + "id": "holdout03d-059", + "language": "de", + "scenario": "general_topics", + "ocr_noise": false, + "text": "Ein Forschungstext beschreibt Shintoismus ohne individuelle Angaben.", + "spans": [ + { + "start": 30, + "end": 41, + "label": "RELIGION", + "text": "Shintoismus" + } + ], + "expected": "Ein Forschungstext beschreibt [RELIGION] ohne individuelle Angaben." + }, + { + "id": "holdout03d-060", + "language": "en", + "scenario": "general_topics", + "ocr_noise": false, + "text": "The librarian assigned Taoism as a subject keyword.", + "spans": [ + { + "start": 23, + "end": 29, + "label": "RELIGION", + "text": "Taoism" + } + ], + "expected": "The librarian assigned [RELIGION] as a subject keyword." + }, + { + "id": "holdout03d-061", + "language": "de", + "scenario": "general_topics", + "ocr_noise": false, + "text": "Die Zeitleiste beschreibt den Protestantismus als Bewegung.", + "spans": [ + { + "start": 30, + "end": 45, + "label": "RELIGION", + "text": "Protestantismus" + } + ], + "expected": "Die Zeitleiste beschreibt den [RELIGION] als Bewegung." + }, + { + "id": "holdout03d-062", + "language": "en", + "scenario": "general_topics", + "ocr_noise": false, + "text": "The audio guide explains Catholicism to an unspecified audience.", + "spans": [ + { + "start": 25, + "end": 36, + "label": "RELIGION", + "text": "Catholicism" + } + ], + "expected": "The audio guide explains [RELIGION] to an unspecified audience." + }, + { + "id": "holdout03d-063", + "language": "de", + "scenario": "general_topics", + "ocr_noise": false, + "text": "Ein allgemeiner Artikel nennt konfessionslos als statistische Kategorie.", + "spans": [ + { + "start": 30, + "end": 44, + "label": "RELIGION", + "text": "konfessionslos" + } + ], + "expected": "Ein allgemeiner Artikel nennt [RELIGION] als statistische Kategorie." + }, + { + "id": "holdout03d-064", + "language": "en", + "scenario": "general_topics", + "ocr_noise": false, + "text": "A terminology sheet explains non-religious; it contains no respondent records.", + "spans": [ + { + "start": 29, + "end": 42, + "label": "RELIGION", + "text": "non-religious" + } + ], + "expected": "A terminology sheet explains [RELIGION]; it contains no respondent records." + }, + { + "id": "holdout03d-065", + "language": "de", + "scenario": "negation_quotation", + "ocr_noise": false, + "text": "Im Protokoll steht: »Ich bin nicht muslimisch.«", + "spans": [ + { + "start": 35, + "end": 45, + "label": "RELIGION", + "text": "muslimisch" + } + ], + "expected": "Im Protokoll steht: »Ich bin nicht [RELIGION].«" + }, + { + "id": "holdout03d-066", + "language": "en", + "scenario": "negation_quotation", + "ocr_noise": false, + "text": "The witness said, \"I am not Christian.\"", + "spans": [ + { + "start": 28, + "end": 37, + "label": "RELIGION", + "text": "Christian" + } + ], + "expected": "The witness said, \"I am not [RELIGION].\"" + }, + { + "id": "holdout03d-067", + "language": "de", + "scenario": "negation_quotation", + "ocr_noise": false, + "text": "Sie widerspricht dem Eintrag katholisch ausdrücklich.", + "spans": [ + { + "start": 29, + "end": 39, + "label": "RELIGION", + "text": "katholisch" + } + ], + "expected": "Sie widerspricht dem Eintrag [RELIGION] ausdrücklich." + }, + { + "id": "holdout03d-068", + "language": "en", + "scenario": "negation_quotation", + "ocr_noise": false, + "text": "The interviewee rejected the label atheist.", + "spans": [ + { + "start": 35, + "end": 42, + "label": "RELIGION", + "text": "atheist" + } + ], + "expected": "The interviewee rejected the label [RELIGION]." + }, + { + "id": "holdout03d-069", + "language": "de", + "scenario": "negation_quotation", + "ocr_noise": false, + "text": "Das Wort »buddhistisch« ist ein Zitat aus einem Lehrbuch.", + "spans": [ + { + "start": 10, + "end": 22, + "label": "RELIGION", + "text": "buddhistisch" + } + ], + "expected": "Das Wort »[RELIGION]« ist ein Zitat aus einem Lehrbuch." + }, + { + "id": "holdout03d-070", + "language": "en", + "scenario": "negation_quotation", + "ocr_noise": false, + "text": "\"Jewish\" appears in a quotation, not a claim about the editor.", + "spans": [ + { + "start": 1, + "end": 7, + "label": "RELIGION", + "text": "Jewish" + } + ], + "expected": "\"[RELIGION]\" appears in a quotation, not a claim about the editor." + }, + { + "id": "holdout03d-071", + "language": "de", + "scenario": "negation_quotation", + "ocr_noise": false, + "text": "Niemand behauptet hier, die Autorin sei hinduistisch.", + "spans": [ + { + "start": 40, + "end": 52, + "label": "RELIGION", + "text": "hinduistisch" + } + ], + "expected": "Niemand behauptet hier, die Autorin sei [RELIGION]." + }, + { + "id": "holdout03d-072", + "language": "en", + "scenario": "negation_quotation", + "ocr_noise": false, + "text": "No one alleged that the resident was Muslim.", + "spans": [ + { + "start": 37, + "end": 43, + "label": "RELIGION", + "text": "Muslim" + } + ], + "expected": "No one alleged that the resident was [RELIGION]." + }, + { + "id": "holdout03d-073", + "language": "de", + "scenario": "negation_quotation", + "ocr_noise": false, + "text": "»Ohne Konfession« wurde als mögliche Antwort vorgelesen.", + "spans": [ + { + "start": 1, + "end": 16, + "label": "RELIGION", + "text": "Ohne Konfession" + } + ], + "expected": "»[RELIGION]« wurde als mögliche Antwort vorgelesen." + }, + { + "id": "holdout03d-074", + "language": "en", + "scenario": "negation_quotation", + "ocr_noise": false, + "text": "The clerk read the option \"no religion\" aloud.", + "spans": [ + { + "start": 27, + "end": 38, + "label": "RELIGION", + "text": "no religion" + } + ], + "expected": "The clerk read the option \"[RELIGION]\" aloud." + }, + { + "id": "holdout03d-075", + "language": "de", + "scenario": "negation_quotation", + "ocr_noise": false, + "text": "Nicht evangelisch, sondern agnostisch: so lautet die Korrektur.", + "spans": [ + { + "start": 6, + "end": 17, + "label": "RELIGION", + "text": "evangelisch" + }, + { + "start": 27, + "end": 37, + "label": "RELIGION", + "text": "agnostisch" + } + ], + "expected": "Nicht [RELIGION], sondern [RELIGION]: so lautet die Korrektur." + }, + { + "id": "holdout03d-076", + "language": "en", + "scenario": "negation_quotation", + "ocr_noise": false, + "text": "The note corrects Catholic to nonreligious.", + "spans": [ + { + "start": 18, + "end": 26, + "label": "RELIGION", + "text": "Catholic" + }, + { + "start": 30, + "end": 42, + "label": "RELIGION", + "text": "nonreligious" + } + ], + "expected": "The note corrects [RELIGION] to [RELIGION]." + }, + { + "id": "holdout03d-077", + "language": "de", + "scenario": "negation_quotation", + "ocr_noise": false, + "text": "Die Rezension zitiert Judentum, ohne einen Glauben zuzuschreiben.", + "spans": [ + { + "start": 22, + "end": 30, + "label": "RELIGION", + "text": "Judentum" + } + ], + "expected": "Die Rezension zitiert [RELIGION], ohne einen Glauben zuzuschreiben." + }, + { + "id": "holdout03d-078", + "language": "en", + "scenario": "negation_quotation", + "ocr_noise": false, + "text": "A reviewer underlined \"Baha'i\" in the supplied excerpt.", + "spans": [ + { + "start": 23, + "end": 29, + "label": "RELIGION", + "text": "Baha'i" + } + ], + "expected": "A reviewer underlined \"[RELIGION]\" in the supplied excerpt." + }, + { + "id": "holdout03d-079", + "language": "de", + "scenario": "negation_quotation", + "ocr_noise": false, + "text": "Die Behauptung einer islamischen Zugehörigkeit wurde zurückgewiesen.", + "spans": [ + { + "start": 21, + "end": 32, + "label": "RELIGION", + "text": "islamischen" + } + ], + "expected": "Die Behauptung einer [RELIGION] Zugehörigkeit wurde zurückgewiesen." + }, + { + "id": "holdout03d-080", + "language": "en", + "scenario": "negation_quotation", + "ocr_noise": false, + "text": "The statement \"Buddhist\" was disputed in the correction sheet.", + "spans": [ + { + "start": 15, + "end": 23, + "label": "RELIGION", + "text": "Buddhist" + } + ], + "expected": "The statement \"[RELIGION]\" was disputed in the correction sheet." + }, + { + "id": "holdout03d-081", + "language": "de", + "scenario": "ambiguity", + "ocr_noise": false, + "text": "Ein orthodoxes Sortierverfahren ist hier eine mathematische Metapher.", + "spans": [], + "expected": "Ein orthodoxes Sortierverfahren ist hier eine mathematische Metapher." + }, + { + "id": "holdout03d-082", + "language": "en", + "scenario": "ambiguity", + "ocr_noise": false, + "text": "Her catholic tastes include jazz and astronomy.", + "spans": [], + "expected": "Her catholic tastes include jazz and astronomy." + }, + { + "id": "holdout03d-083", + "language": "de", + "scenario": "ambiguity", + "ocr_noise": false, + "text": "Die evangelikale Werbemetapher beschreibt keine Religionszugehörigkeit.", + "spans": [], + "expected": "Die evangelikale Werbemetapher beschreibt keine Religionszugehörigkeit." + }, + { + "id": "holdout03d-084", + "language": "en", + "scenario": "ambiguity", + "ocr_noise": false, + "text": "An evangelical sales pitch is the example in this marketing exercise.", + "spans": [], + "expected": "An evangelical sales pitch is the example in this marketing exercise." + }, + { + "id": "holdout03d-085", + "language": "de", + "scenario": "ambiguity", + "ocr_noise": false, + "text": "Der Patient wurde orthodox getauft, wie der geschützte Eintrag berichtet.", + "spans": [ + { + "start": 18, + "end": 26, + "label": "RELIGION", + "text": "orthodox" + } + ], + "expected": "Der Patient wurde [RELIGION] getauft, wie der geschützte Eintrag berichtet." + }, + { + "id": "holdout03d-086", + "language": "en", + "scenario": "ambiguity", + "ocr_noise": false, + "text": "The baptismal record identifies the denomination as Orthodox.", + "spans": [ + { + "start": 52, + "end": 60, + "label": "RELIGION", + "text": "Orthodox" + } + ], + "expected": "The baptismal record identifies the denomination as [RELIGION]." + }, + { + "id": "holdout03d-087", + "language": "de", + "scenario": "ambiguity", + "ocr_noise": false, + "text": "Konfession: evangelisch; Algorithmus: konventionell.", + "spans": [ + { + "start": 12, + "end": 23, + "label": "RELIGION", + "text": "evangelisch" + } + ], + "expected": "Konfession: [RELIGION]; Algorithmus: konventionell." + }, + { + "id": "holdout03d-088", + "language": "en", + "scenario": "ambiguity", + "ocr_noise": false, + "text": "The faith survey selected Evangelical under denomination.", + "spans": [ + { + "start": 26, + "end": 37, + "label": "RELIGION", + "text": "Evangelical" + } + ], + "expected": "The faith survey selected [RELIGION] under denomination." + }, + { + "id": "holdout03d-089", + "language": "de", + "scenario": "ambiguity", + "ocr_noise": false, + "text": "Eine säkulare Rechtsordnung nennt keine persönlichen Bekenntnisse.", + "spans": [], + "expected": "Eine säkulare Rechtsordnung nennt keine persönlichen Bekenntnisse." + }, + { + "id": "holdout03d-090", + "language": "en", + "scenario": "ambiguity", + "ocr_noise": false, + "text": "The meeting has an unaffiliated chair and no membership declarations.", + "spans": [], + "expected": "The meeting has an unaffiliated chair and no membership declarations." + }, + { + "id": "holdout03d-091", + "language": "de", + "scenario": "ambiguity", + "ocr_noise": false, + "text": "Als Religion ist christlich vermerkt, nicht als Personenname.", + "spans": [ + { + "start": 17, + "end": 27, + "label": "RELIGION", + "text": "christlich" + } + ], + "expected": "Als Religion ist [RELIGION] vermerkt, nicht als Personenname." + }, + { + "id": "holdout03d-092", + "language": "en", + "scenario": "ambiguity", + "ocr_noise": false, + "text": "The religion field is Christian, distinct from the given-name column.", + "spans": [ + { + "start": 22, + "end": 31, + "label": "RELIGION", + "text": "Christian" + } + ], + "expected": "The religion field is [RELIGION], distinct from the given-name column." + }, + { + "id": "holdout03d-093", + "language": "de", + "scenario": "ambiguity", + "ocr_noise": false, + "text": "Die goldene Regel im Handwerk ist kein personenbezogenes Religionsmerkmal.", + "spans": [], + "expected": "Die goldene Regel im Handwerk ist kein personenbezogenes Religionsmerkmal." + }, + { + "id": "holdout03d-094", + "language": "en", + "scenario": "ambiguity", + "ocr_noise": false, + "text": "The orthodox spelling of a variable is a programming concern.", + "spans": [], + "expected": "The orthodox spelling of a variable is a programming concern." + }, + { + "id": "holdout03d-095", + "language": "de", + "scenario": "ambiguity", + "ocr_noise": false, + "text": "Im Glaubensregister steht jüdisch als vertrauliche Angabe.", + "spans": [ + { + "start": 26, + "end": 33, + "label": "RELIGION", + "text": "jüdisch" + } + ], + "expected": "Im Glaubensregister steht [RELIGION] als vertrauliche Angabe." + }, + { + "id": "holdout03d-096", + "language": "en", + "scenario": "ambiguity", + "ocr_noise": false, + "text": "The theological syllabus labels a tradition Reformed.", + "spans": [ + { + "start": 44, + "end": 52, + "label": "RELIGION", + "text": "Reformed" + } + ], + "expected": "The theological syllabus labels a tradition [RELIGION]." + }, + { + "id": "holdout03d-097", + "language": "de", + "scenario": "ocr_noise", + "ocr_noise": true, + "text": "Religion laut Scan: Buddh1smus.", + "spans": [ + { + "start": 20, + "end": 30, + "label": "RELIGION", + "text": "Buddh1smus" + } + ], + "expected": "Religion laut Scan: [RELIGION]." + }, + { + "id": "holdout03d-098", + "language": "en", + "scenario": "ocr_noise", + "ocr_noise": true, + "text": "The scanned belief field reads Christlanity.", + "spans": [ + { + "start": 31, + "end": 43, + "label": "RELIGION", + "text": "Christlanity" + } + ], + "expected": "The scanned belief field reads [RELIGION]." + }, + { + "id": "holdout03d-099", + "language": "de", + "scenario": "ocr_noise", + "ocr_noise": true, + "text": "Konfession im OCR-Text: konfessions-\nlos.", + "spans": [ + { + "start": 24, + "end": 40, + "label": "RELIGION", + "text": "konfessions-\nlos" + } + ], + "expected": "Konfession im OCR-Text: [RELIGION]." + }, + { + "id": "holdout03d-100", + "language": "en", + "scenario": "ocr_noise", + "ocr_noise": true, + "text": "Faith from the scan: non-\nreligious.", + "spans": [ + { + "start": 21, + "end": 35, + "label": "RELIGION", + "text": "non-\nreligious" + } + ], + "expected": "Faith from the scan: [RELIGION]." + }, + { + "id": "holdout03d-101", + "language": "de", + "scenario": "ocr_noise", + "ocr_noise": true, + "text": "Die Lesart katho1isch ist im Faksimile markiert.", + "spans": [ + { + "start": 11, + "end": 21, + "label": "RELIGION", + "text": "katho1isch" + } + ], + "expected": "Die Lesart [RELIGION] ist im Faksimile markiert." + }, + { + "id": "holdout03d-102", + "language": "en", + "scenario": "ocr_noise", + "ocr_noise": true, + "text": "Belief value after OCR: Hinduisrn.", + "spans": [ + { + "start": 24, + "end": 33, + "label": "RELIGION", + "text": "Hinduisrn" + } + ], + "expected": "Belief value after OCR: [RELIGION]." + }, + { + "id": "holdout03d-103", + "language": "de", + "scenario": "ocr_noise", + "ocr_noise": true, + "text": "Im Formular wurde agnos-\ntisch getrennt gesetzt.", + "spans": [ + { + "start": 18, + "end": 30, + "label": "RELIGION", + "text": "agnos-\ntisch" + } + ], + "expected": "Im Formular wurde [RELIGION] getrennt gesetzt." + }, + { + "id": "holdout03d-104", + "language": "en", + "scenario": "ocr_noise", + "ocr_noise": true, + "text": "The scanned heading says Juda-\nism.", + "spans": [ + { + "start": 25, + "end": 34, + "label": "RELIGION", + "text": "Juda-\nism" + } + ], + "expected": "The scanned heading says [RELIGION]." + }, + { + "id": "holdout03d-105", + "language": "de", + "scenario": "ocr_noise", + "ocr_noise": true, + "text": "Eine Zeile enthält lslam mit verwechseltem Anfangsbuchstaben.", + "spans": [ + { + "start": 19, + "end": 24, + "label": "RELIGION", + "text": "lslam" + } + ], + "expected": "Eine Zeile enthält [RELIGION] mit verwechseltem Anfangsbuchstaben." + }, + { + "id": "holdout03d-106", + "language": "en", + "scenario": "ocr_noise", + "ocr_noise": true, + "text": "The OCR extracted S1khism in the private religion column.", + "spans": [ + { + "start": 18, + "end": 25, + "label": "RELIGION", + "text": "S1khism" + } + ], + "expected": "The OCR extracted [RELIGION] in the private religion column." + }, + { + "id": "holdout03d-107", + "language": "de", + "scenario": "ocr_noise", + "ocr_noise": true, + "text": "Die handschriftliche Angabe lautet buddhis-\ntisch.", + "spans": [ + { + "start": 35, + "end": 49, + "label": "RELIGION", + "text": "buddhis-\ntisch" + } + ], + "expected": "Die handschriftliche Angabe lautet [RELIGION]." + }, + { + "id": "holdout03d-108", + "language": "en", + "scenario": "ocr_noise", + "ocr_noise": true, + "text": "The line-broken subject is Zoroas-\ntrianism.", + "spans": [ + { + "start": 27, + "end": 43, + "label": "RELIGION", + "text": "Zoroas-\ntrianism" + } + ], + "expected": "The line-broken subject is [RELIGION]." + }, + { + "id": "holdout03d-109", + "language": "de", + "scenario": "ocr_noise", + "ocr_noise": true, + "text": "Aus dem Scan: Fenja Torwald, Bekenntnis Judenturn.", + "spans": [ + { + "start": 14, + "end": 27, + "label": "PERSON", + "text": "Fenja Torwald" + }, + { + "start": 40, + "end": 49, + "label": "RELIGION", + "text": "Judenturn" + } + ], + "expected": "Aus dem Scan: [PERSON], Bekenntnis [RELIGION]." + }, + { + "id": "holdout03d-110", + "language": "en", + "scenario": "ocr_noise", + "ocr_noise": true, + "text": "The noisy note names Orin Wexford and Cath0licism.", + "spans": [ + { + "start": 21, + "end": 33, + "label": "PERSON", + "text": "Orin Wexford" + }, + { + "start": 38, + "end": 49, + "label": "RELIGION", + "text": "Cath0licism" + } + ], + "expected": "The noisy note names [PERSON] and [RELIGION]." + }, + { + "id": "holdout03d-111", + "language": "de", + "scenario": "ocr_noise", + "ocr_noise": true, + "text": "Der OCR-Wert konfesslonsfrei bleibt vertraulich.", + "spans": [ + { + "start": 13, + "end": 28, + "label": "RELIGION", + "text": "konfesslonsfrei" + } + ], + "expected": "Der OCR-Wert [RELIGION] bleibt vertraulich." + }, + { + "id": "holdout03d-112", + "language": "en", + "scenario": "ocr_noise", + "ocr_noise": true, + "text": "Scanned status text: agn0stic.", + "spans": [ + { + "start": 21, + "end": 29, + "label": "RELIGION", + "text": "agn0stic" + } + ], + "expected": "Scanned status text: [RELIGION]." + }, + { + "id": "holdout03d-113", + "language": "de", + "scenario": "multiple_entities", + "ocr_noise": false, + "text": "Silja Dornbach aus Rostock meldet Judentum beim Morgenkai Institut.", + "spans": [ + { + "start": 0, + "end": 14, + "label": "PERSON", + "text": "Silja Dornbach" + }, + { + "start": 19, + "end": 26, + "label": "CITY", + "text": "Rostock" + }, + { + "start": 34, + "end": 42, + "label": "RELIGION", + "text": "Judentum" + }, + { + "start": 48, + "end": 66, + "label": "ORGANIZATION", + "text": "Morgenkai Institut" + } + ], + "expected": "[PERSON] aus [CITY] meldet [RELIGION] beim [ORGANIZATION]." + }, + { + "id": "holdout03d-114", + "language": "en", + "scenario": "multiple_entities", + "ocr_noise": false, + "text": "Nolan Briarwood in York lists Jainism on the Cobalt Quay Trust form.", + "spans": [ + { + "start": 0, + "end": 15, + "label": "PERSON", + "text": "Nolan Briarwood" + }, + { + "start": 19, + "end": 23, + "label": "CITY", + "text": "York" + }, + { + "start": 30, + "end": 37, + "label": "RELIGION", + "text": "Jainism" + }, + { + "start": 45, + "end": 62, + "label": "ORGANIZATION", + "text": "Cobalt Quay Trust" + } + ], + "expected": "[PERSON] in [CITY] lists [RELIGION] on the [ORGANIZATION] form." + }, + { + "id": "holdout03d-115", + "language": "de", + "scenario": "multiple_entities", + "ocr_noise": false, + "text": "Bernsteinpfad Stiftung vermerkt Henrik Wiesenau als konfessionslos.", + "spans": [ + { + "start": 0, + "end": 22, + "label": "ORGANIZATION", + "text": "Bernsteinpfad Stiftung" + }, + { + "start": 32, + "end": 47, + "label": "PERSON", + "text": "Henrik Wiesenau" + }, + { + "start": 52, + "end": 66, + "label": "RELIGION", + "text": "konfessionslos" + } + ], + "expected": "[ORGANIZATION] vermerkt [PERSON] als [RELIGION]." + }, + { + "id": "holdout03d-116", + "language": "en", + "scenario": "multiple_entities", + "ocr_noise": false, + "text": "The Silver Estuary Council stores Ada Merefield's atheism response.", + "spans": [ + { + "start": 4, + "end": 26, + "label": "ORGANIZATION", + "text": "Silver Estuary Council" + }, + { + "start": 34, + "end": 47, + "label": "PERSON", + "text": "Ada Merefield" + }, + { + "start": 50, + "end": 57, + "label": "RELIGION", + "text": "atheism" + } + ], + "expected": "The [ORGANIZATION] stores [PERSON]'s [RELIGION] response." + }, + { + "id": "holdout03d-117", + "language": "de", + "scenario": "multiple_entities", + "ocr_noise": false, + "text": "Ida Talbrück nennt Islam; Kontakt: ida.tal@example.invalid.", + "spans": [ + { + "start": 0, + "end": 12, + "label": "PERSON", + "text": "Ida Talbrück" + }, + { + "start": 19, + "end": 24, + "label": "RELIGION", + "text": "Islam" + }, + { + "start": 35, + "end": 58, + "label": "EMAIL", + "text": "ida.tal@example.invalid" + } + ], + "expected": "[PERSON] nennt [RELIGION]; Kontakt: [EMAIL]." + }, + { + "id": "holdout03d-118", + "language": "en", + "scenario": "multiple_entities", + "ocr_noise": false, + "text": "Lyle Ashcombe wrote Buddhism; contact lyle.a@example.invalid.", + "spans": [ + { + "start": 0, + "end": 13, + "label": "PERSON", + "text": "Lyle Ashcombe" + }, + { + "start": 20, + "end": 28, + "label": "RELIGION", + "text": "Buddhism" + }, + { + "start": 38, + "end": 60, + "label": "EMAIL", + "text": "lyle.a@example.invalid" + } + ], + "expected": "[PERSON] wrote [RELIGION]; contact [EMAIL]." + }, + { + "id": "holdout03d-119", + "language": "de", + "scenario": "multiple_entities", + "ocr_noise": false, + "text": "Rückfrage an Arvid Sandborn: +49 381 482701; Status religionslos.", + "spans": [ + { + "start": 13, + "end": 27, + "label": "PERSON", + "text": "Arvid Sandborn" + }, + { + "start": 29, + "end": 43, + "label": "PHONE", + "text": "+49 381 482701" + }, + { + "start": 52, + "end": 64, + "label": "RELIGION", + "text": "religionslos" + } + ], + "expected": "Rückfrage an [PERSON]: [PHONE]; Status [RELIGION]." + }, + { + "id": "holdout03d-120", + "language": "en", + "scenario": "multiple_entities", + "ocr_noise": false, + "text": "Call Mira Fenwick at +44 1904 582701; belief Hinduism.", + "spans": [ + { + "start": 5, + "end": 17, + "label": "PERSON", + "text": "Mira Fenwick" + }, + { + "start": 21, + "end": 36, + "label": "PHONE", + "text": "+44 1904 582701" + }, + { + "start": 45, + "end": 53, + "label": "RELIGION", + "text": "Hinduism" + } + ], + "expected": "Call [PERSON] at [PHONE]; belief [RELIGION]." + }, + { + "id": "holdout03d-121", + "language": "de", + "scenario": "multiple_entities", + "ocr_noise": false, + "text": "Erfurt: Svea Morgenau korrigiert katholisch zu agnostisch.", + "spans": [ + { + "start": 0, + "end": 6, + "label": "CITY", + "text": "Erfurt" + }, + { + "start": 8, + "end": 21, + "label": "PERSON", + "text": "Svea Morgenau" + }, + { + "start": 33, + "end": 43, + "label": "RELIGION", + "text": "katholisch" + }, + { + "start": 47, + "end": 57, + "label": "RELIGION", + "text": "agnostisch" + } + ], + "expected": "[CITY]: [PERSON] korrigiert [RELIGION] zu [RELIGION]." + }, + { + "id": "holdout03d-122", + "language": "en", + "scenario": "multiple_entities", + "ocr_noise": false, + "text": "Bristol: Owen Thornley replaced Christianity with no religion.", + "spans": [ + { + "start": 0, + "end": 7, + "label": "CITY", + "text": "Bristol" + }, + { + "start": 9, + "end": 22, + "label": "PERSON", + "text": "Owen Thornley" + }, + { + "start": 32, + "end": 44, + "label": "RELIGION", + "text": "Christianity" + }, + { + "start": 50, + "end": 61, + "label": "RELIGION", + "text": "no religion" + } + ], + "expected": "[CITY]: [PERSON] replaced [RELIGION] with [RELIGION]." + }, + { + "id": "holdout03d-123", + "language": "de", + "scenario": "multiple_entities", + "ocr_noise": false, + "text": "Jona Falkenau und Jona Falkenau erscheinen mit Sikhismus zweimal.", + "spans": [ + { + "start": 0, + "end": 13, + "label": "PERSON", + "text": "Jona Falkenau" + }, + { + "start": 18, + "end": 31, + "label": "PERSON", + "text": "Jona Falkenau" + }, + { + "start": 47, + "end": 56, + "label": "RELIGION", + "text": "Sikhismus" + } + ], + "expected": "[PERSON] und [PERSON] erscheinen mit [RELIGION] zweimal." + }, + { + "id": "holdout03d-124", + "language": "en", + "scenario": "multiple_entities", + "ocr_noise": false, + "text": "Esme Rowanby repeated Taoism, then signed Esme Rowanby again.", + "spans": [ + { + "start": 0, + "end": 12, + "label": "PERSON", + "text": "Esme Rowanby" + }, + { + "start": 22, + "end": 28, + "label": "RELIGION", + "text": "Taoism" + }, + { + "start": 42, + "end": 54, + "label": "PERSON", + "text": "Esme Rowanby" + } + ], + "expected": "[PERSON] repeated [RELIGION], then signed [PERSON] again." + }, + { + "id": "holdout03d-125", + "language": "de", + "scenario": "multiple_entities", + "ocr_noise": false, + "text": "Interner Datensatz des Hafenbogen Archivs: Kiel, evangelisch.", + "spans": [ + { + "start": 23, + "end": 41, + "label": "ORGANIZATION", + "text": "Hafenbogen Archivs" + }, + { + "start": 43, + "end": 47, + "label": "CITY", + "text": "Kiel" + }, + { + "start": 49, + "end": 60, + "label": "RELIGION", + "text": "evangelisch" + } + ], + "expected": "Interner Datensatz des [ORGANIZATION]: [CITY], [RELIGION]." + }, + { + "id": "holdout03d-126", + "language": "en", + "scenario": "multiple_entities", + "ocr_noise": false, + "text": "A Maple Tide Institute record from Durham mentions Judaism.", + "spans": [ + { + "start": 2, + "end": 22, + "label": "ORGANIZATION", + "text": "Maple Tide Institute" + }, + { + "start": 35, + "end": 41, + "label": "CITY", + "text": "Durham" + }, + { + "start": 51, + "end": 58, + "label": "RELIGION", + "text": "Judaism" + } + ], + "expected": "A [ORGANIZATION] record from [CITY] mentions [RELIGION]." + }, + { + "id": "holdout03d-127", + "language": "de", + "scenario": "multiple_entities", + "ocr_noise": false, + "text": "Liv Hohenrain nennt Bonn und Bahai; liv.h@example.invalid bleibt intern.", + "spans": [ + { + "start": 0, + "end": 13, + "label": "PERSON", + "text": "Liv Hohenrain" + }, + { + "start": 20, + "end": 24, + "label": "CITY", + "text": "Bonn" + }, + { + "start": 29, + "end": 34, + "label": "RELIGION", + "text": "Bahai" + }, + { + "start": 36, + "end": 57, + "label": "EMAIL", + "text": "liv.h@example.invalid" + } + ], + "expected": "[PERSON] nennt [CITY] und [RELIGION]; [EMAIL] bleibt intern." + }, + { + "id": "holdout03d-128", + "language": "en", + "scenario": "multiple_entities", + "ocr_noise": false, + "text": "Reed Alderwick, Bath, agnosticism; reed.a@example.invalid is private.", + "spans": [ + { + "start": 0, + "end": 14, + "label": "PERSON", + "text": "Reed Alderwick" + }, + { + "start": 16, + "end": 20, + "label": "CITY", + "text": "Bath" + }, + { + "start": 22, + "end": 33, + "label": "RELIGION", + "text": "agnosticism" + }, + { + "start": 35, + "end": 57, + "label": "EMAIL", + "text": "reed.a@example.invalid" + } + ], + "expected": "[PERSON], [CITY], [RELIGION]; [EMAIL] is private." + }, + { + "id": "holdout03d-129", + "language": "de", + "scenario": "formats_repetition", + "ocr_noise": false, + "text": "| Bekenntnis | Islam | Freigabe | nein |", + "spans": [ + { + "start": 15, + "end": 20, + "label": "RELIGION", + "text": "Islam" + } + ], + "expected": "| Bekenntnis | [RELIGION] | Freigabe | nein |" + }, + { + "id": "holdout03d-130", + "language": "en", + "scenario": "formats_repetition", + "ocr_noise": false, + "text": "{\"belief\":\"Buddhism\",\"public\":false}", + "spans": [ + { + "start": 11, + "end": 19, + "label": "RELIGION", + "text": "Buddhism" + } + ], + "expected": "{\"belief\":\"[RELIGION]\",\"public\":false}" + }, + { + "id": "holdout03d-131", + "language": "de", + "scenario": "formats_repetition", + "ocr_noise": false, + "text": "Notiz A: konfessionsfrei. Notiz B: konfessionsfrei.", + "spans": [ + { + "start": 9, + "end": 24, + "label": "RELIGION", + "text": "konfessionsfrei" + }, + { + "start": 35, + "end": 50, + "label": "RELIGION", + "text": "konfessionsfrei" + } + ], + "expected": "Notiz A: [RELIGION]. Notiz B: [RELIGION]." + }, + { + "id": "holdout03d-132", + "language": "en", + "scenario": "formats_repetition", + "ocr_noise": false, + "text": "Row one: atheist; row two: atheist.", + "spans": [ + { + "start": 9, + "end": 16, + "label": "RELIGION", + "text": "atheist" + }, + { + "start": 27, + "end": 34, + "label": "RELIGION", + "text": "atheist" + } + ], + "expected": "Row one: [RELIGION]; row two: [RELIGION]." + }, + { + "id": "holdout03d-133", + "language": "de", + "scenario": "formats_repetition", + "ocr_noise": false, + "text": "RELIGION=Judentum; QUELLE=Selbstauskunft", + "spans": [ + { + "start": 9, + "end": 17, + "label": "RELIGION", + "text": "Judentum" + } + ], + "expected": "RELIGION=[RELIGION]; QUELLE=Selbstauskunft" + }, + { + "id": "holdout03d-134", + "language": "en", + "scenario": "formats_repetition", + "ocr_noise": false, + "text": "BELIEF=Shinto; SOURCE=self-report", + "spans": [ + { + "start": 7, + "end": 13, + "label": "RELIGION", + "text": "Shinto" + } + ], + "expected": "BELIEF=[RELIGION]; SOURCE=self-report" + }, + { + "id": "holdout03d-135", + "language": "de", + "scenario": "formats_repetition", + "ocr_noise": false, + "text": "• Weltanschauung: Agnostizismus\n• Weitergabe: gesperrt", + "spans": [ + { + "start": 18, + "end": 31, + "label": "RELIGION", + "text": "Agnostizismus" + } + ], + "expected": "• Weltanschauung: [RELIGION]\n• Weitergabe: gesperrt" + }, + { + "id": "holdout03d-136", + "language": "en", + "scenario": "formats_repetition", + "ocr_noise": false, + "text": "1. Faith: Sikhism\n2. Access: restricted", + "spans": [ + { + "start": 10, + "end": 17, + "label": "RELIGION", + "text": "Sikhism" + } + ], + "expected": "1. Faith: [RELIGION]\n2. Access: restricted" + }, + { + "id": "holdout03d-137", + "language": "de", + "scenario": "formats_repetition", + "ocr_noise": false, + "text": "Seitenrand Katholizismus / Fußnote Katholizismus.", + "spans": [ + { + "start": 11, + "end": 24, + "label": "RELIGION", + "text": "Katholizismus" + }, + { + "start": 35, + "end": 48, + "label": "RELIGION", + "text": "Katholizismus" + } + ], + "expected": "Seitenrand [RELIGION] / Fußnote [RELIGION]." + }, + { + "id": "holdout03d-138", + "language": "en", + "scenario": "formats_repetition", + "ocr_noise": false, + "text": "Margin Hinduism; footnote Hinduism.", + "spans": [ + { + "start": 7, + "end": 15, + "label": "RELIGION", + "text": "Hinduism" + }, + { + "start": 26, + "end": 34, + "label": "RELIGION", + "text": "Hinduism" + } + ], + "expected": "Margin [RELIGION]; footnote [RELIGION]." + }, + { + "id": "holdout03d-139", + "language": "de", + "scenario": "formats_repetition", + "ocr_noise": false, + "text": "Eingabefeld «ohne Konfession» — nur intern.", + "spans": [ + { + "start": 13, + "end": 28, + "label": "RELIGION", + "text": "ohne Konfession" + } + ], + "expected": "Eingabefeld «[RELIGION]» — nur intern." + }, + { + "id": "holdout03d-140", + "language": "en", + "scenario": "formats_repetition", + "ocr_noise": false, + "text": "Checkbox ☑ non-religious; do not publish.", + "spans": [ + { + "start": 11, + "end": 24, + "label": "RELIGION", + "text": "non-religious" + } + ], + "expected": "Checkbox ☑ [RELIGION]; do not publish." + }, + { + "id": "holdout03d-141", + "language": "de", + "scenario": "formats_repetition", + "ocr_noise": false, + "text": "Christentum → Atheismus ist als Änderung notiert.", + "spans": [ + { + "start": 0, + "end": 11, + "label": "RELIGION", + "text": "Christentum" + }, + { + "start": 14, + "end": 23, + "label": "RELIGION", + "text": "Atheismus" + } + ], + "expected": "[RELIGION] → [RELIGION] ist als Änderung notiert." + }, + { + "id": "holdout03d-142", + "language": "en", + "scenario": "formats_repetition", + "ocr_noise": false, + "text": "Christianity → agnosticism is the recorded correction.", + "spans": [ + { + "start": 0, + "end": 12, + "label": "RELIGION", + "text": "Christianity" + }, + { + "start": 15, + "end": 26, + "label": "RELIGION", + "text": "agnosticism" + } + ], + "expected": "[RELIGION] → [RELIGION] is the recorded correction." + }, + { + "id": "holdout03d-143", + "language": "de", + "scenario": "formats_repetition", + "ocr_noise": false, + "text": "Bekenntnis\tBuddhismus\tgeprüft=offen", + "spans": [ + { + "start": 11, + "end": 21, + "label": "RELIGION", + "text": "Buddhismus" + } + ], + "expected": "Bekenntnis\t[RELIGION]\tgeprüft=offen" + }, + { + "id": "holdout03d-144", + "language": "en", + "scenario": "formats_repetition", + "ocr_noise": false, + "text": "religion\tJainism\treview=pending", + "spans": [ + { + "start": 9, + "end": 16, + "label": "RELIGION", + "text": "Jainism" + } + ], + "expected": "religion\t[RELIGION]\treview=pending" + }, + { + "id": "holdout03d-145", + "language": "de", + "scenario": "negative_controls", + "ocr_noise": false, + "text": "Die Lieferung enthält zwölf Schrauben und einen Lieferschein.", + "spans": [], + "expected": "Die Lieferung enthält zwölf Schrauben und einen Lieferschein." + }, + { + "id": "holdout03d-146", + "language": "en", + "scenario": "negative_controls", + "ocr_noise": false, + "text": "A blue folder sits beside the empty filing tray.", + "spans": [], + "expected": "A blue folder sits beside the empty filing tray." + }, + { + "id": "holdout03d-147", + "language": "de", + "scenario": "negative_controls", + "ocr_noise": false, + "text": "Das Formular enthält keine ausgefüllten persönlichen Felder.", + "spans": [], + "expected": "Das Formular enthält keine ausgefüllten persönlichen Felder." + }, + { + "id": "holdout03d-148", + "language": "en", + "scenario": "negative_controls", + "ocr_noise": false, + "text": "The spelling correction changes a comma to a semicolon.", + "spans": [], + "expected": "The spelling correction changes a comma to a semicolon." + }, + { + "id": "holdout03d-149", + "language": "de", + "scenario": "negative_controls", + "ocr_noise": false, + "text": "Der Kalender zeigt nur Wochentage ohne Termine.", + "spans": [], + "expected": "Der Kalender zeigt nur Wochentage ohne Termine." + }, + { + "id": "holdout03d-150", + "language": "en", + "scenario": "negative_controls", + "ocr_noise": false, + "text": "The maintenance note records a loose hinge.", + "spans": [], + "expected": "The maintenance note records a loose hinge." + }, + { + "id": "holdout03d-151", + "language": "de", + "scenario": "negative_controls", + "ocr_noise": false, + "text": "Eine Überschrift erklärt die Bedienung der Taschenlampe.", + "spans": [], + "expected": "Eine Überschrift erklärt die Bedienung der Taschenlampe." + }, + { + "id": "holdout03d-152", + "language": "en", + "scenario": "negative_controls", + "ocr_noise": false, + "text": "A measurement table lists height and width without identifiers.", + "spans": [], + "expected": "A measurement table lists height and width without identifiers." + }, + { + "id": "holdout03d-153", + "language": "de", + "scenario": "negative_controls", + "ocr_noise": false, + "text": "Das Prüfprotokoll enthält ausschließlich Materialnummern aus Buchstaben.", + "spans": [], + "expected": "Das Prüfprotokoll enthält ausschließlich Materialnummern aus Buchstaben." + }, + { + "id": "holdout03d-154", + "language": "en", + "scenario": "negative_controls", + "ocr_noise": false, + "text": "The anonymous inventory includes pencils and unmarked boxes.", + "spans": [], + "expected": "The anonymous inventory includes pencils and unmarked boxes." + }, + { + "id": "holdout03d-155", + "language": "de", + "scenario": "negative_controls", + "ocr_noise": false, + "text": "Die Zutatenliste nennt Mehl, Wasser und Salz.", + "spans": [], + "expected": "Die Zutatenliste nennt Mehl, Wasser und Salz." + }, + { + "id": "holdout03d-156", + "language": "en", + "scenario": "negative_controls", + "ocr_noise": false, + "text": "The diagram shows arrows connecting unlabeled circles.", + "spans": [], + "expected": "The diagram shows arrows connecting unlabeled circles." + }, + { + "id": "holdout03d-157", + "language": "de", + "scenario": "negative_controls", + "ocr_noise": false, + "text": "Der Absatz behandelt die Grammatik unpersönlicher Sätze.", + "spans": [], + "expected": "Der Absatz behandelt die Grammatik unpersönlicher Sätze." + }, + { + "id": "holdout03d-158", + "language": "en", + "scenario": "negative_controls", + "ocr_noise": false, + "text": "A draft contains only instructions for replacing printer paper.", + "spans": [], + "expected": "A draft contains only instructions for replacing printer paper." + }, + { + "id": "holdout03d-159", + "language": "de", + "scenario": "negative_controls", + "ocr_noise": false, + "text": "Die leere Tabelle hat Spalten für Farbe und Gewicht.", + "spans": [], + "expected": "Die leere Tabelle hat Spalten für Farbe und Gewicht." + }, + { + "id": "holdout03d-160", + "language": "en", + "scenario": "negative_controls", + "ocr_noise": false, + "text": "No religious status, personal name, address or contact is supplied.", + "spans": [], + "expected": "No religious status, personal name, address or contact is supplied." + } +] diff --git a/ocr/anonymization/fixtures/holdout03d/freeze.json b/ocr/anonymization/fixtures/holdout03d/freeze.json new file mode 100644 index 0000000000000000000000000000000000000000..33f5864c4d4fe5bd204e2845224bfc8c5c991282 --- /dev/null +++ b/ocr/anonymization/fixtures/holdout03d/freeze.json @@ -0,0 +1,129 @@ +{ + "schema_version": "frozen-holdout03d/v1", + "cases": 160, + "sha256": "7180e176f6d8bd259da56ad729ede6fd2c798aca15a786ba0ff0adcd862f21a1", + "source_head": "4c5cbbc30801f5911dcf52e1997a70404026d425", + "model_revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", + "prompts": { + "PERSON": "person", + "CITY": "location", + "RELIGION": "religion", + "EMAIL": "email address", + "PHONE": "phone number", + "ORGANIZATION": "organization" + }, + "threshold": 0.5, + "religion_terms": [ + "religionslos", + "konfessionslos", + "konfessionsfrei", + "ohne Konfession", + "nicht religiös", + "no religious affiliation", + "no religion", + "nonreligious", + "non-religious", + "atheist", + "atheists", + "atheism", + "atheistisch", + "Atheismus", + "agnostic", + "agnostics", + "agnosticism", + "agnostisch", + "Agnostizismus", + "Christentum", + "Christianity", + "Christen", + "Christian", + "Christians", + "christlich", + "christliche", + "christlichen", + "Katholizismus", + "Catholicism", + "katholisch", + "katholische", + "katholischen", + "Catholic", + "Catholics", + "Protestantismus", + "Protestantism", + "Protestant", + "Protestants", + "evangelisch", + "evangelische", + "evangelischen", + "Judentum", + "Judaism", + "jüdisch", + "jüdische", + "jüdischen", + "Jewish", + "Jude", + "Jüdin", + "Juden", + "Jews", + "Islam", + "Muslim", + "Muslims", + "muslimisch", + "muslimische", + "muslimischen", + "islamisch", + "islamische", + "islamischen", + "Buddhismus", + "Buddhism", + "Buddhist", + "Buddhists", + "buddhistisch", + "buddhistische", + "buddhistischen", + "Hinduismus", + "Hinduism", + "Hindu", + "Hindus", + "hinduistisch", + "hinduistische", + "hinduistischen", + "Sikhismus", + "Sikhism", + "Sikh", + "Sikhs", + "sikhistisch", + "Jainismus", + "Jainism", + "Jain", + "Jains", + "jainistisch", + "Bahai", + "Baháʼí Faith", + "Bahá'í", + "Baha'i", + "Shinto", + "Shintoismus", + "Shintoism", + "Taoismus", + "Taoism", + "Daoismus", + "Daoism", + "Zoroastrismus", + "Zoroastrianism", + "Orthodox Christianity", + "Orthodoxie" + ], + "implementation_sha256": { + "ocr/anonymization/recognizer.py": "f695a30f0ff8540ba345d5557f0bf2bfc225635b228010da6ee6f5ccbc71ce45", + "ocr/anonymization/normalization.py": "3b2598200933a86cedf2dd20affb911c4ad1ac4f7fdc00182c4044b7a2873f10", + "ocr/anonymization/detectors.py": "c543532c579e8880e4915b3e669063f11e8e39034886f889e9e4e20942bf6a10", + "ocr/anonymization/terminology.py": "370b61beb1b5339549f2c8143a31b546f4a32dbbabd26c061f009341de16babd", + "ocr/anonymization/engine.py": "d784774d57e35b25fdac5bef095df4859db9d606e83759625526cbf90cfc8763", + "ocr/anonymization/replacement.py": "1f048dc237a3355c400a6f085b13848e1c0c49fe2f7f9d5ab8203114ef3e4cd8", + "ocr/anonymization/evaluate.py": "b9ed1453d6b7c704be3eef18b2a32b9cf8e857075355c42025e5d3bdc863d6b0", + "ocr/anonymization/qualify03d.py": "ead5767304b8e10e46b855ab0235c2b6f97ad4fe82271d366bfd730de0e1ebb0" + }, + "evaluation_run_before_freeze": false, + "design": "160 individually authored synthetic cases, 80 DE / 80 EN, 10 scenario strata. No repeated 03c entity-set templates. Selected configuration based only on 40+10+120 development suites. First evaluation and corpus retained even on failure." +} diff --git a/ocr/anonymization/fixtures/qualification.json b/ocr/anonymization/fixtures/qualification.json new file mode 100644 index 0000000000000000000000000000000000000000..c7bd1497dffebb3e917f805b8282cff87e67b6a8 --- /dev/null +++ b/ocr/anonymization/fixtures/qualification.json @@ -0,0 +1,647 @@ +[ + { + "text": "Lina Wolkenstein wohnt in Berlin.", + "expected": "[PERSON] wohnt in [CITY].", + "spans": [ + { + "start": 0, + "end": 16, + "label": "PERSON", + "text": "Lina Wolkenstein" + }, + { + "start": 26, + "end": 32, + "label": "CITY", + "text": "Berlin" + } + ], + "id": "baseline-00", + "language": "de", + "ocr_noise": false + }, + { + "text": "Hans Müller kauft ein.", + "expected": "[PERSON] kauft ein.", + "spans": [ + { + "start": 0, + "end": 11, + "label": "PERSON", + "text": "Hans Müller" + } + ], + "id": "baseline-01", + "language": "de", + "ocr_noise": false + }, + { + "text": "Das Auto von Dr. Klaus Schmidt.", + "expected": "Das Auto von Dr. [PERSON].", + "spans": [ + { + "start": 17, + "end": 30, + "label": "PERSON", + "text": "Klaus Schmidt" + } + ], + "id": "baseline-02", + "language": "de", + "ocr_noise": false + }, + { + "text": "Friedrich Wilhelm geht nach Hause.", + "expected": "[PERSON] geht nach Hause.", + "spans": [ + { + "start": 0, + "end": 17, + "label": "PERSON", + "text": "Friedrich Wilhelm" + } + ], + "id": "baseline-03", + "language": "de", + "ocr_noise": false + }, + { + "text": "Ich kenne Frau Anne-Marie Schneider nicht.", + "expected": "Ich kenne Frau [PERSON] nicht.", + "spans": [ + { + "start": 15, + "end": 35, + "label": "PERSON", + "text": "Anne-Marie Schneider" + } + ], + "id": "baseline-04", + "language": "de", + "ocr_noise": false + }, + { + "text": "John Doe is a good man.", + "expected": "[PERSON] is a good man.", + "spans": [ + { + "start": 0, + "end": 8, + "label": "PERSON", + "text": "John Doe" + } + ], + "id": "baseline-05", + "language": "en", + "ocr_noise": false + }, + { + "text": "Alice and Bob went to the store.", + "expected": "[PERSON] and [PERSON] went to the store.", + "spans": [ + { + "start": 0, + "end": 5, + "label": "PERSON", + "text": "Alice" + }, + { + "start": 10, + "end": 13, + "label": "PERSON", + "text": "Bob" + } + ], + "id": "baseline-06", + "language": "en", + "ocr_noise": false + }, + { + "text": "Is Mary Johnson there?", + "expected": "Is [PERSON] there?", + "spans": [ + { + "start": 3, + "end": 15, + "label": "PERSON", + "text": "Mary Johnson" + } + ], + "id": "baseline-07", + "language": "en", + "ocr_noise": false + }, + { + "text": "Meeting with Mr. Charles Montgomery.", + "expected": "Meeting with Mr. [PERSON].", + "spans": [ + { + "start": 17, + "end": 35, + "label": "PERSON", + "text": "Charles Montgomery" + } + ], + "id": "baseline-08", + "language": "en", + "ocr_noise": false + }, + { + "text": "Elizabeth Taylor was famous.", + "expected": "[PERSON] was famous.", + "spans": [ + { + "start": 0, + "end": 16, + "label": "PERSON", + "text": "Elizabeth Taylor" + } + ], + "id": "baseline-09", + "language": "en", + "ocr_noise": false + }, + { + "text": "I live in New York.", + "expected": "I live in [CITY].", + "spans": [ + { + "start": 10, + "end": 18, + "label": "CITY", + "text": "New York" + } + ], + "id": "baseline-10", + "language": "en", + "ocr_noise": false + }, + { + "text": "Der Zug nach München fährt ab.", + "expected": "Der Zug nach [CITY] fährt ab.", + "spans": [ + { + "start": 13, + "end": 20, + "label": "CITY", + "text": "München" + } + ], + "id": "baseline-11", + "language": "de", + "ocr_noise": false + }, + { + "text": "Paris is beautiful in spring.", + "expected": "[CITY] is beautiful in spring.", + "spans": [ + { + "start": 0, + "end": 5, + "label": "CITY", + "text": "Paris" + } + ], + "id": "baseline-12", + "language": "en", + "ocr_noise": false + }, + { + "text": "Flug von Frankfurt nach Tokyo.", + "expected": "Flug von [CITY] nach [CITY].", + "spans": [ + { + "start": 9, + "end": 18, + "label": "CITY", + "text": "Frankfurt" + }, + { + "start": 24, + "end": 29, + "label": "CITY", + "text": "Tokyo" + } + ], + "id": "baseline-13", + "language": "de", + "ocr_noise": false + }, + { + "text": "Welcome to Los Angeles, California.", + "expected": "Welcome to [CITY], [CITY].", + "spans": [ + { + "start": 11, + "end": 22, + "label": "CITY", + "text": "Los Angeles" + }, + { + "start": 24, + "end": 34, + "label": "CITY", + "text": "California" + } + ], + "id": "baseline-14", + "language": "en", + "ocr_noise": false + }, + { + "text": "Ihre Religionszugehörigkeit ist evangelisch.", + "expected": "Ihre Religionszugehörigkeit ist [RELIGION].", + "spans": [ + { + "start": 32, + "end": 43, + "label": "RELIGION", + "text": "evangelisch" + } + ], + "id": "baseline-15", + "language": "de", + "ocr_noise": false + }, + { + "text": "Er ist katholisch.", + "expected": "Er ist [RELIGION].", + "spans": [ + { + "start": 7, + "end": 17, + "label": "RELIGION", + "text": "katholisch" + } + ], + "id": "baseline-16", + "language": "de", + "ocr_noise": false + }, + { + "text": "They practice Judaism.", + "expected": "They practice [RELIGION].", + "spans": [ + { + "start": 14, + "end": 21, + "label": "RELIGION", + "text": "Judaism" + } + ], + "id": "baseline-17", + "language": "en", + "ocr_noise": false + }, + { + "text": "He is a devout Muslim.", + "expected": "He is a devout [RELIGION].", + "spans": [ + { + "start": 15, + "end": 21, + "label": "RELIGION", + "text": "Muslim" + } + ], + "id": "baseline-18", + "language": "en", + "ocr_noise": false + }, + { + "text": "Buddhist monks live there.", + "expected": "[RELIGION] monks live there.", + "spans": [ + { + "start": 0, + "end": 8, + "label": "RELIGION", + "text": "Buddhist" + } + ], + "id": "baseline-19", + "language": "en", + "ocr_noise": false + }, + { + "text": "John Doe from New York met Lina Wolkenstein in Berlin.", + "expected": "[PERSON] from [CITY] met [PERSON] in [CITY].", + "spans": [ + { + "start": 0, + "end": 8, + "label": "PERSON", + "text": "John Doe" + }, + { + "start": 14, + "end": 22, + "label": "CITY", + "text": "New York" + }, + { + "start": 27, + "end": 43, + "label": "PERSON", + "text": "Lina Wolkenstein" + }, + { + "start": 47, + "end": 53, + "label": "CITY", + "text": "Berlin" + } + ], + "id": "baseline-20", + "language": "mixed", + "ocr_noise": false + }, + { + "text": "David is Catholic and lives in Rome.", + "expected": "[PERSON] is [RELIGION] and lives in [CITY].", + "spans": [ + { + "start": 0, + "end": 5, + "label": "PERSON", + "text": "David" + }, + { + "start": 9, + "end": 17, + "label": "RELIGION", + "text": "Catholic" + }, + { + "start": 31, + "end": 35, + "label": "CITY", + "text": "Rome" + } + ], + "id": "baseline-21", + "language": "en", + "ocr_noise": false + }, + { + "text": "John likes John's new car.", + "expected": "[PERSON] likes [PERSON]'s new car.", + "spans": [ + { + "start": 0, + "end": 4, + "label": "PERSON", + "text": "John" + }, + { + "start": 11, + "end": 15, + "label": "PERSON", + "text": "John" + } + ], + "id": "baseline-22", + "language": "en", + "ocr_noise": false + }, + { + "text": "Berlin, oh Berlin, my favorite city.", + "expected": "[CITY], oh [CITY], my favorite city.", + "spans": [ + { + "start": 0, + "end": 6, + "label": "CITY", + "text": "Berlin" + }, + { + "start": 11, + "end": 17, + "label": "CITY", + "text": "Berlin" + } + ], + "id": "baseline-23", + "language": "en", + "ocr_noise": false + }, + { + "text": "Washington is the capital.", + "expected": "[CITY] is the capital.", + "spans": [ + { + "start": 0, + "end": 10, + "label": "CITY", + "text": "Washington" + } + ], + "id": "baseline-24", + "language": "en", + "ocr_noise": false + }, + { + "text": "George Washington was the president.", + "expected": "[PERSON] was the president.", + "spans": [ + { + "start": 0, + "end": 17, + "label": "PERSON", + "text": "George Washington" + } + ], + "id": "baseline-25", + "language": "en", + "ocr_noise": false + }, + { + "text": "I visited the Jordan river.", + "expected": "I visited the [CITY] river.", + "spans": [ + { + "start": 14, + "end": 20, + "label": "CITY", + "text": "Jordan" + } + ], + "id": "baseline-26", + "language": "en", + "ocr_noise": false + }, + { + "text": "Michael Jordan is a basketball player.", + "expected": "[PERSON] is a basketball player.", + "spans": [ + { + "start": 0, + "end": 14, + "label": "PERSON", + "text": "Michael Jordan" + } + ], + "id": "baseline-27", + "language": "en", + "ocr_noise": false + }, + { + "text": "Mr. Smith-\nson", + "expected": "Mr. [PERSON]", + "spans": [ + { + "start": 4, + "end": 14, + "label": "PERSON", + "text": "Smith-\nson" + } + ], + "id": "baseline-28", + "language": "en", + "ocr_noise": true + }, + { + "text": "Ber-\nlin", + "expected": "[CITY]", + "spans": [ + { + "start": 0, + "end": 8, + "label": "CITY", + "text": "Ber-\nlin" + } + ], + "id": "baseline-29", + "language": "de", + "ocr_noise": true + }, + { + "text": "Jchn D0e went home.", + "expected": "[PERSON] went home.", + "spans": [ + { + "start": 0, + "end": 8, + "label": "PERSON", + "text": "Jchn D0e" + } + ], + "id": "baseline-30", + "language": "en", + "ocr_noise": true + }, + { + "text": "Welcome to Nevv Y0rk.", + "expected": "Welcome to [CITY].", + "spans": [ + { + "start": 11, + "end": 20, + "label": "CITY", + "text": "Nevv Y0rk" + } + ], + "id": "baseline-31", + "language": "en", + "ocr_noise": true + }, + { + "text": "The cat sat on the mat.", + "expected": "The cat sat on the mat.", + "spans": [], + "id": "baseline-32", + "language": "en", + "ocr_noise": false + }, + { + "text": "12345 is a number.", + "expected": "12345 is a number.", + "spans": [], + "id": "baseline-33", + "language": "en", + "ocr_noise": false + }, + { + "text": "This is a simple test.", + "expected": "This is a simple test.", + "spans": [], + "id": "baseline-34", + "language": "en", + "ocr_noise": false + }, + { + "text": "My email is test@example.com.", + "expected": "My email is [EMAIL].", + "spans": [ + { + "start": 12, + "end": 28, + "label": "EMAIL", + "text": "test@example.com" + } + ], + "id": "baseline-35", + "language": "en", + "ocr_noise": false + }, + { + "text": "Call me at 555-0199.", + "expected": "Call me at [PHONE].", + "spans": [ + { + "start": 11, + "end": 19, + "label": "PHONE", + "text": "555-0199" + } + ], + "id": "baseline-36", + "language": "en", + "ocr_noise": false + }, + { + "text": "I work for Google Inc.", + "expected": "I work for [ORGANIZATION].", + "spans": [ + { + "start": 11, + "end": 22, + "label": "ORGANIZATION", + "text": "Google Inc." + } + ], + "id": "baseline-37", + "language": "en", + "ocr_noise": false + }, + { + "text": "Contact info@company.de", + "expected": "Contact [EMAIL]", + "spans": [ + { + "start": 8, + "end": 23, + "label": "EMAIL", + "text": "info@company.de" + } + ], + "id": "baseline-38", + "language": "en", + "ocr_noise": false + }, + { + "text": "The UN is an organization.", + "expected": "The [ORGANIZATION] is an organization.", + "spans": [ + { + "start": 4, + "end": 6, + "label": "ORGANIZATION", + "text": "UN" + } + ], + "id": "baseline-39", + "language": "en", + "ocr_noise": false + } +] diff --git a/ocr/anonymization/fixtures/validation.json b/ocr/anonymization/fixtures/validation.json new file mode 100644 index 0000000000000000000000000000000000000000..aea39e151a69cdee15a8a4e5ae4f51d65112aad2 --- /dev/null +++ b/ocr/anonymization/fixtures/validation.json @@ -0,0 +1,210 @@ +[ + { + "id": "validation-00", + "language": "de", + "text": "Mara Kühn arbeitet bei Siemens in Dresden.", + "spans": [ + { + "start": 0, + "end": 9, + "label": "PERSON", + "text": "Mara Kühn" + }, + { + "start": 23, + "end": 30, + "label": "ORGANIZATION", + "text": "Siemens" + }, + { + "start": 34, + "end": 41, + "label": "CITY", + "text": "Dresden" + } + ], + "expected": "[PERSON] arbeitet bei [ORGANIZATION] in [CITY].", + "ocr_noise": false + }, + { + "id": "validation-01", + "language": "en", + "text": "Emma Wilson works for Acme Corporation in London.", + "spans": [ + { + "start": 0, + "end": 11, + "label": "PERSON", + "text": "Emma Wilson" + }, + { + "start": 22, + "end": 38, + "label": "ORGANIZATION", + "text": "Acme Corporation" + }, + { + "start": 42, + "end": 48, + "label": "CITY", + "text": "London" + } + ], + "expected": "[PERSON] works for [ORGANIZATION] in [CITY].", + "ocr_noise": false + }, + { + "id": "validation-02", + "language": "de", + "text": "Sie ist muslimisch. Kontakt: mara.kuehn@example.org, +49 351 7654321.", + "spans": [ + { + "start": 8, + "end": 18, + "label": "RELIGION", + "text": "muslimisch" + }, + { + "start": 29, + "end": 51, + "label": "EMAIL", + "text": "mara.kuehn@example.org" + }, + { + "start": 53, + "end": 68, + "label": "PHONE", + "text": "+49 351 7654321" + } + ], + "expected": "Sie ist [RELIGION]. Kontakt: [EMAIL], [PHONE].", + "ocr_noise": false + }, + { + "id": "validation-03", + "language": "en", + "text": "His religion is Hinduism. Contact emma@example.net or +1 212 555 0188.", + "spans": [ + { + "start": 16, + "end": 24, + "label": "RELIGION", + "text": "Hinduism" + }, + { + "start": 34, + "end": 50, + "label": "EMAIL", + "text": "emma@example.net" + }, + { + "start": 54, + "end": 69, + "label": "PHONE", + "text": "+1 212 555 0188" + } + ], + "expected": "His religion is [RELIGION]. Contact [EMAIL] or [PHONE].", + "ocr_noise": false + }, + { + "id": "validation-04", + "language": "de", + "text": "Nora Winter besucht Köln. Nora Winter bleibt dort.", + "spans": [ + { + "start": 0, + "end": 11, + "label": "PERSON", + "text": "Nora Winter" + }, + { + "start": 20, + "end": 24, + "label": "CITY", + "text": "Köln" + }, + { + "start": 26, + "end": 37, + "label": "PERSON", + "text": "Nora Winter" + } + ], + "expected": "[PERSON] besucht [CITY]. [PERSON] bleibt dort.", + "ocr_noise": false + }, + { + "id": "validation-05", + "language": "en", + "text": "Jordan is a river, and Jordan Miller is a person.", + "spans": [ + { + "start": 23, + "end": 36, + "label": "PERSON", + "text": "Jordan Miller" + } + ], + "expected": "Jordan is a river, and [PERSON] is a person.", + "ocr_noise": false + }, + { + "id": "validation-06", + "language": "de", + "text": "Religion ist ein Gesprächsthema. Keine Zugehörigkeit ist angegeben.", + "spans": [], + "expected": "Religion ist ein Gesprächsthema. Keine Zugehörigkeit ist angegeben.", + "ocr_noise": false + }, + { + "id": "validation-07", + "language": "en", + "text": "A name alone does not establish religious affiliation.", + "spans": [], + "expected": "A name alone does not establish religious affiliation.", + "ocr_noise": false + }, + { + "id": "validation-08", + "language": "de", + "text": "Frau Eva-Luise Schröder wohnt in Ham-\nburg.", + "spans": [ + { + "start": 5, + "end": 23, + "label": "PERSON", + "text": "Eva-Luise Schröder" + }, + { + "start": 33, + "end": 42, + "label": "CITY", + "text": "Ham-\nburg" + } + ], + "expected": "Frau [PERSON] wohnt in [CITY].", + "ocr_noise": true + }, + { + "id": "validation-09", + "language": "en", + "text": "M4ry Br0wn lives in L0ndon.", + "spans": [ + { + "start": 0, + "end": 10, + "label": "PERSON", + "text": "M4ry Br0wn" + }, + { + "start": 20, + "end": 26, + "label": "CITY", + "text": "L0ndon" + } + ], + "expected": "[PERSON] lives in [CITY].", + "ocr_noise": true + } +] diff --git a/ocr/anonymization/historical.py b/ocr/anonymization/historical.py new file mode 100644 index 0000000000000000000000000000000000000000..a99ecca67a6c601a8fcbcd7fab24ebfb386f02ae --- /dev/null +++ b/ocr/anonymization/historical.py @@ -0,0 +1,36 @@ +"""Verify immutable historical artifacts; never evaluate updated code as an old holdout.""" + +import hashlib +import json +from pathlib import Path + +FROZEN_ARTIFACT_SHA256 = { + "docs/anonymization-evaluation/gate03c-holdout-first.json": "257daa6d43285925c4899471c3f615293293fa0fa8ed17545ebb9d8e99802e83", + "docs/anonymization-evaluation/gate03c-holdout-first.provenance.json": "bf19f6ae26e146aa6e04aaed9a54705c0f910af308ea33a8c5eacfe4d1724fbb", + "docs/anonymization-evaluation/gate03d-holdout-first.json": "08f86d2b78871683835c51b627e251c66b83fc33f828d8a6c8e9742c8bd47d9f", + "docs/anonymization-evaluation/gate03d-holdout-first.provenance.json": "9d5fa15c5427fcfa499262b20453ed4adfe59da86709dffc745ec205e0661bca", + "ocr/anonymization/fixtures/holdout03c/cases.json": "0174fb41d34d0a4db18877c68fb70149eaa67c05da86608d5da4ff8765a52c15", + "ocr/anonymization/fixtures/holdout03c/freeze.json": "c707239993099dab4a48b83f6c2aac58674de5423ede75cd1aced17d290708a6", + "ocr/anonymization/fixtures/holdout03d/cases.json": "7180e176f6d8bd259da56ad729ede6fd2c798aca15a786ba0ff0adcd862f21a1", + "ocr/anonymization/fixtures/holdout03d/freeze.json": "448dc5d7b0f3ef00f7d0f90084866f7f2e85311aac6af32ed0ad3d2658f8b1f5", +} + + +def verify_historical(): + repo = Path(__file__).parents[2] + for name, value in FROZEN_ARTIFACT_SHA256.items(): + if hashlib.sha256((repo / name).read_bytes()).hexdigest() != value: + raise ValueError("historical_evidence_changed") + root = Path(__file__).parent / "fixtures/holdout03d" + freeze = json.loads((root / "freeze.json").read_bytes()) + report = json.loads( + (repo / "docs/anonymization-evaluation/gate03d-holdout-first.json").read_bytes() + ) + cases = json.loads((root / "cases.json").read_bytes()) + if ( + report["implementation_sha256"] != freeze["implementation_sha256"] + or report["corpus_sha256"] != freeze["sha256"] + or {p["case_id"] for p in report["per_case"]} != {c["id"] for c in cases} + ): + raise ValueError("historical_evidence_changed") + return root, freeze, cases, report diff --git a/ocr/anonymization/holdout.py b/ocr/anonymization/holdout.py new file mode 100644 index 0000000000000000000000000000000000000000..bab1d71e1d6a5685ed5640cc850e55f8a1803d44 --- /dev/null +++ b/ocr/anonymization/holdout.py @@ -0,0 +1,79 @@ +"""First-result, frozen synthetic holdout; refuses prompt/weight changes.""" + +import hashlib +import importlib.metadata +import json +import resource +import subprocess +import sys +import time +from pathlib import Path + +from .atomic_output import write_candidate +from .detectors import detect_contacts +from .evaluate import score_cases +from .model import MODEL_REVISION +from .recognizer import GLiNERRecognizer + + +def run(cache, output): + root = Path(__file__).parent / "fixtures/holdout03c" + frozen = json.loads((root / "freeze.json").read_text()) + data = (root / "cases.json").read_bytes() + if hashlib.sha256(data).hexdigest() != frozen["sha256"]: + raise ValueError("holdout_integrity_failed") + denied = [] + + def no_network(event, args): + if event in ("socket.connect", "socket.bind", "socket.getaddrinfo"): + denied.append(event) + raise PermissionError("qualification_network_disabled") + + sys.addaudithook(no_network) + started = time.perf_counter() + recognizer = GLiNERRecognizer(cache) + loading_ms = (time.perf_counter() - started) * 1000 + if ( + recognizer.prompts != frozen["control_prompts"] + or recognizer.threshold != frozen["threshold"] + or MODEL_REVISION != frozen["model_revision"] + ): + raise ValueError("holdout_control_changed") + recognizer.recognize("A plain synthetic warmup sentence.") + report = dict( + schema_version="holdout03c-first-control/v1", + corpus_sha256=frozen["sha256"], + code_sha=subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), + freeze_commit=subprocess.check_output( + ["git", "log", "-1", "--format=%H", "--", str(root / "cases.json")], text=True + ).strip(), + model_revision=MODEL_REVISION, + prompts=recognizer.prompts, + threshold=recognizer.threshold, + loading_ms=loading_ms, + runtime_versions={ + k: importlib.metadata.version(k) + for k in ("gliner", "torch", "transformers", "numpy", "pydantic") + }, + evaluation=score_cases(recognizer, json.loads(data), detector=detect_contacts), + peak_rss_mib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, + denied_network_calls=denied, + automated_release_eligible=False, + ) + write_candidate(output, (json.dumps(report, indent=2) + "\n").encode()) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description="Evaluate frozen holdout once with existing control" + ) + parser.add_argument("--cache", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + try: + run(args.cache, args.output) + except Exception: + raise SystemExit("holdout_evaluation_failed") from None + print("holdout_evaluation_recorded_check_qualification_result") diff --git a/ocr/anonymization/host_adapter.py b/ocr/anonymization/host_adapter.py new file mode 100644 index 0000000000000000000000000000000000000000..64126892385d36f43709144653b168c0c8907082 --- /dev/null +++ b/ocr/anonymization/host_adapter.py @@ -0,0 +1,167 @@ +"""Host security contract. No identity provider or request-identity fallback.""" + +import time +from typing import Protocol + +from pydantic import Field + +from ocr.schemas import SafeError + +from .release_policy import ReleaseAuthority +from .schemas import Destination, StrictModel, VerifiedReview + + +class AuthenticatedIdentity(StrictModel): + subject: str = Field(pattern=r"^[a-zA-Z0-9:_-]{1,128}$") + session_id: str = Field(pattern=r"^[a-zA-Z0-9:_-]{1,128}$") + expires_at: int + + +class CompletedReview(StrictModel): + attestation: VerifiedReview + completed: bool + revoked: bool + expires_at: int + + +class ExportAuthorizationRecord(StrictModel): + artifact_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + candidate_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + review_id: str + review_evidence_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + sensitivity: str + destination_id: str + expires_at: int + revoked: bool + + +class HostSecurityAdapter(Protocol): + """Implement against authenticated server context and authoritative stores. + + credentials are opaque authentication evidence, never a requested subject. + Review content/evidence must be immutable; revocation is separate mutable state. + Host ACLs and destination policy are looked up on EVERY operation. + """ + + def authenticate(self, credential: object) -> AuthenticatedIdentity: ... + def load_review(self, review_id: str) -> CompletedReview: ... + def candidate_exists(self, candidate_digest: str) -> bool: ... + def load_export(self, artifact_digest: str) -> ExportAuthorizationRecord: ... + def current_acl( + self, identity: AuthenticatedIdentity, scope: str, sensitivity: str, destination_id: str + ) -> bool: ... + def destination(self, destination_id: str) -> Destination: ... + def corrected_release_allowed(self, candidate, review: VerifiedReview) -> bool: ... + def legacy_release_allowed(self, candidate, review: VerifiedReview) -> bool: ... + + +class HostAuthorizationBridge: + def __init__(self, host: HostSecurityAdapter): + self.host = host + + def identity(self, credential): + identity = AuthenticatedIdentity.model_validate(self.host.authenticate(credential)) + if identity.expires_at <= time.time(): + raise SafeError("host_authorization_denied") + return identity + + def review(self, credential, review_id): + try: + identity = self.identity(credential) + record = CompletedReview.model_validate(self.host.load_review(review_id)) + review = record.attestation + if ( + not record.completed + or record.revoked + or record.expires_at <= time.time() + or review.reviewer_subject != identity.subject + or review.review_id != review_id + or self.host.candidate_exists(review.candidate_digest) is not True + ): + raise ValueError() + destination = Destination.model_validate(self.host.destination(review.destination_id)) + if ( + self.host.current_acl( + identity, + "anonymization:release", + review.sensitivity, + destination.destination_id, + ) + is not True + ): + raise ValueError() + return review + except Exception: + raise SafeError("host_authorization_denied") from None + + def acl(self, review, scope, destination): + # Identity/ACL already checked with the current session by review(). This + # extra check binds the live operator policy to the approval snapshot. + return ( + Destination.model_validate(self.host.destination(destination.destination_id)) + == destination + ) + + def corrected_release(self, candidate, review): + # Missing production integration denies corrected candidates. + try: + return self.host.corrected_release_allowed(candidate, review) is True + except Exception: + return False + + def legacy_release(self, candidate, review): + # v1/v2 require an explicit authoritative source/schema eligibility policy. + try: + return self.host.legacy_release_allowed(candidate, review) is True + except Exception: + return False + + def authority(self, key, ledger, destination_ids, **kwargs): + return ReleaseAuthority( + key, + self.acl, + verify_review=self.review, + consume_approval=ledger.consume, + correction_policy=self.corrected_release, + legacy_policy=self.legacy_release, + destinations=tuple(self.host.destination(i) for i in destination_ids), + **kwargs, + ) + + def transfer_authorized(self, credential, export_digest, sensitivity, destination_id): + """Separate current permission; a release approval alone never calls this.""" + try: + identity = self.identity(credential) + destination = Destination.model_validate(self.host.destination(destination_id)) + if ( + destination.destination_class != "private" + or sensitivity not in destination.allowed_sensitivities + ): + return False + record = ExportAuthorizationRecord.model_validate(self.host.load_export(export_digest)) + review = CompletedReview.model_validate(self.host.load_review(record.review_id)) + if ( + record.artifact_digest != export_digest + or record.sensitivity != sensitivity + or record.destination_id != destination_id + or record.revoked + or record.expires_at <= time.time() + or not review.completed + or review.revoked + or review.expires_at <= time.time() + or review.attestation.review_id != record.review_id + or review.attestation.candidate_digest != record.candidate_digest + or review.attestation.evidence_digest != record.review_evidence_digest + or review.attestation.sensitivity != sensitivity + or review.attestation.destination_id != destination_id + or self.host.candidate_exists(record.candidate_digest) is not True + ): + return False + return ( + self.host.current_acl( + identity, "anonymization:private-transfer", sensitivity, destination_id + ) + is True + ) + except Exception: + raise SafeError("host_authorization_denied") from None diff --git a/ocr/anonymization/investigate.py b/ocr/anonymization/investigate.py new file mode 100644 index 0000000000000000000000000000000000000000..ddbce1e5547581e6fe7fd419075cb13009893608 --- /dev/null +++ b/ocr/anonymization/investigate.py @@ -0,0 +1,142 @@ +"""Explicit opt-in experiment on repository SYNTHETIC fixtures only. + +Unlike aggregate evaluation, this artifact intentionally contains synthetic gold +and prediction text for diagnosis. Never use this command with private documents. +""" + +import json +import resource +import time +from pathlib import Path + +from .detectors import detect +from .evaluate import score_cases +from .model import MODEL_ID, MODEL_REVISION +from .normalization import dehyphenate +from .recognizer import DEFAULT_PROMPTS, GLiNERRecognizer +from .replacement import resolve_overlaps, validate_entities + +REVIEWED_PROMPTS = DEFAULT_PROMPTS | {"CITY": "city", "ORGANIZATION": "organization"} + +EXPERIMENTS = ( + ("reviewed_baseline", {}, 0.5), + ("lower_threshold", {}, 0.35), + ("geographic_prompt", {"CITY": "location"}, 0.5), + ( + "geographic_and_company_prompt", + {"CITY": "location", "ORGANIZATION": "company or organization"}, + 0.5, + ), +) + + +class Recorded: + def __init__(self, recognizer): + self.recognizer = recognizer + self.predictions = {} + + def recognize(self, text): + found = self.recognizer.recognize(text) + self.predictions[text] = found + return found + + +def run(cache, output): + start = time.perf_counter() + model = GLiNERRecognizer(cache) + loading_ms = (time.perf_counter() - start) * 1000 + root = Path(__file__).parent / "fixtures" + suites = { + n: json.loads((root / f"{n}.json").read_text()) for n in ("qualification", "validation") + } + report = dict( + model=MODEL_ID, + revision=MODEL_REVISION, + loading_ms=loading_ms, + experiments={}, + baseline_diagnosis=[], + ) + model.recognize("The cat sat on the mat.") + for name, changes, threshold in EXPERIMENTS: + model.prompts = REVIEWED_PROMPTS | changes + model.threshold = threshold + recorder = Recorded(model) + scores = {n: score_cases(recorder, cases) for n, cases in suites.items()} + report["experiments"][name] = dict( + prompts=model.prompts.copy(), threshold=threshold, suites=scores + ) + if name != "reviewed_baseline": + continue + for suite, cases in suites.items(): + for case in cases: + raw = recorder.predictions[case["text"]] + combined = resolve_overlaps( + validate_entities(case["text"], raw + detect(case["text"])) + ) + gold = {(s["start"], s["end"], s["label"]) for s in case["spans"]} + predicted = {(s.start, s.end, s.label) for s in combined} + errors = { + label: dict( + false_positives=[ + s.model_dump() + for s in combined + if s.label == label and (s.start, s.end, s.label) not in gold + ], + false_negatives=[ + s + for s in case["spans"] + if s["label"] == label + and (s["start"], s["end"], label) not in predicted + ], + ) + for label in ("CITY", "ORGANIZATION") + } + if not any(v["false_positives"] or v["false_negatives"] for v in errors.values()): + continue + normalized, _ = dehyphenate(case["text"]) + item = dict( + suite=suite, + case_id=case["id"], + text=case["text"], + normalized=normalized, + gold=case["spans"], + predictions=[s.model_dump() for s in combined], + errors=errors, + ) + uncovered_city = [ + s + for s in case["spans"] + if s["label"] == "CITY" + and any( + case["text"][i].isalnum() + and not any(p.start <= i < p.end for p in combined) + for i in range(s["start"], s["end"]) + ) + ] + if uncovered_city: + tokenizer = model.model.data_processor.transformer_tokenizer + ids = tokenizer(normalized)["input_ids"] + item["uncovered_city"] = uncovered_city + item["tokens"] = tokenizer.convert_ids_to_tokens(ids) + item["low_threshold_predictions"] = model.model.predict_entities( + normalized, list(model.prompts.values()), threshold=0.1, flat_ner=True + ) + report["baseline_diagnosis"].append(item) + report["peak_rss_mib"] = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 + Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description="Compare fixed experiments on synthetic repository corpora" + ) + parser.add_argument("--cache", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + try: + run(args.cache, args.output) + except Exception: + raise SystemExit("synthetic_recognition_investigation_failed") from None + print("synthetic_recognition_investigation_completed") diff --git a/ocr/anonymization/model.py b/ocr/anonymization/model.py new file mode 100644 index 0000000000000000000000000000000000000000..4afd1b679f253571e2603b14c5c86ca6c3ab647b --- /dev/null +++ b/ocr/anonymization/model.py @@ -0,0 +1,81 @@ +"""Pinned artifacts for native GLiNER; setup is the only network-enabled operation.""" + +import hashlib +import json +from pathlib import Path + +from ocr.schemas import SafeError + +MODEL_ID = "VAGOsolutions/SauerkrautLM-GLiNER" +MODEL_REVISION = "ba967ee030b7e679dd5422cb16fcd1b379da38e1" +FILES = ( + "gliner_config.json", + "pytorch_model.bin", + "special_tokens_map.json", + "tokenizer.json", + "tokenizer_config.json", +) + + +def file_hash(path): + h = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def verify_cache(path): + try: + path = Path(path) + manifest = json.loads((path / "manifest.json").read_text()) + expected = json.loads(Path(__file__).with_name("artifacts.json").read_text()) + if manifest != expected: + raise ValueError() + if manifest["model"] != MODEL_ID or manifest["revision"] != MODEL_REVISION: + raise ValueError() + if set(manifest["files"]) != set(FILES): + raise ValueError() + for name in FILES: + if file_hash(path / name) != manifest["files"][name]: + raise ValueError() + return path + except Exception: + raise SafeError("anonymization_model_unavailable") from None + + +def install(path): + """Explicit operator command. Download the fixed snapshot, record file integrity.""" + from huggingface_hub import snapshot_download + + path = Path(path) + if (path / "manifest.json").exists(): + return verify_cache(path) + if path.exists() and any(p.name not in {*FILES, ".cache"} for p in path.iterdir()): + raise SafeError("anonymization_model_setup_directory_not_empty") + snapshot_download( + MODEL_ID, + revision=MODEL_REVISION, + local_dir=path, + allow_patterns=list(FILES), + ) + manifest = { + "model": MODEL_ID, + "revision": MODEL_REVISION, + "files": {name: file_hash(path / name) for name in FILES}, + } + (path / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + return verify_cache(path) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Explicit pinned local anonymization model setup") + parser.add_argument("--cache", required=True) + args = parser.parse_args() + try: + install(args.cache) + except Exception: + raise SystemExit("anonymization_model_setup_failed") from None + print(f"Cached {MODEL_ID}@{MODEL_REVISION}") diff --git a/ocr/anonymization/normalization.py b/ocr/anonymization/normalization.py new file mode 100644 index 0000000000000000000000000000000000000000..2636f5042d8f2305a078ef7a4ee8f973fb490f00 --- /dev/null +++ b/ocr/anonymization/normalization.py @@ -0,0 +1,11 @@ +"""One reversible source-coordinate view for hyphenated OCR line wraps.""" + +import re + +LINE_WRAP = re.compile(r"(?<=[^\W\d_])-\r?\n(?=[^\W\d_])", re.UNICODE) + + +def dehyphenate(text): + removed = {i for match in LINE_WRAP.finditer(text) for i in range(match.start(), match.end())} + positions = tuple(i for i in range(len(text)) if i not in removed) + return "".join(text[i] for i in positions), positions diff --git a/ocr/anonymization/outcomes.py b/ocr/anonymization/outcomes.py new file mode 100644 index 0000000000000000000000000000000000000000..1e939b59a5aab07a1da665a90a177210d2c7376b --- /dev/null +++ b/ocr/anonymization/outcomes.py @@ -0,0 +1,124 @@ +"""Development accuracy and actual candidate coverage, with explicit abstentions. + +The historical evaluator and its saved first reports are retained unchanged. +This reporting interface separates blocked evidence from emitted candidate text. +""" + +from collections import Counter + +from ocr.schemas import SafeError + +from .detectors import detect +from .evaluate import metrics, score_cases +from .schemas import LABELS + +OUTCOMES = ( + "complete_no_measured_leakage", + "candidate_partial_mask", + "candidate_fully_unmasked", + "blocked_conflicting_spans", + "blocked_model_runtime", + "other_incomplete", +) +RUNTIME_FAILURES = frozenset( + ("anonymization_model_unavailable", "anonymization_inference_failed", "anonymization_failed") +) + + +def score_outcomes(recognizer, cases, *, detector=detect): + class SafeRecognizer: + def recognize(self, text): + try: + return recognizer.recognize(text) + except SafeError: + raise + except Exception: + raise SafeError("anonymization_inference_failed") from None + + def safe_detector(text): + try: + return detector(text) + except SafeError: + raise + except Exception: + raise SafeError("anonymization_failed") from None + + details = [] + accuracy = score_cases(SafeRecognizer(), cases, detector=safe_detector, case_reports=details) + # Historical aggregate coverage counts incomplete documents as uncovered. + # Name that conservative scope explicitly; it is not emitted-candidate leakage. + for old, new in ( + ("unredacted_canaries", "uncovered_source_canaries_including_incomplete"), + ("literal_unredacted_canaries", "literal_presence_or_incomplete"), + ("partial_canaries", "partially_covered_source_canaries_including_incomplete"), + ("fully_uncovered_canaries", "fully_uncovered_source_canaries_including_incomplete"), + ("alphanumeric_canary_coverage", "all_document_source_coverage"), + ("qualification_passed", "conservative_accuracy_gate_passed"), + ): + accuracy[new] = accuracy.pop(old) + accuracy["scope"] = ( + "all source spans; incomplete processing counts as missing, not leaked output" + ) + counts, partial, full, candidate_spans = Counter(), Counter(), Counter(), Counter() + for case, detail in zip(cases, details, strict=True): + error = detail["error_code"] + if error: + outcome = ( + "blocked_conflicting_spans" + if error == "ambiguous_entity_overlap" + else "blocked_model_runtime" + if error in RUNTIME_FAILURES + else "other_incomplete" + ) + detail["candidate_returned"] = False + detail["candidate_canaries"] = [] + else: + flags = [] + for span in detail["canaries"]: + missing = span["uncovered_positions"] + total = sum(c.isalnum() for c in span["text"]) + if missing: + kind = "fully_unmasked" if len(missing) == total else "partial_mask" + (full if kind == "fully_unmasked" else partial)[span["label"]] += 1 + flags.append( + dict(label=span["label"], start=span["start"], end=span["end"], kind=kind) + ) + detail["candidate_returned"] = True + detail["candidate_canaries"] = flags + outcome = ( + "candidate_fully_unmasked" + if any(f["kind"] == "fully_unmasked" for f in flags) + else "candidate_partial_mask" + if flags + else "complete_no_measured_leakage" + ) + gold = {(s["start"], s["end"], s["label"]) for s in case["spans"]} + predicted = {(s["start"], s["end"], s["label"]) for s in detail["combined_predictions"]} + for labelset, kind in ( + (gold & predicted, "tp"), + (predicted - gold, "fp"), + (gold - predicted, "fn"), + ): + for _, _, label in labelset: + candidate_spans[(label, kind)] += 1 + detail["outcome"] = outcome + counts[outcome] += 1 + returned = sum(counts[k] for k in OUTCOMES[:3]) + safe = counts["blocked_conflicting_spans"] + counts["blocked_model_runtime"] + return dict( + schema_version="development-candidate-outcomes/v1", + accuracy=accuracy, + candidate_outcomes={k: counts[k] for k in OUTCOMES}, + candidate_return_rate=returned / len(cases), + safe_abstention_rate=safe / len(cases), + incomplete_rate=(len(cases) - returned) / len(cases), + no_measured_leakage_rate_among_returned=counts[OUTCOMES[0]] / returned + if returned + else None, + candidate_partial_canaries={k: partial[k] for k in LABELS}, + candidate_fully_unmasked_canaries={k: full[k] for k in LABELS}, + returned_candidate_entity_metrics=metrics(candidate_spans), + per_case=details, + automated_release_eligible=False, + recognition_qualified=False, # development evidence is not independent qualification + ) diff --git a/ocr/anonymization/private_review.py b/ocr/anonymization/private_review.py new file mode 100644 index 0000000000000000000000000000000000000000..c98d184d83b9e4aba56655682eb118ec58406112 --- /dev/null +++ b/ocr/anonymization/private_review.py @@ -0,0 +1,200 @@ +"""Authorized in-process review views. No HTTP endpoint, exporter or IdP. + +Only a real host adapter can attest bindings, ACLs and immutable source storage. +Views remain classified; showing a view or source page never completes review or +creates release permission. Original pages must be reviewed, not only detected spans. +""" + +import hashlib +import re +import time +from dataclasses import dataclass, field +from typing import Literal, Protocol + +from pydantic import Field, model_validator + +from ocr.schemas import SafeError + +from .host_adapter import HostAuthorizationBridge, HostSecurityAdapter +from .release_policy import digest +from .schemas import ( + CorrectedCandidate, + LegacyReleaseCandidate, + RecognitionProvenance, + ReleaseCandidate, + StrictModel, +) + + +class ReviewBinding(StrictModel): + review_ref: str = Field(pattern=r"^[a-zA-Z0-9_-]{1,128}$") + environment_id: str = Field(pattern=r"^[a-zA-Z0-9_-]{1,128}$") + sensitivity: Literal["D0", "D1", "D2"] + expires_at: int + revoked: bool + + +class SourcePageReference(StrictModel): + evidence_ref: str = Field(pattern=r"^[a-f0-9]{64}$") + page_index: int = Field(ge=0) + sensitivity: Literal["D0", "D1", "D2"] + sha256: str = Field(pattern=r"^[a-f0-9]{64}$") + characters: int = Field(ge=0, le=2_000_000) + + +class PrivateReviewMaterial(StrictModel): + binding: ReviewBinding + evidence_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + sources: tuple[SourcePageReference, ...] = Field(min_length=1, max_length=100, repr=False) + candidate: CorrectedCandidate | ReleaseCandidate | LegacyReleaseCandidate | None = Field( + repr=False + ) + candidate_digest: str | None = Field(pattern=r"^[a-f0-9]{64}$") + provenance: RecognitionProvenance | None + blocking_reason: ( + Literal["conflicting_spans", "model_runtime_failure", "other_incomplete"] | None + ) + anonymous: Literal[False] = False + automated_release_eligible: Literal[False] = False + release_review_required: Literal[True] = True + + @model_validator(mode="after") + def consistent_record(self): + sources = {p.page_index: p for p in self.sources} + if ( + len(sources) != len(self.sources) + or sum(p.characters for p in self.sources) > 2_000_000 + or any(p.sensitivity != self.binding.sensitivity for p in self.sources) + ): + raise ValueError("invalid_private_review_material") + if self.candidate is None: + if ( + self.candidate_digest is not None + or self.blocking_reason is None + or self.provenance is None + ): + raise ValueError("invalid_private_review_material") + else: + if ( + self.candidate_digest != digest(self.candidate) + or self.blocking_reason is not None + or self.candidate.sensitivity != self.binding.sensitivity + or len(self.sources) != len(self.candidate.pages) + or {page.page_index for page in self.candidate.pages} != set(sources) + ): + raise ValueError("invalid_private_review_material") + if type(self.candidate) in (ReleaseCandidate, CorrectedCandidate): + if self.provenance != self.candidate.provenance: + raise ValueError("invalid_private_review_material") + elif self.provenance is not None: + raise ValueError("legacy_provenance_unavailable") + for page in self.candidate.pages: + if page.page_index not in sources or any( + not 0 <= tag.start < tag.end <= sources[page.page_index].characters + for tag in page.tags + ): + raise ValueError("invalid_private_review_material") + return self + + @property + def categories(self): + return ( + tuple(sorted({t.label for p in self.candidate.pages for t in p.tags})) + if self.candidate + else () + ) + + @property + def review_flags(self): + # No fixture/gold knowledge or model success ever removes these flags. + ocr = self.candidate is not None and any( + "status-ocr" in t.normalization for p in self.candidate.pages for t in p.tags + ) + return ( + ( + "recognition_incomplete_possible", + "model_scores_uncalibrated", + "review_complete_source_pages", + ) + + (("processing_blocked",) if self.candidate is None else self.candidate.review_reasons) + + ( + ("legacy_configuration_unknown",) + if type(self.candidate) is LegacyReleaseCandidate + else () + ) + ) + (("ocr_status_normalization",) if ocr else ()) + + +@dataclass(frozen=True) +class ClassifiedPageView: + sensitivity: Literal["D0", "D1", "D2"] + page_index: int + text: str = field(repr=False) + + +class PrivateReviewHost(HostSecurityAdapter, Protocol): + def review_binding(self, review_ref: str) -> ReviewBinding: ... + def private_environment(self, environment_id: str) -> bool: ... + def load_review_material(self, review_ref: str) -> PrivateReviewMaterial: ... + def read_classified_page(self, evidence_ref: str, page_index: int) -> bytes: ... + + +class PrivateReviewBridge: + def __init__(self, host: PrivateReviewHost): + self.host = host + self.authorization = HostAuthorizationBridge(host) + + def _binding(self, credential, review_ref): + if type(review_ref) is not str or not re.fullmatch(r"[a-zA-Z0-9_-]{1,128}", review_ref): + raise ValueError() + identity = self.authorization.identity(credential) + binding = ReviewBinding.model_validate(self.host.review_binding(review_ref)) + if ( + binding.review_ref != review_ref + or binding.revoked + or binding.expires_at <= time.time() + or self.host.private_environment(binding.environment_id) is not True + or self.host.current_acl( + identity, "anonymization:review", binding.sensitivity, binding.environment_id + ) + is not True + ): + raise ValueError() + return binding + + def open(self, credential, review_ref): + try: + binding = self._binding(credential, review_ref) + material = PrivateReviewMaterial.model_validate( + self.host.load_review_material(review_ref) + ) + if ( + material.candidate is not None + and self.host.candidate_exists(material.candidate_digest) is not True + ): + raise ValueError() + if material.binding != binding or self._binding(credential, review_ref) != binding: + raise ValueError() + return material + except Exception: + raise SafeError("private_review_access_denied") from None + + def source_page(self, credential, review_ref, page_index): + try: + if type(page_index) is not int: + raise ValueError() + material = self.open(credential, review_ref) + source = next(p for p in material.sources if p.page_index == page_index) + data = self.host.read_classified_page(source.evidence_ref, page_index) + if ( + not isinstance(data, bytes) + or len(data) > 8 * 1024 * 1024 + or hashlib.sha256(data).hexdigest() != source.sha256 + ): + raise ValueError() + text = data.decode("utf-8", errors="strict") + if len(text) != source.characters or self.open(credential, review_ref) != material: + raise ValueError() + return ClassifiedPageView(material.binding.sensitivity, page_index, text) + except Exception: + raise SafeError("private_review_access_denied") from None diff --git a/ocr/anonymization/private_transport.py b/ocr/anonymization/private_transport.py new file mode 100644 index 0000000000000000000000000000000000000000..dacfe457c53a94ce3b112d94a17a1f2ebe4a14f5 --- /dev/null +++ b/ocr/anonymization/private_transport.py @@ -0,0 +1,155 @@ +"""Explicit local private delivery only. No URL client, redirects or fallback.""" + +import fcntl +import hashlib +import os +import stat +import uuid +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Literal + +from pydantic import Field + +from ocr.schemas import SafeError + +from .schemas import Destination, StrictModel + + +class ReviewedExport(StrictModel): + schema_version: Literal["ocr-reviewed-export/v2"] + text: str + sensitivity: Literal["D0", "D1", "D2"] + anonymous: Literal[False] + release_review_required: Literal[False] + release_approved: Literal[True] + candidate_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + destination_id: str = Field(pattern=r"^[a-zA-Z0-9:_-]{1,128}$") + destination_class: Literal["private"] + transmission_authorized: Literal[False] + + +@dataclass(frozen=True) +class LocalDestination: + policy: Destination + root: Path + + +class PrivateFileTransport: + def __init__(self, destinations, *, authorize_transfer): + if not callable(authorize_transfer): + raise SafeError("private_transport_not_configured") + self._authorize = authorize_transfer + registry = {} + for entry in destinations: + try: + root = Path(entry.root) + if ( + not root.is_absolute() + or root.resolve() != root + or entry.policy.destination_class != "private" + ): + raise ValueError() + fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + metadata = os.fstat(fd) + if metadata.st_uid != os.getuid() or metadata.st_mode & 0o077: + raise ValueError() + finally: + os.close(fd) + if entry.policy.destination_id in registry: + raise ValueError() + registry[entry.policy.destination_id] = (entry, metadata.st_dev, metadata.st_ino) + except Exception: + raise SafeError("private_destination_unavailable") from None + self._registry = MappingProxyType(registry) + + def deliver(self, exported_json, *, credential, destination_id): + directory, temporary = None, None + try: + if ( + destination_id not in self._registry + or not isinstance(exported_json, str) + or len(exported_json) > 8 * 1024 * 1024 + ): + raise ValueError() + payload = ReviewedExport.model_validate_json(exported_json).model_dump() + entry, device, inode = self._registry[destination_id] + if ( + payload.get("schema_version") != "ocr-reviewed-export/v2" + or payload.get("release_approved") is not True + or payload.get("transmission_authorized") is not False + or payload.get("destination_id") != destination_id + or payload.get("destination_class") != "private" + or payload.get("sensitivity") not in entry.policy.allowed_sensitivities + ): + raise ValueError() + data = exported_json.encode() + artifact_digest = hashlib.sha256(data).hexdigest() + if ( + self._authorize(credential, artifact_digest, payload["sensitivity"], destination_id) + is not True + ): + raise ValueError() + directory = os.open(entry.root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + fcntl.flock(directory, fcntl.LOCK_SH) + metadata = os.fstat(directory) + if ( + not stat.S_ISDIR(metadata.st_mode) + or metadata.st_mode & 0o077 + or metadata.st_uid != os.getuid() + or (metadata.st_dev, metadata.st_ino) != (device, inode) + ): + raise ValueError() + temporary = ".private-transfer-" + uuid.uuid4().hex + fd = os.open( + temporary, + os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, + 0o600, + dir_fd=directory, + ) + with os.fdopen(fd, "wb") as stream: + if stream.write(data) != len(data): + raise ValueError() + stream.flush() + os.fsync(stream.fileno()) + # Disk I/O may outlast a session/ACL/review/destination authorization. + # Recheck immediately before the atomic publication point. A real + # host must serialize revocation/publication if it needs stronger + # cross-store atomic semantics; this callback is not a transaction. + metadata = os.fstat(directory) + if ( + metadata.st_mode & 0o077 + or metadata.st_uid != os.getuid() + or self._authorize( + credential, artifact_digest, payload["sensitivity"], destination_id + ) + is not True + ): + raise ValueError() + os.link( + temporary, + artifact_digest + ".json", + src_dir_fd=directory, + dst_dir_fd=directory, + follow_symlinks=False, + ) + os.fsync(directory) + return dict( + destination_id=destination_id, + artifact_digest=artifact_digest, + sensitivity=payload["sensitivity"], + transport="local-private-file", + automated_release_eligible=False, + ) + except Exception: + raise SafeError("private_transfer_denied_or_failed") from None + finally: + if directory is not None: + if temporary is not None: + try: + os.unlink(temporary, dir_fd=directory) + except OSError: + pass + os.close(directory) diff --git a/ocr/anonymization/provenance.py b/ocr/anonymization/provenance.py new file mode 100644 index 0000000000000000000000000000000000000000..719b5cbfb4a5300bf12a5472bfb7603c42e3b2b1 --- /dev/null +++ b/ocr/anonymization/provenance.py @@ -0,0 +1,56 @@ +"""Canonical configuration commitment; no raw terms/prompts in candidate metadata.""" + +import hashlib +import json +import math + +from ocr.schemas import SafeError + +from .normalization import LINE_WRAP +from .replacement import REPLACEMENT_POLICY_VERSION +from .schemas import LABELS, RecognitionProvenance +from .terminology import TERMINOLOGY_REVISION + + +def provenance(recognizer, terminology, status_policy, placeholders): + try: + prompts = dict(recognizer.prompts) + threshold = recognizer.threshold + if ( + set(prompts) != set(LABELS) + or any( + not isinstance(v, str) or not 1 <= len(v) <= 80 or not v.strip() + for v in prompts.values() + ) + or len(set(prompts.values())) != len(LABELS) + or type(threshold) not in (float, int) + or not math.isfinite(threshold) + or not 0 < threshold < 1 + ): + raise ValueError() + replacements = {k: f"[{k}]" for k in LABELS} if placeholders is None else dict(placeholders) + # replace() validates the complete placeholder syntax independently. + payload = dict( + schema_version="recognition-configuration/v1", + model_revision=recognizer.revision, + threshold=float(threshold), + prompts=prompts, + terminology=dict(version=TERMINOLOGY_REVISION, terms=sorted(terminology.terms)), + status=status_policy.configuration(), + replacement=dict(version=REPLACEMENT_POLICY_VERSION, placeholders=replacements), + normalization=dict( + line_wrap=LINE_WRAP.pattern, + unicode_casefold=True, + offset_mapping="source-codepoint-v1", + ), + contact_policy="contact-patterns-v1", + ) + encoded = json.dumps( + payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False + ).encode() + return RecognitionProvenance( + model_revision=recognizer.revision, + configuration_digest=hashlib.sha256(encoded).hexdigest(), + ) + except Exception: + raise SafeError("invalid_recognition_configuration") from None diff --git a/ocr/anonymization/qualify03d.py b/ocr/anonymization/qualify03d.py new file mode 100644 index 0000000000000000000000000000000000000000..04cae543171ec628cf4202e76bffa67080961564 --- /dev/null +++ b/ocr/anonymization/qualify03d.py @@ -0,0 +1,121 @@ +"""First-result evaluation of the frozen Gate03d synthetic holdout. + +No prompt/threshold search, no remote access, no release authorization. Corpus, +configuration and implementation hashes are checked before model initialization. +An exclusive output is mandatory; the report is saved before acceptance is tested. +""" + +import hashlib +import importlib.metadata +import json +import resource +import statistics +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +from .atomic_output import write_candidate +from .detectors import detect +from .evaluate import score_cases +from .model import MODEL_REVISION +from .recognizer import DEFAULT_PROMPTS, GLiNERRecognizer +from .terminology import DEFAULT_TERMS + + +def verify_freeze(): + root = Path(__file__).parent / "fixtures/holdout03d" + freeze = json.loads((root / "freeze.json").read_bytes()) + data = (root / "cases.json").read_bytes() + repo = Path(__file__).parents[2] + if ( + hashlib.sha256(data).hexdigest() != freeze["sha256"] + or freeze["model_revision"] != MODEL_REVISION + or freeze["prompts"] != DEFAULT_PROMPTS + or freeze["threshold"] != 0.5 + or freeze["religion_terms"] != list(DEFAULT_TERMS) + or any( + hashlib.sha256((repo / name).read_bytes()).hexdigest() != digest + for name, digest in freeze["implementation_sha256"].items() + ) + ): + raise ValueError("holdout_integrity_or_configuration_changed") + cases = json.loads(data) + if len(cases) != freeze["cases"] or len({c["id"] for c in cases}) != len(cases): + raise ValueError("holdout_integrity_failed") + return root, freeze, cases + + +def run(cache, output): + if Path(output).exists(): + raise FileExistsError("first_result_already_exists") + root, freeze, cases = verify_freeze() + denied = [] + + def offline(event, args): + if event in ("socket.connect", "socket.bind", "socket.getaddrinfo"): + denied.append(event) + raise PermissionError("qualification_network_disabled") + + sys.addaudithook(offline) + started = datetime.now(timezone.utc).isoformat() + start = time.perf_counter() + model = GLiNERRecognizer(cache) + load_ms = (time.perf_counter() - start) * 1000 + model.recognize("A synthetic warmup sentence.") + details = [] + evaluation = score_cases(model, cases, case_reports=details) + # Cost of the deterministic layer separately, with no extra model inference. + rule_ms = [] + for case in cases: + start = time.perf_counter() + detect(case["text"]) + rule_ms.append((time.perf_counter() - start) * 1000) + report = dict( + schema_version="gate03d-independent-first-selected/v1", + started_utc=started, + finished_utc=datetime.now(timezone.utc).isoformat(), + code_sha=subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), + freeze_commit=subprocess.check_output( + ["git", "log", "-1", "--format=%H", "--", str(root / "cases.json")], text=True + ).strip(), + corpus_sha256=freeze["sha256"], + implementation_sha256=freeze["implementation_sha256"], + model_revision=MODEL_REVISION, + prompts=model.prompts, + threshold=model.threshold, + religion_terms=list(DEFAULT_TERMS), + runtime_versions={ + k: importlib.metadata.version(k) + for k in ("gliner", "torch", "transformers", "numpy", "pydantic") + }, + loading_ms=load_ms, + evaluation=evaluation, + per_case=details, + deterministic_latency_ms=dict(median=statistics.median(rule_ms), max=max(rule_ms)), + peak_rss_mib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, + denied_network_calls=denied, + automated_release_eligible=False, + ) + write_candidate(output, (json.dumps(report, ensure_ascii=False, indent=2) + "\n").encode()) + return report + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Evaluate frozen Gate03d holdout once") + parser.add_argument("--cache", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + result = run(args.cache, args.output) + print( + json.dumps( + dict( + qualification_passed=result["evaluation"]["qualification_passed"], + automated_release_eligible=False, + ) + ) + ) + raise SystemExit(0 if result["evaluation"]["qualification_passed"] else 1) diff --git a/ocr/anonymization/recognizer.py b/ocr/anonymization/recognizer.py new file mode 100644 index 0000000000000000000000000000000000000000..00ee0fde92aa297b6e85f1d1f0a047f4ff396ba3 --- /dev/null +++ b/ocr/anonymization/recognizer.py @@ -0,0 +1,152 @@ +"""Local native GLiNER runtime. Character offsets always refer to untouched text.""" + +import math +import re +import threading + +from ocr.schemas import SafeError + +from .model import MODEL_REVISION, verify_cache +from .normalization import dehyphenate +from .replacement import validate_entities +from .schemas import LABELS, Entity + +DEFAULT_PROMPTS = { + "PERSON": "person", + # Privacy category includes geopolitical places, matching the existing + # qualification taxonomy (cities, states and named geographic locations). + "CITY": "location", + "RELIGION": "religion", + "EMAIL": "email address", + "PHONE": "phone number", + "ORGANIZATION": "organization", +} + + +class GLiNERRecognizer: + local = True + revision = MODEL_REVISION + categories = LABELS + + def __init__(self, cache, *, device="cpu", threshold=0.5, prompts=None, threads=2): + self._lock = threading.Lock() + try: + self.prompts = dict(DEFAULT_PROMPTS if prompts is None else prompts) + except Exception: + raise SafeError("unsupported_required_categories") from None + if set(self.prompts) != set(LABELS) or ( + any( + not isinstance(v, str) or not v.strip() or len(v) > 80 + for v in self.prompts.values() + ) + or len(set(self.prompts.values())) != len(LABELS) + ): + raise SafeError("unsupported_required_categories") + if ( + type(threshold) not in (int, float) + or not math.isfinite(threshold) + or not 0 < threshold < 1 + ): + raise SafeError("invalid_recognizer_configuration") + if device not in ("cpu", "cuda") or type(threads) is not int or not 1 <= threads <= 16: + raise SafeError("invalid_recognizer_configuration") + self.threshold = threshold + try: + path = verify_cache(cache) + import torch + from gliner import GLiNER + + torch.set_num_threads(threads) + self.model = GLiNER.from_pretrained( + str(path), + local_files_only=True, + map_location=device, + _attn_implementation="eager", + strict=True, + ) + self.model.eval() + except Exception: + raise SafeError("anonymization_model_unavailable") from None + + def recognize(self, text): + if not isinstance(text, str) or len(text) > 2_000_000: + raise SafeError("anonymization_text_limit") + normalized, positions = dehyphenate(text) + predictions = self._recognize_view(normalized) + if normalized == text: + return predictions + mapped = [] + for entity in predictions: + start = positions[entity.start] + end = positions[entity.end - 1] + 1 + mapped.append( + entity.model_copy( + update={ + "start": start, + "end": end, + "text": text[start:end], + "normalization": "dehyphenation-v1", + } + ) + ) + return validate_entities(text, mapped) + + def _recognize_view(self, text): + if not isinstance(text, str) or len(text) > 2_000_000: + raise SafeError("anonymization_text_limit") + # Overlapping windows cover all original characters without normalizing OCR noise. + # Hard bounds prevent silent tokenizer/model truncation on unusual long tokens. + words = list(re.finditer(r"\S+", text)) + if any(len(w[0]) > 256 for w in words): + raise SafeError("anonymization_token_limit") + entities = [] + reverse = {v: k for k, v in self.prompts.items()} + try: + import torch + + with self._lock, torch.inference_mode(): + for start in range(0, len(words), 96): + window = words[start : start + 128] + left, right = window[0].start(), window[-1].end() + chunk = text[left:right] + encoded = self.model.data_processor.transformer_tokenizer(chunk)["input_ids"] + if len(encoded) > 1536: + raise SafeError("anonymization_token_limit") + raw = self.model.predict_entities( + chunk, + list(self.prompts.values()), + threshold=self.threshold, + flat_ner=True, + ) + if not isinstance(raw, list): + raise ValueError() + for item in raw: + if not isinstance(item, dict) or set(item) != { + "start", + "end", + "text", + "label", + "score", + }: + raise ValueError() + # Validate runtime output before adding the original window offset. + local = Entity( + start=item["start"], + end=item["end"], + text=item["text"], + label=reverse[item["label"]], + score=item["score"], + source="multilingual_ner", + model_revision=self.revision, + ) + validate_entities(chunk, [local]) + entities.append( + local.model_copy( + update={"start": local.start + left, "end": local.end + left} + ) + ) + return validate_entities(text, entities) + except SafeError: + raise + except Exception: + raise SafeError("anonymization_inference_failed") from None diff --git a/ocr/anonymization/recovery.py b/ocr/anonymization/recovery.py new file mode 100644 index 0000000000000000000000000000000000000000..8de6a50dbc53cc7ac0dc688c300305d219c6a58e --- /dev/null +++ b/ocr/anonymization/recovery.py @@ -0,0 +1,43 @@ +"""Explicit classified staging recovery; no source-evidence deletion or logging.""" + +import fcntl +import os +import re +import stat +import time + +from ocr.schemas import SafeError + +STAGING = re.compile(r"\.(?:anonymization|private-transfer)-[A-Za-z0-9_-]{6,64}") + + +def recover_staging(directory, *, minimum_age_seconds=3600): + descriptor = None + try: + if type(minimum_age_seconds) is not int or minimum_age_seconds < 60: + raise ValueError() + descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + metadata = os.fstat(descriptor) + if metadata.st_uid != os.getuid() or metadata.st_mode & 0o077: + raise ValueError() + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + removed = 0 + for name in os.listdir(descriptor): + if not STAGING.fullmatch(name): + continue + info = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + if ( + stat.S_ISREG(info.st_mode) + and info.st_uid == os.getuid() + and not info.st_mode & 0o077 + and time.time() - info.st_mtime >= minimum_age_seconds + ): + os.unlink(name, dir_fd=descriptor) + removed += 1 + os.fsync(descriptor) + return dict(removed_staging_files=removed, secure_erasure_guaranteed=False) + except Exception: + raise SafeError("classified_staging_recovery_failed") from None + finally: + if descriptor is not None: + os.close(descriptor) diff --git a/ocr/anonymization/release_policy.py b/ocr/anonymization/release_policy.py new file mode 100644 index 0000000000000000000000000000000000000000..1bb9d530f35a2e5fe94a66a1a90ee202a414cacd --- /dev/null +++ b/ocr/anonymization/release_policy.py @@ -0,0 +1,213 @@ +"""Private, destination-bound, single-use reviewed export. No transmission API. + +The host supplies authenticated review verification, ACL authorization, an +operator-controlled destination registry and atomic persistent replay storage. +There are no permissive defaults or built-in identity-provider claims. +""" + +import hashlib +import hmac +import json +import time +import uuid + +from ocr.schemas import SafeError + +from .schemas import ( + Approval, + CorrectedCandidate, + Destination, + LegacyReleaseCandidate, + ReleaseCandidate, + VerifiedReview, +) + + +def digest(candidate): + try: + if type(candidate) not in (LegacyReleaseCandidate, ReleaseCandidate, CorrectedCandidate): + raise ValueError() + validated = type(candidate).model_validate(candidate) + if not validated.pages or any( + p.original_sensitivity != validated.sensitivity for p in validated.pages + ): + raise ValueError() + return hashlib.sha256(validated.model_dump_json().encode()).hexdigest() + except Exception: + raise SafeError("invalid_release_candidate") from None + + +class ReleaseAuthority: + def __init__( + self, + key: bytes, + authorize, + *, + verify_review, + consume_approval, + destinations, + ttl=900, + correction_policy=None, + legacy_policy=None, + ): + if ( + not isinstance(key, bytes) + or len(key) < 32 + or not all(callable(c) for c in (authorize, verify_review, consume_approval)) + or (correction_policy is not None and not callable(correction_policy)) + or (legacy_policy is not None and not callable(legacy_policy)) + or type(ttl) is not int + or not 1 <= ttl <= 900 + ): + raise SafeError("invalid_release_authority") + try: + entries = tuple(Destination.model_validate(d) for d in destinations) + self.destinations = {d.destination_id: d for d in entries} + if not entries or len(self.destinations) != len(entries): + raise ValueError() + except Exception: + raise SafeError("invalid_release_destinations") from None + self._key, self._authorize = key, authorize + self._verify_review, self._consume = verify_review, consume_approval + self.ttl = ttl + self._correction_policy = correction_policy + self._legacy_policy = legacy_policy + + def _destination(self, identifier, sensitivity, format): + if format not in ("json", "text"): + raise SafeError("unsupported_release_export") + if not isinstance(identifier, str) or identifier not in self.destinations: + raise SafeError("release_destination_not_approved") + destination = self.destinations[identifier] + if sensitivity not in destination.allowed_sensitivities or ( + sensitivity != "D0" and destination.destination_class != "private" + ): + raise SafeError("release_destination_not_approved") + if sensitivity != "D0" and format != "json": + raise SafeError("classified_export_requires_json") + return destination + + def _review(self, actor, review_id, value, level, destination): + try: + # The callback must verify the session/actor and completed review record, + # not simply echo values from the request or accept a completed=True flag. + review = VerifiedReview.model_validate(self._verify_review(actor, review_id)) + now = int(time.time()) + if ( + review.review_id != review_id + or review.candidate_digest != value + or review.sensitivity != level + or review.destination_id != destination.destination_id + or not 0 <= now - review.completed_at < self.ttl + or self._authorize(review, "anonymization:release", destination) is not True + ): + raise ValueError() + return review + except Exception: + raise SafeError("release_not_approved") from None + + def _candidate_policy(self, candidate, review): + try: + if type(candidate) is CorrectedCandidate: + if review.evidence_digest != candidate.correction.original_evidence_digest: + raise ValueError() + policy = self._correction_policy + else: + # v1/v2 do not carry correction history. The operator must + # explicitly attest authoritative schema/source eligibility; + # removing v3 metadata cannot select an implicit allow branch. + policy = self._legacy_policy + if policy is None or policy(candidate, review) is not True: + raise ValueError() + except Exception: + raise SafeError("release_not_approved") from None + + def _signature(self, fields): + payload = json.dumps(fields, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hmac.new(self._key, payload.encode(), hashlib.sha256).hexdigest() + + def approve(self, candidate, *, actor, review_id, destination, format="json"): + value = digest(candidate) + target = self._destination(destination, candidate.sensitivity, format) + review = self._review(actor, review_id, value, candidate.sensitivity, target) + self._candidate_policy(candidate, review) + now = int(time.time()) + fields = dict( + schema_version="ocr-review-approval/v2", + approval_id=uuid.uuid4().hex, + candidate_digest=value, + review=review.model_dump(mode="json"), + destination=target.model_dump(mode="json"), + format=format, + issued_at=now, + expires_at=min(now + self.ttl, review.completed_at + self.ttl), + ) + return Approval.model_validate_json( + json.dumps(fields | {"signature": self._signature(fields)}) + ) + + def export(self, candidate, approval=None, *, actor, destination, format="json"): + value = digest(candidate) + target = self._destination(destination, candidate.sensitivity, format) + try: + approval = Approval.model_validate(approval) + now = int(time.time()) + if ( + approval.candidate_digest != value + or approval.destination != target + or approval.format != format + or not approval.issued_at <= now < approval.expires_at + or approval.expires_at > approval.issued_at + self.ttl + or not hmac.compare_digest( + approval.signature, + self._signature(approval.model_dump(mode="json", exclude={"signature"})), + ) + ): + raise ValueError() + except Exception: + raise SafeError("release_not_approved") from None + review = self._review( + actor, approval.review.review_id, value, candidate.sensitivity, target + ) + self._candidate_policy(candidate, review) + if review != approval.review: + raise SafeError("release_not_approved") + text = "\n\n".join(p.anonymized_text for p in candidate.pages) + output = ( + text + if format == "text" + else json.dumps( + dict( + schema_version="ocr-reviewed-export/v2", + text=text, + sensitivity=candidate.sensitivity, + anonymous=False, + release_review_required=False, + release_approved=True, + candidate_digest=value, + destination_id=target.destination_id, + destination_class=target.destination_class, + transmission_authorized=False, + ), + ensure_ascii=False, + ) + ) + try: + # Consume AFTER serialization, BEFORE returning bytes. A crash after + # consumption burns the approval. Reissue requires authorization again. + if self._consume(approval.approval_id, approval.expires_at) is not True: + raise ValueError() + except Exception: + raise SafeError("release_approval_consumed_or_unavailable") from None + # Persistent consumption can wait for a database lock. Reauthorize + # after it completes, before classified bytes leave this boundary. A + # failed final check burns the nonce; it never rolls the ledger back. + final_review = self._review( + actor, approval.review.review_id, value, candidate.sensitivity, target + ) + self._candidate_policy(candidate, final_review) + if final_review != approval.review or not ( + approval.issued_at <= int(time.time()) < approval.expires_at + ): + raise SafeError("release_not_approved") + return output diff --git a/ocr/anonymization/replacement.py b/ocr/anonymization/replacement.py new file mode 100644 index 0000000000000000000000000000000000000000..a5c4a91076afc7da3c70a5b0791dec16f585ffdf --- /dev/null +++ b/ocr/anonymization/replacement.py @@ -0,0 +1,76 @@ +"""Deterministic source-coordinate replacement; no text normalization.""" + +import re + +from pydantic import ValidationError + +from ocr.schemas import SafeError + +from .schemas import LABELS, Entity, Replacement + +REPLACEMENT_POLICY_VERSION = "enclosing-spans-failclosed-v1" + + +def validate_entities(text, predictions): + try: + if not isinstance(text, str) or not isinstance(predictions, (list, tuple)): + raise ValueError() + if len(text) > 2_000_000 or len(predictions) > 100_000: + raise ValueError() + entities = [Entity.model_validate(p) for p in predictions] + for p in entities: + if not 0 <= p.start < p.end <= len(text) or text[p.start : p.end] != p.text: + raise ValueError() + return entities + except (ValidationError, ValueError, TypeError): + raise SafeError("invalid_entity_spans") from None + + +def resolve_overlaps(entities): + """Deduplicate, prefer enclosing spans; reject crossing spans/conflicting labels. + + Containment preserves coverage. Picking one crossing span could expose the + other span's tail, so ambiguity blocks the entire document instead. + """ + chosen = [] + for p in sorted(entities, key=lambda p: (p.start, -p.end, p.label)): + if chosen and p.start < chosen[-1].end: + previous = chosen[-1] + if p.start == previous.start and p.end == previous.end: + if p.label != previous.label: + raise SafeError("ambiguous_entity_overlap") + # Prefer deterministic evidence; never fabricate a confidence. + if p.source == "deterministic": + chosen[-1] = p + elif p.end > previous.end: + raise SafeError("ambiguous_entity_overlap") + else: + chosen.append(p) + return chosen + + +def replace(text, predictions, placeholders=None): + placeholders = placeholders if placeholders is not None else {k: f"[{k}]" for k in LABELS} + if ( + not isinstance(placeholders, dict) + or set(placeholders) != set(LABELS) + or any( + not isinstance(v, str) or not re.fullmatch(r"\[[A-Z_]{1,40}\]", v) + for v in placeholders.values() + ) + ): + raise SafeError("unsupported_required_categories") + entities = resolve_overlaps(validate_entities(text, predictions)) + chunks, tags, cursor = [], [], 0 + for p in entities: + placeholder = placeholders[p.label] + chunks.extend((text[cursor : p.start], placeholder)) + tags.append( + Replacement( + **p.model_dump(exclude={"text"}), + placeholder=placeholder, + ) + ) + cursor = p.end + chunks.append(text[cursor:]) + return "".join(chunks), tuple(tags) diff --git a/ocr/anonymization/requirements.in b/ocr/anonymization/requirements.in new file mode 100644 index 0000000000000000000000000000000000000000..b9b8b8dd35df58a8b240edda90b3f0d21bed5b7c --- /dev/null +++ b/ocr/anonymization/requirements.in @@ -0,0 +1,10 @@ +# Trusted local environment only. Never add these to the HF OCR worker image. +--extra-index-url https://download.pytorch.org/whl/cpu +# CPU wheel; GPU operators must produce a separate reviewed lock. +torch==2.8.0+cpu +gliner==0.2.29 +transformers==4.57.1 +huggingface-hub==0.36.2 +numpy==2.4.6 +safetensors==0.7.0 +pydantic==2.14.1 diff --git a/ocr/anonymization/requirements.lock b/ocr/anonymization/requirements.lock new file mode 100644 index 0000000000000000000000000000000000000000..d7c900707fa523d65b307e0fcf2aee80a44376a1 --- /dev/null +++ b/ocr/anonymization/requirements.lock @@ -0,0 +1,1018 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --python /workspace/.anonymization-venv/bin/python ocr/anonymization/requirements.in --index-strategy unsafe-best-match --emit-index-url --generate-hashes -o ocr/anonymization/requirements.lock +--index-url https://pypi.org/simple +--extra-index-url https://download.pytorch.org/whl/cpu + +annotated-types==0.8.0 \ + --hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \ + --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 + # via pydantic +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 + # via requests +charset-normalizer==3.5.2 \ + --hash=sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e \ + --hash=sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf \ + --hash=sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5 \ + --hash=sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56 \ + --hash=sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26 \ + --hash=sha256:0891b9d3903c5571c03771ca669a4b0ec5618ca722a5c957d3d29cd4e5062848 \ + --hash=sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718 \ + --hash=sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93 \ + --hash=sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640 \ + --hash=sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3 \ + --hash=sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875 \ + --hash=sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e \ + --hash=sha256:1461ac396c4fdb983a675f20aa555624f0ee18ac83d832b9244ffff3d8055275 \ + --hash=sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204 \ + --hash=sha256:15bb4005af6320d259dc7593ca84a38d7fe06a421dbcf7b910ae23979101e787 \ + --hash=sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234 \ + --hash=sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3 \ + --hash=sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98 \ + --hash=sha256:195c26fb65950f8fce54e26349852b7bdd7c5f120aeefbcc440b8a20faaed4a3 \ + --hash=sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187 \ + --hash=sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d \ + --hash=sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f \ + --hash=sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7 \ + --hash=sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011 \ + --hash=sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f \ + --hash=sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869 \ + --hash=sha256:254eb48b9fa5ee9898a3c445825a1f340fe53712a098904b39b0bddba8ea3cb1 \ + --hash=sha256:2625388c6c754520c37abaf3b41eb34d1cc4a373f457898f08606c8e362b891d \ + --hash=sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847 \ + --hash=sha256:28a15fdad492a99b6eccfaaed66ef3f74050680545ea61ec8b2f4c538f1f1320 \ + --hash=sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9 \ + --hash=sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93 \ + --hash=sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd \ + --hash=sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00 \ + --hash=sha256:2cc961b171b3f3440f410489ab3573e86aea8736134ebbb40ea1338b7f0831bc \ + --hash=sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0 \ + --hash=sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09 \ + --hash=sha256:304d5463e65a35d7bb0850550e0780395395f6fcf452f04db7d5ca7cecc425ac \ + --hash=sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621 \ + --hash=sha256:30fcd120b732aa79317f08dee04d7de0847822e4cf7ee0e9f445bb958832252c \ + --hash=sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8 \ + --hash=sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a \ + --hash=sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51 \ + --hash=sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0 \ + --hash=sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef \ + --hash=sha256:3d14b50de6bf4d0edf857a9386836846f982b8f524e188e2e68b96d702bcf4aa \ + --hash=sha256:3d21b8b13c7592db2ac5e544a6d83187b995257472b0c9e8351b6d507ae37ed6 \ + --hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 \ + --hash=sha256:3ddacd27458c45bdacd6bd6db644bfb730efbf9e830310186e3045c9c5be8fb2 \ + --hash=sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229 \ + --hash=sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e \ + --hash=sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd \ + --hash=sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115 \ + --hash=sha256:447441e76ec720b15e64418d32e092297340387053047c7c694f579efb0ee1d9 \ + --hash=sha256:4495c5002a7b28557e7e222e77e0b661183e432b7d6d2e788101e3f240e05b8c \ + --hash=sha256:44bd4fbb29dfbeba60e7d2bd000c59e4b21ddb3cc53912b14048d37092706d7c \ + --hash=sha256:4685902cf26edf013ed7a3da0f426ebba7a00ebb9541386d835afbf002c11cab \ + --hash=sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253 \ + --hash=sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995 \ + --hash=sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438 \ + --hash=sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0 \ + --hash=sha256:50e3adfb96fc189eb27b1cf62d3b598b89b4bb0420d93a3d3e42e137409011be \ + --hash=sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b \ + --hash=sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7 \ + --hash=sha256:55ea99acb17b9325618de155a0cd6a2e8f5d10be008113e1d433bbb58db543b2 \ + --hash=sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a \ + --hash=sha256:588461c2e8384d309bd63e5826019b6977bc66d629b99ac8737bb795d7b2cb5a \ + --hash=sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a \ + --hash=sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c \ + --hash=sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5 \ + --hash=sha256:59f63901b0031c3136cf64704dcb21de0bbae62ce2c9529bc39d27665463de37 \ + --hash=sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e \ + --hash=sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4 \ + --hash=sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800 \ + --hash=sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055 \ + --hash=sha256:619799369eeef6366ed3e8755a5670f4f2f0fb6b30a0fd7264dc0fdc2357058e \ + --hash=sha256:62588a277bfb59def052abd940703fa35107152bf479781a878617d60faf8fb5 \ + --hash=sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c \ + --hash=sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b \ + --hash=sha256:68eb192d85ab8e5f6ec69c2bc6ac0179fbf04a5ac1569d12fbef74883fe102d0 \ + --hash=sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80 \ + --hash=sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a \ + --hash=sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4 \ + --hash=sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2 \ + --hash=sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58 \ + --hash=sha256:75a3ceed0724d625d64b86ca20aba182e4df462e04c2414fc941c0f523f06aac \ + --hash=sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc \ + --hash=sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639 \ + --hash=sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf \ + --hash=sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d \ + --hash=sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f \ + --hash=sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c \ + --hash=sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc \ + --hash=sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4 \ + --hash=sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253 \ + --hash=sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade \ + --hash=sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858 \ + --hash=sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26 \ + --hash=sha256:87e50a3e7cb90af586b6c5faf23e302a970415ac73bd7bd90a515a04b427ef96 \ + --hash=sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8 \ + --hash=sha256:8a893cc101149f80a653f82062ebc95b34525a2614382e1da5458fe7c6997249 \ + --hash=sha256:8b2bfab86aa71ae13aa41a6a26aab338e0db2b8bc75434b05aea89e011ff35a4 \ + --hash=sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13 \ + --hash=sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1 \ + --hash=sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03 \ + --hash=sha256:93223adc95033dd47133a46ccfc316a0139176fd79085762e27202ec56018f03 \ + --hash=sha256:9373ad13ef0d2c0fb761e04e55bfdee5a08b52cef2c882c8fbe9935b1517152e \ + --hash=sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364 \ + --hash=sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4 \ + --hash=sha256:9bde855991b7e362c146535e3136a50bfaffc0487d38b33ca7e5edefc6e23849 \ + --hash=sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0 \ + --hash=sha256:9cf9b1a857e25c4baceeb3624e92a56df3668f398c4acba74e174d81fb4d1d3a \ + --hash=sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036 \ + --hash=sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3 \ + --hash=sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21 \ + --hash=sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3 \ + --hash=sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e \ + --hash=sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413 \ + --hash=sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21 \ + --hash=sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346 \ + --hash=sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429 \ + --hash=sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685 \ + --hash=sha256:b736353c0a625bbd5fcec108576e2385db3496f4f771f785ff32e108d3c3bc45 \ + --hash=sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f \ + --hash=sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c \ + --hash=sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d \ + --hash=sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad \ + --hash=sha256:bd16aabe4a02a297c23417aa17ac6299dbd8c49f673bcd645b4929b11f5a4400 \ + --hash=sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb \ + --hash=sha256:c6708715abcf3c73b99508253e961a9967f02fe536532834149574eda6de0d1c \ + --hash=sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc \ + --hash=sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c \ + --hash=sha256:c9790464842f85f437dbbb54417eda1e0e6bfc52dd8d22d6fd1c994b73b2dc74 \ + --hash=sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf \ + --hash=sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604 \ + --hash=sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f \ + --hash=sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105 \ + --hash=sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a \ + --hash=sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d \ + --hash=sha256:d4a7319f304a774bed22115bc891618e45f85065ab44ea6acd07d274e750519a \ + --hash=sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1 \ + --hash=sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5 \ + --hash=sha256:d913de495d90407cd859d263bee2e5d1a4ed3eb6573c04e70d9ec619a7cbed7f \ + --hash=sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e \ + --hash=sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709 \ + --hash=sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874 \ + --hash=sha256:ddf19c062bea7a0cc80f519243d2c01dd091be0cf952a0750d4ad576709559f5 \ + --hash=sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc \ + --hash=sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95 \ + --hash=sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd \ + --hash=sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0 \ + --hash=sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d \ + --hash=sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3 \ + --hash=sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c \ + --hash=sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3 \ + --hash=sha256:e80e6c2f55656b4824d72065abb4ddd6a525c74bd78a0aab5d9fc2cf4fb5af50 \ + --hash=sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491 \ + --hash=sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5 \ + --hash=sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5 \ + --hash=sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655 \ + --hash=sha256:ef4fcbf3327382cd4c9f540babd61248208af7b93eec4de397b4d5f58a09e288 \ + --hash=sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd \ + --hash=sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084 \ + --hash=sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d \ + --hash=sha256:f5833ad231be5eb6553de524a70f48d71b2c8563101750531e0b80184e175cd4 \ + --hash=sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915 \ + --hash=sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1 \ + --hash=sha256:fb9e68df06293761f9fe66ade60a9bc6d0f5e42b8acf2939a9158af86ab0e5bd \ + --hash=sha256:fc14a032f813bf5fe624d991960ea83e9715adc27e4c1830a2361eb1d02ac341 \ + --hash=sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424 \ + --hash=sha256:fd1fbe0f116b6e55da77aca2c6ddcddcfac2186cbf78bdebf40fc156efca389d \ + --hash=sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f + # via requests +filelock==4.2.1 \ + --hash=sha256:7be2662e9e826b9bba28afc0f843a123280d799a2ea44bb5d35ccec67905ae44 \ + --hash=sha256:c4c937ccac2a813cee448e662a56366cfb374c0f8702e75f295c412cc1c5d5a3 + # via + # huggingface-hub + # torch + # transformers +fsspec==2026.9.0 \ + --hash=sha256:0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe \ + --hash=sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f + # via + # huggingface-hub + # torch +gliner==0.2.29 \ + --hash=sha256:0c8cfb9f5c2daf7aff329ebb5ab1609052d7ad0aca0ef26049b9476b018b3fde \ + --hash=sha256:39fa8f33c027627e1d6724bff972ab197bf37cea27e401cd3d40f2f95909b0a4 + # via -r ocr/anonymization/requirements.in +hf-xet==1.7.0 \ + --hash=sha256:0a9e802f33bf50c851abe45fc5380e61f959e2d369647d6742b79ad9d6c27cab \ + --hash=sha256:19c0e64f14175ccb6a1aff69e0d2ab9ec5269a560e6687abaf2b3fa4f73de7cd \ + --hash=sha256:2814a6e999d13464c4d679b788cc5d784eb5a4edfc638a31f10e9a11ab531ef8 \ + --hash=sha256:2b7bb5727889b0f2436dbaaad8fc4c3e66b8240d992716989e0c086b4278b1bc \ + --hash=sha256:4ee5e05a627f5ab5bad7a86582277d645556ea1e199903aae19e033a392aa13a \ + --hash=sha256:57bc157b8b7fe3bee9dcb9af7f3da8de41801c3b31a9ef68a77a33c6a6be382f \ + --hash=sha256:59fba37039233c7fcbe196817d6cdcf1b40dfb17b410f229d85b0cf0a1848da4 \ + --hash=sha256:757168feb5679647c0bb13ee5d0faebe799c4dff9051419885a566ebd79f949d \ + --hash=sha256:80f79dae613ce9e0ea1fd1ae15616ca9ac74aed4c770aabc199c4f03ebecc863 \ + --hash=sha256:87dab080f8f7d32781c2586904e3603f4e60d09bfc727706c3ae419e0829beeb \ + --hash=sha256:acc3851cf2576a8fb2ae926da863f4efabe21303cf292e9a44332802ab0dcc6a \ + --hash=sha256:b01fe18dbbd151a2403d2c64ed30dc6547b00d6babab9a617d77c7acdb81ee66 \ + --hash=sha256:b91569d5f1b61c34b043687da02c05dd3604f3d329e7868510bf3f7971599006 \ + --hash=sha256:d406ec79053c0871817f700c2ac8c36ba0d87f9c34b7458b0f0063bb218b0466 \ + --hash=sha256:e3e88a7a75d7d95cbee1f37dc31341d6201124cf21c6c4b1dfab8ccba9b09e0f \ + --hash=sha256:fa029678be1ba7f953c409b0b27bf15cc69cd1c9b3a674fbd78856ebefca1052 \ + --hash=sha256:fcfd6c22418e57dd5b3aea649e813b2e2cfb2aebf317b210d90f1fe4b3018b52 + # via huggingface-hub +huggingface-hub==0.36.2 \ + --hash=sha256:1934304d2fb224f8afa3b87007d58501acfda9215b334eed53072dd5e815ff7a \ + --hash=sha256:48f0c8eac16145dfce371e9d2d7772854a4f591bcb56c9cf548accf531d54270 + # via + # -r ocr/anonymization/requirements.in + # gliner + # tokenizers + # transformers +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via requests +jinja2==3.1.6 \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via torch +markupsafe==3.0.4 \ + --hash=sha256:007e1ffd9bf65bb6ee96df7b258fc632a4868dd5566037986c64781f35a36e98 \ + --hash=sha256:02fa4acbc6a3fc5c693c34d4dd8c1130b7fe99cc915181b0ddd6f72aeb296002 \ + --hash=sha256:03470d1a8268e692ecf79ecd565593e59d44219377a7ead61f1f1b94c1f7ff6b \ + --hash=sha256:04e7902ba80ee4bac1d50a549606527a1dcf0476cd81403db41099d3b60ec653 \ + --hash=sha256:051417f74bcaaefa316276e0ff723f541616ca51043d070da00249d9bddd3e3c \ + --hash=sha256:05295589e619b9bed252a86b532b8e27350abc372d18ba89b59375325e91ec1e \ + --hash=sha256:06de8ef6331f6e822c28d577dc8bf43fe398800477c49498f38fc38b67ff33fc \ + --hash=sha256:0764a13d34cae40db7bbf3a09b7e9b491bf4603e20b263a7a9d6b8e324975d0a \ + --hash=sha256:077293e425f28ec737dbcad442a71752e28f8ae27cde3d68acd1fb212091cd92 \ + --hash=sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f \ + --hash=sha256:0cee7cb0f9a1b6892ea482237d9403b3d1b4603aee057d0ff01f0fac2d019a97 \ + --hash=sha256:0d9c47709875fdb321452056622e930c52afbc07a7d780762fbb8b4d91ce6fa4 \ + --hash=sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7 \ + --hash=sha256:12a606a492de952afcb43b59a14aaaaad120e708d3663dd0fdf2d738d427a691 \ + --hash=sha256:14bd2d845d62ab678eaf81da89d7b621b51756c72346745c1a594c09d49207a2 \ + --hash=sha256:15ba9e28640feef770374b116a6f019c21f52404aeabe516aa7f800587b98cfc \ + --hash=sha256:18a801868a884f216e784d7d14db2a4077143ce7610440aee2ce8f734e7cfcde \ + --hash=sha256:1c0df495a977d10460a94941799c72d5b5ab03d3858d949b55b5a66c8f371c99 \ + --hash=sha256:1caa2fa5a6184fb233153b35f654e6687bd555476f6170f29d8ee9be1a8b0af9 \ + --hash=sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df \ + --hash=sha256:1f1f9477e174582b0a1b583d60b66e1f2cf5d3fe12cee985e4aedf44766600e5 \ + --hash=sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17 \ + --hash=sha256:26e9867520db70d37f7fb421a7f0d8adb40171011fb84ce869afa1a83370dfa8 \ + --hash=sha256:2a6ef68ae94aed8721934072b27a3b654ea2100b97e4ab864cf1489c90926fbc \ + --hash=sha256:2b2b1e18af909b448bb3cf9e3433366f7a8726271fc214e8b10e0f62a78c724b \ + --hash=sha256:2cb3dd71fc6be918ad4264346a8ed69485f9b7ed7bf35495d8e22807cd6b8bea \ + --hash=sha256:2d1b7d9308288661f56672b1b157d75fc536714d3638487bbea17b6318a78248 \ + --hash=sha256:2dad610540cb2e6272855c178f08ae9a1c7ac258a7fb71660553a5f104b42741 \ + --hash=sha256:2e5a7cd7fdd14fcb1ae5d7d8bf23d24fbd1daefd1fbca2580132e1ea75f098b5 \ + --hash=sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6 \ + --hash=sha256:340cbb1957ba99929cbf19a75626d36ba1ae21d1730b287d1cf7f824a20c4fc7 \ + --hash=sha256:34bdde374c5932765d7dc685c4a1d191a3207852d67e8e0a9eb6ea85156181f1 \ + --hash=sha256:353bd63081912ab8cfa6a0c7d185934cdf8426f04c618bba6bc4b394f2069b67 \ + --hash=sha256:387d8cd30e69b3f0a72877b9ae717033396404e19095b17fe89753a981fda44f \ + --hash=sha256:3882fb412298575bae3b9c46868251f15cc69307359f87bb1b382e53d6e5a2c9 \ + --hash=sha256:38fc55594dab834470b6733dead2ee9e3f657fb0608c769dcafa0ba5ab52f45c \ + --hash=sha256:396ec4e65cc889f69786b3b89478b471cee5a3bcf468b9d9bb03e1a30fb291fc \ + --hash=sha256:39dbacefc411633db5b4378b066a9aca70a3d7e2922c9e578d825f844026eeba \ + --hash=sha256:3a93d9616ddecfb393727a0041a562cf0b15a244e20f2bd25efc7949be4c4f17 \ + --hash=sha256:3d23795802fc8bd72534836d64489bbf0f67c088959091bdb22e10735a5107bf \ + --hash=sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6 \ + --hash=sha256:436e3ffc6310d3c41878c601db29098102fe5d8a467c49da4a4125254e0980f2 \ + --hash=sha256:489505b03f692c3f376394e49194fa7a7f9e8558d6e293a7056a0032b0c38163 \ + --hash=sha256:4a540e2d3192792fc84eced57bef37851ccb2b41f73291bb17408eea77bcd278 \ + --hash=sha256:4a7cdc2a420ca01058182da4253329764d4bfa055564d1eced90e6ba1e8b1d3d \ + --hash=sha256:4bced6e2a6dba6a28f7dd3c6ce14df1b2dd495923f16ea484cad03decd463b2b \ + --hash=sha256:4cf3468d5ec187ffffcaca8e61929a37448f215dafc1386a12c750a72fe53634 \ + --hash=sha256:4e2c4809c14559aa7ef426f27fb35afbb38104c349a903bf8f3600456764bb38 \ + --hash=sha256:4ed644d75aa94a2baf7ec3a96eaa160ea58c742eb9d27c6506053c5c40fc84ed \ + --hash=sha256:4f6e0852a0283b1b1fd776eeb7b766a5f440b3e2bd31ab51af3b400585f3965c \ + --hash=sha256:5066b244f576f91afc8ee3ba029a89f99d39c79b1853fe9d39bea9f0afbec148 \ + --hash=sha256:5086f9975abb1ab531ee6afca1761e4b59a19b446f3f6522ed776963228cfe5a \ + --hash=sha256:50b5bedc9ed8a94fc8857a42ef4f84a81ea88f8d4f05dc8705fb23ee6d8dcca7 \ + --hash=sha256:52704c5d36eb6dda8866493decd61111fff86244c9b1ad225ca01b9e91e5970f \ + --hash=sha256:55ffd6ce583d97dc71dc92e930324c8c0d25aea7e3ade6ae54ef77cedb096811 \ + --hash=sha256:569d65055d367e3dcdf30c3f41119467b73d9ee9faf332bdf40402644f5ac08e \ + --hash=sha256:57f9947a7e57a081c1e3e0a2dd0d2dcf290a4531450e6f611e30084c222a7295 \ + --hash=sha256:5989cb26b2e1efc6a42216a9f6b5ee495ce5ace2e5b352a9af489976b32d1ee2 \ + --hash=sha256:5c22873ad1f0532ba40fa1727f3c0fc1bbbaab6d373d4cbe3f0dc74b2e2521c7 \ + --hash=sha256:5e8b3d0b18fd623afa12ecb2ce8d8becef69f9b5440c6330c7972200e0bb84b0 \ + --hash=sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6 \ + --hash=sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed \ + --hash=sha256:6669c1bf34080161ce49c589cc512ef24d4c704ac9d2b2d3667f519c60418378 \ + --hash=sha256:672d207103e6b16ca098611b0f9efad6bc00afd47c03d6ef62186495ca677dc0 \ + --hash=sha256:6768d67d1bce64270e0fdc2e69309d68b9b18ae56ddf6c711d168e9d051c2cac \ + --hash=sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b \ + --hash=sha256:6bd9e1788e15bfcf6a9082de42e30387e7b85d211ab21e57a939bb8cfaaf8d96 \ + --hash=sha256:6d2a9efe686f9de00d0d1ea32a4a5a86d558a2277501bd78d964214eab625e59 \ + --hash=sha256:6da83a088f8ef93b2d483a8232a4dbf4d69d3d8496b568a03c56becac43e1808 \ + --hash=sha256:7018d4af1cd272e847aa5917983ab5e83e4f6579f9dbfecd4a79c0ca80b144c2 \ + --hash=sha256:71f88e749ea29f67f21f3b36433c1dc54c7729ed2a6d9e2da2e0d9e0d7b224eb \ + --hash=sha256:737c9c3981998eba27f11786f84fddcbabc74068b72a4a1f454ea02094b57b65 \ + --hash=sha256:73e77980c7207854f00fc4e71fb1626868d5740ab4012623d55c7a99ad122a72 \ + --hash=sha256:799c39bdf5e2f1292fedd3009f7b3c9e760f10b2420cb9638d56920840ff6db8 \ + --hash=sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e \ + --hash=sha256:7d3391b2188d18737cb2fa147028b1096236eaa7e156446c650a489fa2cadc91 \ + --hash=sha256:7e1636da3d8dfc220b6dd10264db5f2b165e4888c4518594898fbe381049af8a \ + --hash=sha256:805c8b84534fa10891890f0e4be39f3a99e94615d93e8836bf9fa1fdca2feeb2 \ + --hash=sha256:811d02d5122171c1941357efd8f9bf4ffe907b7f0a1a4e729a880e4be3f46e3e \ + --hash=sha256:8138eb83940ec7299024d92d4dee45f601b9e6c5ffde9d25f4e35e326203c707 \ + --hash=sha256:83b3944fea42a8400edf92fd1770fb8d0d4f7de651353bd2d8525a92dba69a21 \ + --hash=sha256:849dd2bb0e5e4ab2b71c7191726a4a8d5aa8a610daa584728cbee0b710ddc4ef \ + --hash=sha256:8698d70a8081ee8c090dbb394768b5789a1da8b131b5499f89d071dd3cfaf6be \ + --hash=sha256:8781a792a070cf2bd1b86d3aa943894115faaba6e88122a7bf32d62072742453 \ + --hash=sha256:88d59b473bfb03259722600839af9bbd7fa13a2eb514beefeedb95997882f69a \ + --hash=sha256:8909c2f1c6dd65e054ac4b573a91c8384d1492281e55d82d159d653f7a13adf6 \ + --hash=sha256:8965520ac587c94a4ac48b729be3d8b8de00af39699b17585dfb599babe77977 \ + --hash=sha256:8b5d563170ff8ba3181caa967c99a3c804d1dedb702c7cb93a6a7c32247da978 \ + --hash=sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581 \ + --hash=sha256:8f0fac8b13d14bb06c68195f849371924ae53dd7b1c00fed24650f704383b692 \ + --hash=sha256:9240187afb63d2f9ddc3e032c670356fe941f6e20662ea168a5dc3f1f317e1b3 \ + --hash=sha256:925f929d6b59a8b3f8b8c6ac363cd0af7eecc81efb3071770b3c6717c450a369 \ + --hash=sha256:9348cbb300d224fe3b89793262cb093504d4ae927004468463f745188a193e4a \ + --hash=sha256:9388003072b95f2f1e3fd908604194d653ba21330d811961a78b7da1a77e9e36 \ + --hash=sha256:9438a2648b2195980cb2dd8e53ed7b8df91319e2d0b70ae61a9e1d1bc8d3bec9 \ + --hash=sha256:94e4c421742086aeee4c32a506eec8859d7634aad943f7e6aacf70f813478768 \ + --hash=sha256:94f5407f7bc64fa6463906b896f9904beeeb7dd8dc116ee8e9056c8714ff9916 \ + --hash=sha256:971a3bbb75d97ae4e2e8f7d4834236f86f85f0c85e04ab2e191db1123b04f80b \ + --hash=sha256:9e227f3dbe6bde7491cf0a9965d00b88c6b1a4a95d11480ddf88bb96d397c19f \ + --hash=sha256:9e25feb9e330b63edb0278a0acdf85e50d0cb0fbf49c3084abbe4e24ae195346 \ + --hash=sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c \ + --hash=sha256:a18f38cafc329bac5e3c2b96c765b4c96d3d103421ed22ab7988c1e3fce27464 \ + --hash=sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9 \ + --hash=sha256:a5fcffb37e602b0b3c1638a97746b9b96125caa9bcf6fa41d337a9261de231ee \ + --hash=sha256:a8e9f292fcda89b324f2f5c91d13f1424a153e40fc2756f38ee23b15835ff300 \ + --hash=sha256:a9f54054101545a9a9cccefddf54316aa6e4491611fcbef9e91b3b6bebec04f6 \ + --hash=sha256:aa2c838cc024642cc04c6854232f32b43e5e22833dd11119c1766c7873b8370d \ + --hash=sha256:ac0c7c9f1609b0c4c114feb1d7a3409564c7fb77e360bed9e97e5d25dfeaf868 \ + --hash=sha256:add96447a86d205ab616665d53b2950ee81083757f56e6ea833c8b2917646b46 \ + --hash=sha256:ae9dcb8fbe244cb82f8a6458b455b927a03685e383d9bacf1ea5ce180b96dc97 \ + --hash=sha256:b4a635a0487774f841cb1fb62e907e7195cc95bc761e053184b8acc3ceb20733 \ + --hash=sha256:b4d12837e0203bbace818ff4a7461afdcd78bcd782351cea148139180d7bcffe \ + --hash=sha256:b61687d0828e72bf5cda24a2690188f37170bd31c9359ac97e4e66569f120a16 \ + --hash=sha256:b807e598953730f82e4eae3bd30f6a122cf6b31c398c6b504c0e04c13c170429 \ + --hash=sha256:b8cd1f918b26fd7b1832ece557cc18f2d8747309ff8b3f0ef9d4250c5ad67a39 \ + --hash=sha256:b91cc9d336957239ff200f30097e6fea2dc6d6fb3c81e853eaa09eac904fd894 \ + --hash=sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c \ + --hash=sha256:be6cb0c799abb0e2ba3e618e6d28ddddf7e485f6c2ce938dfa237daf3905072c \ + --hash=sha256:befb4158af32106b9a93db8d6d1d1cbbd418c0d5aca0cabb7b1780abf0c89169 \ + --hash=sha256:bf053da3c97a4bc5ecfbb218cdd2983febd91c617be8367d139882aa11e490aa \ + --hash=sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77 \ + --hash=sha256:c1bc67752d5f21013cfe430df4062441714eab79f65a6a05e01505957e9c35fe \ + --hash=sha256:c61750fadcd119d0825bcb7d7d675dd264dcc89cc05292aab5be68ebdbb374ad \ + --hash=sha256:c90d5b3d4e944e065a301d741b3c1d784f6bd1f503aa68b4967e32b2ba313d85 \ + --hash=sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e \ + --hash=sha256:cb96e6e088d6cf71c1ea977510948320234824cf226e32f6f6e044f7a9c82b34 \ + --hash=sha256:cf63c214fe879a65e69a386f915e36104fc84254ab141240f8854602d8e0be2a \ + --hash=sha256:d1aca03ede943eb80ab3d63bb082c84b7aab85ea83bd0fd0c200260945fb49d9 \ + --hash=sha256:d2e56fd3b00222722abfb3f5f0759ddbae4b90811b5ad4343c64030ad1bde70c \ + --hash=sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749 \ + --hash=sha256:d882a373d8093c2941e01291b7ced96e9cbe4781da9a7751ca7e6c70385e5214 \ + --hash=sha256:d920abdfa61279ba1a2ef9484aab07bf03331f8c08a10120fa332353d06e6932 \ + --hash=sha256:da2af0d7aebfc2074080d72efa6ab8317c62481ef1f896f65d9999c1c01f4494 \ + --hash=sha256:dd8ea6ebee7aedbf7c749fa80521d9ccf1ba473e0d1e14805caafbaad281c889 \ + --hash=sha256:de8b364c423ef0a4bad9069657d617f9a5d2b2062457a89b1fa16ee199c399c1 \ + --hash=sha256:df1ae86ff54725a01fa1a0510b914ca53a161b7050be74f6204e24aded5971d0 \ + --hash=sha256:dff05cb7016dff1e9fd68f4122c127b65dfc59de5306cfb7ad92f956f230bee2 \ + --hash=sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786 \ + --hash=sha256:e49fb0d1ce92cfa0cb198cc5b1b11cdf9d0638658e2a2db2687e39db7c87fc78 \ + --hash=sha256:e5c802729725bd07e2bc3ab7b76dc7e0bbfc53129d8f1eb1c002c24cf774717e \ + --hash=sha256:e841068dc0be4cb6dfb5c890eb88cbdcff2f4a332393c7ec94e8e618bd32c1a8 \ + --hash=sha256:e916035e3e9930cbdfdd10abf48861340221857f45509565898e012263f7b289 \ + --hash=sha256:eba154571c16e032112afac0dc2dfe9e63c2ceb7aedd07bb7eecf2ce26d4dd4c \ + --hash=sha256:f03460ff076f70ab595bb45a0205ccea1971443575b6920c52e755dec2b3fbfe \ + --hash=sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237 \ + --hash=sha256:f291bcf42ae98eb5107edb162c3c998b4a89648fd8e99ed4cbd12705292788cd \ + --hash=sha256:f61efe1d2fe0de16158a5fe1d1cf3c14bdb6aecd54d8938fd26512c525c1f624 \ + --hash=sha256:f68edfc67aabac33708941f26f22a7b8e9f81429bc0cf249fcf7d66b23af8d19 \ + --hash=sha256:fa95848c929b6a75f6848d3c9793e59db365ee436776e57db835cdbfa79ba977 \ + --hash=sha256:fd9f8797427910198f95bced71ddfed61130d7e349213bfb8466c9c99e2c46a8 \ + --hash=sha256:fdb4ca07ab75ffadab4a8b135ad59cdbb3156b99310f3d565370da74a15d6bd3 + # via jinja2 +mpmath==1.3.0 \ + --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ + --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c + # via sympy +networkx==3.6.1 \ + --hash=sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509 \ + --hash=sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762 + # via torch +numpy==2.4.6 \ + --hash=sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1 \ + --hash=sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4 \ + --hash=sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f \ + --hash=sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079 \ + --hash=sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096 \ + --hash=sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47 \ + --hash=sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66 \ + --hash=sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d \ + --hash=sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1 \ + --hash=sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e \ + --hash=sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147 \ + --hash=sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd \ + --hash=sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75 \ + --hash=sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063 \ + --hash=sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73 \ + --hash=sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab \ + --hash=sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4 \ + --hash=sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41 \ + --hash=sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402 \ + --hash=sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698 \ + --hash=sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7 \ + --hash=sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8 \ + --hash=sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b \ + --hash=sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8 \ + --hash=sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0 \ + --hash=sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662 \ + --hash=sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91 \ + --hash=sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0 \ + --hash=sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f \ + --hash=sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3 \ + --hash=sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f \ + --hash=sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67 \ + --hash=sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6 \ + --hash=sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997 \ + --hash=sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b \ + --hash=sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e \ + --hash=sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538 \ + --hash=sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627 \ + --hash=sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93 \ + --hash=sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02 \ + --hash=sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853 \ + --hash=sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c \ + --hash=sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43 \ + --hash=sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd \ + --hash=sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8 \ + --hash=sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089 \ + --hash=sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778 \ + --hash=sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1 \ + --hash=sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb \ + --hash=sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261 \ + --hash=sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb \ + --hash=sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a \ + --hash=sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8 \ + --hash=sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359 \ + --hash=sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5 \ + --hash=sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7 \ + --hash=sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751 \ + --hash=sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8 \ + --hash=sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605 \ + --hash=sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e \ + --hash=sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45 \ + --hash=sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2 \ + --hash=sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895 \ + --hash=sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe \ + --hash=sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb \ + --hash=sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a \ + --hash=sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577 \ + --hash=sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d \ + --hash=sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a \ + --hash=sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda \ + --hash=sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6 \ + --hash=sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20 + # via + # -r ocr/anonymization/requirements.in + # gliner + # transformers +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # gliner + # huggingface-hub + # transformers +pydantic==2.14.1 \ + --hash=sha256:9195d967ec791692a04438115466764fb8b9a27b31f14a760437694f40d6b454 \ + --hash=sha256:94f478203dd03404682a1ada216965651dd74b1d2d5ffd62e00e0837caab5c26 + # via -r ocr/anonymization/requirements.in +pydantic-core==2.50.1 \ + --hash=sha256:0036473f5583e6a60e50b8b21651511564277a3f05cc5dab8cf579f552cd5f6c \ + --hash=sha256:0048b6dddc8ef4b64fccaad878bd143b0c3882ea9936279dc11d613f6b7dd1bc \ + --hash=sha256:009634b83993777ddcd69cad0ffcace43dabde692109528e35f0fde91e386a8b \ + --hash=sha256:028e2f212273d4a39b1ec1e0de8166b1165a65fc0f1111452a9d94fc7c625c63 \ + --hash=sha256:062e891facce5ca296a1c37098e5e466780457f86413894b399f0cf22934f769 \ + --hash=sha256:06e01fbbfdb9be777b316a71b6c49efaf4a08b615d0a98d678cda3023f79d019 \ + --hash=sha256:06ead20d39ffd6f2f6f2a8f8a6de67ff8bb1b4f14a8a30e058502514ee2ac685 \ + --hash=sha256:0b3a6f334c6a2345ca15318ff894502a90012536404b37c844a976c76c846e0b \ + --hash=sha256:0c003c3b7f49debb893d2d85ae099ac5959c9839e2f330fadb1fcdf7a6594482 \ + --hash=sha256:131059670f1d2444269b8585cb888963994871932447c08b39ac6a51fcfef658 \ + --hash=sha256:132529c83901437ff642f585216831bf5fd7a91df66829907e155192ead62498 \ + --hash=sha256:17e722e156d0444ecaefbe640bdb60928752bf2013e2b7a11cdb099aaae19bec \ + --hash=sha256:1a9006395dece0e32e704c315eff8a00bede494f6108546cfc5539c89fef4f9a \ + --hash=sha256:1c8632d4ac04e6f91128fca584b3a8a507d81604c24eeaaad00d4be42765c32b \ + --hash=sha256:1c96fd793b73d1b92e65570132505498fe7b21eaef73cdf74e67e5dfba7ac9e4 \ + --hash=sha256:1cf41f1ae3fa155cf167a72689ad044bcc1e3c97e064123677149bdfb5dafc4a \ + --hash=sha256:1deeacb112d14d3f4fcb16b165f7dbaf76c70ba6e82f37ba042bdab51970a0b8 \ + --hash=sha256:1ef800dd7d85bcdadf4c3076e4c94e43939493558a3b69a1ea830c706d4617bb \ + --hash=sha256:1fa4c8bc12c1354c5550c0c35c1852c8c1901e89e06561724e03f8d0342e1f87 \ + --hash=sha256:2005207aafe1231315718bf6ed5d064a7300fb4772754af35ee72fc68159492e \ + --hash=sha256:23923ab9292c40da026330b1ecf4dc2618c8e86e0422e5d1fbf50d94d64ca4f8 \ + --hash=sha256:23edad659e8dbd8ca7e4e877fe6c81573abbdf215bd25a68b53e1272f58b80c7 \ + --hash=sha256:2ab756b72bd5054e4c7ef3ded331b35786cbd3cf931531a508f79a9537517064 \ + --hash=sha256:2cbd1b75b09e976ed0d6b6ca297675632ca35df86130088457cdc60ef36970ae \ + --hash=sha256:2cf91809d0721ab81592ba67bea7694821679c10b1a2e3c3460082b286c1918a \ + --hash=sha256:2df1ff41884de2bc4b307bafd7c40a691094fad2ff8e767e5b45a319257bcf4e \ + --hash=sha256:2eb75304506894a281d346220a4f7481a1b8729577c5ed2a05395991966a8396 \ + --hash=sha256:2eedf82ee4753cdab8e50044c6bd569577eebc3859b11fecf4eb9223761ff966 \ + --hash=sha256:30ddf019d082c117b5d309e5b86710c2a78909907ec1a9381feec3eec02eca0b \ + --hash=sha256:325c23f3e35cfbf0fe3486fa5f7260d1e45885173002d30a28ca019994124255 \ + --hash=sha256:32fad3a91e51b6d2039c572db04a5a873260b399f6bd62c3552671fa7a4a2899 \ + --hash=sha256:36c426eac0af8d1529ff8467e612b933346caec1fdc0d774f78f67a1a11e16c1 \ + --hash=sha256:3a5fce22f1e87d181e924e12da7d81cfe031fb3881a5ddf26ad28f141756ca43 \ + --hash=sha256:3aa9de446b793de2beb6fa2d9d0961803126c4e2a99c2f25ab59b9fd6ea125c0 \ + --hash=sha256:3e46a9eb0a0901dd6275e6b06ac3a464885ef350ec4121fe486869de8053e4bb \ + --hash=sha256:3fde4fdc6487a58d944ca87cf5adc95d5f266e872c19599f5f4c0a8a1b1f9f9f \ + --hash=sha256:409e0ea40ec30d9158f33574fd758e689f6045a0f2596701828c27816ca9687d \ + --hash=sha256:40f523349960fa30f3ea51404308ff50f9997a90df639590f47a057c1f32b415 \ + --hash=sha256:41bc8237121bd8dc8d888dfd6279fc166ffc88c1f1bf3a8bf00869680533ca4c \ + --hash=sha256:42b54c2c90ad348b5e3a85e03e715d572c1fde357ef104cdfe3b03b697a404ea \ + --hash=sha256:455a773617b5913bf5c20d0692e5787b119e52c4d40ea644ca31f5758fd31be2 \ + --hash=sha256:45b11cac094aa25725581d9304eee93c9028516b9ea80dd9e175e13a5a2c840e \ + --hash=sha256:45c6266d071c241f2a168d45bf8c54344f0effce35e7e6b73afdec11f3687568 \ + --hash=sha256:46b3301d3b5c886f77de7546e47274a5842c622ea2020b8c6524c6b66913b4a6 \ + --hash=sha256:476f6ed8e43cd1e0b460920e23571700872b284e77331cb30c4faf459cf48a4b \ + --hash=sha256:48569b0ade9edfbe065cad1d700175546592aebbb42f02adcebcc26e75b896fe \ + --hash=sha256:49c2cbb2397fe4d0987e84606e691af6cb87bc0ee1bd3e7b737f7e10b4c142f9 \ + --hash=sha256:4a53d13cdfbedbfa87f08b83c1a0a5efcc767d785a4b41934fa9cb672670493a \ + --hash=sha256:4be846f55c9477f5f3ddde8f2ce941137e16862a56d018ed885d422bb6ae02f2 \ + --hash=sha256:4df197990c15b5a37c5a277d131d9f2c67de6133f2e5dafd80d9bba4b99f46f9 \ + --hash=sha256:4e834f6a8e4ff772dcc34f58ef5504147a3ea5b0f4eeb13b0f8eb2ca75ac57f1 \ + --hash=sha256:57f51b31ff826e2859120cf4737c5a758a48d96f3e97da40ccee1796d58078ff \ + --hash=sha256:5958c72adb417c39b12ac87525ac60b0d73315fcdc59e21f44ee4a5e2512c9ef \ + --hash=sha256:5dfe41f232befddb9c4377f6cfc702b51595e2d78ed082672adf8758d2c4619f \ + --hash=sha256:5f3cae32fc46121f787cb2486de9cf95a8bf72aec5cc78f64c606fa1735a6ef5 \ + --hash=sha256:64f6047f62a6c5ae08d0a6afb035667aa2d97c3d20d69762e034c5ea144d92a5 \ + --hash=sha256:6a733778df2f7087ec1100ed0b41533e4f3001976e99570fa34f57c66e7f8e3e \ + --hash=sha256:6b20a4bffabdad0db2927ac034ae3b8a681b1f7a0182f3e60b479ad2fde21ebb \ + --hash=sha256:6dbcbee53bf17196a7f745aa9bf5a9603953a1e365b1f020be3207c676a3e7c4 \ + --hash=sha256:6ed4f3cef55164b026fefb41341b7754cc6b624c75dfe7142d2ecceb5ad21c87 \ + --hash=sha256:704075d10b74f2f3c6e15407c696d88701df35fc8953f434a431add0d0074db0 \ + --hash=sha256:739dc730e6be3bd5ec2f4ab5cfc7eb047cc45fc1497b3bafec74ff2ed07df597 \ + --hash=sha256:7456d699b13954e9c0164dcb267250a10ae0dfb03e6e26d6796ab0d46e189c84 \ + --hash=sha256:756d669f04e62ec4148ecfe22be6a4484d9b1181a6ef32e205ebfd200540858b \ + --hash=sha256:7689580e72a642ab5ec64d5f55b2e33636fa43b4ebe63c0c2c965ef307c7d1aa \ + --hash=sha256:76e2e83fa6ec8cdc972d438dafc2522b3a47bee4ec0ae668b29cfb1977ab5242 \ + --hash=sha256:7816e98acc08119dc0f340ab167048ecc54126316330c1f0caf7c6756c88e28f \ + --hash=sha256:79490e33c4c0fcb933bbbcfc3a62184d8803b99f535863dfbb925e1bcb6945ad \ + --hash=sha256:7f476456ac2bb0d937f75191494a09c83a30765fea4f70f3b404942fe25f6cdf \ + --hash=sha256:844b869f118e22a41a091bdcedda8a71bc1b0f62c38d1a0c3211cece47e1d8fc \ + --hash=sha256:84bc765b282a9d5b7fe0348b8648904f25a6a04b2139da52b1dd30c8ac3a2c8f \ + --hash=sha256:84f34323a61a365b4e9295de6028474754829aaddd59c7bf1a040e7487ef8f3c \ + --hash=sha256:8812592c85d0edf423f10eadcef42716d71e8219085ad9e85b775057b7306133 \ + --hash=sha256:88e492e8b9d0312e7dc13667c30222abf284dc3b79b5302b3607b41a5784ce61 \ + --hash=sha256:8a6791afa2245e6c6b180122d105941644f5bd410bb18623b408808cc41a3102 \ + --hash=sha256:8b4c3df25bd323bf1d36a648d563cf1fc69d717451569927151bdad7cad07a77 \ + --hash=sha256:8daa7ee75245d43ad7d747e5c9ecc1b1d06552f72b14887e9276f787d57375f4 \ + --hash=sha256:93ba4e9d8210d941c200431a56b2c0400b131865947903937ed3ec5404307d2e \ + --hash=sha256:94845ff54dc5193f228cab81b2662a04bfbb892e95bdc15edf7399000ce57d54 \ + --hash=sha256:94be440c03fede26969a5ce75468e0e6a9927a1b46d9b679ee8adc1b057b0350 \ + --hash=sha256:9572c1369e9c9da2d64a7b7992c786d90ff295abc93964cfe3125e4290768070 \ + --hash=sha256:983a662de2571cb2502fc8ff47b6770b03d025d2eb314c92f77b3f07c74720ed \ + --hash=sha256:992c3514ec891fa7858099183e4d64e6bd5a5d4ff452fae29df22faa77a006bb \ + --hash=sha256:99ba9bc2b8062ea0c326a990f7f00e6530c23579de66dd246e72c4cafef950a5 \ + --hash=sha256:9d1bed94af6a63835461f3cf7502058eb166c58c4778e11d0f433cfb1bd69e19 \ + --hash=sha256:9e4472072de0137ee0d8e72d6620e85939c271d2f90f6bbb4b15c24638b79f92 \ + --hash=sha256:a27c09d86600f1bf2fe3f37e1ae697faf3143931c09322cd799da94deee923b5 \ + --hash=sha256:a29a061fec0b4e2d714f277e70a3a18125ecff803f2fea6eade2f2e53711d112 \ + --hash=sha256:a3cda0e538208e5d722bbf3698b24f19c0a7d05bc8d5f8a7f9b121ea7fa243d9 \ + --hash=sha256:a44101320cfe99432db74237545a63057dc7a88dfe792cbcad0647f2af56cb81 \ + --hash=sha256:a4aaaa791bdae1c972a7e81765f4f3571c926b8e0b9b6e47346499fb80079665 \ + --hash=sha256:a51eee75939cf811ac09b278745a6cee7dc873ccfbc8b9af3cc88fe4b7ce25b5 \ + --hash=sha256:a7c58106de36ac6a56314182958de20db8d3a29dfd5db527192cc754e4f8e7fb \ + --hash=sha256:aa8224f10880d9bf1b5993988ba153d42a8b4f3f4f511f93b1f09c93ff613c72 \ + --hash=sha256:acbf31f37c53a5ac0c34706c80b4f5107ba20b05fdd3816124bf236ef0c57dd2 \ + --hash=sha256:adc06d218a1cadfd2ec4628424d7d79ce4eba69c2965e7e7b55106f0da5208c8 \ + --hash=sha256:ae28183297fb0d2b8dc46a1f01d51f5e45825fc5afe76a835a6cb7fb34821295 \ + --hash=sha256:b0135bcdcaa0f23573f286e4cb5e0fd2962700964ed13df085b85f2b97aeab9e \ + --hash=sha256:b087b1c5be7ac687cf22eabfe4b6b608d40df23610651e93611e1f49118baf84 \ + --hash=sha256:b0d955195bbbe489ad343fcc956eacea9357b79cb22192c66cacdefcbc14b32f \ + --hash=sha256:b281a3b0f0822618fe5e3e0d8a2048b6356b14388505dc9374ccffeb69989713 \ + --hash=sha256:b6d0c2183008c188e19f4906d426b293bdc4f67ab17df8e180fe16cda208fa71 \ + --hash=sha256:bbce99252ba3167b2b6277f1829d5bf4b43b754524bddf7f944707c3db7d2253 \ + --hash=sha256:bc1f08f68dac9f9e83845a8039880aba2ab553eb9b2259c3243a313182c253fe \ + --hash=sha256:bc94f474417604bd383d2cd445d071b07dd55fedceed3ce33407bf1fcc107290 \ + --hash=sha256:bed5163e03b98bc1fa2eb05d74c63d9c5c95d8ed6254985481640fbf5e237dea \ + --hash=sha256:c17799a62c142d61b8a3c51752a7cbc87fe2ad4ccfab10e628a77b405075c662 \ + --hash=sha256:c18db21573bd2c6489f9a544b7499f0df2853958c568e5e783536ee1f690af41 \ + --hash=sha256:c3ede305158e75510be50869b319550ab072008c13d64d4ab1e094fb286b6f44 \ + --hash=sha256:c516cc5367ca3448995d42cb994bf3f4c9002d2a7c22eac9622551269ad1b807 \ + --hash=sha256:c531166c42ea7bdfecc8c50049581f05dd1993b09cc7c52bb36a14e96deaec7d \ + --hash=sha256:c73622ef819328873b53109ee4f77ceb598bffedd02daf916102be3228866b78 \ + --hash=sha256:c8dce1f1e0e5358b682a6ad3fa5e31b31d4560997b8e61417e9217c8d60f8a0c \ + --hash=sha256:cb57f304525a5e3c13333b772bf9a473f36326e9c821b2e8e1b2fd36f80ae2c3 \ + --hash=sha256:ce8c25ca38cc0e3d7753ba180808de2c0c8cb24eae0df64491e40921454e9831 \ + --hash=sha256:ceff0acc940be2715bd6ad17b24c0e5304abf44f6efd0f81ee8499e640f9dc86 \ + --hash=sha256:cf356f70551d40374eaffb1aa63f1eb6d2006681cbd7a9faea173ce0f4dd7cd2 \ + --hash=sha256:d2d82aa62521c55ddfb000ae70f88cdd8de974078f6024e821dfe5addd0c818f \ + --hash=sha256:d32f3acc081cc3923386d88f422cde8892335e95f034e0104bb4cf9310d9915f \ + --hash=sha256:d4193206b6587047437f6f11d7e776df23e1c1e23af2a54d9347275614791e10 \ + --hash=sha256:d5c0e32fdbce7f1e8ef4d11f655694bf5f4175c757a9f1dc2be09b8864e5bcf5 \ + --hash=sha256:d5e062c01286d861fd6a1c4ff6e063547b3e713067f2df033c0ff97ac2ca006b \ + --hash=sha256:d8f9e8a6c4ab04b78d61f78627370d834eb004b2869dcb28cfffa647b4ea1980 \ + --hash=sha256:d939de9c82e2126f7f48a7e658f8a85ed46d57662d53f44c49b8895fe94a3eb7 \ + --hash=sha256:de531ce1e2a3364e8767878b58f4ff728a434b4fde089781fe30b1e08e2396e0 \ + --hash=sha256:e50d7b94baac6c7d09927fa5ca5800a0c7ee5015c7fcff65beb3a1931b5a6e09 \ + --hash=sha256:e5faeaee74a57d32b3ab3aebad2e348f06d3ba946fc5d28c1728455f00a3d13a \ + --hash=sha256:e6f0cc1bb9900dc558960894adeb30b0c083366fc1d69b856209fb2ca5c36fe5 \ + --hash=sha256:e8e1d6ce820aa23317e8209a86bd65a540973c12dc7552b48a4f6c8e9926815e \ + --hash=sha256:ed1e728b39a383c81035b2459cfcb35d99dfb01f7d6ebe3a913bc1cc5b81e459 \ + --hash=sha256:ee6db2fbed51a7991302e8fac498cd67e336246026d0dfa84cf5166ce1412760 \ + --hash=sha256:efbecf43d321f7b9281441f1f213f7c21c66988b0e06c2730ba13ed47a46bb08 \ + --hash=sha256:f2c634642694e6a0dad2ab1d375589fa671fd442edd5caf7d9737b8f6ca22906 \ + --hash=sha256:f3377c8c2b3ce898423c5e5dd94c7982e30aa7717a7e6ab2470b9de364963709 \ + --hash=sha256:f5187624823423e1d1b82b1072ac41dc837389e18d3d0572cc19bbee46cd550a \ + --hash=sha256:f77ac30b19221cd9bd3fcfa3d4614eff93140d0572ab730cded17b64adca05f3 \ + --hash=sha256:fe90228920fd8ff2be62622b6bb8a2b11acd65046d50c6b130614b5879605a20 + # via pydantic +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # huggingface-hub + # transformers +regex==2026.9.29 \ + --hash=sha256:01000ddf0e3ffef97f2413ceb514f6313040106b6d18a03ee00a4fe35c1eb1db \ + --hash=sha256:0166844493626c5015c6088ee15c9ca2fd060ca15b7641d1657da6a58432ae33 \ + --hash=sha256:044265d77d94f5e3cb2fd72c76723807c429cb8c533e9d4672d0334a6f14f588 \ + --hash=sha256:0476e5bcbe6e1ba3d1c4cc7bbb1c3ba78e3b979b5c8a88d0a6a8cdd4992b8c84 \ + --hash=sha256:066d0e3dbfdd739bce2bf8c2a41dd16f73e3d8adc2eb06dd803a36a307f56075 \ + --hash=sha256:0b65c72739f981377c9c22e0c5c3cd7f42da7bd8a3c9209330fac772c7d893ed \ + --hash=sha256:0c992c19cd45058a4b92f68f139c93db168b48fb1f322c9a7cd620806afb6b51 \ + --hash=sha256:0cc63b5e47c12a48d90c7e9d7de6a035dd14f62868aaedbb4e0ff8ba2b8bfe7b \ + --hash=sha256:0dd8af32e9f7b56b7f95cc1fd79b23054c3bdc172392ae560acc24d57b7ffe71 \ + --hash=sha256:0def9fb6abac55492d6d51cddb7225d07d6f279e774e0adc08569a54a5fc8d46 \ + --hash=sha256:0fd2c901cc307a745ad4bc87f20060d7a0825a3371d1e93488af22e7a387f78f \ + --hash=sha256:1043aedf5917caa861bcb25a9c11460049656bdf0017a90a309fa8f255467725 \ + --hash=sha256:121a76a0985db80ceae9e171c337f8c927868e37d01b54e3ce87bc87f9c6a208 \ + --hash=sha256:143533cc4b6fbc5b95aca0a5b8d541088d374831593def000ec89322c220221d \ + --hash=sha256:14e953ff3607c92d7675bf79c4d4509ef6782aa8c08509f179f9b3d6d0679e86 \ + --hash=sha256:18ae8eed4526e35bdb754d61562b90bf5c00a67fdcf3cc1380dd59597486631b \ + --hash=sha256:19959129885356df0e97556856f77eb2888380dac18bed075a7c05c5128c618d \ + --hash=sha256:1ba8c6a416569ce0d37e83e28a254a61dc99a419084dfb6476cea02d997f74fa \ + --hash=sha256:1c2a0026062abcc321a53db4a185ceba0b59a66b5d37b0808917a88b55a5257f \ + --hash=sha256:1d9fe8091b2e89d470df68a9331111ed008ae8aae6bf1e8e1fba4086a495c84e \ + --hash=sha256:2089fe39c406784d90101c726755ffa1497bb74638fd434300d2b88006186de8 \ + --hash=sha256:23ae6fdad9e63e54038f5ef78aba2933faca61e24d432786589e737bc5522ebb \ + --hash=sha256:26ec4ccce55aa533fbd603d08911b01101a8fcfec987845ac3ae2c7087b2bde3 \ + --hash=sha256:2f7f7aa47b229f2b39a2ae2596d2ad5625d77b5eb9856fac2dab3eb506cdd0a0 \ + --hash=sha256:31b003f9a070335e2a8233ee9b14a3ca8e6d792012ae011f741bf0aaf11744c5 \ + --hash=sha256:32ab11df9677ca80bcbb5fe4eb1da9109a5019239a054836efc6fa1c64e683cf \ + --hash=sha256:33026515aebc0e70d1c89978e53e8d695d35d9e472f8d5b34465ba3c74028650 \ + --hash=sha256:34b6925af9853bf461950e6508910f179fd6e9b1a7ec8548e069606b7e51a26b \ + --hash=sha256:352cf115a810b357caa35193ab656ecf5ef41056855e82f292c99e8514f8d954 \ + --hash=sha256:39ab5894d971f9ac68baa6eca5c50387db579cfcacf36ae8df3feceb1815e6d0 \ + --hash=sha256:3a21a9509d0ee88e7a70e1ad228cd2f0e0fd1e187458db132e8a8d18c97daf9d \ + --hash=sha256:3c5c2ef13797466aa64170cbb66ad98a32351dd4127694cea7199f80f213750d \ + --hash=sha256:3e778bfccd63075167709136afbc251c1f683758d5bf49c803c60ac3f894ce6b \ + --hash=sha256:3f1e6cb402a89457582cd696f982559217d13484a193202c394015297968c86d \ + --hash=sha256:42e82e578c904445d4c8a35b8f28052cf567593215fa5db06266fbc6f77aaa2e \ + --hash=sha256:4408b2b27a95ca8cc48b7411945753773353b5c93b307754781086c99d3a576f \ + --hash=sha256:446654b29bfaa30500d80947eda42cef1449dc8a87f4e3cf061cc8485d3a1f0b \ + --hash=sha256:45010bcfe66df41522d56c9b6114e87ecc597a08970ff6a2ced24415c141ae5f \ + --hash=sha256:49ee178ca31c94621294bf9b8b676a92a2e6bba8af0529591753719e57edb621 \ + --hash=sha256:4d7d93613b01b0199961330e49cfc52d479b3d5776c56c691db31130c0a07d91 \ + --hash=sha256:4fb41211d2333eb930a51e0546a65999761cf1f572a4da56ef9b8a62966c06f2 \ + --hash=sha256:4fe97894d1b306c919b4e50def1e6f6c522f4d03a7283811f4d108f1ce5d3ac2 \ + --hash=sha256:554bffadcbcb6d5f4e5fb10a61cc52084b9a63d1dab5f10bcd2c4343972e8e2c \ + --hash=sha256:59b49507f47479e299a9e1bc41b5cb83a7afda0540625f1dbae886615978acbf \ + --hash=sha256:5eeb8edc6110d9194a4d0d54610f64c37a31c605b5dbb7e407fc6ec7fa34a4a1 \ + --hash=sha256:612b709381c0355b70d89cdb51b7f670591ed5cbbc0e3b5337488019dc667b65 \ + --hash=sha256:61956f074ecd123f55adca68ee3eab46e6a07ad3f8e64e6db95dfacb444f55c4 \ + --hash=sha256:6398d5145689503412cc1748895242598d8846b8967b851133b20dc2ed1e21e8 \ + --hash=sha256:65b408d8fcb273e3499e7ef2ce796810da1becd208c7fb4373692a242d79d461 \ + --hash=sha256:686ac5350fceae63830bb98805fcb8039325bf4c06d9f6f048ff65229d5bffa5 \ + --hash=sha256:6a1a824fbed817e0a891103886b68f063b1e83cc51bc97192a90a60195a9291f \ + --hash=sha256:6abb75ab16bc3281714a5b99548a2225db70dba1f995f6d7f7419b76eb5a8fbe \ + --hash=sha256:6f7121a8914ed13fcfe2099f895341bfb789f004d4c5a0bdece8fa667da10849 \ + --hash=sha256:7020ed44df30b3aa492c00ee3b52d0548c1f30c2c6c5bb13ae897680900d3413 \ + --hash=sha256:720537c7ea6f80dc61913184edb0ce2497a306b39ef19f28505b322553d52bdb \ + --hash=sha256:724184b4aafed865e4f13ca313fdcb43024300c028ec67319cfa16847d84685e \ + --hash=sha256:7c03031610e3e6ed1768a2b7a8fc84637c1257b50c5eacaf094c6e17a84fc563 \ + --hash=sha256:80a5ea3b4fd9d6a5b9a44f7976a9acaaab35aa3c1f6b29e5bd857dfabaded223 \ + --hash=sha256:80c7cadd3fd2bfde5df8aa0787e315812cad0c313a753095d02f4c2b6c01677b \ + --hash=sha256:80ea96f5c1a30bf09007d48466521d9c294bebe197c708c3359096e3e3691632 \ + --hash=sha256:864e9b87ac33c3fb9fb4ad48166d4fdb579c351d5c77deb0d34bccb36a775cd9 \ + --hash=sha256:87fb80cbe3557e27e7b28b995c2b2eedf689b8886f941ab93e0e288f0976518a \ + --hash=sha256:8873c4a11c50b9989168881aeb3f08859f469d809941866aa1feefd8be5431f6 \ + --hash=sha256:888d60953908dcf761aa320c3e390ab8556efbdb551ace63921de90f6ae0848d \ + --hash=sha256:8b5fcc4771732191b2b7d1dd68d8f0353f47f8d90b6150f6dce58bf1112442cb \ + --hash=sha256:8f39588af4731c8923c26810eb3b33f76f17633985e40f59c3cd45a33805a895 \ + --hash=sha256:9173db3be74a35cb6731701094b98120f7ee4876a287882a59cdea1fa7da342f \ + --hash=sha256:92f05c9c42bde5785dc48770bc2194d9f7442544156f951e19cd31b096cec562 \ + --hash=sha256:951733b1bbdb71e377cec567b409f1a7881b47cfcad84121aa74cb575fa425ea \ + --hash=sha256:957bb708e8057ab1649ba566456429d691ec9b90d1c9ad1af1ba7ffbbeaf05f2 \ + --hash=sha256:9916fda742cd4eede63b286f58c06718324265d727ce0856eb1aac86d0d150d6 \ + --hash=sha256:9e1d3a4cb7993b708f0ada8d0c84590efd853f169e7147d2202c9da503180242 \ + --hash=sha256:9e4482589065c8ecd761cff522dcd85f2d39e62f551e37e025d1c7d54772def3 \ + --hash=sha256:a5300757f8a68f5b6cc33f57338d72a0e3589c5cc9ad5f8504ea06f028be582a \ + --hash=sha256:a540abfab208e1b7ef2df231c40ef3b6cbb30a0aad6204e9b6a81c10a6794628 \ + --hash=sha256:a5758353650079898dc1b2b0e95aa51fa23a30d020e06f62c430dd08ee56cdd8 \ + --hash=sha256:a64b85a4760337cfefdb27d42da6ed8b58e8cde3f2d57b6ef43e76ef6ea9ef47 \ + --hash=sha256:a655d34b2a6943af32401f3d94f72e9d731f6ad16285815550bf2b4ee69d420a \ + --hash=sha256:a714befaacbd10092ffe4cea0d3c5f008fb9efe9bc322c715bcdfdee414b9a3d \ + --hash=sha256:a760da040b47767b4b873adfb7c3b691e9ba2fc60f113f9d0b88f1a62f323e85 \ + --hash=sha256:addd736a0547d553283adaf4e05d7104e7f2c7b0b092e9b4d28756825f14531f \ + --hash=sha256:ae4613d7d9dda60fcba95f846cc6f808017f1843f392cf9daad14a6534493d71 \ + --hash=sha256:b11b589e00095ec69cf79841a76360f9b079e95b0368a25b5ebb951ab0c157ff \ + --hash=sha256:b3e445b66c80b4eb4234e855ce94d9adc183eedbd632816228d89930b91b2c5b \ + --hash=sha256:b7b893976e7fe42053da64f2aa27239c24252fd2ec6df471e1be197c0addc3b1 \ + --hash=sha256:b84f186a7f0536fe4ff9a9fa12d06d007b9b71d4b5352ddcc41f59ad6522a312 \ + --hash=sha256:b89efc38431793d28b7cd91227e2f952ad7c48df19132b17f43a5fec3c14143b \ + --hash=sha256:b97a38fb4c732b6832db6bf108963adbcd82ef1268ba2025dce390f45af75efa \ + --hash=sha256:b9d74e4eee9ddb64c2e92d5d61472c59c21684c059eb7b68767be9628e977859 \ + --hash=sha256:bb90e7177944b6684738c1fc36aabd2dd00d1de3be7dbe09f91e196f1bc0dc81 \ + --hash=sha256:bec37990e3d6121f29ecfb594bd8f1bf009e9f7926daba2e50e3b27d3892a783 \ + --hash=sha256:bf3c49863c23a1ad6da9c30351aed6cff8d5ddbeb63c5c8420ae54e98c7d0138 \ + --hash=sha256:bf48516e35cf848390ea68850aba53e7c333720d2945b4d2c25b69fc5171723f \ + --hash=sha256:bfc71e6d970419c1309b3640305298643e2a734cad3f7cfb6d2ddee4175ab53d \ + --hash=sha256:c0094897d7d01f184b2d7fe8c56c66d64efe01b31f4b7d34205b391387df1111 \ + --hash=sha256:c03c6eb6ece86dfdcbb34799efaa339b093132e1aceed491ba5e08fe06cdf699 \ + --hash=sha256:c1a9a6651197fbed6f0212591418b9def774fc3f8324f78d1bf0e6a63e5f8aa1 \ + --hash=sha256:c3589f40749acce747510bf5d589d54e376cb0930ea58b35effac97e5312b0c1 \ + --hash=sha256:c4e38dd8f39c43a91d2410ad2b85610701b0979342c3df1d69eaf8e838c757d8 \ + --hash=sha256:c6c8fabf1dafc1f1ddcbb67896d3f93efb092e8c4b6322d7389b944e76a484e5 \ + --hash=sha256:c90fcf7804ea0a54b896ce0f2b9565350220b8d4890fd0db461a476a4c687963 \ + --hash=sha256:c9b602fae1e00b7c035d661ce85575365719192a7b46784bd71cf64c68053aa0 \ + --hash=sha256:ccb64d887a9db1cd76dbc0f92051a1a478a2a67e7f56c62d915cb881d7734704 \ + --hash=sha256:d06fcdecc10fc7954d7c8f27a03c96055fe525274dc84a7b0dbdc3d6b9e03dab \ + --hash=sha256:d0c3082bf79bcd6a614d55916590ad4b8f93200e10b97f463ea5d9d07c9b5f23 \ + --hash=sha256:d49c18f1ea294cf4adde2e5ac256e98c82ea9d708462ce4bf799dffa7cfe8a2c \ + --hash=sha256:d60030baaa7bfbb02d650c126cdcddcb6e33dbff14d819434c8fa2fdcaeeeba5 \ + --hash=sha256:d7cab119d0df0b9413f106b4d7fc34f2872d3574ed3806fb48959c830b1537da \ + --hash=sha256:d9b77b25b4f395f92de6099ab08e8ae2bc7e51dfe157f22900902243a5cc90c7 \ + --hash=sha256:dabee8f4935e731fb46b2a3091bdda0d3d94b3bbfb907d2b4f12eefce4009619 \ + --hash=sha256:db5e82ba15c142425b8406690032df89e39cca4a2e8afbbb9a3d84edc2373ac3 \ + --hash=sha256:dc79d36d0618752265f0d575915bdc5c5130ecb9c9f6b3bcefeae32e4bdfafcf \ + --hash=sha256:ddfa987262763c3c22a8367d2a49c244b018a74c3a8e3ab1a864119ad45c5633 \ + --hash=sha256:e1172147d28d8fbcf8cb8d26c41506169f5ad8fe9ec969cb116835a19d4d8eca \ + --hash=sha256:e11edba5bc344a32b029a7af9d4b3173982dd79eeafa0b9dbd787364414b0509 \ + --hash=sha256:e2c89e9b762c57f59d5e99ee8b20202adb892e35f8d3485741340999ca55058e \ + --hash=sha256:e31f72490b7c12f7790e1e25c3afffd20503ee1bfb43461d7838b871ff244b19 \ + --hash=sha256:e8c65ef3862a8ad6e86492b6ed9327805dd66904c012bd3649dc67d822ed6c34 \ + --hash=sha256:ebb8912f565b8cdbbf27debfe00df04202c20e2f651b9e32767930c5eace3621 \ + --hash=sha256:ed511a0708e2297e1d6431e7fb217e3402791e491e02da800658ace4973df1bb \ + --hash=sha256:edf06545875f3efa31560d94121e95c7fd70d98b1dfedc0157097d79b13b52ea \ + --hash=sha256:f0fe9834e5aeccaf19a0d8feb296d66a24be1a7c9922002f842a682cd5abb787 \ + --hash=sha256:f1a0d5117230dd46b399a30a38afa44f79c99f3168988fdc4f425c3f928b39df \ + --hash=sha256:f37964e4a5e993d2fd45147741e9dff7f34a2d8c00ab94c4ea0514a4677f959e \ + --hash=sha256:f57dc6b8fef170f105d2cf5cdce254f47b137d7755086cf7050f47e16582abba \ + --hash=sha256:f93bc1c3486ef3747e07c9d7c1d0a147b8fbaab975f80e348aed6f71309dfaca \ + --hash=sha256:fb00027a09a8f9f08028b40dce4c933cf73e4833240ed356583fdc9cfa721566 \ + --hash=sha256:fb99cc9d45f48895d9d67f6a0b8a57f08d39c174d9f25ad97a313e0470267b1c \ + --hash=sha256:fdd88ed5e20b1bcdd234421e454962c971aa44b653bdb7f1ea9ef683e90fb649 \ + --hash=sha256:fe3fa1dd453ed5c7f5ea23a26218329790ed7197a99b90e94330e313959a7f52 + # via transformers +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via + # huggingface-hub + # transformers +safetensors==0.7.0 \ + --hash=sha256:0071bffba4150c2f46cae1432d31995d77acfd9f8db598b5d1a2ce67e8440ad2 \ + --hash=sha256:07663963b67e8bd9f0b8ad15bb9163606cd27cc5a1b96235a50d8369803b96b0 \ + --hash=sha256:12f49080303fa6bb424b362149a12949dfbbf1e06811a88f2307276b0c131afd \ + --hash=sha256:1d060c70284127fa805085d8f10fbd0962792aed71879d00864acda69dbab981 \ + --hash=sha256:42cb091236206bb2016d245c377ed383aa7f78691748f3bb6ee1bfa51ae2ce6a \ + --hash=sha256:473b32699f4200e69801bf5abf93f1a4ecd432a70984df164fc22ccf39c4a6f3 \ + --hash=sha256:54bef08bf00a2bff599982f6b08e8770e09cc012d7bba00783fc7ea38f1fb37d \ + --hash=sha256:5d72abdb8a4d56d4020713724ba81dac065fedb7f3667151c4a637f1d3fb26c0 \ + --hash=sha256:672132907fcad9f2aedcb705b2d7b3b93354a2aec1b2f706c4db852abe338f85 \ + --hash=sha256:6999421eb8ba9df4450a16d9184fcb7bef26240b9f98e95401f17af6c2210b71 \ + --hash=sha256:7b95a3fa7b3abb9b5b0e07668e808364d0d40f6bbbf9ae0faa8b5b210c97b140 \ + --hash=sha256:8469155f4cb518bafb4acf4865e8bb9d6804110d2d9bdcaa78564b9fd841e104 \ + --hash=sha256:94fd4858284736bb67a897a41608b5b0c2496c9bdb3bf2af1fa3409127f20d57 \ + --hash=sha256:b0f6d66c1c538d5a94a73aa9ddca8ccc4227e6c9ff555322ea40bdd142391dd4 \ + --hash=sha256:c74af94bf3ac15ac4d0f2a7c7b4663a15f8c2ab15ed0fc7531ca61d0835eccba \ + --hash=sha256:c82f4d474cf725255d9e6acf17252991c3c8aac038d6ef363a4bf8be2f6db517 \ + --hash=sha256:cdab83a366799fa730f90a4ebb563e494f28e9e92c4819e556152ad55e43591b \ + --hash=sha256:cfdead2f57330d76aa7234051dadfa7d4eedc0e5a27fd08e6f96714a92b00f09 \ + --hash=sha256:d1239932053f56f3456f32eb9625590cc7582e905021f94636202a864d470755 \ + --hash=sha256:dac7252938f0696ddea46f5e855dd3138444e82236e3be475f54929f0c510d48 \ + --hash=sha256:dc92bc2db7b45bda4510e4f51c59b00fe80b2d6be88928346e4294ce1c2abe7c \ + --hash=sha256:e07d91d0c92a31200f25351f4acb2bc6aff7f48094e13ebb1d0fb995b54b6542 \ + --hash=sha256:f4729811a6640d019a4b7ba8638ee2fd21fa5ca8c7e7bdf0fed62068fcaac737 + # via + # -r ocr/anonymization/requirements.in + # gliner + # transformers +sentencepiece==0.2.2 \ + --hash=sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497 \ + --hash=sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e \ + --hash=sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0 \ + --hash=sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719 \ + --hash=sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107 \ + --hash=sha256:16c84ddef8d3084a8af37208acd365b08092ca089080f1a71fbfdd911adda9b3 \ + --hash=sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b \ + --hash=sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a \ + --hash=sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c \ + --hash=sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838 \ + --hash=sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5 \ + --hash=sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6 \ + --hash=sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d \ + --hash=sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf \ + --hash=sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25 \ + --hash=sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d \ + --hash=sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e \ + --hash=sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b \ + --hash=sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd \ + --hash=sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936 \ + --hash=sha256:54a83df9260a89c1734256e620fe1f1a6bfedd7547139d4dc1384efac11a3a85 \ + --hash=sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b \ + --hash=sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708 \ + --hash=sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb \ + --hash=sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9 \ + --hash=sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0 \ + --hash=sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790 \ + --hash=sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e \ + --hash=sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78 \ + --hash=sha256:741b4b367140e9b5c36b5a14c72179f2c946d991ea9a7c031a2a1ee6ad097b99 \ + --hash=sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9 \ + --hash=sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d \ + --hash=sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8 \ + --hash=sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906 \ + --hash=sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383 \ + --hash=sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b \ + --hash=sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53 \ + --hash=sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b \ + --hash=sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab \ + --hash=sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0 \ + --hash=sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914 \ + --hash=sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91 \ + --hash=sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e \ + --hash=sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a \ + --hash=sha256:c76c9b3324efd79029eeb0fd2ced1964bdbeca7d45e030b46fa3ef3cf74f8032 \ + --hash=sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711 \ + --hash=sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da \ + --hash=sha256:caad9566e2ef0e5640d36032c69b0edc7ac6028277b93d93815898804fac450c \ + --hash=sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a \ + --hash=sha256:cd810878180a52950e5a61f25ada5248a453bbdbafe474f89514135fbc1f633d \ + --hash=sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151 \ + --hash=sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563 \ + --hash=sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811 \ + --hash=sha256:eb8da9d9a9b418422c21a07fd19b9d9228692b7a7468a45eec6b11642d3c808b \ + --hash=sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820 \ + --hash=sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090 \ + --hash=sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c + # via gliner +sympy==1.14.0 \ + --hash=sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517 \ + --hash=sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5 + # via torch +tokenizers==0.22.2 \ + --hash=sha256:143b999bdc46d10febb15cbffb4207ddd1f410e2c755857b5a0797961bbdc113 \ + --hash=sha256:1a62ba2c5faa2dd175aaeed7b15abf18d20266189fb3406c5d0550dd34dd5f37 \ + --hash=sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e \ + --hash=sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001 \ + --hash=sha256:1e50f8554d504f617d9e9d6e4c2c2884a12b388a97c5c77f0bc6cf4cd032feee \ + --hash=sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7 \ + --hash=sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd \ + --hash=sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4 \ + --hash=sha256:319f659ee992222f04e58f84cbf407cfa66a65fe3a8de44e8ad2bc53e7d99012 \ + --hash=sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67 \ + --hash=sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a \ + --hash=sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5 \ + --hash=sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917 \ + --hash=sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c \ + --hash=sha256:64d94e84f6660764e64e7e0b22baa72f6cd942279fdbb21d46abd70d179f0195 \ + --hash=sha256:753d47ebd4542742ef9261d9da92cd545b2cacbb48349a1225466745bb866ec4 \ + --hash=sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a \ + --hash=sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc \ + --hash=sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92 \ + --hash=sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5 \ + --hash=sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48 \ + --hash=sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b \ + --hash=sha256:e10bf9113d209be7cd046d40fbabbaf3278ff6d18eb4da4c500443185dc1896c \ + --hash=sha256:f01a9c019878532f98927d2bacb79bbb404b43d3437455522a00a30718cdedb5 + # via transformers +torch==2.8.0+cpu \ + --hash=sha256:0e34e276722ab7dd0dffa9e12fe2135a9b34a0e300c456ed7ad6430229404eb5 \ + --hash=sha256:16d75fa4e96ea28a785dfd66083ca55eb1058b6d6c5413f01656ca965ee2077e \ + --hash=sha256:2be20b2c05a0cce10430cc25f32b689259640d273232b2de357c35729132256d \ + --hash=sha256:2bfc013dd6efdc8f8223a0241d3529af9f315dffefb53ffa3bf14d3f10127da6 \ + --hash=sha256:41e6fc5ec0914fcdce44ccf338b1d19a441b55cafdd741fd0bf1af3f9e4cfd14 \ + --hash=sha256:5239ef35402000844b676a9b79ed76d5ae6b028a6762bbdfebdf8421a0f4d2aa \ + --hash=sha256:5d255d259fbc65439b671580e40fdb8faea4644761b64fed90d6904ffe71bbc1 \ + --hash=sha256:610f600c102386e581327d5efc18c0d6edecb9820b4140d26163354a99cd800d \ + --hash=sha256:64c187345509f2b1bb334feed4666e2c781ca381874bde589182f81247e61f88 \ + --hash=sha256:680129efdeeec3db5da3f88ee5d28c1b1e103b774aef40f9d638e2cce8f8d8d8 \ + --hash=sha256:6d93a7165419bc4b2b907e859ccab0dea5deeab261448ae9a5ec5431f14c0e64 \ + --hash=sha256:7631ef49fbd38d382909525b83696dc12a55d68492ade4ace3883c62b9fc140f \ + --hash=sha256:7cc4af6ba954f36c2163eab98cf113c137fc25aa8bbf1b06ef155968627beed2 \ + --hash=sha256:8b5882276633cf91fe3d2d7246c743b94d44a7e660b27f1308007fdb1bb89f7d \ + --hash=sha256:8f81dedb4c6076ec325acc3b47525f9c550e5284a18eae1d9061c543f7b6e7de \ + --hash=sha256:99fc421a5d234580e45957a7b02effbf3e1c884a5dd077afc85352c77bf41434 \ + --hash=sha256:a5064b5e23772c8d164068cc7c12e01a75faf7b948ecd95a0d4007d7487e5f25 \ + --hash=sha256:a9dbb6f64f63258bc811e2c0c99640a81e5af93c531ad96e95c5ec777ea46dab \ + --hash=sha256:af81283ac671f434b1b25c95ba295f270e72db1fad48831eb5e4748ff9840041 \ + --hash=sha256:b2149858b8340aeeb1f3056e0bff5b82b96e43b596fe49a9dba3184522261213 \ + --hash=sha256:bda4f93d64dcd9ae5d51844bbccc6fcb7d603522bcc95d256b5fe3bdb9dccca3 \ + --hash=sha256:cb06175284673a581dd91fb1965662ae4ecaba6e5c357aa0ea7bb8b84b6b7eeb \ + --hash=sha256:cb9a8ba8137ab24e36bf1742cb79a1294bd374db570f09fc15a5e1318160db4e \ + --hash=sha256:e1ee1b2346ade3ea90306dfbec7e8ff17bc220d344109d189ae09078333b0856 \ + --hash=sha256:e3c3fce24ebaac954b837d1498e36d484ad0d93e2a1ed5b6b0c55a02ea748fab \ + --hash=sha256:eac8b7ef5c7ca106daec5e829dfa8ca56ca47601db13b402d2608861ad3ab926 + # via + # -r ocr/anonymization/requirements.in + # gliner +tqdm==4.70.1 \ + --hash=sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73 \ + --hash=sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4 + # via + # gliner + # huggingface-hub + # transformers +transformers==4.57.1 \ + --hash=sha256:b10d05da8fa67dc41644dbbf9bc45a44cb86ae33da6f9295f5fbf5b7890bd267 \ + --hash=sha256:f06c837959196c75039809636cd964b959f6604b75b8eeec6fdfc0440b89cc55 + # via + # -r ocr/anonymization/requirements.in + # gliner +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # huggingface-hub + # pydantic + # pydantic-core + # torch + # typing-inspection +typing-inspection==0.4.4 \ + --hash=sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47 \ + --hash=sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147 + # via pydantic +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via requests diff --git a/ocr/anonymization/schemas.py b/ocr/anonymization/schemas.py new file mode 100644 index 0000000000000000000000000000000000000000..850983c00bbeb77b43063b3f4ce041d910c408de --- /dev/null +++ b/ocr/anonymization/schemas.py @@ -0,0 +1,207 @@ +"""Classified detection evidence and allowlisted candidate data are separate types.""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +Label = Literal["PERSON", "CITY", "RELIGION", "EMAIL", "PHONE", "ORGANIZATION"] +LABELS = ("PERSON", "CITY", "RELIGION", "EMAIL", "PHONE", "ORGANIZATION") + + +class StrictModel(BaseModel): + model_config = ConfigDict( + extra="forbid", + frozen=True, + strict=True, + hide_input_in_errors=True, + revalidate_instances="always", + ) + + +class Entity(StrictModel): + """Private evidence only: never serialize this type into a release candidate.""" + + start: int = Field(ge=0) + end: int = Field(gt=0) + label: Label + text: str + score: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False) + source: Literal["multilingual_ner", "deterministic"] + model_revision: str = Field(pattern=r"^[a-zA-Z0-9._-]{1,64}$") + normalization: Literal[ + "identity", "dehyphenation-v1", "status-ocr-v1", "dehyphenation-status-ocr-v1" + ] = "identity" + + +class Replacement(StrictModel): + start: int = Field(ge=0) + end: int = Field(gt=0) + label: Label + placeholder: str = Field(pattern=r"^\[[A-Z_]{1,40}\]$") + score: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False) + source: Literal["multilingual_ner", "deterministic", "human_correction"] + model_revision: str = Field(pattern=r"^[a-zA-Z0-9._-]{1,64}$") + normalization: Literal[ + "identity", "dehyphenation-v1", "status-ocr-v1", "dehyphenation-status-ocr-v1" + ] = "identity" + + @model_validator(mode="after") + def original_source_bounds(self): + if self.start >= self.end: + raise ValueError("invalid_replacement_bounds") + # The source length is not the anonymized text length. Its upper bound + # is checked by PrivateReviewMaterial when original references exist. + return self + + +class CandidatePage(StrictModel): + page_index: int = Field(ge=0) + anonymized_text: str + tags: tuple[Replacement, ...] + source: Literal["embedded_text", "cpu_ocr", "alias_ocr"] + original_sensitivity: Literal["D0", "D1", "D2"] + + +class LegacyReleaseCandidate(StrictModel): + schema_version: Literal["ocr-anonymization-candidate/v1"] = "ocr-anonymization-candidate/v1" + candidate_id: str = Field(pattern=r"^[a-f0-9]{32}$") + sensitivity: Literal["D0", "D1", "D2"] + pages: tuple[CandidatePage, ...] = Field(min_length=1) + model_revision: str = Field(pattern=r"^[a-f0-9]{40}$") + anonymous: Literal[False] = False + release_review_required: Literal[True] = True + release_approved: Literal[False] = False + automated_release_eligible: Literal[False] = False + review_reasons: tuple[Literal["recognition_incomplete_possible", "empty_prediction"], ...] + + @model_validator(mode="after") + def document_integrity(self): + indexes = [page.page_index for page in self.pages] + if len(set(indexes)) != len(indexes) or any( + page.original_sensitivity != self.sensitivity for page in self.pages + ): + raise ValueError("invalid_candidate_document") + # No new contiguous numbering rule for valid legacy documents. Exact + # source coverage is checked where immutable source references exist. + return self + + @model_validator(mode="after") + def legacy_normalization(self): + if self.schema_version in ( + "ocr-anonymization-candidate/v1", + "ocr-anonymization-candidate/v2", + ) and any(t.source == "human_correction" for p in self.pages for t in p.tags): + raise ValueError("unsupported_human_correction_version") + if self.schema_version == "ocr-anonymization-candidate/v1" and any( + tag.normalization not in ("identity", "dehyphenation-v1") + for page in self.pages + for tag in page.tags + ): + raise ValueError("unsupported_legacy_normalization") + return self + + +class RecognitionProvenance(StrictModel): + schema_version: Literal["recognition-provenance/v1"] = "recognition-provenance/v1" + model_revision: str = Field(pattern=r"^[a-f0-9]{40}$") + configuration_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + terminology_version: Literal["religious-terminology-v1"] = "religious-terminology-v1" + status_version: Literal["religious-status-v1"] = "religious-status-v1" + replacement_version: Literal["enclosing-spans-failclosed-v1"] = "enclosing-spans-failclosed-v1" + normalization_version: Literal["source-codepoint-v1"] = "source-codepoint-v1" + + +class ReleaseCandidate(LegacyReleaseCandidate): + schema_version: Literal["ocr-anonymization-candidate/v2"] = "ocr-anonymization-candidate/v2" + provenance: RecognitionProvenance + + @model_validator(mode="after") + def consistent_provenance(self): + if self.model_revision != self.provenance.model_revision: + raise ValueError("candidate_provenance_mismatch") + return self + + +class CorrectionProvenance(StrictModel): + schema_version: Literal["ocr-correction-provenance/v1"] = "ocr-correction-provenance/v1" + policy_version: Literal["reviewed-corrections-v1"] = "reviewed-corrections-v1" + correction_id: str = Field(pattern=r"^[a-f0-9]{32}$") + proposal_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + original_evidence_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + machine_snapshot_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + source_references_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + prior_candidate_digest: str | None = Field(pattern=r"^[a-f0-9]{64}$") + prior_state_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + effective_state_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + author_subject_digests: tuple[str, ...] = Field(min_length=1, max_length=1000) + edit_count: int = Field(ge=1, le=1000) + + @model_validator(mode="after") + def valid_authors(self): + import re + + if len(set(self.author_subject_digests)) != len(self.author_subject_digests) or any( + not re.fullmatch(r"[a-f0-9]{64}", s) for s in self.author_subject_digests + ): + raise ValueError("invalid_correction_authors") + return self + + +class CorrectedCandidate(ReleaseCandidate): + schema_version: Literal["ocr-anonymization-candidate/v3"] = "ocr-anonymization-candidate/v3" + correction: CorrectionProvenance + review_reasons: tuple[ + Literal[ + "recognition_incomplete_possible", + "empty_prediction", + "human_correction_requires_review", + ], + ..., + ] + + @model_validator(mode="after") + def reviewed_correction_only(self): + if "human_correction_requires_review" not in self.review_reasons or any( + t.source == "human_correction" + and ( + t.score is not None + or t.model_revision != "reviewed-corrections-v1" + or t.normalization != "identity" + ) + for p in self.pages + for t in p.tags + ): + raise ValueError("invalid_corrected_candidate") + return self + + +class Destination(StrictModel): + """Operator-controlled destination registry entry, never a caller supplied URL.""" + + destination_id: str = Field(pattern=r"^[a-zA-Z0-9:_-]{1,128}$") + destination_class: Literal["private", "public"] + allowed_sensitivities: tuple[Literal["D0", "D1", "D2"], ...] + + +class VerifiedReview(StrictModel): + """Attestation returned by the host application's authenticated review store.""" + + reviewer_subject: str = Field(pattern=r"^[a-zA-Z0-9:_-]{1,128}$") + review_id: str = Field(pattern=r"^[a-zA-Z0-9:_-]{1,128}$") + evidence_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + candidate_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + sensitivity: Literal["D0", "D1", "D2"] + destination_id: str = Field(pattern=r"^[a-zA-Z0-9:_-]{1,128}$") + completed_at: int = Field(ge=0) + + +class Approval(StrictModel): + schema_version: Literal["ocr-review-approval/v2"] = "ocr-review-approval/v2" + approval_id: str = Field(pattern=r"^[a-f0-9]{32}$") + candidate_digest: str = Field(pattern=r"^[a-f0-9]{64}$") + review: VerifiedReview + destination: Destination + format: Literal["json", "text"] + issued_at: int = Field(ge=0) + expires_at: int = Field(ge=0) + signature: str = Field(pattern=r"^[a-f0-9]{64}$") diff --git a/ocr/anonymization/staging_probe.py b/ocr/anonymization/staging_probe.py new file mode 100644 index 0000000000000000000000000000000000000000..440bb87a96654bd84d86097c71e61274bf4a1669 --- /dev/null +++ b/ocr/anonymization/staging_probe.py @@ -0,0 +1,138 @@ +"""Bounded offline runtime qualification, never a service or production launcher.""" + +import argparse +import hashlib +import json +import multiprocessing +import resource +import sys +import time +import traceback +from pathlib import Path + + +def pressure(ready, stop): + allocation = bytearray(1536 * 1024 * 1024) + for i in range(0, len(allocation), 4096): + allocation[i] = 1 + ready.set() + value = 0 + while not stop.is_set(): + value = (value * 1664525 + 1013904223) % (2**32) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--cache", required=True) + parser.add_argument("--output", required=True) + parser.add_argument( + "--hybrid", action="store_true", help="Measure selected model + terminology path" + ) + parser.add_argument( + "--paired", action="store_true", help="Alternate same-text model/control timings" + ) + args = parser.parse_args() + if args.paired and not args.hybrid: + parser.error("paired measurement requires hybrid") + denied = [] + + def audit(event, values): + if event in ("socket.connect", "socket.bind", "socket.getaddrinfo"): + denied.append( + dict( + event=event, callsite=[frame.name for frame in traceback.extract_stack(limit=5)] + ) + ) + raise PermissionError("staging_network_disabled") + + sys.addaudithook(audit) + # The Docker network namespace is additionally network=none, with no ports. + context = multiprocessing.get_context("spawn") + ready, stop = context.Event(), context.Event() + load = context.Process(target=pressure, args=(ready, stop)) + load.start() + try: + if not ready.wait(30): + raise RuntimeError("pressure_not_ready") + from .atomic_output import write_candidate + from .model import verify_cache + from .recognizer import GLiNERRecognizer + + start = time.perf_counter() + verify_cache(args.cache) + integrity_ms = (time.perf_counter() - start) * 1000 + start = time.perf_counter() + model = GLiNERRecognizer(args.cache) + loading_ms = (time.perf_counter() - start) * 1000 + from .engine import AnonymizationEngine + + engine = AnonymizationEngine(model) if args.hybrid else None + samples, baseline_samples = [], [] + for iteration in range(8): + start = time.perf_counter() + text = "Synthetic reviewer Petra Ashcroft works at Harbor Research in Norwich." + if engine is None: + entities = model.recognize(text) + else: + _, entities, _ = engine.transform(text + " Religious status: agnostic.") + if not entities: + raise RuntimeError("model_not_ready") + if args.paired: + # Both arms use the identical independent synthetic sentence. + # Alternate order to expose warmup/order effects, not hide them. + from .detectors import detect_contacts + from .replacement import replace + + paired_text = text + " Religious status: agnostic." + order = ("control", "hybrid") if iteration % 2 else ("hybrid", "control") + for arm in order: + start = time.perf_counter() + if arm == "control": + replace( + paired_text, model.recognize(paired_text) + detect_contacts(paired_text) + ) + else: + engine.transform(paired_text) + duration = (time.perf_counter() - start) * 1000 + (samples if arm == "hybrid" else baseline_samples).append(duration) + else: + samples.append((time.perf_counter() - start) * 1000) + limit = int(Path("/sys/fs/cgroup/memory.max").read_text()) + current = int(Path("/sys/fs/cgroup/memory.current").read_text()) + peak = int(Path("/sys/fs/cgroup/memory.peak").read_text()) + listening = 0 + for table in ("tcp", "tcp6"): + for line in Path("/proc/net/" + table).read_text().splitlines()[1:]: + listening += line.split()[3] == "0A" + result = dict( + probe="single-worker-with-1536MiB-concurrent-CPU-memory-load", + recognition_path="hybrid-terminology" if args.hybrid else "model-only", + integrity_ms=integrity_ms, + loading_ms=loading_ms, + warm_ms=samples, + same_text_control_ms=baseline_samples, + process_peak_rss_mib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, + cgroup_memory_limit_mib=limit / 2**20, + cgroup_current_mib=current / 2**20, + cgroup_peak_mib=peak / 2**20, + measured_headroom_mib=(limit - peak) / 2**20, + cpu_quota=Path("/sys/fs/cgroup/cpu.max").read_text().strip(), + denied_network_calls=denied, + native_tcp_listeners_at_measurement=listening, + source_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + automated_release_eligible=False, + ) + write_candidate(args.output, (json.dumps(result, indent=2) + "\n").encode()) + finally: + stop.set() + load.join(10) + if load.is_alive(): + load.kill() + load.join() + + +if __name__ == "__main__": + try: + main() + except Exception: + raise SystemExit("private_staging_probe_failed") from None diff --git a/ocr/anonymization/status.py b/ocr/anonymization/status.py new file mode 100644 index 0000000000000000000000000000000000000000..d04e84522f1d1e555090f357037e9ef4dd279664 --- /dev/null +++ b/ocr/anonymization/status.py @@ -0,0 +1,97 @@ +"""Finite religious-status grammar and optional single-substitution OCR evidence. + +Protection of text only: never attributes a belief to a person. OCR matching is +restricted to long unambiguous status words, not arbitrary identifiers or names. +""" + +import re +from dataclasses import dataclass + +from ocr.schemas import SafeError + +from .normalization import dehyphenate +from .schemas import Entity +from .terminology import _word + +STATUS_VERSION = "religious-status-v1" +STATUS_WORDS = ("konfessionslos", "konfessionsfrei", "religionslos", "nonreligious") +STATUS_PATTERNS = ( + r"keine[rmn]?[ \t]{1,4}(?:Religion|Konfession)", + r"ohne[ \t]{1,4}(?:Religion|Konfession)", + r"religiously[ \t]{1,4}(?:unaffiliated|uncommitted|non-affiliated)", + r"(?:no|without)[ \t]{1,4}(?:religion|religious affiliation)", +) +OCR_CLASSES = (frozenset("il1"), frozenset("o0")) + + +@dataclass(frozen=True) +class StatusPolicy: + ocr_enabled: bool = True + ocr_terms: tuple[str, ...] = STATUS_WORDS + + def __post_init__(self): + if type(self.ocr_enabled) is not bool or ( + type(self.ocr_terms) is not tuple + or not 1 <= len(self.ocr_terms) <= 64 + or any( + not isinstance(t, str) or not re.fullmatch(r"[a-z]{12,40}", t) + for t in self.ocr_terms + ) + ): + raise SafeError("invalid_status_configuration") + + def configuration(self): + return dict( + version=STATUS_VERSION, + regex_flags=["IGNORECASE"], + ocr_ascii_only=True, + word_boundaries="unicode-alnum-combining-underscore-v1", + patterns=STATUS_PATTERNS, + ocr_enabled=self.ocr_enabled, + ocr_terms=sorted(set(self.ocr_terms)), + ocr_min_length=12, + ocr_max_substitutions=1, + ocr_classes=[sorted(c) for c in OCR_CLASSES], + ) + + def detect(self, text): + if not isinstance(text, str) or len(text) > 2_000_000: + raise SafeError("anonymization_text_limit") + view, positions = dehyphenate(text) + found = {} + + def add(left, right, ocr=False): + if (left and _word(view[left - 1])) or (right < len(view) and _word(view[right])): + return + start, end = positions[left], positions[right - 1] + 1 + normalization = "identity" if view == text else "dehyphenation-v1" + if ocr: + normalization = "status-ocr-v1" if view == text else "dehyphenation-status-ocr-v1" + found[(start, end)] = Entity( + start=start, + end=end, + label="RELIGION", + text=text[start:end], + source="deterministic", + model_revision=STATUS_VERSION, + normalization=normalization, + ) + if len(found) > 100_000: + raise SafeError("invalid_entity_spans") + + for pattern in STATUS_PATTERNS: + for match in re.finditer(pattern, view, re.IGNORECASE): + add(match.start(), match.end()) + if self.ocr_enabled: + for match in re.finditer(r"[A-Za-z01]{12,40}", view): + word = match[0].lower() + for term in self.ocr_terms: + if len(word) != len(term): + continue + differences = [(a, b) for a, b in zip(word, term, strict=True) if a != b] + if len(differences) == 1 and any( + set(differences[0]) <= group for group in OCR_CLASSES + ): + add(match.start(), match.end(), ocr=True) + break + return list(found.values()) diff --git a/ocr/anonymization/terminology.py b/ocr/anonymization/terminology.py new file mode 100644 index 0000000000000000000000000000000000000000..fb8ef73ecb5f7d9c07b1a61e571c7fb47ae5d1d3 --- /dev/null +++ b/ocr/anonymization/terminology.py @@ -0,0 +1,188 @@ +"""Bounded literal protection of religious terminology, not personal attribution. + +A match says only that the source contains a protected religious/status term. +Negations, quotations and general topics are intentionally protected too. No +relationship between a term and a nearby person is inferred. This finite lexicon +is neither a spelling corrector nor an exhaustive religion/affiliation detector. +""" + +import unicodedata + +from ocr.schemas import SafeError + +from .normalization import dehyphenate +from .schemas import Entity + +TERMINOLOGY_REVISION = "religious-terminology-v1" +# Explicit EN/DE words and inflections, never fixture IDs, names or context templates. +# Ambiguous adjectives (e.g. orthodox, evangelical) remain contextual model decisions. +DEFAULT_TERMS = ( + "religionslos", + "konfessionslos", + "konfessionsfrei", + "ohne Konfession", + "nicht religiös", + "no religious affiliation", + "no religion", + "nonreligious", + "non-religious", + "atheist", + "atheists", + "atheism", + "atheistisch", + "Atheismus", + "agnostic", + "agnostics", + "agnosticism", + "agnostisch", + "Agnostizismus", + "Christentum", + "Christianity", + "Christen", + "Christian", + "Christians", + "christlich", + "christliche", + "christlichen", + "Katholizismus", + "Catholicism", + "katholisch", + "katholische", + "katholischen", + "Catholic", + "Catholics", + "Protestantismus", + "Protestantism", + "Protestant", + "Protestants", + "evangelisch", + "evangelische", + "evangelischen", + "Judentum", + "Judaism", + "jüdisch", + "jüdische", + "jüdischen", + "Jewish", + "Jude", + "Jüdin", + "Juden", + "Jews", + "Islam", + "Muslim", + "Muslims", + "muslimisch", + "muslimische", + "muslimischen", + "islamisch", + "islamische", + "islamischen", + "Buddhismus", + "Buddhism", + "Buddhist", + "Buddhists", + "buddhistisch", + "buddhistische", + "buddhistischen", + "Hinduismus", + "Hinduism", + "Hindu", + "Hindus", + "hinduistisch", + "hinduistische", + "hinduistischen", + "Sikhismus", + "Sikhism", + "Sikh", + "Sikhs", + "sikhistisch", + "Jainismus", + "Jainism", + "Jain", + "Jains", + "jainistisch", + "Bahai", + "Baháʼí Faith", + "Bahá'í", + "Baha'i", + "Shinto", + "Shintoismus", + "Shintoism", + "Taoismus", + "Taoism", + "Daoismus", + "Daoism", + "Zoroastrismus", + "Zoroastrianism", + "Orthodox Christianity", + "Orthodoxie", +) + + +def _word(c): + # Combining marks and connector punctuation belong to a word too. This avoids + # matching a bare term inside a decomposed or underscore-connected identifier. + return c.isalnum() or c == "_" or unicodedata.category(c).startswith("M") + + +class ReligiousTerminology: + """Operator-configurable literal terms; no user supplied regular expressions. + + At most 256 terms, 80 code points each, 2M input code points and 100k matches. + Case folding and OCR line dehyphenation retain exact original source offsets. + Configuration is trusted operator policy, not per-request model output. + """ + + def __init__(self, terms=DEFAULT_TERMS): + if not isinstance(terms, (tuple, list)) or not 1 <= len(terms) <= 256: + raise SafeError("invalid_terminology_configuration") + if any( + not isinstance(t, str) + or not 1 <= len(t) <= 80 + or t != t.strip() + or not all(c.isalpha() or c in " -'ʼáí" for c in t) + for t in terms + ): + raise SafeError("invalid_terminology_configuration") + self.terms = tuple(sorted({t.casefold() for t in terms}, key=lambda t: (-len(t), t))) + + def detect(self, text): + if not isinstance(text, str) or len(text) > 2_000_000: + raise SafeError("anonymization_text_limit") + normalized, source_positions = dehyphenate(text) + folded_parts, folded_positions = [], [] + for i, char in enumerate(normalized): + part = char.casefold() + folded_parts.append(part) + folded_positions.extend([i] * len(part)) + folded = "".join(folded_parts) + found = {} + for term in self.terms: + offset = 0 + while (left := folded.find(term, offset)) >= 0: + right = left + len(term) + offset = right + if (left and _word(folded[left - 1])) or ( + right < len(folded) and _word(folded[right]) + ): + continue + # Never accept part of an expanding case fold (e.g. one s in ß). + if (left and folded_positions[left - 1] == folded_positions[left]) or ( + right < len(folded_positions) + and folded_positions[right - 1] == folded_positions[right] + ): + continue + start = source_positions[folded_positions[left]] + end = source_positions[folded_positions[right - 1]] + 1 + found[(start, end)] = Entity( + start=start, + end=end, + label="RELIGION", + text=text[start:end], + source="deterministic", + model_revision=TERMINOLOGY_REVISION, + normalization="identity" if normalized == text else "dehyphenation-v1", + ) + if len(found) > 100_000: + raise SafeError("invalid_entity_spans") + return list(found.values()) diff --git a/ocr/api.py b/ocr/api.py new file mode 100644 index 0000000000000000000000000000000000000000..1ecbaa28acc0a7be8a3c414f83639c2aebdbf2cc --- /dev/null +++ b/ocr/api.py @@ -0,0 +1,208 @@ +"""Private dispatcher ASGI app. Does not bind a listener or change production.""" + +from contextlib import asynccontextmanager + +from fastapi import FastAPI, File, Form, Request, UploadFile +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse + +from .config import Settings +from .http_security import Guard, owner +from .jobs import Jobs, status +from .routing import Dispatcher +from .schemas import OCROptions, SafeError +from .security import sensitivity + +CODES = { + "unauthorized": 401, + "job_not_found": 404, + "no_authorized_destination": 403, + "idempotency_conflict": 409, + "result_not_ready": 409, + "job_capacity": 429, + "queue_full": 429, + "upload_limit": 413, + "model_not_ready": 503, +} + + +def create_app(settings=None, cpu=None, gpu=None): + settings = settings or Settings.from_env() + + @asynccontextmanager + async def lifespan(app): + nonlocal cpu, gpu + probe_task = None + if cpu is None: + if settings.cpu_url: + from .adapters.cpu_http import RemoteCPU + + cpu = RemoteCPU(settings.cpu_url, settings.worker_token, settings.timeout) + else: + from .adapters.rapidocr_cpu import RapidCPU + + cpu = RapidCPU(settings.models_dir) + try: + await cpu.initialize() + except SafeError: + pass # /ready reports failure; never pretend warmed readiness. + if gpu is None and settings.gpu_enabled and settings.freellm_url and settings.freellm_key: + from .adapters.freellm_alias import FreeLLMAlias + + gpu = FreeLLMAlias(settings.freellm_url, settings.freellm_key, settings.timeout) + + async def probe(): + try: + await gpu.probe() + except Exception: + gpu.qualified = False + + import asyncio + + probe_task = asyncio.create_task(probe()) + dispatcher = Dispatcher(cpu, gpu, settings) + dispatcher.start() + jobs = Jobs(dispatcher, settings) + jobs.start() + app.state.jobs = jobs + app.state.dispatcher = dispatcher + try: + yield + finally: + if probe_task: + probe_task.cancel() + await asyncio.gather(probe_task, return_exceptions=True) + await jobs.close() + await dispatcher.close() + + app = FastAPI(title="Private Hybrid OCR", version="1.0.0", lifespan=lifespan) + app.add_middleware( + Guard, + token=settings.token, + max_bytes=settings.max_upload, + rate=settings.rate_per_minute, + ) + + @app.exception_handler(SafeError) + async def safe_error(request, exc): + return JSONResponse({"code": exc.code}, status_code=CODES.get(exc.code, 422)) + + @app.exception_handler(RequestValidationError) + async def invalid(request, exc): + return JSONResponse({"code": "invalid_request"}, status_code=422) + + @app.exception_handler(Exception) + async def unexpected(request, exc): + return JSONResponse({"code": "internal_error"}, status_code=500) + + def parse_options(raw): + try: + if len(raw) > 8192: + raise ValueError() + opts = OCROptions.model_validate_json(raw) + opts.sensitivity = sensitivity(opts.sensitivity, settings.min_sensitivity) + opts.max_pages = min(opts.max_pages, settings.max_pages) + return opts + except Exception: + raise SafeError("invalid_options") from None + + async def upload(file): + try: + data = await file.read(settings.max_upload + 1) + finally: + await file.close() + if len(data) > settings.max_upload: + raise SafeError("upload_limit") + return data + + @app.get("/health") + async def health(): + return {"status": "ok"} + + @app.get("/ready") + async def ready(): + cpu_health = await app.state.dispatcher.cpu.backend.health() + return JSONResponse( + {"ready": cpu_health.ready}, status_code=200 if cpu_health.ready else 503 + ) + + @app.post("/v1/ocr/jobs", status_code=202) + async def submit(request: Request, file: UploadFile = File(...), options: str = Form("{}")): + opts = parse_options(options) + data = await upload(file) + key = request.headers.get("idempotency-key") + if key and len(key) > 128: + raise SafeError("invalid_idempotency_key") + j = app.state.jobs.submit(data, file.filename or "", opts, owner(request), key) + return dict( + job_id=j.id, + status=j.status, + request_id=j.request_id, + status_url=f"/v1/ocr/jobs/{j.id}", + result_url=f"/v1/ocr/jobs/{j.id}/result", + ) + + @app.get("/v1/ocr/jobs/{job_id}") + async def get(job_id: str, request: Request): + return status(app.state.jobs.get(job_id, owner(request))) + + @app.get("/v1/ocr/jobs/{job_id}/result") + async def result(job_id: str, request: Request): + j = app.state.jobs.get(job_id, owner(request)) + if j.result is None: + raise SafeError("result_not_ready") + return j.result + + @app.delete("/v1/ocr/jobs/{job_id}") + async def cancel(job_id: str, request: Request): + j = app.state.jobs.get(job_id, owner(request)) + app.state.jobs.cancel(j) + return status(j) + + @app.post("/v1/ocr/page") + async def page(request: Request, file: UploadFile = File(...), options: str = Form("{}")): + opts = parse_options(options) + opts.max_pages = 1 + pages = await app.state.jobs.parse(await upload(file), file.filename or "", opts) + return await app.state.dispatcher.page(pages[0], opts) + + @app.get("/metrics") + async def metrics(): + import numpy as np + + lanes = [app.state.dispatcher.cpu] + ( + [app.state.dispatcher.gpu] if app.state.dispatcher.gpu else [] + ) + return { + lane.backend.name: dict( + queued=lane.queue.qsize(), + active=lane.active, + completed=lane.completed, + errors=lane.errors, + ewma_ms=lane.ewma, + cold_ms=lane.cold_ms, + warm_latency_ms={ + str(p): float(np.percentile(lane.samples, p)) if lane.samples else None + for p in (50, 95, 99) + }, + ) + for lane in lanes + } + + from .openai_facade import ChatRequest, completion, image_from_request + + @app.post("/v1/chat/completions") + async def chat(body: ChatRequest): + data, filename = image_from_request(body, settings.max_upload) + opts = body.ocr_options.model_copy(deep=True) + opts.sensitivity = sensitivity(opts.sensitivity, settings.min_sensitivity) + opts.mode = {"ocr-auto": "auto", "ocr-cpu": "cpu_only", "ocr-gpu": "gpu_only"}[body.model] + opts.max_pages = 1 + pages = await app.state.jobs.parse(data, filename, opts) + result = await app.state.dispatcher.page(pages[0], opts) + return completion(result, body.model) + + return app + + +app = create_app() diff --git a/ocr/benchmark.py b/ocr/benchmark.py new file mode 100644 index 0000000000000000000000000000000000000000..0c28fbf31e5cb9f9c4f59dba8fbb33f9ae3aba84 --- /dev/null +++ b/ocr/benchmark.py @@ -0,0 +1,179 @@ +"""Real CPU benchmark. Mocked provider timings never qualify overflow.""" + +import argparse +import asyncio +import hashlib +import json +import os +import platform +import resource +import shutil +import subprocess +import time +from pathlib import Path + +import numpy as np +import psutil + +from .adapters.rapidocr_cpu import RapidCPU +from .config import Settings +from .fixtures.generate import generate +from .ingest import ingest +from .routing import Dispatcher +from .schemas import OCROptions, SafeError + + +def distance(a, b): + prev = list(range(len(b) + 1)) + for i, x in enumerate(a, 1): + row = [i] + for j, y in enumerate(b, 1): + row.append(min(row[-1] + 1, prev[j] + 1, prev[j - 1] + (x != y))) + prev = row + return prev[-1] + + +async def benchmark(args): + manifest = generate(args.fixtures) + settings = Settings.from_env() + cpu = RapidCPU(settings.models_dir) + process = psutil.Process() + cpu_start = time.process_time() + init = time.perf_counter() + await cpu.initialize() + init_ms = (time.perf_counter() - init) * 1000 + dispatcher = Dispatcher(cpu, None, settings) + dispatcher.start() + options = OCROptions(mode="cpu_only", sensitivity="D0") + rows = [] + start = time.perf_counter() + try: + for fixture in manifest: + path = Path(args.fixtures) / fixture["file"] + pages = ingest(path.read_bytes(), path.name, options, settings) + times = [] + result = None + for _ in range(args.runs): + t = time.perf_counter() + try: + result = await dispatcher.page(pages[0], options) + except SafeError as e: + rows.append( + dict( + fixture=fixture["file"], + category=fixture["category"], + source="failed", + failure=e.code, + latency_ms=[(time.perf_counter() - t) * 1000], + cer=None, + wer=None, + ) + ) + break + times.append((time.perf_counter() - t) * 1000) + if result is None: + continue + reference = " ".join(fixture["text"].split()) + actual = " ".join(result.text.split()) + rows.append( + dict( + fixture=fixture["file"], + category=fixture["category"], + source=result.source, + latency_ms=times, + cer=distance(reference, actual) / max(1, len(reference)), + wer=distance(reference.split(), actual.split()) + / max(1, len(reference.split())), + lines=len(result.blocks), + flags=result.flags, + box_bounds_valid=all( + all(0 <= v <= 1000 for v in b.bbox_1000) + for b in result.blocks + if b.bbox_1000 + ), + ) + ) + batches = [] + for count in args.batch_sizes: + path = Path(args.fixtures) / f"mixed-{count}.pdf" + batch_start = time.perf_counter() + pages = ingest(path.read_bytes(), path.name, options, settings) + # Bound per-job fanout as in production, while preserving original order. + sem = asyncio.Semaphore(settings.pages_per_job) + + async def run(p, sem=sem): + async with sem: + return await dispatcher.page(p, options) + + results = await asyncio.gather(*(run(p) for p in pages)) + elapsed = time.perf_counter() - batch_start + batches.append( + dict( + pages=count, + seconds=elapsed, + pages_per_minute=count / elapsed * 60, + ordered=[r.page_index for r in results] == list(range(count)), + embedded=sum(r.source == "embedded_text" for r in results), + cpu=sum(r.source == "cpu_ocr" for r in results), + mode="cpu_only", + gpu_calls=0, + ) + ) + elapsed = time.perf_counter() - start + lat = [v for row in rows if row["source"] == "cpu_ocr" for v in row["latency_ms"]] + root = Path(__file__).parent + fingerprint = hashlib.sha256() + for path in sorted(root.rglob("*.py")): + if "tests" not in path.parts: + fingerprint.update(str(path.relative_to(root)).encode()) + fingerprint.update(path.read_bytes()) + revision = os.getenv("OCR_SOURCE_REVISION") + if not revision and shutil.which("git"): + try: + revision = subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL + ).strip() + except subprocess.CalledProcessError: + pass + report = dict( + schema_version="ocr-benchmark/v1", + commit=revision, + runtime_sha256=fingerprint.hexdigest(), + cpu_quota=Path("/sys/fs/cgroup/cpu.max").read_text().strip() + if Path("/sys/fs/cgroup/cpu.max").exists() + else None, + memory_limit=Path("/sys/fs/cgroup/memory.max").read_text().strip() + if Path("/sys/fs/cgroup/memory.max").exists() + else None, + platform=platform.platform(), + python=platform.python_version(), + logical_cpus=os.cpu_count(), + engine="PP-OCRv6 Small / RapidOCR 3.10.0 / ORT CPU 1.24.1", + model_init_ms=init_ms, + warm_latency_ms={str(p): float(np.percentile(lat, p)) for p in (50, 95, 99)}, + failed_fixtures=sum(row["source"] == "failed" for row in rows), + process_threads=process.num_threads(), + peak_rss_mib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, + cpu_seconds=time.process_time() - cpu_start, + wall_seconds=elapsed, + rows=rows, + batches=batches, + limitations=[ + "Synthetic printed ground truth only; no human handwriting corpus", + "No live GPU, quota/cost, HF Basic, OpenVINO, table semantics or ground-truth box IoU qualification", + "No mixed CPU/GPU throughput gain measured; overflow remains disabled", + ], + ) + Path(args.output).write_text(json.dumps(report, indent=2)) + print(json.dumps({k: v for k, v in report.items() if k not in ("rows",)}, indent=2)) + finally: + await dispatcher.close() + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--fixtures", default="/tmp/ocr-fixtures") + p.add_argument("--output", default="/tmp/ocr-benchmark.json") + p.add_argument("--runs", type=int, default=3) + p.add_argument("--batch-sizes", type=int, nargs="*", default=[20, 100]) + asyncio.run(benchmark(p.parse_args())) diff --git a/ocr/classify.py b/ocr/classify.py new file mode 100644 index 0000000000000000000000000000000000000000..30562b3ed020826fdbcf8d0d3bca468ec4509ef7 --- /dev/null +++ b/ocr/classify.py @@ -0,0 +1,34 @@ +import io + +import numpy as np +from PIL import Image, ImageFilter + + +def features(image: bytes) -> dict: + with Image.open(io.BytesIO(image)) as im: + gray = im.convert("L").resize((256, 256)) + a = np.asarray(gray, dtype=float) + ink = a < 180 + contrast = float(a.std()) + edge = float(np.asarray(gray.filter(ImageFilter.FIND_EDGES), dtype=float).var()) + horizontal = int((ink.mean(axis=1) > 0.7).sum()) + vertical = int((ink.mean(axis=0) > 0.7).sum()) + # These are proxies, never semantic handwriting/table claims. + density = float(ink.mean()) + ink_contrast = contrast / max(0.001, density**0.5) + return dict( + blank=density < 0.0001, + contrast=contrast, + ink_contrast=ink_contrast, + edge_variance=edge, + line_density=density, + table_rulings=horizontal + vertical, + possible_columns=bool( + ink[:, :100].mean() > 0.03 + and ink[:, 156:].mean() > 0.03 + and ink[:, 120:136].mean() < 0.01 + ), + complex=horizontal + vertical > 8 or (ink_contrast < 145 and density > 0.001), + handwriting_proxy="unknown", + orientation="landscape" if im.width > im.height else "portrait", + ) diff --git a/ocr/cli.py b/ocr/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..caa940faae566ef66eeb60eb304bdc19efdc20e2 --- /dev/null +++ b/ocr/cli.py @@ -0,0 +1,56 @@ +import argparse +import asyncio +import json +from pathlib import Path + +from .config import Settings +from .jobs import Jobs +from .routing import Dispatcher +from .schemas import OCROptions + + +async def main(args): + settings = Settings.from_env() + if args.command == "probe-alias": + from .adapters.freellm_alias import FreeLLMAlias + + backend = FreeLLMAlias(settings.freellm_url, settings.freellm_key) + try: + print(json.dumps(await backend.probe(), indent=2)) + finally: + await backend.close() + return + from .adapters.rapidocr_cpu import RapidCPU + + cpu = RapidCPU(settings.models_dir) + await cpu.initialize() + dispatcher = Dispatcher(cpu, None, settings) + dispatcher.start() + jobs = Jobs(dispatcher, settings) + options = OCROptions(mode="cpu_only", sensitivity=args.sensitivity) + try: + path = Path(args.file) + job = jobs.submit(path.read_bytes(), path.name, options, "cli") + await job.task + if not job.result: + raise RuntimeError("job_failed") + if args.format == "markdown": + print(job.result.markdown) + else: + print(job.result.model_dump_json(indent=2)) + if job.status != "completed": + raise SystemExit(1) + finally: + await jobs.close() + await dispatcher.close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command", required=True) + run = sub.add_parser("run") + run.add_argument("file") + run.add_argument("--sensitivity", choices=["D0", "D1", "D2"], default="D2") + run.add_argument("--format", choices=["json", "markdown"], default="json") + sub.add_parser("probe-alias") + asyncio.run(main(parser.parse_args())) diff --git a/ocr/config.py b/ocr/config.py new file mode 100644 index 0000000000000000000000000000000000000000..16e286d8d198f8d3246a0b2b473531cabd430410 --- /dev/null +++ b/ocr/config.py @@ -0,0 +1,121 @@ +import os +from dataclasses import dataclass +from urllib.parse import urlparse + + +def flag(name, default=False): + return os.getenv(name, str(default)).lower() == "true" + + +@dataclass +class Settings: + token: str = "" + worker_token: str = "" + cpu_url: str = "" + freellm_url: str = "" + freellm_key: str = "" + gpu_enabled: bool = False + overflow_enabled: bool = False + overflow_qualified: bool = False + dual_run: bool = False + min_sensitivity: str = "D2" + external_cpu_classes: tuple = ("D0",) + max_upload: int = 25 * 1024 * 1024 + max_pages: int = 100 + max_pixels: int = 20_000_000 + total_pixels: int = 200_000_000 + cpu_workers: int = 1 + gpu_workers: int = 2 + queue_size: int = 32 + active_jobs: int = 4 + retained_jobs: int = 128 + max_result_bytes: int = 8 * 1024 * 1024 + retained_bytes: int = 64 * 1024 * 1024 + pages_per_job: int = 2 + ttl: float = 1800 + timeout: float = 90 + job_timeout: float = 900 + gpu_budget: int = 20 + rate_per_minute: int = 60 + models_dir: str = "/tmp/ocr-models" + + def __post_init__(self): + if self.min_sensitivity not in ("D0", "D1", "D2"): + raise ValueError("invalid_min_sensitivity") + for n in ( + "max_upload", + "max_pages", + "max_pixels", + "total_pixels", + "cpu_workers", + "gpu_workers", + "queue_size", + "active_jobs", + "retained_jobs", + "pages_per_job", + "ttl", + "timeout", + "job_timeout", + "gpu_budget", + "rate_per_minute", + ): + if getattr(self, n) <= 0: + raise ValueError("invalid_limit") + if ( + self.max_upload > 25 * 1024 * 1024 + or self.max_pages > 100 + or self.max_pixels > 20_000_000 + ): + raise ValueError("limit_exceeds_safety_ceiling") + for url in (self.cpu_url, self.freellm_url): + if not url: + continue + p = urlparse(url) + if ( + p.username + or p.password + or p.query + or p.fragment + or p.scheme not in ("http", "https") + ): + raise ValueError("invalid_backend_url") + if p.scheme == "http" and p.hostname not in ( + "localhost", + "127.0.0.1", + "::1", + ): + raise ValueError("remote_tls_required") + + @classmethod + def from_env(cls): + if os.getenv("OCR_GPU_MODEL", "alias-ocr") != "alias-ocr": + raise ValueError("unsupported_gpu_model") + return cls( + token=os.getenv("OCR_DISPATCHER_TOKEN", ""), + worker_token=os.getenv("OCR_CPU_AUTH_TOKEN", ""), + cpu_url=os.getenv("OCR_CPU_URL", "").rstrip("/"), + freellm_url=os.getenv("FREELLMAPI_BASE_URL", "").rstrip("/"), + freellm_key=os.getenv("FREELLMAPI_API_KEY", ""), + gpu_enabled=flag("OCR_ENABLE_GPU"), + overflow_enabled=flag("OCR_ENABLE_GPU_OVERFLOW"), + overflow_qualified=flag("OCR_OVERFLOW_BENCHMARK_QUALIFIED"), + dual_run=flag("OCR_QUALITY_DUAL_RUN"), + min_sensitivity=os.getenv("OCR_MIN_SENSITIVITY", "D2"), + external_cpu_classes=tuple( + c for c in ("D0", "D1", "D2") if flag("OCR_ALLOW_EXTERNAL_CPU_" + c, c == "D0") + ), + max_upload=int(os.getenv("OCR_MAX_UPLOAD_MIB", "25")) * 1024 * 1024, + max_pages=int(os.getenv("OCR_MAX_PAGES", "100")), + total_pixels=int(os.getenv("OCR_MAX_TOTAL_PIXELS", "200000000")), + queue_size=int(os.getenv("OCR_BACKEND_QUEUE_CAPACITY", "32")), + active_jobs=int(os.getenv("OCR_MAX_ACTIVE_JOBS", "4")), + retained_jobs=int(os.getenv("OCR_MAX_RETAINED_JOBS", "128")), + timeout=float(os.getenv("OCR_BACKEND_TIMEOUT_SECONDS", "90")), + job_timeout=float(os.getenv("OCR_JOB_TIMEOUT_SECONDS", "900")), + gpu_budget=int(os.getenv("OCR_GPU_REQUESTS_PER_MINUTE", "20")), + max_pixels=int(os.getenv("OCR_MAX_PIXELS", "20000000")), + cpu_workers=int(os.getenv("OCR_CPU_CONCURRENCY", "1")), + gpu_workers=int(os.getenv("OCR_GPU_CONCURRENCY", "2")), + ttl=float(os.getenv("OCR_JOB_TTL_MIN", "30")) * 60, + models_dir=os.getenv("OCR_MODELS_DIR", "/tmp/ocr-models"), + ) diff --git a/ocr/deploy/hf-cpu/Dockerfile b/ocr/deploy/hf-cpu/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..7308f85a2c1ef79f4d1a23bb633e791b109c7a9d --- /dev/null +++ b/ocr/deploy/hf-cpu/Dockerfile @@ -0,0 +1,23 @@ +# syntax=docker/dockerfile:1 +FROM python:3.11.17-slim-bookworm +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 OMP_NUM_THREADS=2 OPENBLAS_NUM_THREADS=1 \ + OCR_MODELS_DIR=/opt/ocr-models HOME=/home/ocr +RUN apt-get update && apt-get install -y --no-install-recommends fonts-dejavu-core && rm -rf /var/lib/apt/lists/* +ARG OCR_SOURCE_REVISION=unknown +ENV OCR_SOURCE_REVISION=$OCR_SOURCE_REVISION +WORKDIR /app +# Build from repository root. Only OCR dependencies and code enter this separate worker. +COPY ocr/requirements.lock /tmp/requirements.lock +RUN --mount=type=secret,id=proxy_ca \ + if [ -f /run/secrets/proxy_ca ]; then export PIP_CERT=/run/secrets/proxy_ca; fi; \ + pip install --no-cache-dir --require-hashes -r /tmp/requirements.lock +COPY ocr /app/ocr +# Download once at build time, verify pinned SHA256s, keep inference offline. +RUN --mount=type=secret,id=proxy_ca \ + if [ -f /run/secrets/proxy_ca ]; then export SSL_CERT_FILE=/run/secrets/proxy_ca; fi; \ + python -m ocr.download_models +RUN useradd --uid 1000 --create-home ocr && chmod -R a+rX /opt/ocr-models /app +USER ocr +EXPOSE 7860 +HEALTHCHECK --interval=30s --timeout=5s CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:7860/health',timeout=3)" +CMD ["uvicorn", "ocr.worker_api:app", "--host", "0.0.0.0", "--port", "7860", "--workers", "1", "--no-access-log", "--limit-concurrency", "4"] diff --git a/ocr/deploy/hf-cpu/README.md b/ocr/deploy/hf-cpu/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0d27bb0db4d27e6dd1dcc0b0effb8f4964022302 --- /dev/null +++ b/ocr/deploy/hf-cpu/README.md @@ -0,0 +1,28 @@ +--- +title: Authenticated CPU OCR +emoji: 📄 +colorFrom: blue +colorTo: gray +sdk: docker +app_port: 7860 +license: mit +--- + +Separate stateless PP-OCRv6 Small CPU worker. This is not the existing Augmentoolkit +Space. No public demo or document persistence. Set **OCR_CPU_AUTH_TOKEN** as a Space +secret. `/health` is public and content-free; all processing, capabilities and +readiness endpoints require `Authorization: Bearer ...`. + +Create a separate CPU Basic Space only after owner approval. From repository root, +copy `ocr/` and this Dockerfile into a staging directory, with this README at its +root. Build context must contain `ocr/requirements.lock`, code, and `ocr/models.json`. +Do not upload the entire Augmentoolkit repository, inputs, credentials or outputs. +CPU Basic sizing (2 vCPU/16 GB) must be confirmed on the selected account. + +One process warms detector and recognizer. Readiness is 503 if models fail integrity +or initialization. Weights are downloaded and checksum-verified at build time; +runtime inference does not need model-download egress. Space sleep destroys +processes; readiness must be rechecked after wake. Token privacy is not D2 processing +authorization. Default dispatcher policy allows remote CPU only for public D0. + +See `ocr/docs/deployment.md` for build, validation, approval and rollback steps. diff --git a/ocr/deploy/private-anonymization/README.md b/ocr/deploy/private-anonymization/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8847b698a8147f5668500bac022b95d6a3e9852a --- /dev/null +++ b/ocr/deploy/private-anonymization/README.md @@ -0,0 +1,106 @@ +# Private staging specification — NOT deployed + +The declarative `staging.json` and `.service.disabled` template are review artifacts. +`deployment_enabled=false`, production identity is BLOCKED, and both the real host +adapter and worker entrypoints are unset. This task does not supply a fake identity +provider or a default production worker. Do not enable the unit as-is or use the +isolated test adapter as production wiring. There are no new endpoints or ports. + +## Qualified resource profile and remaining host gates + +The tested baseline is ONE persistent model instance, two inference threads, +one active job, four vCPU and an 8-GiB memory ceiling. Tests used a bounded Docker +probe on this cloud machine, with an additional live 1.5-GiB memory/CPU load. +The first two runs peaked around 4.4 GiB total cgroup usage, leaving about 3.6 GiB. +See the exact per-run artifacts for startup, warm latency, process RSS and headroom. +The third probe also measures native TCP listeners. These are synthetic short-text +measurements, not qualification of maximal documents or the actual Debian host. + +Propose at least 12 GiB host RAM to reserve room beyond this 8-GiB worker profile; +measure the real host's competing applications before staging. This is a deployment +baseline, not proof that 12 GiB is the universal minimum. The host application must +enforce the manifest's bounded queue, input, retention and timeout settings. No such +production host application was available to qualify here. Do not use multiple +model instances under this memory budget without a new load test. + +## Provisioning sequence after separate authorization + +1. Pin the reviewed code SHA and Python 3.11 runtime. Install the existing OCR and + separate anonymization hash-locked dependencies in a dedicated venv. No NER + dependency or checkpoint is added to the public HF worker image. +2. Provision the fixed model snapshot explicitly, verify `artifacts.json`, then make + its cache read-only to the worker. Runtime sets HF/Transformers offline and + disables telemetry. The supplied unit isolates networking and allows no public + listeners; external transport is not implemented or authorized. +3. Allocate a dedicated non-login UID and encrypted persistent local filesystem. + State/review/candidate/export directories must be owned by that UID, mode 0700; + classified files and SQLite database are 0600, with UMask=0077. Original OCR + evidence is immutable, private and separately retained. Model weights are public + artifacts but immutable/read-only at runtime. Do not log classified paths/text. +4. Provision the ledger ONCE using `SQLiteApprovalLedger(path, initialize=True)` + in the private 0700 state directory, then record `ledger.ledger_id` in private + configuration. Normal startup must use `SQLiteApprovalLedger(path, + expected_id=recorded_id)`; it MUST NOT use initialize=True. Startup refuses + missing/corrupt/wrong-ID storage instead of resetting authorization history. + Existing pre-Gate03c ledgers require an explicit audited migration or key rotation; + automatic destructive migration is deliberately absent. +5. Integrate `HostSecurityAdapter` against the real identity/session verifier, + immutable completed-review and candidate/export stores, current ACL/revocation + data and operator destination registry. Authenticate opaque credentials, not + requested subject IDs. Review and destination data must be rechecked at export. + The concrete adapter and private host entrypoint remain BLOCKED here. +6. Load a signing key from private secret storage (the unit uses systemd credentials). + All consumers in a single key domain must use the same durable ledger. This + implementation supports single-host SQLite, or distinct actual signing keys per + host. `validate_key_topology(..., signing_keys=...)` compares computed fingerprints; + different aliases for the same secret do not bypass the guard. Shared-key + multi-host deployment is BLOCKED: no shared durable backend has been implemented. + A configuration flag claiming shared durability is not qualification evidence. +7. After identity, physical destination and fresh recognition gates are resolved, + repeat the resource/network/permission tests on the actual private staging host + and obtain separate deployment authorization. Never point D1/D2 at Agent-0 or + an external model in anticipation of anonymization. + +## Destination and transport boundary + +`PrivateFileTransport` supports only operator-configured private local directories. +It accepts a destination ID, never a path/URL, opens the verified directory without +symlink following, pins its device/inode, writes mode-0600 bytes and atomically +publishes under the artifact SHA-256 filename. It has no HTTP client, redirects, +public fallback or remote destination mechanism. Changed roots/policies are denied. + +A successful export does NOT call this transport. A separate operation must +authenticate again, check an authoritative exported-artifact digest, and grant +`anonymization:private-transfer` for the retained classification and destination. +An approved JSON export alone does not authorize this. D1/D2 remain D1/D2 and the +payload retains `transmission_authorized=false`; the transport receipt records only +the explicitly authorized local delivery. Remote transmission stays disabled. + +## Failure recovery and rollback + +- Stop intake and the private worker before rollback; do not restart any production + service as part of this PR. Drain/cancel pending work without publishing candidates. +- Keep the current ledger, pinned ledger ID and signing-key domain intact across + code rollback. NEVER restore an older ledger snapshot while its signing key is + still valid: this could resurrect consumed approvals. If durable state is lost, + corrupt or replaced, fail closed; rotate the key and invalidate outstanding + approvals before explicit new provisioning. SQLite backups/migrations require an + application-consistent procedure, not a copy of a live database file. +- Switch the private code/venv pointer to the previously reviewed SHA only if that + version understands the current ledger schema and contracts. Otherwise keep the + worker stopped and perform an audited migration. Do not silently downgrade to + the pre-Gate03c auto-create ledger behavior or legacy anonymizer. +- Preserve original evidence and all unapproved candidates as classified. Recovery + may remove only abandoned `.anonymization-*` / `.private-transfer-*` staging files + through `recover_staging(private_directory)`. It obtains an exclusive directory + lock; active writers hold a shared lock. Symlinks and ordinary evidence filenames + are retained. Killed-writer recovery is tested with real process termination. +- Unlinking is not secure erasure on SSDs/copy-on-write/snapshots. Encryption, + retention, snapshot access and secure destruction remain private-host operator + duties. No classified-file cleanup is scheduled or activated automatically here. +- A code rollback does not enable release. `automated_release_eligible=false`, + deployment disabled and manual review/authorization remain mandatory. + +The newly frozen holdout still has unmasked religious-reference canaries. Therefore +private deployment and real sensitive-document processing remain NO-GO, independent +of the successful local security-boundary tests. See the Gate 03c report. diff --git a/ocr/deploy/private-anonymization/anonymization-staging.service.disabled b/ocr/deploy/private-anonymization/anonymization-staging.service.disabled new file mode 100644 index 0000000000000000000000000000000000000000..64034686a6f80c7858aa87556dd984366960f2d1 --- /dev/null +++ b/ocr/deploy/private-anonymization/anonymization-staging.service.disabled @@ -0,0 +1,36 @@ +# TEMPLATE ONLY. Not installed/enabled; deployment and real host bindings BLOCKED. +[Unit] +Description=Private anonymization staging host (manual approval required) +ConditionPathExists=/etc/private-anonymization/APPROVED-STAGING-CONFIG + +[Service] +Type=simple +User=augmentanon +Group=augmentanon +UMask=0077 +EnvironmentFile=/etc/private-anonymization/host.env +Environment=HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_HUB_DISABLE_TELEMETRY=1 +WorkingDirectory=/opt/private-anonymization/current +# PRIVATE_ANONYMIZATION_HOST_MODULE must be the separately audited real host app. +# No default/mock module is supplied and this template must not be enabled as-is. +ExecStart=/opt/private-anonymization/venv/bin/python -m ${PRIVATE_ANONYMIZATION_HOST_MODULE} +Restart=no +TimeoutStopSec=30 +StateDirectory=private-anonymization +StateDirectoryMode=0700 +MemoryMax=8G +CPUQuota=400% +TasksMax=64 +NoNewPrivileges=yes +PrivateNetwork=yes +RestrictAddressFamilies=AF_UNIX +IPAddressDeny=any +PrivateTmp=yes +ProtectSystem=strict +ProtectHome=yes +ReadWritePaths=/var/lib/private-anonymization +ReadOnlyPaths=/var/cache/private-anonymization/model +LoadCredential=release-signing-key:/etc/private-anonymization/release-signing-key + +[Install] +WantedBy=multi-user.target diff --git a/ocr/deploy/private-anonymization/staging.json b/ocr/deploy/private-anonymization/staging.json new file mode 100644 index 0000000000000000000000000000000000000000..a0e73402a746b0b0d80c404ce0e96daff8473325 --- /dev/null +++ b/ocr/deploy/private-anonymization/staging.json @@ -0,0 +1,18 @@ +{ + "schema_version": "private-anonymization-staging/v1", + "deployment_enabled": false, + "production_identity_integration": "BLOCKED", + "host_adapter_entrypoint": null, + "worker_entrypoint": null, + "replicas": 1, + "ledger_backend": "single-host-local-sqlite", + "multi_host_shared_key_support": "BLOCKED: no shared durable backend implemented", + "host_key_domains": "Each host requires a distinct actual signing key; compare computed fingerprints, not aliases", + "resources": {"minimum_qualification_profile_vcpu": 4, "worker_memory_max_gib": 8, "recommended_host_memory_gib": 12, "model_instances": 1, "inference_threads": 2, "concurrent_jobs": 1}, + "bounds": {"staging_max_text_characters": 10000, "maximum_job_runtime_seconds": 300, "queue_capacity": 2, "bounded_result_retention_required": true}, + "model": {"id": "VAGOsolutions/SauerkrautLM-GLiNER", "revision": "ba967ee030b7e679dd5422cb16fcd1b379da38e1", "manifest": "ocr/anonymization/artifacts.json", "cache": "/var/cache/private-anonymization/model", "runtime_downloads": false}, + "runtime": {"python": "3.11", "locks": ["ocr/requirements.lock", "ocr/anonymization/requirements.lock"], "persistent_process": true, "entrypoint_status": "BLOCKED until real host application and adapters exist", "automatic_restart": false}, + "network": {"enabled": false, "public_ports": [], "listeners": [], "remote_transmission": false, "transport": "explicit approved local private filesystem only"}, + "filesystem": {"state_root": "/var/lib/private-anonymization", "directories_mode": "0700", "classified_files_mode": "0600", "model_cache_read_only": true, "source_evidence_immutable": true, "encrypted_volume_required": true, "ledger_id_pinned_in_private_configuration": true, "startup_ledger_creation": false}, + "authorization": {"authenticated_identity_required": true, "immutable_completed_review_store_required": true, "current_acl_and_revocation_required": true, "export_artifact_registry_required": true, "separate_transfer_scope": "anonymization:private-transfer", "automated_release_eligible": false, "automatic_exports": false} +} diff --git a/ocr/docs/adr.md b/ocr/docs/adr.md new file mode 100644 index 0000000000000000000000000000000000000000..8333b6afa11e98fe9cb515a44371fbe60a1a7c47 --- /dev/null +++ b/ocr/docs/adr.md @@ -0,0 +1,29 @@ +# Decisions and deviations from spec.md + +1. **Canonical spec**: preserve `spec (10).md`; copy it byte-for-byte to `spec.md`. +2. **Unavailable infrastructure**: user explicitly deferred Debian/alias access. + Deliver runtime-configurable OpenAI-compatible integration and offline mocks; + do not claim actual model, provider, grounding or D2 qualification. +3. **D2 fail closed**: there is no trustworthy executed-destination/fallback + attestation in this checkout. No metadata/environment flag enables D2 alias or + remote HF routing. The approved catalogue is recorded without changing its + existing policies. Local CPU remains available for D2. +4. **Ingestion isolation**: use pypdf with decompression/work caps plus PDFium in a + disposable limited parser process. No persistence, credential inheritance or + arbitrary remote fetch. This is additional resource containment, not a sandbox + claim. Linux is the supported dispatcher deployment target. +5. **CPU classification**: pin detector and recognizer only. Do not silently add a + differently tiered orientation model. EXIF and PDF rotation are preserved; + scanned text rotated inside the page is a known accuracy limitation. +6. **Provenance first**: output selection controls assembled representations; + normalized source-page evidence and sensitivity remain available. Alias boxes, + confidence, semantic tables/math and verified provider identity remain null or + false unless empirically qualified. Quality alternatives retain disagreements. +7. **Deployment isolation**: new `ocr.worker_api:app` and `ocr.api:app` are separate + from existing Augmentoolkit. The explicit client/anonymization handoff bridge + does not silently replace Tesseract or the legacy anonymizer in production. +8. **Performance gates**: CPU measurements on this cloud machine are not HF CPU + Basic measurements. OpenVINO, genuine handwriting, human-annotated box IoU, + structural table fidelity and live hybrid cost/quota qualification remain + outstanding. Overflow remains disabled. Synthetic corpus labels are known + rendered source text; no independent human-reviewed document corpus is claimed. diff --git a/ocr/docs/api.md b/ocr/docs/api.md new file mode 100644 index 0000000000000000000000000000000000000000..1764d66c46de01458899fe62e157acb0dcb7aa46 --- /dev/null +++ b/ocr/docs/api.md @@ -0,0 +1,67 @@ +# Private OCR API + +Run the separate `ocr.api:app` ASGI application through an existing trusted ingress +or an explicitly approved loopback/Unix socket. It is not mounted into or started +by the existing Augmentoolkit application. Runtime settings are read from process +environment (Space secrets or Debian secret injection), never bundled in code. + +All routes except `/health` require `Authorization: Bearer $OCR_DISPATCHER_TOKEN`. +`GET /ready` returns 200 only when CPU is warmed, otherwise 503. `/docs` and +`/openapi.json` provide the generated versioned schema. `/metrics` is authenticated +and returns content-free lane counters, queue depth, cold/warm latency percentiles. + +| Method/path | Contract | +|---|---| +| POST `/v1/ocr/jobs` | Multipart `file`, `options` JSON; 202 with job/status/result URLs | +| GET `/v1/ocr/jobs/{id}` | queued/running/completed/partial/failed/cancelled, counts, timestamps, safe errors | +| GET `/v1/ocr/jobs/{id}/result` | `ocr-result/v1` JSON, only when a result is retained; 409 otherwise | +| DELETE `/v1/ocr/jobs/{id}` | Cancel pending work and purge retained result; no promise of undoing remote work | +| POST `/v1/ocr/page` | One image or one-page PDF, normalized `ocr-page/v1` result | +| POST `/v1/chat/completions` | Restricted single-image façade described below | + +Use `Idempotency-Key` (max 128 characters) on job creation. Keys are scoped to the +authenticated service credential, not globally. Reuse with different effective +options/bytes returns 409. Multiple users sharing a token share that scope; this +is not a multi-tenant identity provider. 404 does not disclose another credential's +job. TTL eviction removes result/idempotency records. Errors include 401 auth, +403 unauthorized destination, 413 size, 422 input/backend errors, 429 capacity/rate, +409 result/idempotency state, 503 unavailable readiness. + +```sh +# Existing private endpoint supplied by the operator; never use a guessed port. +curl --fail-with-body -H "Authorization: Bearer $OCR_DISPATCHER_TOKEN" \ + -F 'file=@public-example.pdf' \ + -F 'options={"mode":"cpu_only","sensitivity":"D0","output":"both"}' \ + "$OCR_DISPATCHER_URL/v1/ocr/jobs" +``` + +Options: `mode` (auto/cpu_only/gpu_only/quality/throughput), `sensitivity`, `output` +(text/markdown/structured/both), `language_hints`, `layout`, `preserve_boxes`, +`extract_pdf_text_layer`, `force_ocr`, `prefer_cpu`, `allow_external_cpu`, +`allow_gpu_overflow`, `max_pages`. Unknown fields are rejected. `output` selects +assembled text/Markdown fields; per-page source evidence remains in the envelope. +Language hints do not replace the pinned multilingual recognition model. CPU +preference is a hint subject to explicit mode, quality and capability requirements. + +Results contain stable zero-based page indexes and SHA256-derived page identities, +engine/backend versions, sensitivity, normalized blocks, flags, route reasons, +attempt counts/timing/destinations and alternative evidence. Markdown delimiters +are one-based ``. Geometry is finite 0..1000 in source-raster +orientation. Missing geometry/confidence stays null; alias destination stays null +when not verifiable. No invented token usage or LightOn confidence. + +The optional façade accepts `model: ocr-auto | ocr-cpu | ocr-gpu`, `stream:false`, +and exactly one user message containing exactly one image_url with +`data:image/png;base64,...` (JPEG/TIFF also supported). An `ocr_options` extension +can increase sensitivity or restrict routing. It returns standard non-streaming +chat choices plus an `ocr` provenance extension, without invented `usage`. +Arbitrary text messages, remote URLs, tools/functions, extra parameters, streaming +and PDFs are rejected. This is not full OpenAI API compatibility and does not +modify FreeLLMAPI. The raw JSON body cap makes effective inline-image capacity +slightly smaller than the 25 MiB binary multipart limit. + +Worker API: separate token `OCR_CPU_AUTH_TOKEN`, `POST /internal/ocr/page` multipart +one raster image (`file`, optional bounded `request_id`, empty `{}` `options`). +`GET /internal/capabilities` returns conservative model capabilities. It admits one +inference at a time and rejects excess work, rather than accumulating an unbounded +queue. It does not receive FreeLLMAPI credentials or document classification policy. diff --git a/ocr/docs/audit.md b/ocr/docs/audit.md new file mode 100644 index 0000000000000000000000000000000000000000..ef8018923589b62596672bb2cb201c6143f8e82e --- /dev/null +++ b/ocr/docs/audit.md @@ -0,0 +1,48 @@ +# Phase 0 audit + +Requested branch: `deploy-hf-space-augmentanonymous-14035769368160495102`. +Fetched HEAD: `fe293a173364dfbfc0166f9dd26b0ed1eaef8c57`. +Implementation worktree: `/workspace/augmentoolkit-ocr`, branch `implement/hybrid-ocr`. +The pre-existing `/workspace/augmentoolkit` checkout and its services are separate. +No existing tracked changes were present when the worktree was created. + +The supplied specification is `spec (10).md`; `spec.md` is an exact canonical copy. +`Agent.md` describes the existing `Leon4gr45/augmentanonymous` Docker Space and +its unified port. It is not authorization to replace that deployment. The OCR +worker is packaged separately; no production upload or service mutation occurred. + +Existing ingestion: `generation/core_components/chunking.py` uses PyMuPDF and +Tesseract when PDF text is short. Existing `augmentoolkit/utils/anonymization.py` +uses Presidio/NLTK and can return unchanged text on failure. Neither is replaced +or silently invoked by the new service. `ocr.handoff` supplies an explicit bridge +and classified evidence for a reviewed downstream anonymizer. + +No existing OCR dispatcher, FreeLLMAPI endpoint, `alias-ocr` configuration, +provider enforcement protocol, or D2 router implementation was found in this +checkout. There were no FreeLLMAPI/OCR/HF credential variable bindings on this +machine, no outbound identities, and no configured VPN. This machine's hostname +is not Debian-Devil. Existing listeners at audit time were loopback 5173, 6379, +8000 and platform port 1040. They were not stopped or reconfigured. + +The user deferred infrastructure access and live alias verification. Runtime +settings accept the eventual private FreeLLMAPI URL/key from environment secrets; +no address or credential was guessed. Actual serving model, downstream provider, +multimodal path, grounding, auth and quotas remain unverified live. GPU defaults +off. D1/D2 alias egress cannot be enabled by an environment flag. + +RapidOCR 3.10.0 and ORT 1.24.1 were installed and exercised. RapidOCR's exact +`default_models.yaml` entries provide both PP-OCRv6 Small ONNX SHA256s, retained +in `ocr/models.json`; actual downloaded files matched. ONNX metadata includes the +recognition character dictionary. No classifier/alternate tier is downloaded. +The two models initialize on `CPUExecutionProvider`; GPU/CUDA is not installed. + +ModelScope was initially denied by the environment proxy. Required ModelScope +hosts were added to the environment configuration draft while preserving existing +Hugging Face host additions. Subsequent HTTPS downloads succeeded with checksum +verification. This is artifact acquisition access, not document egress approval. +No model weights, real documents or secrets were committed. + +The ModelScope public model metadata API declares Apache License 2.0 for +RapidAI/RapidOCR. Installed distribution metadata identifies RapidOCR Apache-2.0, +ORT MIT, pypdf BSD-3-Clause, Pillow MIT-CMU, and pypdfium2 BSD-3-Clause/Apache-2.0 +with dependency licenses; retain bundled PDFium notices in redistribution. diff --git a/ocr/docs/benchmark.md b/ocr/docs/benchmark.md new file mode 100644 index 0000000000000000000000000000000000000000..2f4fa1b901246c131be682927388220ae8c73f6c --- /dev/null +++ b/ocr/docs/benchmark.md @@ -0,0 +1,39 @@ +# Reproducible OCR benchmarks + +The generator creates CC0 synthetic public DE/EN printed pages (100/200/300 DPI), +text-layer/scanned PDFs, blur/contrast/rotation, ruled tables, columns, a receipt, +blank/damaged input and mixed 20-/100-page PDFs. No real personal data or private +source documents are used. DejaVu Sans is a prerequisite, provided in the worker +image. Ground truth is rendered source text; independent human review, genuine +handwriting and annotated layout/table ground truth remain outstanding. + +```sh +OCR_MODELS_DIR=/path/to/verified/models OMP_NUM_THREADS=2 OPENBLAS_NUM_THREADS=1 \ + python -m ocr.benchmark --runs 3 --fixtures /tmp/ocr-fixtures \ + --output /tmp/ocr-report.json +``` + +The report records revision/runtime source digest, cgroup limits, platform, +engine versions, model-init and warm latency percentiles, CER/WER, per-fixture +failures/flags/geometry bounds, batches/pages/min, CPU time and peak process RSS. +It does not persist documents or OCR text. CPU-only mixed batches use both embedded +text and CPU OCR; they are not proof of CPU/GPU overflow speedup. + +For the two-CPU reproduction, build the reviewed image from repository root, with +an optional ephemeral proxy CA mount in a managed environment, then: + +```sh +docker run --rm --network none --cpus 2 --memory 2g --read-only \ + --tmpfs /tmp:rw,noexec,nosuid,size=512m hybrid-ocr-cpu:REVIEWED_REVISION \ + sh -c 'python -m ocr.benchmark --runs 3 --fixtures /tmp/fixtures --output /tmp/report.json > /tmp/summary.json && cat /tmp/report.json' +``` + +No host ports are published. The benchmark overrides the worker server command; +the image's inherited liveness healthcheck may report unhealthy because no server +is expected in that container. Its state is not an OCR outcome or readiness result. +A separate actual-server smoke test validates readiness/auth/OCR. Limits do not +make this an actual Hugging Face CPU Basic deployment, nor an exclusive-core test. + +See `docs/ocr-benchmarks/` and `docs/OCR_RELEASE_READINESS.md` for retained measurements, +accuracy limitations, exact source revision and production qualification gaps. +Never enable overflow based on these CPU-only measurements or mocked alias timing. diff --git a/ocr/docs/deployment.md b/ocr/docs/deployment.md new file mode 100644 index 0000000000000000000000000000000000000000..93abf5090dbddb114534b25d047f3aaff7b4bbe5 --- /dev/null +++ b/ocr/docs/deployment.md @@ -0,0 +1,92 @@ +# Installation, deployment review and rollback + +Do not run the original `linux.sh`, root Dockerfile, or an HF upload as part of this +OCR rollout: they manage the existing application. No production deploy/restart +has been authorized. Work in the isolated implementation worktree. + +## Reproduce locally + +Python 3.11, Linux x86_64. Use a separate venv; no dependency change to the existing +application is necessary. Keep TLS and package/model checksum verification enabled. + +```sh +python3.11 -m venv /tmp/ocr-venv +/tmp/ocr-venv/bin/pip install --require-hashes -r ocr/requirements.lock +/tmp/ocr-venv/bin/pip install -r ocr/requirements-dev.txt +export OCR_MODELS_DIR=/opt/ocr-models # select an operator-writable location +/tmp/ocr-venv/bin/python -m ocr.download_models +/tmp/ocr-venv/bin/python -m pytest -c pytest-ocr.ini +OCR_RUN_LIVE_CPU=1 /tmp/ocr-venv/bin/python -m pytest -c pytest-ocr.ini ocr/tests/test_live.py +/tmp/ocr-venv/bin/python -m ocr.fixtures.generate /tmp/ocr-fixtures +/tmp/ocr-venv/bin/python -m ocr.cli run /tmp/ocr-fixtures/de-scan.pdf --format markdown +``` + +`ocr/.env.example` lists settings without secrets. The application reads environment +variables directly; it does not load arbitrary `.env` files. On Debian, a reviewed +service manager can inject an EnvironmentFile with mode 0600. On HF, enter values +under Space Settings → Secrets; those become environment variables at startup. + +The dispatcher uses `OCR_DISPATCHER_TOKEN`; the CPU Space uses only +`OCR_CPU_AUTH_TOKEN`. Configure the dispatcher with that token plus `OCR_CPU_URL`. +`FREELLMAPI_BASE_URL` must be the actual existing private API URL including `/v1`; +`FREELLMAPI_API_KEY` stays on Debian. `OCR_GPU_MODEL` must remain `alias-ocr`. +They can be filled later from runtime secrets, as requested; there are no stored +credential values or guessed endpoints. Never place FreeLLMAPI keys in a public +CPU worker. If a separate HF-hosted dispatcher is later desired, it needs its own +trust review and cannot reach a Debian loopback address by itself. + +Start the dispatcher only on an operator-approved loopback socket or Unix socket, +for example through existing private ingress using `uvicorn ocr.api:app --uds +/path/to/approved/ocr.sock --workers 1 --no-access-log`. Select the approved writable +socket location; do not create new public listeners. One process is required for +consistent in-memory jobs. Existing services need no restart. + +## Separate CPU Space + +Review the target Space identity, privacy, token and D0-only egress policy first. +`Agent.md` names the existing application Space, not a newly selected OCR Space. +Do not overwrite it. Stage only `ocr/`, `ocr/deploy/hf-cpu/Dockerfile` as root +`Dockerfile`, and `ocr/deploy/hf-cpu/README.md` as root `README.md`. The README +contains Docker SDK/app_port 7860 metadata. CPU worker never calls Debian. + +Build from repository root: + +```sh +docker build -f ocr/deploy/hf-cpu/Dockerfile -t hybrid-ocr-cpu:review . +``` + +In the managed cloud proxy, supply its public trust bundle as an ephemeral BuildKit +secret (`--secret id=proxy_ca,src=/etc/ssl/certs/ca-certificates.crt`). Do not disable +TLS verification. Build-time destinations: package registry, Debian signed package +mirrors and pinned ModelScope model URLs/CDNs. Models are checksum-verified and +baked into the image. No inference-time model download is needed. + +The reviewed image runs non-root, one Uvicorn worker, ORT intra-op 2/inter-op 1 and +OpenCV 2 threads. Test with CPU/memory limits and no host port publication. A real +HF CPU Basic deployment must be explicitly approved, followed by authenticated +health/readiness, public-fixture OCR, memory/cold-start measurements, sleep/wake +checks and D0-only dispatcher integration. No availability guarantee for sleeping +Spaces is made. + +## Alias qualification later + +Set the private URL/key securely, leave sensitive data out, then run +`python -m ocr.cli probe-alias`. Review observed model/provider output with the +FreeLLMAPI operator. `/v1/models` or an alias name alone cannot qualify routing. +`OCR_ENABLE_GPU=true` permits only D0 after a successful multimodal challenge; +D1/D2 remain blocked. Grounding/provider enforcement must be separately audited +before any D2 extension. Reprobe after model/config changes; qualification expires +after one hour, after which requests fail closed until a controlled restart/reprobe. + +Overflow stays disabled until real CPU/GPU mixed 20+/100-page batches demonstrate +>15% end-to-end gain under approved budgets. Enable both operator gates plus caller +permission only after retaining that evidence. Mocked timing tests are not evidence. + +## Rollback + +Disable new dispatcher ingress or set `OCR_ENABLE_GPU=false` and select `cpu_only`. +Stop only this new worker/dispatcher after owner approval. Revert the implementation +commit in a review branch if needed; no migration or existing service rollback is +required. Retained in-memory jobs are lost on restart; clients must resubmit with +explicit awareness that remote work may already have executed. Keep D2 local until +an audited destination enforcement design is delivered. diff --git a/ocr/docs/routing.md b/ocr/docs/routing.md new file mode 100644 index 0000000000000000000000000000000000000000..c625b7e1d327bedee1156c59deb63c2a7a9b25ae --- /dev/null +++ b/ocr/docs/routing.md @@ -0,0 +1,49 @@ +# Routing and reliability + +`ocr.routing.Dispatcher` owns two independent bounded lanes. Defaults: CPU 1, +alias 2 workers, 32 pending items per lane, two active pages per job, four active +jobs, 128 retained job records, 30 minute creation-time TTL. One parser process +runs at a time; its address space is capped at 2 GiB, CPU time at 30 seconds and +wall time at 40 seconds. It receives no credential environment. An output cap +bounds the parent transfer. Native parser isolation is resource containment, not +a general hostile-code sandbox; keep PDFium/Pillow/parser dependencies current. + +Input limits: 25 MiB, 100 pages, 20 MP/page, 200 MP/document. Limits can be lowered. +Post-processing limits: 8 MiB serialized result/job, 64 MiB aggregate retained +results. Capacity exhaustion is explicit. Source bytes/raster pages are transient; +results contain classified OCR evidence until purge or TTL. Restart loses jobs. + +Usable PDF text skips OCR and carries `text_layer_heuristic` provenance. This is a +heuristic, not proof that a malicious text layer matches visible content. Use +`force_ocr` for suspicious files. PDFium renders at 200 DPI; EXIF orientation is +applied to images. Boxes refer to the resulting source raster, not an unrotated +PDF coordinate system. Embedded text currently has no invented geometry. + +- `auto`: CPU first; complex pages can use alias only with verified layout support. + CPU quality problems selectively escalate to an eligible alias. +- `cpu_only`/`gpu_only`: explicit lane; no cross-lane fallback. +- `quality`: selective comparison for complex pages when the operator enables + `OCR_QUALITY_DUAL_RUN`. Keep both results and disagreement/review flags. +- `throughput`: requires caller permission, operator enablement, benchmark + qualification and both warm EWMAs. Alias must predict completion at least 15% + sooner. Default overflow is off; no real hybrid speedup is yet qualified. + +Authorize immediately before every attempt, including retries, fallback and +escalation. Health, circuit breaker and per-minute alias budget are checked there. +At most one retry follows 429/502/503/transport timeout with jitter; three failures +open the lane for 30 seconds. No silent retry changes the model or destination. +The first successful lane call is recorded separately from the warm EWMA. + +Initial quality thresholds are uncalibrated heuristics: p10 line score <0.7, +empty nonblank input, implausibly short text, repetition, replacement characters, +column-order warning, low-contrast/blur proxy and malformed Markdown. The blur/contrast proxy uses image variance normalized by square-root ink density +(initial threshold 145, checked on the synthetic corpus, not a general calibration). +CPU scores are never synthesized +for alias text. Unknown boxes stay null. Clear quality failures produce explicit +page failures or review flags. Rotated scans and blur are known limitations. + +Cancellation purges a retained result and stops pending page work. A request +already executing remotely may finish and incur cost. Native ORT work retains its +serialization lock until it finishes; it cannot be force-interrupted safely. +Jobs that have failures and successes are `partial`, not `completed`. Results and +Markdown delimiters use original page order even if completions arrive out of order. diff --git a/ocr/docs/threat-model.md b/ocr/docs/threat-model.md new file mode 100644 index 0000000000000000000000000000000000000000..6ac88247dedd26e15e0caaab3442b5bbf52f4a31 --- /dev/null +++ b/ocr/docs/threat-model.md @@ -0,0 +1,52 @@ +# OCR trust boundaries + +D0 = public, D1 = internal, D2 = sensitive. Unknown input defaults to D2. +Effective sensitivity is the stricter of caller label and server minimum. +A deployment must set the server minimum to its classification policy; callers +cannot be trusted to classify a mixed-trust upload collection correctly. + +| Destination | D0 | D1 | D2 | +|---|---|---|---| +| Trusted local CPU | allowed | allowed | allowed | +| Authenticated remote HF CPU worker | explicit server/caller permission | blocked | blocked | +| Probed alias, operator enabled | allowed | blocked | blocked | +| Unprobed/unhealthy alias | blocked | blocked | blocked | + +An HTTPS URL, private Space, token or model name does not approve a data processor. +The established D2 inference catalogue is helmholtz, blablador, instituteforX. +`auto-smart`/`jev-router` permission is confined to that catalogue; generic `auto` +is not D2 authorization. This implementation changes none of those production +policies. It has no audited enforceable executed-destination/fallback attestation +protocol, so D2 alias routing is hard-disabled even when metadata claims an +approved provider. `OCR_GPU_D2_ENABLED` is intentionally unsupported. Implement +and audit a real enforcement contract as a separate change before enabling it. + +The D0 alias probe renders a random synthetic string in an image without putting +that string in the prompt. Successful extraction establishes only a tested +multimodal path. Returned model/provider fields are observations, not trustworthy +proof of destination enforcement. A model change revokes probe qualification. +Grounding/table/math capabilities are not enabled by alias/model name. + +No arbitrary URL ingestion, HTTP redirects to other processors, remote file paths, +OpenAI tools, PDF chat attachments or general-purpose chat execution are exposed. +Remote service URLs are operator configuration and require HTTPS; HTTP is allowed +only for loopback. OpenAI façade accepts one inline image and no arbitrary prompt. + +Client and worker tokens are separate. Authentication precedes multipart parsing; +fixed/chunked bodies and rates are bounded. No wildcard CORS. All endpoints except +content-free `/health` require auth, including docs, metrics and readiness. +Service must bind loopback/trusted IPC on Debian. No Caddy, Funnel, public Debian +port, existing production container or FreeLLMAPI routing change is needed. + +Operational errors contain constant codes. Provider exception bodies, images, +base64, bearer tokens and OCR text are not logged. Native PDF parser stderr is +suppressed in its disposable process; ASGI guards prevent unexpected exceptions +from escaping into server tracebacks. Server access logs should be disabled. +Source raster bytes are not persisted; multipart spool files are closed by the +request lifecycle. There is no durable queue or at-rest document store. A future +persistent queue needs encryption, tenant authorization and retention review. + +Anonymization handoff retains source page identity and original sensitivity. +OCR results are never presumed anonymous. The bridge requires a typed tagger +result and fails closed on errors; it does not use the legacy fail-open fallback. +Actual GLiNER/PII model qualification and release approval remain downstream work. diff --git a/ocr/download_models.py b/ocr/download_models.py new file mode 100644 index 0000000000000000000000000000000000000000..f197cb07984236382e9faf4b7cff37d13384924c --- /dev/null +++ b/ocr/download_models.py @@ -0,0 +1,41 @@ +"""Explicit, checksum-verified model acquisition. Never called during inference.""" + +import hashlib +import json +import os +from pathlib import Path + +import httpx + + +def download(directory): + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + manifest = json.loads(Path(__file__).with_name("models.json").read_text()) + for item in manifest.values(): + dest = directory / item["filename"] + if dest.exists() and hashlib.sha256(dest.read_bytes()).hexdigest() == item["sha256"]: + continue + tmp = dest.with_suffix(".part") + digest = hashlib.sha256() + try: + # Fixed authoritative URLs, redirects permitted for artifact CDN only (no credentials). + with httpx.stream("GET", item["url"], follow_redirects=True, timeout=120) as response: + response.raise_for_status() + with tmp.open("wb") as f: + size = 0 + for chunk in response.iter_bytes(): + size += len(chunk) + if size > 150 * 1024 * 1024: + raise RuntimeError("model_size_limit") + f.write(chunk) + digest.update(chunk) + if digest.hexdigest() != item["sha256"]: + raise RuntimeError("model_checksum_mismatch") + tmp.replace(dest) + finally: + tmp.unlink(missing_ok=True) + + +if __name__ == "__main__": + download(os.getenv("OCR_MODELS_DIR", "/tmp/ocr-models")) diff --git a/ocr/fixtures/generate.py b/ocr/fixtures/generate.py new file mode 100644 index 0000000000000000000000000000000000000000..c957880706784b321d0d58ce4c03ce15f282bbf7 --- /dev/null +++ b/ocr/fixtures/generate.py @@ -0,0 +1,123 @@ +"""CC0 synthetic corpus, deterministic, no real identities or personal data.""" + +import io +import json +from pathlib import Path + +from PIL import Image, ImageDraw, ImageFilter, ImageFont +from pypdf import PdfReader, PdfWriter +from pypdf.generic import DecodedStreamObject, DictionaryObject, NameObject + +TEXT_EN = "Public OCR benchmark.\nThe quick brown fox jumps over the lazy dog.\nInvoice total: 123.45 EUR." +TEXT_DE = "Öffentlicher OCR-Test.\nGrüße aus Köln: Äpfel, Öl und süße Birnen.\nRechnungsbetrag: 123,45 EUR." + + +def raster(text, dpi=200, style="clean"): + scale = dpi / 100 + im = Image.new("RGB", (int(700 * scale), int(430 * scale)), "white") + d = ImageDraw.Draw(im) + font = ImageFont.truetype("DejaVuSans.ttf", int(23 * scale)) + d.multiline_text( + (int(25 * scale), int(35 * scale)), + text, + font=font, + fill=(150, 150, 150) if style == "contrast" else "black", + spacing=int(16 * scale), + ) + if style == "table": + for y in (170, 220, 270): + d.line((25 * scale, y * scale, 650 * scale, y * scale), fill="black", width=2) + for x in (25, 300, 650): + d.line((x * scale, 170 * scale, x * scale, 270 * scale), fill="black", width=2) + if style == "blur": + im = im.filter(ImageFilter.GaussianBlur(1.5 * scale)) + if style == "rotated": + im = im.rotate(90, expand=True, fillcolor="white") + return im + + +def born_digital(text=TEXT_EN): + w = PdfWriter() + p = w.add_blank_page(width=600, height=400) + font = DictionaryObject( + { + NameObject("/Type"): NameObject("/Font"), + NameObject("/Subtype"): NameObject("/Type1"), + NameObject("/BaseFont"): NameObject("/Helvetica"), + NameObject("/Encoding"): NameObject("/WinAnsiEncoding"), + } + ) + p[NameObject("/Resources")] = DictionaryObject( + {NameObject("/Font"): DictionaryObject({NameObject("/F1"): w._add_object(font)})} + ) + + def escape(s): + return s.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") + + stream = DecodedStreamObject() + stream.set_data( + ( + "BT /F1 18 Tf 30 340 Td 30 TL " + + " T* ".join("(" + escape(line) + ") Tj" for line in text.splitlines()) + + " ET" + ).encode("cp1252") + ) + p[NameObject("/Contents")] = w._add_object(stream) + b = io.BytesIO() + w.write(b) + return b.getvalue() + + +def generate(root): + root = Path(root) + root.mkdir(parents=True, exist_ok=True) + manifest = [] + for lang, text in [("en", TEXT_EN), ("de", TEXT_DE)]: + for dpi in (100, 200, 300): + name = f"{lang}-{dpi}.png" + raster(text, dpi).save(root / name) + manifest.append(dict(file=name, text=text, category="printed", language=lang, dpi=dpi)) + name = f"{lang}-text.pdf" + (root / name).write_bytes(born_digital(text)) + manifest.append(dict(file=name, text=text, category="embedded", language=lang)) + for style in ("contrast", "blur", "rotated", "table"): + name = f"en-{style}.png" + raster(TEXT_EN, style=style).save(root / name) + manifest.append(dict(file=name, text=TEXT_EN, category=style, language="en")) + col = "Left column: public text.\nOne two three. Right column: public text.\n Four five six." + raster(col).save(root / "columns.png") + manifest.append(dict(file="columns.png", text=col, category="columns", language="en")) + raster("Receipt\nApples 2.00 EUR\nBread 3.00 EUR\nTOTAL 5.00 EUR").save(root / "receipt.png") + manifest.append( + dict( + file="receipt.png", + text="Receipt\nApples 2.00 EUR\nBread 3.00 EUR\nTOTAL 5.00 EUR", + category="receipt", + language="en", + ) + ) + Image.new("RGB", (700, 430), "white").save(root / "blank.png") + manifest.append(dict(file="blank.png", text="", category="blank", language="en")) + # Scanned PDF and mixed 20/100 page batches (no files with real PII). + b = io.BytesIO() + raster(TEXT_DE).save(b, format="PDF", resolution=200) + scan = b.getvalue() + (root / "de-scan.pdf").write_bytes(scan) + manifest.append(dict(file="de-scan.pdf", text=TEXT_DE, category="scanned_pdf", language="de")) + for count in (20, 100): + w = PdfWriter() + for i in range(count): + w.append(PdfReader(io.BytesIO(born_digital(TEXT_EN) if i % 2 == 0 else scan))) + with (root / f"mixed-{count}.pdf").open("wb") as f: + w.write(f) + (root / "damaged.pdf").write_bytes(b"%PDF-1.7\ncorrupted") + (root / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)) + return manifest + + +if __name__ == "__main__": + import argparse + + p = argparse.ArgumentParser() + p.add_argument("directory") + generate(p.parse_args().directory) diff --git a/ocr/handoff.py b/ocr/handoff.py new file mode 100644 index 0000000000000000000000000000000000000000..a266dcac9645aa9cf8d5385d171aa93f4055d62c --- /dev/null +++ b/ocr/handoff.py @@ -0,0 +1,82 @@ +"""Explicit opt-in Augmentoolkit bridge; original evidence remains classified.""" + +from .schemas import SafeError + + +def anonymization_handoff(result, tagger): + """Pass AnonymizationEngine for the supported validated candidate workflow. + + Historical callback compatibility: tagger receives text and returns {'text': ..., 'tags': [...]}. + + No implicit fallback to the repository's fail-open anonymizer. The caller owns + review and release; the original OCR result is not mutated or declassified. + """ + from .anonymization import AnonymizationEngine + + if isinstance(tagger, AnonymizationEngine): + return tagger.handoff(result) + # Historical characterization bridge only. This dictionary is classified + # evidence, never a ReleaseCandidate accepted by ReleaseAuthority. + pages = [] + for p in result.pages: + try: + tagged = tagger(p.text) + except Exception: + raise SafeError("anonymization_failed") from None + if ( + not isinstance(tagged, dict) + or not isinstance(tagged.get("text"), str) + or not isinstance(tagged.get("tags"), list) + ): + raise SafeError("invalid_anonymization_result") + pages.append( + dict( + page_index=p.page_index, + page_id=p.page_id, + original_sensitivity=p.sensitivity, + source=p.source, + engine=p.engine, + provider_destination=p.provider_destination, + anonymized_text=tagged["text"], + tags=tagged["tags"], + release_review_required=True, + ) + ) + return dict( + schema_version="ocr-anonymization-handoff/v1", + sensitivity=result.sensitivity, + pages=pages, + original_evidence_retained_by_caller=True, + anonymous=False, + release_approved=False, + release_review_required=True, + ) + + +async def augmentoolkit_extract(path, client, token, *, sensitivity="D2"): + """Trusted CLI/client filesystem only. Uses the private dispatcher, not remote model API.""" + import asyncio + import json + from pathlib import Path + + path = Path(path) + with path.open("rb") as f: + r = await client.post( + "/v1/ocr/jobs", + headers={"Authorization": "Bearer " + token}, + files={"file": (path.name, f)}, + data={"options": json.dumps({"sensitivity": sensitivity})}, + ) + r.raise_for_status() + job = r.json() + for _ in range(300): + r = await client.get(job["status_url"], headers={"Authorization": "Bearer " + token}) + r.raise_for_status() + if r.json()["status"] in ("completed", "partial", "failed", "cancelled"): + break + await asyncio.sleep(1) + else: + raise SafeError("client_poll_timeout") + r = await client.get(job["result_url"], headers={"Authorization": "Bearer " + token}) + r.raise_for_status() + return r.json() # Classified evidence; never silently invoke anonymization/publication. diff --git a/ocr/http_security.py b/ocr/http_security.py new file mode 100644 index 0000000000000000000000000000000000000000..eb064f6db28e8c756cf6e7485803e0f776254abf --- /dev/null +++ b/ocr/http_security.py @@ -0,0 +1,104 @@ +import hashlib +import hmac +import time +from collections import deque + +from fastapi.responses import JSONResponse + + +class Guard: + """Authenticate before multipart parsing; bound chunked and fixed-length bodies.""" + + def __init__(self, app, token, max_bytes, rate=60): + self.app = app + self.token = token + self.max_bytes = max_bytes + 64 * 1024 + self.rate = rate + self.calls = deque() + + async def __call__(self, scope, receive, send): + if scope["type"] != "http": + return await self.app(scope, receive, send) + headers = dict(scope.get("headers", [])) + path = scope["path"] + # Only /health and documentation endpoints are unauthenticated + if path not in ("/health", "/docs", "/openapi.json", "/redoc"): + if not self.token: + return await JSONResponse({"code": "service_unconfigured"}, status_code=503)( + scope, receive, send + ) + auth = headers.get(b"authorization", b"") + expected = b"Bearer " + self.token.encode("utf-8") + if not hmac.compare_digest(auth, expected): + return await JSONResponse({"code": "unauthorized"}, status_code=401)( + scope, receive, send + ) + if scope["method"] == "POST": + now = time.monotonic() + while self.calls and now - self.calls[0] > 60: + self.calls.popleft() + if len(self.calls) >= self.rate: + return await JSONResponse({"code": "rate_limit"}, status_code=429)( + scope, receive, send + ) + self.calls.append(now) + try: + if int(headers.get(b"content-length", b"0")) > self.max_bytes: + return await JSONResponse({"code": "upload_limit"}, status_code=413)( + scope, receive, send + ) + except ValueError: + return await JSONResponse({"code": "invalid_length"}, status_code=400)( + scope, receive, send + ) + # Buffer at most the bounded body before Starlette sees it; no spool files on oversize. + body = bytearray() + if scope["method"] in ("POST", "PUT", "PATCH"): + while True: + message = await receive() + if message["type"] == "http.disconnect": + return + body.extend(message.get("body", b"")) + if len(body) > self.max_bytes: + return await JSONResponse({"code": "upload_limit"}, status_code=413)( + scope, receive, send + ) + if not message.get("more_body", False): + break + delivered = False + + async def bounded_receive(): + nonlocal delivered + if not delivered: + delivered = True + return { + "type": "http.request", + "body": bytes(body), + "more_body": False, + } + return await receive() + + return await self._invoke(scope, bounded_receive, send) + return await self._invoke(scope, receive, send) + + async def _invoke(self, scope, receive, send): + started = False + + async def safe_send(message): + nonlocal started + if message["type"] == "http.response.start": + started = True + await send(message) + + try: + await self.app(scope, receive, safe_send) + except Exception: + # Avoid leaking unexpected parser/provider exceptions into server tracebacks. + if not started: + await JSONResponse({"code": "internal_error"}, status_code=500)( + scope, receive, send + ) + + +def owner(request): + return hashlib.sha256(request.headers.get("authorization", "").encode()).hexdigest() diff --git a/ocr/ingest.py b/ocr/ingest.py new file mode 100644 index 0000000000000000000000000000000000000000..02d3cdccbd055b49b630f6928d3f365093cca250 --- /dev/null +++ b/ocr/ingest.py @@ -0,0 +1,167 @@ +"""Bounded parsing. No URLs, no caller-selected server paths, no persistence.""" + +import hashlib +import io +import math +import warnings +from pathlib import Path + +from PIL import Image, ImageOps +from pypdf import apply_configuration + +from .classify import features +from .schemas import RasterPage, SafeError + +EXT = {".png": "PNG", ".jpg": "JPEG", ".jpeg": "JPEG", ".tif": "TIFF", ".tiff": "TIFF"} + + +def usable_text(text): + s = text.strip() + return ( + len(s) >= 24 + and sum(c.isalpha() for c in s) >= 15 + and s.count("\ufffd") / len(s) < 0.01 + and sum(c.isprintable() or c.isspace() for c in s) / len(s) > 0.98 + ) + + +def encode(im): + b = io.BytesIO() + im.convert("RGB").save(b, format="PNG") + return b.getvalue() + + +def decode_image(data, max_pixels): + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + im = Image.open(io.BytesIO(data)) + if im.format not in ("PNG", "JPEG", "TIFF") or im.width * im.height > max_pixels: + raise SafeError("image_limit") + im.load() + return ImageOps.exif_transpose(im).convert("RGB") + except SafeError: + raise + except (Exception, Image.DecompressionBombWarning): + raise SafeError("invalid_image") from None + + +def ingest(data: bytes, filename: str, options, settings): + with apply_configuration( + maximum_declared_stream_length=8_000_000, + zlib_maximum_output_length=8_000_000, + array_based_stream_maximum_output_length=8_000_000, + lzw_maximum_output_length=8_000_000, + run_length_maximum_output_length=8_000_000, + stream_decoding_work_maximum_length=32_000_000, + page_tree_maximum_entries=1000, + image_maximum_buffer_size=60_000_000, + ): + return _ingest(data, filename, options, settings) + + +def _ingest(data: bytes, filename: str, options, settings): + if not data or len(data) > settings.max_upload: + raise SafeError("upload_limit") + ext = Path(filename).suffix.lower() + digest = hashlib.sha256(data).hexdigest() + pages = [] + total = 0 + limit = min(options.max_pages, settings.max_pages) + + def add(im, index, text=None, rotation=0): + nonlocal total + if im.width * im.height > settings.max_pixels: + raise SafeError("image_limit") + total += im.width * im.height + if total > settings.total_pixels: + raise SafeError("document_pixel_limit") + image = encode(im) + pages.append( + RasterPage( + index, + image, + im.width, + im.height, + f"{digest}:{index}", + rotation, + text, + features(image), + ) + ) + + try: + if ext == ".pdf": + if not data.startswith(b"%PDF-"): + raise SafeError("signature_mismatch") + import pypdfium2 as pdfium + from pypdf import PdfReader + + reader = PdfReader(io.BytesIO(data), strict=True) + if reader.is_encrypted: + raise SafeError("encrypted_pdf") + if not 0 < len(reader.pages) <= limit: + raise SafeError("page_limit") + with pdfium.PdfDocument(data) as doc: + for i, p in enumerate(reader.pages): + # Avoid unbounded text-stream decompression before extraction. + contents = p.get_contents() + if contents and len(contents.get_data()) > 8 * 1024 * 1024: + raise SafeError("pdf_stream_limit") + text = ( + p.extract_text() or "" + if options.extract_pdf_text_layer and not options.force_ocr + else "" + ) + text = text if usable_text(text) else None + page = doc[i] + try: + w, h = page.get_size() + scale = 200 / 72 + if ( + not math.isfinite(w * h) + or w <= 0 + or h <= 0 + or w * h * scale * scale > settings.max_pixels + ): + raise SafeError("image_limit") + # Embedded pages need no raster for OCR; keep source page dimensions. + if text: + add( + Image.new("RGB", (max(1, int(w)), max(1, int(h))), "white"), + i, + text, + int(p.get("/Rotate", 0)), + ) + else: + bitmap = page.render(scale=scale) + try: + add( + bitmap.to_pil(), + i, + rotation=int(p.get("/Rotate", 0)), + ) + finally: + bitmap.close() + finally: + page.close() + elif ext in EXT: + with Image.open(io.BytesIO(data)) as source: + if source.format != EXT[ext]: + raise SafeError("signature_mismatch") + count = getattr(source, "n_frames", 1) + if count > limit: + raise SafeError("page_limit") + for i in range(count): + source.seek(i) + if source.width * source.height > settings.max_pixels: + raise SafeError("image_limit") + source.load() + add(ImageOps.exif_transpose(source).convert("RGB"), i) + else: + raise SafeError("unsupported_media") + except SafeError: + raise + except Exception: + raise SafeError("invalid_document") from None + return pages diff --git a/ocr/jobs.py b/ocr/jobs.py new file mode 100644 index 0000000000000000000000000000000000000000..6d32dd8c9de70ca6116400f09756b03da764fc6f --- /dev/null +++ b/ocr/jobs.py @@ -0,0 +1,218 @@ +import asyncio +import hashlib +import time +import uuid +from dataclasses import dataclass, field + +from .normalize import markdown_pages +from .parse_process import parse +from .schemas import Failure, OCRResult, SafeError + + +@dataclass +class Job: + id: str + owner: str + request_id: str + sensitivity: str + created: float = field(default_factory=time.time) + updated: float = field(default_factory=time.time) + status: str = "queued" + page_count: int = 0 + completed_pages: int = 0 + failures: list = field(default_factory=list) + result: OCRResult | None = None + task: asyncio.Task | None = None + fingerprint: str = "" + stored_bytes: int = 0 + + +class Jobs: + def __init__(self, dispatcher, settings): + self.dispatcher = dispatcher + self.settings = settings + self.jobs = {} + self.keys = {} + self.cleaner = None + self.parse_slots = asyncio.Semaphore(1) + + def start(self): + self.cleaner = asyncio.create_task(self._cleanup()) + + async def close(self): + if self.cleaner: + self.cleaner.cancel() + tasks = [j.task for j in self.jobs.values() if j.task and not j.task.done()] + for t in tasks: + t.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + if self.cleaner: + await asyncio.gather(self.cleaner, return_exceptions=True) + self.jobs.clear() + self.keys.clear() + + def purge_expired(self): + now = time.time() + for id, j in list(self.jobs.items()): + if now - j.created > self.settings.ttl: + if j.task and not j.task.done(): + j.task.cancel() + del self.jobs[id] + for key, value in list(self.keys.items()): + if value == id: + del self.keys[key] + + async def _cleanup(self): + while True: + await asyncio.sleep(min(30, self.settings.ttl)) + self.purge_expired() + + def get(self, id, owner): + self.purge_expired() + j = self.jobs.get(id) + if not j or j.owner != owner: + raise SafeError("job_not_found") + return j + + def submit(self, data, filename, options, owner, key=None): + self.purge_expired() + fingerprint = hashlib.sha256(data + options.model_dump_json().encode()).hexdigest() + if key and (owner, key) in self.keys: + j = self.get(self.keys[(owner, key)], owner) + if j.fingerprint != fingerprint: + raise SafeError("idempotency_conflict") + return j + if ( + len(self.jobs) >= self.settings.retained_jobs + or sum(j.status in ("queued", "running") for j in self.jobs.values()) + >= self.settings.active_jobs + ): + raise SafeError("job_capacity") + j = Job( + str(uuid.uuid4()), + owner, + str(uuid.uuid4()), + options.sensitivity, + fingerprint=fingerprint, + ) + self.jobs[j.id] = j + if key: + self.keys[(owner, key)] = j.id + j.task = asyncio.create_task(self._run(j, data, filename, options)) + return j + + async def parse(self, data, filename, options): + async with self.parse_slots: + return await parse(data, filename, options, self.settings) + + async def _run(self, j, data, filename, options): + results = [] + tasks = [] + try: + async with asyncio.timeout(self.settings.job_timeout): + pages = await self.parse(data, filename, options) + data = b"" + j.page_count = len(pages) + j.status = "running" + slots = asyncio.Semaphore(self.settings.pages_per_job) + + async def one(page): + async with slots: + try: + result = await self.dispatcher.page(page, options) + current = sum(len(p.text) + len(p.markdown) for p in results) + if ( + current + len(result.model_dump_json()) + > self.settings.max_result_bytes + ): + raise SafeError("result_limit") + results.append(result) + except SafeError as e: + j.failures.append(Failure(page_index=page.index, code=e.code)) + except Exception: + j.failures.append(Failure(page_index=page.index, code="page_failed")) + finally: + j.completed_pages += 1 + j.updated = time.time() + + tasks = [asyncio.create_task(one(p)) for p in pages] + try: + await asyncio.gather(*tasks) + finally: + for t in tasks: + if not t.done(): + t.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + pages.clear() + j.status = ( + "partial" if results and j.failures else "failed" if j.failures else "completed" + ) + results.sort(key=lambda p: p.page_index) + j.result = OCRResult( + job_id=j.id, + request_id=j.request_id, + status=j.status, + page_count=j.page_count, + sensitivity=j.sensitivity, + markdown=markdown_pages(results) + if options.output in ("markdown", "both") + else "", + text="\n\n".join(p.text for p in results) + if options.output in ("text", "both") + else "", + pages=results, + failures=sorted(j.failures, key=lambda p: p.page_index), + ) + size = len(j.result.model_dump_json().encode("utf-8")) + if ( + size > self.settings.max_result_bytes + or size + sum(other.stored_bytes for other in self.jobs.values()) + > self.settings.retained_bytes + ): + j.result = None + raise SafeError("result_capacity") + j.stored_bytes = size + except asyncio.CancelledError: + j.status = "cancelled" + j.result = None + j.failures = [] + raise + except Exception as e: + j.status = "failed" + j.failures = [ + Failure( + page_index=-1, + code=e.code + if isinstance(e, SafeError) + else "job_timeout" + if isinstance(e, TimeoutError) + else "job_failed", + ) + ] + finally: + data = b"" + j.updated = time.time() + + def cancel(self, j): + if j.task and not j.task.done(): + j.task.cancel() + j.status = "cancelled" + j.stored_bytes = 0 + j.result = None + j.failures = [] + for key, id in list(self.keys.items()): + if id == j.id: + del self.keys[key] + + +def status(j): + return dict( + job_id=j.id, + request_id=j.request_id, + status=j.status, + page_count=j.page_count, + completed_pages=j.completed_pages, + created=j.created, + updated=j.updated, + failures=[f.model_dump() for f in j.failures], + ) diff --git a/ocr/models.json b/ocr/models.json new file mode 100644 index 0000000000000000000000000000000000000000..96a5e31b652f417e3f8a1a4ec00284040e7e80be --- /dev/null +++ b/ocr/models.json @@ -0,0 +1,4 @@ +{ + "Det": {"filename":"PP-OCRv6_det_small.onnx","url":"https://www.modelscope.cn/models/RapidAI/RapidOCR/resolve/v3.10.0/onnx/PP-OCRv6/det/PP-OCRv6_det_small.onnx","sha256":"090f04abcd9d9a7498bc4ebf677e4cb9bdce1fe4197ddb7e529f1ef44e1ff94f"}, + "Rec": {"filename":"PP-OCRv6_rec_small.onnx","url":"https://www.modelscope.cn/models/RapidAI/RapidOCR/resolve/v3.10.0/onnx/PP-OCRv6/rec/PP-OCRv6_rec_small.onnx","sha256":"6f327246b50388f3c176ae304bd95767ea6dc0c9ae92153ef8cbe210b3c14884"} +} diff --git a/ocr/normalize.py b/ocr/normalize.py new file mode 100644 index 0000000000000000000000000000000000000000..719810fea0dac1d5f140aeaa4492fac006e64562 --- /dev/null +++ b/ocr/normalize.py @@ -0,0 +1,44 @@ +import re + +from .schemas import Block, SafeError + + +def polygon(points, width, height): + p = [ + [ + round(min(1000, max(0, float(x) / width * 1000)), 3), + round(min(1000, max(0, float(y) / height * 1000)), 3), + ] + for x, y in points + ] + return p, [ + min(x[0] for x in p), + min(x[1] for x in p), + max(x[0] for x in p), + max(x[1] for x in p), + ] + + +def grounding(text): + blocks = [] + for match in re.finditer(r"!\[(text|table|figure|title|formula)\]\(([^)]*)\)", text): + try: + box = [float(v) for v in match[2].split(",")] + if ( + len(box) != 4 + or not all(0 <= v <= 1000 for v in box) + or box[0] > box[2] + or box[1] > box[3] + ): + raise ValueError() + blocks.append(Block(type=match[1], bbox_1000=box)) + except ValueError: + raise SafeError("invalid_grounding") from None + return blocks + + +def markdown_pages(pages): + return "\n\n".join( + f"\n\n{p.markdown}" + for p in sorted(pages, key=lambda p: p.page_index) + ) diff --git a/ocr/openai_facade.py b/ocr/openai_facade.py new file mode 100644 index 0000000000000000000000000000000000000000..a7a9a1334a78e9adba308375084bf62d9427f989 --- /dev/null +++ b/ocr/openai_facade.py @@ -0,0 +1,76 @@ +"""Deliberately limited single-image chat-completions facade.""" + +import base64 +import binascii +import time +import uuid +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from .schemas import OCROptions, SafeError + + +class ImageURL(BaseModel): + model_config = ConfigDict(extra="forbid") + url: str + + +class ImageContent(BaseModel): + model_config = ConfigDict(extra="forbid") + type: Literal["image_url"] + image_url: ImageURL + + +class Message(BaseModel): + model_config = ConfigDict(extra="forbid") + role: Literal["user"] + content: list[ImageContent] = Field(min_length=1, max_length=1) + + +class ChatRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + model: Literal["ocr-auto", "ocr-cpu", "ocr-gpu", "ocr-alias"] + messages: list[Message] = Field(min_length=1, max_length=1) + stream: Literal[False] = False + # OCR extension, not an OpenAI parameter. It cannot lower server sensitivity. + ocr_options: OCROptions = Field(default_factory=OCROptions) + + +def image_from_request(body, max_bytes): + url = body.messages[0].content[0].image_url.url + formats = { + "data:image/png;base64,": ".png", + "data:image/jpeg;base64,": ".jpg", + "data:image/tiff;base64,": ".tiff", + } + for prefix, ext in formats.items(): + if url.startswith(prefix): + payload = url[len(prefix) :] + if len(payload) > 4 * ((max_bytes + 2) // 3): + raise SafeError("upload_limit") + try: + data = base64.b64decode(payload, validate=True) + except (ValueError, binascii.Error): + raise SafeError("invalid_data_url") from None + if len(data) > max_bytes: + raise SafeError("upload_limit") + return data, "page" + ext + raise SafeError("only_image_data_urls_supported") + + +def completion(result, model): + return { + "id": "chatcmpl-" + uuid.uuid4().hex, + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": result.markdown}, + "finish_reason": "stop", + } + ], + "ocr": result.model_dump(), + } diff --git a/ocr/parse_process.py b/ocr/parse_process.py new file mode 100644 index 0000000000000000000000000000000000000000..c2f6c59c7e14570b4866bfd5112e473de4cb9ad1 --- /dev/null +++ b/ocr/parse_process.py @@ -0,0 +1,86 @@ +import asyncio +import base64 +import json +import sys +from pathlib import Path + +from .schemas import RasterPage, SafeError + + +async def parse(data, filename, options, settings): + # Only non-secret limits cross the parser boundary; credentials are never inherited. + limits = { + name: getattr(settings, name) + for name in ("max_upload", "max_pages", "max_pixels", "total_pixels") + } + payload = json.dumps( + dict( + data=base64.b64encode(data).decode(), + filename=filename, + options=options.model_dump(), + limits=limits, + ) + ).encode() + proc = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "ocr.parse_worker", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + cwd=str(Path(__file__).resolve().parents[1]), + env={ + "PATH": "/usr/bin:/bin", + "LANG": "C.UTF-8", + "OPENBLAS_NUM_THREADS": "1", + "OMP_NUM_THREADS": "1", + }, + ) + + async def transfer(): + async def write(): + try: + proc.stdin.write(payload) + await proc.stdin.drain() + proc.stdin.close() + except (BrokenPipeError, ConnectionResetError): + pass + + task = asyncio.create_task(write()) + try: + out = bytearray() + while chunk := await proc.stdout.read(64 * 1024): + out.extend(chunk) + if len(out) > 128 * 1024 * 1024: + raise SafeError("parser_output_limit") + await task + await proc.wait() + if proc.returncode: + raise SafeError("parser_resource_limit") + try: + result = json.loads(out) + if "error" in result: + raise SafeError(result["error"]) + pages = [ + RasterPage(**{**p, "image": base64.b64decode(p["image"], validate=True)}) + for p in result["pages"] + ] + if not 1 <= len(pages) <= min(options.max_pages, settings.max_pages): + raise SafeError("page_limit") + return pages + except SafeError: + raise + except Exception: + raise SafeError("invalid_document") from None + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + try: + return await asyncio.wait_for(transfer(), 40) + except asyncio.TimeoutError: + raise SafeError("parser_timeout") from None + finally: + if proc.returncode is None: + proc.kill() + await proc.wait() diff --git a/ocr/parse_worker.py b/ocr/parse_worker.py new file mode 100644 index 0000000000000000000000000000000000000000..f40ed3ef1a84f3512fe1c0cadee1bba9f27e1e01 --- /dev/null +++ b/ocr/parse_worker.py @@ -0,0 +1,51 @@ +"""Disposable parser process: bounded CPU/address space, no credential environment.""" + +import base64 +import json +import resource +import sys + +# Limits are set before importing native document/image libraries. +resource.setrlimit(resource.RLIMIT_AS, (2 * 1024**3, 2 * 1024**3)) +resource.setrlimit(resource.RLIMIT_CPU, (30, 30)) +resource.setrlimit(resource.RLIMIT_FSIZE, (0, 0)) + +from .config import Settings # noqa: E402 +from .ingest import ingest # noqa: E402 +from .schemas import OCROptions, SafeError # noqa: E402 + + +def main(): + try: + raw = sys.stdin.buffer.read(36 * 1024 * 1024 + 1) + if len(raw) > 36 * 1024 * 1024: + raise SafeError("upload_limit") + payload = json.loads(raw) + pages = ingest( + base64.b64decode(payload["data"], validate=True), + payload["filename"], + OCROptions.model_validate(payload["options"]), + Settings(**payload["limits"]), + ) + out = [ + dict( + index=p.index, + image=base64.b64encode(p.image).decode(), + width=p.width, + height=p.height, + page_id=p.page_id, + rotation=p.rotation, + embedded=p.embedded, + features=p.features, + ) + for p in pages + ] + sys.stdout.write(json.dumps({"pages": out})) + except Exception as e: + sys.stdout.write( + json.dumps({"error": e.code if isinstance(e, SafeError) else "invalid_document"}) + ) + + +if __name__ == "__main__": + main() diff --git a/ocr/quality.py b/ocr/quality.py new file mode 100644 index 0000000000000000000000000000000000000000..96ea6d7cde5e4c00573d4a09d1ae800a1b7b9953 --- /dev/null +++ b/ocr/quality.py @@ -0,0 +1,24 @@ +from collections import Counter + + +def quality(result, page, threshold=0.7): + flags = [] + if not page.features.get("blank") and page.features.get("ink_contrast", 1000) < 145: + flags.append("low_contrast_or_blur") + if not result.text.strip() and not page.features.get("blank"): + flags.append("empty_nonblank") + if 0 < len(result.text.strip()) < 5 and page.features.get("line_density", 0) > 0.02: + flags.append("suspiciously_short") + scores = sorted(b.confidence for b in result.blocks if b.confidence is not None) + if scores and scores[int((len(scores) - 1) * 0.1)] < threshold: + flags.append("low_confidence") + lines = [s.strip() for s in result.text.splitlines() if s.strip()] + if len(lines) > 5 and max(Counter(lines).values()) > len(lines) * 0.6: + flags.append("repeated_text") + if result.text.count("```") % 2: + flags.append("markdown_unbalanced") + if "\ufffd" in result.text: + flags.append("unresolved_noise") + if page.features.get("possible_columns") and result.source == "cpu_ocr": + flags.append("reading_order_unverified") + return list(dict.fromkeys(result.flags + flags)) diff --git a/ocr/requirements-dev.txt b/ocr/requirements-dev.txt new file mode 100644 index 0000000000000000000000000000000000000000..1ce334d9b1d40ff6df563d7096059306eb242e29 --- /dev/null +++ b/ocr/requirements-dev.txt @@ -0,0 +1,4 @@ +# Install requirements.lock with --require-hashes separately, then this file. +pytest==9.1.1 +pytest-asyncio==1.4.0 +ruff==0.17.1 diff --git a/ocr/requirements.in b/ocr/requirements.in new file mode 100644 index 0000000000000000000000000000000000000000..8f9cb0563a822edf9d384c5cfcb6f2e7cdd6736d --- /dev/null +++ b/ocr/requirements.in @@ -0,0 +1,11 @@ +rapidocr==3.10.0 +onnxruntime==1.24.1 +fastapi==0.143.3 +uvicorn==0.54.0 +httpx==0.28.1 +python-multipart==0.0.32 +pydantic==2.14.1 +pypdf==6.21.0 +pypdfium2==5.14.0 +Pillow==12.3.0 +psutil==7.2.2 diff --git a/ocr/requirements.lock b/ocr/requirements.lock new file mode 100644 index 0000000000000000000000000000000000000000..6f73af92ca1289825868091a585a9cbc6c9ceaf1 --- /dev/null +++ b/ocr/requirements.lock @@ -0,0 +1,963 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile ocr/requirements.in --python /workspace/.ocr-venv/bin/python -c /tmp/ocr-tested-constraints.txt --generate-hashes -o ocr/requirements.lock +annotated-doc==0.0.5 \ + --hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 \ + --hash=sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb + # via + # -c /tmp/ocr-tested-constraints.txt + # fastapi +annotated-types==0.8.0 \ + --hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \ + --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 + # via + # -c /tmp/ocr-tested-constraints.txt + # pydantic +anyio==4.15.1 \ + --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \ + --hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94 + # via + # -c /tmp/ocr-tested-constraints.txt + # httpx + # starlette +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 + # via + # -c /tmp/ocr-tested-constraints.txt + # httpcore + # httpx + # requests +charset-normalizer==3.5.2 \ + --hash=sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e \ + --hash=sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf \ + --hash=sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5 \ + --hash=sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56 \ + --hash=sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26 \ + --hash=sha256:0891b9d3903c5571c03771ca669a4b0ec5618ca722a5c957d3d29cd4e5062848 \ + --hash=sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718 \ + --hash=sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93 \ + --hash=sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640 \ + --hash=sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3 \ + --hash=sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875 \ + --hash=sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e \ + --hash=sha256:1461ac396c4fdb983a675f20aa555624f0ee18ac83d832b9244ffff3d8055275 \ + --hash=sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204 \ + --hash=sha256:15bb4005af6320d259dc7593ca84a38d7fe06a421dbcf7b910ae23979101e787 \ + --hash=sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234 \ + --hash=sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3 \ + --hash=sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98 \ + --hash=sha256:195c26fb65950f8fce54e26349852b7bdd7c5f120aeefbcc440b8a20faaed4a3 \ + --hash=sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187 \ + --hash=sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d \ + --hash=sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f \ + --hash=sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7 \ + --hash=sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011 \ + --hash=sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f \ + --hash=sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869 \ + --hash=sha256:254eb48b9fa5ee9898a3c445825a1f340fe53712a098904b39b0bddba8ea3cb1 \ + --hash=sha256:2625388c6c754520c37abaf3b41eb34d1cc4a373f457898f08606c8e362b891d \ + --hash=sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847 \ + --hash=sha256:28a15fdad492a99b6eccfaaed66ef3f74050680545ea61ec8b2f4c538f1f1320 \ + --hash=sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9 \ + --hash=sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93 \ + --hash=sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd \ + --hash=sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00 \ + --hash=sha256:2cc961b171b3f3440f410489ab3573e86aea8736134ebbb40ea1338b7f0831bc \ + --hash=sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0 \ + --hash=sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09 \ + --hash=sha256:304d5463e65a35d7bb0850550e0780395395f6fcf452f04db7d5ca7cecc425ac \ + --hash=sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621 \ + --hash=sha256:30fcd120b732aa79317f08dee04d7de0847822e4cf7ee0e9f445bb958832252c \ + --hash=sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8 \ + --hash=sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a \ + --hash=sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51 \ + --hash=sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0 \ + --hash=sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef \ + --hash=sha256:3d14b50de6bf4d0edf857a9386836846f982b8f524e188e2e68b96d702bcf4aa \ + --hash=sha256:3d21b8b13c7592db2ac5e544a6d83187b995257472b0c9e8351b6d507ae37ed6 \ + --hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 \ + --hash=sha256:3ddacd27458c45bdacd6bd6db644bfb730efbf9e830310186e3045c9c5be8fb2 \ + --hash=sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229 \ + --hash=sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e \ + --hash=sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd \ + --hash=sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115 \ + --hash=sha256:447441e76ec720b15e64418d32e092297340387053047c7c694f579efb0ee1d9 \ + --hash=sha256:4495c5002a7b28557e7e222e77e0b661183e432b7d6d2e788101e3f240e05b8c \ + --hash=sha256:44bd4fbb29dfbeba60e7d2bd000c59e4b21ddb3cc53912b14048d37092706d7c \ + --hash=sha256:4685902cf26edf013ed7a3da0f426ebba7a00ebb9541386d835afbf002c11cab \ + --hash=sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253 \ + --hash=sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995 \ + --hash=sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438 \ + --hash=sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0 \ + --hash=sha256:50e3adfb96fc189eb27b1cf62d3b598b89b4bb0420d93a3d3e42e137409011be \ + --hash=sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b \ + --hash=sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7 \ + --hash=sha256:55ea99acb17b9325618de155a0cd6a2e8f5d10be008113e1d433bbb58db543b2 \ + --hash=sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a \ + --hash=sha256:588461c2e8384d309bd63e5826019b6977bc66d629b99ac8737bb795d7b2cb5a \ + --hash=sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a \ + --hash=sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c \ + --hash=sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5 \ + --hash=sha256:59f63901b0031c3136cf64704dcb21de0bbae62ce2c9529bc39d27665463de37 \ + --hash=sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e \ + --hash=sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4 \ + --hash=sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800 \ + --hash=sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055 \ + --hash=sha256:619799369eeef6366ed3e8755a5670f4f2f0fb6b30a0fd7264dc0fdc2357058e \ + --hash=sha256:62588a277bfb59def052abd940703fa35107152bf479781a878617d60faf8fb5 \ + --hash=sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c \ + --hash=sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b \ + --hash=sha256:68eb192d85ab8e5f6ec69c2bc6ac0179fbf04a5ac1569d12fbef74883fe102d0 \ + --hash=sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80 \ + --hash=sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a \ + --hash=sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4 \ + --hash=sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2 \ + --hash=sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58 \ + --hash=sha256:75a3ceed0724d625d64b86ca20aba182e4df462e04c2414fc941c0f523f06aac \ + --hash=sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc \ + --hash=sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639 \ + --hash=sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf \ + --hash=sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d \ + --hash=sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f \ + --hash=sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c \ + --hash=sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc \ + --hash=sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4 \ + --hash=sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253 \ + --hash=sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade \ + --hash=sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858 \ + --hash=sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26 \ + --hash=sha256:87e50a3e7cb90af586b6c5faf23e302a970415ac73bd7bd90a515a04b427ef96 \ + --hash=sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8 \ + --hash=sha256:8a893cc101149f80a653f82062ebc95b34525a2614382e1da5458fe7c6997249 \ + --hash=sha256:8b2bfab86aa71ae13aa41a6a26aab338e0db2b8bc75434b05aea89e011ff35a4 \ + --hash=sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13 \ + --hash=sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1 \ + --hash=sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03 \ + --hash=sha256:93223adc95033dd47133a46ccfc316a0139176fd79085762e27202ec56018f03 \ + --hash=sha256:9373ad13ef0d2c0fb761e04e55bfdee5a08b52cef2c882c8fbe9935b1517152e \ + --hash=sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364 \ + --hash=sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4 \ + --hash=sha256:9bde855991b7e362c146535e3136a50bfaffc0487d38b33ca7e5edefc6e23849 \ + --hash=sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0 \ + --hash=sha256:9cf9b1a857e25c4baceeb3624e92a56df3668f398c4acba74e174d81fb4d1d3a \ + --hash=sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036 \ + --hash=sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3 \ + --hash=sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21 \ + --hash=sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3 \ + --hash=sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e \ + --hash=sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413 \ + --hash=sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21 \ + --hash=sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346 \ + --hash=sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429 \ + --hash=sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685 \ + --hash=sha256:b736353c0a625bbd5fcec108576e2385db3496f4f771f785ff32e108d3c3bc45 \ + --hash=sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f \ + --hash=sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c \ + --hash=sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d \ + --hash=sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad \ + --hash=sha256:bd16aabe4a02a297c23417aa17ac6299dbd8c49f673bcd645b4929b11f5a4400 \ + --hash=sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb \ + --hash=sha256:c6708715abcf3c73b99508253e961a9967f02fe536532834149574eda6de0d1c \ + --hash=sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc \ + --hash=sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c \ + --hash=sha256:c9790464842f85f437dbbb54417eda1e0e6bfc52dd8d22d6fd1c994b73b2dc74 \ + --hash=sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf \ + --hash=sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604 \ + --hash=sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f \ + --hash=sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105 \ + --hash=sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a \ + --hash=sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d \ + --hash=sha256:d4a7319f304a774bed22115bc891618e45f85065ab44ea6acd07d274e750519a \ + --hash=sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1 \ + --hash=sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5 \ + --hash=sha256:d913de495d90407cd859d263bee2e5d1a4ed3eb6573c04e70d9ec619a7cbed7f \ + --hash=sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e \ + --hash=sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709 \ + --hash=sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874 \ + --hash=sha256:ddf19c062bea7a0cc80f519243d2c01dd091be0cf952a0750d4ad576709559f5 \ + --hash=sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc \ + --hash=sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95 \ + --hash=sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd \ + --hash=sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0 \ + --hash=sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d \ + --hash=sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3 \ + --hash=sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c \ + --hash=sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3 \ + --hash=sha256:e80e6c2f55656b4824d72065abb4ddd6a525c74bd78a0aab5d9fc2cf4fb5af50 \ + --hash=sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491 \ + --hash=sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5 \ + --hash=sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5 \ + --hash=sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655 \ + --hash=sha256:ef4fcbf3327382cd4c9f540babd61248208af7b93eec4de397b4d5f58a09e288 \ + --hash=sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd \ + --hash=sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084 \ + --hash=sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d \ + --hash=sha256:f5833ad231be5eb6553de524a70f48d71b2c8563101750531e0b80184e175cd4 \ + --hash=sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915 \ + --hash=sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1 \ + --hash=sha256:fb9e68df06293761f9fe66ade60a9bc6d0f5e42b8acf2939a9158af86ab0e5bd \ + --hash=sha256:fc14a032f813bf5fe624d991960ea83e9715adc27e4c1830a2361eb1d02ac341 \ + --hash=sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424 \ + --hash=sha256:fd1fbe0f116b6e55da77aca2c6ddcddcfac2186cbf78bdebf40fc156efca389d \ + --hash=sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f + # via + # -c /tmp/ocr-tested-constraints.txt + # requests +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ + --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 + # via + # -c /tmp/ocr-tested-constraints.txt + # uvicorn +colorlog==6.12.0 \ + --hash=sha256:2a7924c1dadf18b22a0eb8b06d1c7b01d5341707ec1641eb6fcc4fde0c3e8e5f \ + --hash=sha256:30d392604e9110045a2c2aeefc27d7a017abbab63f3a8aee594eac0801df784e + # via + # -c /tmp/ocr-tested-constraints.txt + # rapidocr +fastapi==0.143.3 \ + --hash=sha256:75e8bd68e2f52f77d29b2879749c257d64de2d85282d25270caf2bfccce5efe5 \ + --hash=sha256:b9ab567ee57afb4454004ce04c889f8f0160ac47946b99795887cd6e88484848 + # via + # -c /tmp/ocr-tested-constraints.txt + # -r ocr/requirements.in +flatbuffers==25.12.19 \ + --hash=sha256:7634f50c427838bb021c2d66a3d1168e9d199b0607e6329399f04846d42e20b4 + # via + # -c /tmp/ocr-tested-constraints.txt + # onnxruntime +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via + # -c /tmp/ocr-tested-constraints.txt + # httpcore + # uvicorn +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via + # -c /tmp/ocr-tested-constraints.txt + # httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via + # -c /tmp/ocr-tested-constraints.txt + # -r ocr/requirements.in +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via + # -c /tmp/ocr-tested-constraints.txt + # anyio + # httpx + # requests +mpmath==1.3.0 \ + --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ + --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c + # via + # -c /tmp/ocr-tested-constraints.txt + # sympy +numpy==2.4.6 \ + --hash=sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1 \ + --hash=sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4 \ + --hash=sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f \ + --hash=sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079 \ + --hash=sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096 \ + --hash=sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47 \ + --hash=sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66 \ + --hash=sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d \ + --hash=sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1 \ + --hash=sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e \ + --hash=sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147 \ + --hash=sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd \ + --hash=sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75 \ + --hash=sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063 \ + --hash=sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73 \ + --hash=sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab \ + --hash=sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4 \ + --hash=sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41 \ + --hash=sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402 \ + --hash=sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698 \ + --hash=sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7 \ + --hash=sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8 \ + --hash=sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b \ + --hash=sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8 \ + --hash=sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0 \ + --hash=sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662 \ + --hash=sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91 \ + --hash=sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0 \ + --hash=sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f \ + --hash=sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3 \ + --hash=sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f \ + --hash=sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67 \ + --hash=sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6 \ + --hash=sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997 \ + --hash=sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b \ + --hash=sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e \ + --hash=sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538 \ + --hash=sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627 \ + --hash=sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93 \ + --hash=sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02 \ + --hash=sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853 \ + --hash=sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c \ + --hash=sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43 \ + --hash=sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd \ + --hash=sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8 \ + --hash=sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089 \ + --hash=sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778 \ + --hash=sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1 \ + --hash=sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb \ + --hash=sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261 \ + --hash=sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb \ + --hash=sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a \ + --hash=sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8 \ + --hash=sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359 \ + --hash=sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5 \ + --hash=sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7 \ + --hash=sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751 \ + --hash=sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8 \ + --hash=sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605 \ + --hash=sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e \ + --hash=sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45 \ + --hash=sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2 \ + --hash=sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895 \ + --hash=sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe \ + --hash=sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb \ + --hash=sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a \ + --hash=sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577 \ + --hash=sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d \ + --hash=sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a \ + --hash=sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda \ + --hash=sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6 \ + --hash=sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20 + # via + # -c /tmp/ocr-tested-constraints.txt + # onnxruntime + # opencv-python-headless + # rapidocr + # shapely +omegaconf==2.4.0 \ + --hash=sha256:7e219a70bea371da250285b9c01c7db027bc421722799c2cc4ff14bc4a2a753b \ + --hash=sha256:f141a664c777582405e3cc6e0545ecc79429e79f844995b1a6e71c35946a025c + # via + # -c /tmp/ocr-tested-constraints.txt + # rapidocr +onnxruntime==1.24.1 \ + --hash=sha256:1099f3629832580fedf415cfce2462a56cc9ca2b560d6300c24558e2ac049134 \ + --hash=sha256:1cabe71ca14dcfbf812d312aab0a704507ac909c137ee6e89e4908755d0fc60e \ + --hash=sha256:2dfbba602da840615ed5b431facda4b3a43b5d8276cf9e0dbf13d842df105838 \ + --hash=sha256:3273c330f5802b64b4103e87b5bbc334c0355fff1b8935d8910b0004ce2f20c8 \ + --hash=sha256:34488aa760fb5c2e6d06a7ca9241124eb914a6a06f70936a14c669d1b3df9598 \ + --hash=sha256:34b6119526eda12613f0d0498e2ae59563c247c370c9cef74c2fc93133dde157 \ + --hash=sha256:4da1b84b3bdeec543120df169e5e62a1445bf732fc2c7fb036c2f8a4090455e8 \ + --hash=sha256:557753ec345efa227c6a65139f3d29c76330fcbd54cc10dd1b64232ebb939c13 \ + --hash=sha256:6361dda4270f3939a625670bd67ae0982a49b7f923207450e28433abc9c3a83b \ + --hash=sha256:7307aab9e2e879c0171f37e0eb2808a5b4aec7ba899bb17c5f0cedfc301a8ac2 \ + --hash=sha256:780add442ce2d4175fafb6f3102cdc94243acffa3ab16eacc03dd627cc7b1b54 \ + --hash=sha256:79b3119ab9f4f3817062e6dbe7f4a44937de93905e3a31ba34313d18cb49e7be \ + --hash=sha256:82e367770e8fba8a87ba9f4c04bb527e6d4d7204540f1390f202c27a3b759fb4 \ + --hash=sha256:86bc43e922b1f581b3de26a3dc402149c70e5542fceb5bec6b3a85542dbeb164 \ + --hash=sha256:88a2b54dca00c90fca6303eedf13d49b5b4191d031372c2e85f5cffe4d86b79e \ + --hash=sha256:890c503ca187bc883c3aa72c53f2a604ec8e8444bdd1bf6ac243ec6d5e085202 \ + --hash=sha256:bd1e4aefe73b6b99aa303cd72562ab6de3cccb09088100f8ad1c974be13079c7 \ + --hash=sha256:df0af2f1cfcfff9094971c7eb1d1dfae7ccf81af197493c4dc4643e4342c0946 \ + --hash=sha256:ea4942104805e868f3ddddfa1fbb58b04503a534d489ab2d1452bbfa345c78c2 \ + --hash=sha256:ea8963a99e0f10489acdf00ef3383c3232b7e44aa497b063c63be140530d9f85 + # via + # -c /tmp/ocr-tested-constraints.txt + # -r ocr/requirements.in +opencv-python-headless==5.0.0.93 \ + --hash=sha256:030ca5e0837a2963ab36ef896baa9767eb8d2b83353fb28af5a521e40dd8756f \ + --hash=sha256:09a872a157c1376ab922a69bbf22f9a95bcc7b658a9d8b436a60212b02b2eeb4 \ + --hash=sha256:10818d91510e05c04568ae12b5cd120779c70c01bf897b001a6221fe430df80f \ + --hash=sha256:1e55af3abfb462eeeabe5c775f12bdb36216d8a93a3583d69e6bd6e1d6ba7d00 \ + --hash=sha256:829717b6a95554f273e49e357cee3b3a2a26b6f4842fbc1bed2b45bdd8f87e0e \ + --hash=sha256:840bd717c21e5c11cadadc022a823315ea417f961213d06b4df010e019eb16f4 \ + --hash=sha256:b82f9831daab90b725c7c1ee1b36cb5732c367096ac76d119e64e14eb70d5f3c \ + --hash=sha256:c6bcd96b185975ea240d22cfdb15a1f6d080cc95264cfbe2621f21bb144d89b9 \ + --hash=sha256:ed709fdf9aa0bd1f2ed8549e71d19449b03a675bb581eb292285f6861953be37 + # via + # -c /tmp/ocr-tested-constraints.txt + # rapidocr +opentelemetry-api==1.45.1 \ + --hash=sha256:aa38ed19bcc084ba42782a73255b3582283eced7ad6dddbd6695189e69adfb75 \ + --hash=sha256:b31553efa588ae44bc306f863c785c5333a9ecc091248c6ee68b4b6c87fdedfb + # via + # -c /tmp/ocr-tested-constraints.txt + # fastapi +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via + # -c /tmp/ocr-tested-constraints.txt + # onnxruntime +pillow==12.3.0 \ + --hash=sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756 \ + --hash=sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a \ + --hash=sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59 \ + --hash=sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45 \ + --hash=sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3 \ + --hash=sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df \ + --hash=sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139 \ + --hash=sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b \ + --hash=sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39 \ + --hash=sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e \ + --hash=sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8 \ + --hash=sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1 \ + --hash=sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8 \ + --hash=sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89 \ + --hash=sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5 \ + --hash=sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130 \ + --hash=sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd \ + --hash=sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d \ + --hash=sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b \ + --hash=sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed \ + --hash=sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace \ + --hash=sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb \ + --hash=sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931 \ + --hash=sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510 \ + --hash=sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6 \ + --hash=sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1 \ + --hash=sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce \ + --hash=sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385 \ + --hash=sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e \ + --hash=sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c \ + --hash=sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7 \ + --hash=sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace \ + --hash=sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c \ + --hash=sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f \ + --hash=sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64 \ + --hash=sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f \ + --hash=sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a \ + --hash=sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827 \ + --hash=sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17 \ + --hash=sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4 \ + --hash=sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a \ + --hash=sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701 \ + --hash=sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e \ + --hash=sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91 \ + --hash=sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66 \ + --hash=sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468 \ + --hash=sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217 \ + --hash=sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658 \ + --hash=sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418 \ + --hash=sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a \ + --hash=sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c \ + --hash=sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330 \ + --hash=sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402 \ + --hash=sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09 \ + --hash=sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930 \ + --hash=sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f \ + --hash=sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec \ + --hash=sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a \ + --hash=sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94 \ + --hash=sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468 \ + --hash=sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b \ + --hash=sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965 \ + --hash=sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8 \ + --hash=sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd \ + --hash=sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7 \ + --hash=sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c \ + --hash=sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777 \ + --hash=sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35 \ + --hash=sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9 \ + --hash=sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f \ + --hash=sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f \ + --hash=sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0 \ + --hash=sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c \ + --hash=sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71 \ + --hash=sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3 \ + --hash=sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838 \ + --hash=sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf \ + --hash=sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321 \ + --hash=sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26 \ + --hash=sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec \ + --hash=sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9 \ + --hash=sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65 \ + --hash=sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5 \ + --hash=sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e \ + --hash=sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d \ + --hash=sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198 \ + --hash=sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7 + # via + # -c /tmp/ocr-tested-constraints.txt + # -r ocr/requirements.in + # rapidocr +protobuf==7.36.2 \ + --hash=sha256:497d0463ff3316681da6c0b9e8d06cb465d61abce00b613ab42226175644d1bb \ + --hash=sha256:89f23aa53c24553a2416fd4fd1ec06f74fa42b14b546d8883128813f775bbfd2 \ + --hash=sha256:912c1221170e16c08d1f086762f563dd61ff83c18b5fa6652952dfaded66f728 \ + --hash=sha256:a300819d441e078a5608c0d3c709796bb548136058fda017ae51d425b44fd353 \ + --hash=sha256:bdb3a345d48db958e6ce1f18e508beb0cc981d64f24088427549c866cd039f1e \ + --hash=sha256:cbc70b17ee27e28894c7fee8bb04be1abead49e936bc70eb60052531eee2079e \ + --hash=sha256:e11e1f0180583a2af89db6a2ecd9e8dc40aa6d2988ca175bfd0e6d12ea72d74e \ + --hash=sha256:f4fee11ec330d238b34a05c9b675f693c20415d1c5bd7d5320cc2f8a798eb9cf + # via + # -c /tmp/ocr-tested-constraints.txt + # onnxruntime +psutil==7.2.2 \ + --hash=sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372 \ + --hash=sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9 \ + --hash=sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841 \ + --hash=sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63 \ + --hash=sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979 \ + --hash=sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a \ + --hash=sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b \ + --hash=sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9 \ + --hash=sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee \ + --hash=sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312 \ + --hash=sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b \ + --hash=sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9 \ + --hash=sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e \ + --hash=sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc \ + --hash=sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1 \ + --hash=sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf \ + --hash=sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea \ + --hash=sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988 \ + --hash=sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486 \ + --hash=sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00 \ + --hash=sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8 + # via + # -c /tmp/ocr-tested-constraints.txt + # -r ocr/requirements.in +pyclipper==1.4.0 \ + --hash=sha256:0a4d2736fb3c42e8eb1d38bf27a720d1015526c11e476bded55138a977c17d9d \ + --hash=sha256:0b74a9dd44b22a7fd35d65fb1ceeba57f3817f34a97a28c3255556362e491447 \ + --hash=sha256:0b8c2105b3b3c44dbe1a266f64309407fe30bf372cf39a94dc8aaa97df00da5b \ + --hash=sha256:14c8bdb5a72004b721c4e6f448d2c2262d74a7f0c9e3076aeff41e564a92389f \ + --hash=sha256:1b6c8d75ba20c6433c9ea8f1a0feb7e4d3ac06a09ad1fd6d571afc1ddf89b869 \ + --hash=sha256:222ac96c8b8281b53d695b9c4fedc674f56d6d4320ad23f1bdbd168f4e316140 \ + --hash=sha256:29dae3e0296dff8502eeb7639fcfee794b0eec8590ba3563aee28db269da6b04 \ + --hash=sha256:37bfec361e174110cdddffd5ecd070a8064015c99383d95eb692c253951eee8a \ + --hash=sha256:3ef44b64666ebf1cb521a08a60c3e639d21b8c50bfbe846ba7c52a0415e936f4 \ + --hash=sha256:58e29d7443d7cc0e83ee9daf43927730386629786d00c63b04fe3b53ac01462c \ + --hash=sha256:6a97b961f182b92d899ca88c1bb3632faea2e00ce18d07c5f789666ebb021ca4 \ + --hash=sha256:6c317e182590c88ec0194149995e3d71a979cfef3b246383f4e035f9d4a11826 \ + --hash=sha256:773c0e06b683214dcfc6711be230c83b03cddebe8a57eae053d4603dd63582f9 \ + --hash=sha256:7c87480fc91a5af4c1ba310bdb7de2f089a3eeef5fe351a3cedc37da1fcced1c \ + --hash=sha256:81d8bb2d1fb9d66dc7ea4373b176bb4b02443a7e328b3b603a73faec088b952e \ + --hash=sha256:8d42b07a2f6cfe2d9b87daf345443583f00a14e856927782fde52f3a255e305a \ + --hash=sha256:9882bd889f27da78add4dd6f881d25697efc740bf840274e749988d25496c8e1 \ + --hash=sha256:98b2a40f98e1fc1b29e8a6094072e7e0c7dfe901e573bf6cfc6eb7ce84a7ae87 \ + --hash=sha256:9bc45f2463d997848450dbed91c950ca37c6cf27f84a49a5cad4affc0b469e39 \ + --hash=sha256:a8d2b5fb75ebe57e21ce61e79a9131edec2622ff23cc665e4d1d1f201bc1a801 \ + --hash=sha256:a9f11ad133257c52c40d50de7a0ca3370a0cdd8e3d11eec0604ad3c34ba549e9 \ + --hash=sha256:adcb7ca33c5bdc33cd775e8b3eadad54873c802a6d909067a57348bcb96e7a2d \ + --hash=sha256:b3b3630051b53ad2564cb079e088b112dd576e3d91038338ad1cc7915e0f14dc \ + --hash=sha256:bafad70d2679c187120e8c44e1f9a8b06150bad8c0aecf612ad7dfbfa9510f73 \ + --hash=sha256:bbc827b77442c99deaeee26e0e7f172355ddb097a5e126aea206d447d3b26286 \ + --hash=sha256:c9a3faa416ff536cee93417a72bfb690d9dea136dc39a39dbbe1e5dadf108c9c \ + --hash=sha256:ce1f83c9a4e10ea3de1959f0ae79e9a5bd41346dff648fee6228ba9eaf8b3872 \ + --hash=sha256:d1e5498d883b706a4ce636247f0d830c6eb34a25b843a1b78e2c969754ca9037 \ + --hash=sha256:d1f807e2b4760a8e5c6d6b4e8c1d71ef52b7fe1946ff088f4fa41e16a881a5ca \ + --hash=sha256:d49df13cbb2627ccb13a1046f3ea6ebf7177b5504ec61bdef87d6a704046fd6e \ + --hash=sha256:d4b2d7c41086f1927d14947c563dfc7beed2f6c0d9af13c42fe3dcdc20d35832 \ + --hash=sha256:e9b973467d9c5fa9bc30bb6ac95f9f4d7c3d9fc25f6cf2d1cc972088e5955c01 \ + --hash=sha256:f160a2c6ba036f7eaf09f1f10f4fbfa734234af9112fb5187877efed78df9303 \ + --hash=sha256:f2a50c22c3a78cb4e48347ecf06930f61ce98cf9252f2e292aa025471e9d75b1 \ + --hash=sha256:f3672dbafbb458f1b96e1ee3e610d174acb5ace5bd2ed5d1252603bb797f2fc6 \ + --hash=sha256:fd24849d2b94ec749ceac7c34c9f01010d23b6e9d9216cf2238b8481160e703d + # via + # -c /tmp/ocr-tested-constraints.txt + # rapidocr +pydantic==2.14.1 \ + --hash=sha256:9195d967ec791692a04438115466764fb8b9a27b31f14a760437694f40d6b454 \ + --hash=sha256:94f478203dd03404682a1ada216965651dd74b1d2d5ffd62e00e0837caab5c26 + # via + # -c /tmp/ocr-tested-constraints.txt + # -r ocr/requirements.in + # fastapi +pydantic-core==2.50.1 \ + --hash=sha256:0036473f5583e6a60e50b8b21651511564277a3f05cc5dab8cf579f552cd5f6c \ + --hash=sha256:0048b6dddc8ef4b64fccaad878bd143b0c3882ea9936279dc11d613f6b7dd1bc \ + --hash=sha256:009634b83993777ddcd69cad0ffcace43dabde692109528e35f0fde91e386a8b \ + --hash=sha256:028e2f212273d4a39b1ec1e0de8166b1165a65fc0f1111452a9d94fc7c625c63 \ + --hash=sha256:062e891facce5ca296a1c37098e5e466780457f86413894b399f0cf22934f769 \ + --hash=sha256:06e01fbbfdb9be777b316a71b6c49efaf4a08b615d0a98d678cda3023f79d019 \ + --hash=sha256:06ead20d39ffd6f2f6f2a8f8a6de67ff8bb1b4f14a8a30e058502514ee2ac685 \ + --hash=sha256:0b3a6f334c6a2345ca15318ff894502a90012536404b37c844a976c76c846e0b \ + --hash=sha256:0c003c3b7f49debb893d2d85ae099ac5959c9839e2f330fadb1fcdf7a6594482 \ + --hash=sha256:131059670f1d2444269b8585cb888963994871932447c08b39ac6a51fcfef658 \ + --hash=sha256:132529c83901437ff642f585216831bf5fd7a91df66829907e155192ead62498 \ + --hash=sha256:17e722e156d0444ecaefbe640bdb60928752bf2013e2b7a11cdb099aaae19bec \ + --hash=sha256:1a9006395dece0e32e704c315eff8a00bede494f6108546cfc5539c89fef4f9a \ + --hash=sha256:1c8632d4ac04e6f91128fca584b3a8a507d81604c24eeaaad00d4be42765c32b \ + --hash=sha256:1c96fd793b73d1b92e65570132505498fe7b21eaef73cdf74e67e5dfba7ac9e4 \ + --hash=sha256:1cf41f1ae3fa155cf167a72689ad044bcc1e3c97e064123677149bdfb5dafc4a \ + --hash=sha256:1deeacb112d14d3f4fcb16b165f7dbaf76c70ba6e82f37ba042bdab51970a0b8 \ + --hash=sha256:1ef800dd7d85bcdadf4c3076e4c94e43939493558a3b69a1ea830c706d4617bb \ + --hash=sha256:1fa4c8bc12c1354c5550c0c35c1852c8c1901e89e06561724e03f8d0342e1f87 \ + --hash=sha256:2005207aafe1231315718bf6ed5d064a7300fb4772754af35ee72fc68159492e \ + --hash=sha256:23923ab9292c40da026330b1ecf4dc2618c8e86e0422e5d1fbf50d94d64ca4f8 \ + --hash=sha256:23edad659e8dbd8ca7e4e877fe6c81573abbdf215bd25a68b53e1272f58b80c7 \ + --hash=sha256:2ab756b72bd5054e4c7ef3ded331b35786cbd3cf931531a508f79a9537517064 \ + --hash=sha256:2cbd1b75b09e976ed0d6b6ca297675632ca35df86130088457cdc60ef36970ae \ + --hash=sha256:2cf91809d0721ab81592ba67bea7694821679c10b1a2e3c3460082b286c1918a \ + --hash=sha256:2df1ff41884de2bc4b307bafd7c40a691094fad2ff8e767e5b45a319257bcf4e \ + --hash=sha256:2eb75304506894a281d346220a4f7481a1b8729577c5ed2a05395991966a8396 \ + --hash=sha256:2eedf82ee4753cdab8e50044c6bd569577eebc3859b11fecf4eb9223761ff966 \ + --hash=sha256:30ddf019d082c117b5d309e5b86710c2a78909907ec1a9381feec3eec02eca0b \ + --hash=sha256:325c23f3e35cfbf0fe3486fa5f7260d1e45885173002d30a28ca019994124255 \ + --hash=sha256:32fad3a91e51b6d2039c572db04a5a873260b399f6bd62c3552671fa7a4a2899 \ + --hash=sha256:36c426eac0af8d1529ff8467e612b933346caec1fdc0d774f78f67a1a11e16c1 \ + --hash=sha256:3a5fce22f1e87d181e924e12da7d81cfe031fb3881a5ddf26ad28f141756ca43 \ + --hash=sha256:3aa9de446b793de2beb6fa2d9d0961803126c4e2a99c2f25ab59b9fd6ea125c0 \ + --hash=sha256:3e46a9eb0a0901dd6275e6b06ac3a464885ef350ec4121fe486869de8053e4bb \ + --hash=sha256:3fde4fdc6487a58d944ca87cf5adc95d5f266e872c19599f5f4c0a8a1b1f9f9f \ + --hash=sha256:409e0ea40ec30d9158f33574fd758e689f6045a0f2596701828c27816ca9687d \ + --hash=sha256:40f523349960fa30f3ea51404308ff50f9997a90df639590f47a057c1f32b415 \ + --hash=sha256:41bc8237121bd8dc8d888dfd6279fc166ffc88c1f1bf3a8bf00869680533ca4c \ + --hash=sha256:42b54c2c90ad348b5e3a85e03e715d572c1fde357ef104cdfe3b03b697a404ea \ + --hash=sha256:455a773617b5913bf5c20d0692e5787b119e52c4d40ea644ca31f5758fd31be2 \ + --hash=sha256:45b11cac094aa25725581d9304eee93c9028516b9ea80dd9e175e13a5a2c840e \ + --hash=sha256:45c6266d071c241f2a168d45bf8c54344f0effce35e7e6b73afdec11f3687568 \ + --hash=sha256:46b3301d3b5c886f77de7546e47274a5842c622ea2020b8c6524c6b66913b4a6 \ + --hash=sha256:476f6ed8e43cd1e0b460920e23571700872b284e77331cb30c4faf459cf48a4b \ + --hash=sha256:48569b0ade9edfbe065cad1d700175546592aebbb42f02adcebcc26e75b896fe \ + --hash=sha256:49c2cbb2397fe4d0987e84606e691af6cb87bc0ee1bd3e7b737f7e10b4c142f9 \ + --hash=sha256:4a53d13cdfbedbfa87f08b83c1a0a5efcc767d785a4b41934fa9cb672670493a \ + --hash=sha256:4be846f55c9477f5f3ddde8f2ce941137e16862a56d018ed885d422bb6ae02f2 \ + --hash=sha256:4df197990c15b5a37c5a277d131d9f2c67de6133f2e5dafd80d9bba4b99f46f9 \ + --hash=sha256:4e834f6a8e4ff772dcc34f58ef5504147a3ea5b0f4eeb13b0f8eb2ca75ac57f1 \ + --hash=sha256:57f51b31ff826e2859120cf4737c5a758a48d96f3e97da40ccee1796d58078ff \ + --hash=sha256:5958c72adb417c39b12ac87525ac60b0d73315fcdc59e21f44ee4a5e2512c9ef \ + --hash=sha256:5dfe41f232befddb9c4377f6cfc702b51595e2d78ed082672adf8758d2c4619f \ + --hash=sha256:5f3cae32fc46121f787cb2486de9cf95a8bf72aec5cc78f64c606fa1735a6ef5 \ + --hash=sha256:64f6047f62a6c5ae08d0a6afb035667aa2d97c3d20d69762e034c5ea144d92a5 \ + --hash=sha256:6a733778df2f7087ec1100ed0b41533e4f3001976e99570fa34f57c66e7f8e3e \ + --hash=sha256:6b20a4bffabdad0db2927ac034ae3b8a681b1f7a0182f3e60b479ad2fde21ebb \ + --hash=sha256:6dbcbee53bf17196a7f745aa9bf5a9603953a1e365b1f020be3207c676a3e7c4 \ + --hash=sha256:6ed4f3cef55164b026fefb41341b7754cc6b624c75dfe7142d2ecceb5ad21c87 \ + --hash=sha256:704075d10b74f2f3c6e15407c696d88701df35fc8953f434a431add0d0074db0 \ + --hash=sha256:739dc730e6be3bd5ec2f4ab5cfc7eb047cc45fc1497b3bafec74ff2ed07df597 \ + --hash=sha256:7456d699b13954e9c0164dcb267250a10ae0dfb03e6e26d6796ab0d46e189c84 \ + --hash=sha256:756d669f04e62ec4148ecfe22be6a4484d9b1181a6ef32e205ebfd200540858b \ + --hash=sha256:7689580e72a642ab5ec64d5f55b2e33636fa43b4ebe63c0c2c965ef307c7d1aa \ + --hash=sha256:76e2e83fa6ec8cdc972d438dafc2522b3a47bee4ec0ae668b29cfb1977ab5242 \ + --hash=sha256:7816e98acc08119dc0f340ab167048ecc54126316330c1f0caf7c6756c88e28f \ + --hash=sha256:79490e33c4c0fcb933bbbcfc3a62184d8803b99f535863dfbb925e1bcb6945ad \ + --hash=sha256:7f476456ac2bb0d937f75191494a09c83a30765fea4f70f3b404942fe25f6cdf \ + --hash=sha256:844b869f118e22a41a091bdcedda8a71bc1b0f62c38d1a0c3211cece47e1d8fc \ + --hash=sha256:84bc765b282a9d5b7fe0348b8648904f25a6a04b2139da52b1dd30c8ac3a2c8f \ + --hash=sha256:84f34323a61a365b4e9295de6028474754829aaddd59c7bf1a040e7487ef8f3c \ + --hash=sha256:8812592c85d0edf423f10eadcef42716d71e8219085ad9e85b775057b7306133 \ + --hash=sha256:88e492e8b9d0312e7dc13667c30222abf284dc3b79b5302b3607b41a5784ce61 \ + --hash=sha256:8a6791afa2245e6c6b180122d105941644f5bd410bb18623b408808cc41a3102 \ + --hash=sha256:8b4c3df25bd323bf1d36a648d563cf1fc69d717451569927151bdad7cad07a77 \ + --hash=sha256:8daa7ee75245d43ad7d747e5c9ecc1b1d06552f72b14887e9276f787d57375f4 \ + --hash=sha256:93ba4e9d8210d941c200431a56b2c0400b131865947903937ed3ec5404307d2e \ + --hash=sha256:94845ff54dc5193f228cab81b2662a04bfbb892e95bdc15edf7399000ce57d54 \ + --hash=sha256:94be440c03fede26969a5ce75468e0e6a9927a1b46d9b679ee8adc1b057b0350 \ + --hash=sha256:9572c1369e9c9da2d64a7b7992c786d90ff295abc93964cfe3125e4290768070 \ + --hash=sha256:983a662de2571cb2502fc8ff47b6770b03d025d2eb314c92f77b3f07c74720ed \ + --hash=sha256:992c3514ec891fa7858099183e4d64e6bd5a5d4ff452fae29df22faa77a006bb \ + --hash=sha256:99ba9bc2b8062ea0c326a990f7f00e6530c23579de66dd246e72c4cafef950a5 \ + --hash=sha256:9d1bed94af6a63835461f3cf7502058eb166c58c4778e11d0f433cfb1bd69e19 \ + --hash=sha256:9e4472072de0137ee0d8e72d6620e85939c271d2f90f6bbb4b15c24638b79f92 \ + --hash=sha256:a27c09d86600f1bf2fe3f37e1ae697faf3143931c09322cd799da94deee923b5 \ + --hash=sha256:a29a061fec0b4e2d714f277e70a3a18125ecff803f2fea6eade2f2e53711d112 \ + --hash=sha256:a3cda0e538208e5d722bbf3698b24f19c0a7d05bc8d5f8a7f9b121ea7fa243d9 \ + --hash=sha256:a44101320cfe99432db74237545a63057dc7a88dfe792cbcad0647f2af56cb81 \ + --hash=sha256:a4aaaa791bdae1c972a7e81765f4f3571c926b8e0b9b6e47346499fb80079665 \ + --hash=sha256:a51eee75939cf811ac09b278745a6cee7dc873ccfbc8b9af3cc88fe4b7ce25b5 \ + --hash=sha256:a7c58106de36ac6a56314182958de20db8d3a29dfd5db527192cc754e4f8e7fb \ + --hash=sha256:aa8224f10880d9bf1b5993988ba153d42a8b4f3f4f511f93b1f09c93ff613c72 \ + --hash=sha256:acbf31f37c53a5ac0c34706c80b4f5107ba20b05fdd3816124bf236ef0c57dd2 \ + --hash=sha256:adc06d218a1cadfd2ec4628424d7d79ce4eba69c2965e7e7b55106f0da5208c8 \ + --hash=sha256:ae28183297fb0d2b8dc46a1f01d51f5e45825fc5afe76a835a6cb7fb34821295 \ + --hash=sha256:b0135bcdcaa0f23573f286e4cb5e0fd2962700964ed13df085b85f2b97aeab9e \ + --hash=sha256:b087b1c5be7ac687cf22eabfe4b6b608d40df23610651e93611e1f49118baf84 \ + --hash=sha256:b0d955195bbbe489ad343fcc956eacea9357b79cb22192c66cacdefcbc14b32f \ + --hash=sha256:b281a3b0f0822618fe5e3e0d8a2048b6356b14388505dc9374ccffeb69989713 \ + --hash=sha256:b6d0c2183008c188e19f4906d426b293bdc4f67ab17df8e180fe16cda208fa71 \ + --hash=sha256:bbce99252ba3167b2b6277f1829d5bf4b43b754524bddf7f944707c3db7d2253 \ + --hash=sha256:bc1f08f68dac9f9e83845a8039880aba2ab553eb9b2259c3243a313182c253fe \ + --hash=sha256:bc94f474417604bd383d2cd445d071b07dd55fedceed3ce33407bf1fcc107290 \ + --hash=sha256:bed5163e03b98bc1fa2eb05d74c63d9c5c95d8ed6254985481640fbf5e237dea \ + --hash=sha256:c17799a62c142d61b8a3c51752a7cbc87fe2ad4ccfab10e628a77b405075c662 \ + --hash=sha256:c18db21573bd2c6489f9a544b7499f0df2853958c568e5e783536ee1f690af41 \ + --hash=sha256:c3ede305158e75510be50869b319550ab072008c13d64d4ab1e094fb286b6f44 \ + --hash=sha256:c516cc5367ca3448995d42cb994bf3f4c9002d2a7c22eac9622551269ad1b807 \ + --hash=sha256:c531166c42ea7bdfecc8c50049581f05dd1993b09cc7c52bb36a14e96deaec7d \ + --hash=sha256:c73622ef819328873b53109ee4f77ceb598bffedd02daf916102be3228866b78 \ + --hash=sha256:c8dce1f1e0e5358b682a6ad3fa5e31b31d4560997b8e61417e9217c8d60f8a0c \ + --hash=sha256:cb57f304525a5e3c13333b772bf9a473f36326e9c821b2e8e1b2fd36f80ae2c3 \ + --hash=sha256:ce8c25ca38cc0e3d7753ba180808de2c0c8cb24eae0df64491e40921454e9831 \ + --hash=sha256:ceff0acc940be2715bd6ad17b24c0e5304abf44f6efd0f81ee8499e640f9dc86 \ + --hash=sha256:cf356f70551d40374eaffb1aa63f1eb6d2006681cbd7a9faea173ce0f4dd7cd2 \ + --hash=sha256:d2d82aa62521c55ddfb000ae70f88cdd8de974078f6024e821dfe5addd0c818f \ + --hash=sha256:d32f3acc081cc3923386d88f422cde8892335e95f034e0104bb4cf9310d9915f \ + --hash=sha256:d4193206b6587047437f6f11d7e776df23e1c1e23af2a54d9347275614791e10 \ + --hash=sha256:d5c0e32fdbce7f1e8ef4d11f655694bf5f4175c757a9f1dc2be09b8864e5bcf5 \ + --hash=sha256:d5e062c01286d861fd6a1c4ff6e063547b3e713067f2df033c0ff97ac2ca006b \ + --hash=sha256:d8f9e8a6c4ab04b78d61f78627370d834eb004b2869dcb28cfffa647b4ea1980 \ + --hash=sha256:d939de9c82e2126f7f48a7e658f8a85ed46d57662d53f44c49b8895fe94a3eb7 \ + --hash=sha256:de531ce1e2a3364e8767878b58f4ff728a434b4fde089781fe30b1e08e2396e0 \ + --hash=sha256:e50d7b94baac6c7d09927fa5ca5800a0c7ee5015c7fcff65beb3a1931b5a6e09 \ + --hash=sha256:e5faeaee74a57d32b3ab3aebad2e348f06d3ba946fc5d28c1728455f00a3d13a \ + --hash=sha256:e6f0cc1bb9900dc558960894adeb30b0c083366fc1d69b856209fb2ca5c36fe5 \ + --hash=sha256:e8e1d6ce820aa23317e8209a86bd65a540973c12dc7552b48a4f6c8e9926815e \ + --hash=sha256:ed1e728b39a383c81035b2459cfcb35d99dfb01f7d6ebe3a913bc1cc5b81e459 \ + --hash=sha256:ee6db2fbed51a7991302e8fac498cd67e336246026d0dfa84cf5166ce1412760 \ + --hash=sha256:efbecf43d321f7b9281441f1f213f7c21c66988b0e06c2730ba13ed47a46bb08 \ + --hash=sha256:f2c634642694e6a0dad2ab1d375589fa671fd442edd5caf7d9737b8f6ca22906 \ + --hash=sha256:f3377c8c2b3ce898423c5e5dd94c7982e30aa7717a7e6ab2470b9de364963709 \ + --hash=sha256:f5187624823423e1d1b82b1072ac41dc837389e18d3d0572cc19bbee46cd550a \ + --hash=sha256:f77ac30b19221cd9bd3fcfa3d4614eff93140d0572ab730cded17b64adca05f3 \ + --hash=sha256:fe90228920fd8ff2be62622b6bb8a2b11acd65046d50c6b130614b5879605a20 + # via + # -c /tmp/ocr-tested-constraints.txt + # pydantic +pypdf==6.21.0 \ + --hash=sha256:3aec619699dacb1b73f57e87d9adaa4424814517b2cc028de20adba5a37af4c4 \ + --hash=sha256:c16174c8653abafbe4d023b3c9227d226938d9579ffb5a73b4fac7756c0895a6 + # via + # -c /tmp/ocr-tested-constraints.txt + # -r ocr/requirements.in +pypdfium2==5.14.0 \ + --hash=sha256:09b99c8f0cb427eb17fec13c0862ed598bba34b4843df153f70fff806a2820bc \ + --hash=sha256:11f281613fa22313d9c7ab89947665e84eccf8ebe40e1198a84a88352305648d \ + --hash=sha256:149fd5c6397b8df8bf7911a93506eff0be874f877afe7ac936cf5d37d21a6a06 \ + --hash=sha256:1951f0aed469150b13c62eabd501a9839e608ab9983ca8579be9eb73213b72b6 \ + --hash=sha256:2de384df66ba55fcaab0775f30f28ec1090af3dfa60276a07821efc96d993118 \ + --hash=sha256:382de7fe20d32c42993a274d7b6c555a5623a97570dfc1d2f5e0a16fe0d5d482 \ + --hash=sha256:51d9e9b64ebc34effaf57f9b6d4511b3f66ad3744bd1690d2cc6700853173dcf \ + --hash=sha256:593f2c952ae3ffdca0efcbb3d9464fbccb876254386114ff900cabef21157c3f \ + --hash=sha256:605ab9d0d4c5e223599c9065b88d16b2c1f131c807c80dea8adbb16f1433e95b \ + --hash=sha256:790e2cac1641a65912b73bd7243f45195d36f1663c85a3e1a126a8f5867c82a3 \ + --hash=sha256:9f4d77db5232826dd03a63481f32164331b96c21fd68f0667b2e43dbae141a93 \ + --hash=sha256:9fd5cc94a389d50298e4d8cb79af6b9b8e0d785606e2a937725dc6e271c9c6e6 \ + --hash=sha256:b40a0913196a1483f0fdc22a53f8719c3aef87f1c4d8d9c38d2ad4e207500fdf \ + --hash=sha256:bed597b2cea3990164e43f9003f71db18959d0abd5d73adc9c176e7be2d84b98 \ + --hash=sha256:c5f009b3157f10e97dceb55963f5910eff92feb00587ba10a76f12b87ce1a4b6 \ + --hash=sha256:c73be14076bedebd9bcaf9b062579c95c668580043bccd29eb0db502101d5716 \ + --hash=sha256:d436ee9e024f981e68f5775f5a9d115f93ea14ee6c2c6efd35dd17d83edf4942 \ + --hash=sha256:dbfd6deff68cc46b134acd6be380d98d694a9f018fbb622c07229225c85db389 \ + --hash=sha256:e4e203ea9710fd00e5448edb6f1615dc8587035357f75f40b432dde0c33e8da1 \ + --hash=sha256:e70d87cb0577eab38f2106f9c9606b458930beef612a1b5f298772ed259f5ec0 \ + --hash=sha256:eb8aeca157808f323e39ea298cc6d6c8e080c192ea2efb1ca81daa0f0ff4d095 \ + --hash=sha256:f1b696e6901e16f114a2ec6332e5e3f8f5033a901614ead28499ab18ca6024f5 \ + --hash=sha256:f6f13bbcc5f4adabc2676e52f662c6cb375de86b314790b0ae08f3ab62eb116a + # via + # -c /tmp/ocr-tested-constraints.txt + # -r ocr/requirements.in +python-multipart==0.0.32 \ + --hash=sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e \ + --hash=sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23 + # via + # -c /tmp/ocr-tested-constraints.txt + # -r ocr/requirements.in +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # -c /tmp/ocr-tested-constraints.txt + # omegaconf + # rapidocr +rapidocr==3.10.0 \ + --hash=sha256:2fc34e26cd0f48514804a94f8832283039c7334076bff60768b1fed9ad068050 + # via + # -c /tmp/ocr-tested-constraints.txt + # -r ocr/requirements.in +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via + # -c /tmp/ocr-tested-constraints.txt + # rapidocr +shapely==2.2.0 \ + --hash=sha256:000c0ce2a3ba49427e6288b7add9de5d8525d4e65d6ebc8840103040d4d57b86 \ + --hash=sha256:0a63e6b68ec785ef3aae3935c4aa9fb8edccced94e23c79d5d85276442c60859 \ + --hash=sha256:0edec813c81effaf4e20c18b1aa86827925ce27c0315621f2a1a080e22e0de5e \ + --hash=sha256:17434cb9819c9974c3331333a3b878fa5bf8f85dd69cc3fb7ff5d260f6fbc102 \ + --hash=sha256:1af6935acde1db0b6a1bcbea30cbad5ae900723dfd398367ae1488470dc53667 \ + --hash=sha256:1eaa2cb64cdedaf65d6bc86f2819c9cd7d6d68f969aa3ebfdc93743ab581f437 \ + --hash=sha256:24b175c570efc91d1180ac6cd527dc80e863bb7de37f8b2771703d822c65e023 \ + --hash=sha256:287ec7602f7a114b862ae0123880e57160cebe059843a4c7028aaee9e74287f6 \ + --hash=sha256:2fd87e55d7a7d310553b527378545cdc6ef8702473ed9294926b892c3cfb2ba0 \ + --hash=sha256:3575a323b7665d7a2e391b16a626caa6b6f6348f399183aca3fc656febd7cf04 \ + --hash=sha256:40871d7135cd723f965d200181aa28418e9ec029fd85bdd010488259d1c01906 \ + --hash=sha256:446b2d5a323bddd1c2a27f41325fdb3a3e8e33c1f8f0f840bdb63e8c1515b29e \ + --hash=sha256:48dd1d961391f314ab7fa8812c86ca2a727bee2bdca1478730eacaea007da18e \ + --hash=sha256:4e5830637c080bdc646c5982ad6f7cc296b93038879649f7a6acd8e0f1c4db04 \ + --hash=sha256:4ed79beb8d4b6cc7c67780fd381feed25848a5f9b8a2385ac5711eccd115647a \ + --hash=sha256:556f20346a7d96fefbb71b74640d84ca14041703d60f0d2ff47b29d9b3e0093d \ + --hash=sha256:593e51cd04fe1122f1ab3fae87b306c36b2be0184a5e0d9c26849c55ff4580dc \ + --hash=sha256:596b7994ceafa526b6e0522ca29fbc41d19f86459161d6efe1f251d0acd49f3f \ + --hash=sha256:5b740c9a197e5feb30bdc6e64a5eb3ca2a7324d11498844136dfc317daac6a99 \ + --hash=sha256:5d61088e2ef71dafad0dd4fae8a521cc1f20da4a89d3096bab5b3260b39b3052 \ + --hash=sha256:626fe4c0d32860a98e75ecffabf5a62254c6168eac96b633ad313cd62a38bb2b \ + --hash=sha256:650a5f4d8a8e3c96982079d8c99b6ddbe6602bbd1e34c75c2b95dbc0d28ac997 \ + --hash=sha256:6ec5178a39803fa8626322f69d298037f182461dd28e3ae96c2c7a4309a6bf30 \ + --hash=sha256:715561ceda03b09ca1c6baf9922179392d8c2bc53a1b877965225f0dfb487a58 \ + --hash=sha256:74028f468e05e461b30a479b08c1fb5094fa45062abeeec8e7905a6711761436 \ + --hash=sha256:7416db8ff3a1003687d4118e741343b3cf9ac2a4a925a59d44d98a865ac4e9e7 \ + --hash=sha256:74f4313af38d6e49ea83532d6cedfb4fe5e6c5485d7c40202bd61b19d6ff09bf \ + --hash=sha256:770d4db5cf0bfeed931a1c4aaf4f4eadad0f43f5fc72c27c88fe1f07904ae767 \ + --hash=sha256:7765e0e5d51d63eae0a911861cbda87165a01677bc9bce6ed20d06858ccde99f \ + --hash=sha256:776cc8571d53e42be8fa6d42ad52a599b8e2186dd0c752922831508099af71e2 \ + --hash=sha256:778421a19085bef1fb38bc0699db1ee9b08fdd0e30a8768788d601a4371f2de0 \ + --hash=sha256:7c0b262116bb75b86751440b42e19673911bc0a8f0d5ce723ce294c3d6e4d5c0 \ + --hash=sha256:7cf5b3a801b9b4febf774efde2e31280e647388deae8452693d8e6420b3a1ff2 \ + --hash=sha256:7f68c1fbacab81c0c066d1c3051eeb0f680b7a7a2c511e741f77741640187896 \ + --hash=sha256:806d399418b23eee7241736d572ad1e0b784782f9241d7c8e2cfceb00787831d \ + --hash=sha256:8bc985ad731da2f2cedde9c3cfb3c3d946fe6fc63d2ca557673dc33dd1e389b9 \ + --hash=sha256:8d6ffe94710f37535a47161120cd5f7f0f0d9bb800c2fddebbd089cb7f1b3453 \ + --hash=sha256:90895df6542ae039fc6557dec6194e3509e883fbd6f5788e3c3e7a38fe46b257 \ + --hash=sha256:9147ebc3b116a0511dca043937f85caf1a41690815643d5b89c8bc472f51c850 \ + --hash=sha256:96e5101ad2d73df869255bae4c55537f372d32066e2328c376e09841f0f66800 \ + --hash=sha256:9ee11aeba1759d15a525ded58e17916d3edfa60d52110fd8df6a7609a871f066 \ + --hash=sha256:a851e077f0f02a3383923e02eca5447a29ddbf234e39593b91c8b7ac75218133 \ + --hash=sha256:a9a380624cdd7a7e661bf15a4d1625082766f07ccd2540cb0a9e0df1ad4f6c11 \ + --hash=sha256:b2338ac40e6652c8bfb857936ea9be9a16f43a362c6f67eb3bad741b05fd5683 \ + --hash=sha256:b8cb04906b74db26f848f76744fa995cd6abeae9145d27cc405277de1f949660 \ + --hash=sha256:c037369c35510f51100dd6d386ee3203bac32f164d53e27ca12c3cea5bb643b1 \ + --hash=sha256:c2915ae1b858e73d5832be7fb5e89497cc5140fa505da40a45223029dc6deace \ + --hash=sha256:c36ccbff5c3374c349c370bfdac22c7676b268b4a707c98e9031f498965aa02d \ + --hash=sha256:c3caa4c6308e7eaf18f4661134a1575eb290a56df78d0ae1b02f919a4cc7bd9d \ + --hash=sha256:c4127c064bc71f8b7f9b3f341d6627ed39977fd0b61a17c68d09179f5e0089ae \ + --hash=sha256:c88b21a0e9599ebb741e08f71a95c8f07a434af909efb088828a9874d234d06d \ + --hash=sha256:cbe184e1946cfe115a9dfeadd2effd88ab4a237ab1a4335d106defa80fbc2d82 \ + --hash=sha256:ce858295be3947143a3f44f145fa6dbacd5dcc5c4103801d42cd3be4a2034614 \ + --hash=sha256:d75957716368f919c63016dae1977a0d007e15f06861cd178701edb91b08d2b0 \ + --hash=sha256:d9b11d712ac72f1d869f2b6964dea5bd9f20b89901adcd796d6712496144ab22 \ + --hash=sha256:da47a0cc9e630b4dff0db46e8972b29d2d27f337425ce9d4c77fd046ce48eabd \ + --hash=sha256:dc5faa593948aa64d9afae48331b80f43f7aacc68425d99064a4d6772f53f1ad \ + --hash=sha256:e414c78bc81aadd76a429111a350f4ef3d05fc13019805617b524951258468e5 \ + --hash=sha256:e8865e553d874a1ec4a032057ea81fca9def37b188cd8fb550af3b3480b3f88c \ + --hash=sha256:f340e7f99aaee3df5acd6b247cddf723051a7c93d1e1ef09025b80d84e4c0ded \ + --hash=sha256:f79b3b34ad2d067207f21f821489c720b14ce40f3bfda931987a193165f80133 \ + --hash=sha256:f8cd733a66a2a10f461a70dde9fad7b2b62c6a48c7a66cea57ee6f1cd9f2bd2f \ + --hash=sha256:fdb599ec540cea5b635ac47bf24fca4cdfd1c39730ffc0b6cf0d2666b0dd9a33 \ + --hash=sha256:ff9e87b534edf35af65758fafb31ad3b797354cba9323899e263f450c69a2ff2 + # via + # -c /tmp/ocr-tested-constraints.txt + # rapidocr +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via + # -c /tmp/ocr-tested-constraints.txt + # rapidocr +starlette==1.8.0 \ + --hash=sha256:1565dc0b35d5737a271ed1e0e04e949f4e81198799f216d2667b0a0fb9cf9522 \ + --hash=sha256:dfdd6b29c26483288088d990eee59631dedadd66ce20d203402a7ca8e3c4656f + # via + # -c /tmp/ocr-tested-constraints.txt + # fastapi +sympy==1.14.0 \ + --hash=sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517 \ + --hash=sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5 + # via + # -c /tmp/ocr-tested-constraints.txt + # onnxruntime +tqdm==4.70.1 \ + --hash=sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73 \ + --hash=sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4 + # via + # -c /tmp/ocr-tested-constraints.txt + # rapidocr +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # -c /tmp/ocr-tested-constraints.txt + # anyio + # fastapi + # opentelemetry-api + # pydantic + # pydantic-core + # starlette + # typing-inspection +typing-inspection==0.4.4 \ + --hash=sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47 \ + --hash=sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147 + # via + # -c /tmp/ocr-tested-constraints.txt + # fastapi + # pydantic +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via + # -c /tmp/ocr-tested-constraints.txt + # requests +uvicorn==0.54.0 \ + --hash=sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf \ + --hash=sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620 + # via + # -c /tmp/ocr-tested-constraints.txt + # -r ocr/requirements.in diff --git a/ocr/routing.py b/ocr/routing.py new file mode 100644 index 0000000000000000000000000000000000000000..0750615207327ce16ce0913e4654ed0ffddc1a50 --- /dev/null +++ b/ocr/routing.py @@ -0,0 +1,338 @@ +"""Independent bounded lanes with retry-time policy checks and warm EWMAs.""" + +import asyncio +import importlib.metadata +import random +import time +from collections import deque +from dataclasses import dataclass + +from .quality import quality +from .schemas import Attempt, SafeError +from .security import authorize + + +@dataclass +class Work: + page: object + options: object + future: asyncio.Future + queued: float + + +class Lane: + def __init__(self, backend, workers, settings): + self.backend = backend + self.settings = settings + self.workers = workers + self.queue = asyncio.Queue(settings.queue_size) + self.tasks = [] + self.active = 0 + self.ewma = None + self.samples = deque(maxlen=500) + self.failures = 0 + self.open_until = 0 + self.calls = deque() + self.completed = 0 + self.errors = 0 + self.cold_ms = None + + def start(self): + self.tasks = [asyncio.create_task(self._worker()) for _ in range(self.workers)] + + async def close(self): + for t in self.tasks: + t.cancel() + await asyncio.gather(*self.tasks, return_exceptions=True) + while not self.queue.empty(): + item = self.queue.get_nowait() + if not item.future.done(): + item.future.cancel() + self.queue.task_done() + close = getattr(self.backend, "close", None) + if close: + await close() + + async def eligible(self, options): + if time.monotonic() < self.open_until: + return False + try: + authorize(self.backend, options, self.settings, await self.backend.capabilities()) + return ( + await asyncio.wait_for(self.backend.health(), min(5, self.settings.timeout)) + ).ready + except (SafeError, asyncio.TimeoutError): + return False + + def estimate(self): + return ( + (self.queue.qsize() + self.active + self.workers) / self.workers * (self.ewma or 5000) + ) + + async def submit(self, page, options): + f = asyncio.get_running_loop().create_future() + try: + self.queue.put_nowait(Work(page, options, f, time.monotonic())) + except asyncio.QueueFull: + raise SafeError("queue_full", True) from None + try: + return await f + except asyncio.CancelledError: + f.cancel() + raise + + async def _worker(self): + while True: + w = await self.queue.get() + counted = False + try: + if w.future.cancelled(): + continue + self.active += 1 + counted = True + try: + value = await self._execute(w) + if not w.future.done(): + w.future.set_result(value) + except SafeError as e: + if not w.future.done(): + w.future.set_exception(e) + finally: + if counted: + self.active = max(0, self.active - 1) + self.queue.task_done() + + async def _execute(self, w): + attempts = [] + for retry in range(2): + if w.future.cancelled(): + raise SafeError("cancelled") + # Policy and health are re-evaluated immediately before EVERY request. + authorize( + self.backend, + w.options, + self.settings, + await self.backend.capabilities(), + ) + if time.monotonic() < self.open_until: + raise SafeError("circuit_open") + if not (await self.backend.health()).ready: + raise SafeError("backend_unavailable", True) + if self.backend.name == "gpu": + now = time.monotonic() + while self.calls and now - self.calls[0] > 60: + self.calls.popleft() + if len(self.calls) >= self.settings.gpu_budget: + raise SafeError("gpu_budget_exhausted") + self.calls.append(now) + start = time.monotonic() + wait = (start - w.queued) * 1000 + try: + result = await asyncio.wait_for( + self.backend.recognize(w.page, w.options), self.settings.timeout + ) + result.flags = quality(result, w.page) + if "empty_nonblank" in result.flags: + raise SafeError("empty_nonblank") + ms = (time.monotonic() - start) * 1000 + attempts.append( + Attempt( + backend=self.backend.name, + destination=result.provider_destination, + engine=result.engine, + outcome="ok", + latency_ms=ms, + queue_wait_ms=wait, + ) + ) + self.failures = 0 + self.completed += 1 + # Exclude first successful request (cold) from warm capacity estimates. + if self.cold_ms is None: + self.cold_ms = ms + else: + self.samples.append(ms) + self.ewma = ms if self.ewma is None else 0.2 * ms + 0.8 * self.ewma + result.attempts = attempts + result.attempt_count = len(attempts) + result.latency_ms = sum(a.latency_ms for a in attempts) + return result + except Exception as e: + safe = ( + e + if isinstance(e, SafeError) + else SafeError( + "backend_timeout" + if isinstance(e, asyncio.TimeoutError) + else "backend_failure", + isinstance(e, asyncio.TimeoutError), + ) + ) + self.failures += 1 + self.errors += 1 + attempts.append( + Attempt( + backend=self.backend.name, + outcome=safe.code, + latency_ms=(time.monotonic() - start) * 1000, + queue_wait_ms=wait, + ) + ) + if self.failures >= 3: + self.open_until = time.monotonic() + 30 + if retry == 0 and safe.retryable and self.failures < 3: + await asyncio.sleep(0.1 + random.random() * 0.1) + continue + safe.attempts = attempts + raise safe from None + + +class Dispatcher: + def __init__(self, cpu, gpu, settings): + self.settings = settings + self.cpu = Lane(cpu, settings.cpu_workers, settings) + self.gpu = Lane(gpu, settings.gpu_workers, settings) if gpu else None + + def start(self): + self.cpu.start() + if self.gpu: + self.gpu.start() + + async def close(self): + await self.cpu.close() + if self.gpu: + await self.gpu.close() + + async def page(self, page, options): + from .schemas import OCRPageResult + + if page.embedded and not options.force_ocr: + return OCRPageResult( + page_index=page.index, + page_id=page.page_id, + source="embedded_text", + engine="pypdf", + backend_version=importlib.metadata.version("pypdf"), + provider_destination="local", + text=page.embedded, + markdown=page.embedded, + image_width=page.width, + image_height=page.height, + rotation=page.rotation, + route_reason="usable_pdf_text_layer", + sensitivity=options.sensitivity, + flags=["text_layer_heuristic"], + ) + mode = options.mode + if mode == "cpu_only": + cpu_ok, gpu_ok = await self.cpu.eligible(options), False + elif mode == "gpu_only": + cpu_ok, gpu_ok = False, bool(self.gpu and await self.gpu.eligible(options)) + else: + + async def gpu_eligible(): + return bool(self.gpu and await self.gpu.eligible(options)) + + cpu_ok, gpu_ok = await asyncio.gather(self.cpu.eligible(options), gpu_eligible()) + reason = "cpu_default" + if mode == "cpu_only": + lane = self.cpu if cpu_ok else None + reason = "explicit_cpu" + elif mode == "gpu_only": + lane = self.gpu if gpu_ok else None + reason = "explicit_gpu" + else: + lane = self.cpu if cpu_ok else self.gpu if gpu_ok else None + if gpu_ok and page.features.get("complex") and mode in ("auto", "quality"): + caps = await self.gpu.backend.capabilities() + if caps.layout_blocks: + lane = self.gpu + reason = "complex_layout_verified" + if ( + mode == "throughput" + and gpu_ok + and cpu_ok + and options.allow_gpu_overflow + and self.settings.overflow_enabled + and self.settings.overflow_qualified + ): + # No speculative speedup: both warm measured capacities are mandatory. + if ( + self.cpu.ewma + and self.gpu.ewma + and self.gpu.estimate() < self.cpu.estimate() * 0.85 + ): + lane = self.gpu + reason = "measured_completion_time" + if lane is None: + raise SafeError("no_authorized_destination") + other = self.gpu if lane is self.cpu else self.cpu + failed_attempts = [] + attempted_names = {lane.backend.name} + try: + result = await lane.submit(page, options) + except SafeError as e: + failed_attempts = getattr(e, "attempts", []) + if mode in ("cpu_only", "gpu_only") or not other or not await other.eligible(options): + raise + attempted_names.add(other.backend.name) + result = await other.submit(page, options) + lane = other + other = self.gpu if lane is self.cpu else self.cpu + result.flags += ["fallback_used"] + reason = "approved_fallback" + quality_failed = any( + f in result.flags + for f in ( + "low_confidence", + "low_contrast_or_blur", + "suspiciously_short", + "repeated_text", + "unresolved_noise", + ) + ) + dual = mode == "quality" and self.settings.dual_run and page.features.get("complex") + if ( + (quality_failed or dual) + and mode not in ("cpu_only", "gpu_only") + and other + and other.backend.name not in attempted_names + and await other.eligible(options) + ): + try: + alt = await other.submit(page, options) + if alt.text.strip() != result.text.strip(): + result.flags += ["disagreement", "needs_review"] + if quality_failed and len(alt.flags) < len(result.flags): + alt.alternatives = [result] + if alt.text.strip() != result.text.strip(): + alt.flags += ["disagreement", "needs_review"] + result = alt + reason = "quality_escalation" + else: + result.alternatives = [alt] + except SafeError as exc: + result.attempts += getattr(exc, "attempts", []) + result.flags += ["escalation_unavailable", "needs_review"] + elif quality_failed: + result.flags += ["needs_review"] + result.attempts = failed_attempts + result.attempts + result.attempt_count = len(result.attempts) + sum( + a.attempt_count for a in result.alternatives + ) + if options.layout == "blocks" and not result.blocks: + result.flags += ["layout_unavailable", "needs_review"] + result.route_reason = reason + result.page_id = page.page_id + result.rotation = page.rotation + result.sensitivity = options.sensitivity + for alternative in result.alternatives: + alternative.sensitivity = options.sensitivity + alternative.page_id = page.page_id + alternative.rotation = page.rotation + if not options.preserve_boxes: + for candidate in [result] + result.alternatives: + for b in candidate.blocks: + b.bbox_1000 = None + b.polygon_1000 = None + return result diff --git a/ocr/ruff.toml b/ocr/ruff.toml new file mode 100644 index 0000000000000000000000000000000000000000..ff0f73d5b4c5b6622a99505beaeb19cb4a3b5433 --- /dev/null +++ b/ocr/ruff.toml @@ -0,0 +1,6 @@ +target-version = "py311" +line-length = 100 +[lint] +select = ["E4", "E7", "E9", "F", "I", "B"] +# FastAPI documents File/Form dependency declarations in defaults. +ignore = ["B008"] diff --git a/ocr/schemas.py b/ocr/schemas.py new file mode 100644 index 0000000000000000000000000000000000000000..0e627162d2ea743563f4d182ed0df626afa32f15 --- /dev/null +++ b/ocr/schemas.py @@ -0,0 +1,166 @@ +import math +from dataclasses import dataclass, field +from typing import Literal, Protocol + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +Sensitivity = Literal["D0", "D1", "D2"] + + +class SafeError(Exception): + """Only constant machine codes may leave the service boundary.""" + + def __init__(self, code: str, retryable: bool = False): + self.code, self.retryable = code, retryable + super().__init__(code) + + +class OCROptions(BaseModel): + model_config = ConfigDict(extra="forbid") + mode: Literal["auto", "throughput", "quality", "cpu_only", "gpu_only"] = "auto" + sensitivity: Sensitivity = "D2" + output: Literal["text", "markdown", "structured", "both"] = "both" + language_hints: list[str] = Field(default_factory=lambda: ["de", "en"], max_length=10) + layout: Literal["auto", "text", "blocks"] = "auto" + preserve_boxes: bool = True + extract_pdf_text_layer: bool = True + force_ocr: bool = False + prefer_cpu: bool = True + allow_external_cpu: bool = True + allow_gpu_overflow: bool = False + max_pages: int = Field(default=100, ge=1, le=100) + + +class Block(BaseModel): + type: str = "text" + text: str = "" + bbox_1000: list[float] | None = None + polygon_1000: list[list[float]] | None = None + confidence: float | None = None + confidence_kind: str | None = None + + @field_validator("bbox_1000") + @classmethod + def validate_bbox(cls, box): + if box is not None and ( + len(box) != 4 + or not all(math.isfinite(v) and 0 <= v <= 1000 for v in box) + or box[0] > box[2] + or box[1] > box[3] + ): + raise ValueError("invalid_bbox") + return box + + @field_validator("polygon_1000") + @classmethod + def validate_polygon(cls, points): + if points is not None and ( + len(points) != 4 + or any( + len(p) != 2 or not all(math.isfinite(v) and 0 <= v <= 1000 for v in p) + for p in points + ) + ): + raise ValueError("invalid_polygon") + return points + + @field_validator("confidence") + @classmethod + def validate_confidence(cls, value): + if value is not None and not (math.isfinite(value) and 0 <= value <= 1): + raise ValueError("invalid_confidence") + return value + + +class Attempt(BaseModel): + backend: str + destination: str | None = None + engine: str | None = None + outcome: str + latency_ms: float = 0 + queue_wait_ms: float = 0 + + +class OCRPageResult(BaseModel): + schema_version: str = "ocr-page/v1" + page_index: int + page_id: str = "" + source: Literal["embedded_text", "cpu_ocr", "alias_ocr"] + engine: str | None = None + backend_version: str | None = None + provider_destination: str | None = None + text: str + markdown: str + image_width: int + image_height: int + rotation: int = 0 + transform: str = "source-raster; normalized 0..1000" + blocks: list[Block] = Field(default_factory=list) + flags: list[str] = Field(default_factory=list) + route_reason: str = "" + latency_ms: float = 0 + attempt_count: int = 0 + attempts: list[Attempt] = Field(default_factory=list) + alternatives: list["OCRPageResult"] = Field(default_factory=list) + sensitivity: Sensitivity = "D2" + + +class BackendCapabilities(BaseModel): + name: str + text: bool = True + bounding_boxes: bool = False + layout_blocks: bool = False + tables: bool = False + math: bool = False + languages: list[str] = Field(default_factory=list) + max_image_px: int = 20_000_000 + supports_sensitive_classes: list[str] = Field(default_factory=list) + effective_destination_verifiable: bool = False + grounding_verified: bool = False + multimodal_verified: bool = False + + +class BackendHealth(BaseModel): + ready: bool + code: str = "ok" + + +@dataclass +class RasterPage: + index: int + image: bytes + width: int + height: int + page_id: str + rotation: int = 0 + embedded: str | None = None + features: dict = field(default_factory=dict) + + +class OCRBackend(Protocol): + name: str + destination: str | None + external: bool + + async def capabilities(self) -> BackendCapabilities: ... + async def health(self) -> BackendHealth: ... + async def recognize(self, page: RasterPage, options: OCROptions) -> OCRPageResult: ... + + +class Failure(BaseModel): + page_index: int + code: str + + +class OCRResult(BaseModel): + schema_version: str = "ocr-result/v1" + job_id: str + request_id: str + status: str + page_count: int + sensitivity: Sensitivity + markdown: str + text: str + pages: list[OCRPageResult] + failures: list[Failure] + anonymization_required: bool = True diff --git a/ocr/security.py b/ocr/security.py new file mode 100644 index 0000000000000000000000000000000000000000..63d41aa0e0fc94b8966e3173df94f6e37489650e --- /dev/null +++ b/ocr/security.py @@ -0,0 +1,32 @@ +"""Fail closed before egress. Client options can only restrict server policy.""" + +from .schemas import SafeError + +D2_PROVIDERS = frozenset({"helmholtz", "blablador", "instituteforx"}) + + +def sensitivity(requested, minimum): + return max(requested, minimum, key=lambda x: ("D0", "D1", "D2").index(x)) + + +def authorize(backend, options, settings, caps): + level = options.sensitivity + if not caps.text or not caps.multimodal_verified: + raise SafeError("backend_unqualified") + if backend.name == "cpu": + if backend.external and ( + not options.allow_external_cpu or level not in settings.external_cpu_classes + ): + raise SafeError("no_authorized_destination") + if backend.external and level != "D0": + # No remote D1/D2 approval protocol is implemented; configuration alone is insufficient. + raise SafeError("no_authorized_destination") + return + if not settings.gpu_enabled: + raise SafeError("gpu_disabled") + # No existing FreeLLMAPI destination enforcement contract was available for audit. + # D2 stays hard-disabled, including hidden retry/fallback and caller overrides. + if level != "D0": + raise SafeError("no_authorized_destination") + if not backend.qualified: + raise SafeError("backend_unqualified") diff --git a/ocr/tests/conftest.py b/ocr/tests/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..a7f935cb107b568c7e38e82ff750eb80136cd3f7 --- /dev/null +++ b/ocr/tests/conftest.py @@ -0,0 +1,69 @@ +import asyncio +import io + +import pytest +from ocr.schemas import BackendCapabilities, BackendHealth, Block, OCRPageResult +from PIL import Image + + +@pytest.fixture +def png(): + b = io.BytesIO() + Image.new("RGB", (200, 120), "white").save(b, format="PNG") + return b.getvalue() + + +class Fake: + external = False + + def __init__( + self, + name="cpu", + *, + text="Public fixture text", + delay=0, + fail=None, + confidence=0.98, + external=False, + ): + self.name = name + self.text = text + self.delay = delay + self.fail = fail + self.confidence = confidence + self.calls = [] + self.external = external + self.destination = "fake-local" if not external else "https://unapproved.invalid" + self.qualified = True + self.ready = True + + async def capabilities(self): + return BackendCapabilities(name=self.name, text=True, multimodal_verified=True) + + async def health(self): + return BackendHealth(ready=self.ready) + + async def recognize(self, page, options): + self.calls.append(page.index) + await asyncio.sleep(self.delay * (3 - page.index % 3)) + if self.fail: + error = self.fail() if callable(self.fail) else self.fail + if error: + raise error + return OCRPageResult( + page_index=page.index, + source="cpu_ocr" if self.name == "cpu" else "alias_ocr", + text=self.text + str(page.index), + markdown=self.text + str(page.index), + engine="test-only", + provider_destination=self.destination, + image_width=page.width, + image_height=page.height, + blocks=[ + Block( + text=self.text, + confidence=self.confidence if self.name == "cpu" else None, + confidence_kind="test-only", + ) + ], + ) diff --git a/ocr/tests/fixtures/candidate_v1.json b/ocr/tests/fixtures/candidate_v1.json new file mode 100644 index 0000000000000000000000000000000000000000..44f65e578e70eb960f032e1a7a490b3e64b69d41 --- /dev/null +++ b/ocr/tests/fixtures/candidate_v1.json @@ -0,0 +1 @@ +{"schema_version":"ocr-anonymization-candidate/v1","candidate_id":"00000000000000000000000000000000","sensitivity":"D2","pages":[{"page_index":0,"anonymized_text":"Synthetic [PERSON].","tags":[],"source":"cpu_ocr","original_sensitivity":"D2"}],"model_revision":"ba967ee030b7e679dd5422cb16fcd1b379da38e1","anonymous":false,"release_review_required":true,"release_approved":false,"automated_release_eligible":false,"review_reasons":["recognition_incomplete_possible"]} \ No newline at end of file diff --git a/ocr/tests/fixtures/correction_host.py b/ocr/tests/fixtures/correction_host.py new file mode 100644 index 0000000000000000000000000000000000000000..1e884b8954a767219122a423ea8fdabdb53477f0 --- /dev/null +++ b/ocr/tests/fixtures/correction_host.py @@ -0,0 +1,251 @@ +"""ISOLATED synthetic correction store/identity fixture. NOT durable production storage. + +Transactions use an RLock/CAS only. Production must independently qualify durable +atomic source/candidate/proposal retention and current authorization inside commits. +""" + +import threading +import time + +from ocr.anonymization.corrections import ( + CorrectionContext, + CorrectionProposal, + PublicationReceipt, + StoredProposal, + author_digest, + commitment, +) +from ocr.anonymization.host_adapter import AuthenticatedIdentity, CompletedReview +from ocr.anonymization.release_policy import digest +from ocr.anonymization.schemas import CorrectedCandidate, Destination, VerifiedReview + + +class CorrectionHostFixture: + production_ready = False + durable_corrections = False + + def __init__(self, context, texts, *, separate_approver): + if type(separate_approver) is not bool: + raise ValueError() + self.separate_approver = separate_approver + self.lock = threading.RLock() + self.author_credential, self.approver_credential = object(), object() + self.identities = { + self.author_credential: AuthenticatedIdentity( + subject="synthetic-author", + session_id="author-session", + expires_at=int(time.time()) + 600, + ), + self.approver_credential: AuthenticatedIdentity( + subject="synthetic-approver", + session_id="approver-session", + expires_at=int(time.time()) + 600, + ), + } + self.contexts = {context.material.binding.review_ref: context} + self.bindings = {context.material.binding.review_ref: context.material.binding} + self.heads = {commitment(context.machine): commitment(context)} + self.source = { + (s.evidence_ref, s.page_index): texts[s.page_index].encode() + for s in context.machine.sources + } + self.candidates = {} + if context.material.candidate is not None: + self.candidates[digest(context.material.candidate)] = context.material.candidate + self.proposals, self.completed, self.exports = {}, {}, {} + self.allowed = set() + binding = context.material.binding + for identity in self.identities.values(): + self.allowed.add( + ( + identity.subject, + "anonymization:review", + binding.sensitivity, + binding.environment_id, + ) + ) + self.allowed.add( + ( + identity.subject, + "anonymization:release", + binding.sensitivity, + "synthetic-private", + ) + ) + for action in ("inspect", "submit", "apply"): + self.allowed.add( + ( + "synthetic-author", + "anonymization:correction:" + action, + binding.sensitivity, + binding.environment_id, + ) + ) + self.policy = Destination( + destination_id="synthetic-private", + destination_class="private", + allowed_sensitivities=("D0", "D1", "D2"), + ) + self.context_reads, self.proposal_reads = 0, 0 + self.before_publish = None + + def legacy_release_allowed(self, candidate, review): + return ( + self.candidates.get(digest(candidate)) == candidate + and digest(candidate) == review.candidate_digest + ) + + def authenticate(self, credential): + return self.identities[credential] + + def current_acl(self, identity, scope, sensitivity, destination): + return (identity.subject, scope, sensitivity, destination) in self.allowed + + def private_environment(self, environment): + return environment == "synthetic-room" + + def review_binding(self, reference): + return self.bindings[reference] + + def load_review_material(self, reference): + return self.contexts[reference].material + + def read_classified_page(self, reference, index): + return self.source[(reference, index)] + + def candidate_exists(self, value): + return value in self.candidates + + def load_correction_context(self, reference): + self.context_reads += 1 + return self.contexts[reference] + + def correction_binding(self, identifier): + return self.proposals[identifier].proposal.review_ref + + def load_correction(self, identifier): + self.proposal_reads += 1 + return self.proposals[identifier] + + def destination(self, identifier): + if identifier != self.policy.destination_id: + raise KeyError() + return self.policy + + def load_review(self, identifier): + return self.completed[identifier] + + def load_export(self, value): + return self.exports[value] + + def _check(self, credential, proposal, expected, action): + identity = self.authenticate(credential) + context = self.contexts[proposal.review_ref] + binding = self.review_binding(proposal.review_ref) + if ( + identity.subject != proposal.author_subject + or identity.expires_at <= time.time() + or binding != context.material.binding + or binding.revoked + or binding.expires_at <= time.time() + or not self.private_environment(binding.environment_id) + or not self.current_acl( + identity, "anonymization:review", binding.sensitivity, binding.environment_id + ) + or not self.current_acl( + identity, + "anonymization:correction:" + action, + binding.sensitivity, + binding.environment_id, + ) + or commitment(context) != expected + or proposal.context_digest != expected + or self.heads[commitment(context.machine)] != expected + or not proposal.created_at <= time.time() < proposal.expires_at + ): + raise PermissionError("synthetic commit denied") + return context + + def store_correction(self, credential, proposal, expected_context): + with self.lock: + proposal = CorrectionProposal.model_validate(proposal) + self._check(credential, proposal, expected_context, "submit") + if proposal.correction_id in self.proposals: + raise ValueError() + self.proposals[proposal.correction_id] = StoredProposal( + proposal=proposal, revoked=False, published_candidate_digest=None + ) + return True + + def publish_correction(self, credential, proposal, context, expected_context): + with self.lock: + if self.before_publish is not None: + self.before_publish() + old = self._check(credential, proposal, expected_context, "apply") + context = CorrectionContext.model_validate(context) + record = self.proposals[proposal.correction_id] + candidate = context.material.candidate + if ( + record.proposal != proposal + or record.revoked + or record.published_candidate_digest is not None + or context.machine != old.machine + or type(candidate) is not CorrectedCandidate + or candidate.correction.proposal_digest != commitment(proposal) + or candidate.correction.prior_candidate_digest != old.material.candidate_digest + or context.material.binding.sensitivity != old.material.binding.sensitivity + or context.material.binding.environment_id != old.material.binding.environment_id + or context.material.binding.expires_at != old.material.binding.expires_at + or context.material.binding.review_ref in self.contexts + ): + raise ValueError() + value = digest(candidate) + reference = context.material.binding.review_ref + self.contexts[reference] = context + self.bindings[reference] = context.material.binding + self.candidates[value] = candidate + self.proposals[proposal.correction_id] = record.model_copy( + update={"published_candidate_digest": value, "published_review_ref": reference} + ) + self.heads[commitment(context.machine)] = commitment(context) + return PublicationReceipt( + correction_id=proposal.correction_id, + proposal_digest=commitment(proposal), + candidate_digest=value, + context_digest=commitment(context), + review_ref=reference, + ) + + def complete_review(self, credential, value): + """Privileged synthetic test setup ONLY, not a completion API/default.""" + identity = self.authenticate(credential) + candidate = self.candidates[value] + context = next(c for c in self.contexts.values() if c.material.candidate_digest == value) + reference = "completed-" + str(len(self.completed)) + self.completed[reference] = CompletedReview( + attestation=VerifiedReview( + reviewer_subject=identity.subject, + review_id=reference, + evidence_digest=context.machine.evidence_digest, + candidate_digest=value, + sensitivity=candidate.sensitivity, + destination_id=self.policy.destination_id, + completed_at=int(time.time()), + ), + completed=True, + revoked=False, + expires_at=int(time.time()) + 600, + ) + return reference + + def corrected_release_allowed(self, candidate, review): + if ( + self.candidates.get(digest(candidate)) != candidate + or type(candidate) is not CorrectedCandidate + ): + return False + return ( + not self.separate_approver + or author_digest(review.reviewer_subject) + not in candidate.correction.author_subject_digests + ) diff --git a/ocr/tests/fixtures/gate03h_review_pocs.py.txt b/ocr/tests/fixtures/gate03h_review_pocs.py.txt new file mode 100644 index 0000000000000000000000000000000000000000..abc7c0fbf33afe4396eee00507e932f9856ee607 --- /dev/null +++ b/ocr/tests/fixtures/gate03h_review_pocs.py.txt @@ -0,0 +1,44 @@ +"""Review-only PoCs. No production files or PR tests are modified.""" + +import hashlib +import json +import time + +import pytest +from ocr.anonymization.candidate_io import parse_candidate +from ocr.anonymization.engine import AnonymizationEngine +from ocr.anonymization.private_review import PrivateReviewMaterial, ReviewBinding, SourcePageReference +from ocr.anonymization.release_policy import digest +from ocr.schemas import SafeError +from ocr.tests.test_anonymization_security import StubRecognizer, evidence + + +def test_duplicate_page_manifest_should_be_rejected(): + texts = ("Synthetic first page", "Synthetic second page") + candidate = AnonymizationEngine(StubRecognizer()).handoff(evidence("D2", texts)) + data = json.loads(candidate.model_dump_json()) + data["pages"][1] = data["pages"][0] + duplicate = parse_candidate(json.dumps(data)) + # Matching cardinality is not a bijection: page 1 has disappeared. + assert [p.page_index for p in duplicate.pages] == [0, 0] + sources = tuple(SourcePageReference( + evidence_ref=hashlib.sha256(f"synthetic-page-{i}".encode()).hexdigest(), + page_index=i, sensitivity="D2", sha256=hashlib.sha256(t.encode()).hexdigest(), + characters=len(t), + ) for i, t in enumerate(texts)) + with pytest.raises((ValueError, SafeError)): + PrivateReviewMaterial( + binding=ReviewBinding(review_ref="review-poc", environment_id="private-poc", + sensitivity="D2", expires_at=int(time.time())+600, revoked=False), + evidence_digest="a"*64, sources=sources, candidate=duplicate, + candidate_digest=digest(duplicate), provenance=duplicate.provenance, + blocking_reason=None, + ) + + +def test_invalid_replacement_bounds_should_be_rejected_by_parser(): + candidate = AnonymizationEngine(StubRecognizer()).handoff(evidence("D2", ("Synthetic",))) + data = json.loads(candidate.model_dump_json()) + data["pages"][0]["tags"][0].update(start=9, end=-1) + with pytest.raises((ValueError, SafeError)): + parse_candidate(json.dumps(data)) diff --git a/ocr/tests/fixtures/gate03h_valid_candidates.json b/ocr/tests/fixtures/gate03h_valid_candidates.json new file mode 100644 index 0000000000000000000000000000000000000000..cb84edabc984eb6dd7b3d4e3bd8b495b36b92561 --- /dev/null +++ b/ocr/tests/fixtures/gate03h_valid_candidates.json @@ -0,0 +1,39 @@ +{ + "generated_at_source_sha": "8b0aaf4ee0d22db977db600d24b96785aeca6305", + "text": [ + "\ud83e\udded Gr\u00fc\u00dfe SyntheticAlpha.", + "\u6f22\u5b57 SecondSyntheticName." + ], + "cases": { + "v1-0": { + "candidate_json": "{\"schema_version\":\"ocr-anonymization-candidate/v1\",\"candidate_id\":\"270c4a6298254dfa82d5a968e6b5d2a5\",\"sensitivity\":\"D2\",\"pages\":[{\"page_index\":0,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"},{\"page_index\":1,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"}],\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"anonymous\":false,\"release_review_required\":true,\"release_approved\":false,\"automated_release_eligible\":false,\"review_reasons\":[\"recognition_incomplete_possible\"]}", + "candidate_digest": "1c601de96820bf15d369b8d12906d6d4a10f14d3d99ddf01137bfd9b29067cbb", + "material_json": "{\"binding\":{\"review_ref\":\"original-review\",\"environment_id\":\"synthetic-room\",\"sensitivity\":\"D2\",\"expires_at\":1792359689,\"revoked\":false},\"evidence_digest\":\"e92d5714a8d75252ece99e73592ac817dafcece741b2c5442ffa01ffbf9bac95\",\"sources\":[{\"evidence_ref\":\"36384d06f622ef817c2c6ffcab437ea1913432529b1c2986f9e924e84c420cac\",\"page_index\":0,\"sensitivity\":\"D2\",\"sha256\":\"6e8b79fa149157fbfa19c51a3386078de28cdc93a62ec910b8fe73b476cf145d\",\"characters\":23},{\"evidence_ref\":\"19747a0b391f98c39168b01efb18223955afceb08e33a71421fdf2c9c61a5f87\",\"page_index\":1,\"sensitivity\":\"D2\",\"sha256\":\"03887c7a53fc37b09011d6baf70fd94f862a9e409185c0cb8007076cd91771c9\",\"characters\":23}],\"candidate\":{\"schema_version\":\"ocr-anonymization-candidate/v1\",\"candidate_id\":\"270c4a6298254dfa82d5a968e6b5d2a5\",\"sensitivity\":\"D2\",\"pages\":[{\"page_index\":0,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"},{\"page_index\":1,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"}],\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"anonymous\":false,\"release_review_required\":true,\"release_approved\":false,\"automated_release_eligible\":false,\"review_reasons\":[\"recognition_incomplete_possible\"]},\"candidate_digest\":\"1c601de96820bf15d369b8d12906d6d4a10f14d3d99ddf01137bfd9b29067cbb\",\"provenance\":null,\"blocking_reason\":null,\"anonymous\":false,\"automated_release_eligible\":false,\"release_review_required\":true}" + }, + "v2-0": { + "candidate_json": "{\"schema_version\":\"ocr-anonymization-candidate/v2\",\"candidate_id\":\"270c4a6298254dfa82d5a968e6b5d2a5\",\"sensitivity\":\"D2\",\"pages\":[{\"page_index\":0,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"},{\"page_index\":1,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"}],\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"anonymous\":false,\"release_review_required\":true,\"release_approved\":false,\"automated_release_eligible\":false,\"review_reasons\":[\"recognition_incomplete_possible\"],\"provenance\":{\"schema_version\":\"recognition-provenance/v1\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"configuration_digest\":\"9ea9537bb8cbb7b3e5d21c566acf10a80bebc1ee2a4aad53f3e2618f586d50c8\",\"terminology_version\":\"religious-terminology-v1\",\"status_version\":\"religious-status-v1\",\"replacement_version\":\"enclosing-spans-failclosed-v1\",\"normalization_version\":\"source-codepoint-v1\"}}", + "candidate_digest": "b9775811c16eb082e73bf76fa4eb3646d7eaa7019a19a94a652e0f30235a6e98", + "material_json": "{\"binding\":{\"review_ref\":\"original-review\",\"environment_id\":\"synthetic-room\",\"sensitivity\":\"D2\",\"expires_at\":1792359689,\"revoked\":false},\"evidence_digest\":\"e92d5714a8d75252ece99e73592ac817dafcece741b2c5442ffa01ffbf9bac95\",\"sources\":[{\"evidence_ref\":\"36384d06f622ef817c2c6ffcab437ea1913432529b1c2986f9e924e84c420cac\",\"page_index\":0,\"sensitivity\":\"D2\",\"sha256\":\"6e8b79fa149157fbfa19c51a3386078de28cdc93a62ec910b8fe73b476cf145d\",\"characters\":23},{\"evidence_ref\":\"19747a0b391f98c39168b01efb18223955afceb08e33a71421fdf2c9c61a5f87\",\"page_index\":1,\"sensitivity\":\"D2\",\"sha256\":\"03887c7a53fc37b09011d6baf70fd94f862a9e409185c0cb8007076cd91771c9\",\"characters\":23}],\"candidate\":{\"schema_version\":\"ocr-anonymization-candidate/v2\",\"candidate_id\":\"270c4a6298254dfa82d5a968e6b5d2a5\",\"sensitivity\":\"D2\",\"pages\":[{\"page_index\":0,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"},{\"page_index\":1,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"}],\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"anonymous\":false,\"release_review_required\":true,\"release_approved\":false,\"automated_release_eligible\":false,\"review_reasons\":[\"recognition_incomplete_possible\"],\"provenance\":{\"schema_version\":\"recognition-provenance/v1\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"configuration_digest\":\"9ea9537bb8cbb7b3e5d21c566acf10a80bebc1ee2a4aad53f3e2618f586d50c8\",\"terminology_version\":\"religious-terminology-v1\",\"status_version\":\"religious-status-v1\",\"replacement_version\":\"enclosing-spans-failclosed-v1\",\"normalization_version\":\"source-codepoint-v1\"}},\"candidate_digest\":\"b9775811c16eb082e73bf76fa4eb3646d7eaa7019a19a94a652e0f30235a6e98\",\"provenance\":{\"schema_version\":\"recognition-provenance/v1\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"configuration_digest\":\"9ea9537bb8cbb7b3e5d21c566acf10a80bebc1ee2a4aad53f3e2618f586d50c8\",\"terminology_version\":\"religious-terminology-v1\",\"status_version\":\"religious-status-v1\",\"replacement_version\":\"enclosing-spans-failclosed-v1\",\"normalization_version\":\"source-codepoint-v1\"},\"blocking_reason\":null,\"anonymous\":false,\"automated_release_eligible\":false,\"release_review_required\":true}" + }, + "v3-0": { + "candidate_json": "{\"schema_version\":\"ocr-anonymization-candidate/v3\",\"candidate_id\":\"8ed4bc38ade7432eadc49e4d8e894a11\",\"sensitivity\":\"D2\",\"pages\":[{\"page_index\":0,\"anonymized_text\":\"[ORGANIZATION]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"ORGANIZATION\",\"placeholder\":\"[ORGANIZATION]\",\"score\":null,\"source\":\"human_correction\",\"model_revision\":\"reviewed-corrections-v1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"},{\"page_index\":1,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"}],\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"anonymous\":false,\"release_review_required\":true,\"release_approved\":false,\"automated_release_eligible\":false,\"review_reasons\":[\"recognition_incomplete_possible\",\"human_correction_requires_review\"],\"provenance\":{\"schema_version\":\"recognition-provenance/v1\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"configuration_digest\":\"9ea9537bb8cbb7b3e5d21c566acf10a80bebc1ee2a4aad53f3e2618f586d50c8\",\"terminology_version\":\"religious-terminology-v1\",\"status_version\":\"religious-status-v1\",\"replacement_version\":\"enclosing-spans-failclosed-v1\",\"normalization_version\":\"source-codepoint-v1\"},\"correction\":{\"schema_version\":\"ocr-correction-provenance/v1\",\"policy_version\":\"reviewed-corrections-v1\",\"correction_id\":\"b989595a388944c889cfe4f0c74f12ee\",\"proposal_digest\":\"5e95249a57b3680e062661e4302f1133f5f42212017e49c9d1513e6b36479eba\",\"original_evidence_digest\":\"e92d5714a8d75252ece99e73592ac817dafcece741b2c5442ffa01ffbf9bac95\",\"machine_snapshot_digest\":\"a198f81d187708579e1debb7a353bca02dd2f8f53c723002b2cc872eef254856\",\"source_references_digest\":\"45baf17369b5d44293adc6c045c45c8315862346e8ae86d4efe2aa593771210a\",\"prior_candidate_digest\":\"b9775811c16eb082e73bf76fa4eb3646d7eaa7019a19a94a652e0f30235a6e98\",\"prior_state_digest\":\"ecb497582c2690cc98e29a8128197a9a8da758d6249454dc754769133aa215c7\",\"effective_state_digest\":\"dea92cb7c070164a07f11afa1730e01f56a6bcde46102ed1285c3f155af781c3\",\"author_subject_digests\":[\"b673638fbcf3408727574d4a880a49360fab63056a3af9164463d414c31a3993\"],\"edit_count\":1}}", + "candidate_digest": "b0ea11aeecd83b48f902ccf722b5af3be100897f58e92e049cc63e315703dafb", + "material_json": "{\"binding\":{\"review_ref\":\"correction-9fc01a58b85c40e086801c1562705f60\",\"environment_id\":\"synthetic-room\",\"sensitivity\":\"D2\",\"expires_at\":1792359689,\"revoked\":false},\"evidence_digest\":\"e92d5714a8d75252ece99e73592ac817dafcece741b2c5442ffa01ffbf9bac95\",\"sources\":[{\"evidence_ref\":\"36384d06f622ef817c2c6ffcab437ea1913432529b1c2986f9e924e84c420cac\",\"page_index\":0,\"sensitivity\":\"D2\",\"sha256\":\"6e8b79fa149157fbfa19c51a3386078de28cdc93a62ec910b8fe73b476cf145d\",\"characters\":23},{\"evidence_ref\":\"19747a0b391f98c39168b01efb18223955afceb08e33a71421fdf2c9c61a5f87\",\"page_index\":1,\"sensitivity\":\"D2\",\"sha256\":\"03887c7a53fc37b09011d6baf70fd94f862a9e409185c0cb8007076cd91771c9\",\"characters\":23}],\"candidate\":{\"schema_version\":\"ocr-anonymization-candidate/v3\",\"candidate_id\":\"8ed4bc38ade7432eadc49e4d8e894a11\",\"sensitivity\":\"D2\",\"pages\":[{\"page_index\":0,\"anonymized_text\":\"[ORGANIZATION]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"ORGANIZATION\",\"placeholder\":\"[ORGANIZATION]\",\"score\":null,\"source\":\"human_correction\",\"model_revision\":\"reviewed-corrections-v1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"},{\"page_index\":1,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"}],\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"anonymous\":false,\"release_review_required\":true,\"release_approved\":false,\"automated_release_eligible\":false,\"review_reasons\":[\"recognition_incomplete_possible\",\"human_correction_requires_review\"],\"provenance\":{\"schema_version\":\"recognition-provenance/v1\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"configuration_digest\":\"9ea9537bb8cbb7b3e5d21c566acf10a80bebc1ee2a4aad53f3e2618f586d50c8\",\"terminology_version\":\"religious-terminology-v1\",\"status_version\":\"religious-status-v1\",\"replacement_version\":\"enclosing-spans-failclosed-v1\",\"normalization_version\":\"source-codepoint-v1\"},\"correction\":{\"schema_version\":\"ocr-correction-provenance/v1\",\"policy_version\":\"reviewed-corrections-v1\",\"correction_id\":\"b989595a388944c889cfe4f0c74f12ee\",\"proposal_digest\":\"5e95249a57b3680e062661e4302f1133f5f42212017e49c9d1513e6b36479eba\",\"original_evidence_digest\":\"e92d5714a8d75252ece99e73592ac817dafcece741b2c5442ffa01ffbf9bac95\",\"machine_snapshot_digest\":\"a198f81d187708579e1debb7a353bca02dd2f8f53c723002b2cc872eef254856\",\"source_references_digest\":\"45baf17369b5d44293adc6c045c45c8315862346e8ae86d4efe2aa593771210a\",\"prior_candidate_digest\":\"b9775811c16eb082e73bf76fa4eb3646d7eaa7019a19a94a652e0f30235a6e98\",\"prior_state_digest\":\"ecb497582c2690cc98e29a8128197a9a8da758d6249454dc754769133aa215c7\",\"effective_state_digest\":\"dea92cb7c070164a07f11afa1730e01f56a6bcde46102ed1285c3f155af781c3\",\"author_subject_digests\":[\"b673638fbcf3408727574d4a880a49360fab63056a3af9164463d414c31a3993\"],\"edit_count\":1}},\"candidate_digest\":\"b0ea11aeecd83b48f902ccf722b5af3be100897f58e92e049cc63e315703dafb\",\"provenance\":{\"schema_version\":\"recognition-provenance/v1\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"configuration_digest\":\"9ea9537bb8cbb7b3e5d21c566acf10a80bebc1ee2a4aad53f3e2618f586d50c8\",\"terminology_version\":\"religious-terminology-v1\",\"status_version\":\"religious-status-v1\",\"replacement_version\":\"enclosing-spans-failclosed-v1\",\"normalization_version\":\"source-codepoint-v1\"},\"blocking_reason\":null,\"anonymous\":false,\"automated_release_eligible\":false,\"release_review_required\":true}" + }, + "v1-1": { + "candidate_json": "{\"schema_version\":\"ocr-anonymization-candidate/v1\",\"candidate_id\":\"9add00157e1a4860b0bcf7529c54e56c\",\"sensitivity\":\"D2\",\"pages\":[{\"page_index\":1,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"},{\"page_index\":2,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"}],\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"anonymous\":false,\"release_review_required\":true,\"release_approved\":false,\"automated_release_eligible\":false,\"review_reasons\":[\"recognition_incomplete_possible\"]}", + "candidate_digest": "33d3cd62eb3fc3815da10391b314faeff5f8d83cd02ce58d52cba8facab35cc8", + "material_json": "{\"binding\":{\"review_ref\":\"original-review\",\"environment_id\":\"synthetic-room\",\"sensitivity\":\"D2\",\"expires_at\":1792359689,\"revoked\":false},\"evidence_digest\":\"9fba256420a0134b41f46353e97270eb2723dcf6de96d6e0fcbaf3da9a456dff\",\"sources\":[{\"evidence_ref\":\"67dc23b648fc0daa76d5f2b5ada77714b2a2c0906724efc1594eade1e147d809\",\"page_index\":1,\"sensitivity\":\"D2\",\"sha256\":\"6e8b79fa149157fbfa19c51a3386078de28cdc93a62ec910b8fe73b476cf145d\",\"characters\":23},{\"evidence_ref\":\"2dfa9253458ec4286bd64ce7e7f3da669169dcb021f39e701b1afdb06c83a0bc\",\"page_index\":2,\"sensitivity\":\"D2\",\"sha256\":\"03887c7a53fc37b09011d6baf70fd94f862a9e409185c0cb8007076cd91771c9\",\"characters\":23}],\"candidate\":{\"schema_version\":\"ocr-anonymization-candidate/v1\",\"candidate_id\":\"9add00157e1a4860b0bcf7529c54e56c\",\"sensitivity\":\"D2\",\"pages\":[{\"page_index\":1,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"},{\"page_index\":2,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"}],\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"anonymous\":false,\"release_review_required\":true,\"release_approved\":false,\"automated_release_eligible\":false,\"review_reasons\":[\"recognition_incomplete_possible\"]},\"candidate_digest\":\"33d3cd62eb3fc3815da10391b314faeff5f8d83cd02ce58d52cba8facab35cc8\",\"provenance\":null,\"blocking_reason\":null,\"anonymous\":false,\"automated_release_eligible\":false,\"release_review_required\":true}" + }, + "v2-1": { + "candidate_json": "{\"schema_version\":\"ocr-anonymization-candidate/v2\",\"candidate_id\":\"9add00157e1a4860b0bcf7529c54e56c\",\"sensitivity\":\"D2\",\"pages\":[{\"page_index\":1,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"},{\"page_index\":2,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"}],\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"anonymous\":false,\"release_review_required\":true,\"release_approved\":false,\"automated_release_eligible\":false,\"review_reasons\":[\"recognition_incomplete_possible\"],\"provenance\":{\"schema_version\":\"recognition-provenance/v1\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"configuration_digest\":\"9ea9537bb8cbb7b3e5d21c566acf10a80bebc1ee2a4aad53f3e2618f586d50c8\",\"terminology_version\":\"religious-terminology-v1\",\"status_version\":\"religious-status-v1\",\"replacement_version\":\"enclosing-spans-failclosed-v1\",\"normalization_version\":\"source-codepoint-v1\"}}", + "candidate_digest": "d56dd423fa430c8ca7623b69855cbe2f9d0307c565e35eb3c669991597dc049d", + "material_json": "{\"binding\":{\"review_ref\":\"original-review\",\"environment_id\":\"synthetic-room\",\"sensitivity\":\"D2\",\"expires_at\":1792359689,\"revoked\":false},\"evidence_digest\":\"9fba256420a0134b41f46353e97270eb2723dcf6de96d6e0fcbaf3da9a456dff\",\"sources\":[{\"evidence_ref\":\"67dc23b648fc0daa76d5f2b5ada77714b2a2c0906724efc1594eade1e147d809\",\"page_index\":1,\"sensitivity\":\"D2\",\"sha256\":\"6e8b79fa149157fbfa19c51a3386078de28cdc93a62ec910b8fe73b476cf145d\",\"characters\":23},{\"evidence_ref\":\"2dfa9253458ec4286bd64ce7e7f3da669169dcb021f39e701b1afdb06c83a0bc\",\"page_index\":2,\"sensitivity\":\"D2\",\"sha256\":\"03887c7a53fc37b09011d6baf70fd94f862a9e409185c0cb8007076cd91771c9\",\"characters\":23}],\"candidate\":{\"schema_version\":\"ocr-anonymization-candidate/v2\",\"candidate_id\":\"9add00157e1a4860b0bcf7529c54e56c\",\"sensitivity\":\"D2\",\"pages\":[{\"page_index\":1,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"},{\"page_index\":2,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"}],\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"anonymous\":false,\"release_review_required\":true,\"release_approved\":false,\"automated_release_eligible\":false,\"review_reasons\":[\"recognition_incomplete_possible\"],\"provenance\":{\"schema_version\":\"recognition-provenance/v1\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"configuration_digest\":\"9ea9537bb8cbb7b3e5d21c566acf10a80bebc1ee2a4aad53f3e2618f586d50c8\",\"terminology_version\":\"religious-terminology-v1\",\"status_version\":\"religious-status-v1\",\"replacement_version\":\"enclosing-spans-failclosed-v1\",\"normalization_version\":\"source-codepoint-v1\"}},\"candidate_digest\":\"d56dd423fa430c8ca7623b69855cbe2f9d0307c565e35eb3c669991597dc049d\",\"provenance\":{\"schema_version\":\"recognition-provenance/v1\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"configuration_digest\":\"9ea9537bb8cbb7b3e5d21c566acf10a80bebc1ee2a4aad53f3e2618f586d50c8\",\"terminology_version\":\"religious-terminology-v1\",\"status_version\":\"religious-status-v1\",\"replacement_version\":\"enclosing-spans-failclosed-v1\",\"normalization_version\":\"source-codepoint-v1\"},\"blocking_reason\":null,\"anonymous\":false,\"automated_release_eligible\":false,\"release_review_required\":true}" + }, + "v3-1": { + "candidate_json": "{\"schema_version\":\"ocr-anonymization-candidate/v3\",\"candidate_id\":\"21aee163100943dca8c081b412a88bf9\",\"sensitivity\":\"D2\",\"pages\":[{\"page_index\":1,\"anonymized_text\":\"[ORGANIZATION]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"ORGANIZATION\",\"placeholder\":\"[ORGANIZATION]\",\"score\":null,\"source\":\"human_correction\",\"model_revision\":\"reviewed-corrections-v1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"},{\"page_index\":2,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"}],\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"anonymous\":false,\"release_review_required\":true,\"release_approved\":false,\"automated_release_eligible\":false,\"review_reasons\":[\"recognition_incomplete_possible\",\"human_correction_requires_review\"],\"provenance\":{\"schema_version\":\"recognition-provenance/v1\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"configuration_digest\":\"9ea9537bb8cbb7b3e5d21c566acf10a80bebc1ee2a4aad53f3e2618f586d50c8\",\"terminology_version\":\"religious-terminology-v1\",\"status_version\":\"religious-status-v1\",\"replacement_version\":\"enclosing-spans-failclosed-v1\",\"normalization_version\":\"source-codepoint-v1\"},\"correction\":{\"schema_version\":\"ocr-correction-provenance/v1\",\"policy_version\":\"reviewed-corrections-v1\",\"correction_id\":\"d48f4de9be114047b8a82f1e6e8d4331\",\"proposal_digest\":\"4707bfc61d8ad090d71c5f386ad130ebc975e64dfd08c213ce4018ff11054047\",\"original_evidence_digest\":\"9fba256420a0134b41f46353e97270eb2723dcf6de96d6e0fcbaf3da9a456dff\",\"machine_snapshot_digest\":\"13230ebc918eeecad7fb07ea6ef71b4d98212336b15b38e647f41ee4e27c54a5\",\"source_references_digest\":\"1e61c8d9922d3289892dee5c07be4bb80775c02cb878b75e26c48669d21c4088\",\"prior_candidate_digest\":\"d56dd423fa430c8ca7623b69855cbe2f9d0307c565e35eb3c669991597dc049d\",\"prior_state_digest\":\"4f11826b9debdc282e29cae9b09977f367af04aa6b5e37c0be219494b868447c\",\"effective_state_digest\":\"12f2b1b1c3bada50d0f302d39af12b7035719d10b0b1d17c84bfc2d3b3493f99\",\"author_subject_digests\":[\"b673638fbcf3408727574d4a880a49360fab63056a3af9164463d414c31a3993\"],\"edit_count\":1}}", + "candidate_digest": "d4ab8c6c42d76253b030fc1ee5b77f5bc0f54bc4263bb081af744e4e3f662e20", + "material_json": "{\"binding\":{\"review_ref\":\"correction-01a57e670261433e95b493209065082e\",\"environment_id\":\"synthetic-room\",\"sensitivity\":\"D2\",\"expires_at\":1792359689,\"revoked\":false},\"evidence_digest\":\"9fba256420a0134b41f46353e97270eb2723dcf6de96d6e0fcbaf3da9a456dff\",\"sources\":[{\"evidence_ref\":\"67dc23b648fc0daa76d5f2b5ada77714b2a2c0906724efc1594eade1e147d809\",\"page_index\":1,\"sensitivity\":\"D2\",\"sha256\":\"6e8b79fa149157fbfa19c51a3386078de28cdc93a62ec910b8fe73b476cf145d\",\"characters\":23},{\"evidence_ref\":\"2dfa9253458ec4286bd64ce7e7f3da669169dcb021f39e701b1afdb06c83a0bc\",\"page_index\":2,\"sensitivity\":\"D2\",\"sha256\":\"03887c7a53fc37b09011d6baf70fd94f862a9e409185c0cb8007076cd91771c9\",\"characters\":23}],\"candidate\":{\"schema_version\":\"ocr-anonymization-candidate/v3\",\"candidate_id\":\"21aee163100943dca8c081b412a88bf9\",\"sensitivity\":\"D2\",\"pages\":[{\"page_index\":1,\"anonymized_text\":\"[ORGANIZATION]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"ORGANIZATION\",\"placeholder\":\"[ORGANIZATION]\",\"score\":null,\"source\":\"human_correction\",\"model_revision\":\"reviewed-corrections-v1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"},{\"page_index\":2,\"anonymized_text\":\"[PERSON]\",\"tags\":[{\"start\":0,\"end\":23,\"label\":\"PERSON\",\"placeholder\":\"[PERSON]\",\"score\":0.9,\"source\":\"multilingual_ner\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"normalization\":\"identity\"}],\"source\":\"cpu_ocr\",\"original_sensitivity\":\"D2\"}],\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"anonymous\":false,\"release_review_required\":true,\"release_approved\":false,\"automated_release_eligible\":false,\"review_reasons\":[\"recognition_incomplete_possible\",\"human_correction_requires_review\"],\"provenance\":{\"schema_version\":\"recognition-provenance/v1\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"configuration_digest\":\"9ea9537bb8cbb7b3e5d21c566acf10a80bebc1ee2a4aad53f3e2618f586d50c8\",\"terminology_version\":\"religious-terminology-v1\",\"status_version\":\"religious-status-v1\",\"replacement_version\":\"enclosing-spans-failclosed-v1\",\"normalization_version\":\"source-codepoint-v1\"},\"correction\":{\"schema_version\":\"ocr-correction-provenance/v1\",\"policy_version\":\"reviewed-corrections-v1\",\"correction_id\":\"d48f4de9be114047b8a82f1e6e8d4331\",\"proposal_digest\":\"4707bfc61d8ad090d71c5f386ad130ebc975e64dfd08c213ce4018ff11054047\",\"original_evidence_digest\":\"9fba256420a0134b41f46353e97270eb2723dcf6de96d6e0fcbaf3da9a456dff\",\"machine_snapshot_digest\":\"13230ebc918eeecad7fb07ea6ef71b4d98212336b15b38e647f41ee4e27c54a5\",\"source_references_digest\":\"1e61c8d9922d3289892dee5c07be4bb80775c02cb878b75e26c48669d21c4088\",\"prior_candidate_digest\":\"d56dd423fa430c8ca7623b69855cbe2f9d0307c565e35eb3c669991597dc049d\",\"prior_state_digest\":\"4f11826b9debdc282e29cae9b09977f367af04aa6b5e37c0be219494b868447c\",\"effective_state_digest\":\"12f2b1b1c3bada50d0f302d39af12b7035719d10b0b1d17c84bfc2d3b3493f99\",\"author_subject_digests\":[\"b673638fbcf3408727574d4a880a49360fab63056a3af9164463d414c31a3993\"],\"edit_count\":1}},\"candidate_digest\":\"d4ab8c6c42d76253b030fc1ee5b77f5bc0f54bc4263bb081af744e4e3f662e20\",\"provenance\":{\"schema_version\":\"recognition-provenance/v1\",\"model_revision\":\"ba967ee030b7e679dd5422cb16fcd1b379da38e1\",\"configuration_digest\":\"9ea9537bb8cbb7b3e5d21c566acf10a80bebc1ee2a4aad53f3e2618f586d50c8\",\"terminology_version\":\"religious-terminology-v1\",\"status_version\":\"religious-status-v1\",\"replacement_version\":\"enclosing-spans-failclosed-v1\",\"normalization_version\":\"source-codepoint-v1\"},\"blocking_reason\":null,\"anonymous\":false,\"automated_release_eligible\":false,\"release_review_required\":true}" + } + } +} diff --git a/ocr/tests/fixtures/isolated_host.py b/ocr/tests/fixtures/isolated_host.py new file mode 100644 index 0000000000000000000000000000000000000000..b9b1ed30c4364b29c24d29d3d57ffb9b3f0b4142 --- /dev/null +++ b/ocr/tests/fixtures/isolated_host.py @@ -0,0 +1,102 @@ +"""ISOLATED TEST ADAPTER ONLY. Not a production identity provider or startup default.""" + +import hashlib +import json +import time +from types import MappingProxyType + +from ocr.anonymization.host_adapter import ( + AuthenticatedIdentity, + CompletedReview, + ExportAuthorizationRecord, +) +from ocr.anonymization.release_policy import digest +from ocr.anonymization.schemas import Destination, VerifiedReview + + +class IsolatedHostAdapter: + production_ready = False + + def __init__(self, candidate): + self.credential = object() # Never accepted from a requested subject string. + self.identity = AuthenticatedIdentity( + subject="fixture-subject", + session_id="fixture-session", + expires_at=int(time.time()) + 600, + ) + self.policy = Destination( + destination_id="private-fixture", + destination_class="private", + allowed_sensitivities=("D0", "D1", "D2"), + ) + self.destinations = {self.policy.destination_id: self.policy} + value = digest(candidate) + attestation = VerifiedReview( + reviewer_subject=self.identity.subject, + review_id="fixture-review", + evidence_digest=hashlib.sha256(b"immutable synthetic completed review").hexdigest(), + candidate_digest=value, + sensitivity=candidate.sensitivity, + destination_id=self.policy.destination_id, + completed_at=int(time.time()), + ) + self.records = MappingProxyType( + { + attestation.review_id: CompletedReview( + attestation=attestation, + completed=True, + revoked=False, + expires_at=int(time.time()) + 600, + ) + } + ) + self.candidates = frozenset((value,)) + self.exports = {} + self.revoked = set() + self.allowed = { + ( + self.identity.subject, + "anonymization:release", + candidate.sensitivity, + self.policy.destination_id, + ) + } + + def legacy_release_allowed(self, candidate, review): + return digest(candidate) in self.candidates and digest(candidate) == review.candidate_digest + + def authenticate(self, credential): + if credential is not self.credential: + raise PermissionError("fixture_credentials_rejected") + return self.identity + + def load_review(self, review_id): + return self.records[review_id].model_copy(update={"revoked": review_id in self.revoked}) + + def candidate_exists(self, candidate_digest): + return candidate_digest in self.candidates + + def load_export(self, artifact_digest): + return self.exports[artifact_digest] + + def register_export(self, serialized_export): + """Privileged fixture store setup after actual reviewed export, never a public API.""" + payload = json.loads(serialized_export) + value = hashlib.sha256(serialized_export.encode()).hexdigest() + review = self.records["fixture-review"] + self.exports[value] = ExportAuthorizationRecord( + artifact_digest=value, + candidate_digest=payload["candidate_digest"], + review_id="fixture-review", + review_evidence_digest=review.attestation.evidence_digest, + sensitivity=payload["sensitivity"], + destination_id=payload["destination_id"], + expires_at=review.expires_at, + revoked=False, + ) + + def current_acl(self, identity, scope, sensitivity, destination_id): + return (identity.subject, scope, sensitivity, destination_id) in self.allowed + + def destination(self, destination_id): + return self.destinations[destination_id] diff --git a/ocr/tests/test_agent_api.py b/ocr/tests/test_agent_api.py new file mode 100644 index 0000000000000000000000000000000000000000..2556da8e535e94b557331bb0187bf994b24ec3c9 --- /dev/null +++ b/ocr/tests/test_agent_api.py @@ -0,0 +1,100 @@ +import os + +os.environ["OCR_CPU_AUTH_TOKEN"] = "test-token" + +import pytest +from fastapi.testclient import TestClient +from ocr.config import Settings +from ocr.worker_api import create_app + + +class MockBackend: + class Health: + ready = True + + async def health(self): + return self.Health() + + async def capabilities(self): + return {"models": []} + + async def recognize(self, p, opts): + return {} + + +@pytest.fixture(scope="module") +def client(): + settings = Settings() + settings.worker_token = "test-token" + app = create_app(settings) + with TestClient(app) as c: + app.state.backend = MockBackend() + yield c + + +def test_health_public(client): + response = client.get("/health") + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +def test_ready_unauthorized(client): + response = client.get("/ready") + assert response.status_code == 401 + + +def test_v1_models_authorized(client): + response = client.get("/v1/models", headers={"Authorization": "Bearer test-token"}) + assert response.status_code == 200 + assert response.json()["data"][0]["id"] == "ocr-cpu" + + +def test_v1_chat_completions_invalid_model(client): + req = { + "model": "gpt-4", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}, + } + ], + } + ], + } + response = client.post( + "/v1/chat/completions", headers={"Authorization": "Bearer test-token"}, json=req + ) + assert response.status_code == 422 + assert response.json()["code"] == "invalid_request" + + +def test_v1_chat_completions_invalid_url(client): + req = { + "model": "ocr-cpu", + "messages": [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": "https://example.com/image.png"}} + ], + } + ], + } + response = client.post( + "/v1/chat/completions", headers={"Authorization": "Bearer test-token"}, json=req + ) + assert response.status_code == 422 + assert response.json()["code"] == "only_image_data_urls_supported" + + +def test_v1_ocr_page_wrong_sensitivity(client): + files = {"file": ("test.png", b"fake-image-data", "image/png")} + data = {"sensitivity": "D1"} + response = client.post( + "/v1/ocr/page", headers={"Authorization": "Bearer test-token"}, files=files, data=data + ) + assert response.status_code == 422 + assert response.json()["code"] == "invalid_worker_options" diff --git a/ocr/tests/test_alias.py b/ocr/tests/test_alias.py new file mode 100644 index 0000000000000000000000000000000000000000..a9e566dd2fe1f6c111db275dd1a1f02951a9f5c2 --- /dev/null +++ b/ocr/tests/test_alias.py @@ -0,0 +1,147 @@ +import json + +import httpx +import pytest +from ocr.adapters.freellm_alias import FreeLLMAlias +from ocr.normalize import grounding, polygon +from ocr.schemas import OCROptions, RasterPage, SafeError + + +@pytest.mark.parametrize( + "response,code", + [ + ( + {"choices": [{"finish_reason": "stop", "message": {"content": ""}}]}, + "alias_empty", + ), + ( + {"choices": [{"finish_reason": "length", "message": {"content": "some"}}]}, + "alias_truncated", + ), + ( + {"choices": [{"finish_reason": "stop", "message": {"content": None}}]}, + "alias_empty", + ), + ( + { + "choices": [ + { + "finish_reason": "stop", + "message": {"content": [{"type": "text", "text": "some"}]}, + } + ] + }, + "alias_empty", + ), + ( + { + "choices": [ + { + "finish_reason": "stop", + "message": {"content": "text"}, + } + ] + }, + "alias_reasoning_output", + ), + ], +) +async def test_invalid_reply(png, response, code): + def handle(r): + data = json.loads(r.content) + assert data["model"] == "alias-ocr" and data["stream"] is False + assert data["messages"][0]["content"][0]["image_url"]["url"].startswith( + "data:image/png;base64," + ) + assert "enable_thinking" not in data + return httpx.Response(200, json=response) + + a = FreeLLMAlias("http://127.0.0.1:1/v1", "fake", transport=httpx.MockTransport(handle)) + a.qualified = True + a.qualified_at = __import__("time").monotonic() + try: + with pytest.raises(SafeError, match=code): + await a.recognize(RasterPage(0, png, 200, 120, "id"), OCROptions()) + finally: + await a.close() + + +@pytest.mark.parametrize("status", [429, 502, 503]) +async def test_retryable(status, png): + a = FreeLLMAlias( + "http://127.0.0.1:1/v1", + "fake", + transport=httpx.MockTransport(lambda r: httpx.Response(status)), + ) + try: + with pytest.raises(SafeError) as exc: + await a._completion(png) + assert exc.value.retryable + finally: + await a.close() + + +def test_grounding_and_boxes(): + assert grounding("![table](0,10,500,1000)")[0].confidence is None + with pytest.raises(SafeError): + grounding("![text](-1,0,2,3)") + with pytest.raises(SafeError): + grounding("![text](nan,0,2,3)") + p, b = polygon([(0, 0), (200, 0), (200, 100), (0, 100)], 200, 100) + assert b == [0, 0, 1000, 1000] + + +def test_untrusted_geometry_rejected(): + from ocr.schemas import Block + from pydantic import ValidationError + + for box in ([1, 2, 3], [-1, 2, 3, 4], [5, 5, 2, 2], [0, 0, float("nan"), 3]): + with pytest.raises(ValidationError): + Block(bbox_1000=box) + assert Block(text="plain Markdown").bbox_1000 is None + + +@pytest.mark.parametrize("valid", [True, False]) +async def test_probe_does_not_qualify_destination(monkeypatch, valid): + monkeypatch.setattr("secrets.token_hex", lambda n: "abcd99") + + def handle(request): + body = json.loads(request.content) + assert "ABCD99" not in body["messages"][0]["content"][1]["text"] + return httpx.Response( + 200, + json={ + "model": "lightonai/LightOnOCR-3-4B", + "provider": "helmholtz", + "choices": [ + { + "finish_reason": "stop", + "message": {"content": "OCR ABCD99" if valid else "empty white image"}, + } + ], + }, + ) + + a = FreeLLMAlias("http://127.0.0.1:1/v1", "fake", transport=httpx.MockTransport(handle)) + try: + if valid: + report = await a.probe() + assert report["multimodal_verified"] and report["observed_provider"] == "helmholtz" + assert not report["d2_qualified"] and not report["effective_destination_verified"] + assert not (await a.capabilities()).grounding_verified + from ocr.config import Settings + from ocr.security import authorize + + with pytest.raises(SafeError, match="no_authorized_destination"): + authorize( + a, + OCROptions(sensitivity="D2"), + Settings(gpu_enabled=True), + await a.capabilities(), + ) + else: + with pytest.raises(SafeError, match="alias_multimodal_probe_failed"): + await a.probe() + assert not a.qualified + finally: + await a.close() diff --git a/ocr/tests/test_anonymization_engine.py b/ocr/tests/test_anonymization_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..8f53724f4509091dab62a5d048d3a4bfaa839219 --- /dev/null +++ b/ocr/tests/test_anonymization_engine.py @@ -0,0 +1,134 @@ +"""Replacement correctness is independent of model recognition accuracy.""" + +import pytest +from ocr.anonymization.detectors import detect +from ocr.anonymization.model import MODEL_REVISION +from ocr.anonymization.replacement import replace +from ocr.anonymization.schemas import Entity +from ocr.schemas import SafeError + + +def entity(text, label, start, end): + return Entity( + start=start, + end=end, + label=label, + text=text[start:end], + source="multilingual_ner", + model_revision=MODEL_REVISION, + score=0.8, + ) + + +def test_every_qualification_gold_span_replaces_exactly(): + # Preserve the merged historical corpus byte-for-byte, and use its gold spans + # to measure replacement alone. This is NOT a recognition-accuracy test. + from ocr.tests.test_anonymization_qualification import SYNTHETIC_CASES + + for case in SYNTHETIC_CASES: + predictions = [ + entity(case["text"], s["label"], s["start"], s["end"]) for s in case["spans"] + ] + output, tags = replace(case["text"], predictions) + # Historical Google Inc. fixture includes the terminal period in its + # gold span but also expects a period outside replacement. Preserve the + # fixture; source spans take precedence over that inconsistent string. + expected = case["expected"] + if case["text"] == "I work for Google Inc.": + expected = "I work for [ORGANIZATION]" + assert output == expected + assert len(tags) == len(predictions) + assert all("text" not in t.model_dump() for t in tags) + + +def test_repeated_umlaut_hyphenated_broken_words_preserve_other_spacing(): + text = "Änne-Marie grüßt\nÄnne-Marie.\n\nBer-\nlin!" + spans = [ + entity(text, "PERSON", 0, 10), + entity(text, "PERSON", 18, 28), + entity(text, "CITY", 31, 39), + ] + output, _ = replace(text, spans) + assert output == "[PERSON] grüßt\n[PERSON].\n\n[CITY]!" + + +def test_containment_and_duplicate_policy(): + text = "Mary Jane Doe" + spans = [ + entity(text, "PERSON", 0, len(text)), + entity(text, "PERSON", 5, len(text)), + entity(text, "PERSON", 0, len(text)), + ] + output, tags = replace(text, spans) + assert output == "[PERSON]" and len(tags) == 1 + + +def test_crossing_or_conflicting_overlap_blocks_whole_transformation(): + text = "Mary Jane Doe" + with pytest.raises(SafeError, match="ambiguous_entity_overlap"): + replace(text, [entity(text, "PERSON", 0, 9), entity(text, "PERSON", 5, len(text))]) + with pytest.raises(SafeError, match="ambiguous_entity_overlap"): + replace(text, [entity(text, "PERSON", 0, 4), entity(text, "CITY", 0, 4)]) + + +@pytest.mark.parametrize( + "text,expected", + [ + ("Mail: föö@example.de.\nCall +49 30 12345678!", "Mail: [EMAIL].\nCall [PHONE]!"), + ("test@example.com and 555-0199.", "[EMAIL] and [PHONE]."), + ("12345 is a number.", "12345 is a number."), + ], +) +def test_contact_patterns(text, expected): + output, tags = replace(text, detect(text)) + assert output == expected + assert all(t.score is None for t in tags) + + +def test_dehyphenation_preserves_original_character_mapping(): + from ocr.anonymization.normalization import dehyphenate + + text = "Än-\nne in Ber-\r\nlin, and Anne-Marie." + normalized, positions = dehyphenate(text) + assert normalized == "Änne in Berlin, and Anne-Marie." + start = normalized.index("Berlin") + end = start + len("Berlin") + assert text[positions[start] : positions[end - 1] + 1] == "Ber-\r\nlin" + assert all(normalized[i] == text[p] for i, p in enumerate(positions)) + + +def test_recognizer_maps_normalized_predictions_without_inventing_offsets(): + from ocr.anonymization.recognizer import GLiNERRecognizer + + model = GLiNERRecognizer.__new__(GLiNERRecognizer) + + def view(text): + if text != "Berlin": + return [] + return [entity(text, "CITY", 0, 6)] + + model._recognize_view = view + found = model.recognize("Ber-\nlin") + assert len(found) == 1 and found[0].text == "Ber-\nlin" + assert found[0].normalization == "dehyphenation-v1" + assert replace("Ber-\nlin", found)[0] == "[CITY]" + + +def test_entity_level_evaluator_separates_labels_coverage_and_partial_masking(): + from ocr.anonymization.evaluate import score_cases + + class Recognition: + def recognize(self, text): + return [entity(text, "ORGANIZATION", 0, 3)] + + case = dict( + text="ABC DEF", + expected="[PERSON]", + spans=[dict(start=0, end=7, label="PERSON", text="ABC DEF")], + ) + report = score_cases(Recognition(), [case]) + assert report["combined_entity_metrics"]["PERSON"]["fn"] == 1 + assert report["combined_entity_metrics"]["ORGANIZATION"]["fp"] == 1 + assert report["unredacted_canaries"]["PERSON"] == 1 + assert report["literal_unredacted_canaries"]["PERSON"] == 0 + assert report["qualification_passed"] is False diff --git a/ocr/tests/test_anonymization_integration.py b/ocr/tests/test_anonymization_integration.py new file mode 100644 index 0000000000000000000000000000000000000000..a3c9f428418c55ce4d48b8410ba0748c1cf11150 --- /dev/null +++ b/ocr/tests/test_anonymization_integration.py @@ -0,0 +1,151 @@ +"""Actual pinned recognition is separate from deterministic boundary tests.""" + +import json +import os +from pathlib import Path + +import pytest +from ocr.anonymization import AnonymizationEngine +from ocr.anonymization.detectors import detect_contacts +from ocr.anonymization.evaluate import score_cases +from ocr.anonymization.recognizer import GLiNERRecognizer +from ocr.handoff import anonymization_handoff +from ocr.tests.test_anonymization_security import evidence + + +@pytest.mark.skipif( + os.getenv("RUN_ANONYMIZATION_MODEL") != "1", reason="Explicit opt-in pinned local model" +) +def test_real_pinned_recognition_and_ocr_handoff(): + cache = os.environ["ANONYMIZATION_MODEL_CACHE"] + model = GLiNERRecognizer(cache) + root = Path(__file__).parents[1] / "anonymization" / "fixtures" + validation = json.loads((root / "validation.json").read_text()) + text = validation[0]["text"] + result = anonymization_handoff(evidence("D2", (text, text)), AnonymizationEngine(model)) + assert len(result.pages) == 2 and result.sensitivity == "D2" + assert "[PERSON]" in result.pages[0].anonymized_text + assert not result.release_approved and result.release_review_required + + +@pytest.mark.skipif( + os.getenv("RUN_ANONYMIZATION_MODEL") != "1", reason="Explicit opt-in pinned local model" +) +def test_real_synthetic_canary_release_qualification(): + model = GLiNERRecognizer(os.environ["ANONYMIZATION_MODEL_CACHE"]) + root = Path(__file__).parents[1] / "anonymization" / "fixtures" + reports = [ + score_cases(model, json.loads((root / f"{name}.json").read_text())) + for name in ("qualification", "validation") + ] + # Quality qualification is deliberately stricter than smoke recognition. + # A nonzero canary count is a failing gate, not an xfail or an assumed pass. + assert all(r["qualification_passed"] for r in reports), ( + "Pinned recognizer has leakage; automated release blocked" + ) + + +@pytest.mark.skipif( + os.getenv("RUN_GATE03C_HOLDOUT") != "1", reason="Opt-in historical Gate03c control" +) +def test_gate03c_frozen_holdout_canary_acceptance(): + import hashlib + + root = Path(__file__).parents[1] / "anonymization/fixtures/holdout03c" + frozen = json.loads((root / "freeze.json").read_text()) + data = (root / "cases.json").read_bytes() + assert hashlib.sha256(data).hexdigest() == frozen["sha256"] + model = GLiNERRecognizer(os.environ["ANONYMIZATION_MODEL_CACHE"]) + assert model.prompts == frozen["control_prompts"] and model.threshold == frozen["threshold"] + report = score_cases(model, json.loads(data), detector=detect_contacts) + assert report["qualification_passed"], ( + "Historical Gate03c control FAILED; original first-result evidence remains unchanged" + ) + + +@pytest.mark.skipif( + os.getenv("RUN_GATE03D_MODEL") != "1", reason="Opt-in selected hybrid development regression" +) +def test_gate03d_selected_hybrid_development_regression(): + model = GLiNERRecognizer(os.environ["ANONYMIZATION_MODEL_CACHE"]) + root = Path(__file__).parents[1] / "anonymization/fixtures" + for relative in ("qualification.json", "validation.json", "holdout03c/cases.json"): + assert score_cases(model, json.loads((root / relative).read_bytes()))[ + "qualification_passed" + ] + + +@pytest.mark.skipif( + os.getenv("RUN_GATE03D_HOLDOUT") != "1", + reason="Opt-in historical Gate03d first-result acceptance", +) +def test_gate03d_recorded_first_holdout_acceptance(): + # The standalone first-result command performs inference once. This test + # checks that actual recorded gate without rerunning or overwriting it. + from ocr.anonymization.historical import verify_historical + + _, freeze, cases, report = verify_historical() + assert report["corpus_sha256"] == freeze["sha256"] + assert report["model_revision"] == freeze["model_revision"] + assert report["implementation_sha256"] == freeze["implementation_sha256"] + assert {p["case_id"] for p in report["per_case"]} == {c["id"] for c in cases} + assert report["evaluation"]["qualification_passed"], ( + "Historical Gate03d first result FAILED; not a benchmark of the updated candidate" + ) + + +@pytest.mark.skipif( + os.getenv("RUN_GATE03E_MODEL") != "1", + reason="Opt-in current development candidate coverage gate", +) +def test_gate03e_recorded_current_candidate_coverage_gate(): + report = json.loads(Path(os.environ["GATE03E_DEVELOPMENT_OUTPUT"]).read_bytes()) + assert report["independent_qualification"] is False + selected = report["experiments"]["status_scoped_ocr"] + # Measured returned-candidate coverage remains a failing development gate + # if any partial or full canary survives. Blocking is reported separately. + assert all( + sum(s["candidate_partial_canaries"].values()) == 0 + and sum(s["candidate_fully_unmasked_canaries"].values()) == 0 + for s in selected.values() + ), "Current Gate03e returned candidate still has a canary; model qualification remains NO-GO" + + +@pytest.mark.skipif( + os.getenv("RUN_GATE03F_MODEL") != "1", + reason="Opt-in recorded pinned-model synthetic correction demonstration", +) +def test_gate03f_recorded_private_correction_utility(): + import hashlib + + report = json.loads(Path(os.environ["GATE03F_CORRECTION_OUTPUT"]).read_bytes()) + source = Path(__file__).parents[1] / "anonymization/fixtures/holdout03d/cases.json" + assert report["corpus_sha256"] == hashlib.sha256(source.read_bytes()).hexdigest() + assert not report["independent_qualification"] and not report["model_qualification_passed"] + assert not report["corrected_outputs_count_as_model_true_positives"] + assert not report["automated_release_eligible"] + assert report["actual_inference_calls"] == len(report["cases"]) == 9 + corrected = [c for c in report["cases"] if c["new_candidate_produced"]] + assert len(corrected) == 6 + assert sum(c["original_outcome"] == "blocked_conflicting_spans" for c in corrected) == 5 + assert sum(c["original_outcome"] == "candidate_partial_mask" for c in corrected) == 1 + assert all( + c["malformed_correction_rejected"] and c["unauthorized_correction_rejected"] + for c in corrected + ) + assert all( + c["source_canaries_covered_after"] + and c["release_blocked"] + and c["final_sensitivity"] == "D2" + and c["completed_release_reviews"] == 0 + for c in report["cases"] + ) + + assert all(c["document_exact_match_after"] for c in corrected) + controls = json.loads(Path(os.environ["GATE03F_CONTROLS_OUTPUT"]).read_bytes()) + assert controls["correct_controls_only"] and controls["actual_inference_calls"] == 2 + assert all( + c["document_exact_match_after"] and not c["new_candidate_produced"] and c["release_blocked"] + for c in controls["cases"] + ) + assert {c["case_id"] for c in controls["cases"]} == {"holdout03d-004", "holdout03d-005"} diff --git a/ocr/tests/test_anonymization_qualification.py b/ocr/tests/test_anonymization_qualification.py new file mode 100644 index 0000000000000000000000000000000000000000..f5be0baa1fe9e9b79702e34dcae5ad8a736a0470 --- /dev/null +++ b/ocr/tests/test_anonymization_qualification.py @@ -0,0 +1,482 @@ +import os +import time +from unittest.mock import patch + +import pytest +from ocr.handoff import anonymization_handoff +from ocr.schemas import Block, OCRPageResult, OCRResult, SafeError + +# 1. Synthetic dataset generation with span-aware evaluation +SYNTHETIC_CASES = [ + # German Personal Names + { + "text": "Lina Wolkenstein wohnt in Berlin.", + "expected": "[PERSON] wohnt in [CITY].", + "spans": [ + {"start": 0, "end": 16, "label": "PERSON", "text": "Lina Wolkenstein"}, + {"start": 26, "end": 32, "label": "CITY", "text": "Berlin"}, + ], + }, + { + "text": "Hans Müller kauft ein.", + "expected": "[PERSON] kauft ein.", + "spans": [{"start": 0, "end": 11, "label": "PERSON", "text": "Hans Müller"}], + }, + { + "text": "Das Auto von Dr. Klaus Schmidt.", + "expected": "Das Auto von Dr. [PERSON].", + "spans": [{"start": 17, "end": 30, "label": "PERSON", "text": "Klaus Schmidt"}], + }, + { + "text": "Friedrich Wilhelm geht nach Hause.", + "expected": "[PERSON] geht nach Hause.", + "spans": [{"start": 0, "end": 17, "label": "PERSON", "text": "Friedrich Wilhelm"}], + }, + { + "text": "Ich kenne Frau Anne-Marie Schneider nicht.", + "expected": "Ich kenne Frau [PERSON] nicht.", + "spans": [{"start": 15, "end": 35, "label": "PERSON", "text": "Anne-Marie Schneider"}], + }, + # English Personal Names + { + "text": "John Doe is a good man.", + "expected": "[PERSON] is a good man.", + "spans": [{"start": 0, "end": 8, "label": "PERSON", "text": "John Doe"}], + }, + { + "text": "Alice and Bob went to the store.", + "expected": "[PERSON] and [PERSON] went to the store.", + "spans": [ + {"start": 0, "end": 5, "label": "PERSON", "text": "Alice"}, + {"start": 10, "end": 13, "label": "PERSON", "text": "Bob"}, + ], + }, + { + "text": "Is Mary Johnson there?", + "expected": "Is [PERSON] there?", + "spans": [{"start": 3, "end": 15, "label": "PERSON", "text": "Mary Johnson"}], + }, + { + "text": "Meeting with Mr. Charles Montgomery.", + "expected": "Meeting with Mr. [PERSON].", + "spans": [{"start": 17, "end": 35, "label": "PERSON", "text": "Charles Montgomery"}], + }, + { + "text": "Elizabeth Taylor was famous.", + "expected": "[PERSON] was famous.", + "spans": [{"start": 0, "end": 16, "label": "PERSON", "text": "Elizabeth Taylor"}], + }, + # Cities and Locations + { + "text": "I live in New York.", + "expected": "I live in [CITY].", + "spans": [{"start": 10, "end": 18, "label": "CITY", "text": "New York"}], + }, + { + "text": "Der Zug nach München fährt ab.", + "expected": "Der Zug nach [CITY] fährt ab.", + "spans": [{"start": 13, "end": 20, "label": "CITY", "text": "München"}], + }, + { + "text": "Paris is beautiful in spring.", + "expected": "[CITY] is beautiful in spring.", + "spans": [{"start": 0, "end": 5, "label": "CITY", "text": "Paris"}], + }, + { + "text": "Flug von Frankfurt nach Tokyo.", + "expected": "Flug von [CITY] nach [CITY].", + "spans": [ + {"start": 9, "end": 18, "label": "CITY", "text": "Frankfurt"}, + {"start": 24, "end": 29, "label": "CITY", "text": "Tokyo"}, + ], + }, + { + "text": "Welcome to Los Angeles, California.", + "expected": "Welcome to [CITY], [CITY].", + "spans": [ + {"start": 11, "end": 22, "label": "CITY", "text": "Los Angeles"}, + {"start": 24, "end": 34, "label": "CITY", "text": "California"}, + ], + }, + # Religious Affiliations + { + "text": "Ihre Religionszugehörigkeit ist evangelisch.", + "expected": "Ihre Religionszugehörigkeit ist [RELIGION].", + "spans": [{"start": 32, "end": 43, "label": "RELIGION", "text": "evangelisch"}], + }, + { + "text": "Er ist katholisch.", + "expected": "Er ist [RELIGION].", + "spans": [{"start": 7, "end": 17, "label": "RELIGION", "text": "katholisch"}], + }, + { + "text": "They practice Judaism.", + "expected": "They practice [RELIGION].", + "spans": [{"start": 14, "end": 21, "label": "RELIGION", "text": "Judaism"}], + }, + { + "text": "He is a devout Muslim.", + "expected": "He is a devout [RELIGION].", + "spans": [{"start": 15, "end": 21, "label": "RELIGION", "text": "Muslim"}], + }, + { + "text": "Buddhist monks live there.", + "expected": "[RELIGION] monks live there.", + "spans": [{"start": 0, "end": 8, "label": "RELIGION", "text": "Buddhist"}], + }, + # Multiple entities per sentence + { + "text": "John Doe from New York met Lina Wolkenstein in Berlin.", + "expected": "[PERSON] from [CITY] met [PERSON] in [CITY].", + "spans": [ + {"start": 0, "end": 8, "label": "PERSON", "text": "John Doe"}, + {"start": 14, "end": 22, "label": "CITY", "text": "New York"}, + {"start": 27, "end": 43, "label": "PERSON", "text": "Lina Wolkenstein"}, + {"start": 47, "end": 53, "label": "CITY", "text": "Berlin"}, + ], + }, + { + "text": "David is Catholic and lives in Rome.", + "expected": "[PERSON] is [RELIGION] and lives in [CITY].", + "spans": [ + {"start": 0, "end": 5, "label": "PERSON", "text": "David"}, + {"start": 9, "end": 17, "label": "RELIGION", "text": "Catholic"}, + {"start": 31, "end": 35, "label": "CITY", "text": "Rome"}, + ], + }, + # Repeated entities + { + "text": "John likes John's new car.", + "expected": "[PERSON] likes [PERSON]'s new car.", + "spans": [ + {"start": 0, "end": 4, "label": "PERSON", "text": "John"}, + {"start": 11, "end": 15, "label": "PERSON", "text": "John"}, + ], + }, + { + "text": "Berlin, oh Berlin, my favorite city.", + "expected": "[CITY], oh [CITY], my favorite city.", + "spans": [ + {"start": 0, "end": 6, "label": "CITY", "text": "Berlin"}, + {"start": 11, "end": 17, "label": "CITY", "text": "Berlin"}, + ], + }, + # Ambiguous names and place names + { + "text": "Washington is the capital.", + "expected": "[CITY] is the capital.", + "spans": [{"start": 0, "end": 10, "label": "CITY", "text": "Washington"}], + }, + { + "text": "George Washington was the president.", + "expected": "[PERSON] was the president.", + "spans": [{"start": 0, "end": 17, "label": "PERSON", "text": "George Washington"}], + }, + { + "text": "I visited the Jordan river.", + "expected": "I visited the [CITY] river.", + "spans": [{"start": 14, "end": 20, "label": "CITY", "text": "Jordan"}], + }, + { + "text": "Michael Jordan is a basketball player.", + "expected": "[PERSON] is a basketball player.", + "spans": [{"start": 0, "end": 14, "label": "PERSON", "text": "Michael Jordan"}], + }, + # Names split over lines (Simulating OCR artifacts) + { + "text": "Mr. Smith-\nson", + "expected": "Mr. [PERSON]", + "spans": [{"start": 4, "end": 14, "label": "PERSON", "text": "Smith-\nson"}], + }, + { + "text": "Ber-\nlin", + "expected": "[CITY]", + "spans": [{"start": 0, "end": 8, "label": "CITY", "text": "Ber-\nlin"}], + }, + # OCR spelling errors + { + "text": "Jchn D0e went home.", + "expected": "[PERSON] went home.", + "spans": [{"start": 0, "end": 8, "label": "PERSON", "text": "Jchn D0e"}], + }, + { + "text": "Welcome to Nevv Y0rk.", + "expected": "Welcome to [CITY].", + "spans": [{"start": 11, "end": 20, "label": "CITY", "text": "Nevv Y0rk"}], + }, + # Documents without sensitive entities + {"text": "The cat sat on the mat.", "expected": "The cat sat on the mat.", "spans": []}, + {"text": "12345 is a number.", "expected": "12345 is a number.", "spans": []}, + {"text": "This is a simple test.", "expected": "This is a simple test.", "spans": []}, + # Email addresses, phone numbers, organizations (Unsupported) + { + "text": "My email is test@example.com.", + "expected": "My email is [EMAIL].", + "spans": [{"start": 12, "end": 28, "label": "EMAIL", "text": "test@example.com"}], + }, + { + "text": "Call me at 555-0199.", + "expected": "Call me at [PHONE].", + "spans": [{"start": 11, "end": 19, "label": "PHONE", "text": "555-0199"}], + }, + { + "text": "I work for Google Inc.", + "expected": "I work for [ORGANIZATION].", + "spans": [{"start": 11, "end": 22, "label": "ORGANIZATION", "text": "Google Inc."}], + }, + { + "text": "Contact info@company.de", + "expected": "Contact [EMAIL]", + "spans": [{"start": 8, "end": 23, "label": "EMAIL", "text": "info@company.de"}], + }, + { + "text": "The UN is an organization.", + "expected": "The [ORGANIZATION] is an organization.", + "spans": [{"start": 4, "end": 6, "label": "ORGANIZATION", "text": "UN"}], + }, +] + + +def tagger_wrapper(text): + pytest.importorskip("nltk") + from augmentoolkit.utils.anonymization import anonymize_text + + result_text = anonymize_text(text) + return {"text": result_text, "tags": []} + + +class TestAnonymizationQualification: + def test_synthetic_fixtures_spans_are_valid(self): + """ + Verify that all 40 gold-standard synthetic spans correctly match their offsets. + This test always runs in CI and does not depend on NLTK/Presidio. + """ + for case in SYNTHETIC_CASES: + original = case["text"] + for span in case["spans"]: + assert original[span["start"] : span["end"]] == span["text"] + + def evaluate_metrics(self): + """ + Calculates unredacted canary count and document-level exact match rates. + We explicitly mark entity-level Precision, Recall, and F1 as "UNAVAILABLE" + until the recognizer provides usable span evidence. + """ + pytest.importorskip("nltk") + from augmentoolkit.utils.anonymization import anonymize_text + + entity_types = ["PERSON", "CITY", "RELIGION", "EMAIL", "PHONE", "ORGANIZATION"] + + unredacted_canaries = {e: 0 for e in entity_types} + total_time = 0 + exact_match_count = 0 + + for case in SYNTHETIC_CASES: + original = case["text"] + expected = case["expected"] + spans = case["spans"] + + start_time = time.time() + actual = anonymize_text(original) + total_time += time.time() - start_time + + if actual == expected: + exact_match_count += 1 + + # Track false negatives and leaking canaries. + for span in spans: + label = span["label"] + text = span["text"] + # We check if the expected required word is still present in the output + if text.lower() in actual.lower(): + unredacted_canaries[label] += 1 + + results = {} + for entity in entity_types: + results[entity] = { + "precision": "UNAVAILABLE", + "recall": "UNAVAILABLE", + "f1": "UNAVAILABLE", + "unredacted_canaries": unredacted_canaries[entity], + } + + results["document_exact_match_rate"] = exact_match_count / len(SYNTHETIC_CASES) + results["avg_runtime_ms"] = (total_time / len(SYNTHETIC_CASES)) * 1000 + return results + + def test_evaluate_synthetic_dataset(self): + """ + Evaluate the synthetic dataset and print metrics for the qualification report. + """ + results = self.evaluate_metrics() + print("\n--- Evaluation Metrics ---") + for k, v in results.items(): + print(f"{k}: {v}") + print("--------------------------\n") + + assert results["PERSON"]["unredacted_canaries"] > 0 + + # === Security and Fail-Closed Testing === + # BASELINE CHARACTERIZATION + def test_baseline_nltk_fail_open_security_flaw(self): + """ + BASELINE DEFECT: If NLTK fails, it currently returns the original text (fail-open flaw). + """ + pytest.importorskip("nltk") + from augmentoolkit.utils.anonymization import anonymize_text + + with patch( + "augmentoolkit.utils.anonymization.word_tokenize", + side_effect=Exception("Simulated failure"), + ): + original_text = "Lina Wolkenstein wohnt in Berlin." + anonymized = anonymize_text(original_text) + + # This assert demonstrates the flaw: original text is returned. + assert anonymized == original_text + assert "Wolkenstein" in anonymized + + # ACCEPTANCE TESTS + def test_acceptance_fail_closed_handoff(self): + """ + ACCEPTANCE TEST: The handoff wrapper must raise SafeError if the tagger + errors out, preventing silent release of unredacted text. + """ + page = OCRPageResult( + page_index=1, + page_id="p1", + text="Secret Data", + markdown="Secret Data", + image_width=800, + image_height=100, + blocks=[], + sensitivity="D2", + source="cpu_ocr", + engine="mocked", + provider_destination="local", + ) + result = OCRResult( + schema_version="1", + job_id="j1", + request_id="r1", + file_id="f1", + page_count=1, + markdown="Secret Data", + text="Secret Data", + failures=[], + status="completed", + sensitivity="D2", + pages=[page], + anonymization_required=True, + ) + + def bad_tagger(text): + raise Exception("Tagger completely failed") + + with pytest.raises(SafeError, match="anonymization_failed"): + anonymization_handoff(result, bad_tagger) + + def test_acceptance_malformed_outputs_blocked(self): + """ + ACCEPTANCE TEST: Empty or None strings passed to handoff must be safely handled + by raising a SafeError or ensuring it doesn't fail open. + """ + page = OCRPageResult( + page_index=1, + page_id="p1", + text="", + markdown="", + image_width=800, + image_height=100, + blocks=[], + sensitivity="D2", + source="cpu_ocr", + engine="mocked", + provider_destination="local", + ) + result = OCRResult( + schema_version="1", + job_id="j1", + request_id="r1", + file_id="f1", + page_count=1, + markdown="", + text="", + failures=[], + status="completed", + sensitivity="D2", + pages=[page], + anonymization_required=True, + ) + + def invalid_tagger(text): + return "This string doesn't match the dict format" + + with pytest.raises(SafeError, match="invalid_anonymization_result"): + anonymization_handoff(result, invalid_tagger) + + # === OCR-to-Anonymization Integration === + + @pytest.mark.skipif( + os.environ.get("RUN_OPTIONAL_OCR_INTEGRATION") != "1", reason="Opt-in OCR integration test" + ) + def test_ocr_to_anonymization_integration(self): + """ + Verify that a representative OCR result successfully passes through handoff. + This test uses a deterministic mock result to avoid relying on external ML OCR tools. + """ + pytest.importorskip("nltk") + ocr_text = "Lina Wolkenstein wohnt in Berlin. Ihre Religionszugehoerigkeit ist evangelisch." + page = OCRPageResult( + page_index=1, + page_id="p1", + text=ocr_text, + markdown=ocr_text, + image_width=800, + image_height=100, + blocks=[ + Block(type="text", text=ocr_text, bbox_1000=[0.0, 0.0, 10.0, 10.0], confidence=0.9) + ], + sensitivity="D2", + source="cpu_ocr", + engine="mocked", + provider_destination="local", + ) + result = OCRResult( + schema_version="1", + job_id="j1", + request_id="r1", + file_id="f1", + page_count=1, + markdown=ocr_text, + text=ocr_text, + failures=[], + status="completed", + sensitivity="D2", + pages=[page], + anonymization_required=True, + ) + + handoff_result = anonymization_handoff(result, tagger_wrapper) + + assert len(handoff_result["pages"]) == 1 + page_res = handoff_result["pages"][0] + + # Verify required handoff structure and states + assert page_res["release_review_required"] is True + assert handoff_result["anonymous"] is False + assert handoff_result["original_evidence_retained_by_caller"] is True + + # Verify actual text replacement output leakage independently (Baseline Defect / NO-GO) + # We assert that the legacy model leaks sensitive content, demonstrating it is unsafe for release. + anonymized_text = page_res["anonymized_text"].lower() + leaks = { + "Lina": "lina" in anonymized_text, + "Wolkenstein": "wolkenstein" in anonymized_text, + "Berlin": "berlin" in anonymized_text, + "evangelisch": "evangelisch" in anonymized_text, + } + print("Integration Test Leaks:", leaks) + + # We expect at least some required canaries to leak in the baseline. + assert any(leaks.values()), "Baseline should leak canaries in opt-in test" diff --git a/ocr/tests/test_anonymization_release_policy.py b/ocr/tests/test_anonymization_release_policy.py new file mode 100644 index 0000000000000000000000000000000000000000..c7e7eab27e58ed78daad8b441d2ad0d949ebd4b9 --- /dev/null +++ b/ocr/tests/test_anonymization_release_policy.py @@ -0,0 +1,352 @@ +"""Synthetic identity/review/ACL callbacks are test fixtures, never production defaults.""" + +import json +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from unittest.mock import Mock + +import pytest +from ocr.anonymization import ReleaseAuthority +from ocr.anonymization.approval_ledger import SQLiteApprovalLedger +from ocr.anonymization.release_policy import digest +from ocr.anonymization.schemas import Destination, VerifiedReview +from ocr.handoff import anonymization_handoff +from ocr.schemas import SafeError +from ocr.tests.test_anonymization_security import CANARY, candidate, evidence + +PRIVATE = Destination( + destination_id="private-archive", + destination_class="private", + allowed_sensitivities=("D0", "D1", "D2"), +) +OTHER = Destination( + destination_id="private-other", + destination_class="private", + allowed_sensitivities=("D0", "D1", "D2"), +) +PUBLIC = Destination( + destination_id="public-download", + destination_class="public", + allowed_sensitivities=("D0", "D1", "D2"), +) + + +@dataclass +class ApplicationFixture: + value: object + review: VerifiedReview + authority: ReleaseAuthority + ledger: SQLiteApprovalLedger + authorizer: object + verifier: object + + def approve(self, **changes): + return self.authority.approve( + self.value, + **( + dict( + actor="authenticated-session", + review_id="review-1", + destination=PRIVATE.destination_id, + ) + | changes + ), + ) + + def export(self, approval=None, **changes): + return self.authority.export( + self.value, + approval, + **(dict(actor="authenticated-session", destination=PRIVATE.destination_id) | changes), + ) + + +def application(tmp_path, *, level="D2"): + value = candidate(level) + review = VerifiedReview( + reviewer_subject="opaque-subject-1", + review_id="review-1", + evidence_digest="a" * 64, + candidate_digest=digest(value), + sensitivity=level, + destination_id=PRIVATE.destination_id, + completed_at=int(time.time()), + ) + + def verify(actor, review_id): + if actor != "authenticated-session" or review_id != "review-1": + raise ValueError(CANARY) + return review + + verifier = Mock(side_effect=verify) + authorizer = Mock( + side_effect=lambda r, scope, dest: ( + r.reviewer_subject == "opaque-subject-1" + and scope == "anonymization:release" + and dest.destination_id == PRIVATE.destination_id + ) + ) + ledger = SQLiteApprovalLedger(tmp_path / "approvals.sqlite", initialize=True) + authority = ReleaseAuthority( + b"x" * 32, + authorizer, + verify_review=verifier, + consume_approval=ledger.consume, + legacy_policy=lambda c, r: digest(c) == r.candidate_digest, + destinations=(PRIVATE, OTHER, PUBLIC), + ) + return ApplicationFixture(value, review, authority, ledger, authorizer, verifier) + + +@pytest.mark.parametrize("level", ["D0", "D1", "D2"]) +def test_reviewed_export_is_destination_bound_classified_and_single_use(tmp_path, level): + app = application(tmp_path, level=level) + assert app.value.automated_release_eligible is False + with pytest.raises(SafeError, match="release_not_approved"): + app.export() + approval = app.approve() + output = json.loads(app.export(approval)) + assert output["sensitivity"] == level and output["anonymous"] is False + assert ( + output["destination_id"] == PRIVATE.destination_id + and output["destination_class"] == "private" + ) + assert output["transmission_authorized"] is False + assert CANARY not in json.dumps(output) + assert "reviewer_subject" not in output and "evidence_digest" not in output + with pytest.raises(SafeError, match="release_approval_consumed_or_unavailable"): + app.export(approval) + + +@pytest.mark.parametrize( + "changed", + [ + dict(actor="forged-session"), + dict(review_id="missing-review"), + dict(destination="https://attacker.invalid"), + dict(destination=PUBLIC.destination_id), + dict(destination=OTHER.destination_id), + dict(format="markdown"), + dict(format="text"), + ], +) +def test_issue_requires_authenticated_review_and_approved_private_destination(tmp_path, changed): + app = application(tmp_path) + with pytest.raises(SafeError): + app.approve(**changed) + + +@pytest.mark.parametrize( + "changed", + [ + dict(actor="forged-session"), + dict(destination=OTHER.destination_id), + dict(destination=PUBLIC.destination_id), + dict(destination="arbitrary"), + dict(format="text"), + dict(format="markdown"), + ], +) +def test_export_cannot_change_identity_destination_or_format(tmp_path, changed): + app = application(tmp_path) + approval = app.approve() + with pytest.raises(SafeError): + app.export(approval, **changed) + # A rejected operation must not burn a still valid approval. + assert json.loads(app.export(approval))["release_approved"] is True + + +@pytest.mark.parametrize( + "field,value", + [ + ("reviewer_subject", "another-subject"), + ("candidate_digest", "b" * 64), + ("sensitivity", "D0"), + ("destination_id", OTHER.destination_id), + ("completed_at", 0), + ("completed_at", 9_999_999_999), + ("review_id", "wrong"), + ], +) +def test_incomplete_stale_or_mismatched_review_evidence_rejected(tmp_path, field, value): + app = application(tmp_path) + app.verifier.side_effect = None + app.verifier.return_value = app.review.model_copy(update={field: value}) + with pytest.raises(SafeError, match="release_not_approved"): + app.approve() + + +def test_boolean_review_claims_are_not_verified_evidence(tmp_path): + app = application(tmp_path) + app.verifier.side_effect = None + app.verifier.return_value = {"review_completed": True} + with pytest.raises(SafeError, match="release_not_approved"): + app.approve() + + +@pytest.mark.parametrize( + "field,value", + [ + ("signature", "0" * 64), + ("expires_at", 0), + ("issued_at", 9_999_999_999), + ("candidate_digest", "0" * 64), + ("approval_id", "1" * 32), + ("format", "text"), + ("destination", OTHER), + ], +) +def test_approval_tampering_or_expiration_rejected(tmp_path, field, value): + app = application(tmp_path) + approval = app.approve() + with pytest.raises(SafeError, match="release_not_approved"): + app.export(approval.model_copy(update={field: value})) + + +def test_expired_approval_and_acl_revocation_rechecked_at_export(tmp_path, monkeypatch): + app = application(tmp_path) + approval = app.approve() + app.authorizer.side_effect = None + app.authorizer.return_value = False + with pytest.raises(SafeError, match="release_not_approved"): + app.export(approval) + app.authorizer.return_value = True + monkeypatch.setattr("ocr.anonymization.release_policy.time.time", lambda: approval.expires_at) + with pytest.raises(SafeError, match="release_not_approved"): + app.export(approval) + + +def test_review_record_changes_invalidate_previous_approval(tmp_path): + app = application(tmp_path) + approval = app.approve() + app.verifier.side_effect = None + app.verifier.return_value = app.review.model_copy(update={"evidence_digest": "b" * 64}) + with pytest.raises(SafeError, match="release_not_approved"): + app.export(approval) + + +def test_candidate_changes_and_forged_copies_rejected(tmp_path): + app = application(tmp_path) + approval = app.approve() + with pytest.raises(SafeError, match="release_not_approved"): + app.authority.export( + candidate(), approval, actor="authenticated-session", destination=PRIVATE.destination_id + ) + with pytest.raises(SafeError, match="invalid_release_candidate"): + app.authority.approve( + app.value.model_copy(update={"sensitivity": "D0"}), + actor="authenticated-session", + review_id="review-1", + destination=PRIVATE.destination_id, + ) + + +def test_replay_ledger_persists_across_authority_instances_and_races(tmp_path): + app = application(tmp_path) + approval = app.approve() + restarted_ledger = SQLiteApprovalLedger(tmp_path / "approvals.sqlite") + restarted = ReleaseAuthority( + b"x" * 32, + app.authorizer, + verify_review=app.verifier, + consume_approval=restarted_ledger.consume, + legacy_policy=app.authority._legacy_policy, + destinations=(PRIVATE,), + ) + + def attempt(authority): + try: + authority.export( + app.value, + approval, + actor="authenticated-session", + destination=PRIVATE.destination_id, + ) + return True + except SafeError: + return False + + with ThreadPoolExecutor(max_workers=2) as pool: + assert sum(pool.map(attempt, [app.authority, restarted])) == 1 + assert not attempt(restarted) + assert (tmp_path / "approvals.sqlite").stat().st_mode & 0o777 == 0o600 + + +def test_ledger_unavailability_never_returns_export(tmp_path): + app = application(tmp_path) + approval = app.approve() + app.authority._consume = Mock(side_effect=RuntimeError(CANARY)) + with pytest.raises(SafeError, match="release_approval_consumed_or_unavailable") as exc: + app.export(approval) + assert CANARY not in str(exc.value) + + +def test_sensitive_public_destination_is_denied_even_by_permissive_application(tmp_path): + app = application(tmp_path) + app.authorizer.side_effect = None + app.authorizer.return_value = True # Deliberately malicious policy in this regression only. + with pytest.raises(SafeError, match="release_destination_not_approved"): + app.approve(destination=PUBLIC.destination_id) + + +def test_d0_text_requires_separately_scoped_approval(tmp_path): + app = application(tmp_path, level="D0") + approval = app.approve(format="text") + with pytest.raises(SafeError, match="release_not_approved"): + app.export(approval, format="json") + assert app.export(approval, format="text") == "[PERSON]" + + +def test_legacy_dictionary_cannot_receive_approval(tmp_path): + app = application(tmp_path) + legacy = anonymization_handoff(evidence(), lambda text: {"text": text, "tags": []}) + with pytest.raises(SafeError, match="invalid_release_candidate"): + app.authority.approve( + legacy, + actor="authenticated-session", + review_id="review-1", + destination=PRIVATE.destination_id, + ) + + +def test_authority_has_no_default_identity_review_or_replay_authorization(): + with pytest.raises(TypeError): + ReleaseAuthority(b"x" * 32, lambda *args: True) + + +def test_legacy_boolean_review_or_v1_approval_cannot_bypass_new_boundary(tmp_path): + app = application(tmp_path) + with pytest.raises(TypeError): + app.authority.approve(app.value, actor="authenticated-session", review_completed=True) + with pytest.raises(SafeError, match="release_not_approved"): + app.export( + { + "candidate_digest": digest(app.value), + "expires_at": int(time.time()) + 900, + "signature": "a" * 64, + } + ) + + +def test_replay_ledger_rejects_shared_permissions_or_symlink(tmp_path): + bad = tmp_path / "bad.sqlite" + bad.write_bytes(b"") + bad.chmod(0o644) + with pytest.raises(SafeError, match="approval_ledger_unavailable"): + SQLiteApprovalLedger(bad) + link = tmp_path / "link.sqlite" + link.symlink_to(bad) + with pytest.raises(SafeError, match="approval_ledger_unavailable"): + SQLiteApprovalLedger(link) + + +def test_signed_approval_json_roundtrip_preserves_identity_and_destination_binding(tmp_path): + from ocr.anonymization.schemas import Approval + + app = application(tmp_path) + original = app.approve() + restored = Approval.model_validate_json(original.model_dump_json()) + assert json.loads(app.export(restored))["destination_id"] == PRIVATE.destination_id + with pytest.raises(SafeError, match="release_approval_consumed_or_unavailable"): + app.export(original) diff --git a/ocr/tests/test_anonymization_runtime_safety.py b/ocr/tests/test_anonymization_runtime_safety.py new file mode 100644 index 0000000000000000000000000000000000000000..354cac70723f6f87f1b8361fea1076c7de62e566 --- /dev/null +++ b/ocr/tests/test_anonymization_runtime_safety.py @@ -0,0 +1,203 @@ +import ast +import importlib.util +import sys +import types +from pathlib import Path +from unittest.mock import Mock + +import pytest +from ocr.anonymization.atomic_output import write_candidate +from ocr.schemas import SafeError + +ROOT = Path(__file__).parents[2] + + +def offline_helpers(): + spec = importlib.util.spec_from_file_location( + "offline_chunking_helpers", ROOT / "augmentoolkit/utils/offline_chunking.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("stage", ["write", "flush", "fsync", "link"]) +def test_atomic_candidate_failure_has_no_partial_target_or_staging_file( + tmp_path, monkeypatch, stage +): + import ocr.anonymization.atomic_output as module + + if stage in ("write", "flush"): + original = module.os.fdopen + + class BrokenStream: + def __init__(self, fd, mode): + self.real = original(fd, mode) + + def __enter__(self): + return self + + def __exit__(self, *args): + return self.real.__exit__(*args) + + def write(self, data): + if stage == "write": + self.real.write(data[:3]) + raise OSError("synthetic disk failure") + return self.real.write(data) + + def flush(self): + if stage == "flush": + raise OSError("synthetic disk failure") + + monkeypatch.setattr(module.os, "fdopen", BrokenStream) + else: + monkeypatch.setattr(module.os, stage, Mock(side_effect=OSError("synthetic disk failure"))) + with pytest.raises(SafeError, match="anonymization_candidate_write_failed"): + write_candidate(tmp_path / "candidate.json", b'{"complete":true}') + assert not list(tmp_path.iterdir()) + + +def test_publication_is_complete_atomic_private_and_no_overwrite(tmp_path, monkeypatch): + import ocr.anonymization.atomic_output as module + + target = tmp_path / "candidate.json" + data = b'{"complete":true}' + original = module.os.link + + def inspect_link(source, dest): + assert not target.exists() + assert Path(source).read_bytes() == data + assert Path(source).stat().st_mode & 0o777 == 0o600 + original(source, dest) + + monkeypatch.setattr(module.os, "link", inspect_link) + write_candidate(target, data) + assert target.read_bytes() == data and len(list(tmp_path.iterdir())) == 1 + monkeypatch.setattr(module.os, "link", original) + with pytest.raises(SafeError): + write_candidate(target, b"replacement") + assert target.read_bytes() == data + + +def test_symlink_or_racing_writer_cannot_be_overwritten(tmp_path, monkeypatch): + import ocr.anonymization.atomic_output as module + + original = module.os.link + target, other = tmp_path / "candidate.json", tmp_path / "other" + other.write_bytes(b"existing") + target.symlink_to(other) + with pytest.raises(SafeError): + write_candidate(target, b"replacement") + assert other.read_bytes() == b"existing" + target.unlink() + + def racing_link(source, dest): + Path(dest).write_bytes(b"winner") + original(source, dest) + + monkeypatch.setattr(module.os, "link", racing_link) + with pytest.raises(SafeError): + write_candidate(target, b"loser") + assert target.read_bytes() == b"winner" + assert not list(tmp_path.glob(".anonymization-*")) + + +def test_chunking_import_does_not_download_nltk_or_model_files(monkeypatch): + module = offline_helpers() + fake_nltk = types.ModuleType("nltk") + fake_nltk.download = Mock(side_effect=AssertionError("no runtime download")) + loader = Mock(side_effect=AssertionError("no import-time model load")) + monkeypatch.setitem(sys.modules, "nltk", fake_nltk) + monkeypatch.setitem( + sys.modules, + "transformers", + types.SimpleNamespace(AutoTokenizer=types.SimpleNamespace(from_pretrained=loader)), + ) + # Import the actual legacy module with optional format packages absent; no + # extraction/backend/package download is needed to exercise ordinary chunking. + monkeypatch.setitem(sys.modules, "pdf2image", None) + spec = importlib.util.spec_from_file_location( + "private_chunking_runtime_test", ROOT / "generation/core_components/chunking.py" + ) + chunking = importlib.util.module_from_spec(spec) + monkeypatch.setenv("TOKENIZERS_PARALLELISM", "false") + spec.loader.exec_module(chunking) + fake_nltk.download.assert_not_called() + loader.assert_not_called() + chunking.count_tokens = lambda text: len(text.split()) + text = "Ordinary public text for a useful chunking smoke test that contains more than fifty characters." + assert ( + chunking.chunking_algorithm_str(content=text, source_name="public.txt")[0]["text"] == text + ) + # No syntactic import-time download call may reappear in this module. + tree = ast.parse((ROOT / "generation/core_components/chunking.py").read_text()) + assert not any( + isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) and n.func.attr == "download" + for n in ast.walk(tree) + ) + assert module.OfflineTokenizer + + +def test_missing_cached_tokenizer_uses_reversible_conservative_fallback(monkeypatch): + module = offline_helpers() + loader = Mock(side_effect=OSError("cache unavailable")) + monkeypatch.setitem( + sys.modules, + "transformers", + types.SimpleNamespace(AutoTokenizer=types.SimpleNamespace(from_pretrained=loader)), + ) + tokenizer = module.OfflineTokenizer("operator-provisioned-model") + text = "Ein gewöhnlicher Text." + tokens = tokenizer.encode(text) + assert len(tokens) == len(text.encode()) + 2 + assert tokenizer.decode(tokens) == text + assert tokenizer.encode(text) == tokens + loader.assert_called_once_with("operator-provisioned-model", local_files_only=True) + + +def test_cached_tokenizer_and_missing_nltk_corpus_preserve_chunking(monkeypatch): + module = offline_helpers() + backend = types.SimpleNamespace( + encode=lambda text: [1, 2], decode=lambda tokens, **kw: "decoded" + ) + monkeypatch.setitem( + sys.modules, + "transformers", + types.SimpleNamespace( + AutoTokenizer=types.SimpleNamespace(from_pretrained=Mock(return_value=backend)) + ), + ) + tokenizer = module.OfflineTokenizer("cached") + assert tokenizer.encode("text") == [1, 2] and tokenizer.decode([1, 2]) == "decoded" + tokenize = types.ModuleType("nltk.tokenize") + tokenize.sent_tokenize = Mock(side_effect=LookupError("corpus absent")) + tokenize.PunktSentenceTokenizer = lambda: types.SimpleNamespace( + tokenize=lambda text: ["First.", "Second."] + ) + monkeypatch.setitem(sys.modules, "nltk.tokenize", tokenize) + assert module.sent_tokenize("First. Second.") == ["First.", "Second."] + + +def test_short_write_never_publishes_partial_candidate(tmp_path, monkeypatch): + import ocr.anonymization.atomic_output as module + + original = module.os.fdopen + + class ShortStream: + def __init__(self, fd, mode): + self.real = original(fd, mode) + + def __enter__(self): + return self + + def __exit__(self, *args): + return self.real.__exit__(*args) + + def write(self, data): + return self.real.write(data[:3]) + + monkeypatch.setattr(module.os, "fdopen", ShortStream) + with pytest.raises(SafeError, match="anonymization_candidate_write_failed"): + write_candidate(tmp_path / "candidate.json", b'{"complete":true}') + assert not list(tmp_path.iterdir()) diff --git a/ocr/tests/test_anonymization_security.py b/ocr/tests/test_anonymization_security.py new file mode 100644 index 0000000000000000000000000000000000000000..cd218eac7d97d064e480defe54825c6bb6be1ca0 --- /dev/null +++ b/ocr/tests/test_anonymization_security.py @@ -0,0 +1,341 @@ +"""No model download or inference dependency in boundary qualification.""" + +import json +from unittest.mock import Mock + +import pytest +from ocr.anonymization import AnonymizationEngine, Entity +from ocr.anonymization.model import MODEL_REVISION +from ocr.anonymization.recognizer import DEFAULT_PROMPTS, GLiNERRecognizer +from ocr.anonymization.replacement import replace +from ocr.anonymization.schemas import LABELS +from ocr.handoff import anonymization_handoff +from ocr.schemas import Block, Failure, OCRPageResult, OCRResult, SafeError + +CANARY = "SYNTHETIC_UNREDACTED_SECRET" + + +class StubRecognizer: + threshold = 0.5 + prompts = DEFAULT_PROMPTS.copy() + local = True + revision = MODEL_REVISION + categories = LABELS + + def recognize(self, text): + return ( + [ + Entity( + start=0, + end=len(text), + label="PERSON", + text=text, + source="multilingual_ner", + model_revision=self.revision, + score=0.9, + ) + ] + if text + else [] + ) + + +def evidence(level="D2", texts=(CANARY,)): + pages = [ + OCRPageResult( + page_index=i, + page_id=CANARY, + source="cpu_ocr", + engine=CANARY, + provider_destination=CANARY, + text=t, + markdown=CANARY, + image_width=1, + image_height=1, + blocks=[Block(text=CANARY)], + sensitivity=level, + ) + for i, t in enumerate(texts) + ] + pages[0].alternatives = [pages[0].model_copy(update={"alternatives": []})] + return OCRResult( + job_id=CANARY, + request_id=CANARY, + status="completed", + page_count=len(pages), + sensitivity=level, + text=CANARY, + markdown=CANARY, + pages=pages, + failures=[], + ) + + +def candidate(level="D2"): + return anonymization_handoff(evidence(level), AnonymizationEngine(StubRecognizer())) + + +@pytest.mark.parametrize("level", ["D0", "D1", "D2"]) +def test_new_handoff_retains_classification_and_discards_reidentifying_metadata(level): + source = evidence(level, (CANARY, CANARY)) + snapshot = source.model_dump_json() + value = anonymization_handoff(source, AnonymizationEngine(StubRecognizer())) + assert source.model_dump_json() == snapshot + assert CANARY not in value.model_dump_json() + assert len(value.pages) == 2 + assert value.sensitivity == level + assert all(p.original_sensitivity == level for p in value.pages) + assert value.release_review_required is True + assert value.release_approved is False and value.anonymous is False + assert value.automated_release_eligible is False + + +@pytest.mark.parametrize( + "failure", + [ + "exception", + "none", + "string", + "extra", + "wrong_slice", + "invalid_score", + "invalid_label", + "negative", + "past_end", + "bool_offset", + "nan_score", + ], +) +def test_untrusted_model_outputs_never_become_candidates(failure, capsys): + model = StubRecognizer() + good = model.recognize(CANARY)[0].model_dump() + if failure == "exception": + model.recognize = Mock(side_effect=RuntimeError(CANARY)) + else: + bad = { + "none": None, + "string": CANARY, + "extra": [{**good, "preview": CANARY}], + "wrong_slice": [{**good, "text": "wrong"}], + "invalid_score": [{**good, "score": 1.1}], + "invalid_label": [{**good, "label": "NAME"}], + "negative": [{**good, "start": -1}], + "past_end": [{**good, "end": 999999}], + "bool_offset": [{**good, "start": False}], + "nan_score": [{**good, "score": float("nan")}], + }[failure] + model.recognize = Mock(return_value=bad) + with pytest.raises(SafeError) as error: + AnonymizationEngine(model).handoff(evidence()) + assert CANARY not in str(error.value) + assert CANARY not in str(capsys.readouterr()) + + +def test_model_unavailable_fails_without_any_download(tmp_path): + with pytest.raises(SafeError, match="anonymization_model_unavailable"): + GLiNERRecognizer(tmp_path) + + +@pytest.mark.parametrize("attr,value", [("categories", ("PERSON",)), ("local", False)]) +def test_unsupported_categories_or_external_recognizer_blocked(attr, value): + model = StubRecognizer() + setattr(model, attr, value) + with pytest.raises(SafeError): + AnonymizationEngine(model).handoff(evidence()) + + +@pytest.mark.parametrize( + "mutation", ["partial", "failure", "missing", "duplicate", "classification"] +) +def test_incomplete_or_inconsistent_evidence_fails_closed(mutation): + result = evidence(texts=(CANARY, CANARY)) + if mutation == "partial": + result.status = "partial" + elif mutation == "failure": + result.failures = [Failure(page_index=3, code="failed")] + elif mutation == "missing": + result.page_count += 1 + elif mutation == "duplicate": + result.pages[1].page_index = 0 + else: + result.pages[1].sensitivity = "D0" + with pytest.raises(SafeError): + AnonymizationEngine(StubRecognizer()).handoff(result) + + +def test_failure_on_second_page_returns_no_partial_candidate(): + model = StubRecognizer() + model.recognize = Mock(side_effect=[model.recognize(CANARY), RuntimeError(CANARY)]) + with pytest.raises(SafeError, match="anonymization_failed"): + AnonymizationEngine(model).handoff(evidence(texts=(CANARY, CANARY))) + + +def test_empty_predictions_require_review_and_never_claim_success(): + model = StubRecognizer() + model.recognize = Mock(return_value=[]) + value = AnonymizationEngine(model).handoff(evidence()) + assert "empty_prediction" in value.review_reasons + assert not value.automated_release_eligible and not value.release_approved + + +def test_bad_placeholders_block_transformation(): + with pytest.raises(SafeError, match="unsupported_required_categories"): + replace(CANARY, [], {"PERSON": "unsafe"}) + + +def test_untrusted_safe_error_cannot_smuggle_source_into_error_response(): + model = StubRecognizer() + model.recognize = Mock(side_effect=SafeError(CANARY)) + with pytest.raises(SafeError, match="^anonymization_failed$"): + AnonymizationEngine(model).handoff(evidence()) + + +def test_model_copy_cannot_bypass_span_validation(): + forged = StubRecognizer().recognize(CANARY)[0].model_copy(update={"start": -1}) + with pytest.raises(SafeError, match="invalid_entity_spans"): + replace(CANARY, [forged]) + + +def test_model_manifest_corruption_is_rejected(tmp_path): + from ocr.anonymization.model import FILES, MODEL_ID, file_hash, verify_cache + + for name in FILES: + (tmp_path / name).write_text("synthetic artifact") + manifest = dict( + model=MODEL_ID, + revision=MODEL_REVISION, + files={name: file_hash(tmp_path / name) for name in FILES}, + ) + (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + # Authentic artifact hashes are pinned in the repository, so even a forged + # local manifest with internally consistent hashes must fail. + with pytest.raises(SafeError, match="anonymization_model_unavailable"): + verify_cache(tmp_path) + (tmp_path / FILES[0]).write_text("tampered") + with pytest.raises(SafeError, match="anonymization_model_unavailable"): + verify_cache(tmp_path) + + +def test_inference_without_runtime_is_sanitized(monkeypatch): + import sys + + model = GLiNERRecognizer.__new__(GLiNERRecognizer) + model._lock = __import__("threading").Lock() + model.prompts = {k: k for k in LABELS} + monkeypatch.setitem(sys.modules, "torch", None) + with pytest.raises(SafeError, match="anonymization_inference_failed"): + model.recognize(CANARY) + + +def test_legacy_pipeline_flag_blocks_before_processing_or_external_egress(): + # Execute the real chunking entry function without importing its legacy + # module-level NLTK/tokenizer downloads. No data generation backend is needed. + import ast + import os + from pathlib import Path + + path = Path(__file__).parents[2] / "generation/core_components/chunking.py" + tree = ast.parse(path.read_text()) + function = next( + n + for n in tree.body + if isinstance(n, ast.FunctionDef) and n.name == "chunking_algorithm_str" + ) + namespace = {"os": os, "SafeError": SafeError} + exec(compile(ast.Module(body=[function], type_ignores=[]), str(path), "exec"), namespace) + with pytest.raises(SafeError, match="legacy_anonymization_requires_reviewed_workflow"): + namespace["chunking_algorithm_str"](content=CANARY, anonymize=True) + + +def test_model_initialization_exception_is_sanitized(tmp_path, monkeypatch): + import sys + import types + + import ocr.anonymization.recognizer as module + + monkeypatch.setattr(module, "verify_cache", lambda path: path) + monkeypatch.setitem(sys.modules, "torch", types.SimpleNamespace(set_num_threads=Mock())) + monkeypatch.setitem( + sys.modules, + "gliner", + types.SimpleNamespace( + GLiNER=types.SimpleNamespace(from_pretrained=Mock(side_effect=RuntimeError(CANARY))) + ), + ) + with pytest.raises(SafeError, match="^anonymization_model_unavailable$"): + GLiNERRecognizer(tmp_path) + + +@pytest.mark.parametrize( + "options", + [ + dict(threshold="bad"), + dict(threshold=float("nan")), + dict(prompts=[]), + dict(prompts={k: [] for k in LABELS}), + dict(threads=True), + ], +) +def test_invalid_runtime_configuration_is_sanitized(tmp_path, options): + with pytest.raises(SafeError): + GLiNERRecognizer(tmp_path, **options) + + +def test_cli_creates_private_classified_candidate_and_never_overwrites( + tmp_path, monkeypatch, capsys +): + import sys + + from ocr.anonymization import __main__ as cli + + source, output = tmp_path / "evidence.json", tmp_path / "candidate.json" + source.write_text(evidence().model_dump_json()) + monkeypatch.setattr(cli, "GLiNERRecognizer", lambda cache: StubRecognizer()) + monkeypatch.setattr( + sys, + "argv", + [ + "anonymization", + "--model-cache", + str(tmp_path), + "--input", + str(source), + "--output", + str(output), + ], + ) + cli.main() + original = output.read_bytes() + assert CANARY.encode() not in original + assert output.stat().st_mode & 0o777 == 0o600 + assert json.loads(original)["sensitivity"] == "D2" + with pytest.raises(SystemExit, match="anonymization_candidate_failed"): + cli.main() + assert output.read_bytes() == original + assert CANARY not in str(capsys.readouterr()) + + +def test_cli_failure_never_writes_a_candidate(tmp_path, monkeypatch): + import sys + + from ocr.anonymization import __main__ as cli + + source, output = tmp_path / "evidence.json", tmp_path / "candidate.json" + source.write_text(evidence().model_dump_json()) + monkeypatch.setattr(cli, "GLiNERRecognizer", Mock(side_effect=RuntimeError(CANARY))) + monkeypatch.setattr( + sys, + "argv", + [ + "anonymization", + "--model-cache", + str(tmp_path), + "--input", + str(source), + "--output", + str(output), + ], + ) + with pytest.raises(SystemExit, match="anonymization_candidate_failed"): + cli.main() + assert not output.exists() diff --git a/ocr/tests/test_api_jobs.py b/ocr/tests/test_api_jobs.py new file mode 100644 index 0000000000000000000000000000000000000000..6494ea3e49ff3c95fee911a99b10439b345450db --- /dev/null +++ b/ocr/tests/test_api_jobs.py @@ -0,0 +1,159 @@ +import asyncio + +import httpx +from conftest import Fake +from ocr.api import create_app +from ocr.config import Settings +from ocr.fixtures.generate import generate +from ocr.worker_api import create_app as worker_app + +H = {"Authorization": "Bearer test"} + + +async def wait(client, id): + for _ in range(200): + s = (await client.get("/v1/ocr/jobs/" + id, headers=H)).json() + if s["status"] in ("completed", "partial", "failed", "cancelled"): + return s + await asyncio.sleep(0.02) + raise AssertionError("job did not finish") + + +async def test_auth_downgrade_idempotency_cancel(png): + cpu = Fake() + app = create_app(Settings(token="test"), cpu) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c, + ): + assert (await c.post("/v1/ocr/jobs", content=b"not multipart")).status_code == 401 + r = await c.post( + "/v1/ocr/jobs", + headers={**H, "Idempotency-Key": "key"}, + files={"file": ("a.png", png)}, + data={"options": '{"sensitivity":"D0"}'}, + ) + assert r.status_code == 202 + id = r.json()["job_id"] + assert (await wait(c, id))["status"] == "completed" + result = (await c.get(f"/v1/ocr/jobs/{id}/result", headers=H)).json() + assert result["sensitivity"] == "D2" + assert result["pages"][0]["page_index"] == 0 + again = await c.post( + "/v1/ocr/jobs", + headers={**H, "Idempotency-Key": "key"}, + files={"file": ("a.png", png)}, + data={"options": '{"sensitivity":"D0"}'}, + ) + assert again.json()["job_id"] == id + bad = await c.post( + "/v1/ocr/jobs", + headers={**H, "Idempotency-Key": "key"}, + files={"file": ("a.png", png)}, + data={"options": '{"sensitivity":"D2","force_ocr":true}'}, + ) + assert bad.status_code == 409 + assert (await c.get(f"/v1/ocr/jobs/{id}/result")).status_code == 401 + await c.delete(f"/v1/ocr/jobs/{id}", headers=H) + assert (await c.get(f"/v1/ocr/jobs/{id}/result", headers=H)).status_code == 409 + + +async def test_out_of_order_mixed_and_partial(tmp_path): + generate(tmp_path) + cpu = Fake(delay=0.01) + app = create_app(Settings(token="test"), cpu) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c, + ): + r = await c.post( + "/v1/ocr/jobs", + headers=H, + files={"file": ("a.pdf", (tmp_path / "mixed-20.pdf").read_bytes())}, + ) + id = r.json()["job_id"] + s = await wait(c, id) + assert s["status"] == "completed" + result = (await c.get(f"/v1/ocr/jobs/{id}/result", headers=H)).json() + assert [p["page_index"] for p in result["pages"]] == list(range(20)) + assert "" in result["markdown"] and "" in result["markdown"] + assert len(cpu.calls) == 10 + + +async def test_worker_auth_limits_ready(png): + app = worker_app(Settings(worker_token="test"), Fake()) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c, + ): + assert (await c.get("/health")).status_code == 200 + assert (await c.get("/ready")).status_code == 401 + assert (await c.get("/ready", headers=H)).json()["ready"] + assert ( + await c.post("/internal/ocr/page", headers=H, files={"file": ("a.png", png)}) + ).status_code == 200 + assert ( + await c.post("/internal/ocr/page", headers=H, files={"file": ("a.png", b"%PDF-")}) + ).status_code == 422 + + +async def test_ttl_and_capacity(png): + app = create_app(Settings(token="test", ttl=0.1, active_jobs=1), Fake(delay=0.5)) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c, + ): + r = await c.post("/v1/ocr/jobs", headers=H, files={"file": ("a.png", png)}) + id = r.json()["job_id"] + assert ( + await c.post("/v1/ocr/jobs", headers=H, files={"file": ("a.png", png)}) + ).status_code == 429 + await asyncio.sleep(0.15) + assert (await c.get("/v1/ocr/jobs/" + id, headers=H)).status_code == 404 + + +async def test_two_lane_out_of_order_assembly(): + import io + + from PIL import Image + + completed = [] + + class Record(Fake): + async def recognize(self, page, options): + r = await super().recognize(page, options) + completed.append(page.index) + return r + + gpu = Record("gpu", delay=0.02) + app = create_app( + Settings( + token="test", + min_sensitivity="D0", + gpu_enabled=True, + overflow_enabled=True, + overflow_qualified=True, + ), + Fake(delay=0.5), + gpu, + ) + b = io.BytesIO() + im = Image.new("RGB", (100, 100), "white") + im.save(b, format="TIFF", save_all=True, append_images=[im, im]) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c, + ): + app.state.dispatcher.cpu.ewma = 10000 + app.state.dispatcher.gpu.ewma = 1 + r = await c.post( + "/v1/ocr/jobs", + headers=H, + files={"file": ("test.tiff", b.getvalue())}, + data={"options": '{"sensitivity":"D0","mode":"throughput","allow_gpu_overflow":true}'}, + ) + id = r.json()["job_id"] + await app.state.jobs.jobs[id].task + r = (await c.get(f"/v1/ocr/jobs/{id}/result", headers=H)).json() + assert r["status"] == "completed" and [p["page_index"] for p in r["pages"]] == [0, 1, 2] + assert completed != [0, 1, 2] and len(gpu.calls) == 3 diff --git a/ocr/tests/test_facade.py b/ocr/tests/test_facade.py new file mode 100644 index 0000000000000000000000000000000000000000..154db65e9d95a598299f56c36a94f7a49eaf569b --- /dev/null +++ b/ocr/tests/test_facade.py @@ -0,0 +1,45 @@ +import base64 + +import httpx +from conftest import Fake +from ocr.api import create_app +from ocr.config import Settings + + +async def test_facade_and_rejections(png): + app = create_app(Settings(token="test"), Fake()) + headers = {"Authorization": "Bearer test"} + body = { + "model": "ocr-cpu", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64," + base64.b64encode(png).decode() + }, + } + ], + } + ], + } + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client, + ): + r = await client.post("/v1/chat/completions", headers=headers, json=body) + assert r.status_code == 200, r.text + assert r.json()["choices"][0]["message"]["content"] + assert "usage" not in r.json() and r.json()["ocr"]["sensitivity"] == "D2" + for extra in ({"stream": True}, {"tools": []}, {"model": "alias-ocr"}): + assert ( + await client.post("/v1/chat/completions", headers=headers, json={**body, **extra}) + ).status_code == 422 + body["messages"][0]["content"][0]["image_url"]["url"] = ( + "https://unapproved.invalid/image.png" + ) + assert ( + await client.post("/v1/chat/completions", headers=headers, json=body) + ).status_code == 422 diff --git a/ocr/tests/test_gate03c_holdout_utf8.py b/ocr/tests/test_gate03c_holdout_utf8.py new file mode 100644 index 0000000000000000000000000000000000000000..d04f8d51654548d303e5357b5b12e94823f223f5 --- /dev/null +++ b/ocr/tests/test_gate03c_holdout_utf8.py @@ -0,0 +1,74 @@ +import hashlib +import json + +import pytest +from ocr.schemas import SafeError +from ocr.tests.test_anonymization_runtime_safety import offline_helpers + + +@pytest.mark.parametrize( + "text", ["plain ASCII", "München, Grüße!", "naïve café 🌍", "漢字🙂aéß", "नमस्ते दुनिया"] +) +def test_full_and_every_truncation_boundary_roundtrip(text): + tokenizer = offline_helpers().OfflineTokenizer("absent-cache") + tokenizer._attempted = True + tokens = tokenizer.encode(text) + assert tokenizer.decode(tokens) == text + for direction in ("prefix", "suffix"): + for limit in range(len(tokens) + 1): + shortened = tokenizer.truncate_tokens(tokens, limit, keep=direction) + decoded = tokenizer.decode(shortened) + assert len(shortened) <= limit + assert text.startswith(decoded) if direction == "prefix" else text.endswith(decoded) + assert tokenizer.decode(tokenizer.encode(decoded)) == decoded + assert "\ufffd" not in decoded + + +def test_direct_partial_utf8_decode_fails_instead_of_silently_losing_character(): + tokenizer = offline_helpers().OfflineTokenizer("absent-cache") + tokenizer._attempted = True + with pytest.raises(SafeError, match="tokenizer_invalid_or_truncated_utf8"): + tokenizer.decode(tokenizer.encode("é")[:2]) + with pytest.raises(SafeError): + tokenizer.decode([256, 999, 257]) + + +def test_frozen_holdout_integrity_and_control_unchanged(): + from pathlib import Path + + from ocr.anonymization.recognizer import DEFAULT_PROMPTS + + root = Path(__file__).parents[1] / "anonymization/fixtures/holdout03c" + freeze = json.loads((root / "freeze.json").read_text()) + raw = (root / "cases.json").read_bytes() + cases = json.loads(raw) + assert hashlib.sha256(raw).hexdigest() == freeze["sha256"] + assert len(cases) >= 100 and len({c["id"] for c in cases}) == len(cases) + assert freeze["control_prompts"] == DEFAULT_PROMPTS + assert {c["language"] for c in cases} == {"de", "en"} + for case in cases: + for span in case["spans"]: + assert case["text"][span["start"] : span["end"]] == span["text"] + + +def test_existing_conversation_suffix_truncation_preserves_utf8_characters(): + import ast + from pathlib import Path + + tokenizer = offline_helpers().OfflineTokenizer("absent-cache") + tokenizer._attempted = True + path = Path(__file__).parents[2] / "generation/core_components/chunking.py" + tree = ast.parse(path.read_text()) + function = next( + n + for n in tree.body + if isinstance(n, ast.FunctionDef) and n.name == "format_conversation_history" + ) + scope = {"tokenizer": tokenizer, "count_tokens": lambda text: len(tokenizer.encode(text))} + exec(compile(ast.Module(body=[function], type_ignores=[]), str(path), "exec"), scope) + source = "é" * 20001 + "🙂" + output = scope["format_conversation_history"]([{"from": "human", "value": source}]) + assert output.startswith("...truncated...") + payload = output.removeprefix("...truncated...") + assert source.endswith(payload) and payload.endswith("🙂") + assert tokenizer.decode(tokenizer.encode(payload)) == payload diff --git a/ocr/tests/test_gate03c_runtime_state.py b/ocr/tests/test_gate03c_runtime_state.py new file mode 100644 index 0000000000000000000000000000000000000000..c5788f1d97cbb9e7878134dc7fef1814b609cbaa --- /dev/null +++ b/ocr/tests/test_gate03c_runtime_state.py @@ -0,0 +1,185 @@ +import fcntl +import json +import multiprocessing +import os +import sqlite3 +import time +from concurrent.futures import ProcessPoolExecutor + +import pytest +from ocr.anonymization.approval_ledger import SQLiteApprovalLedger, validate_key_topology +from ocr.anonymization.recovery import recover_staging +from ocr.schemas import SafeError +from ocr.tests.test_anonymization_release_policy import application + + +def consume_worker(args): + path, identity, nonce, expires = args + return SQLiteApprovalLedger(path, expected_id=identity).consume(nonce, expires) + + +def open_transaction(path, ready): + con = sqlite3.connect(path) + con.execute("BEGIN IMMEDIATE") + con.execute("INSERT INTO consumed VALUES (?,?)", ("uncommitted", int(time.time()) + 300)) + ready.set() + time.sleep(60) + + +def test_persistent_multiprocess_race_restart_and_rotation(tmp_path): + path = tmp_path / "ledger.sqlite" + ledger = SQLiteApprovalLedger(path, initialize=True) + args = (str(path), ledger.ledger_id, "shared-nonce", int(time.time()) + 300) + with ProcessPoolExecutor(max_workers=4) as executor: + assert sum(executor.map(consume_worker, [args] * 8)) == 1 + assert consume_worker(args) is False + assert not ledger.consume("expired", int(time.time()) - 1) + # Actual authority key rotation: old signatures are rejected with new key, + # while the durable replay ledger is retained (no database reset). + from ocr.anonymization import ReleaseAuthority + from ocr.tests.test_anonymization_release_policy import PRIVATE + + app = application(tmp_path) + approval = app.approve() + rotated = ReleaseAuthority( + b"n" * 32, + app.authorizer, + verify_review=app.verifier, + consume_approval=app.ledger.consume, + legacy_policy=app.authority._legacy_policy, + destinations=(PRIVATE,), + ) + with pytest.raises(SafeError): + rotated.export( + app.value, approval, actor="authenticated-session", destination=PRIVATE.destination_id + ) + + fresh = rotated.approve( + app.value, + actor="authenticated-session", + review_id="review-1", + destination=PRIVATE.destination_id, + ) + assert ( + json.loads( + rotated.export( + app.value, fresh, actor="authenticated-session", destination=PRIVATE.destination_id + ) + )["release_approved"] + is True + ) + + +def test_termination_rolls_back_and_committed_consumption_survives(tmp_path): + path = tmp_path / "ledger.sqlite" + ledger = SQLiteApprovalLedger(path, initialize=True) + assert ledger.consume("committed", int(time.time()) + 300) + ctx = multiprocessing.get_context("spawn") + ready = ctx.Event() + process = ctx.Process(target=open_transaction, args=(str(path), ready)) + process.start() + try: + assert ready.wait(10) + finally: + process.kill() + process.join(10) + restarted = SQLiteApprovalLedger(path, expected_id=ledger.ledger_id) + assert not restarted.consume("committed", int(time.time()) + 300) + assert restarted.consume("uncommitted", int(time.time()) + 300) + + +@pytest.mark.parametrize( + "failure", ["missing", "corrupt", "permissions", "replacement", "identity_mismatch"] +) +def test_storage_failure_never_reinitializes_or_releases(tmp_path, failure): + path = tmp_path / "ledger.sqlite" + ledger = SQLiteApprovalLedger(path, initialize=True) + if failure == "missing": + path.unlink() + elif failure == "corrupt": + path.write_bytes(b"not a database") + elif failure == "permissions": + path.chmod(0o644) + elif failure == "replacement": + path.rename(tmp_path / "old.sqlite") + SQLiteApprovalLedger(path, initialize=True) + if failure == "identity_mismatch": + with pytest.raises(SafeError): + SQLiteApprovalLedger(path, expected_id="wrong") + else: + with pytest.raises(SafeError): + ledger.consume("x", int(time.time()) + 300) + if failure != "replacement": + with pytest.raises(SafeError): + SQLiteApprovalLedger(path) + if failure == "missing": + assert not path.exists() + + +def test_multi_host_keys_cannot_share_independent_local_ledgers(): + a = dict( + host_id="host-a", key_id="key-a", ledger_id="a", backend="sqlite", shared_durable=False + ) + b = a | dict(host_id="host-b", key_id="key-b", ledger_id="b") + # Different labels for identical raw keys must not bypass the guard. + with pytest.raises(SafeError): + validate_key_topology([a, b], signing_keys={"key-a": b"x" * 32, "key-b": b"x" * 32}) + validate_key_topology([a, b], signing_keys={"key-a": b"x" * 32, "key-b": b"y" * 32}) + # An operator boolean claiming shared durability is not a real shared backend. + with pytest.raises(SafeError): + validate_key_topology( + [ + a | dict(backend="shared", shared_durable=True), + b | dict(backend="shared", shared_durable=True), + ], + signing_keys={"key-a": b"x" * 32, "key-b": b"x" * 32}, + ) + + +def test_classified_cleanup_retains_evidence_and_respects_active_writer_lock(tmp_path): + old = tmp_path / ".anonymization-abandoned123" + old.write_bytes(b"classified synthetic staging") + old.chmod(0o600) + os.utime(old, (0, 0)) + source = tmp_path / "original-evidence.json" + source.write_bytes(b"preserve classified evidence") + linked = tmp_path / ".anonymization-symlink123" + linked.symlink_to(source) + fd = os.open(tmp_path, os.O_RDONLY | os.O_DIRECTORY) + try: + fcntl.flock(fd, fcntl.LOCK_SH) + with pytest.raises(SafeError): + recover_staging(tmp_path, minimum_age_seconds=60) + finally: + os.close(fd) + result = recover_staging(tmp_path, minimum_age_seconds=60) + assert result["removed_staging_files"] == 1 and result["secure_erasure_guaranteed"] is False + assert source.exists() and linked.is_symlink() + + +def leave_partial_staging(directory, ready): + import tempfile + + descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) + fcntl.flock(descriptor, fcntl.LOCK_SH) + fd, name = tempfile.mkstemp(prefix=".anonymization-", dir=directory) + os.write(fd, b"partial synthetic classified output") + os.utime(name, (0, 0)) + ready.set() + time.sleep(30) + + +def test_killed_writer_recovery_removes_only_private_abandoned_stage(tmp_path): + context = multiprocessing.get_context("spawn") + ready = context.Event() + process = context.Process(target=leave_partial_staging, args=(str(tmp_path), ready)) + process.start() + try: + assert ready.wait(10) + assert not (tmp_path / "candidate.json").exists() + finally: + process.kill() + process.join(10) + result = recover_staging(tmp_path, minimum_age_seconds=60) + assert result["removed_staging_files"] == 1 + assert not list(tmp_path.iterdir()) diff --git a/ocr/tests/test_gate03e_outcomes_status.py b/ocr/tests/test_gate03e_outcomes_status.py new file mode 100644 index 0000000000000000000000000000000000000000..87184d6f1f6b3c5005724f6498fab19552326859 --- /dev/null +++ b/ocr/tests/test_gate03e_outcomes_status.py @@ -0,0 +1,183 @@ +"""Synthetic category/coverage separation and bounded status-word evidence.""" + +import pytest +from ocr.anonymization.outcomes import score_outcomes +from ocr.anonymization.replacement import replace +from ocr.anonymization.schemas import Entity +from ocr.anonymization.status import StatusPolicy +from ocr.schemas import SafeError + + +@pytest.mark.parametrize( + "text,expected", + [ + ("Angabe: keine Konfession.", ["keine Konfession"]), + ("Er bestritt 'keiner Religion' anzugehören.", ["keiner Religion"]), + ("A religiously uncommitted position is quoted.", ["religiously uncommitted"]), + ("The essay concerns religiously unaffiliated citizens.", ["religiously unaffiliated"]), + ("Ohne Religion, nicht ohne Humor.", ["Ohne Religion"]), + ("Status: NONRELIGI0US", ["NONRELIGI0US"]), + ("Scan: konfessionsfre1", ["konfessionsfre1"]), + ("Scan: konfessi0nsfrei", ["konfessi0nsfrei"]), + ("💡 nonreli-\ngi0us", ["nonreli-\ngi0us"]), + ], +) +def test_general_status_grammar_and_exact_offsets(text, expected): + predictions = StatusPolicy().detect(text) + assert [p.text for p in predictions] == expected + assert all(p.score is None and p.source == "deterministic" for p in predictions) + output, _ = replace(text, predictions) + assert output.count("[RELIGION]") == len(expected) + assert all(text[p.start : p.end] == p.text for p in predictions) + + +@pytest.mark.parametrize( + "text", + [ + "politically unaffiliated", + "Religiously sophisticated art", + "ohne Religionsbuch", + "keine Religionszugehörigkeit behauptet", + "nonreligiousness", + "nonreligi00s", + "prenom_nonreligi0us", + "nonreligi0us_suffix", + "übernonreligi0us", + "none religious", + "a 12-character number", + "nonreligi0us\u0301", + "Archivs", + "Hafenbogen", + "konfessXonsfrei", + "konfess10nsfre1", + ], +) +def test_no_broad_fuzzy_matching_or_attribution(text): + assert not StatusPolicy().detect(text) + + +def test_disable_ocr_and_bounded_operator_policy(): + assert not StatusPolicy(ocr_enabled=False).detect("NONRELIGI0US") + for kwargs in ( + {"ocr_enabled": 1}, + {"ocr_terms": ()}, + {"ocr_terms": ("short",)}, + {"ocr_terms": ("a.*",)}, + {"ocr_terms": ("a" * 41,)}, + ): + with pytest.raises(SafeError, match="invalid_status_configuration"): + StatusPolicy(**kwargs) + + +def test_span_conflicts_are_abstentions_not_returned_leaks(): + text = "Synthetic" + gold = dict(start=0, end=9, text=text, label="RELIGION") + case = dict(id="unit", text=text, spans=[gold], expected="[RELIGION]") + + class Recognizer: + def recognize(self, text): + return [ + Entity( + **(gold | {"label": "PERSON"}), source="deterministic", model_revision="unit" + ) + ] + + report = score_outcomes( + Recognizer(), + [case], + detector=lambda t: [Entity(**gold, source="deterministic", model_revision="unit")], + ) + assert report["candidate_outcomes"]["blocked_conflicting_spans"] == 1 + assert report["safe_abstention_rate"] == 1 and report["candidate_return_rate"] == 0 + assert sum(report["candidate_fully_unmasked_canaries"].values()) == 0 + assert sum(report["candidate_partial_canaries"].values()) == 0 + assert report["no_measured_leakage_rate_among_returned"] is None + assert report["accuracy"]["uncovered_source_canaries_including_incomplete"]["RELIGION"] == 1 + assert report["accuracy"]["document_exact_match_rate"] == 0 + assert not report["recognition_qualified"] + + +@pytest.mark.parametrize( + "code,outcome", + [ + ("anonymization_inference_failed", "blocked_model_runtime"), + ("incomplete_ocr_document", "other_incomplete"), + ("anonymization_text_limit", "other_incomplete"), + ], +) +def test_other_failures_never_count_as_success(code, outcome): + class Failed: + def recognize(self, text): + raise SafeError(code) + + report = score_outcomes( + Failed(), + [dict(id="negative", text="plain", spans=[], expected="plain")], + detector=lambda t: [], + ) + assert report["candidate_outcomes"][outcome] == 1 + assert report["candidate_outcomes"]["complete_no_measured_leakage"] == 0 + assert report["candidate_return_rate"] == 0 + + +def test_partial_full_and_taxonomy_mismatch_are_distinct(): + cases = [ + dict( + id=str(i), + text="abc def", + spans=[dict(start=0, end=7, text="abc def", label="PERSON")], + expected="[PERSON]", + ) + for i in range(3) + ] + + class Model: + def __init__(self): + self.n = 0 + + def recognize(self, text): + self.n += 1 + if self.n == 1: + return [] + return [ + Entity( + start=0, + end=3 if self.n == 2 else 7, + text=text[:3] if self.n == 2 else text, + label="CITY", + source="deterministic", + model_revision="unit", + ) + ] + + report = score_outcomes(Model(), cases, detector=lambda t: []) + assert [p["outcome"] for p in report["per_case"]] == [ + "candidate_fully_unmasked", + "candidate_partial_mask", + "complete_no_measured_leakage", + ] + assert report["accuracy"]["document_exact_match_rate"] == 0 + assert report["returned_candidate_entity_metrics"]["PERSON"]["fn"] == 3 + assert ( + report["candidate_partial_canaries"]["PERSON"] + == report["candidate_fully_unmasked_canaries"]["PERSON"] + == 1 + ) + + +def test_unexpected_runtime_failure_is_sanitized_and_abstains(): + class Broken: + def recognize(self, text): + raise RuntimeError("SYNTHETIC_PRIVATE_RUNTIME_PAYLOAD") + + result = score_outcomes(Broken(), [dict(id="unit", text="plain", spans=[], expected="plain")]) + assert result["candidate_outcomes"]["blocked_model_runtime"] == 1 + assert result["per_case"][0]["error_code"] == "anonymization_inference_failed" + assert "SYNTHETIC_PRIVATE_RUNTIME_PAYLOAD" not in str(result) + + +def test_historical_first_runner_refuses_updated_implementation(): + from ocr.anonymization.qualify03d import verify_freeze + + with pytest.raises(ValueError, match="holdout_integrity_or_configuration_changed"): + verify_freeze() diff --git a/ocr/tests/test_gate03e_provenance_review.py b/ocr/tests/test_gate03e_provenance_review.py new file mode 100644 index 0000000000000000000000000000000000000000..30f34a9be92af07def2981eb50baaa846c62a50e --- /dev/null +++ b/ocr/tests/test_gate03e_provenance_review.py @@ -0,0 +1,382 @@ +"""Actual approval compatibility and synthetic private-host review security.""" + +import hashlib +import json +import time +from pathlib import Path + +import pytest +from ocr.anonymization.candidate_io import parse_candidate +from ocr.anonymization.engine import AnonymizationEngine +from ocr.anonymization.private_review import ( + PrivateReviewBridge, + PrivateReviewMaterial, + ReviewBinding, + SourcePageReference, +) +from ocr.anonymization.release_policy import digest +from ocr.anonymization.schemas import LABELS, LegacyReleaseCandidate +from ocr.anonymization.status import StatusPolicy +from ocr.schemas import SafeError +from ocr.tests.fixtures.isolated_host import IsolatedHostAdapter +from ocr.tests.test_anonymization_release_policy import PRIVATE, application +from ocr.tests.test_anonymization_security import CANARY, StubRecognizer, candidate, evidence + + +def test_canonical_provenance_and_no_raw_config_metadata(): + m = StubRecognizer() + a = AnonymizationEngine(m).handoff(evidence()) + m.prompts = dict(reversed(list(m.prompts.items()))) + b = AnonymizationEngine(m).handoff(evidence()) + assert a.provenance == b.provenance + m.prompts = m.prompts | {"PERSON": CANARY} + c = AnonymizationEngine(m).handoff(evidence()) + assert c.provenance != a.provenance + assert CANARY not in c.model_dump_json() + assert c.schema_version == "ocr-anonymization-candidate/v2" + assert not c.anonymous and not c.automated_release_eligible + + +@pytest.mark.parametrize("change", ["threshold", "terms", "ocr", "placeholders"]) +def test_each_policy_change_changes_digest(change): + m = StubRecognizer() + original = AnonymizationEngine(m).handoff(evidence()).provenance.configuration_digest + kwargs = {} + if change == "threshold": + m.threshold = 0.4 + if change == "terms": + kwargs["religion_terms"] = ("Shinto",) + if change == "ocr": + kwargs["status_policy"] = StatusPolicy(ocr_enabled=False) + if change == "placeholders": + kwargs["placeholders"] = {k: "[HIDDEN_" + k + "]" for k in LABELS} + assert ( + AnonymizationEngine(m, **kwargs).handoff(evidence()).provenance.configuration_digest + != original + ) + + +def test_runtime_configuration_drift_blocks_candidate(): + class Drift(StubRecognizer): + def recognize(self, text): + self.threshold = 0.4 + return super().recognize(text) + + with pytest.raises(SafeError, match="recognition_configuration_changed"): + AnonymizationEngine(Drift()).handoff(evidence()) + + +def test_old_candidate_serialization_and_approval_remain_valid(tmp_path): + raw = (Path(__file__).parent / "fixtures/candidate_v1.json").read_bytes() + old = parse_candidate(raw) + assert type(old) is LegacyReleaseCandidate + assert old.model_dump_json().encode() == raw + assert digest(old) == "eb3ab0d101360654f07aa4c614f12873646d44e1d21b1e6cf096033225dec435" + app = application(tmp_path) + oldreview = app.review.model_copy(update={"candidate_digest": digest(old)}) + + def verify(actor, review_id): + assert actor == "authenticated-session" and review_id == "review-1" + return oldreview + + app.verifier.side_effect = verify + app.value = old + approval = app.approve() + new = candidate() + with pytest.raises(SafeError, match="release_not_approved"): + app.authority.export( + new, approval, actor="authenticated-session", destination=PRIVATE.destination_id + ) + out = json.loads(app.export(approval)) + assert out["candidate_digest"] == digest(old) and out["sensitivity"] == "D2" + assert not out["transmission_authorized"] + + +@pytest.mark.parametrize("change", ["unknown_version", "missing_provenance", "model_mismatch"]) +def test_unsupported_or_forged_candidate_fails_closed(change): + data = json.loads(candidate().model_dump_json()) + if change == "unknown_version": + data["schema_version"] = "ocr-anonymization-candidate/v999" + if change == "missing_provenance": + del data["provenance"] + if change == "model_mismatch": + data["provenance"]["model_revision"] = "f" * 40 + with pytest.raises(SafeError, match="unsupported_or_invalid_candidate"): + parse_candidate(json.dumps(data)) + + +def test_modified_provenance_invalidates_new_approval(tmp_path): + app = application(tmp_path) + approval = app.approve() + forged = app.value.model_copy( + update={ + "provenance": app.value.provenance.model_copy(update={"configuration_digest": "f" * 64}) + } + ) + with pytest.raises(SafeError, match="release_not_approved"): + app.authority.export( + forged, approval, actor="authenticated-session", destination=PRIVATE.destination_id + ) + assert json.loads(app.export(approval))["candidate_digest"] == digest(app.value) + + +class SyntheticReviewHost(IsolatedHostAdapter): + """Isolated test host only; opaque credentials and no production fallback.""" + + def __init__(self): + value = candidate() + super().__init__(value) + self.binding = ReviewBinding( + review_ref="synthetic-review", + environment_id="private-lab", + sensitivity="D2", + expires_at=int(time.time()) + 300, + revoked=False, + ) + self.source = CANARY.encode() + self.material = PrivateReviewMaterial( + binding=self.binding, + evidence_digest="a" * 64, + sources=( + SourcePageReference( + evidence_ref="b" * 64, + sensitivity="D2", + page_index=0, + sha256=hashlib.sha256(self.source).hexdigest(), + characters=len(CANARY), + ), + ), + candidate=value, + candidate_digest=digest(value), + provenance=value.provenance, + blocking_reason=None, + ) + self.allowed.add((self.identity.subject, "anonymization:review", "D2", "private-lab")) + self.reads = 0 + + def review_binding(self, reference): + if reference != self.binding.review_ref: + raise PermissionError(CANARY) + return self.binding + + def private_environment(self, identifier): + return identifier == "private-lab" + + def load_review_material(self, reference): + self.reads += 1 + return self.material + + def read_classified_page(self, reference, index): + assert reference == "b" * 64 and index == 0 + return self.source + + +def test_private_review_has_classified_source_and_no_approval_power(): + host = SyntheticReviewHost() + bridge = PrivateReviewBridge(host) + material = bridge.open(host.credential, host.binding.review_ref) + assert material.candidate.pages[0].anonymized_text == "[PERSON]" + assert material.categories == ("PERSON",) + assert "review_complete_source_pages" in material.review_flags + assert material.provenance == material.candidate.provenance + assert not material.automated_release_eligible and not material.anonymous + assert CANARY not in repr(material) + original = bridge.source_page(host.credential, host.binding.review_ref, 0) + assert original.text == CANARY and original.sensitivity == "D2" + assert CANARY not in repr(original) + with pytest.raises(SafeError, match="invalid_release_candidate"): + digest(material) + assert host.production_ready is False + + +@pytest.mark.parametrize( + "mode", ["credentials", "acl", "expired", "revoked", "public_environment", "candidate_missing"] +) +def test_private_review_refuses_missing_authority(mode): + host = SyntheticReviewHost() + credential = host.credential + if mode == "credentials": + credential = {"subject": host.identity.subject, "approved": True} + if mode == "acl": + host.allowed.clear() + if mode == "expired": + host.identity = host.identity.model_copy(update={"expires_at": 0}) + if mode == "revoked": + host.binding = host.binding.model_copy(update={"revoked": True}) + if mode == "public_environment": + host.private_environment = lambda identifier: False + if mode == "candidate_missing": + host.candidates = frozenset() + with pytest.raises(SafeError, match="^private_review_access_denied$"): + PrivateReviewBridge(host).open(credential, host.binding.review_ref) + if mode != "candidate_missing": + assert host.reads == 0 + + +@pytest.mark.parametrize( + "mode", + [ + "changed_source", + "changed_class", + "changed_digest", + "revoked_during_read", + "session_expired_during_read", + "url_reference", + ], +) +def test_private_source_integrity_and_current_authorization(mode): + host = SyntheticReviewHost() + bridge = PrivateReviewBridge(host) + if mode == "changed_source": + host.source = b"SYNTHETIC_ALTERED" + if mode == "changed_class": + host.material = host.material.model_copy( + update={"binding": host.binding.model_copy(update={"sensitivity": "D0"})} + ) + if mode == "changed_digest": + host.material = host.material.model_copy(update={"candidate_digest": "f" * 64}) + if mode == "url_reference": + host.material = host.material.model_copy( + update={ + "sources": ( + host.material.sources[0].model_copy( + update={"evidence_ref": "https://example.invalid"} + ), + ) + } + ) + if mode in ("revoked_during_read", "session_expired_during_read"): + original = host.read_classified_page + + def changed(reference, index): + if mode == "revoked_during_read": + host.binding = host.binding.model_copy(update={"revoked": True}) + else: + host.identity = host.identity.model_copy(update={"expires_at": 0}) + return original(reference, index) + + host.read_classified_page = changed + with pytest.raises(SafeError, match="^private_review_access_denied$"): + bridge.source_page(host.credential, host.binding.review_ref, 0) + + +def test_blocked_review_record_has_reason_and_original_but_no_candidate(): + host = SyntheticReviewHost() + host.material = host.material.model_copy( + update={"candidate": None, "candidate_digest": None, "blocking_reason": "conflicting_spans"} + ) + material = PrivateReviewBridge(host).open(host.credential, host.binding.review_ref) + assert material.candidate is None and material.categories == () + assert "processing_blocked" in material.review_flags and material.release_review_required + assert ( + PrivateReviewBridge(host).source_page(host.credential, host.binding.review_ref, 0).text + == CANARY + ) + + +def test_legacy_review_cannot_invent_configuration_or_accept_new_normalization(): + host = SyntheticReviewHost() + old = parse_candidate((Path(__file__).parent / "fixtures/candidate_v1.json").read_bytes()) + host.material = host.material.model_copy( + update={"candidate": old, "candidate_digest": digest(old), "provenance": None} + ) + host.candidates = frozenset((digest(old),)) + packet = PrivateReviewBridge(host).open(host.credential, host.binding.review_ref) + assert "legacy_configuration_unknown" in packet.review_flags and packet.provenance is None + forged = json.loads(old.model_dump_json()) + forged["pages"][0]["tags"] = [ + dict( + start=0, + end=1, + label="PERSON", + placeholder="[PERSON]", + source="deterministic", + model_revision="unit", + normalization="status-ocr-v1", + ) + ] + with pytest.raises(SafeError, match="unsupported_or_invalid_candidate"): + parse_candidate(json.dumps(forged)) + + +def test_replacement_version_and_normalization_configuration_are_committed(monkeypatch): + import importlib + import re + + module = importlib.import_module("ocr.anonymization.provenance") + base = AnonymizationEngine(StubRecognizer()).handoff(evidence()).provenance + monkeypatch.setattr(module, "REPLACEMENT_POLICY_VERSION", "future-policy-version") + changed = AnonymizationEngine(StubRecognizer()).handoff(evidence()).provenance + assert changed.configuration_digest != base.configuration_digest + monkeypatch.setattr(module, "LINE_WRAP", re.compile(r"not-the-current-normalization")) + assert ( + AnonymizationEngine(StubRecognizer()).handoff(evidence()).provenance.configuration_digest + != changed.configuration_digest + ) + + +def test_unknown_recognizer_configuration_is_never_defaulted(): + class Missing(StubRecognizer): + threshold = None + + with pytest.raises(SafeError, match="invalid_recognition_configuration"): + AnonymizationEngine(Missing()).handoff(evidence()) + + +def test_revocation_while_loading_review_material_prevents_return(): + host = SyntheticReviewHost() + original = host.load_review_material + + def changed(reference): + result = original(reference) + host.binding = host.binding.model_copy(update={"revoked": True}) + return result + + host.load_review_material = changed + with pytest.raises(SafeError, match="^private_review_access_denied$"): + PrivateReviewBridge(host).open(host.credential, host.binding.review_ref) + + +def test_source_reference_classification_cannot_be_downgraded(): + host = SyntheticReviewHost() + host.material = host.material.model_copy( + update={"sources": (host.material.sources[0].model_copy(update={"sensitivity": "D0"}),)} + ) + with pytest.raises(SafeError, match="^private_review_access_denied$"): + PrivateReviewBridge(host).source_page(host.credential, host.binding.review_ref, 0) + + +@pytest.mark.parametrize("text", ["Grüße aus dem Testlabor", "漢字🙂synthetic", "नमस्ते"]) +def test_private_source_preserves_full_unicode(text): + host = SyntheticReviewHost() + value = AnonymizationEngine(StubRecognizer()).handoff(evidence("D2", (text,))) + host.source = text.encode("utf-8") + host.candidates = frozenset((digest(value),)) + source = host.material.sources[0].model_copy( + update={"sha256": hashlib.sha256(host.source).hexdigest(), "characters": len(text)} + ) + host.material = host.material.model_copy( + update={ + "candidate": value, + "candidate_digest": digest(value), + "provenance": value.provenance, + "sources": (source,), + } + ) + assert ( + PrivateReviewBridge(host).source_page(host.credential, host.binding.review_ref, 0).text + == text + ) + + +@pytest.mark.parametrize( + "reference", ["https://example.invalid", "../private-file", "x" * 129, {"approved": True}] +) +def test_caller_reference_rejected_before_host_lookup(reference): + from unittest.mock import Mock + + host = SyntheticReviewHost() + host.review_binding = Mock(side_effect=AssertionError("must not reach host lookup")) + with pytest.raises(SafeError, match="^private_review_access_denied$"): + PrivateReviewBridge(host).open(host.credential, reference) + host.review_binding.assert_not_called() diff --git a/ocr/tests/test_gate03g_security_audit.py b/ocr/tests/test_gate03g_security_audit.py new file mode 100644 index 0000000000000000000000000000000000000000..8d04095249269f8fbd83960de1d5239d2ae7f631 --- /dev/null +++ b/ocr/tests/test_gate03g_security_audit.py @@ -0,0 +1,274 @@ +"""Independent adversarial checks of review, release and publication boundaries. + +Synthetic identity/review fixtures; real SQLite locking and filesystem operations. +""" + +import sqlite3 +from concurrent.futures import ThreadPoolExecutor +from threading import Event +from types import SimpleNamespace + +import pytest +from ocr.anonymization import approval_ledger, private_transport +from ocr.anonymization.approval_ledger import SQLiteApprovalLedger +from ocr.anonymization.host_adapter import HostAuthorizationBridge +from ocr.anonymization.private_transport import LocalDestination, PrivateFileTransport +from ocr.anonymization.release_policy import ReleaseAuthority +from ocr.schemas import SafeError +from ocr.tests.test_anonymization_release_policy import PRIVATE, application +from ocr.tests.test_private_host_adapters import setup +from ocr.tests.test_reviewed_corrections import apply_one, edit_for, host_fixture + + +def test_corrected_completed_review_must_bind_original_ocr_evidence(tmp_path): + host = host_fixture() + _, receipt, candidate = apply_one( + host, edit_for(host, "Silver Lantern Archives", "ORGANIZATION", "incomplete_boundary") + ) + review_id = host.complete_review(host.approver_credential, receipt.candidate_digest) + record = host.completed[review_id] + host.completed[review_id] = record.model_copy( + update={"attestation": record.attestation.model_copy(update={"evidence_digest": "f" * 64})} + ) + authority = HostAuthorizationBridge(host).authority( + b"x" * 32, + SQLiteApprovalLedger(tmp_path / "ledger.sqlite", initialize=True), + ("synthetic-private",), + ) + with pytest.raises(SafeError, match="^release_not_approved$"): + authority.approve( + candidate, + actor=host.approver_credential, + review_id=review_id, + destination="synthetic-private", + ) + + +def test_expiry_during_real_sqlite_lock_wait_does_not_consume(tmp_path, monkeypatch): + path = tmp_path / "ledger.sqlite" + ledger = SQLiteApprovalLedger(path, initialize=True) + waiting = Event() + clock = [1000] + monkeypatch.setattr(approval_ledger, "time", SimpleNamespace(time=lambda: clock[0])) + + class Connection(sqlite3.Connection): + def execute(self, sql, *args, **kwargs): + if sql == "BEGIN IMMEDIATE": + waiting.set() + return super().execute(sql, *args, **kwargs) + + monkeypatch.setattr( + ledger, "_connect", lambda: sqlite3.connect(path, timeout=5, factory=Connection) + ) + blocking = sqlite3.connect(path) + blocking.execute("BEGIN IMMEDIATE") + try: + with ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(ledger.consume, "expires-under-contention", 1001) + try: + assert waiting.wait(2) + clock[0] = 1001 + finally: + blocking.rollback() + assert future.result(timeout=5) is False + finally: + blocking.close() + with sqlite3.connect(path) as check: + assert check.execute("SELECT id FROM consumed").fetchall() == [] + + +def test_private_delivery_revocation_during_write_prevents_publication(tmp_path, monkeypatch): + value, host, bridge, authority, approval = setup(tmp_path) + sink = tmp_path / "sink" + sink.mkdir(mode=0o700) + transport = PrivateFileTransport( + [LocalDestination(host.policy, sink)], authorize_transfer=bridge.transfer_authorized + ) + exported = authority.export( + value, approval, actor=host.credential, destination=host.policy.destination_id + ) + host.register_export(exported) + host.allowed.add( + (host.identity.subject, "anonymization:private-transfer", "D2", host.policy.destination_id) + ) + original_fsync = private_transport.os.fsync + + def revoke_after_write(fd): + original_fsync(fd) + host.allowed.clear() + + monkeypatch.setattr(private_transport.os, "fsync", revoke_after_write) + with pytest.raises(SafeError, match="^private_transfer_denied_or_failed$"): + transport.deliver( + exported, credential=host.credential, destination_id=host.policy.destination_id + ) + assert list(sink.iterdir()) == [] + + +def test_uncorrected_release_requires_explicit_operator_policy(tmp_path): + app = application(tmp_path) + unconfigured = ReleaseAuthority( + b"x" * 32, + app.authorizer, + verify_review=app.verifier, + consume_approval=app.ledger.consume, + destinations=(PRIVATE,), + ) + with pytest.raises(SafeError, match="^release_not_approved$"): + unconfigured.approve( + app.value, + actor="authenticated-session", + review_id="review-1", + destination=PRIVATE.destination_id, + ) + + +def test_transfer_rejects_mismatched_completed_review_lookup(tmp_path): + value, host, bridge, authority, approval = setup(tmp_path) + exported = authority.export( + value, approval, actor=host.credential, destination=host.policy.destination_id + ) + host.register_export(exported) + artifact = next(iter(host.exports)) + host.allowed.add( + (host.identity.subject, "anonymization:private-transfer", "D2", host.policy.destination_id) + ) + original = host.load_review + host.load_review = lambda key: original(key).model_copy( + update={ + "attestation": original(key).attestation.model_copy( + update={"review_id": "different-review"} + ) + } + ) + assert ( + bridge.transfer_authorized(host.credential, artifact, "D2", host.policy.destination_id) + is False + ) + + +@pytest.mark.parametrize("change", ["acl", "legacy_policy"]) +def test_revocation_during_consumption_burns_approval_without_returning_output(tmp_path, change): + app = application(tmp_path) + approval = app.approve() + consume = app.authority._consume + + def consume_then_revoke(identifier, expiry): + consumed = consume(identifier, expiry) + if change == "acl": + app.authorizer.side_effect = None + app.authorizer.return_value = False + else: + app.authority._legacy_policy = None + return consumed + + app.authority._consume = consume_then_revoke + with pytest.raises(SafeError, match="^release_not_approved$"): + app.export(approval) + # Fail-closed final authorization burns the original nonce; no retry release. + assert not app.ledger.consume(approval.approval_id, approval.expires_at) + + +@pytest.mark.parametrize("policy", [None, False, 1, "allow", RuntimeError("private-canary")]) +@pytest.mark.parametrize("phase", ["approve", "export"]) +def test_legacy_policy_is_explicit_literal_true_and_current(tmp_path, policy, phase): + app = application(tmp_path) + approval = app.approve() if phase == "export" else None + + def decision(candidate, review): + if isinstance(policy, Exception): + raise policy + return policy + + app.authority._legacy_policy = None if policy is None else decision + with pytest.raises(SafeError, match="^release_not_approved$") as caught: + app.export(approval) if phase == "export" else app.approve() + assert "private-canary" not in str(caught.value) + + +def test_missing_host_legacy_policy_denies_existing_approval(tmp_path): + value, host, bridge, authority, approval = setup(tmp_path) + host.legacy_release_allowed = None + with pytest.raises(SafeError, match="^release_not_approved$"): + authority.export( + value, approval, actor=host.credential, destination=host.policy.destination_id + ) + + +@pytest.mark.parametrize("version", ["v1", "v2"]) +def test_version_coercion_does_not_bypass_corrected_default_deny(tmp_path, version): + from ocr.anonymization.candidate_io import parse_candidate + from ocr.anonymization.release_policy import digest + from ocr.anonymization.schemas import LegacyReleaseCandidate, ReleaseCandidate + + host = host_fixture(specs=(("weather", "PERSON"),), text="weather") + _, receipt, corrected = apply_one( + host, edit_for(host, "weather", "PERSON", "false_positive", action="suppress") + ) + review_id = host.complete_review(host.approver_credential, receipt.candidate_digest) + bridge = HostAuthorizationBridge(host) + authority = ReleaseAuthority( + b"x" * 32, + bridge.acl, + verify_review=bridge.review, + consume_approval=SQLiteApprovalLedger(tmp_path / "ledger.sqlite", initialize=True).consume, + destinations=(host.policy,), + legacy_policy=lambda *args: True, + ) + # Even explicit v1/v2 permission is not a fallback correction permission. + with pytest.raises(SafeError, match="^release_not_approved$"): + authority.approve( + corrected, + actor=host.approver_credential, + review_id=review_id, + destination=host.policy.destination_id, + ) + parent = LegacyReleaseCandidate if version == "v1" else ReleaseCandidate + with pytest.raises(ValueError): + parent.model_validate(corrected) + malformed = corrected.model_copy( + update={"schema_version": "ocr-anonymization-candidate/" + version} + ) + with pytest.raises(SafeError, match="^invalid_release_candidate$"): + digest(malformed) + with pytest.raises(SafeError, match="^unsupported_or_invalid_candidate$"): + parse_candidate(malformed.model_dump_json()) + + +def test_post_link_sync_failure_leaves_only_complete_private_artifact(tmp_path, monkeypatch): + import hashlib + import json + import stat + + value, host, bridge, authority, approval = setup(tmp_path) + exported = authority.export( + value, approval, actor=host.credential, destination=host.policy.destination_id + ) + host.register_export(exported) + host.allowed.add( + (host.identity.subject, "anonymization:private-transfer", "D2", host.policy.destination_id) + ) + sink = tmp_path / "sink" + sink.mkdir(mode=0o700) + transport = PrivateFileTransport( + [LocalDestination(host.policy, sink)], authorize_transfer=bridge.transfer_authorized + ) + original = private_transport.os.fsync + + def fail_directory_sync(fd): + if stat.S_ISDIR(private_transport.os.fstat(fd).st_mode): + raise OSError("private-canary") + original(fd) + + monkeypatch.setattr(private_transport.os, "fsync", fail_directory_sync) + with pytest.raises(SafeError, match="^private_transfer_denied_or_failed$"): + transport.deliver( + exported, credential=host.credential, destination_id=host.policy.destination_id + ) + files = list(sink.iterdir()) + assert ( + len(files) == 1 and files[0].name == hashlib.sha256(exported.encode()).hexdigest() + ".json" + ) + assert files[0].read_bytes() == exported.encode() + assert files[0].stat().st_mode & 0o777 == 0o600 + assert json.loads(files[0].read_bytes())["sensitivity"] == "D2" diff --git a/ocr/tests/test_gate03h_candidate_integrity.py b/ocr/tests/test_gate03h_candidate_integrity.py new file mode 100644 index 0000000000000000000000000000000000000000..920a9c80f7e0344e699b815b1d516f59d4df9f36 --- /dev/null +++ b/ocr/tests/test_gate03h_candidate_integrity.py @@ -0,0 +1,313 @@ +"""Structural remediation only; synthetic fixtures, no model or new holdout.""" + +import hashlib +import json +import runpy +import time +from pathlib import Path +from unittest.mock import Mock + +import pytest +from ocr.anonymization.approval_ledger import SQLiteApprovalLedger +from ocr.anonymization.candidate_io import parse_candidate +from ocr.anonymization.corrections import CorrectionService +from ocr.anonymization.private_review import PrivateReviewBridge, PrivateReviewMaterial +from ocr.anonymization.release_policy import ReleaseAuthority, digest +from ocr.anonymization.schemas import Destination, VerifiedReview +from ocr.schemas import SafeError +from ocr.tests.test_reviewed_corrections import edit_for, host_fixture + +FIXTURES = Path(__file__).parent / "fixtures" +COMPAT = json.loads((FIXTURES / "gate03h_valid_candidates.json").read_bytes()) +VERSIONS = ("v1", "v2", "v3") + + +def valid(version, first=0): + return parse_candidate(COMPAT["cases"][f"{version}-{first}"]["candidate_json"]) + + +def material_for(candidate, version, first=0): + original = PrivateReviewMaterial.model_validate_json( + COMPAT["cases"][f"{version}-{first}"]["material_json"] + ) + return original.model_copy( + update={"candidate": candidate, "candidate_digest": digest(candidate)} + ) + + +@pytest.mark.parametrize( + "name", + [ + "test_duplicate_page_manifest_should_be_rejected", + "test_invalid_replacement_bounds_should_be_rejected_by_parser", + ], +) +def test_original_review_poc_rejection(name): + # Preserve and execute the exact original functions. The original duplicate + # PoC parsed before its raises block; stronger schema rejection now happens + # earlier. Only that precise sanitized early rejection is accepted here. + path = FIXTURES / "gate03h_review_pocs.py.txt" + assert hashlib.sha256(path.read_bytes()).hexdigest() == ( + "de869531a2276c99da4cabbd23ab2bf19c09a1e2116a759d1f70f8fb677b3b2c" + ) + try: + runpy.run_path(str(path))[name]() + except SafeError as exc: + assert name == "test_duplicate_page_manifest_should_be_rejected" + assert exc.code == "unsupported_or_invalid_candidate" + + +@pytest.mark.parametrize("version", VERSIONS) +@pytest.mark.parametrize("indices", [(0, 0), (1, 1), ()]) +def test_schema_rejects_duplicate_and_empty_pages_all_entry_paths(version, indices): + candidate = valid(version) + pages = tuple(candidate.pages[i] for i in indices) + values = candidate.model_dump() | {"pages": pages} + with pytest.raises(ValueError): + type(candidate)(**values) + forged = candidate.model_copy(update={"pages": pages}) + with pytest.raises(ValueError): + type(candidate).model_validate(forged) + with pytest.raises(SafeError, match="^unsupported_or_invalid_candidate$"): + parse_candidate(forged.model_dump_json()) + with pytest.raises(SafeError, match="^invalid_release_candidate$"): + digest(forged) + + +@pytest.mark.parametrize("version", VERSIONS) +@pytest.mark.parametrize("indices", [(0, 2), (0,), (0, 1, 2), (9, 10)]) +def test_private_review_requires_exact_original_page_set(version, indices): + candidate = valid(version) + pages = tuple(candidate.pages[min(i, 1)].model_copy(update={"page_index": i}) for i in indices) + # Unique positive indices are allowed structurally; only immutable source + # references establish whether a source page is absent or unexpected. + changed = type(candidate).model_validate(candidate.model_copy(update={"pages": pages})) + material = material_for(changed, version) + with pytest.raises(ValueError): + PrivateReviewMaterial.model_validate(material) + with pytest.raises(ValueError): + PrivateReviewMaterial.model_validate_json(material.model_dump_json()) + + +BAD_BOUNDS = [ + (-1, 2), + (0, -1), + (0, 0), + (1, 1), + (2, 1), + ("0", 2), + (0, "2"), + (False, 2), + (0, True), + (0.0, 2), + (0, 2.0), + (None, 2), + (0, None), +] + + +@pytest.mark.parametrize("version", VERSIONS) +@pytest.mark.parametrize("start,end", BAD_BOUNDS) +def test_strict_replacement_bounds_direct_json_and_digest(version, start, end): + candidate = valid(version) + page, tag = candidate.pages[0], candidate.pages[0].tags[0] + with pytest.raises(ValueError): + type(tag)(**(tag.model_dump() | {"start": start, "end": end})) + badtag = tag.model_copy(update={"start": start, "end": end}) + badpage = page.model_copy(update={"tags": (badtag,) + page.tags[1:]}) + forged = candidate.model_copy(update={"pages": (badpage,) + candidate.pages[1:]}) + with pytest.raises(ValueError): + type(candidate).model_validate(forged) + with pytest.raises(ValueError): + type(candidate)(**(candidate.model_dump() | {"pages": forged.pages})) + # Preserve the malformed JSON value's type directly. Serializing an + # intentionally invalid model_copy may coerce bools into integer output. + payload = json.loads(candidate.model_dump_json()) + payload["pages"][0]["tags"][0].update(start=start, end=end) + with pytest.raises(SafeError, match="^unsupported_or_invalid_candidate$"): + parse_candidate(json.dumps(payload)) + with pytest.raises(SafeError, match="^invalid_release_candidate$"): + digest(forged) + + +@pytest.mark.parametrize("version", VERSIONS) +def test_source_aware_upper_bound_and_original_coordinate_semantics(version): + candidate = valid(version) + original = material_for(candidate, version) + # Real valid tags extend beyond the shortened output, using Unicode + # codepoints rather than UTF-8 bytes or placeholder offsets. + for page, source in zip(candidate.pages, original.sources, strict=True): + assert page.tags[0].end == source.characters > len(page.anonymized_text) + text = COMPAT["text"][page.page_index] + assert source.characters == len(text) < len(text.encode()) + assert PrivateReviewMaterial.model_validate(original) == original + page = candidate.pages[0] + tag = page.tags[0].model_copy(update={"end": original.sources[0].characters + 1}) + forged = candidate.model_copy( + update={"pages": (page.model_copy(update={"tags": (tag,)}),) + candidate.pages[1:]} + ) + # No source length is serialized in candidates, so the upper bound belongs + # at the source-aware boundary. Structural/digest validity is not approval. + parsed = parse_candidate(forged.model_dump_json()) + with pytest.raises(ValueError): + PrivateReviewMaterial.model_validate(material_for(parsed, version)) + + +@pytest.mark.parametrize("version", VERSIONS) +@pytest.mark.parametrize("first", [0, 1]) +def test_valid_pre_fix_bytes_digests_and_indexing_remain_identical(version, first): + record = COMPAT["cases"][f"{version}-{first}"] + candidate = valid(version, first) + assert candidate.model_dump_json() == record["candidate_json"] + assert digest(candidate) == record["candidate_digest"] + assert [p.page_index for p in candidate.pages] == [first, first + 1] + material = PrivateReviewMaterial.model_validate_json(record["material_json"]) + assert material.model_dump_json() == record["material_json"] + assert candidate.sensitivity == "D2" and not candidate.anonymous + assert candidate.release_review_required and not candidate.automated_release_eligible + + +@pytest.mark.parametrize("version", VERSIONS) +@pytest.mark.parametrize("kind", ["duplicate", "empty", "bounds", "classification"]) +def test_bad_typed_candidate_cannot_approve_or_export_and_does_not_consume(tmp_path, version, kind): + candidate = valid(version) + target = Destination( + destination_id="test-private", destination_class="private", allowed_sensitivities=("D2",) + ) + material = material_for(candidate, version) + review = VerifiedReview( + reviewer_subject="synthetic-final-reviewer", + review_id="synthetic-review", + evidence_digest=material.evidence_digest, + candidate_digest=digest(candidate), + sensitivity="D2", + destination_id=target.destination_id, + completed_at=int(time.time()), + ) + verifier = Mock(return_value=review) + ledger = SQLiteApprovalLedger(tmp_path / "ledger.sqlite", initialize=True) + + # Explicit synthetic policy callbacks; never production defaults. + def policy(c, r): + return digest(c) == review.candidate_digest + + authority = ReleaseAuthority( + b"x" * 32, + lambda *args: True, + verify_review=verifier, + consume_approval=ledger.consume, + destinations=(target,), + legacy_policy=policy, + correction_policy=policy, + ) + approval = authority.approve( + candidate, actor="session", review_id=review.review_id, destination=target.destination_id + ) + page = candidate.pages[0] + if kind == "duplicate": + pages = (page, page) + elif kind == "empty": + pages = () + elif kind == "bounds": + pages = ( + page.model_copy(update={"tags": (page.tags[0].model_copy(update={"start": -1}),)}), + ) + candidate.pages[1:] + else: + pages = (page.model_copy(update={"original_sensitivity": "D0"}),) + candidate.pages[1:] + forged = candidate.model_copy(update={"pages": pages}) + verifier.reset_mock() + for action in ( + lambda: authority.approve( + forged, actor="session", review_id=review.review_id, destination=target.destination_id + ), + lambda: authority.export( + forged, approval, actor="session", destination=target.destination_id + ), + ): + with pytest.raises(SafeError, match="^invalid_release_candidate$"): + action() + verifier.assert_not_called() + # Original exact-digest approval is still usable; malformed attempts did not + # mutate old bytes/history or consume its durable nonce. + output = json.loads( + authority.export(candidate, approval, actor="session", destination=target.destination_id) + ) + assert output["sensitivity"] == "D2" and not output["transmission_authorized"] + with pytest.raises(SafeError, match="^release_approval_consumed_or_unavailable$"): + authority.export(candidate, approval, actor="session", destination=target.destination_id) + + +@pytest.mark.parametrize("kind", ["foreign_page", "upper_bound"]) +def test_private_review_and_correction_reject_corrupt_stored_material(kind): + host = host_fixture() + service = CorrectionService(host) + proposal = service.propose( + host.author_credential, + "original-review", + (edit_for(host, "Silver Lantern Archives", "ORGANIZATION", "incomplete_boundary"),), + ) + host.context_reads = 0 + context = host.contexts["original-review"] + candidate = context.material.candidate + page = candidate.pages[0] + if kind == "foreign_page": + page = page.model_copy(update={"page_index": 19}) + else: + tag = page.tags[0].model_copy(update={"end": context.machine.sources[0].characters + 1}) + page = page.model_copy(update={"tags": (tag,)}) + changed = type(candidate).model_validate(candidate.model_copy(update={"pages": (page,)})) + host.candidates[digest(changed)] = changed + material = context.material.model_copy( + update={"candidate": changed, "candidate_digest": digest(changed)} + ) + host.contexts["original-review"] = context.model_copy(update={"material": material}) + with pytest.raises(SafeError, match="^private_review_access_denied$"): + PrivateReviewBridge(host).open(host.author_credential, "original-review") + with pytest.raises(SafeError, match="^private_correction_denied$"): + service.inspect(host.author_credential, "original-review") + with pytest.raises(SafeError, match="^private_correction_denied$"): + service.apply(host.author_credential, proposal.correction_id) + assert host.context_reads == 0 and len(host.contexts) == 1 + assert host.proposals[proposal.correction_id].published_candidate_digest is None + + +@pytest.mark.parametrize("version", VERSIONS) +def test_malformed_document_containers_and_classification_fail_closed(version): + candidate = valid(version) + for pages in (None, {}, "pages", [None]): + data = json.loads(candidate.model_dump_json()) + data["pages"] = pages + with pytest.raises(SafeError, match="^unsupported_or_invalid_candidate$"): + parse_candidate(json.dumps(data)) + mismatch = candidate.model_copy(update={"sensitivity": "D0"}) + with pytest.raises(SafeError, match="^invalid_release_candidate$"): + digest(mismatch) + + +@pytest.mark.parametrize("version", ["v1", "v2"]) +def test_v3_duplicate_pages_cannot_bypass_with_version_coercion(version): + corrected = valid("v3") + malformed = corrected.model_copy( + update={ + "schema_version": "ocr-anonymization-candidate/" + version, + "pages": (corrected.pages[0], corrected.pages[0]), + } + ) + with pytest.raises(SafeError, match="^invalid_release_candidate$"): + digest(malformed) + with pytest.raises(SafeError, match="^unsupported_or_invalid_candidate$"): + parse_candidate(malformed.model_dump_json()) + + +def test_legacy_archive_unique_indices_are_not_forced_contiguous(): + legacy = valid("v1") + archived = legacy.model_copy( + update={ + "pages": tuple( + page.model_copy(update={"page_index": index}) + for page, index in zip(legacy.pages, (7, 19), strict=True) + ) + } + ) + assert parse_candidate(archived.model_dump_json()) == archived + assert digest(archived) diff --git a/ocr/tests/test_ingest.py b/ocr/tests/test_ingest.py new file mode 100644 index 0000000000000000000000000000000000000000..bbcaf0c8f2991e1e180902eba80e0beff286ca69 --- /dev/null +++ b/ocr/tests/test_ingest.py @@ -0,0 +1,62 @@ +import io + +import pytest +from ocr.config import Settings +from ocr.fixtures.generate import born_digital, generate +from ocr.ingest import decode_image, ingest, usable_text +from ocr.schemas import OCROptions, SafeError +from PIL import Image + + +def test_embedded_skip_and_force(): + p = ingest(born_digital(), "file.pdf", OCROptions(), Settings()) + assert len(p) == 1 and p[0].embedded and p[0].index == 0 + assert ( + ingest(born_digital(), "file.pdf", OCROptions(force_ocr=True), Settings())[0].embedded + is None + ) + + +@pytest.mark.parametrize( + "data,name", [(b"%PDF-broken", "a.pdf"), (b"hello", "a.png"), (b"hello", "a.txt")] +) +def test_malformed(data, name): + with pytest.raises(SafeError): + ingest(data, name, OCROptions(), Settings()) + + +def test_signature_and_limits(png): + with pytest.raises(SafeError, match="signature_mismatch"): + ingest(png, "a.jpg", OCROptions(), Settings()) + with pytest.raises(SafeError, match="image_limit"): + ingest(png, "a.png", OCROptions(), Settings(max_pixels=100)) + with pytest.raises(SafeError, match="upload_limit"): + ingest(png, "a.png", OCROptions(), Settings(max_upload=10)) + assert not usable_text("a\ufffd\ufffd") + + +def test_twenty_page_mixed(tmp_path): + generate(tmp_path) + pages = ingest((tmp_path / "mixed-20.pdf").read_bytes(), "a.pdf", OCROptions(), Settings()) + assert len(pages) == 20 and [p.index for p in pages] == list(range(20)) + assert sum(p.embedded is not None for p in pages) == 10 + with pytest.raises(SafeError, match="page_limit"): + ingest( + (tmp_path / "mixed-20.pdf").read_bytes(), + "a.pdf", + OCROptions(max_pages=2), + Settings(), + ) + + +def test_tiff_and_bomb(png): + b = io.BytesIO() + Image.new("RGB", (40, 40), "white").save( + b, + format="TIFF", + save_all=True, + append_images=[Image.new("RGB", (40, 40), "black")], + ) + assert len(ingest(b.getvalue(), "pages.tiff", OCROptions(), Settings())) == 2 + with pytest.raises(SafeError): + decode_image(png, 1) diff --git a/ocr/tests/test_limits_provenance.py b/ocr/tests/test_limits_provenance.py new file mode 100644 index 0000000000000000000000000000000000000000..0a770dbbb1486b7665f950642a47d2803fff7d77 --- /dev/null +++ b/ocr/tests/test_limits_provenance.py @@ -0,0 +1,163 @@ +import io + +import httpx +import pytest +from conftest import Fake +from ocr.adapters.cpu_http import RemoteCPU +from ocr.api import create_app +from ocr.config import Settings +from ocr.handoff import anonymization_handoff +from ocr.routing import Dispatcher +from ocr.schemas import OCROptions, OCRResult, RasterPage, SafeError +from PIL import Image + +H = {"Authorization": "Bearer test"} + + +async def test_partial_order_and_logs(png, caplog): + class FailOne(Fake): + async def recognize(self, page, options): + if page.index == 1: + raise RuntimeError("PRIVATE_SECRET_IMAGE_TEXT") + return await super().recognize(page, options) + + b = io.BytesIO() + im = Image.new("RGB", (100, 100), "white") + im.save(b, format="TIFF", save_all=True, append_images=[im, im]) + app = create_app(Settings(token="test"), FailOne()) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c, + ): + response = await c.post( + "/v1/ocr/jobs", headers=H, files={"file": ("test.tiff", b.getvalue())} + ) + id = response.json()["job_id"] + job = app.state.jobs.jobs[id] + await job.task + r = await c.get(f"/v1/ocr/jobs/{id}/result", headers=H) + body = r.json() + assert body["status"] == "partial" + assert [p["page_index"] for p in body["pages"]] == [0, 2] + assert body["failures"] == [{"page_index": 1, "code": "backend_failure"}] + assert "PRIVATE_SECRET_IMAGE_TEXT" not in r.text + caplog.text + metrics = (await c.get("/metrics", headers=H)).text + assert "PRIVATE_SECRET" not in metrics and "Public fixture" not in metrics + + +async def test_upload_chunked_rate_options(png): + app = create_app(Settings(token="test", max_upload=50, rate_per_minute=3), Fake()) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c, + ): + + async def chunks(): + yield b"x" * 40000 + yield b"x" * 40000 + + assert (await c.post("/v1/ocr/jobs", headers=H, content=chunks())).status_code == 413 + r = await c.post("/v1/ocr/page", headers=H, files={"file": ("a.png", png)}) + assert r.status_code == 413 + assert ( + await c.post("/v1/chat/completions", headers=H, json={"private": "never echoed"}) + ).status_code == 422 + assert (await c.post("/v1/chat/completions", headers=H, json={})).status_code == 429 + + +async def test_no_token_and_remote_unavailable(png): + app = create_app(Settings(), Fake()) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c, + ): + assert (await c.get("/ready")).status_code == 503 + remote = RemoteCPU( + "https://worker.invalid", + "fake", + transport=httpx.MockTransport(lambda r: httpx.Response(503)), + ) + try: + assert not (await remote.health()).ready + finally: + await remote.close() + + +async def test_quality_evidence_and_handoff(png): + d = Dispatcher( + Fake(confidence=0.1), + Fake("gpu", text="Other public transcription"), + Settings(gpu_enabled=True), + ) + d.start() + try: + p = await d.page(RasterPage(0, png, 200, 120, "id"), OCROptions(sensitivity="D0")) + assert p.alternatives and "needs_review" in p.flags and "disagreement" in p.flags + assert p.attempt_count == 2 and p.alternatives[0].sensitivity == "D0" + r = OCRResult( + job_id="j", + request_id="r", + status="completed", + page_count=1, + sensitivity="D0", + markdown=p.markdown, + text=p.text, + pages=[p], + failures=[], + ) + h = anonymization_handoff(r, lambda text: {"text": text, "tags": []}) + assert h["anonymous"] is False and h["pages"][0]["page_id"] == "id" and p.text == r.text + with pytest.raises(SafeError): + anonymization_handoff(r, lambda text: "silently declassified") + finally: + await d.close() + + +async def test_timeout_and_result_capacity(png): + d = Dispatcher(Fake(delay=0.1), None, Settings(timeout=0.01)) + d.start() + try: + with pytest.raises(SafeError, match="backend_timeout"): + await d.page(RasterPage(0, png, 200, 120, "id"), OCROptions()) + assert len(d.cpu.backend.calls) == 2 + finally: + await d.close() + app = create_app(Settings(token="test", max_result_bytes=20), Fake()) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c, + ): + r = await c.post("/v1/ocr/jobs", headers=H, files={"file": ("a.png", png)}) + j = app.state.jobs.jobs[r.json()["job_id"]] + await j.task + assert j.status == "failed" and j.result is None + + +async def test_missing_models_readiness_and_output(png, tmp_path): + from ocr.adapters.rapidocr_cpu import RapidCPU + + cpu = RapidCPU(tmp_path) + with pytest.raises(SafeError, match="model_integrity_unavailable"): + await cpu.initialize() + assert not (await cpu.health()).ready + app = create_app(Settings(token="test"), cpu) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c, + ): + assert (await c.get("/ready", headers=H)).status_code == 503 + app = create_app(Settings(token="test"), Fake()) + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c, + ): + r = await c.post( + "/v1/ocr/jobs", + headers=H, + files={"file": ("a.png", png)}, + data={"options": '{"output":"text"}'}, + ) + id = r.json()["job_id"] + await app.state.jobs.jobs[id].task + result = (await c.get(f"/v1/ocr/jobs/{id}/result", headers=H)).json() + assert result["text"] and not result["markdown"] and result["pages"][0]["text"] diff --git a/ocr/tests/test_live.py b/ocr/tests/test_live.py new file mode 100644 index 0000000000000000000000000000000000000000..3d6e5f6b8e80312433fe6b852ce6c5029c7520bd --- /dev/null +++ b/ocr/tests/test_live.py @@ -0,0 +1,76 @@ +import os + +import pytest +from ocr.adapters.rapidocr_cpu import RapidCPU +from ocr.config import Settings +from ocr.fixtures.generate import raster +from ocr.ingest import encode, ingest +from ocr.routing import Dispatcher +from ocr.schemas import OCROptions + + +@pytest.mark.skipif(os.getenv("OCR_RUN_LIVE_CPU") != "1", reason="opt-in real CPU models") +async def test_real_cpu(): + s = Settings.from_env() + cpu = RapidCPU(s.models_dir) + await cpu.initialize() + d = Dispatcher(cpu, None, s) + d.start() + try: + page = ingest(encode(raster("Public OCR benchmark.")), "test.png", OCROptions(), s)[0] + r = await d.page(page, OCROptions(mode="cpu_only")) + assert "Public OCR benchmark" in r.text + assert r.blocks and r.engine == "pp-ocrv6-small" + finally: + await d.close() + + +@pytest.mark.skipif( + os.getenv("OCR_RUN_LIVE_ALIAS") != "1", + reason="requires approved D0 endpoint and credentials", +) +async def test_live_alias(): + from ocr.adapters.freellm_alias import FreeLLMAlias + + s = Settings.from_env() + a = FreeLLMAlias(s.freellm_url, s.freellm_key) + try: + report = await a.probe() + assert report["multimodal_verified"] + assert not report["d2_qualified"] + finally: + await a.close() + + +@pytest.mark.skipif(os.getenv("OCR_RUN_LIVE_CPU") != "1", reason="opt-in real CPU models") +async def test_real_api_image_and_pdf(tmp_path): + import io + + import httpx + from ocr.api import create_app + from ocr.fixtures.generate import raster + + s = Settings(token="synthetic", models_dir=os.environ["OCR_MODELS_DIR"]) + cpu = RapidCPU(s.models_dir) + await cpu.initialize() + app = create_app(s, cpu) + h = {"Authorization": "Bearer synthetic"} + async with ( + app.router.lifespan_context(app), + httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as c, + ): + im = raster("Public OCR benchmark.") + image = encode(im) + r = await c.post("/v1/ocr/page", headers=h, files={"file": ("public.png", image)}) + assert r.status_code == 200 and "Public OCR benchmark" in r.json()["text"] + pdf = io.BytesIO() + im.save(pdf, format="PDF", resolution=200) + r = await c.post("/v1/ocr/jobs", headers=h, files={"file": ("public.pdf", pdf.getvalue())}) + id = r.json()["job_id"] + await app.state.jobs.jobs[id].task + r = await c.get(f"/v1/ocr/jobs/{id}/result", headers=h) + assert r.status_code == 200 and r.json()["status"] == "completed" + assert ( + "Public OCR benchmark" in r.json()["text"] + and r.json()["pages"][0]["source"] == "cpu_ocr" + ) diff --git a/ocr/tests/test_private_host_adapters.py b/ocr/tests/test_private_host_adapters.py new file mode 100644 index 0000000000000000000000000000000000000000..60a955bdc2d9493c01dc4c553b9820b1148781d8 --- /dev/null +++ b/ocr/tests/test_private_host_adapters.py @@ -0,0 +1,216 @@ +import hashlib +import json + +import pytest +from ocr.anonymization.approval_ledger import SQLiteApprovalLedger +from ocr.anonymization.host_adapter import HostAuthorizationBridge +from ocr.anonymization.private_transport import LocalDestination, PrivateFileTransport +from ocr.schemas import SafeError +from ocr.tests.fixtures.isolated_host import IsolatedHostAdapter +from ocr.tests.test_anonymization_security import candidate +from pydantic import ValidationError + + +def setup(tmp_path): + value = candidate() + host = IsolatedHostAdapter(value) + bridge = HostAuthorizationBridge(host) + ledger = SQLiteApprovalLedger(tmp_path / "ledger.sqlite", initialize=True) + authority = bridge.authority(b"z" * 32, ledger, [host.policy.destination_id]) + approval = authority.approve( + value, + actor=host.credential, + review_id="fixture-review", + destination=host.policy.destination_id, + ) + return value, host, bridge, authority, approval + + +@pytest.mark.parametrize( + "failure", + [ + "requested_identity", + "boolean", + "expired_identity", + "revoked_review", + "expired_review", + "incomplete_review", + "revoked_session", + "destination_changed", + "candidate_missing", + "acl_revoked", + "destination_revoked", + ], +) +def test_real_adapter_contract_fails_closed_without_trusting_request_flags(tmp_path, failure): + value, host, bridge, authority, approval = setup(tmp_path) + actor = host.credential + if failure == "requested_identity": + actor = "fixture-subject" + elif failure == "boolean": + actor = True + elif failure == "expired_identity": + host.identity = host.identity.model_copy(update={"expires_at": 0}) + elif failure == "revoked_review": + host.revoked.add("fixture-review") + elif failure == "expired_review": + original = host.load_review + host.load_review = lambda key: original(key).model_copy(update={"expires_at": 0}) + elif failure == "incomplete_review": + original = host.load_review + host.load_review = lambda key: original(key).model_copy(update={"completed": False}) + elif failure == "revoked_session": + host.credential = object() + elif failure == "destination_changed": + host.destinations[host.policy.destination_id] = host.policy.model_copy( + update={"destination_class": "public"} + ) + elif failure == "candidate_missing": + host.candidates = frozenset() + elif failure == "acl_revoked": + host.allowed.clear() + else: + host.destinations.clear() + with pytest.raises(SafeError): + authority.export(value, approval, actor=actor, destination="private-fixture") + assert not host.production_ready + + +def test_review_evidence_is_immutable_and_candidate_bound(tmp_path): + value, host, bridge, authority, approval = setup(tmp_path) + with pytest.raises(TypeError): + host.records["new"] = host.records["fixture-review"] + with pytest.raises(ValidationError): + host.records["fixture-review"].attestation.candidate_digest = "b" * 64 + with pytest.raises(SafeError): + authority.approve( + candidate(), + actor=host.credential, + review_id="fixture-review", + destination="private-fixture", + ) + + +def test_local_delivery_requires_separate_current_authorization_and_retains_classification( + tmp_path, +): + value, host, bridge, authority, approval = setup(tmp_path) + sink = tmp_path / "sink" + sink.mkdir(mode=0o700) + transport = PrivateFileTransport( + [LocalDestination(host.policy, sink)], authorize_transfer=bridge.transfer_authorized + ) + exported = authority.export( + value, approval, actor=host.credential, destination=host.policy.destination_id + ) + with pytest.raises(SafeError): + transport.deliver( + exported, credential=host.credential, destination_id=host.policy.destination_id + ) + assert not list(sink.iterdir()) + # Explicit privileged fixture registration and separate transfer ACL. Neither + # happens on export in this library or automatically in a production service. + artifact = hashlib.sha256(exported.encode()).hexdigest() + host.register_export(exported) + host.allowed.add( + (host.identity.subject, "anonymization:private-transfer", "D2", host.policy.destination_id) + ) + result = transport.deliver( + exported, credential=host.credential, destination_id=host.policy.destination_id + ) + saved = sink / (artifact + ".json") + assert saved.stat().st_mode & 0o777 == 0o600 + assert json.loads(saved.read_text())["sensitivity"] == "D2" + assert json.loads(saved.read_text())["transmission_authorized"] is False + assert result["automated_release_eligible"] is False + with pytest.raises(SafeError): + transport.deliver( + exported, credential=host.credential, destination_id=host.policy.destination_id + ) + + +@pytest.mark.parametrize( + "attack", + [ + "url", + "redirect", + "public_fallback", + "substituted_directory", + "symlink", + "extra_raw_metadata", + "changed_bytes", + ], +) +def test_transport_prohibits_substitution_fallback_and_unregistered_artifacts(tmp_path, attack): + value, host, bridge, authority, approval = setup(tmp_path) + sink = tmp_path / "sink" + sink.mkdir(mode=0o700) + transport = PrivateFileTransport( + [LocalDestination(host.policy, sink)], authorize_transfer=bridge.transfer_authorized + ) + exported = authority.export( + value, approval, actor=host.credential, destination=host.policy.destination_id + ) + host.register_export(exported) + host.allowed.add( + (host.identity.subject, "anonymization:private-transfer", "D2", host.policy.destination_id) + ) + destination = host.policy.destination_id + if attack == "url": + destination = "https://external.invalid/upload" + elif attack == "public_fallback": + destination = "public-worker" + elif attack in ("redirect", "extra_raw_metadata"): + data = json.loads(exported) + data["location" if attack == "redirect" else "raw_original"] = "synthetic forbidden field" + exported = json.dumps(data) + elif attack == "changed_bytes": + exported = exported + " " + else: + sink.rename(tmp_path / "original") + if attack == "symlink": + sink.symlink_to(tmp_path / "original", target_is_directory=True) + else: + sink.mkdir(mode=0o700) + with pytest.raises(SafeError): + transport.deliver(exported, credential=host.credential, destination_id=destination) + assert not list( + (tmp_path / "original" if (tmp_path / "original").exists() else sink).glob("*.json") + ) + + +@pytest.mark.parametrize( + "change", + ["review_revoked", "export_revoked", "export_expired", "review_expired", "acl_revoked"], +) +def test_revocation_after_export_still_blocks_private_delivery(tmp_path, change): + value, host, bridge, authority, approval = setup(tmp_path) + sink = tmp_path / "sink" + sink.mkdir(mode=0o700) + exported = authority.export( + value, approval, actor=host.credential, destination=host.policy.destination_id + ) + artifact = hashlib.sha256(exported.encode()).hexdigest() + host.register_export(exported) + host.allowed.add( + (host.identity.subject, "anonymization:private-transfer", "D2", host.policy.destination_id) + ) + if change == "review_revoked": + host.revoked.add("fixture-review") + elif change == "export_revoked": + host.exports[artifact] = host.exports[artifact].model_copy(update={"revoked": True}) + elif change == "export_expired": + host.exports[artifact] = host.exports[artifact].model_copy(update={"expires_at": 0}) + elif change == "review_expired": + previous = host.load_review + host.load_review = lambda value: previous(value).model_copy(update={"expires_at": 0}) + else: + host.allowed.clear() + transport = PrivateFileTransport( + [LocalDestination(host.policy, sink)], authorize_transfer=bridge.transfer_authorized + ) + with pytest.raises(SafeError): + transport.deliver( + exported, credential=host.credential, destination_id=host.policy.destination_id + ) + assert not list(sink.iterdir()) diff --git a/ocr/tests/test_quality.py b/ocr/tests/test_quality.py new file mode 100644 index 0000000000000000000000000000000000000000..02a3f5fc1609dc5afc0541599110c710cc7bf7ee --- /dev/null +++ b/ocr/tests/test_quality.py @@ -0,0 +1,24 @@ +from ocr.classify import features +from ocr.fixtures.generate import raster +from ocr.ingest import encode +from ocr.quality import quality +from ocr.schemas import OCRPageResult, RasterPage + + +def test_blur_proxy_and_blank(): + def page(style): + data = encode(raster("Public printed fixture text.", style=style)) + return RasterPage(0, data, 1400, 860, "id", features=features(data)) + + p = page("blur") + r = OCRPageResult( + page_index=0, + source="cpu_ocr", + text="Public printed fixture text.", + markdown="text", + image_width=p.width, + image_height=p.height, + ) + assert "low_contrast_or_blur" in quality(r, p) + assert "empty_nonblank" in quality(r.model_copy(update={"text": ""}), p) + assert "low_contrast_or_blur" not in quality(r, page("clean")) diff --git a/ocr/tests/test_religious_terminology.py b/ocr/tests/test_religious_terminology.py new file mode 100644 index 0000000000000000000000000000000000000000..1bde9c04ed5c9e6b47767aaf95117a2b456adedc --- /dev/null +++ b/ocr/tests/test_religious_terminology.py @@ -0,0 +1,185 @@ +"""Independent unit fixtures for lexical protection, never attribution to people.""" + +import pytest +from ocr.anonymization.replacement import replace, validate_entities +from ocr.anonymization.terminology import ReligiousTerminology +from ocr.schemas import SafeError + + +@pytest.mark.parametrize( + "text,terms,expected", + [ + ("Status: religionslos.", ("religionslos",), ["religionslos"]), + ("Angabe: OHNE KONFESSION", ("ohne Konfession",), ["OHNE KONFESSION"]), + ( + "No religious affiliation was stated.", + ("no religious affiliation",), + ["No religious affiliation"], + ), + ("A lecture on Taoism, not a personal assertion.", ("Taoism",), ["Taoism"]), + ("She denied being a Jain.", ("Jain",), ["Jain"]), + ('He quoted "Zoroastrianism".', ("Zoroastrianism",), ["Zoroastrianism"]), + ("Préface: Straße / STRASSE.", ("Straße",), ["Straße", "STRASSE"]), + ("💡 A Bahá'í reference.", ("Bahá'í",), ["Bahá'í"]), + ("An agnos-\ntisch entry.", ("agnostisch",), ["agnos-\ntisch"]), + ("Hinduism Hinduism", ("Hinduism",), ["Hinduism", "Hinduism"]), + ("Islamabad Islamismus preIslam Islam_suffix Islam\u0301", ("Islam",), []), + ("orthodox sorting; catholic taste; evangelical marketing", ("Islam",), []), + ("Buddh1sm is an OCR error", ("Buddhism",), []), + ], +) +def test_literal_offsets_and_no_attribution(text, terms, expected): + found = ReligiousTerminology(terms).detect(text) + assert [p.text for p in sorted(found, key=lambda p: p.start)] == expected + validate_entities(text, found) + assert all(p.score is None and p.source == "deterministic" for p in found) + assert all(p.label == "RELIGION" for p in found) + assert all( + set(p.model_dump()) + == {"start", "end", "label", "text", "score", "source", "model_revision", "normalization"} + for p in found + ) + sanitized, _ = replace(text, found) + assert sanitized.count("[RELIGION]") == len(expected) + + +@pytest.mark.parametrize( + "terms", + [(), [], [""], [" x"], ["a.*"], ["a\nb"], ["a" * 81], [None], ["a"] * 257, "Islam", {"Islam"}], +) +def test_reject_unbounded_or_nonliteral_configuration(terms): + with pytest.raises(SafeError, match="invalid_terminology_configuration"): + ReligiousTerminology(terms) + + +def test_casefold_never_matches_partial_expansion(): + assert not ReligiousTerminology(("s",)).detect("ß") + + +def test_length_limit_and_default_is_not_affiliation_inference(): + detector = ReligiousTerminology() + with pytest.raises(SafeError, match="anonymization_text_limit"): + detector.detect("x" * 2_000_001) + found = detector.detect("A report about Sikhism says nothing about the attendee.") + assert [p.text for p in found] == ["Sikhism"] + assert not detector.detect("An orthodox algorithm handles an unaffiliated candidate.") + + +def test_longest_containing_phrase_preserves_offsets_and_has_no_fake_score(): + found = ReligiousTerminology(("no religion", "religion")).detect("Reported: no religion.") + out, tags = replace("Reported: no religion.", found) + assert out == "Reported: [RELIGION]." and len(tags) == 1 + assert tags[0].score is None and tags[0].model_revision == "religious-terminology-v1" + + +def test_engine_keeps_d2_and_review_flags_with_only_terminology_evidence(): + from ocr.anonymization.engine import AnonymizationEngine + from ocr.anonymization.model import MODEL_REVISION + from ocr.anonymization.schemas import LABELS + from ocr.tests.test_anonymization_security import evidence + + class LocalEmpty: + from ocr.anonymization.recognizer import DEFAULT_PROMPTS + + threshold, prompts = 0.5, DEFAULT_PROMPTS.copy() + local, revision, categories = True, MODEL_REVISION, LABELS + + def recognize(self, text): + return [] + + candidate = AnonymizationEngine(LocalEmpty()).handoff(evidence("D2", ("The topic is Taoism.",))) + assert candidate.pages[0].anonymized_text == "The topic is [RELIGION]." + assert candidate.sensitivity == candidate.pages[0].original_sensitivity == "D2" + assert not candidate.anonymous and not candidate.automated_release_eligible + assert candidate.release_review_required and not candidate.release_approved + assert "empty_prediction" in candidate.review_reasons + + +def test_terminology_does_not_hide_model_failure_or_overlap(): + from ocr.anonymization.engine import AnonymizationEngine + from ocr.anonymization.model import MODEL_REVISION + from ocr.anonymization.schemas import LABELS, Entity + + class Broken: + local, revision, categories = True, MODEL_REVISION, LABELS + + def recognize(self, text): + raise RuntimeError("private payload in model exception") + + with pytest.raises(SafeError, match="^anonymization_failed$"): + AnonymizationEngine(Broken()).transform("Taoism") + + class Conflict(Broken): + def recognize(self, text): + return [ + Entity( + start=0, + end=6, + text=text, + label="PERSON", + score=0.8, + source="multilingual_ner", + model_revision=MODEL_REVISION, + ) + ] + + with pytest.raises(SafeError, match="ambiguous_entity_overlap"): + AnonymizationEngine(Conflict()).transform("Taoism") + + +def test_custom_operator_terminology_uses_same_provenance_validation(): + from ocr.anonymization.detectors import detect + + found = detect("A Rastafari topic.", terminology=ReligiousTerminology(("Rastafari",))) + assert len(found) == 1 and found[0].text == "Rastafari" + assert not detect("Rastafari") + + +def test_historical_frozen_holdout_is_unchanged_well_formed_and_complete(): + import json + from pathlib import Path + + from ocr.anonymization.historical import verify_historical + + root, freeze, cases, _ = verify_historical() + assert len(cases) >= 150 and freeze["evaluation_run_before_freeze"] is False + assert sum(c["language"] == "de" for c in cases) == 80 + assert sum(c["language"] == "en" for c in cases) == 80 + assert len({c["scenario"] for c in cases}) == 10 + oldroot = Path(__file__).parents[1] / "anonymization/fixtures" + oldtexts = { + c["text"] + for p in ( + oldroot / "qualification.json", + oldroot / "validation.json", + oldroot / "holdout03c/cases.json", + ) + for c in json.loads(p.read_bytes()) + } + assert not oldtexts.intersection(c["text"] for c in cases) + for case in cases: + reconstructed = case["text"] + for span in reversed(case["spans"]): + assert case["text"][span["start"] : span["end"]] == span["text"] + reconstructed = ( + reconstructed[: span["start"]] + + "[" + + span["label"] + + "]" + + reconstructed[span["end"] :] + ) + assert reconstructed == case["expected"] + assert root.name == "holdout03d" + + +def test_existing_first_result_refuses_before_initializing_model(tmp_path, monkeypatch): + from ocr.anonymization import qualify03d + + target = tmp_path / "immutable.json" + target.write_bytes(b"first result") + monkeypatch.setattr( + qualify03d, "GLiNERRecognizer", lambda *a, **k: pytest.fail("must not infer twice") + ) + with pytest.raises(FileExistsError, match="first_result_already_exists"): + qualify03d.run("unused-cache", target) + assert target.read_bytes() == b"first result" diff --git a/ocr/tests/test_reviewed_corrections.py b/ocr/tests/test_reviewed_corrections.py new file mode 100644 index 0000000000000000000000000000000000000000..34b6d1374f2d9a26d283968edf9efa89f097bebe --- /dev/null +++ b/ocr/tests/test_reviewed_corrections.py @@ -0,0 +1,619 @@ +"""Source-bound human edits and actual approval/ledger regressions, synthetic only.""" + +import json +import time +from concurrent.futures import ThreadPoolExecutor +from threading import Barrier + +import pytest +from ocr.anonymization.approval_ledger import SQLiteApprovalLedger +from ocr.anonymization.candidate_io import parse_candidate +from ocr.anonymization.corrections import ( + CorrectionEdit, + CorrectionService, + author_digest, + commitment, + prepare_review, +) +from ocr.anonymization.engine import AnonymizationEngine +from ocr.anonymization.host_adapter import HostAuthorizationBridge +from ocr.anonymization.private_review import PrivateReviewBridge +from ocr.anonymization.release_policy import digest +from ocr.anonymization.schemas import CorrectedCandidate, Entity +from ocr.schemas import SafeError +from ocr.tests.fixtures.correction_host import CorrectionHostFixture +from ocr.tests.test_anonymization_security import StubRecognizer, evidence + +TEXT = "PrivateSyntheticName works at Silver Lantern Archives." + + +class Recognizer(StubRecognizer): + def __init__(self, specs): + self.specs = specs + + def recognize(self, text): + return [ + Entity( + start=text.index(word), + end=text.index(word) + len(word), + text=word, + label=label, + score=0.9, + source="multilingual_ner", + model_revision=self.revision, + ) + for word, label in self.specs + ] + + +def host_fixture( + *, + specs=(("PrivateSyntheticName", "PERSON"), ("Silver Lantern", "ORGANIZATION")), + text=TEXT, + level="D2", + job="original-doc", + separate=True, +): + result = evidence(level, (text,)) + result.job_id = job + context = prepare_review( + AnonymizationEngine(Recognizer(specs)), + result, + review_ref="original-review", + environment_id="synthetic-room", + expires_at=int(time.time()) + 600, + ) + return CorrectionHostFixture(context, {0: text}, separate_approver=separate) + + +def edit_for(host, word, category, reason, *, reference="original-review", action="mask", page=0): + service = CorrectionService(host) + credential = host.author_credential + context = service.inspect(credential, reference) + text = PrivateReviewBridge(host).source_page(credential, reference, page).text + start = text.index(word) + end = start + len(word) + span = service.bind(credential, reference, page, start, end) + targets = tuple( + sorted( + { + o.observation_id + for o in context.effective + if o.page_index == page + and o.entity.start < end + and start < o.entity.end + and (action == "mask" or o.entity.label == category) + } + ) + ) + return CorrectionEdit( + span=span, + category=category, + action=action, + rationale=reason, + target_observation_ids=targets, + ) + + +def apply_one(host, edit, *, reference="original-review"): + service = CorrectionService(host) + proposal = service.propose(host.author_credential, reference, (edit,)) + receipt = service.apply(host.author_credential, proposal.correction_id) + return proposal, receipt, host.candidates[receipt.candidate_digest] + + +@pytest.mark.parametrize("case", ["missing", "boundary", "category", "conflict", "false_positive"]) +def test_all_five_supported_corrections(case): + options = { + "missing": dict(specs=(), text="PrivateSyntheticName"), + "boundary": dict( + specs=(("Silver Lantern", "ORGANIZATION"),), text="Silver Lantern Archives" + ), + "category": dict(specs=(("Greystone", "PERSON"),), text="Greystone"), + "conflict": dict(specs=(("Greystone", "PERSON"), ("Greystone", "CITY")), text="Greystone"), + "false_positive": dict(specs=(("weather", "PERSON"),), text="weather"), + } + host = host_fixture(**options[case]) + old = host.contexts["original-review"] + snapshot = old.model_dump_json() + word, category, reason, action = { + "missing": ("PrivateSyntheticName", "PERSON", "missing_entity", "mask"), + "boundary": ("Silver Lantern Archives", "ORGANIZATION", "incomplete_boundary", "mask"), + "category": ("Greystone", "CITY", "wrong_category", "mask"), + "conflict": ("Greystone", "CITY", "span_conflict", "mask"), + "false_positive": ("weather", "PERSON", "false_positive", "suppress"), + }[case] + proposal, receipt, value = apply_one( + host, edit_for(host, word, category, reason, action=action) + ) + assert type(value) is CorrectedCandidate and parse_candidate(value.model_dump_json()) == value + assert value.pages[0].anonymized_text == ( + "weather" if case == "false_positive" else "[" + category + "]" + ) + assert ( + not value.anonymous and not value.release_approved and not value.automated_release_eligible + ) + assert ( + value.release_review_required and "human_correction_requires_review" in value.review_reasons + ) + assert value.correction.proposal_digest == commitment(proposal) + assert value.provenance == old.machine.provenance + assert host.contexts["original-review"].model_dump_json() == snapshot + assert host.contexts[receipt.review_ref].machine == old.machine + assert value.correction.prior_candidate_digest == old.material.candidate_digest + assert value.correction.author_subject_digests == (author_digest("synthetic-author"),) + assert "PrivateSyntheticName" not in value.model_dump_json() + assert "PrivateSyntheticName" not in proposal.model_dump_json() + assert "PrivateSyntheticName" not in repr(host.contexts[receipt.review_ref]) + if case != "false_positive": + assert all(t.source == "human_correction" and t.score is None for t in value.pages[0].tags) + if old.material.candidate is not None: + assert ( + value.candidate_id != old.material.candidate.candidate_id + and digest(value) != old.material.candidate_digest + ) + assert old.material.candidate_digest in host.candidates + else: + with pytest.raises(SafeError, match="invalid_release_candidate"): + digest(old.material.candidate) + + +@pytest.mark.parametrize("operation", ["inspect", "submit", "bind", "inspect_proposal", "apply"]) +def test_no_correction_permission_even_with_review_and_release_roles(operation): + host = host_fixture() + service = CorrectionService(host) + edit = edit_for(host, "Silver Lantern Archives", "ORGANIZATION", "incomplete_boundary") + proposal = service.propose(host.author_credential, "original-review", (edit,)) + host.context_reads = host.proposal_reads = 0 + credential = host.approver_credential + with pytest.raises(SafeError, match="^private_correction_denied$"): + if operation == "inspect": + service.inspect(credential, "original-review") + if operation == "submit": + service.propose(credential, "original-review", (edit,)) + if operation == "bind": + service.bind(credential, "original-review", 0, 0, 4) + if operation == "inspect_proposal": + service.inspect_proposal(credential, proposal.correction_id) + if operation == "apply": + service.apply(credential, proposal.correction_id) + assert host.context_reads == host.proposal_reads == 0 + + +@pytest.mark.parametrize( + "credential", + [None, False, {"subject": "synthetic-author", "approved": True}, "synthetic-author"], +) +def test_no_request_supplied_identity_or_approval_flag(credential): + host = host_fixture() + with pytest.raises(SafeError, match="^private_correction_denied$"): + CorrectionService(host).inspect(credential, "original-review") + + +@pytest.mark.parametrize( + "field,value", + [ + ("evidence_digest", "f" * 64), + ("context_digest", "f" * 64), + ("machine_snapshot_digest", "f" * 64), + ("prior_candidate_digest", "f" * 64), + ("author_subject", "another-author"), + ("expires_at", 1), + ], +) +def test_stale_or_inconsistent_proposal_rejected(field, value): + host = host_fixture() + service = CorrectionService(host) + proposal = service.propose( + host.author_credential, + "original-review", + (edit_for(host, "Silver Lantern Archives", "ORGANIZATION", "incomplete_boundary"),), + ) + record = host.proposals[proposal.correction_id] + host.proposals[proposal.correction_id] = record.model_copy( + update={"proposal": proposal.model_copy(update={field: value})} + ) + with pytest.raises(SafeError, match="^private_correction_denied$"): + service.apply(host.author_credential, proposal.correction_id) + assert len(host.contexts) == 1 + + +@pytest.mark.parametrize( + "change", ["start", "end", "digest", "page", "raw_text", "extra_replacement", "wrong_target"] +) +def test_incorrect_original_offsets_and_malformed_edits_are_sanitized(change): + host = host_fixture() + edit = edit_for(host, "Silver Lantern Archives", "ORGANIZATION", "incomplete_boundary") + data = edit.model_dump() + if change == "start": + data["span"]["start"] -= 1 + if change == "end": + data["span"]["end"] += 1 + if change == "digest": + data["span"]["source_span_digest"] = "f" * 64 + if change == "page": + data["span"]["page_index"] = 3 + if change == "raw_text": + data["span"]["text"] = "PrivateSyntheticName" + if change == "extra_replacement": + data["replacement_text"] = "PrivateSyntheticName" + if change == "wrong_target": + data["target_observation_ids"] = ("f" * 64,) + with pytest.raises(SafeError, match="^private_correction_denied$") as error: + CorrectionService(host).propose(host.author_credential, "original-review", (data,)) + assert "PrivateSyntheticName" not in str(error.value) + assert not host.proposals and len(host.contexts) == 1 + + +def test_cannot_reuse_span_on_other_document_even_same_text(): + one = host_fixture(job="document-one") + two = host_fixture(job="document-two") + edit = edit_for(one, "Silver Lantern Archives", "ORGANIZATION", "incomplete_boundary") + assert ( + one.contexts["original-review"].machine.evidence_digest + != two.contexts["original-review"].machine.evidence_digest + ) + with pytest.raises(SafeError, match="^private_correction_denied$"): + CorrectionService(two).propose(two.author_credential, "original-review", (edit,)) + + +@pytest.mark.parametrize( + "mode", + [ + "overlapping_edits", + "unacknowledged_prediction", + "conflict_not_all_targets", + "unknown_targets", + ], +) +def test_conflicting_or_silent_overrides_fail_closed(mode): + host = host_fixture(specs=(("Greystone", "PERSON"), ("Greystone", "CITY")), text="Greystone") + edit = edit_for(host, "Greystone", "CITY", "span_conflict") + edits = (edit,) + if mode == "overlapping_edits": + edits = (edit, edit.model_copy(update={"category": "PERSON"})) + if mode == "unacknowledged_prediction": + edits = ( + edit.model_copy(update={"target_observation_ids": (), "rationale": "missing_entity"}), + ) + if mode == "conflict_not_all_targets": + edits = ( + edit.model_copy(update={"target_observation_ids": edit.target_observation_ids[:1]}), + ) + if mode == "unknown_targets": + edits = (edit.model_copy(update={"target_observation_ids": ("f" * 64,)}),) + with pytest.raises(SafeError, match="^private_correction_denied$"): + CorrectionService(host).propose(host.author_credential, "original-review", edits) + assert not host.proposals and not host.candidates + + +@pytest.mark.parametrize("level", ["D1", "D2"]) +def test_approval_cannot_follow_correction_and_old_history_remains_valid(tmp_path, level): + host = host_fixture(level=level) + bridge = HostAuthorizationBridge(host) + ledger = SQLiteApprovalLedger(tmp_path / "ledger.sqlite", initialize=True) + authority = bridge.authority(b"x" * 32, ledger, ("synthetic-private",)) + old = host.contexts["original-review"].material.candidate + review_id = host.complete_review(host.author_credential, digest(old)) + oldapproval = authority.approve( + old, actor=host.author_credential, review_id=review_id, destination="synthetic-private" + ) + proposal, receipt, corrected = apply_one( + host, edit_for(host, "Silver Lantern Archives", "ORGANIZATION", "incomplete_boundary") + ) + assert corrected.sensitivity == corrected.pages[0].original_sensitivity == level + for approval in (None, oldapproval): + with pytest.raises(SafeError, match="release_not_approved"): + authority.export( + corrected, approval, actor=host.author_credential, destination="synthetic-private" + ) + output = json.loads( + authority.export( + old, oldapproval, actor=host.author_credential, destination="synthetic-private" + ) + ) + assert output["candidate_digest"] == digest(old) and not output["transmission_authorized"] + with pytest.raises(SafeError, match="release_approval_consumed_or_unavailable"): + authority.export( + old, oldapproval, actor=host.author_credential, destination="synthetic-private" + ) + author_review = host.complete_review(host.author_credential, receipt.candidate_digest) + with pytest.raises(SafeError, match="release_not_approved"): + authority.approve( + corrected, + actor=host.author_credential, + review_id=author_review, + destination="synthetic-private", + ) + final_review = host.complete_review(host.approver_credential, receipt.candidate_digest) + approval = authority.approve( + corrected, + actor=host.approver_credential, + review_id=final_review, + destination="synthetic-private", + ) + released = json.loads( + authority.export( + corrected, approval, actor=host.approver_credential, destination="synthetic-private" + ) + ) + assert ( + released["sensitivity"] == level + and not released["anonymous"] + and not released["transmission_authorized"] + ) + with pytest.raises(SafeError, match="release_approval_consumed_or_unavailable"): + authority.export( + corrected, approval, actor=host.approver_credential, destination="synthetic-private" + ) + + +def test_missing_explicit_corrected_release_policy_denies(tmp_path): + host = host_fixture() + _, receipt, value = apply_one( + host, edit_for(host, "Silver Lantern Archives", "ORGANIZATION", "incomplete_boundary") + ) + host.corrected_release_allowed = None + review_id = host.complete_review(host.approver_credential, receipt.candidate_digest) + authority = HostAuthorizationBridge(host).authority( + b"x" * 32, + SQLiteApprovalLedger(tmp_path / "ledger.sqlite", initialize=True), + ("synthetic-private",), + ) + with pytest.raises(SafeError, match="release_not_approved"): + authority.approve( + value, + actor=host.approver_credential, + review_id=review_id, + destination="synthetic-private", + ) + + +@pytest.mark.parametrize( + "mode", + ["revoked_proposal", "revoked_view", "expired_session", "acl_at_commit", "missing_storage"], +) +def test_current_permission_and_storage_rechecked_at_publication(mode): + host = host_fixture() + service = CorrectionService(host) + proposal = service.propose( + host.author_credential, + "original-review", + (edit_for(host, "Silver Lantern Archives", "ORGANIZATION", "incomplete_boundary"),), + ) + if mode == "revoked_proposal": + host.proposals[proposal.correction_id] = host.proposals[proposal.correction_id].model_copy( + update={"revoked": True} + ) + if mode == "revoked_view": + host.bindings["original-review"] = host.bindings["original-review"].model_copy( + update={"revoked": True} + ) + if mode == "expired_session": + host.identities[host.author_credential] = host.identities[ + host.author_credential + ].model_copy(update={"expires_at": 0}) + if mode == "acl_at_commit": + host.before_publish = lambda: host.allowed.clear() + if mode == "missing_storage": + host.publish_correction = lambda *a: False + with pytest.raises(SafeError, match="^private_correction_denied$"): + service.apply(host.author_credential, proposal.correction_id) + assert len(host.contexts) == 1 + + +def test_simultaneous_apply_and_proposal_replay_publish_once(): + host = host_fixture() + service = CorrectionService(host) + proposal = service.propose( + host.author_credential, + "original-review", + (edit_for(host, "Silver Lantern Archives", "ORGANIZATION", "incomplete_boundary"),), + ) + barrier = Barrier(2) + + def worker(): + barrier.wait() + try: + return service.apply(host.author_credential, proposal.correction_id) + except SafeError: + return None + + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(lambda _: worker(), range(2))) + assert sum(r is not None for r in results) == 1 and len(host.contexts) == 2 + with pytest.raises(SafeError, match="^private_correction_denied$"): + service.apply(host.author_credential, proposal.correction_id) + + +def test_second_correction_uses_original_offsets_and_retains_chain(): + host = host_fixture(specs=(), text="FirstSyntheticName meets SecondSyntheticName.") + first, published, old = apply_one( + host, edit_for(host, "FirstSyntheticName", "PERSON", "missing_entity") + ) + second, receipt, new = apply_one( + host, + edit_for( + host, "SecondSyntheticName", "PERSON", "missing_entity", reference=published.review_ref + ), + reference=published.review_ref, + ) + assert new.pages[0].anonymized_text == "[PERSON] meets [PERSON]." + assert second.edits[0].span.start == len("FirstSyntheticName meets ") + assert new.correction.prior_candidate_digest == digest(old) + assert host.contexts[receipt.review_ref].machine == host.contexts["original-review"].machine + assert len(host.contexts) == 3 and len(host.proposals) == 2 + assert CorrectionService(host).inspect_proposal( + host.author_credential, first.correction_id + ).record.published_candidate_digest == digest(old) + + +def test_stale_parallel_proposal_cannot_discard_a_newer_correction(): + host = host_fixture() + service = CorrectionService(host) + edit = edit_for(host, "Silver Lantern Archives", "ORGANIZATION", "incomplete_boundary") + first = service.propose(host.author_credential, "original-review", (edit,)) + stale = service.propose(host.author_credential, "original-review", (edit,)) + service.apply(host.author_credential, first.correction_id) + with pytest.raises(SafeError, match="^private_correction_denied$"): + service.apply(host.author_credential, stale.correction_id) + assert len(host.contexts) == 2 + + +def test_lost_publication_reply_retains_immutable_recovery_pointer(): + host = host_fixture() + service = CorrectionService(host) + proposal = service.propose( + host.author_credential, + "original-review", + (edit_for(host, "Silver Lantern Archives", "ORGANIZATION", "incomplete_boundary"),), + ) + original = host.publish_correction + + def lost(*args): + original(*args) + raise OSError("SYNTHETIC_PRIVATE_WRITE_ERROR") + + host.publish_correction = lost + with pytest.raises(SafeError, match="^private_correction_denied$"): + service.apply(host.author_credential, proposal.correction_id) + view = service.inspect_proposal(host.author_credential, proposal.correction_id) + assert view.record.published_candidate_digest in host.candidates + assert view.record.published_review_ref in host.contexts + assert len(host.contexts) == 2 + + +def test_unnecessary_empty_correction_has_no_new_candidate_or_approval(): + host = host_fixture() + before = len(host.candidates) + with pytest.raises(SafeError, match="^private_correction_denied$"): + CorrectionService(host).propose(host.author_credential, "original-review", ()) + assert len(host.candidates) == before and not host.proposals + + +def test_runtime_failure_is_not_a_correctable_conflict(): + class Broken(Recognizer): + def recognize(self, text): + raise RuntimeError("PrivateSyntheticName") + + with pytest.raises(SafeError, match="^correction_capture_failed$"): + prepare_review( + AnonymizationEngine(Broken(())), + evidence("D2", (TEXT,)), + review_ref="original-review", + environment_id="synthetic-room", + expires_at=int(time.time()) + 600, + ) + + +def test_empty_prediction_warning_survives_manual_correction(): + host = host_fixture(specs=(), text="PrivateSyntheticName") + _, _, value = apply_one( + host, edit_for(host, "PrivateSyntheticName", "PERSON", "missing_entity") + ) + assert "empty_prediction" in value.review_reasons + + +def test_corrected_policy_revocation_is_checked_again_at_export(tmp_path): + host = host_fixture() + _, receipt, value = apply_one( + host, edit_for(host, "Silver Lantern Archives", "ORGANIZATION", "incomplete_boundary") + ) + authority = HostAuthorizationBridge(host).authority( + b"x" * 32, + SQLiteApprovalLedger(tmp_path / "ledger.sqlite", initialize=True), + ("synthetic-private",), + ) + review = host.complete_review(host.approver_credential, receipt.candidate_digest) + approval = authority.approve( + value, actor=host.approver_credential, review_id=review, destination="synthetic-private" + ) + host.corrected_release_allowed = lambda *args: False + with pytest.raises(SafeError, match="^release_not_approved$"): + authority.export( + value, approval, actor=host.approver_credential, destination="synthetic-private" + ) + + +@pytest.mark.parametrize("mode", ["classification", "malformed", "source", "observation"]) +def test_corrupt_private_records_fail_sanitized(mode, caplog): + host = host_fixture() + context = host.contexts["original-review"] + edit = edit_for(host, "Silver Lantern Archives", "ORGANIZATION", "incomplete_boundary") + if mode == "malformed": + host.load_correction_context = lambda ref: {"PrivateSyntheticName": "secret"} + if mode == "classification": + host.contexts["original-review"] = context.model_copy( + update={ + "material": context.material.model_copy( + update={ + "binding": context.material.binding.model_copy(update={"sensitivity": "D0"}) + } + ) + } + ) + if mode == "source": + host.source = {k: b"PrivateSyntheticName" for k in host.source} + if mode == "observation": + bad = context.machine.observations[0].model_copy(update={"observation_id": "f" * 64}) + host.contexts["original-review"] = context.model_copy( + update={ + "machine": context.machine.model_copy( + update={"observations": (bad,) + context.machine.observations[1:]} + ) + } + ) + with pytest.raises(SafeError, match="^private_correction_denied$") as caught: + service = CorrectionService(host) + service.propose(host.author_credential, "original-review", (edit,)) + assert "PrivateSyntheticName" not in str(caught.value) + caplog.text + assert len(host.contexts) == 1 and not host.proposals + + +@pytest.mark.parametrize("first_page", [0, 1]) +def test_unicode_multipage_corrections_use_original_codepoints(first_page): + texts = ("🧭 Grüße SyntheticAlpha.", "漢字 SecondSyntheticName.") + result = evidence("D2", texts) + for index, page in enumerate(result.pages, first_page): + page.page_index = index + context = prepare_review( + AnonymizationEngine(Recognizer(())), + result, + review_ref="original-review", + environment_id="synthetic-room", + expires_at=int(time.time()) + 600, + ) + host = CorrectionHostFixture( + context, dict(enumerate(texts, first_page)), separate_approver=True + ) + service = CorrectionService(host) + edits = tuple( + edit_for(host, word, "PERSON", "missing_entity", page=page) + for page, word in enumerate(("SyntheticAlpha", "SecondSyntheticName"), first_page) + ) + proposal = service.propose(host.author_credential, "original-review", edits) + receipt = service.apply(host.author_credential, proposal.correction_id) + candidate = host.candidates[receipt.candidate_digest] + assert tuple(p.anonymized_text for p in candidate.pages) == ( + "🧭 Grüße [PERSON].", + "漢字 [PERSON].", + ) + for edit, page, text in zip(edits, candidate.pages, texts, strict=True): + assert edit.span.start == page.tags[0].start + assert len(text[: edit.span.start].encode()) > edit.span.start + + +@pytest.mark.parametrize("version", ["v1", "v2"]) +def test_human_tags_cannot_be_disguised_as_legacy_candidate(version): + host = host_fixture() + _, _, corrected = apply_one( + host, edit_for(host, "Silver Lantern Archives", "ORGANIZATION", "incomplete_boundary") + ) + payload = corrected.model_dump(mode="json") + payload["schema_version"] = "ocr-anonymization-candidate/" + version + del payload["correction"] + payload["review_reasons"] = ["recognition_incomplete_possible"] + if version == "v1": + del payload["provenance"] + with pytest.raises(SafeError, match="^unsupported_or_invalid_candidate$"): + parse_candidate(json.dumps(payload)) diff --git a/ocr/tests/test_security_routing.py b/ocr/tests/test_security_routing.py new file mode 100644 index 0000000000000000000000000000000000000000..bf2262d72f307632e9b36f6015df62e512a02e6c --- /dev/null +++ b/ocr/tests/test_security_routing.py @@ -0,0 +1,229 @@ +import asyncio + +import pytest +from conftest import Fake +from ocr.config import Settings +from ocr.routing import Dispatcher +from ocr.schemas import OCROptions, RasterPage, SafeError +from ocr.security import authorize, sensitivity + + +@pytest.mark.parametrize("level", ["D0", "D1", "D2"]) +@pytest.mark.parametrize("name,external", [("cpu", False), ("cpu", True), ("gpu", True)]) +async def test_security_matrix(level, name, external): + b = Fake(name, external=external) + opts = OCROptions(sensitivity=level) + settings = Settings(gpu_enabled=True) + if external and level != "D0": + with pytest.raises(SafeError, match="no_authorized_destination"): + authorize(b, opts, settings, await b.capabilities()) + assert not b.calls + else: + authorize(b, opts, settings, await b.capabilities()) + + +async def test_d2_hidden_substitution_blocked(png): + cpu = Fake("cpu", external=True) + gpu = Fake("gpu", external=True) + d = Dispatcher(cpu, gpu, Settings(gpu_enabled=True, external_cpu_classes=("D0", "D2"))) + d.start() + try: + with pytest.raises(SafeError, match="no_authorized_destination"): + await d.page( + RasterPage(0, png, 200, 120, "id"), + OCROptions(mode="gpu_only", sensitivity="D2"), + ) + assert not gpu.calls and not cpu.calls + finally: + await d.close() + + +async def test_low_confidence_escalation(png): + cpu = Fake(confidence=0.2) + gpu = Fake("gpu", text="Alternate transcription") + d = Dispatcher(cpu, gpu, Settings(gpu_enabled=True)) + d.start() + try: + result = await d.page(RasterPage(0, png, 200, 120, "id"), OCROptions(sensitivity="D0")) + assert ( + gpu.calls + and result.alternatives + and "needs_review" in result.flags + or result.alternatives[0].flags + ) + assert result.attempt_count == 2 + finally: + await d.close() + + +@pytest.mark.parametrize("mode", ["cpu_only", "gpu_only"]) +async def test_explicit_no_fallback(png, mode): + cpu = Fake(fail=SafeError("failure")) + gpu = Fake("gpu", fail=SafeError("failure")) + d = Dispatcher(cpu, gpu, Settings(gpu_enabled=True)) + d.start() + try: + with pytest.raises(SafeError): + await d.page( + RasterPage(0, png, 200, 120, "id"), + OCROptions(mode=mode, sensitivity="D0"), + ) + assert bool(cpu.calls) == (mode == "cpu_only") + assert bool(gpu.calls) == (mode == "gpu_only") + finally: + await d.close() + + +async def test_retry_policy_rechecked(png): + cpu = Fake(external=True) + + def fail(): + cpu.external = True + return SafeError("backend_http_503", True) + + cpu.fail = fail + settings = Settings() + d = Dispatcher(cpu, None, settings) + d.start() + try: + task = asyncio.create_task( + d.page(RasterPage(0, png, 200, 120, "id"), OCROptions(sensitivity="D0")) + ) + await asyncio.sleep(0.04) + settings.external_cpu_classes = () + with pytest.raises(SafeError): + await task + assert len(cpu.calls) == 1 + finally: + await d.close() + + +async def test_fallback_and_throughput_gates(png): + cpu = Fake(fail=SafeError("cpu_unavailable")) + gpu = Fake("gpu") + d = Dispatcher(cpu, gpu, Settings(gpu_enabled=True)) + d.start() + try: + r = await d.page(RasterPage(0, png, 200, 120, "id"), OCROptions(sensitivity="D0")) + assert r.source == "alias_ocr" and "fallback_used" in r.flags + cpu.fail = None + cpu.calls.clear() + gpu.calls.clear() + d.cpu.ewma = 1000 + d.gpu.ewma = 10 + opts = OCROptions(mode="throughput", allow_gpu_overflow=True, sensitivity="D0") + await d.page(RasterPage(0, png, 200, 120, "id"), opts) + assert cpu.calls and not gpu.calls + d.settings.overflow_enabled = True + d.settings.overflow_qualified = True + await d.page(RasterPage(1, png, 200, 120, "id"), opts) + assert gpu.calls + finally: + await d.close() + + +def test_no_downgrade(): + assert sensitivity("D0", "D2") == "D2" + + +@pytest.mark.parametrize("status", [429, 502, 503]) +async def test_bounded_retries_breaker(png, status): + cpu = Fake(fail=SafeError(f"backend_http_{status}", True)) + d = Dispatcher(cpu, None, Settings()) + d.start() + try: + p = RasterPage(0, png, 200, 120, "id") + with pytest.raises(SafeError): + await d.page(p, OCROptions()) + assert len(cpu.calls) == 2 + with pytest.raises(SafeError): + await d.page(p, OCROptions()) + assert len(cpu.calls) == 3 + with pytest.raises(SafeError): + await d.page(p, OCROptions()) + assert len(cpu.calls) == 3 and d.cpu.open_until > 0 + finally: + await d.close() + + +async def test_budget_health_and_gpu_fallback(png): + cpu = Fake() + gpu = Fake("gpu", fail=SafeError("alias_bad_reply")) + d = Dispatcher(cpu, gpu, Settings(gpu_enabled=True, gpu_budget=1)) + d.start() + try: + p = RasterPage(0, png, 200, 120, "id") + o = OCROptions(sensitivity="D0", mode="gpu_only") + with pytest.raises(SafeError, match="alias_bad_reply"): + await d.page(p, o) + with pytest.raises(SafeError, match="gpu_budget_exhausted"): + await d.page(p, o) + assert len(gpu.calls) == 1 and not cpu.calls + gpu.ready = False + r = await d.page(p, OCROptions(sensitivity="D0")) + assert r.source == "cpu_ocr" + finally: + await d.close() + + +async def test_queue_capacity_and_cancel(png): + cpu = Fake(delay=0.1) + d = Dispatcher(cpu, None, Settings(queue_size=1)) + d.start() + try: + p = RasterPage(0, png, 200, 120, "id") + o = OCROptions() + one = asyncio.create_task(d.cpu.submit(p, o)) + await asyncio.sleep(0.02) + two = asyncio.create_task(d.cpu.submit(p, o)) + await asyncio.sleep(0.01) + with pytest.raises(SafeError, match="queue_full"): + await d.cpu.submit(p, o) + two.cancel() + await asyncio.gather(two, return_exceptions=True) + await one + await asyncio.sleep(0.01) + assert len(cpu.calls) == 1 and d.cpu.active == 0 + finally: + await d.close() + + +async def test_empty_reply_one_attempt_and_no_false_success(png): + class Empty(Fake): + async def recognize(self, page, options): + result = await super().recognize(page, options) + result.text = "" + result.markdown = "" + return result + + d = Dispatcher(Empty(), None, Settings()) + d.start() + try: + with pytest.raises(SafeError, match="empty_nonblank") as e: + await d.page( + RasterPage(0, png, 200, 120, "id", features={"blank": False}), + OCROptions(mode="cpu_only"), + ) + assert len(e.value.attempts) == 1 and d.cpu.completed == 0 + finally: + await d.close() + + +async def test_gpu_only_not_blocked_by_cpu_health(png): + class Slow(Fake): + async def health(self): + await asyncio.sleep(10) + return await super().health() + + d = Dispatcher(Slow(), Fake("gpu"), Settings(gpu_enabled=True)) + d.start() + try: + result = await asyncio.wait_for( + d.page( + RasterPage(0, png, 200, 120, "id"), OCROptions(mode="gpu_only", sensitivity="D0") + ), + 0.5, + ) + assert result.source == "alias_ocr" + finally: + await d.close() diff --git a/ocr/worker_api.py b/ocr/worker_api.py new file mode 100644 index 0000000000000000000000000000000000000000..5b222b5596d54f29c4f5383c0ba02444e11e62db --- /dev/null +++ b/ocr/worker_api.py @@ -0,0 +1,289 @@ +import asyncio +from contextlib import asynccontextmanager + +from fastapi import Depends, FastAPI, File, Form, UploadFile +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from fastapi.security import HTTPBearer + +from .classify import features +from .config import Settings +from .http_security import Guard +from .ingest import decode_image, encode +from .openai_facade import ChatRequest, completion, image_from_request +from .schemas import OCROptions, RasterPage, SafeError + +security = HTTPBearer() + + +def create_app(settings=None, backend=None): + settings = settings or Settings.from_env() + + @asynccontextmanager + async def lifespan(app): + nonlocal backend + if backend is None: + import os + + from .adapters.rapidocr_cpu import RapidCPU + + cpu_backend = RapidCPU(settings.models_dir) + try: + await cpu_backend.initialize() + except SafeError: + pass + + alias_backend = None + openai_url = os.getenv("OPENAI_COMPATIBLE_URL") + openai_key = os.getenv("OPENAI_COMPATIBLE_API") + openai_model = os.getenv("OPENAI_COMPATIBLE_MODEL", "alias-ocr") + + if openai_url and openai_key: + from .adapters.freellm_alias import FreeLLMAlias + + alias_backend = FreeLLMAlias(openai_url, openai_key) + alias_backend.actual_model = openai_model + try: + await alias_backend.probe() + except SafeError: + pass + + class AutoBackend: + def __init__(self, cpu, alias, alias_model): + self.cpu = cpu + self.alias = alias + self.alias_model = alias_model + self.name = "auto" + self.destination = None + self.external = False + + async def health(self): + return await self.cpu.health() + + async def capabilities(self): + return await self.cpu.capabilities() + + async def recognize(self, page, options): + model_id = getattr(options, "_req_model", "ocr-cpu") + if model_id == "ocr-cpu": + return await self.cpu.recognize(page, options) + elif model_id == "ocr-alias": + if not self.alias: + raise SafeError("alias_unconfigured") + if not (await self.alias.health()).ready: + raise SafeError("alias_not_ready") + return await self.alias.recognize(page, options) + else: + try: + return await self.cpu.recognize(page, options) + except Exception: + if ( + self.alias + and (await self.alias.health()).ready + and options.sensitivity == "D0" + ): + return await self.alias.recognize(page, options) + raise SafeError("model_not_ready") from None + + backend = AutoBackend(cpu_backend, alias_backend, openai_model) + app.state.backend = backend + yield + + app = FastAPI(title="Authenticated PP-OCRv6 Small worker", version="1.0.0", lifespan=lifespan) + app.add_middleware( + Guard, + token=settings.worker_token, + max_bytes=settings.max_upload, + rate=settings.rate_per_minute, + ) + admission = asyncio.Semaphore(1) + + @app.exception_handler(SafeError) + async def safe(request, exc): + return JSONResponse( + {"code": exc.code}, + status_code=503 if exc.code in ("model_not_ready", "worker_busy") else 422, + ) + + @app.exception_handler(RequestValidationError) + async def invalid(request, exc): + return JSONResponse({"code": "invalid_request"}, status_code=422) + + @app.get("/health") + async def health(): + return {"status": "ok"} + + @app.get("/ready", dependencies=[Depends(security)]) + async def ready(): + h = await app.state.backend.health() + return JSONResponse({"ready": h.ready}, status_code=200 if h.ready else 503) + + @app.get("/internal/capabilities", dependencies=[Depends(security)]) + async def capabilities(): + return await app.state.backend.capabilities() + + @app.post("/internal/ocr/page", dependencies=[Depends(security)]) + async def page( + file: UploadFile = File(...), + request_id: str = Form(""), + options: str = Form("{}"), + ): + # Reject rather than grow an unbounded HTTP queue on a two-core worker. + if admission.locked(): + await file.close() + raise SafeError("worker_busy", True) + async with admission: + try: + data = await file.read(settings.max_upload + 1) + finally: + await file.close() + if len(data) > settings.max_upload: + raise SafeError("upload_limit") + if len(request_id) > 128 or options != "{}": + raise SafeError("invalid_worker_options") + image = await asyncio.to_thread(decode_image, data, settings.max_pixels) + data = encode(image) + p = RasterPage( + 0, + data, + image.width, + image.height, + "worker-page", + features=features(data), + ) + if not (await app.state.backend.health()).ready: + raise SafeError("model_not_ready") + return await app.state.backend.recognize(p, OCROptions()) + + @app.get("/v1/models", dependencies=[Depends(security)]) + async def list_models(): + models = [ + { + "id": "ocr-cpu", + "object": "model", + "created": 1677610602, + "owned_by": "augmentoolkit", + }, + { + "id": "ocr-auto", + "object": "model", + "created": 1677610602, + "owned_by": "augmentoolkit", + }, + ] + + if hasattr(app.state.backend, "alias") and app.state.backend.alias: + try: + h = await app.state.backend.alias.health() + if h.ready: + models.append( + { + "id": "ocr-alias", + "object": "model", + "created": 1677610602, + "owned_by": "augmentoolkit", + } + ) + except Exception: + pass + + return { + "object": "list", + "data": models, + } + + @app.get("/v1/ocr/capabilities", dependencies=[Depends(security)]) + async def v1_capabilities(): + return await app.state.backend.capabilities() + + @app.post("/v1/ocr/page", dependencies=[Depends(security)]) + async def v1_page( + file: UploadFile = File(...), + sensitivity: str = Form(...), + options: str = Form("{}"), + ): + if sensitivity != "D0": + raise SafeError("invalid_worker_options", False) + + if admission.locked(): + await file.close() + raise SafeError("worker_busy", True) + async with admission: + try: + data = await file.read(settings.max_upload + 1) + finally: + await file.close() + if len(data) > settings.max_upload: + raise SafeError("upload_limit") + + from .classify import features + from .ingest import decode_image, encode + + # Reject unsupported media by catching errors or doing a quick check + try: + image = await asyncio.to_thread(decode_image, data, settings.max_pixels) + except Exception: + raise SafeError("unsupported_media") from None + + data_enc = encode(image) + p = RasterPage( + 0, + data_enc, + image.width, + image.height, + "agent-page", + features=features(data_enc), + ) + if not (await app.state.backend.health()).ready: + raise SafeError("model_not_ready") + return await app.state.backend.recognize(p, OCROptions(sensitivity="D0")) + + @app.post("/v1/chat/completions", dependencies=[Depends(security)]) + async def v1_chat_completions(req: ChatRequest): + if req.model not in ("ocr-cpu", "ocr-alias", "ocr-auto"): + raise SafeError("unsupported_model") + + if admission.locked(): + raise SafeError("worker_busy", True) + + async with admission: + if not (await app.state.backend.health()).ready: + raise SafeError("model_not_ready") + + try: + data, ext = image_from_request(req, settings.max_upload) + except SafeError as e: + raise e + except Exception: + raise SafeError("unsupported_media") from None + + from .classify import features + from .ingest import decode_image, encode + + try: + image = await asyncio.to_thread(decode_image, data, settings.max_pixels) + except Exception: + raise SafeError("unsupported_media") from None + + data_enc = encode(image) + p = RasterPage( + 0, + data_enc, + image.width, + image.height, + "agent-chat-page", + features=features(data_enc), + ) + + # Force sensitivity to D0 + opts = req.ocr_options + opts.sensitivity = "D0" + opts._req_model = req.model + + result = await app.state.backend.recognize(p, opts) + return completion(result, req.model) + + return app + + +app = create_app()