code/VESSA_data_preprocess/process_total_reduced.py

#!/usr/bin/env python3
"""
Batch process TotalSegmentator_reduced and TotalSegmentatorMRI_reduced to RL_dataset format.

Datasets:
- TotalSegmentator_reduced
- TotalSegmentatorMRI_reduced

Input:  /projects/p32685/datasets/biomedparse/{TotalSegmentator_reduced,TotalSegmentatorMRI_reduced}
Output: /projects/p32685/RL_dataset/{totalsegmentator,totalsegmentatormri}_10percent_processed/
"""

import os
import sys
import subprocess
from pathlib import Path

SCRIPT_DIR = Path(__file__).resolve().parent
sys.path.insert(0, str(SCRIPT_DIR))

TWO_DATASETS = [
    {
        "name": "TotalSegmentator_reduced",
        "raw_dir": "/projects/p32685/datasets/biomedparse/TotalSegmentator_reduced",
        "description": "TotalSegmentator reduced - 117 anatomical structures (every 10th slice)"
    },
    {
        "name": "TotalSegmentatorMRI_reduced",
        "raw_dir": "/projects/p32685/datasets/biomedparse/TotalSegmentatorMRI_reduced",
        "description": "TotalSegmentator MRI reduced - 50 anatomical structures (every 10th slice)"
    }
]

OUTPUT_BASE_DIR = "/projects/p32685/RL_dataset"
TEMPLATE_PERCENT = 10
DEVICE = "cuda:0"


def process_dataset(dataset_config, skip_convert=False, skip_packing=False, skip_precompute=False):
    """Process a single dataset."""
    dataset_name = dataset_config["name"]
    raw_dir = dataset_config["raw_dir"]

    print("\n" + "="*80)
    print(f"Processing {dataset_name}: {dataset_config['description']}")
    print("="*80)
    print(f"Raw data: {raw_dir}")
    print(f"Output base: {OUTPUT_BASE_DIR}")
    print(f"Template percent: {TEMPLATE_PERCENT}%")

    if not os.path.exists(raw_dir):
        print(f"✗ Error: Raw data directory does not exist: {raw_dir}")
        return False

    cmd = [
        "python3",
        os.path.join(SCRIPT_DIR, "end_to_end_preprocessing.py"),
        "--dataset_name", dataset_name,
        "--template_percent", str(TEMPLATE_PERCENT),
        "--raw_data_dir", raw_dir,
        "--output_base_dir", OUTPUT_BASE_DIR,
        "--device", DEVICE
    ]
    if skip_convert:
        cmd.append("--skip_convert")
    if skip_packing:
        cmd.append("--skip_packing")
    if skip_precompute:
        cmd.append("--skip_precompute")

    print(f"\nRunning: {' '.join(cmd)}")

    try:
        result = subprocess.run(cmd, check=True, cwd=SCRIPT_DIR)
        if result.returncode == 0:
            print(f"\n✓ {dataset_name} processed successfully!")
            return True
        else:
            print(f"\n✗ {dataset_name} failed with return code {result.returncode}")
            return False
    except subprocess.CalledProcessError as e:
        print(f"\n✗ {dataset_name} failed: {e}")
        return False
    except Exception as e:
        print(f"\n✗ {dataset_name} failed with exception: {e}")
        import traceback
        traceback.print_exc()
        return False


def main():
    import argparse

    parser = argparse.ArgumentParser(description="Batch process TotalSegmentator reduced datasets to RL_dataset")
    parser.add_argument("--start_from", type=int, default=1, help="Start from dataset number (1-2)")
    parser.add_argument("--end_at", type=int, default=2, help="End at dataset number (1-2)")
    parser.add_argument("--skip_convert", action="store_true", help="Skip step 1 (conversion)")
    parser.add_argument("--skip_packing", action="store_true", help="Skip step 2 (packing)")
    parser.add_argument("--skip_precompute", action="store_true", help="Skip step 3 (precompute)")
    parser.add_argument("--device", type=str, default="cuda:0", help="Device for template precomputation")
    parser.add_argument("--template_percent", type=int, default=10, help="Template percentage (default: 10)")

    args = parser.parse_args()
    global DEVICE, TEMPLATE_PERCENT
    DEVICE = args.device
    TEMPLATE_PERCENT = args.template_percent

    print("╔" + "="*78 + "╗")
    print("║" + " "*10 + "BATCH PROCESS TOTAL SEGMENTATOR REDUCED DATASETS" + " "*18 + "║")
    print("╔" + "="*78 + "╗")
    print(f"\nDatasets: TotalSegmentator_reduced, TotalSegmentatorMRI_reduced")
    print(f"Processing: {args.start_from} to {args.end_at}")
    print(f"Output base: {OUTPUT_BASE_DIR}")
    print(f"Template percent: {TEMPLATE_PERCENT}%")
    print(f"Device: {DEVICE}")

    if args.skip_convert:
        print("⚠ Skipping Step 1 (conversion)")
    if args.skip_packing:
        print("⚠ Skipping Step 2 (packing)")
    if args.skip_precompute:
        print("⚠ Skipping Step 3 (precompute)")

    success_count = 0
    failed = []

    for i, cfg in enumerate(TWO_DATASETS, start=1):
        if i < args.start_from or i > args.end_at:
            continue
        print(f"\n{'='*80}")
        print(f"Dataset {i}/2: {cfg['name']}")
        print(f"{'='*80}")
        if process_dataset(cfg, args.skip_convert, args.skip_packing, args.skip_precompute):
            success_count += 1
        else:
            failed.append(cfg['name'])

    print("\n" + "="*80)
    print("BATCH PROCESSING SUMMARY")
    print("="*80)
    n_total = args.end_at - args.start_from + 1
    print(f"Successfully processed: {success_count}/{n_total} datasets")
    if failed:
        print(f"Failed: {', '.join(failed)}")
    else:
        print("\n✓ All datasets processed successfully!")
    print(f"\nOutput: {OUTPUT_BASE_DIR}/")
    # Output folder names omit _reduced (totalsegmentator, totalsegmentatormri)
    out_names = ["totalsegmentator", "totalsegmentatormri"]
    for i in range(args.start_from, args.end_at + 1):
        out = f"{out_names[i-1]}_{TEMPLATE_PERCENT}percent_processed"
        print(f"  {out}/")


if __name__ == "__main__":
    main()