tahamajs/RR / Codes /main.py
tahamajs's picture
download
raw
9.32 kB
#!/usr/bin/env python3
"""
Master experiment runner for the Efficient Transformer Toolkit.
Usage:
python run.py --config configs/sst2_galerkin.yaml
python run.py --model distilbert-base-uncased --task sst2 \
--attn_type galerkin --rank 64 --method lora
"""
import argparse
from pathlib import Path
# ---------------------------------------------------------------------------
# 1. Parse command line and load full configuration
# ---------------------------------------------------------------------------
parser = argparse.ArgumentParser()
parser.add_argument("--config", type=str, default=None, help="YAML config file")
parser.add_argument("--model", type=str, default="distilbert-base-uncased")
parser.add_argument("--task", "-t", type=str, default="sst2")
parser.add_argument("--attn_type", type=str, default="full")
parser.add_argument("--compressor", type=str, default="none")
parser.add_argument("--method", type=str, default="baseline")
parser.add_argument("--rank", type=int, default=32)
parser.add_argument("--svd_energy", type=float, default=0.95)
parser.add_argument("--batch_size", type=int, default=16)
parser.add_argument("--epochs", type=int, default=5)
parser.add_argument("--lr", type=float, default=2e-5)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--output_dir", type=str, default="./experiments")
parser.add_argument("--exp_name", type=str, default=None)
parser.add_argument("--device", type=str, default="auto")
parser.add_argument("--fp16", action="store_true")
parser.add_argument("--save_visuals", action="store_true")
parser.add_argument("--max_train_samples", type=int, default=-1)
parser.add_argument("--sweep", action="store_true", help="Run hyperparameter sweep")
args = parser.parse_args()
# Import framework after argument parsing to allow offline mode early
from framework.config import load_config
# Load full configuration (YAML + CLI overrides)
cfg = load_config(args.config, cli_args=args)
# Override with any direct CLI values that the user provided
if args.attn_type != "full":
cfg.compression.attn_type = args.attn_type
if args.rank != 32:
cfg.compression.rank = args.rank
if args.batch_size != 16:
cfg.training.batch_size = args.batch_size
if args.epochs != 5:
cfg.training.epochs = args.epochs
if args.lr != 2e-5:
cfg.training.learning_rate = args.lr
if args.seed != 42:
cfg.training.seed = (
args.seed
) # but TrainingConfig does not have seed? We'll handle seed separately
if args.fp16:
cfg.hardware.enable_fp16 = True
if args.device != "auto":
cfg.hardware.device = args.device
exp_name = (
args.exp_name
or f"{cfg.compression.attn_type}_{cfg.method.method}_{args.task}_{args.seed}"
)
output_dir = Path(args.output_dir) / exp_name
# ---------------------------------------------------------------------------
# 2. Framework imports (offline mode already set via environment if needed)
# ---------------------------------------------------------------------------
from framework.core.utils import get_device
from framework.core.utils import set_seed as set_all_seeds
from framework.data import (
DataConfig,
ExperimentConfig,
TaskType,
create_dataloader,
load_dataset,
preprocess_function,
)
from framework.models import load_model
from framework.results_saver import ResultsSaver, SaveConfig
from framework.tracking import ExperimentTracker, TrackingBackend
from framework.trainer.callbacks import (
GradientNormCallback,
MemoryCallback,
TimeCallback,
)
from framework.training import (
HuggingFaceTrainer,
ResultsManager,
get_method as get_method_fn,
get_task_metrics,
)
# Optional callbacks
try:
from framework.callbacks.custom import EMACallback, SWACallback
except ImportError:
EMACallback = SWACallback = None
try:
from framework.callbacks.analysis import AttentionHeatmapCallback
from framework.callbacks.analysis import (
GradientNormCallback as AnalysisGradCallback,
)
from framework.callbacks.analysis import SVDAttentionCallback
except ImportError:
SVDAttentionCallback = AttentionHeatmapCallback = AnalysisGradCallback = None
# ---------------------------------------------------------------------------
# 3. Setup
# ---------------------------------------------------------------------------
set_all_seeds(args.seed)
device = get_device(cfg.hardware.device)
# ---------------------------------------------------------------------------
# 4. Load model (with attention injection) and tokenizer
# ---------------------------------------------------------------------------
model, tokenizer = load_model(
cfg.model.name,
num_labels=cfg.model.num_labels,
attention_type=cfg.compression.attn_type,
attention_rank=cfg.compression.rank,
variant=cfg.model.variants[0] if cfg.model.variants else "standard",
cache_dir=cfg.model.cache_dir,
)
# ---------------------------------------------------------------------------
# 5. Load dataset and create dataloaders
# ---------------------------------------------------------------------------
exp_config = ExperimentConfig(dataset_name=args.task)
dataset = load_dataset(exp_config)
num_labels = cfg.model.num_labels
is_regression = args.task == "stsb"
sent1_key, sent2_key = None, None
from framework.data.glue_loader import GLUE_TASK_KEYS
if args.task in GLUE_TASK_KEYS:
sent1_key, sent2_key = GLUE_TASK_KEYS[args.task]
data_cfg = DataConfig(
task_type=(
TaskType.SINGLE_CLASSIFICATION
if sent2_key is None
else TaskType.PAIR_CLASSIFICATION
),
max_length=cfg.data.max_seq_length,
batch_size=cfg.training.batch_size,
text_column_1=sent1_key,
text_column_2=sent2_key,
label_column="label",
)
def glue_preprocess(examples):
return preprocess_function(examples, tokenizer, data_cfg)
tokenized = dataset.map(
glue_preprocess, batched=True, remove_columns=dataset["train"].column_names
)
tokenized.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])
train_data = tokenized["train"]
if args.max_train_samples > 0:
train_data = train_data.select(range(min(len(train_data), args.max_train_samples)))
train_loader = create_dataloader(train_data, tokenizer, data_cfg)
val_split = "validation" if "validation" in tokenized else "validation_matched"
val_loader = (
create_dataloader(tokenized[val_split], tokenizer, data_cfg)
if val_split in tokenized
else None
)
# ---------------------------------------------------------------------------
# 6. Apply training method (LoRA, Mixout, etc.)
# ---------------------------------------------------------------------------
if cfg.method.method != "baseline":
method_fn = get_method_fn(cfg.method.method)
if method_fn is None:
raise ValueError(f"Unknown method: {cfg.method.method}")
method_fn(
model, cfg.compression
) # compression config contains method-specific params
# ---------------------------------------------------------------------------
# 7. Set up ResultsManager, Tracker, and Trainer
# ---------------------------------------------------------------------------
results = ResultsManager(
experiment_name=exp_name,
base_dir=str(args.output_dir),
config=cfg.__dict__,
)
tracker = ExperimentTracker(
backend=TrackingBackend.LOCAL,
log_dir=str(output_dir / "tracking"),
experiment_name=exp_name,
)
trainer = HuggingFaceTrainer(
model=model,
train_dataset=train_loader.dataset,
eval_dataset=val_loader.dataset if val_loader else None,
config=cfg.training.__dict__,
results_manager=results,
compute_metrics=get_task_metrics(args.task),
)
# Add basic callbacks
trainer.add_callback(MemoryCallback())
trainer.add_callback(TimeCallback())
trainer.add_callback(GradientNormCallback(log_every_n_steps=100))
if cfg.hardware.enable_fp16:
trainer.args.fp16 = True
# Optional advanced callbacks
if args.attn_type != "full" and SVDAttentionCallback is not None:
trainer.add_callback(SVDAttentionCallback(log_every_steps=500))
if args.save_visuals and AttentionHeatmapCallback is not None:
trainer.add_callback(AttentionHeatmapCallback(output_dir=str(output_dir)))
# ---------------------------------------------------------------------------
# 8. Train
# ---------------------------------------------------------------------------
trainer.train()
eval_metrics = trainer.evaluate()
# ---------------------------------------------------------------------------
# 9. Save everything via ResultsSaver
# ---------------------------------------------------------------------------
saver_cfg = SaveConfig(
save_model=True,
save_plots=True,
model_export_formats=["onnx"] if cfg.hardware.device == "cuda" else [],
storage_backends=["local"],
)
saver = ResultsSaver(
run_id=exp_name,
base_dir=Path(args.output_dir),
config=saver_cfg,
)
# Convert history to a list of dicts for the saver
history = [h for h in trainer.state.log_history] if hasattr(trainer, "state") else []
saver.save_all(model, cfg.__dict__, history, eval_metrics, tokenizer)
saver.push_all()
print("=" * 60)
print(f"Experiment {exp_name} completed successfully!")
print(f"Results saved to {output_dir}")
print(f"Final metrics: {eval_metrics}")
print("=" * 60)

Xet Storage Details

Size:
9.32 kB
·
Xet hash:
0bbe936c5713455372c1f2f3bd84bac05ccc0dfe00ca781b0dcf45b2519597b6

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.