Download train.py from ajkndfjsdfasdf/docker-default: direct link, hf CLI and curl.
- Browser
- Download file 3.87 kB
-
https://huggingface.co/ajkndfjsdfasdf/docker-default/resolve/main/train.py
- Command line
-
hf download hf://ajkndfjsdfasdf/docker-default/train.py
-
curl -L -o train.py https://huggingface.co/ajkndfjsdfasdf/docker-default/resolve/main/train.py
3.87 kB
| from transformers import T5ForConditionalGeneration, T5Tokenizer, Trainer, TrainingArguments | |
| from datasets import load_dataset | |
| import os | |
| import wandb | |
| import torch | |
| # 🔧 Название модели и путь | |
| model_name = "google/flan-t5-large" | |
| run_id = "flan-t5-large-ru-autobatch" | |
| output_dir = f"./{run_id}" | |
| start_batch_size = 20 # ⚠️ Начинаем с небольшого batch, чтобы избежать OOM | |
| step_batch_size = 1 | |
| # 📦 Загружаем модель и токенизатор | |
| model = T5ForConditionalGeneration.from_pretrained(model_name) | |
| tokenizer = T5Tokenizer.from_pretrained(model_name) | |
| # 📂 Загружаем датасет | |
| data_files = { | |
| "train": "mt5_ru_gen_async.jsonl", | |
| "validation": "mt5_ru_gen_eval.jsonl" | |
| } | |
| dataset = load_dataset("json", data_files=data_files) | |
| # 🔠 Токенизация | |
| def tokenize_function(examples): | |
| model_inputs = tokenizer( | |
| examples["text"], max_length=256, truncation=True, padding="max_length" | |
| ) | |
| with tokenizer.as_target_tokenizer(): | |
| labels = tokenizer( | |
| examples["target"], max_length=256, truncation=True, padding="max_length" | |
| ) | |
| # Заменяем PAD-токены на -100, чтобы не учитывать их в подсчёте loss | |
| labels["input_ids"] = [ | |
| [(token if token != tokenizer.pad_token_id else -100) for token in label] | |
| for label in labels["input_ids"] | |
| ] | |
| model_inputs["labels"] = labels["input_ids"] | |
| return model_inputs | |
| tokenized_datasets = dataset.map(tokenize_function, batched=True) | |
| # 🔑 Авторизация W&B | |
| wandb.login(key="5f028bc0142fb7fa45bdacdde3c00dbbaf8bf98e") | |
| # 🚀 Функция автоподбора batch size | |
| def try_training_with_batch_size(batch_size_start): | |
| batch_size = batch_size_start | |
| while batch_size > 0: | |
| try: | |
| print(f"\n🚀 Пробуем batch_size = {batch_size}") | |
| training_args = TrainingArguments( | |
| output_dir=output_dir, | |
| evaluation_strategy="steps", | |
| eval_steps=100, | |
| learning_rate=3e-5, | |
| per_device_train_batch_size=batch_size, | |
| per_device_eval_batch_size=batch_size, | |
| #fp16=True, # Включайте при наличии подходящего GPU (A100 / V100 / T4) | |
| num_train_epochs=10, | |
| logging_steps=100, | |
| warmup_ratio=0.06, | |
| logging_first_step=True, | |
| weight_decay=0.01, | |
| logging_dir="./logs", | |
| save_total_limit=2, | |
| save_strategy="epoch", | |
| report_to="wandb", | |
| run_name=run_id, | |
| disable_tqdm=False, | |
| max_grad_norm=1.0 | |
| ) | |
| trainer = Trainer( | |
| model=model, | |
| args=training_args, | |
| train_dataset=tokenized_datasets["train"], | |
| eval_dataset=tokenized_datasets["validation"] | |
| ) | |
| trainer.train() | |
| return batch_size | |
| except RuntimeError as e: | |
| if "CUDA out of memory" in str(e): | |
| print(f"❌ OOM на batch_size = {batch_size}, уменьшаем...") | |
| torch.cuda.empty_cache() | |
| batch_size -= step_batch_size | |
| else: | |
| raise e | |
| raise RuntimeError("Не удалось подобрать подходящий batch size 😢") | |
| # 🏁 Запуск с автоподбором | |
| optimal_batch_size = try_training_with_batch_size(start_batch_size) | |
| print(f"\n✅ Успешно обучено с batch_size = {optimal_batch_size}") | |
| # 💾 Сохраняем модель | |
| model.save_pretrained(output_dir) | |
| tokenizer.save_pretrained(output_dir) | |
| print(f"✅ Модель сохранена в {output_dir}") | |