Fetching metadata from the HF Docker repository...
-
Upload app.py
-
Upload requirements.txt
-
Update app.py
import torch
import gradio as gr
from transformers import AutoModelForCausalLM, AutoTokenizer, Trainer, TrainingArguments, DataCollatorForLanguageModeling
from datasets import load_dataset, Dataset
def train_model(txt_file):
# Carregar o dataset a partir do arquivo de texto
with open(txt_file.name, "r", encoding="utf-8") as f:
lines = f.readlines()
dataset = Dataset.from_dict({"text": lines})
# Definir modelo e tokenizer
model_name = "distilgpt2"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(model_name)
# Tokenizar os dados
def tokenize_function(examples):
return tokenizer(examples["text"], truncation=True, padding="max_length", max_length=512)
tokenized_datasets = dataset.map(tokenize_function, batched=True)
# Configurar parâmetros de treino
training_args = TrainingArguments(
output_dir="./results",
per_device_train_batch_size=2,
… -
Create app.py
gradio
nltk
scikit-learn -
initial commit
