Download app.py from abastecimiento-lab/app_chatbot: direct link, hf CLI and curl.
- Browser
- Download file 1.49 kB
-
https://huggingface.co/abastecimiento-lab/app_chatbot/resolve/main/app.py
- Command line
-
hf download hf://abastecimiento-lab/app_chatbot/app.py
-
curl -L -o app.py https://huggingface.co/abastecimiento-lab/app_chatbot/resolve/main/app.py
1.49 kB
| from datasets import load_dataset | |
| from transformers import DistilBertTokenizer, DistilBertForQuestionAnswering, Trainer, TrainingArguments | |
| import torch | |
| # Cargar el dataset desde Hugging Face (si lo subiste) | |
| dataset = load_dataset("abastecimiento-lab/dataset_reglamento_limpio") # Cambia el nombre por el de tu dataset | |
| # Cargar el tokenizador y modelo preentrenado de DistilBERT | |
| tokenizer = DistilBertTokenizer.from_pretrained("distilbert-base-uncased") | |
| model = DistilBertForQuestionAnswering.from_pretrained("distilbert-base-uncased") | |
| def preprocess_function(examples): | |
| # Preprocesar el dataset para la tarea de QA | |
| return tokenizer( | |
| examples["contexto"], | |
| examples["pregunta"], | |
| truncation=True, | |
| padding=True, | |
| max_length=512, | |
| return_tensors="pt" | |
| ) | |
| # Aplicar el preprocesamiento al dataset | |
| tokenized_datasets = dataset.map(preprocess_function, batched=True) | |
| # Definir los parámetros de entrenamiento | |
| training_args = TrainingArguments( | |
| output_dir='./results', | |
| evaluation_strategy="epoch", | |
| learning_rate=2e-5, | |
| per_device_train_batch_size=8, | |
| num_train_epochs=3, | |
| weight_decay=0.01 | |
| ) | |
| trainer = Trainer( | |
| model=model, | |
| args=training_args, | |
| train_dataset=tokenized_datasets['train'], | |
| eval_dataset=tokenized_datasets['validation'] | |
| ) | |
| # Entrenar el modelo | |
| trainer.train() | |
| # Guardar el modelo entrenado | |
| model.save_pretrained('./modelo_entrenado') | |
| tokenizer.save_pretrained('./modelo_entrenado') | |