diffusion_baseline / upstreams /candi /main_for_sweeps.py
jyliuAI's picture
Add files using upload-large-folder tool
9a46c96 verified
Raw History Blame Contribute Delete
8.42 kB
from tqdm import tqdm
import json
import os
from typing import List, Dict, Tuple
import fsspec
import hydra
import lightning as L
import omegaconf
import rich.syntax
import rich.tree
import torch
import algo
import dataloader as dataloader
from text_metrics import evaluate_text_metrics
import utils
import pdb
# from lm_eval.models.huggingface import HFLM
# from lm_eval import evaluator
# from hugging_face_wrappers import DiffusionEvalWrapper, CandiEvalWrapper
omegaconf.OmegaConf.register_new_resolver("cwd", os.getcwd)
omegaconf.OmegaConf.register_new_resolver("device_count", torch.cuda.device_count)
omegaconf.OmegaConf.register_new_resolver("eval", eval)
omegaconf.OmegaConf.register_new_resolver("div_up", lambda x, y: (x + y - 1) // y)
def _load_from_checkpoint(diffusion_model, config, tokenizer):
if "hf" in config.algo.backbone:
return diffusion_model(config, tokenizer=tokenizer).to("cuda")
model = diffusion_model.load_from_checkpoint(
config.eval.checkpoint_path, tokenizer=tokenizer, config=config
)
return model
@L.pytorch.utilities.rank_zero_only
def _print_config(
config: omegaconf.DictConfig, resolve: bool = True, save_cfg: bool = True
) -> None:
"""Prints content of DictConfig using Rich library and its tree structure.
Args:
config (DictConfig): Configuration composed by Hydra.
resolve (bool): Whether to resolve reference fields of DictConfig.
save_cfg (bool): Whether to save the configuration tree to a file.
"""
style = "dim"
tree = rich.tree.Tree("CONFIG", style=style, guide_style=style)
fields = config.keys()
for field in fields:
branch = tree.add(field, style=style, guide_style=style)
config_section = config.get(field)
branch_content = str(config_section)
if isinstance(config_section, omegaconf.DictConfig):
branch_content = omegaconf.OmegaConf.to_yaml(
config_section, resolve=resolve
)
branch.add(rich.syntax.Syntax(branch_content, "yaml"))
rich.print(tree)
if save_cfg:
with fsspec.open(
"{}/config_tree.txt".format(config.checkpointing.save_dir), "w"
) as fp:
rich.print(tree, file=fp)
@L.pytorch.utilities.rank_zero_only
def _print_batch(train_ds, valid_ds, tokenizer, k=64, vocab_offset=False):
for dl_type, dl in [("train", train_ds), ("valid", valid_ds)]:
print(f"Printing {dl_type} dataloader batch.")
batch = next(iter(dl))
print("Batch input_ids.shape", batch["input_ids"].shape)
first = batch["input_ids"][0, :k]
last = batch["input_ids"][0, -k:]
print(f"First {k} tokens:", tokenizer.decode(first))
print("ids:", first)
print(f"Last {k} tokens:", tokenizer.decode(last))
print("ids:", last)
def _generate_samples(model, config, logger, tokenizer):
logger.info("Starting Sample Eval.")
if config.eval.disable_ema:
logger.info("Disabling EMA.")
model.ema = None
all_samples = []
print("Using single GPU for generation")
# using compile
for _ in tqdm(range(config.sampling.num_sample_batches)):
samples = model.restore_model_and_sample(num_steps=config.sampling.steps)
model.metrics.record_entropy(samples)
text_samples = model.tokenizer.batch_decode(samples)
model.metrics.record_generative_perplexity(
text_samples, config.model.length, model.device
)
all_samples.extend(list(text_samples))
generative_ppl = 0.0
entropy = 0.0
model.metrics.record_entropy(samples)
gen_sequences = tokenizer.batch_decode(samples)
device = torch.device('cuda:0')
model.metrics.record_generative_perplexity(gen_sequences, config.model.length, device)
all_samples.extend(list(gen_sequences))
generative_ppl = model.metrics.gen_ppl.compute().item()
entropy = model.metrics.sample_entropy.compute().item()
res = {
"generative_ppl": generative_ppl,
"entropy": entropy,
"generated_seqs": all_samples,
}
return res
def _sweep(diffusion_model, config, tokenizer, logger, is_conditional=False):
temps_to_use = torch.linspace(.1, 2.0, steps=30).tolist()
steps_to_use = [8, 16, 32, 64, 128]
model = _load_from_checkpoint(
diffusion_model=diffusion_model, config=config, tokenizer=tokenizer
)
# model.backbone = torch.compile(model.backbone)
total_metadata = {}
for steps in steps_to_use:
cur_step_info = {}
config.sampling.steps = steps
for temp in temps_to_use:
model.temp = temp
model.metrics.reset()
res = _generate_samples(model, config, logger, tokenizer,)
res['steps'] = steps
res['temperature'] = temp
samples_path = config.eval.generated_samples_path
samples_path = samples_path.replace('.json', f'_steps{steps}_temp{temp:.3f}.json')
with fsspec.open(samples_path, "w") as f:
json.dump(res, f,
indent=4,
)
print("Samples saved at:", samples_path)
if config.data.train == 'text8':
train_ds, valid_ds = dataloader.get_dataloaders(config, tokenizer)
train_texts = []
# check if train_texts.txt exists
if os.path.exists("/home/patrick/duo/train_texts.txt"):
print("Loading training texts from train_texts.txt for text metrics...")
train_texts = open("/home/patrick/duo/train_texts.txt", "r").readlines()
else:
print("Loading training texts for text metrics...")
for batch in tqdm(train_ds):
batch_texts = tokenizer.batch_decode(batch["input_ids"], skip_special_tokens=True)
train_texts.extend(batch_texts)
with open("/home/patrick/duo/train_texts.txt", "w") as f:
for text in train_texts:
f.write(f"{text}\n")
text_metrics = evaluate_text_metrics(res["generated_seqs"], train_texts)
cur_step_info[temp] = text_metrics
print(f"Steps: {steps}, Temp: {temp}")
print(text_metrics)
else:
cur_step_info[temp] = {
"perplexity": res["generative_ppl"],
"entropy": res["entropy"],
}
print(f"Steps: {steps}, Temp: {temp}")
print("perplexity: ", res["generative_ppl"])
print("entropy: ", res["entropy"])
total_metadata[steps] = cur_step_info
metadata_path = config.eval.generated_samples_path.replace('.json', f'_sweep_metadata.json')
with fsspec.open(metadata_path, "w") as f:
json.dump(total_metadata, f,
indent=4,
)
return
@hydra.main(version_base=None, config_path="configs", config_name="config")
def main(config):
"""Main entry point for training."""
L.seed_everything(config.seed)
# _print_config(config, resolve=True, save_cfg=True)
logger = utils.get_logger(__name__)
tokenizer = dataloader.get_tokenizer(config)
if config.algo.name == "ar":
diffusion_model = algo.AR
elif config.algo.name == "mdlm":
diffusion_model = algo.MDLM
elif config.algo.name == "duo_base":
diffusion_model = algo.DUO_BASE
elif config.algo.name == "d3pm":
diffusion_model = algo.D3PMAbsorb
elif config.algo.name == "sedd":
diffusion_model = algo.SEDDAbsorb
elif config.algo.name == "duo":
diffusion_model = algo.DUO
elif config.algo.name == "distillation":
diffusion_model = algo.Distillation
elif config.algo.name == "ot-finetune":
diffusion_model = algo.OptimalTransportFinetune
elif config.algo.name == "candi":
diffusion_model = algo.CANDI
else:
raise ValueError(f"Invalid algorithm name: {config.algo.name}")
kwargs = {
"diffusion_model": diffusion_model,
"config": config,
"tokenizer": tokenizer,
"logger": logger,
}
_sweep(**kwargs, is_conditional=False)
if __name__ == "__main__":
main()