Download upstreams/candi/main_for_sweeps.py from jyliuAI/diffusion_baseline: direct link, hf CLI and curl.
- Browser
- Download file 8.42 kB
-
https://huggingface.co/jyliuAI/diffusion_baseline/resolve/main/upstreams/candi/main_for_sweeps.py
- Command line
-
hf download hf://jyliuAI/diffusion_baseline/upstreams/candi/main_for_sweeps.py
-
curl -L -o main_for_sweeps.py https://huggingface.co/jyliuAI/diffusion_baseline/resolve/main/upstreams/candi/main_for_sweeps.py
8.42 kB
| from tqdm import tqdm | |
| import json | |
| import os | |
| from typing import List, Dict, Tuple | |
| import fsspec | |
| import hydra | |
| import lightning as L | |
| import omegaconf | |
| import rich.syntax | |
| import rich.tree | |
| import torch | |
| import algo | |
| import dataloader as dataloader | |
| from text_metrics import evaluate_text_metrics | |
| import utils | |
| import pdb | |
| # from lm_eval.models.huggingface import HFLM | |
| # from lm_eval import evaluator | |
| # from hugging_face_wrappers import DiffusionEvalWrapper, CandiEvalWrapper | |
| omegaconf.OmegaConf.register_new_resolver("cwd", os.getcwd) | |
| omegaconf.OmegaConf.register_new_resolver("device_count", torch.cuda.device_count) | |
| omegaconf.OmegaConf.register_new_resolver("eval", eval) | |
| omegaconf.OmegaConf.register_new_resolver("div_up", lambda x, y: (x + y - 1) // y) | |
| def _load_from_checkpoint(diffusion_model, config, tokenizer): | |
| if "hf" in config.algo.backbone: | |
| return diffusion_model(config, tokenizer=tokenizer).to("cuda") | |
| model = diffusion_model.load_from_checkpoint( | |
| config.eval.checkpoint_path, tokenizer=tokenizer, config=config | |
| ) | |
| return model | |
| def _print_config( | |
| config: omegaconf.DictConfig, resolve: bool = True, save_cfg: bool = True | |
| ) -> None: | |
| """Prints content of DictConfig using Rich library and its tree structure. | |
| Args: | |
| config (DictConfig): Configuration composed by Hydra. | |
| resolve (bool): Whether to resolve reference fields of DictConfig. | |
| save_cfg (bool): Whether to save the configuration tree to a file. | |
| """ | |
| style = "dim" | |
| tree = rich.tree.Tree("CONFIG", style=style, guide_style=style) | |
| fields = config.keys() | |
| for field in fields: | |
| branch = tree.add(field, style=style, guide_style=style) | |
| config_section = config.get(field) | |
| branch_content = str(config_section) | |
| if isinstance(config_section, omegaconf.DictConfig): | |
| branch_content = omegaconf.OmegaConf.to_yaml( | |
| config_section, resolve=resolve | |
| ) | |
| branch.add(rich.syntax.Syntax(branch_content, "yaml")) | |
| rich.print(tree) | |
| if save_cfg: | |
| with fsspec.open( | |
| "{}/config_tree.txt".format(config.checkpointing.save_dir), "w" | |
| ) as fp: | |
| rich.print(tree, file=fp) | |
| def _print_batch(train_ds, valid_ds, tokenizer, k=64, vocab_offset=False): | |
| for dl_type, dl in [("train", train_ds), ("valid", valid_ds)]: | |
| print(f"Printing {dl_type} dataloader batch.") | |
| batch = next(iter(dl)) | |
| print("Batch input_ids.shape", batch["input_ids"].shape) | |
| first = batch["input_ids"][0, :k] | |
| last = batch["input_ids"][0, -k:] | |
| print(f"First {k} tokens:", tokenizer.decode(first)) | |
| print("ids:", first) | |
| print(f"Last {k} tokens:", tokenizer.decode(last)) | |
| print("ids:", last) | |
| def _generate_samples(model, config, logger, tokenizer): | |
| logger.info("Starting Sample Eval.") | |
| if config.eval.disable_ema: | |
| logger.info("Disabling EMA.") | |
| model.ema = None | |
| all_samples = [] | |
| print("Using single GPU for generation") | |
| # using compile | |
| for _ in tqdm(range(config.sampling.num_sample_batches)): | |
| samples = model.restore_model_and_sample(num_steps=config.sampling.steps) | |
| model.metrics.record_entropy(samples) | |
| text_samples = model.tokenizer.batch_decode(samples) | |
| model.metrics.record_generative_perplexity( | |
| text_samples, config.model.length, model.device | |
| ) | |
| all_samples.extend(list(text_samples)) | |
| generative_ppl = 0.0 | |
| entropy = 0.0 | |
| model.metrics.record_entropy(samples) | |
| gen_sequences = tokenizer.batch_decode(samples) | |
| device = torch.device('cuda:0') | |
| model.metrics.record_generative_perplexity(gen_sequences, config.model.length, device) | |
| all_samples.extend(list(gen_sequences)) | |
| generative_ppl = model.metrics.gen_ppl.compute().item() | |
| entropy = model.metrics.sample_entropy.compute().item() | |
| res = { | |
| "generative_ppl": generative_ppl, | |
| "entropy": entropy, | |
| "generated_seqs": all_samples, | |
| } | |
| return res | |
| def _sweep(diffusion_model, config, tokenizer, logger, is_conditional=False): | |
| temps_to_use = torch.linspace(.1, 2.0, steps=30).tolist() | |
| steps_to_use = [8, 16, 32, 64, 128] | |
| model = _load_from_checkpoint( | |
| diffusion_model=diffusion_model, config=config, tokenizer=tokenizer | |
| ) | |
| # model.backbone = torch.compile(model.backbone) | |
| total_metadata = {} | |
| for steps in steps_to_use: | |
| cur_step_info = {} | |
| config.sampling.steps = steps | |
| for temp in temps_to_use: | |
| model.temp = temp | |
| model.metrics.reset() | |
| res = _generate_samples(model, config, logger, tokenizer,) | |
| res['steps'] = steps | |
| res['temperature'] = temp | |
| samples_path = config.eval.generated_samples_path | |
| samples_path = samples_path.replace('.json', f'_steps{steps}_temp{temp:.3f}.json') | |
| with fsspec.open(samples_path, "w") as f: | |
| json.dump(res, f, | |
| indent=4, | |
| ) | |
| print("Samples saved at:", samples_path) | |
| if config.data.train == 'text8': | |
| train_ds, valid_ds = dataloader.get_dataloaders(config, tokenizer) | |
| train_texts = [] | |
| # check if train_texts.txt exists | |
| if os.path.exists("/home/patrick/duo/train_texts.txt"): | |
| print("Loading training texts from train_texts.txt for text metrics...") | |
| train_texts = open("/home/patrick/duo/train_texts.txt", "r").readlines() | |
| else: | |
| print("Loading training texts for text metrics...") | |
| for batch in tqdm(train_ds): | |
| batch_texts = tokenizer.batch_decode(batch["input_ids"], skip_special_tokens=True) | |
| train_texts.extend(batch_texts) | |
| with open("/home/patrick/duo/train_texts.txt", "w") as f: | |
| for text in train_texts: | |
| f.write(f"{text}\n") | |
| text_metrics = evaluate_text_metrics(res["generated_seqs"], train_texts) | |
| cur_step_info[temp] = text_metrics | |
| print(f"Steps: {steps}, Temp: {temp}") | |
| print(text_metrics) | |
| else: | |
| cur_step_info[temp] = { | |
| "perplexity": res["generative_ppl"], | |
| "entropy": res["entropy"], | |
| } | |
| print(f"Steps: {steps}, Temp: {temp}") | |
| print("perplexity: ", res["generative_ppl"]) | |
| print("entropy: ", res["entropy"]) | |
| total_metadata[steps] = cur_step_info | |
| metadata_path = config.eval.generated_samples_path.replace('.json', f'_sweep_metadata.json') | |
| with fsspec.open(metadata_path, "w") as f: | |
| json.dump(total_metadata, f, | |
| indent=4, | |
| ) | |
| return | |
| def main(config): | |
| """Main entry point for training.""" | |
| L.seed_everything(config.seed) | |
| # _print_config(config, resolve=True, save_cfg=True) | |
| logger = utils.get_logger(__name__) | |
| tokenizer = dataloader.get_tokenizer(config) | |
| if config.algo.name == "ar": | |
| diffusion_model = algo.AR | |
| elif config.algo.name == "mdlm": | |
| diffusion_model = algo.MDLM | |
| elif config.algo.name == "duo_base": | |
| diffusion_model = algo.DUO_BASE | |
| elif config.algo.name == "d3pm": | |
| diffusion_model = algo.D3PMAbsorb | |
| elif config.algo.name == "sedd": | |
| diffusion_model = algo.SEDDAbsorb | |
| elif config.algo.name == "duo": | |
| diffusion_model = algo.DUO | |
| elif config.algo.name == "distillation": | |
| diffusion_model = algo.Distillation | |
| elif config.algo.name == "ot-finetune": | |
| diffusion_model = algo.OptimalTransportFinetune | |
| elif config.algo.name == "candi": | |
| diffusion_model = algo.CANDI | |
| else: | |
| raise ValueError(f"Invalid algorithm name: {config.algo.name}") | |
| kwargs = { | |
| "diffusion_model": diffusion_model, | |
| "config": config, | |
| "tokenizer": tokenizer, | |
| "logger": logger, | |
| } | |
| _sweep(**kwargs, is_conditional=False) | |
| if __name__ == "__main__": | |
| main() |