diff --git a/.gitattributes b/.gitattributes index c149555b5474b181cd0ce18afd369315ab44c2ae..e7557fab207778c4ec1497401e30e3c83c454c5c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -48,3 +48,19 @@ qwen3/csd/tokenizer.json filter=lfs diff=lfs merge=lfs -text qwen3/distillm_adaptive_sfkl/checkpoint-1256/tokenizer.json filter=lfs diff=lfs merge=lfs -text qwen3/distillm_adaptive_sfkl/checkpoint-1570/tokenizer.json filter=lfs diff=lfs merge=lfs -text qwen3/distillm_adaptive_sfkl/checkpoint-314/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen3/distillm_adaptive_sfkl/checkpoint-628/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen3/distillm_adaptive_sfkl/checkpoint-942/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen3/distillm_adaptive_sfkl/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen3/distillm_adaptive_srkl/checkpoint-1256/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen3/distillm_adaptive_srkl/checkpoint-1570/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen3/distillm_adaptive_srkl/checkpoint-314/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen3/distillm_adaptive_srkl/checkpoint-628/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen3/distillm_adaptive_srkl/checkpoint-942/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen3/distillm_adaptive_srkl/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen3/fdd_sfkl/checkpoint-1256/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen3/fdd_sfkl/checkpoint-1570/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen3/fdd_sfkl/checkpoint-314/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen3/fdd_sfkl/checkpoint-628/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen3/fdd_sfkl/checkpoint-942/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen3/fdd_sfkl/tokenizer.json filter=lfs diff=lfs merge=lfs -text +qwen3/fdd_srkl/checkpoint-1256/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-628/latest b/qwen3/distillm_adaptive_sfkl/checkpoint-628/latest new file mode 100644 index 0000000000000000000000000000000000000000..af055c3db05dcb2af541005f22308dbd946aba65 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-628/latest @@ -0,0 +1 @@ +global_step628 \ No newline at end of file diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-628/rng_state_0.pth b/qwen3/distillm_adaptive_sfkl/checkpoint-628/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..2322455495cd88cc66104acc67f0503a6f573648 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-628/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d5fad2941403da5d23f24b7da6834213087f8dec2f4451cd284aab44f844f43 +size 14917 diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-628/rng_state_1.pth b/qwen3/distillm_adaptive_sfkl/checkpoint-628/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..2a1cf006281d8a7005ce50ab0c6b2fe5047b1d11 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-628/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21e0209531857c65fa2018e6f770607a67108d795a5c1b4c238bb893982e789a +size 14917 diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-628/scheduler.pt b/qwen3/distillm_adaptive_sfkl/checkpoint-628/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..96c0cabcd99be4945ffbd7897ecc5367ac4beb28 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-628/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb823a2c51d7fb42dfe3b693a7159784285dbf38c5a797e6e73292096123ab7a +size 1465 diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-628/tokenizer.json b/qwen3/distillm_adaptive_sfkl/checkpoint-628/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-628/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-628/tokenizer_config.json b/qwen3/distillm_adaptive_sfkl/checkpoint-628/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-628/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-628/trainer_state.json b/qwen3/distillm_adaptive_sfkl/checkpoint-628/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..68e88ce6f0a0909139ceb39addaeabc4c111de99 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-628/trainer_state.json @@ -0,0 +1,296 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 628, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "distillm_initial_generator_count": 1712.0, + "distillm_initial_generator_exact_match": 0.0, + "distillm_initial_generator_rouge1": 16.2149, + "distillm_initial_generator_rouge2": 3.2276, + "distillm_initial_generator_rougeL": 11.6725, + "distillm_initial_loss": 2.494271755218506, + "distillm_initial_runtime": 2988.1286, + "distillm_initial_samples_per_second": 1.143, + "distillm_initial_selector_accuracy": 52.278, + "distillm_initial_selector_count": 1712.0, + "distillm_initial_steps_per_second": 0.036, + "epoch": 0, + "step": 0 + }, + { + "epoch": 0.06382130035899482, + "grad_norm": 1.4179933071136475, + "learning_rate": 9.996390385474569e-05, + "loss": 0.360275936126709, + "step": 20 + }, + { + "epoch": 0.12764260071798963, + "grad_norm": 0.6145927309989929, + "learning_rate": 9.984797509888031e-05, + "loss": 0.0921450138092041, + "step": 40 + }, + { + "epoch": 0.19146390107698444, + "grad_norm": 0.949298083782196, + "learning_rate": 9.965229938050925e-05, + "loss": 0.06155582666397095, + "step": 60 + }, + { + "epoch": 0.25528520143597927, + "grad_norm": 0.32781288027763367, + "learning_rate": 9.937719005663432e-05, + "loss": 0.03148526251316071, + "step": 80 + }, + { + "epoch": 0.3191065017949741, + "grad_norm": 0.42831170558929443, + "learning_rate": 9.902308769000724e-05, + "loss": 0.0304107666015625, + "step": 100 + }, + { + "epoch": 0.3829278021539689, + "grad_norm": 0.37467315793037415, + "learning_rate": 9.859055934360789e-05, + "loss": 0.018133948743343353, + "step": 120 + }, + { + "epoch": 0.4467491025129637, + "grad_norm": 0.26231899857521057, + "learning_rate": 9.808029767254437e-05, + "loss": 0.023359841108322142, + "step": 140 + }, + { + "epoch": 0.5105704028719585, + "grad_norm": 0.23304614424705505, + "learning_rate": 9.749311981482817e-05, + "loss": 0.015377765893936158, + "step": 160 + }, + { + "epoch": 0.5743917032309533, + "grad_norm": 4.825753688812256, + "learning_rate": 9.682996608280154e-05, + "loss": 0.0134853795170784, + "step": 180 + }, + { + "epoch": 0.6382130035899481, + "grad_norm": 0.29226061701774597, + "learning_rate": 9.609189845731207e-05, + "loss": 0.022103385627269746, + "step": 200 + }, + { + "epoch": 0.702034303948943, + "grad_norm": 0.1759956330060959, + "learning_rate": 9.52800988870462e-05, + "loss": 0.011787685751914977, + "step": 220 + }, + { + "epoch": 0.7658556043079378, + "grad_norm": 0.18942032754421234, + "learning_rate": 9.439586739574513e-05, + "loss": 0.009058185666799546, + "step": 240 + }, + { + "epoch": 0.8296769046669326, + "grad_norm": 0.2657412886619568, + "learning_rate": 9.344062000033414e-05, + "loss": 0.009212753176689148, + "step": 260 + }, + { + "epoch": 0.8934982050259274, + "grad_norm": 0.28189197182655334, + "learning_rate": 9.241588644329919e-05, + "loss": 0.009374459087848664, + "step": 280 + }, + { + "epoch": 0.9573195053849222, + "grad_norm": 0.22191843390464783, + "learning_rate": 9.132330774294252e-05, + "loss": 0.009671492874622345, + "step": 300 + }, + { + "epoch": 1.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 90.0701, + "eval_generator_rouge1": 99.4297, + "eval_generator_rouge2": 99.1758, + "eval_generator_rougeL": 99.0503, + "eval_loss": 0.007805492263287306, + "eval_runtime": 601.5417, + "eval_samples_per_second": 5.675, + "eval_selector_accuracy": 95.5607, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.178, + "step": 314 + }, + { + "epoch": 1.0191463901076985, + "grad_norm": 1.0133247375488281, + "learning_rate": 9.01646335654399e-05, + "loss": 0.013477623462677002, + "step": 320 + }, + { + "epoch": 1.0829676904666932, + "grad_norm": 0.07633373141288757, + "learning_rate": 8.894171942290817e-05, + "loss": 0.0067503504455089566, + "step": 340 + }, + { + "epoch": 1.146788990825688, + "grad_norm": 0.10817505419254303, + "learning_rate": 8.765652370197003e-05, + "loss": 0.005428457632660866, + "step": 360 + }, + { + "epoch": 1.210610291184683, + "grad_norm": 0.13295228779315948, + "learning_rate": 8.631110452757467e-05, + "loss": 0.004656650871038437, + "step": 380 + }, + { + "epoch": 1.2744315915436777, + "grad_norm": 0.09486064314842224, + "learning_rate": 8.490761646709642e-05, + "loss": 0.008230840414762497, + "step": 400 + }, + { + "epoch": 1.3382528919026724, + "grad_norm": 0.3019433617591858, + "learning_rate": 8.344830707998953e-05, + "loss": 0.008267045021057129, + "step": 420 + }, + { + "epoch": 1.4020741922616673, + "grad_norm": 0.06807491928339005, + "learning_rate": 8.193551331852458e-05, + "loss": 0.006827282160520554, + "step": 440 + }, + { + "epoch": 1.4658954926206622, + "grad_norm": 0.31115707755088806, + "learning_rate": 8.037165778537033e-05, + "loss": 0.005447100847959518, + "step": 460 + }, + { + "epoch": 1.5297167929796571, + "grad_norm": 0.09037721902132034, + "learning_rate": 7.875924485401427e-05, + "loss": 0.0037935201078653337, + "step": 480 + }, + { + "epoch": 1.5935380933386518, + "grad_norm": 0.050306856632232666, + "learning_rate": 7.710085665823444e-05, + "loss": 0.004351192712783813, + "step": 500 + }, + { + "epoch": 1.6573593936976465, + "grad_norm": 0.06550924479961395, + "learning_rate": 7.539914895704521e-05, + "loss": 0.004179144278168678, + "step": 520 + }, + { + "epoch": 1.7211806940566414, + "grad_norm": 0.037223804742097855, + "learning_rate": 7.365684688173908e-05, + "loss": 0.006012631207704544, + "step": 540 + }, + { + "epoch": 1.7850019944156363, + "grad_norm": 0.07440082728862762, + "learning_rate": 7.18767405718347e-05, + "loss": 0.0067665338516235355, + "step": 560 + }, + { + "epoch": 1.848823294774631, + "grad_norm": 0.4098421037197113, + "learning_rate": 7.006168070692036e-05, + "loss": 0.007050857692956924, + "step": 580 + }, + { + "epoch": 1.9126445951336257, + "grad_norm": 1.077409029006958, + "learning_rate": 6.821457394154786e-05, + "loss": 0.0025460906326770782, + "step": 600 + }, + { + "epoch": 1.9764658954926206, + "grad_norm": 0.14294511079788208, + "learning_rate": 6.633837825048768e-05, + "loss": 0.0034835845232009887, + "step": 620 + }, + { + "epoch": 2.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 92.9907, + "eval_generator_rouge1": 99.8109, + "eval_generator_rouge2": 99.6137, + "eval_generator_rougeL": 99.5606, + "eval_loss": 0.005652441177517176, + "eval_runtime": 602.3066, + "eval_samples_per_second": 5.668, + "eval_selector_accuracy": 97.722, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.178, + "step": 628 + } + ], + "logging_steps": 20, + "max_steps": 1570, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.346351770520781e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-628/training_args.bin b/qwen3/distillm_adaptive_sfkl/checkpoint-628/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..9965b3704c5694d942c823f03f32ef4aa365d5a2 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-628/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4080ce6c122d894dce2a6242fb58ae879822809092a9484cf3c306a91aacf376 +size 7185 diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-628/zero_to_fp32.py b/qwen3/distillm_adaptive_sfkl/checkpoint-628/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-628/zero_to_fp32.py @@ -0,0 +1,790 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY, + AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY, + AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def _has_autoep_zero3_partitioned_metadata(state_dict): + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY) + if autoep_layers is None: + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY) + if not isinstance(autoep_layers, list): + return False + return any( + isinstance(entry, dict) + and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT + for entry in autoep_layers) + + +def _raise_if_autoep_zero3_partitioned_state(state_dict): + if _has_autoep_zero3_partitioned_metadata(state_dict): + raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. " + "AutoEP expert parameters are partitioned over expert replica groups, so " + "global data-parallel consolidation would produce incomplete expert tensors. " + "Use ds_to_universal.py for expert-aware conversion.") + + +def _raise_if_autoep_zero3_partitioned_checkpoint(model_files): + for file in model_files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints + # before the expensive optimizer-shard load below. + model_files = get_model_state_files(ds_checkpoint_dir) + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-942/README.md b/qwen3/distillm_adaptive_sfkl/checkpoint-942/README.md new file mode 100644 index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-942/README.md @@ -0,0 +1,208 @@ +--- +base_model: Qwen/Qwen3-0.6B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-0.6B +- llama-factory +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-942/adapter_config.json b/qwen3/distillm_adaptive_sfkl/checkpoint-942/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d88d3ddd2a1fe81f4cf32eb7f116cfcaecff483e --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-942/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-0.6B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "down_proj", + "v_proj", + "q_proj", + "up_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-942/adapter_model.safetensors b/qwen3/distillm_adaptive_sfkl/checkpoint-942/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1cd03eae9485350a9a887099f3cacace26660c51 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-942/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f1f927767965a819c3def28535da80f1c6ab3cc57154957d4cef244fe16eca2 +size 40422560 diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-942/chat_template.jinja b/qwen3/distillm_adaptive_sfkl/checkpoint-942/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-942/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-942/distillm_state_rank0.pt b/qwen3/distillm_adaptive_sfkl/checkpoint-942/distillm_state_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..4c4026c6e95aa9d01c436ecad3c6f0b27f11f91d --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-942/distillm_state_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be37e9f05d7d7943ae5068691d49e7fe146b0249ac3bd306b8a17830441b0c7c +size 9467 diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-942/distillm_state_rank1.pt b/qwen3/distillm_adaptive_sfkl/checkpoint-942/distillm_state_rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..15516d1a1e440bac4d70959dfa9a603927bd3bd4 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-942/distillm_state_rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5284c6faa8372751c674221b6912744948fcb78359dd17daa1518f8c1985ddf +size 9339 diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/distillm_adaptive_sfkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d551989f526ed7af118b7c5d11b53de6d7d31ebd --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfb8860f1893d34d8deb96fb152de943d824a2a23a5fd6ab990e3fd198eb0cdf +size 121135557 diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/distillm_adaptive_sfkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eca27588adbb85c3d33e0c7dfc817f4ba1592631 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f743e4456f789d6274d504763dbb0d6d50d7c306212f0fcde2f030d1b6e5b002 +size 121135749 diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt b/qwen3/distillm_adaptive_sfkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6dcac85f58bedbb604bc85d9537026fe21e2cb4d --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15e351ed5f1ef84806ebe7436bd28f9be429c105ea538e0a63b2752d38662c16 +size 351841765 diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-942/latest b/qwen3/distillm_adaptive_sfkl/checkpoint-942/latest new file mode 100644 index 0000000000000000000000000000000000000000..4e9a68e4460aca593c480611a486d2a4cd73ef42 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-942/latest @@ -0,0 +1 @@ +global_step942 \ No newline at end of file diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-942/rng_state_0.pth b/qwen3/distillm_adaptive_sfkl/checkpoint-942/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..29b6eb5b88e945897856deb233f6d7563bdc7eba --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-942/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b75fc62d24d0c049ccd0600506cc8b95377b4bde08176598b9d645b84e39b66 +size 14917 diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-942/rng_state_1.pth b/qwen3/distillm_adaptive_sfkl/checkpoint-942/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..2700ab42800e08514e8144cb7ed580143c7afe22 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-942/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fe1f543ee4f21e08dbf0ed89b5b9c41408b9b3f756ad4b44d26869053ab44cd +size 14917 diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-942/scheduler.pt b/qwen3/distillm_adaptive_sfkl/checkpoint-942/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b1f8af0480bf83ab728abfeff11bc3288214349 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-942/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8076da6f897951c5bdf8649e4fba596e8c49819927746017442ffd85f40b18a3 +size 1465 diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-942/tokenizer.json b/qwen3/distillm_adaptive_sfkl/checkpoint-942/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-942/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-942/tokenizer_config.json b/qwen3/distillm_adaptive_sfkl/checkpoint-942/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-942/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-942/trainer_state.json b/qwen3/distillm_adaptive_sfkl/checkpoint-942/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a525a443e160cce0db76bd7cff2799491885cee8 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-942/trainer_state.json @@ -0,0 +1,423 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 942, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "distillm_initial_generator_count": 1712.0, + "distillm_initial_generator_exact_match": 0.0, + "distillm_initial_generator_rouge1": 16.2149, + "distillm_initial_generator_rouge2": 3.2276, + "distillm_initial_generator_rougeL": 11.6725, + "distillm_initial_loss": 2.494271755218506, + "distillm_initial_runtime": 2988.1286, + "distillm_initial_samples_per_second": 1.143, + "distillm_initial_selector_accuracy": 52.278, + "distillm_initial_selector_count": 1712.0, + "distillm_initial_steps_per_second": 0.036, + "epoch": 0, + "step": 0 + }, + { + "epoch": 0.06382130035899482, + "grad_norm": 1.4179933071136475, + "learning_rate": 9.996390385474569e-05, + "loss": 0.360275936126709, + "step": 20 + }, + { + "epoch": 0.12764260071798963, + "grad_norm": 0.6145927309989929, + "learning_rate": 9.984797509888031e-05, + "loss": 0.0921450138092041, + "step": 40 + }, + { + "epoch": 0.19146390107698444, + "grad_norm": 0.949298083782196, + "learning_rate": 9.965229938050925e-05, + "loss": 0.06155582666397095, + "step": 60 + }, + { + "epoch": 0.25528520143597927, + "grad_norm": 0.32781288027763367, + "learning_rate": 9.937719005663432e-05, + "loss": 0.03148526251316071, + "step": 80 + }, + { + "epoch": 0.3191065017949741, + "grad_norm": 0.42831170558929443, + "learning_rate": 9.902308769000724e-05, + "loss": 0.0304107666015625, + "step": 100 + }, + { + "epoch": 0.3829278021539689, + "grad_norm": 0.37467315793037415, + "learning_rate": 9.859055934360789e-05, + "loss": 0.018133948743343353, + "step": 120 + }, + { + "epoch": 0.4467491025129637, + "grad_norm": 0.26231899857521057, + "learning_rate": 9.808029767254437e-05, + "loss": 0.023359841108322142, + "step": 140 + }, + { + "epoch": 0.5105704028719585, + "grad_norm": 0.23304614424705505, + "learning_rate": 9.749311981482817e-05, + "loss": 0.015377765893936158, + "step": 160 + }, + { + "epoch": 0.5743917032309533, + "grad_norm": 4.825753688812256, + "learning_rate": 9.682996608280154e-05, + "loss": 0.0134853795170784, + "step": 180 + }, + { + "epoch": 0.6382130035899481, + "grad_norm": 0.29226061701774597, + "learning_rate": 9.609189845731207e-05, + "loss": 0.022103385627269746, + "step": 200 + }, + { + "epoch": 0.702034303948943, + "grad_norm": 0.1759956330060959, + "learning_rate": 9.52800988870462e-05, + "loss": 0.011787685751914977, + "step": 220 + }, + { + "epoch": 0.7658556043079378, + "grad_norm": 0.18942032754421234, + "learning_rate": 9.439586739574513e-05, + "loss": 0.009058185666799546, + "step": 240 + }, + { + "epoch": 0.8296769046669326, + "grad_norm": 0.2657412886619568, + "learning_rate": 9.344062000033414e-05, + "loss": 0.009212753176689148, + "step": 260 + }, + { + "epoch": 0.8934982050259274, + "grad_norm": 0.28189197182655334, + "learning_rate": 9.241588644329919e-05, + "loss": 0.009374459087848664, + "step": 280 + }, + { + "epoch": 0.9573195053849222, + "grad_norm": 0.22191843390464783, + "learning_rate": 9.132330774294252e-05, + "loss": 0.009671492874622345, + "step": 300 + }, + { + "epoch": 1.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 90.0701, + "eval_generator_rouge1": 99.4297, + "eval_generator_rouge2": 99.1758, + "eval_generator_rougeL": 99.0503, + "eval_loss": 0.007805492263287306, + "eval_runtime": 601.5417, + "eval_samples_per_second": 5.675, + "eval_selector_accuracy": 95.5607, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.178, + "step": 314 + }, + { + "epoch": 1.0191463901076985, + "grad_norm": 1.0133247375488281, + "learning_rate": 9.01646335654399e-05, + "loss": 0.013477623462677002, + "step": 320 + }, + { + "epoch": 1.0829676904666932, + "grad_norm": 0.07633373141288757, + "learning_rate": 8.894171942290817e-05, + "loss": 0.0067503504455089566, + "step": 340 + }, + { + "epoch": 1.146788990825688, + "grad_norm": 0.10817505419254303, + "learning_rate": 8.765652370197003e-05, + "loss": 0.005428457632660866, + "step": 360 + }, + { + "epoch": 1.210610291184683, + "grad_norm": 0.13295228779315948, + "learning_rate": 8.631110452757467e-05, + "loss": 0.004656650871038437, + "step": 380 + }, + { + "epoch": 1.2744315915436777, + "grad_norm": 0.09486064314842224, + "learning_rate": 8.490761646709642e-05, + "loss": 0.008230840414762497, + "step": 400 + }, + { + "epoch": 1.3382528919026724, + "grad_norm": 0.3019433617591858, + "learning_rate": 8.344830707998953e-05, + "loss": 0.008267045021057129, + "step": 420 + }, + { + "epoch": 1.4020741922616673, + "grad_norm": 0.06807491928339005, + "learning_rate": 8.193551331852458e-05, + "loss": 0.006827282160520554, + "step": 440 + }, + { + "epoch": 1.4658954926206622, + "grad_norm": 0.31115707755088806, + "learning_rate": 8.037165778537033e-05, + "loss": 0.005447100847959518, + "step": 460 + }, + { + "epoch": 1.5297167929796571, + "grad_norm": 0.09037721902132034, + "learning_rate": 7.875924485401427e-05, + "loss": 0.0037935201078653337, + "step": 480 + }, + { + "epoch": 1.5935380933386518, + "grad_norm": 0.050306856632232666, + "learning_rate": 7.710085665823444e-05, + "loss": 0.004351192712783813, + "step": 500 + }, + { + "epoch": 1.6573593936976465, + "grad_norm": 0.06550924479961395, + "learning_rate": 7.539914895704521e-05, + "loss": 0.004179144278168678, + "step": 520 + }, + { + "epoch": 1.7211806940566414, + "grad_norm": 0.037223804742097855, + "learning_rate": 7.365684688173908e-05, + "loss": 0.006012631207704544, + "step": 540 + }, + { + "epoch": 1.7850019944156363, + "grad_norm": 0.07440082728862762, + "learning_rate": 7.18767405718347e-05, + "loss": 0.0067665338516235355, + "step": 560 + }, + { + "epoch": 1.848823294774631, + "grad_norm": 0.4098421037197113, + "learning_rate": 7.006168070692036e-05, + "loss": 0.007050857692956924, + "step": 580 + }, + { + "epoch": 1.9126445951336257, + "grad_norm": 1.077409029006958, + "learning_rate": 6.821457394154786e-05, + "loss": 0.0025460906326770782, + "step": 600 + }, + { + "epoch": 1.9764658954926206, + "grad_norm": 0.14294511079788208, + "learning_rate": 6.633837825048768e-05, + "loss": 0.0034835845232009887, + "step": 620 + }, + { + "epoch": 2.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 92.9907, + "eval_generator_rouge1": 99.8109, + "eval_generator_rouge2": 99.6137, + "eval_generator_rougeL": 99.5606, + "eval_loss": 0.005652441177517176, + "eval_runtime": 602.3066, + "eval_samples_per_second": 5.668, + "eval_selector_accuracy": 97.722, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.178, + "step": 628 + }, + { + "epoch": 2.038292780215397, + "grad_norm": 0.15983566641807556, + "learning_rate": 6.443609819179926e-05, + "loss": 0.0025436000898480414, + "step": 640 + }, + { + "epoch": 2.102114080574392, + "grad_norm": 0.24636076390743256, + "learning_rate": 6.251078009530256e-05, + "loss": 0.005166642367839813, + "step": 660 + }, + { + "epoch": 2.1659353809333863, + "grad_norm": 0.05653794854879379, + "learning_rate": 6.0565507184155677e-05, + "loss": 0.0020019154995679854, + "step": 680 + }, + { + "epoch": 2.2297566812923812, + "grad_norm": 0.020209209993481636, + "learning_rate": 5.860339463735136e-05, + "loss": 0.003964858502149582, + "step": 700 + }, + { + "epoch": 2.293577981651376, + "grad_norm": 0.015605684369802475, + "learning_rate": 5.662758460103895e-05, + "loss": 0.005196084827184677, + "step": 720 + }, + { + "epoch": 2.357399282010371, + "grad_norm": 0.05083553120493889, + "learning_rate": 5.464124115666095e-05, + "loss": 0.002757585048675537, + "step": 740 + }, + { + "epoch": 2.421220582369366, + "grad_norm": 0.023810580372810364, + "learning_rate": 5.264754525396212e-05, + "loss": 0.0015414590016007423, + "step": 760 + }, + { + "epoch": 2.4850418827283605, + "grad_norm": 0.022960327565670013, + "learning_rate": 5.06496896169857e-05, + "loss": 0.0012374447658658027, + "step": 780 + }, + { + "epoch": 2.5488631830873554, + "grad_norm": 0.08331727981567383, + "learning_rate": 4.865087363121384e-05, + "loss": 0.0037563137710094454, + "step": 800 + }, + { + "epoch": 2.6126844834463503, + "grad_norm": 0.3172169327735901, + "learning_rate": 4.6654298220040673e-05, + "loss": 0.0025980696082115173, + "step": 820 + }, + { + "epoch": 2.6765057838053448, + "grad_norm": 0.0642293244600296, + "learning_rate": 4.466316071878229e-05, + "loss": 0.0016859130933880805, + "step": 840 + }, + { + "epoch": 2.74032708416434, + "grad_norm": 0.07357119768857956, + "learning_rate": 4.2680649754432913e-05, + "loss": 0.002408899739384651, + "step": 860 + }, + { + "epoch": 2.8041483845233346, + "grad_norm": 0.12400355935096741, + "learning_rate": 4.070994013936646e-05, + "loss": 0.0020469389855861664, + "step": 880 + }, + { + "epoch": 2.8679696848823295, + "grad_norm": 0.21444188058376312, + "learning_rate": 3.8754187787161155e-05, + "loss": 0.0023752737790346144, + "step": 900 + }, + { + "epoch": 2.9317909852413244, + "grad_norm": 0.07703635096549988, + "learning_rate": 3.681652465868875e-05, + "loss": 0.004145454987883568, + "step": 920 + }, + { + "epoch": 2.995612285600319, + "grad_norm": 0.11405396461486816, + "learning_rate": 3.490005374656184e-05, + "loss": 0.0013555588200688362, + "step": 940 + }, + { + "epoch": 3.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 95.3271, + "eval_generator_rouge1": 99.9061, + "eval_generator_rouge2": 99.7991, + "eval_generator_rougeL": 99.6925, + "eval_loss": 0.003950633108615875, + "eval_runtime": 595.0919, + "eval_samples_per_second": 5.737, + "eval_selector_accuracy": 98.0724, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.18, + "step": 942 + } + ], + "logging_steps": 20, + "max_steps": 1570, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.524157743353037e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-942/training_args.bin b/qwen3/distillm_adaptive_sfkl/checkpoint-942/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..9965b3704c5694d942c823f03f32ef4aa365d5a2 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-942/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4080ce6c122d894dce2a6242fb58ae879822809092a9484cf3c306a91aacf376 +size 7185 diff --git a/qwen3/distillm_adaptive_sfkl/checkpoint-942/zero_to_fp32.py b/qwen3/distillm_adaptive_sfkl/checkpoint-942/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/checkpoint-942/zero_to_fp32.py @@ -0,0 +1,790 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY, + AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY, + AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def _has_autoep_zero3_partitioned_metadata(state_dict): + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY) + if autoep_layers is None: + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY) + if not isinstance(autoep_layers, list): + return False + return any( + isinstance(entry, dict) + and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT + for entry in autoep_layers) + + +def _raise_if_autoep_zero3_partitioned_state(state_dict): + if _has_autoep_zero3_partitioned_metadata(state_dict): + raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. " + "AutoEP expert parameters are partitioned over expert replica groups, so " + "global data-parallel consolidation would produce incomplete expert tensors. " + "Use ds_to_universal.py for expert-aware conversion.") + + +def _raise_if_autoep_zero3_partitioned_checkpoint(model_files): + for file in model_files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints + # before the expensive optimizer-shard load below. + model_files = get_model_state_files(ds_checkpoint_dir) + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/qwen3/distillm_adaptive_sfkl/distillm_state_rank0.pt b/qwen3/distillm_adaptive_sfkl/distillm_state_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..becca175688219a9b4a422f423edefb47590217d --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/distillm_state_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c45fea5636ba58d552a49ba5a651afe08fe7e86b64bd29d82359e1ffb3df111f +size 9403 diff --git a/qwen3/distillm_adaptive_sfkl/distillm_state_rank1.pt b/qwen3/distillm_adaptive_sfkl/distillm_state_rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d7c319e37a117a78ed636f3109afc6ef3c8fd98 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/distillm_state_rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34b6ffca878fd86b6133b9eb70d94fee805ff00ef1d33571b54dc0de3a307df3 +size 9339 diff --git a/qwen3/distillm_adaptive_sfkl/eval_results.json b/qwen3/distillm_adaptive_sfkl/eval_results.json new file mode 100644 index 0000000000000000000000000000000000000000..6436d41b9d29ac9c977fbcfefbcc0fbe8c10e10e --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/eval_results.json @@ -0,0 +1,19 @@ +{ + "epoch": 5.0, + "eval_distillm_threshold": 0.0, + "eval_distillm_threshold_changed": 0.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 96.3785, + "eval_generator_rouge1": 99.9305, + "eval_generator_rouge2": 99.8682, + "eval_generator_rougeL": 99.7529, + "eval_loss": 0.003217515302821994, + "eval_rollout_dataset_steps": 12535, + "eval_rollout_fresh_steps": 0, + "eval_rollout_replay_steps": 0, + "eval_runtime": 579.9368, + "eval_samples_per_second": 5.887, + "eval_selector_accuracy": 98.715, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.185 +} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_sfkl/tokenizer.json b/qwen3/distillm_adaptive_sfkl/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/qwen3/distillm_adaptive_sfkl/tokenizer_config.json b/qwen3/distillm_adaptive_sfkl/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen3/distillm_adaptive_sfkl/train_results.json b/qwen3/distillm_adaptive_sfkl/train_results.json new file mode 100644 index 0000000000000000000000000000000000000000..febdb56ab5bd58dbef21fcb4ba9fc6ed7a3664f9 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 5.0, + "total_flos": 5.871444840310374e+16, + "train_loss": 0.011652989937051847, + "train_runtime": 11256.8941, + "train_samples_per_second": 4.454, + "train_steps_per_second": 0.139 +} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_sfkl/trainer_state.json b/qwen3/distillm_adaptive_sfkl/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..adf331a883f1cfde135ff503241d5efda8bdd679 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/trainer_state.json @@ -0,0 +1,679 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 1570, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "distillm_initial_generator_count": 1712.0, + "distillm_initial_generator_exact_match": 0.0, + "distillm_initial_generator_rouge1": 16.2149, + "distillm_initial_generator_rouge2": 3.2276, + "distillm_initial_generator_rougeL": 11.6725, + "distillm_initial_loss": 2.494271755218506, + "distillm_initial_runtime": 2988.1286, + "distillm_initial_samples_per_second": 1.143, + "distillm_initial_selector_accuracy": 52.278, + "distillm_initial_selector_count": 1712.0, + "distillm_initial_steps_per_second": 0.036, + "epoch": 0, + "step": 0 + }, + { + "epoch": 0.06382130035899482, + "grad_norm": 1.4179933071136475, + "learning_rate": 9.996390385474569e-05, + "loss": 0.360275936126709, + "step": 20 + }, + { + "epoch": 0.12764260071798963, + "grad_norm": 0.6145927309989929, + "learning_rate": 9.984797509888031e-05, + "loss": 0.0921450138092041, + "step": 40 + }, + { + "epoch": 0.19146390107698444, + "grad_norm": 0.949298083782196, + "learning_rate": 9.965229938050925e-05, + "loss": 0.06155582666397095, + "step": 60 + }, + { + "epoch": 0.25528520143597927, + "grad_norm": 0.32781288027763367, + "learning_rate": 9.937719005663432e-05, + "loss": 0.03148526251316071, + "step": 80 + }, + { + "epoch": 0.3191065017949741, + "grad_norm": 0.42831170558929443, + "learning_rate": 9.902308769000724e-05, + "loss": 0.0304107666015625, + "step": 100 + }, + { + "epoch": 0.3829278021539689, + "grad_norm": 0.37467315793037415, + "learning_rate": 9.859055934360789e-05, + "loss": 0.018133948743343353, + "step": 120 + }, + { + "epoch": 0.4467491025129637, + "grad_norm": 0.26231899857521057, + "learning_rate": 9.808029767254437e-05, + "loss": 0.023359841108322142, + "step": 140 + }, + { + "epoch": 0.5105704028719585, + "grad_norm": 0.23304614424705505, + "learning_rate": 9.749311981482817e-05, + "loss": 0.015377765893936158, + "step": 160 + }, + { + "epoch": 0.5743917032309533, + "grad_norm": 4.825753688812256, + "learning_rate": 9.682996608280154e-05, + "loss": 0.0134853795170784, + "step": 180 + }, + { + "epoch": 0.6382130035899481, + "grad_norm": 0.29226061701774597, + "learning_rate": 9.609189845731207e-05, + "loss": 0.022103385627269746, + "step": 200 + }, + { + "epoch": 0.702034303948943, + "grad_norm": 0.1759956330060959, + "learning_rate": 9.52800988870462e-05, + "loss": 0.011787685751914977, + "step": 220 + }, + { + "epoch": 0.7658556043079378, + "grad_norm": 0.18942032754421234, + "learning_rate": 9.439586739574513e-05, + "loss": 0.009058185666799546, + "step": 240 + }, + { + "epoch": 0.8296769046669326, + "grad_norm": 0.2657412886619568, + "learning_rate": 9.344062000033414e-05, + "loss": 0.009212753176689148, + "step": 260 + }, + { + "epoch": 0.8934982050259274, + "grad_norm": 0.28189197182655334, + "learning_rate": 9.241588644329919e-05, + "loss": 0.009374459087848664, + "step": 280 + }, + { + "epoch": 0.9573195053849222, + "grad_norm": 0.22191843390464783, + "learning_rate": 9.132330774294252e-05, + "loss": 0.009671492874622345, + "step": 300 + }, + { + "epoch": 1.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 90.0701, + "eval_generator_rouge1": 99.4297, + "eval_generator_rouge2": 99.1758, + "eval_generator_rougeL": 99.0503, + "eval_loss": 0.007805492263287306, + "eval_runtime": 601.5417, + "eval_samples_per_second": 5.675, + "eval_selector_accuracy": 95.5607, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.178, + "step": 314 + }, + { + "epoch": 1.0191463901076985, + "grad_norm": 1.0133247375488281, + "learning_rate": 9.01646335654399e-05, + "loss": 0.013477623462677002, + "step": 320 + }, + { + "epoch": 1.0829676904666932, + "grad_norm": 0.07633373141288757, + "learning_rate": 8.894171942290817e-05, + "loss": 0.0067503504455089566, + "step": 340 + }, + { + "epoch": 1.146788990825688, + "grad_norm": 0.10817505419254303, + "learning_rate": 8.765652370197003e-05, + "loss": 0.005428457632660866, + "step": 360 + }, + { + "epoch": 1.210610291184683, + "grad_norm": 0.13295228779315948, + "learning_rate": 8.631110452757467e-05, + "loss": 0.004656650871038437, + "step": 380 + }, + { + "epoch": 1.2744315915436777, + "grad_norm": 0.09486064314842224, + "learning_rate": 8.490761646709642e-05, + "loss": 0.008230840414762497, + "step": 400 + }, + { + "epoch": 1.3382528919026724, + "grad_norm": 0.3019433617591858, + "learning_rate": 8.344830707998953e-05, + "loss": 0.008267045021057129, + "step": 420 + }, + { + "epoch": 1.4020741922616673, + "grad_norm": 0.06807491928339005, + "learning_rate": 8.193551331852458e-05, + "loss": 0.006827282160520554, + "step": 440 + }, + { + "epoch": 1.4658954926206622, + "grad_norm": 0.31115707755088806, + "learning_rate": 8.037165778537033e-05, + "loss": 0.005447100847959518, + "step": 460 + }, + { + "epoch": 1.5297167929796571, + "grad_norm": 0.09037721902132034, + "learning_rate": 7.875924485401427e-05, + "loss": 0.0037935201078653337, + "step": 480 + }, + { + "epoch": 1.5935380933386518, + "grad_norm": 0.050306856632232666, + "learning_rate": 7.710085665823444e-05, + "loss": 0.004351192712783813, + "step": 500 + }, + { + "epoch": 1.6573593936976465, + "grad_norm": 0.06550924479961395, + "learning_rate": 7.539914895704521e-05, + "loss": 0.004179144278168678, + "step": 520 + }, + { + "epoch": 1.7211806940566414, + "grad_norm": 0.037223804742097855, + "learning_rate": 7.365684688173908e-05, + "loss": 0.006012631207704544, + "step": 540 + }, + { + "epoch": 1.7850019944156363, + "grad_norm": 0.07440082728862762, + "learning_rate": 7.18767405718347e-05, + "loss": 0.0067665338516235355, + "step": 560 + }, + { + "epoch": 1.848823294774631, + "grad_norm": 0.4098421037197113, + "learning_rate": 7.006168070692036e-05, + "loss": 0.007050857692956924, + "step": 580 + }, + { + "epoch": 1.9126445951336257, + "grad_norm": 1.077409029006958, + "learning_rate": 6.821457394154786e-05, + "loss": 0.0025460906326770782, + "step": 600 + }, + { + "epoch": 1.9764658954926206, + "grad_norm": 0.14294511079788208, + "learning_rate": 6.633837825048768e-05, + "loss": 0.0034835845232009887, + "step": 620 + }, + { + "epoch": 2.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 92.9907, + "eval_generator_rouge1": 99.8109, + "eval_generator_rouge2": 99.6137, + "eval_generator_rougeL": 99.5606, + "eval_loss": 0.005652441177517176, + "eval_runtime": 602.3066, + "eval_samples_per_second": 5.668, + "eval_selector_accuracy": 97.722, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.178, + "step": 628 + }, + { + "epoch": 2.038292780215397, + "grad_norm": 0.15983566641807556, + "learning_rate": 6.443609819179926e-05, + "loss": 0.0025436000898480414, + "step": 640 + }, + { + "epoch": 2.102114080574392, + "grad_norm": 0.24636076390743256, + "learning_rate": 6.251078009530256e-05, + "loss": 0.005166642367839813, + "step": 660 + }, + { + "epoch": 2.1659353809333863, + "grad_norm": 0.05653794854879379, + "learning_rate": 6.0565507184155677e-05, + "loss": 0.0020019154995679854, + "step": 680 + }, + { + "epoch": 2.2297566812923812, + "grad_norm": 0.020209209993481636, + "learning_rate": 5.860339463735136e-05, + "loss": 0.003964858502149582, + "step": 700 + }, + { + "epoch": 2.293577981651376, + "grad_norm": 0.015605684369802475, + "learning_rate": 5.662758460103895e-05, + "loss": 0.005196084827184677, + "step": 720 + }, + { + "epoch": 2.357399282010371, + "grad_norm": 0.05083553120493889, + "learning_rate": 5.464124115666095e-05, + "loss": 0.002757585048675537, + "step": 740 + }, + { + "epoch": 2.421220582369366, + "grad_norm": 0.023810580372810364, + "learning_rate": 5.264754525396212e-05, + "loss": 0.0015414590016007423, + "step": 760 + }, + { + "epoch": 2.4850418827283605, + "grad_norm": 0.022960327565670013, + "learning_rate": 5.06496896169857e-05, + "loss": 0.0012374447658658027, + "step": 780 + }, + { + "epoch": 2.5488631830873554, + "grad_norm": 0.08331727981567383, + "learning_rate": 4.865087363121384e-05, + "loss": 0.0037563137710094454, + "step": 800 + }, + { + "epoch": 2.6126844834463503, + "grad_norm": 0.3172169327735901, + "learning_rate": 4.6654298220040673e-05, + "loss": 0.0025980696082115173, + "step": 820 + }, + { + "epoch": 2.6765057838053448, + "grad_norm": 0.0642293244600296, + "learning_rate": 4.466316071878229e-05, + "loss": 0.0016859130933880805, + "step": 840 + }, + { + "epoch": 2.74032708416434, + "grad_norm": 0.07357119768857956, + "learning_rate": 4.2680649754432913e-05, + "loss": 0.002408899739384651, + "step": 860 + }, + { + "epoch": 2.8041483845233346, + "grad_norm": 0.12400355935096741, + "learning_rate": 4.070994013936646e-05, + "loss": 0.0020469389855861664, + "step": 880 + }, + { + "epoch": 2.8679696848823295, + "grad_norm": 0.21444188058376312, + "learning_rate": 3.8754187787161155e-05, + "loss": 0.0023752737790346144, + "step": 900 + }, + { + "epoch": 2.9317909852413244, + "grad_norm": 0.07703635096549988, + "learning_rate": 3.681652465868875e-05, + "loss": 0.004145454987883568, + "step": 920 + }, + { + "epoch": 2.995612285600319, + "grad_norm": 0.11405396461486816, + "learning_rate": 3.490005374656184e-05, + "loss": 0.0013555588200688362, + "step": 940 + }, + { + "epoch": 3.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 95.3271, + "eval_generator_rouge1": 99.9061, + "eval_generator_rouge2": 99.7991, + "eval_generator_rougeL": 99.6925, + "eval_loss": 0.003950633108615875, + "eval_runtime": 595.0919, + "eval_samples_per_second": 5.737, + "eval_selector_accuracy": 98.0724, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.18, + "step": 942 + }, + { + "epoch": 3.057439170323095, + "grad_norm": 0.01742524839937687, + "learning_rate": 3.3007844105971336e-05, + "loss": 0.0017176205292344093, + "step": 960 + }, + { + "epoch": 3.12126047068209, + "grad_norm": 0.003188655711710453, + "learning_rate": 3.114292593987149e-05, + "loss": 0.0026764504611492155, + "step": 980 + }, + { + "epoch": 3.185081771041085, + "grad_norm": 0.01925867609679699, + "learning_rate": 2.9308285746383408e-05, + "loss": 0.0012796200811862946, + "step": 1000 + }, + { + "epoch": 3.24890307140008, + "grad_norm": 0.02856561355292797, + "learning_rate": 2.7506861536187867e-05, + "loss": 0.003274943307042122, + "step": 1020 + }, + { + "epoch": 3.3127243717590744, + "grad_norm": 0.28238120675086975, + "learning_rate": 2.5741538127566396e-05, + "loss": 0.0020828992128372192, + "step": 1040 + }, + { + "epoch": 3.3765456721180693, + "grad_norm": 0.0034029490780085325, + "learning_rate": 2.4015142526625115e-05, + "loss": 0.0026421770453453064, + "step": 1060 + }, + { + "epoch": 3.4403669724770642, + "grad_norm": 0.08193282783031464, + "learning_rate": 2.233043940009967e-05, + "loss": 0.002248910814523697, + "step": 1080 + }, + { + "epoch": 3.504188272836059, + "grad_norm": 0.006192841101437807, + "learning_rate": 2.0690126647990974e-05, + "loss": 0.0024001479148864748, + "step": 1100 + }, + { + "epoch": 3.568009573195054, + "grad_norm": 0.023488029837608337, + "learning_rate": 1.9096831083121828e-05, + "loss": 0.001568210683763027, + "step": 1120 + }, + { + "epoch": 3.6318308735540485, + "grad_norm": 0.10149848461151123, + "learning_rate": 1.7553104224533324e-05, + "loss": 0.0015563317574560642, + "step": 1140 + }, + { + "epoch": 3.6956521739130435, + "grad_norm": 0.016474615782499313, + "learning_rate": 1.606141821145735e-05, + "loss": 0.003182568401098251, + "step": 1160 + }, + { + "epoch": 3.7594734742720384, + "grad_norm": 0.015502875670790672, + "learning_rate": 1.4624161844408763e-05, + "loss": 0.002123878337442875, + "step": 1180 + }, + { + "epoch": 3.823294774631033, + "grad_norm": 0.013245862908661366, + "learning_rate": 1.324363675973698e-05, + "loss": 0.0021719962358474733, + "step": 1200 + }, + { + "epoch": 3.8871160749900278, + "grad_norm": 0.176155224442482, + "learning_rate": 1.1922053743763052e-05, + "loss": 0.003167513757944107, + "step": 1220 + }, + { + "epoch": 3.9509373753490227, + "grad_norm": 0.00724257156252861, + "learning_rate": 1.0661529192404973e-05, + "loss": 0.0008703418076038361, + "step": 1240 + }, + { + "epoch": 4.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 96.6706, + "eval_generator_rouge1": 99.9312, + "eval_generator_rouge2": 99.8709, + "eval_generator_rougeL": 99.7682, + "eval_loss": 0.0032213269732892513, + "eval_runtime": 594.8325, + "eval_samples_per_second": 5.739, + "eval_selector_accuracy": 98.5981, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.18, + "step": 1256 + }, + { + "epoch": 4.0127642600717985, + "grad_norm": 0.2515179216861725, + "learning_rate": 9.464081721960592e-06, + "loss": 0.0021197797730565073, + "step": 1260 + }, + { + "epoch": 4.076585560430794, + "grad_norm": 0.052773285657167435, + "learning_rate": 8.331628936475818e-06, + "loss": 0.0007191885262727738, + "step": 1280 + }, + { + "epoch": 4.140406860789788, + "grad_norm": 0.020960243418812752, + "learning_rate": 7.2659843568748385e-06, + "loss": 0.0012289942242205143, + "step": 1300 + }, + { + "epoch": 4.204228161148784, + "grad_norm": 0.009606960229575634, + "learning_rate": 6.268854516770008e-06, + "loss": 0.0012783183716237544, + "step": 1320 + }, + { + "epoch": 4.268049461507778, + "grad_norm": 0.03356768190860748, + "learning_rate": 5.341836229602331e-06, + "loss": 0.0015904821455478668, + "step": 1340 + }, + { + "epoch": 4.331870761866773, + "grad_norm": 0.43399015069007874, + "learning_rate": 4.4864140314888216e-06, + "loss": 0.0019017960876226425, + "step": 1360 + }, + { + "epoch": 4.395692062225768, + "grad_norm": 0.09213124215602875, + "learning_rate": 3.703957803871932e-06, + "loss": 0.000992914568632841, + "step": 1380 + }, + { + "epoch": 4.4595133625847625, + "grad_norm": 0.030944187194108963, + "learning_rate": 2.9957205797780495e-06, + "loss": 0.0019235227257013321, + "step": 1400 + }, + { + "epoch": 4.523334662943758, + "grad_norm": 0.04038539156317711, + "learning_rate": 2.362836537198199e-06, + "loss": 0.001333768479526043, + "step": 1420 + }, + { + "epoch": 4.587155963302752, + "grad_norm": 0.009491882286965847, + "learning_rate": 1.8063191828043703e-06, + "loss": 0.0010302967391908168, + "step": 1440 + }, + { + "epoch": 4.650977263661747, + "grad_norm": 0.02417566254734993, + "learning_rate": 1.3270597289100437e-06, + "loss": 0.0008206027559936047, + "step": 1460 + }, + { + "epoch": 4.714798564020742, + "grad_norm": 0.0027866039890795946, + "learning_rate": 9.258256662741205e-07, + "loss": 0.0008394632488489151, + "step": 1480 + }, + { + "epoch": 4.778619864379737, + "grad_norm": 0.01856832019984722, + "learning_rate": 6.032595350337341e-07, + "loss": 0.0011719820089638234, + "step": 1500 + }, + { + "epoch": 4.842441164738732, + "grad_norm": 0.013062495738267899, + "learning_rate": 3.5987789573421356e-07, + "loss": 0.0029238536953926086, + "step": 1520 + }, + { + "epoch": 4.9062624650977265, + "grad_norm": 0.094195157289505, + "learning_rate": 1.9607050210401192e-07, + "loss": 0.0008629086427390575, + "step": 1540 + }, + { + "epoch": 4.970083765456721, + "grad_norm": 0.02496415562927723, + "learning_rate": 1.1209967689925832e-07, + "loss": 0.0011463512666523457, + "step": 1560 + }, + { + "epoch": 5.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 96.4953, + "eval_generator_rouge1": 99.9278, + "eval_generator_rouge2": 99.8599, + "eval_generator_rougeL": 99.777, + "eval_loss": 0.003217515302821994, + "eval_runtime": 592.4524, + "eval_samples_per_second": 5.762, + "eval_selector_accuracy": 98.7734, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.181, + "step": 1570 + }, + { + "epoch": 5.0, + "step": 1570, + "total_flos": 5.871444840310374e+16, + "train_loss": 0.011652989937051847, + "train_runtime": 11256.8941, + "train_samples_per_second": 4.454, + "train_steps_per_second": 0.139 + } + ], + "logging_steps": 20, + "max_steps": 1570, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 5.871444840310374e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/qwen3/distillm_adaptive_sfkl/training_args.bin b/qwen3/distillm_adaptive_sfkl/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..9965b3704c5694d942c823f03f32ef4aa365d5a2 --- /dev/null +++ b/qwen3/distillm_adaptive_sfkl/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4080ce6c122d894dce2a6242fb58ae879822809092a9484cf3c306a91aacf376 +size 7185 diff --git a/qwen3/distillm_adaptive_srkl/README.md b/qwen3/distillm_adaptive_srkl/README.md new file mode 100644 index 0000000000000000000000000000000000000000..939b1c61677e2c9bcbc7efbeb1532f7e1efcaaa2 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/README.md @@ -0,0 +1,79 @@ +--- +library_name: peft +license: other +base_model: Qwen/Qwen3-0.6B +tags: +- base_model:adapter:Qwen/Qwen3-0.6B +- llama-factory +- lora +- transformers +pipeline_tag: text-generation +model-index: +- name: distillm_adaptive_srkl + results: [] +--- + + + +# distillm_adaptive_srkl + +This model is a fine-tuned version of [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) on the cypher_prepared_train dataset. +It achieves the following results on the evaluation set: +- Loss: 0.0035 +- Selector Count: 1712.0 +- Generator Count: 1712.0 +- Selector Accuracy: 98.7734 +- Generator Exact Match: 96.9042 +- Generator Rouge1: 99.9492 +- Generator Rouge2: 99.899 +- Generator Rougel: 99.7905 + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 0.0001 +- train_batch_size: 2 +- eval_batch_size: 16 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 2 +- gradient_accumulation_steps: 8 +- total_train_batch_size: 32 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine_with_min_lr +- num_epochs: 5 + +### Training results + +| Training Loss | Epoch | Step | Validation Loss | Selector Count | Generator Count | Selector Accuracy | Generator Exact Match | Generator Rouge1 | Generator Rouge2 | Generator Rougel | +|:-------------:|:-----:|:----:|:---------------:|:--------------:|:---------------:|:-----------------:|:---------------------:|:----------------:|:----------------:|:----------------:| +| 0.0095 | 1.0 | 314 | 0.0135 | 1712.0 | 1712.0 | 89.9533 | 85.3972 | 99.3064 | 98.9501 | 98.7596 | +| 0.0037 | 2.0 | 628 | 0.0053 | 1712.0 | 1712.0 | 97.7804 | 94.9766 | 99.8874 | 99.7935 | 99.6469 | +| 0.0025 | 3.0 | 942 | 0.0046 | 1712.0 | 1712.0 | 98.014 | 95.6192 | 99.9324 | 99.8567 | 99.7224 | +| 0.0007 | 4.0 | 1256 | 0.0036 | 1712.0 | 1712.0 | 98.1893 | 96.2617 | 99.9463 | 99.8925 | 99.7351 | +| 0.0012 | 5.0 | 1570 | 0.0035 | 1712.0 | 1712.0 | 98.6565 | 96.5537 | 99.9447 | 99.8878 | 99.764 | + + +### Framework versions + +- PEFT 0.18.1 +- Transformers 5.8.0 +- Pytorch 2.9.1+cu128 +- Datasets 4.0.0 +- Tokenizers 0.22.2 \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/adapter_config.json b/qwen3/distillm_adaptive_srkl/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa3291dd38d42da741ab7ac66ac0e7b712287fa0 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-0.6B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/adapter_model.safetensors b/qwen3/distillm_adaptive_srkl/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..821356e29283573fa40c63b52acceee925c617b9 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:032a6e24d0e66536585b1cb2d85d49c3eccc6095f574a95f40aa92f550f07a6d +size 40422560 diff --git a/qwen3/distillm_adaptive_srkl/all_results.json b/qwen3/distillm_adaptive_srkl/all_results.json new file mode 100644 index 0000000000000000000000000000000000000000..0d69916da44794e2e68cefaf44c60071666c73f4 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/all_results.json @@ -0,0 +1,24 @@ +{ + "epoch": 5.0, + "eval_distillm_threshold": 0.0, + "eval_distillm_threshold_changed": 0.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 96.9042, + "eval_generator_rouge1": 99.9492, + "eval_generator_rouge2": 99.899, + "eval_generator_rougeL": 99.7905, + "eval_loss": 0.003477960592135787, + "eval_rollout_dataset_steps": 12535, + "eval_rollout_fresh_steps": 0, + "eval_rollout_replay_steps": 0, + "eval_runtime": 580.143, + "eval_samples_per_second": 5.885, + "eval_selector_accuracy": 98.7734, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.184, + "total_flos": 5.871444840310374e+16, + "train_loss": 0.013134213612907251, + "train_runtime": 10459.448, + "train_samples_per_second": 4.793, + "train_steps_per_second": 0.15 +} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/chat_template.jinja b/qwen3/distillm_adaptive_srkl/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1256/README.md b/qwen3/distillm_adaptive_srkl/checkpoint-1256/README.md new file mode 100644 index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1256/README.md @@ -0,0 +1,208 @@ +--- +base_model: Qwen/Qwen3-0.6B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-0.6B +- llama-factory +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1256/adapter_config.json b/qwen3/distillm_adaptive_srkl/checkpoint-1256/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa3291dd38d42da741ab7ac66ac0e7b712287fa0 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1256/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-0.6B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1256/adapter_model.safetensors b/qwen3/distillm_adaptive_srkl/checkpoint-1256/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..553064b58cc08043f9d0c810a22e6ee5acd1d599 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1256/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b27ed4690aa3a13b0ec69d237692c7feb512db08e7ecc96296963d93234c1a37 +size 40422560 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1256/chat_template.jinja b/qwen3/distillm_adaptive_srkl/checkpoint-1256/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1256/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1256/distillm_state_rank0.pt b/qwen3/distillm_adaptive_srkl/checkpoint-1256/distillm_state_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba55d0738648d06c6b1157efaafadb891c3ede66 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1256/distillm_state_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a9b5a84ccbbe30ec49486231d2b6d24ec7caa2a645eaeb1ef1ac1046bacb30f +size 9403 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1256/distillm_state_rank1.pt b/qwen3/distillm_adaptive_srkl/checkpoint-1256/distillm_state_rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d6f6e0743fd8e8410d9dfaa520a36a19db24f2f --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1256/distillm_state_rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55b029e6dfb2341a0bd92c4bf670fd9d1504f4e3c5bfe7a81b0984f5e033f257 +size 9339 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/distillm_adaptive_srkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..12f080907e9be055cab2670dba0b91d05ff8bf02 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86aa0a0ce5271ea4787655d0e1024f759b22039596227ac25bb9cee1668aed6d +size 121135557 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/distillm_adaptive_srkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b6adba9c6a63b60d51f09b9257951a8d2e06d46 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab414addc3d2fbb92d26560edfed29a2b5f9534c7bff0a3a1cf5a1be77f5efca +size 121135749 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt b/qwen3/distillm_adaptive_srkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..be8fe89eb2ea3fac340ffc1a76c5bf0603359a09 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba3aa53aaa40b350ca8b02f8724c3ce623ebdc91ef564645a9443788b765c244 +size 351841765 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1256/latest b/qwen3/distillm_adaptive_srkl/checkpoint-1256/latest new file mode 100644 index 0000000000000000000000000000000000000000..74abb3d23ea83ebdac59918752db5d1782834ca7 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1256/latest @@ -0,0 +1 @@ +global_step1256 \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1256/rng_state_0.pth b/qwen3/distillm_adaptive_srkl/checkpoint-1256/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..fedc1ee585ad169dec294fec3b1be40b3b5305c8 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1256/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d72ceed2e87ac660095154ed4ea25bb1a12b464ddbe9c56fe118eb449217c9cb +size 14917 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1256/rng_state_1.pth b/qwen3/distillm_adaptive_srkl/checkpoint-1256/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..537f4c729f4bb5085310a1af069f6a1d527fdaf9 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1256/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd5fbee2088191f9b98eb193978cc98cd9988f26d6f602cf6f2151e5edfe0a66 +size 14917 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1256/scheduler.pt b/qwen3/distillm_adaptive_srkl/checkpoint-1256/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..40e863d07cfed22cd912014eb3ba3fd2b16b173f --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1256/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1775297f14e75f7463ae51983a66296f81f4b2e2e5d7234f1a6d295abfa4cf23 +size 1465 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1256/tokenizer.json b/qwen3/distillm_adaptive_srkl/checkpoint-1256/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1256/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1256/tokenizer_config.json b/qwen3/distillm_adaptive_srkl/checkpoint-1256/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1256/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1256/trainer_state.json b/qwen3/distillm_adaptive_srkl/checkpoint-1256/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ffbd54c60c245bce0ead51e01d860c70b1c89d30 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1256/trainer_state.json @@ -0,0 +1,543 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1256, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "distillm_initial_generator_count": 1712.0, + "distillm_initial_generator_exact_match": 0.0, + "distillm_initial_generator_rouge1": 16.2149, + "distillm_initial_generator_rouge2": 3.2276, + "distillm_initial_generator_rougeL": 11.6725, + "distillm_initial_loss": 2.494271755218506, + "distillm_initial_runtime": 2946.4402, + "distillm_initial_samples_per_second": 1.159, + "distillm_initial_selector_accuracy": 52.278, + "distillm_initial_selector_count": 1712.0, + "distillm_initial_steps_per_second": 0.036, + "epoch": 0, + "step": 0 + }, + { + "epoch": 0.06382130035899482, + "grad_norm": 0.9302584528923035, + "learning_rate": 9.996390385474569e-05, + "loss": 0.375650691986084, + "step": 20 + }, + { + "epoch": 0.12764260071798963, + "grad_norm": 0.819491982460022, + "learning_rate": 9.984797509888031e-05, + "loss": 0.10328874588012696, + "step": 40 + }, + { + "epoch": 0.19146390107698444, + "grad_norm": 0.9121766686439514, + "learning_rate": 9.965229938050925e-05, + "loss": 0.06860063076019288, + "step": 60 + }, + { + "epoch": 0.25528520143597927, + "grad_norm": 0.3703226149082184, + "learning_rate": 9.937719005663432e-05, + "loss": 0.055023026466369626, + "step": 80 + }, + { + "epoch": 0.3191065017949741, + "grad_norm": 0.6126606464385986, + "learning_rate": 9.902308769000724e-05, + "loss": 0.03998597264289856, + "step": 100 + }, + { + "epoch": 0.3829278021539689, + "grad_norm": 1.0489412546157837, + "learning_rate": 9.859055934360789e-05, + "loss": 0.03420835733413696, + "step": 120 + }, + { + "epoch": 0.4467491025129637, + "grad_norm": 0.3606373965740204, + "learning_rate": 9.808029767254437e-05, + "loss": 0.03716553449630737, + "step": 140 + }, + { + "epoch": 0.5105704028719585, + "grad_norm": 1.0114212036132812, + "learning_rate": 9.749311981482817e-05, + "loss": 0.02128121256828308, + "step": 160 + }, + { + "epoch": 0.5743917032309533, + "grad_norm": 0.22879505157470703, + "learning_rate": 9.682996608280154e-05, + "loss": 0.016371789574623107, + "step": 180 + }, + { + "epoch": 0.6382130035899481, + "grad_norm": 0.40647953748703003, + "learning_rate": 9.609189845731207e-05, + "loss": 0.02666616439819336, + "step": 200 + }, + { + "epoch": 0.702034303948943, + "grad_norm": 0.38679832220077515, + "learning_rate": 9.52800988870462e-05, + "loss": 0.022025266289711, + "step": 220 + }, + { + "epoch": 0.7658556043079378, + "grad_norm": 0.2795315384864807, + "learning_rate": 9.439586739574513e-05, + "loss": 0.00786939337849617, + "step": 240 + }, + { + "epoch": 0.8296769046669326, + "grad_norm": 0.3553146421909332, + "learning_rate": 9.344062000033414e-05, + "loss": 0.010387377440929412, + "step": 260 + }, + { + "epoch": 0.8934982050259274, + "grad_norm": 0.20546464622020721, + "learning_rate": 9.241588644329919e-05, + "loss": 0.009576073288917542, + "step": 280 + }, + { + "epoch": 0.9573195053849222, + "grad_norm": 0.2525307536125183, + "learning_rate": 9.132330774294252e-05, + "loss": 0.009539780020713807, + "step": 300 + }, + { + "epoch": 1.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 85.3972, + "eval_generator_rouge1": 99.3064, + "eval_generator_rouge2": 98.9501, + "eval_generator_rougeL": 98.7596, + "eval_loss": 0.013466483913362026, + "eval_runtime": 592.176, + "eval_samples_per_second": 5.765, + "eval_selector_accuracy": 89.9533, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.181, + "step": 314 + }, + { + "epoch": 1.0191463901076985, + "grad_norm": 0.7174072265625, + "learning_rate": 9.01646335654399e-05, + "loss": 0.017302539944648743, + "step": 320 + }, + { + "epoch": 1.0829676904666932, + "grad_norm": 0.11603154987096786, + "learning_rate": 8.894171942290817e-05, + "loss": 0.008357083797454834, + "step": 340 + }, + { + "epoch": 1.146788990825688, + "grad_norm": 0.19480980932712555, + "learning_rate": 8.765652370197003e-05, + "loss": 0.005870707705616951, + "step": 360 + }, + { + "epoch": 1.210610291184683, + "grad_norm": 0.16763336956501007, + "learning_rate": 8.631110452757467e-05, + "loss": 0.0035324640572071075, + "step": 380 + }, + { + "epoch": 1.2744315915436777, + "grad_norm": 0.03674647584557533, + "learning_rate": 8.490761646709642e-05, + "loss": 0.005342463403940201, + "step": 400 + }, + { + "epoch": 1.3382528919026724, + "grad_norm": 0.0998690128326416, + "learning_rate": 8.344830707998953e-05, + "loss": 0.006832928955554962, + "step": 420 + }, + { + "epoch": 1.4020741922616673, + "grad_norm": 0.13103976845741272, + "learning_rate": 8.193551331852458e-05, + "loss": 0.005545094236731529, + "step": 440 + }, + { + "epoch": 1.4658954926206622, + "grad_norm": 0.4100915491580963, + "learning_rate": 8.037165778537033e-05, + "loss": 0.007910171896219254, + "step": 460 + }, + { + "epoch": 1.5297167929796571, + "grad_norm": 0.14175599813461304, + "learning_rate": 7.875924485401427e-05, + "loss": 0.0031586486846208574, + "step": 480 + }, + { + "epoch": 1.5935380933386518, + "grad_norm": 0.19752922654151917, + "learning_rate": 7.710085665823444e-05, + "loss": 0.004655975103378296, + "step": 500 + }, + { + "epoch": 1.6573593936976465, + "grad_norm": 0.03805746138095856, + "learning_rate": 7.539914895704521e-05, + "loss": 0.0038994356989860536, + "step": 520 + }, + { + "epoch": 1.7211806940566414, + "grad_norm": 0.7704943418502808, + "learning_rate": 7.365684688173908e-05, + "loss": 0.008619165420532227, + "step": 540 + }, + { + "epoch": 1.7850019944156363, + "grad_norm": 0.10960758477449417, + "learning_rate": 7.18767405718347e-05, + "loss": 0.004270360246300698, + "step": 560 + }, + { + "epoch": 1.848823294774631, + "grad_norm": 0.5830378532409668, + "learning_rate": 7.006168070692036e-05, + "loss": 0.010602302849292755, + "step": 580 + }, + { + "epoch": 1.9126445951336257, + "grad_norm": 0.1323094367980957, + "learning_rate": 6.821457394154786e-05, + "loss": 0.004179984703660012, + "step": 600 + }, + { + "epoch": 1.9764658954926206, + "grad_norm": 0.06579075008630753, + "learning_rate": 6.633837825048768e-05, + "loss": 0.0037369213998317717, + "step": 620 + }, + { + "epoch": 2.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 94.9766, + "eval_generator_rouge1": 99.8874, + "eval_generator_rouge2": 99.7935, + "eval_generator_rougeL": 99.6469, + "eval_loss": 0.005270988680422306, + "eval_runtime": 592.8427, + "eval_samples_per_second": 5.759, + "eval_selector_accuracy": 97.7804, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.18, + "step": 628 + }, + { + "epoch": 2.038292780215397, + "grad_norm": 0.08518785238265991, + "learning_rate": 6.443609819179926e-05, + "loss": 0.001967618428170681, + "step": 640 + }, + { + "epoch": 2.102114080574392, + "grad_norm": 0.1549246907234192, + "learning_rate": 6.251078009530256e-05, + "loss": 0.0023793572559952735, + "step": 660 + }, + { + "epoch": 2.1659353809333863, + "grad_norm": 0.08672250062227249, + "learning_rate": 6.0565507184155677e-05, + "loss": 0.004786616936326027, + "step": 680 + }, + { + "epoch": 2.2297566812923812, + "grad_norm": 0.03346117213368416, + "learning_rate": 5.860339463735136e-05, + "loss": 0.002478102594614029, + "step": 700 + }, + { + "epoch": 2.293577981651376, + "grad_norm": 0.015912503004074097, + "learning_rate": 5.662758460103895e-05, + "loss": 0.002031809091567993, + "step": 720 + }, + { + "epoch": 2.357399282010371, + "grad_norm": 0.03547865152359009, + "learning_rate": 5.464124115666095e-05, + "loss": 0.004523729532957077, + "step": 740 + }, + { + "epoch": 2.421220582369366, + "grad_norm": 0.007262590806931257, + "learning_rate": 5.264754525396212e-05, + "loss": 0.0013824908062815666, + "step": 760 + }, + { + "epoch": 2.4850418827283605, + "grad_norm": 0.024668537080287933, + "learning_rate": 5.06496896169857e-05, + "loss": 0.0008345138281583786, + "step": 780 + }, + { + "epoch": 2.5488631830873554, + "grad_norm": 0.07706174999475479, + "learning_rate": 4.865087363121384e-05, + "loss": 0.0049762561917305, + "step": 800 + }, + { + "epoch": 2.6126844834463503, + "grad_norm": 0.25065284967422485, + "learning_rate": 4.6654298220040673e-05, + "loss": 0.00360473096370697, + "step": 820 + }, + { + "epoch": 2.6765057838053448, + "grad_norm": 0.0715489313006401, + "learning_rate": 4.466316071878229e-05, + "loss": 0.001436175312846899, + "step": 840 + }, + { + "epoch": 2.74032708416434, + "grad_norm": 0.116939976811409, + "learning_rate": 4.2680649754432913e-05, + "loss": 0.0016343716531991958, + "step": 860 + }, + { + "epoch": 2.8041483845233346, + "grad_norm": 0.01825389452278614, + "learning_rate": 4.070994013936646e-05, + "loss": 0.0012953002005815506, + "step": 880 + }, + { + "epoch": 2.8679696848823295, + "grad_norm": 0.10613884776830673, + "learning_rate": 3.8754187787161155e-05, + "loss": 0.000981786847114563, + "step": 900 + }, + { + "epoch": 2.9317909852413244, + "grad_norm": 0.00612761452794075, + "learning_rate": 3.681652465868875e-05, + "loss": 0.0015057422220706939, + "step": 920 + }, + { + "epoch": 2.995612285600319, + "grad_norm": 0.10040487349033356, + "learning_rate": 3.490005374656184e-05, + "loss": 0.0025246324017643928, + "step": 940 + }, + { + "epoch": 3.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 95.6192, + "eval_generator_rouge1": 99.9324, + "eval_generator_rouge2": 99.8567, + "eval_generator_rougeL": 99.7224, + "eval_loss": 0.004592898301780224, + "eval_runtime": 596.6943, + "eval_samples_per_second": 5.722, + "eval_selector_accuracy": 98.014, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.179, + "step": 942 + }, + { + "epoch": 3.057439170323095, + "grad_norm": 0.015967005863785744, + "learning_rate": 3.3007844105971336e-05, + "loss": 0.004021823406219482, + "step": 960 + }, + { + "epoch": 3.12126047068209, + "grad_norm": 0.001990415621548891, + "learning_rate": 3.114292593987149e-05, + "loss": 0.002897568792104721, + "step": 980 + }, + { + "epoch": 3.185081771041085, + "grad_norm": 0.007704610005021095, + "learning_rate": 2.9308285746383408e-05, + "loss": 0.00075986348092556, + "step": 1000 + }, + { + "epoch": 3.24890307140008, + "grad_norm": 0.026389382779598236, + "learning_rate": 2.7506861536187867e-05, + "loss": 0.0032131548970937727, + "step": 1020 + }, + { + "epoch": 3.3127243717590744, + "grad_norm": 0.03544643148779869, + "learning_rate": 2.5741538127566396e-05, + "loss": 0.0007816605269908905, + "step": 1040 + }, + { + "epoch": 3.3765456721180693, + "grad_norm": 0.06217798963189125, + "learning_rate": 2.4015142526625115e-05, + "loss": 0.0025276828557252882, + "step": 1060 + }, + { + "epoch": 3.4403669724770642, + "grad_norm": 0.08190753310918808, + "learning_rate": 2.233043940009967e-05, + "loss": 0.0021929899230599403, + "step": 1080 + }, + { + "epoch": 3.504188272836059, + "grad_norm": 0.006671294569969177, + "learning_rate": 2.0690126647990974e-05, + "loss": 0.0009399710223078727, + "step": 1100 + }, + { + "epoch": 3.568009573195054, + "grad_norm": 0.028819965198636055, + "learning_rate": 1.9096831083121828e-05, + "loss": 0.0007333569228649139, + "step": 1120 + }, + { + "epoch": 3.6318308735540485, + "grad_norm": 0.10018011927604675, + "learning_rate": 1.7553104224533324e-05, + "loss": 0.0007428646087646485, + "step": 1140 + }, + { + "epoch": 3.6956521739130435, + "grad_norm": 1.5251667499542236, + "learning_rate": 1.606141821145735e-05, + "loss": 0.0033368520438671113, + "step": 1160 + }, + { + "epoch": 3.7594734742720384, + "grad_norm": 0.006973282899707556, + "learning_rate": 1.4624161844408763e-05, + "loss": 0.0012252738699316979, + "step": 1180 + }, + { + "epoch": 3.823294774631033, + "grad_norm": 0.005893409717828035, + "learning_rate": 1.324363675973698e-05, + "loss": 0.0025427818298339845, + "step": 1200 + }, + { + "epoch": 3.8871160749900278, + "grad_norm": 0.055641546845436096, + "learning_rate": 1.1922053743763052e-05, + "loss": 0.0033231116831302643, + "step": 1220 + }, + { + "epoch": 3.9509373753490227, + "grad_norm": 0.011760273016989231, + "learning_rate": 1.0661529192404973e-05, + "loss": 0.0006555643863976002, + "step": 1240 + }, + { + "epoch": 4.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 96.2617, + "eval_generator_rouge1": 99.9463, + "eval_generator_rouge2": 99.8925, + "eval_generator_rougeL": 99.7351, + "eval_loss": 0.00355384754948318, + "eval_runtime": 597.0533, + "eval_samples_per_second": 5.718, + "eval_selector_accuracy": 98.1893, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.179, + "step": 1256 + } + ], + "logging_steps": 20, + "max_steps": 1570, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.697643190059008e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1256/training_args.bin b/qwen3/distillm_adaptive_srkl/checkpoint-1256/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c03c7a9fc6ee30bf7a43346e8a9ee99e30a6079d --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1256/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9796a6c4dfa7bcfc7531b6a3163315134b78052b383dd8ed82be57a88f7087f9 +size 7185 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1256/zero_to_fp32.py b/qwen3/distillm_adaptive_srkl/checkpoint-1256/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1256/zero_to_fp32.py @@ -0,0 +1,790 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY, + AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY, + AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def _has_autoep_zero3_partitioned_metadata(state_dict): + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY) + if autoep_layers is None: + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY) + if not isinstance(autoep_layers, list): + return False + return any( + isinstance(entry, dict) + and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT + for entry in autoep_layers) + + +def _raise_if_autoep_zero3_partitioned_state(state_dict): + if _has_autoep_zero3_partitioned_metadata(state_dict): + raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. " + "AutoEP expert parameters are partitioned over expert replica groups, so " + "global data-parallel consolidation would produce incomplete expert tensors. " + "Use ds_to_universal.py for expert-aware conversion.") + + +def _raise_if_autoep_zero3_partitioned_checkpoint(model_files): + for file in model_files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints + # before the expensive optimizer-shard load below. + model_files = get_model_state_files(ds_checkpoint_dir) + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1570/README.md b/qwen3/distillm_adaptive_srkl/checkpoint-1570/README.md new file mode 100644 index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1570/README.md @@ -0,0 +1,208 @@ +--- +base_model: Qwen/Qwen3-0.6B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-0.6B +- llama-factory +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1570/adapter_config.json b/qwen3/distillm_adaptive_srkl/checkpoint-1570/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa3291dd38d42da741ab7ac66ac0e7b712287fa0 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1570/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-0.6B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1570/adapter_model.safetensors b/qwen3/distillm_adaptive_srkl/checkpoint-1570/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..821356e29283573fa40c63b52acceee925c617b9 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1570/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:032a6e24d0e66536585b1cb2d85d49c3eccc6095f574a95f40aa92f550f07a6d +size 40422560 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1570/chat_template.jinja b/qwen3/distillm_adaptive_srkl/checkpoint-1570/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1570/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1570/distillm_state_rank0.pt b/qwen3/distillm_adaptive_srkl/checkpoint-1570/distillm_state_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..becca175688219a9b4a422f423edefb47590217d --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1570/distillm_state_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c45fea5636ba58d552a49ba5a651afe08fe7e86b64bd29d82359e1ffb3df111f +size 9403 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1570/distillm_state_rank1.pt b/qwen3/distillm_adaptive_srkl/checkpoint-1570/distillm_state_rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d7c319e37a117a78ed636f3109afc6ef3c8fd98 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1570/distillm_state_rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34b6ffca878fd86b6133b9eb70d94fee805ff00ef1d33571b54dc0de3a307df3 +size 9339 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/distillm_adaptive_srkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..53595d22268c488da275bd955358b6fd6ad9d70b --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa749e02f92099c5e4c1a9cf23e93b1dded49751383885dcc40d5eac712f6b84 +size 121135557 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/distillm_adaptive_srkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..43e03766c6d88ee14baac90f993ffcf0bb801444 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b50b439a0c58c9c9a00725ad26c0855d6d66c9fb0005c1339028d70dd0a223a +size 121135749 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt b/qwen3/distillm_adaptive_srkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..33cccba1c7d2765e2876d46b3cbd7fef233fc6df --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2369eabce3c6721b7644d134a3b1e663be9117583f310e4d24461a26bf556a65 +size 351841765 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1570/latest b/qwen3/distillm_adaptive_srkl/checkpoint-1570/latest new file mode 100644 index 0000000000000000000000000000000000000000..18c2fcff58fa6daa0a2e7c3a015552a69085778b --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1570/latest @@ -0,0 +1 @@ +global_step1570 \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1570/rng_state_0.pth b/qwen3/distillm_adaptive_srkl/checkpoint-1570/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..acca61ed8ebf072a2976f71efc031234461e8f45 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1570/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:915c9e8e14c76dc9f483b76582cf4077e1d6054d4269fb8c102409f26a917ba0 +size 14917 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1570/rng_state_1.pth b/qwen3/distillm_adaptive_srkl/checkpoint-1570/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..c64535cda167f0a1b022bb998c305c89e054cf03 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1570/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e57a6093e547a7f4bd7f4a3e56236a6f02a4279bc1802ac29e85bd374bc430c +size 14917 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1570/scheduler.pt b/qwen3/distillm_adaptive_srkl/checkpoint-1570/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..e22954463492d8a070ae427c95418df61893c54c --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1570/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6332fac17ae99a07597512c705658da2fd8429d1c39bb776b30e107c2cb7aae +size 1465 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1570/tokenizer.json b/qwen3/distillm_adaptive_srkl/checkpoint-1570/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1570/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1570/tokenizer_config.json b/qwen3/distillm_adaptive_srkl/checkpoint-1570/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1570/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1570/trainer_state.json b/qwen3/distillm_adaptive_srkl/checkpoint-1570/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6dad519a292ed161b325f9acfbfc42bf560c209b --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1570/trainer_state.json @@ -0,0 +1,670 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 1570, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "distillm_initial_generator_count": 1712.0, + "distillm_initial_generator_exact_match": 0.0, + "distillm_initial_generator_rouge1": 16.2149, + "distillm_initial_generator_rouge2": 3.2276, + "distillm_initial_generator_rougeL": 11.6725, + "distillm_initial_loss": 2.494271755218506, + "distillm_initial_runtime": 2946.4402, + "distillm_initial_samples_per_second": 1.159, + "distillm_initial_selector_accuracy": 52.278, + "distillm_initial_selector_count": 1712.0, + "distillm_initial_steps_per_second": 0.036, + "epoch": 0, + "step": 0 + }, + { + "epoch": 0.06382130035899482, + "grad_norm": 0.9302584528923035, + "learning_rate": 9.996390385474569e-05, + "loss": 0.375650691986084, + "step": 20 + }, + { + "epoch": 0.12764260071798963, + "grad_norm": 0.819491982460022, + "learning_rate": 9.984797509888031e-05, + "loss": 0.10328874588012696, + "step": 40 + }, + { + "epoch": 0.19146390107698444, + "grad_norm": 0.9121766686439514, + "learning_rate": 9.965229938050925e-05, + "loss": 0.06860063076019288, + "step": 60 + }, + { + "epoch": 0.25528520143597927, + "grad_norm": 0.3703226149082184, + "learning_rate": 9.937719005663432e-05, + "loss": 0.055023026466369626, + "step": 80 + }, + { + "epoch": 0.3191065017949741, + "grad_norm": 0.6126606464385986, + "learning_rate": 9.902308769000724e-05, + "loss": 0.03998597264289856, + "step": 100 + }, + { + "epoch": 0.3829278021539689, + "grad_norm": 1.0489412546157837, + "learning_rate": 9.859055934360789e-05, + "loss": 0.03420835733413696, + "step": 120 + }, + { + "epoch": 0.4467491025129637, + "grad_norm": 0.3606373965740204, + "learning_rate": 9.808029767254437e-05, + "loss": 0.03716553449630737, + "step": 140 + }, + { + "epoch": 0.5105704028719585, + "grad_norm": 1.0114212036132812, + "learning_rate": 9.749311981482817e-05, + "loss": 0.02128121256828308, + "step": 160 + }, + { + "epoch": 0.5743917032309533, + "grad_norm": 0.22879505157470703, + "learning_rate": 9.682996608280154e-05, + "loss": 0.016371789574623107, + "step": 180 + }, + { + "epoch": 0.6382130035899481, + "grad_norm": 0.40647953748703003, + "learning_rate": 9.609189845731207e-05, + "loss": 0.02666616439819336, + "step": 200 + }, + { + "epoch": 0.702034303948943, + "grad_norm": 0.38679832220077515, + "learning_rate": 9.52800988870462e-05, + "loss": 0.022025266289711, + "step": 220 + }, + { + "epoch": 0.7658556043079378, + "grad_norm": 0.2795315384864807, + "learning_rate": 9.439586739574513e-05, + "loss": 0.00786939337849617, + "step": 240 + }, + { + "epoch": 0.8296769046669326, + "grad_norm": 0.3553146421909332, + "learning_rate": 9.344062000033414e-05, + "loss": 0.010387377440929412, + "step": 260 + }, + { + "epoch": 0.8934982050259274, + "grad_norm": 0.20546464622020721, + "learning_rate": 9.241588644329919e-05, + "loss": 0.009576073288917542, + "step": 280 + }, + { + "epoch": 0.9573195053849222, + "grad_norm": 0.2525307536125183, + "learning_rate": 9.132330774294252e-05, + "loss": 0.009539780020713807, + "step": 300 + }, + { + "epoch": 1.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 85.3972, + "eval_generator_rouge1": 99.3064, + "eval_generator_rouge2": 98.9501, + "eval_generator_rougeL": 98.7596, + "eval_loss": 0.013466483913362026, + "eval_runtime": 592.176, + "eval_samples_per_second": 5.765, + "eval_selector_accuracy": 89.9533, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.181, + "step": 314 + }, + { + "epoch": 1.0191463901076985, + "grad_norm": 0.7174072265625, + "learning_rate": 9.01646335654399e-05, + "loss": 0.017302539944648743, + "step": 320 + }, + { + "epoch": 1.0829676904666932, + "grad_norm": 0.11603154987096786, + "learning_rate": 8.894171942290817e-05, + "loss": 0.008357083797454834, + "step": 340 + }, + { + "epoch": 1.146788990825688, + "grad_norm": 0.19480980932712555, + "learning_rate": 8.765652370197003e-05, + "loss": 0.005870707705616951, + "step": 360 + }, + { + "epoch": 1.210610291184683, + "grad_norm": 0.16763336956501007, + "learning_rate": 8.631110452757467e-05, + "loss": 0.0035324640572071075, + "step": 380 + }, + { + "epoch": 1.2744315915436777, + "grad_norm": 0.03674647584557533, + "learning_rate": 8.490761646709642e-05, + "loss": 0.005342463403940201, + "step": 400 + }, + { + "epoch": 1.3382528919026724, + "grad_norm": 0.0998690128326416, + "learning_rate": 8.344830707998953e-05, + "loss": 0.006832928955554962, + "step": 420 + }, + { + "epoch": 1.4020741922616673, + "grad_norm": 0.13103976845741272, + "learning_rate": 8.193551331852458e-05, + "loss": 0.005545094236731529, + "step": 440 + }, + { + "epoch": 1.4658954926206622, + "grad_norm": 0.4100915491580963, + "learning_rate": 8.037165778537033e-05, + "loss": 0.007910171896219254, + "step": 460 + }, + { + "epoch": 1.5297167929796571, + "grad_norm": 0.14175599813461304, + "learning_rate": 7.875924485401427e-05, + "loss": 0.0031586486846208574, + "step": 480 + }, + { + "epoch": 1.5935380933386518, + "grad_norm": 0.19752922654151917, + "learning_rate": 7.710085665823444e-05, + "loss": 0.004655975103378296, + "step": 500 + }, + { + "epoch": 1.6573593936976465, + "grad_norm": 0.03805746138095856, + "learning_rate": 7.539914895704521e-05, + "loss": 0.0038994356989860536, + "step": 520 + }, + { + "epoch": 1.7211806940566414, + "grad_norm": 0.7704943418502808, + "learning_rate": 7.365684688173908e-05, + "loss": 0.008619165420532227, + "step": 540 + }, + { + "epoch": 1.7850019944156363, + "grad_norm": 0.10960758477449417, + "learning_rate": 7.18767405718347e-05, + "loss": 0.004270360246300698, + "step": 560 + }, + { + "epoch": 1.848823294774631, + "grad_norm": 0.5830378532409668, + "learning_rate": 7.006168070692036e-05, + "loss": 0.010602302849292755, + "step": 580 + }, + { + "epoch": 1.9126445951336257, + "grad_norm": 0.1323094367980957, + "learning_rate": 6.821457394154786e-05, + "loss": 0.004179984703660012, + "step": 600 + }, + { + "epoch": 1.9764658954926206, + "grad_norm": 0.06579075008630753, + "learning_rate": 6.633837825048768e-05, + "loss": 0.0037369213998317717, + "step": 620 + }, + { + "epoch": 2.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 94.9766, + "eval_generator_rouge1": 99.8874, + "eval_generator_rouge2": 99.7935, + "eval_generator_rougeL": 99.6469, + "eval_loss": 0.005270988680422306, + "eval_runtime": 592.8427, + "eval_samples_per_second": 5.759, + "eval_selector_accuracy": 97.7804, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.18, + "step": 628 + }, + { + "epoch": 2.038292780215397, + "grad_norm": 0.08518785238265991, + "learning_rate": 6.443609819179926e-05, + "loss": 0.001967618428170681, + "step": 640 + }, + { + "epoch": 2.102114080574392, + "grad_norm": 0.1549246907234192, + "learning_rate": 6.251078009530256e-05, + "loss": 0.0023793572559952735, + "step": 660 + }, + { + "epoch": 2.1659353809333863, + "grad_norm": 0.08672250062227249, + "learning_rate": 6.0565507184155677e-05, + "loss": 0.004786616936326027, + "step": 680 + }, + { + "epoch": 2.2297566812923812, + "grad_norm": 0.03346117213368416, + "learning_rate": 5.860339463735136e-05, + "loss": 0.002478102594614029, + "step": 700 + }, + { + "epoch": 2.293577981651376, + "grad_norm": 0.015912503004074097, + "learning_rate": 5.662758460103895e-05, + "loss": 0.002031809091567993, + "step": 720 + }, + { + "epoch": 2.357399282010371, + "grad_norm": 0.03547865152359009, + "learning_rate": 5.464124115666095e-05, + "loss": 0.004523729532957077, + "step": 740 + }, + { + "epoch": 2.421220582369366, + "grad_norm": 0.007262590806931257, + "learning_rate": 5.264754525396212e-05, + "loss": 0.0013824908062815666, + "step": 760 + }, + { + "epoch": 2.4850418827283605, + "grad_norm": 0.024668537080287933, + "learning_rate": 5.06496896169857e-05, + "loss": 0.0008345138281583786, + "step": 780 + }, + { + "epoch": 2.5488631830873554, + "grad_norm": 0.07706174999475479, + "learning_rate": 4.865087363121384e-05, + "loss": 0.0049762561917305, + "step": 800 + }, + { + "epoch": 2.6126844834463503, + "grad_norm": 0.25065284967422485, + "learning_rate": 4.6654298220040673e-05, + "loss": 0.00360473096370697, + "step": 820 + }, + { + "epoch": 2.6765057838053448, + "grad_norm": 0.0715489313006401, + "learning_rate": 4.466316071878229e-05, + "loss": 0.001436175312846899, + "step": 840 + }, + { + "epoch": 2.74032708416434, + "grad_norm": 0.116939976811409, + "learning_rate": 4.2680649754432913e-05, + "loss": 0.0016343716531991958, + "step": 860 + }, + { + "epoch": 2.8041483845233346, + "grad_norm": 0.01825389452278614, + "learning_rate": 4.070994013936646e-05, + "loss": 0.0012953002005815506, + "step": 880 + }, + { + "epoch": 2.8679696848823295, + "grad_norm": 0.10613884776830673, + "learning_rate": 3.8754187787161155e-05, + "loss": 0.000981786847114563, + "step": 900 + }, + { + "epoch": 2.9317909852413244, + "grad_norm": 0.00612761452794075, + "learning_rate": 3.681652465868875e-05, + "loss": 0.0015057422220706939, + "step": 920 + }, + { + "epoch": 2.995612285600319, + "grad_norm": 0.10040487349033356, + "learning_rate": 3.490005374656184e-05, + "loss": 0.0025246324017643928, + "step": 940 + }, + { + "epoch": 3.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 95.6192, + "eval_generator_rouge1": 99.9324, + "eval_generator_rouge2": 99.8567, + "eval_generator_rougeL": 99.7224, + "eval_loss": 0.004592898301780224, + "eval_runtime": 596.6943, + "eval_samples_per_second": 5.722, + "eval_selector_accuracy": 98.014, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.179, + "step": 942 + }, + { + "epoch": 3.057439170323095, + "grad_norm": 0.015967005863785744, + "learning_rate": 3.3007844105971336e-05, + "loss": 0.004021823406219482, + "step": 960 + }, + { + "epoch": 3.12126047068209, + "grad_norm": 0.001990415621548891, + "learning_rate": 3.114292593987149e-05, + "loss": 0.002897568792104721, + "step": 980 + }, + { + "epoch": 3.185081771041085, + "grad_norm": 0.007704610005021095, + "learning_rate": 2.9308285746383408e-05, + "loss": 0.00075986348092556, + "step": 1000 + }, + { + "epoch": 3.24890307140008, + "grad_norm": 0.026389382779598236, + "learning_rate": 2.7506861536187867e-05, + "loss": 0.0032131548970937727, + "step": 1020 + }, + { + "epoch": 3.3127243717590744, + "grad_norm": 0.03544643148779869, + "learning_rate": 2.5741538127566396e-05, + "loss": 0.0007816605269908905, + "step": 1040 + }, + { + "epoch": 3.3765456721180693, + "grad_norm": 0.06217798963189125, + "learning_rate": 2.4015142526625115e-05, + "loss": 0.0025276828557252882, + "step": 1060 + }, + { + "epoch": 3.4403669724770642, + "grad_norm": 0.08190753310918808, + "learning_rate": 2.233043940009967e-05, + "loss": 0.0021929899230599403, + "step": 1080 + }, + { + "epoch": 3.504188272836059, + "grad_norm": 0.006671294569969177, + "learning_rate": 2.0690126647990974e-05, + "loss": 0.0009399710223078727, + "step": 1100 + }, + { + "epoch": 3.568009573195054, + "grad_norm": 0.028819965198636055, + "learning_rate": 1.9096831083121828e-05, + "loss": 0.0007333569228649139, + "step": 1120 + }, + { + "epoch": 3.6318308735540485, + "grad_norm": 0.10018011927604675, + "learning_rate": 1.7553104224533324e-05, + "loss": 0.0007428646087646485, + "step": 1140 + }, + { + "epoch": 3.6956521739130435, + "grad_norm": 1.5251667499542236, + "learning_rate": 1.606141821145735e-05, + "loss": 0.0033368520438671113, + "step": 1160 + }, + { + "epoch": 3.7594734742720384, + "grad_norm": 0.006973282899707556, + "learning_rate": 1.4624161844408763e-05, + "loss": 0.0012252738699316979, + "step": 1180 + }, + { + "epoch": 3.823294774631033, + "grad_norm": 0.005893409717828035, + "learning_rate": 1.324363675973698e-05, + "loss": 0.0025427818298339845, + "step": 1200 + }, + { + "epoch": 3.8871160749900278, + "grad_norm": 0.055641546845436096, + "learning_rate": 1.1922053743763052e-05, + "loss": 0.0033231116831302643, + "step": 1220 + }, + { + "epoch": 3.9509373753490227, + "grad_norm": 0.011760273016989231, + "learning_rate": 1.0661529192404973e-05, + "loss": 0.0006555643863976002, + "step": 1240 + }, + { + "epoch": 4.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 96.2617, + "eval_generator_rouge1": 99.9463, + "eval_generator_rouge2": 99.8925, + "eval_generator_rougeL": 99.7351, + "eval_loss": 0.00355384754948318, + "eval_runtime": 597.0533, + "eval_samples_per_second": 5.718, + "eval_selector_accuracy": 98.1893, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.179, + "step": 1256 + }, + { + "epoch": 4.0127642600717985, + "grad_norm": 0.35947123169898987, + "learning_rate": 9.464081721960592e-06, + "loss": 0.002213069796562195, + "step": 1260 + }, + { + "epoch": 4.076585560430794, + "grad_norm": 0.11643707007169724, + "learning_rate": 8.331628936475818e-06, + "loss": 0.0006028454750776291, + "step": 1280 + }, + { + "epoch": 4.140406860789788, + "grad_norm": 0.010412893258035183, + "learning_rate": 7.2659843568748385e-06, + "loss": 0.0008279280737042427, + "step": 1300 + }, + { + "epoch": 4.204228161148784, + "grad_norm": 0.008442528545856476, + "learning_rate": 6.268854516770008e-06, + "loss": 0.0014621550217270851, + "step": 1320 + }, + { + "epoch": 4.268049461507778, + "grad_norm": 0.029130427166819572, + "learning_rate": 5.341836229602331e-06, + "loss": 0.0010409696958959102, + "step": 1340 + }, + { + "epoch": 4.331870761866773, + "grad_norm": 0.3795068562030792, + "learning_rate": 4.4864140314888216e-06, + "loss": 0.0025711465626955033, + "step": 1360 + }, + { + "epoch": 4.395692062225768, + "grad_norm": 0.007296374533325434, + "learning_rate": 3.703957803871932e-06, + "loss": 0.000880935974419117, + "step": 1380 + }, + { + "epoch": 4.4595133625847625, + "grad_norm": 0.04088542237877846, + "learning_rate": 2.9957205797780495e-06, + "loss": 0.0011038152500987054, + "step": 1400 + }, + { + "epoch": 4.523334662943758, + "grad_norm": 0.03055841661989689, + "learning_rate": 2.362836537198199e-06, + "loss": 0.0015731073915958404, + "step": 1420 + }, + { + "epoch": 4.587155963302752, + "grad_norm": 0.016010979190468788, + "learning_rate": 1.8063191828043703e-06, + "loss": 0.0008220190182328224, + "step": 1440 + }, + { + "epoch": 4.650977263661747, + "grad_norm": 0.02590778097510338, + "learning_rate": 1.3270597289100437e-06, + "loss": 0.00046561416238546374, + "step": 1460 + }, + { + "epoch": 4.714798564020742, + "grad_norm": 0.00261318520642817, + "learning_rate": 9.258256662741205e-07, + "loss": 0.00044544278644025325, + "step": 1480 + }, + { + "epoch": 4.778619864379737, + "grad_norm": 0.010697306133806705, + "learning_rate": 6.032595350337341e-07, + "loss": 0.0008411692455410957, + "step": 1500 + }, + { + "epoch": 4.842441164738732, + "grad_norm": 0.011842342093586922, + "learning_rate": 3.5987789573421356e-07, + "loss": 0.003439590334892273, + "step": 1520 + }, + { + "epoch": 4.9062624650977265, + "grad_norm": 0.7014644742012024, + "learning_rate": 1.9607050210401192e-07, + "loss": 0.0016547713428735733, + "step": 1540 + }, + { + "epoch": 4.970083765456721, + "grad_norm": 0.03197485953569412, + "learning_rate": 1.1209967689925832e-07, + "loss": 0.0011944442056119442, + "step": 1560 + }, + { + "epoch": 5.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 96.5537, + "eval_generator_rouge1": 99.9447, + "eval_generator_rouge2": 99.8878, + "eval_generator_rougeL": 99.764, + "eval_loss": 0.003477960592135787, + "eval_runtime": 593.3647, + "eval_samples_per_second": 5.754, + "eval_selector_accuracy": 98.6565, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.18, + "step": 1570 + } + ], + "logging_steps": 20, + "max_steps": 1570, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 5.871444840310374e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1570/training_args.bin b/qwen3/distillm_adaptive_srkl/checkpoint-1570/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c03c7a9fc6ee30bf7a43346e8a9ee99e30a6079d --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1570/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9796a6c4dfa7bcfc7531b6a3163315134b78052b383dd8ed82be57a88f7087f9 +size 7185 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-1570/zero_to_fp32.py b/qwen3/distillm_adaptive_srkl/checkpoint-1570/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-1570/zero_to_fp32.py @@ -0,0 +1,790 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY, + AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY, + AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def _has_autoep_zero3_partitioned_metadata(state_dict): + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY) + if autoep_layers is None: + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY) + if not isinstance(autoep_layers, list): + return False + return any( + isinstance(entry, dict) + and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT + for entry in autoep_layers) + + +def _raise_if_autoep_zero3_partitioned_state(state_dict): + if _has_autoep_zero3_partitioned_metadata(state_dict): + raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. " + "AutoEP expert parameters are partitioned over expert replica groups, so " + "global data-parallel consolidation would produce incomplete expert tensors. " + "Use ds_to_universal.py for expert-aware conversion.") + + +def _raise_if_autoep_zero3_partitioned_checkpoint(model_files): + for file in model_files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints + # before the expensive optimizer-shard load below. + model_files = get_model_state_files(ds_checkpoint_dir) + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-314/README.md b/qwen3/distillm_adaptive_srkl/checkpoint-314/README.md new file mode 100644 index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-314/README.md @@ -0,0 +1,208 @@ +--- +base_model: Qwen/Qwen3-0.6B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-0.6B +- llama-factory +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-314/adapter_config.json b/qwen3/distillm_adaptive_srkl/checkpoint-314/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa3291dd38d42da741ab7ac66ac0e7b712287fa0 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-314/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-0.6B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-314/adapter_model.safetensors b/qwen3/distillm_adaptive_srkl/checkpoint-314/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..868bad6e34f9e18a55cfa25d60ec6f8fde387119 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-314/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:286aeaf2c745560dc8d12f0a59c417ea6376cd591946132db51b32ba8fbb75e1 +size 40422560 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-314/chat_template.jinja b/qwen3/distillm_adaptive_srkl/checkpoint-314/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-314/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-314/distillm_state_rank0.pt b/qwen3/distillm_adaptive_srkl/checkpoint-314/distillm_state_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..c92ac759103ef6b2a0cb377b030a5ee272ca1dbc --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-314/distillm_state_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9208c430a8d43fa0e53ede8a6e5eab75fb565f1ad94e1b66dd5a8fad4f2d279d +size 9403 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-314/distillm_state_rank1.pt b/qwen3/distillm_adaptive_srkl/checkpoint-314/distillm_state_rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..16e0bb3962fa3f007370bb097abbec17d52cdc05 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-314/distillm_state_rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06c0f9ff15c6f6d1dc5269643a7219fdaa4612ecfea5313c5b9359e6d4f4e8a3 +size 9339 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/distillm_adaptive_srkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dab3f235cc65f6e210a461e9c1f36fe93e8ce002 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ed3c77316e426b99c3001d4a8400487b494df70dd197d1aab9639cacf1bbeb4 +size 121135557 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/distillm_adaptive_srkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fdf6c18e9ab5743bba8ee55b810387152693dadc --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6441a1cca8bea7246e889f9a812cd5e8abfae42b66c0c60ced78d70837f1cb8 +size 121135749 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt b/qwen3/distillm_adaptive_srkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9bea2698de463ff3087ceafc5390752ceb53c584 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cbc5cf93d5483d518dd9b0af4d27574312d8d4f9bfeeb51040f8ea32e3b3611 +size 351841765 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-314/latest b/qwen3/distillm_adaptive_srkl/checkpoint-314/latest new file mode 100644 index 0000000000000000000000000000000000000000..a9786513b1a514db2c3a9da7408844635663d55d --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-314/latest @@ -0,0 +1 @@ +global_step314 \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-314/rng_state_0.pth b/qwen3/distillm_adaptive_srkl/checkpoint-314/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..24c30ccc5c35b4be9ca91a0ff675a91814b9a518 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-314/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dedb80d39ab5d9854d2ef2d6f0d3b518c18d4fccf1f6eb83d5259a31fcc6ec3 +size 14917 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-314/rng_state_1.pth b/qwen3/distillm_adaptive_srkl/checkpoint-314/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..f49270b908f3e0796261cc506ac4a07f555a99a9 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-314/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58fb1027661f11de2631778eece32452bc4ab321407c2a54db6b65d5bde91f8a +size 14917 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-314/scheduler.pt b/qwen3/distillm_adaptive_srkl/checkpoint-314/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..a3cb460802cda03bf0833a98a6f1428a3fd4f585 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-314/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17e254b5642bb75983fd5105bec79b35376cfbc99d29702287668b8767d46149 +size 1465 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-314/tokenizer.json b/qwen3/distillm_adaptive_srkl/checkpoint-314/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-314/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-314/tokenizer_config.json b/qwen3/distillm_adaptive_srkl/checkpoint-314/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-314/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-314/trainer_state.json b/qwen3/distillm_adaptive_srkl/checkpoint-314/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b042eafdc557fd8cff03ad6369f15d46f809f51d --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-314/trainer_state.json @@ -0,0 +1,169 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 314, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "distillm_initial_generator_count": 1712.0, + "distillm_initial_generator_exact_match": 0.0, + "distillm_initial_generator_rouge1": 16.2149, + "distillm_initial_generator_rouge2": 3.2276, + "distillm_initial_generator_rougeL": 11.6725, + "distillm_initial_loss": 2.494271755218506, + "distillm_initial_runtime": 2946.4402, + "distillm_initial_samples_per_second": 1.159, + "distillm_initial_selector_accuracy": 52.278, + "distillm_initial_selector_count": 1712.0, + "distillm_initial_steps_per_second": 0.036, + "epoch": 0, + "step": 0 + }, + { + "epoch": 0.06382130035899482, + "grad_norm": 0.9302584528923035, + "learning_rate": 9.996390385474569e-05, + "loss": 0.375650691986084, + "step": 20 + }, + { + "epoch": 0.12764260071798963, + "grad_norm": 0.819491982460022, + "learning_rate": 9.984797509888031e-05, + "loss": 0.10328874588012696, + "step": 40 + }, + { + "epoch": 0.19146390107698444, + "grad_norm": 0.9121766686439514, + "learning_rate": 9.965229938050925e-05, + "loss": 0.06860063076019288, + "step": 60 + }, + { + "epoch": 0.25528520143597927, + "grad_norm": 0.3703226149082184, + "learning_rate": 9.937719005663432e-05, + "loss": 0.055023026466369626, + "step": 80 + }, + { + "epoch": 0.3191065017949741, + "grad_norm": 0.6126606464385986, + "learning_rate": 9.902308769000724e-05, + "loss": 0.03998597264289856, + "step": 100 + }, + { + "epoch": 0.3829278021539689, + "grad_norm": 1.0489412546157837, + "learning_rate": 9.859055934360789e-05, + "loss": 0.03420835733413696, + "step": 120 + }, + { + "epoch": 0.4467491025129637, + "grad_norm": 0.3606373965740204, + "learning_rate": 9.808029767254437e-05, + "loss": 0.03716553449630737, + "step": 140 + }, + { + "epoch": 0.5105704028719585, + "grad_norm": 1.0114212036132812, + "learning_rate": 9.749311981482817e-05, + "loss": 0.02128121256828308, + "step": 160 + }, + { + "epoch": 0.5743917032309533, + "grad_norm": 0.22879505157470703, + "learning_rate": 9.682996608280154e-05, + "loss": 0.016371789574623107, + "step": 180 + }, + { + "epoch": 0.6382130035899481, + "grad_norm": 0.40647953748703003, + "learning_rate": 9.609189845731207e-05, + "loss": 0.02666616439819336, + "step": 200 + }, + { + "epoch": 0.702034303948943, + "grad_norm": 0.38679832220077515, + "learning_rate": 9.52800988870462e-05, + "loss": 0.022025266289711, + "step": 220 + }, + { + "epoch": 0.7658556043079378, + "grad_norm": 0.2795315384864807, + "learning_rate": 9.439586739574513e-05, + "loss": 0.00786939337849617, + "step": 240 + }, + { + "epoch": 0.8296769046669326, + "grad_norm": 0.3553146421909332, + "learning_rate": 9.344062000033414e-05, + "loss": 0.010387377440929412, + "step": 260 + }, + { + "epoch": 0.8934982050259274, + "grad_norm": 0.20546464622020721, + "learning_rate": 9.241588644329919e-05, + "loss": 0.009576073288917542, + "step": 280 + }, + { + "epoch": 0.9573195053849222, + "grad_norm": 0.2525307536125183, + "learning_rate": 9.132330774294252e-05, + "loss": 0.009539780020713807, + "step": 300 + }, + { + "epoch": 1.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 85.3972, + "eval_generator_rouge1": 99.3064, + "eval_generator_rouge2": 98.9501, + "eval_generator_rougeL": 98.7596, + "eval_loss": 0.013466483913362026, + "eval_runtime": 592.176, + "eval_samples_per_second": 5.765, + "eval_selector_accuracy": 89.9533, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.181, + "step": 314 + } + ], + "logging_steps": 20, + "max_steps": 1570, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.172842003562496e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-314/training_args.bin b/qwen3/distillm_adaptive_srkl/checkpoint-314/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c03c7a9fc6ee30bf7a43346e8a9ee99e30a6079d --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-314/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9796a6c4dfa7bcfc7531b6a3163315134b78052b383dd8ed82be57a88f7087f9 +size 7185 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-314/zero_to_fp32.py b/qwen3/distillm_adaptive_srkl/checkpoint-314/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-314/zero_to_fp32.py @@ -0,0 +1,790 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY, + AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY, + AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def _has_autoep_zero3_partitioned_metadata(state_dict): + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY) + if autoep_layers is None: + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY) + if not isinstance(autoep_layers, list): + return False + return any( + isinstance(entry, dict) + and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT + for entry in autoep_layers) + + +def _raise_if_autoep_zero3_partitioned_state(state_dict): + if _has_autoep_zero3_partitioned_metadata(state_dict): + raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. " + "AutoEP expert parameters are partitioned over expert replica groups, so " + "global data-parallel consolidation would produce incomplete expert tensors. " + "Use ds_to_universal.py for expert-aware conversion.") + + +def _raise_if_autoep_zero3_partitioned_checkpoint(model_files): + for file in model_files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints + # before the expensive optimizer-shard load below. + model_files = get_model_state_files(ds_checkpoint_dir) + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-628/README.md b/qwen3/distillm_adaptive_srkl/checkpoint-628/README.md new file mode 100644 index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-628/README.md @@ -0,0 +1,208 @@ +--- +base_model: Qwen/Qwen3-0.6B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-0.6B +- llama-factory +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-628/adapter_config.json b/qwen3/distillm_adaptive_srkl/checkpoint-628/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa3291dd38d42da741ab7ac66ac0e7b712287fa0 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-628/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-0.6B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-628/adapter_model.safetensors b/qwen3/distillm_adaptive_srkl/checkpoint-628/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..671924382e81d9a82f9702f24ffc595dae44ca92 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-628/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad390e01cd9484f10a03ab14ae679d9c9280c3e18f841c22c7706544ac24122a +size 40422560 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-628/chat_template.jinja b/qwen3/distillm_adaptive_srkl/checkpoint-628/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-628/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-628/distillm_state_rank0.pt b/qwen3/distillm_adaptive_srkl/checkpoint-628/distillm_state_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..a9bffbbbd85eed6d205abf4ba497b58a9c2ab686 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-628/distillm_state_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9083bb0185b673b3026a71a4e1b07808667f534cb608256c2f138888949155e4 +size 9403 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-628/distillm_state_rank1.pt b/qwen3/distillm_adaptive_srkl/checkpoint-628/distillm_state_rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..cb46275912c78d8615a32230c877ae737b79e87d --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-628/distillm_state_rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ade08e81abc31e53e5fa43cc5e43d33641c8cb156fb645f56ff2473327d9a0d +size 9339 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/distillm_adaptive_srkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa4bd2dd22ed38957775341f67fe7f1c9b0c5df8 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c35183163b594049a73e859ccfa277974c0a0769af1dabb58d74c1d73d9fcde2 +size 121135557 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/distillm_adaptive_srkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9605d4b48f0c8f99b840fd0b6fab06c5d8832595 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3b173e9238b7f89b8f15da418540a2f72ebb23ed82a92747a7866128bcb3fd1 +size 121135749 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt b/qwen3/distillm_adaptive_srkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..349585f17f3ff95d5e704e07c47ab5608196fcee --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32d85db728a6d326fa9292209e189b6f7f328aa315637880b7388e7d0b8c9252 +size 351841765 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-628/latest b/qwen3/distillm_adaptive_srkl/checkpoint-628/latest new file mode 100644 index 0000000000000000000000000000000000000000..af055c3db05dcb2af541005f22308dbd946aba65 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-628/latest @@ -0,0 +1 @@ +global_step628 \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-628/rng_state_0.pth b/qwen3/distillm_adaptive_srkl/checkpoint-628/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..6d68313b7e0fbb9acf8d75635a59ffc7753e1faf --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-628/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6046bc81e94bcdeb7a97aac19407b11cb95ef5351924f897279b1055ec882f2e +size 14917 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-628/rng_state_1.pth b/qwen3/distillm_adaptive_srkl/checkpoint-628/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..9cd501ff18def43f816787ad82c037b9317f519f --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-628/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1e8a8b98e7357104dac0de9634dc786ec647f63cef4082573eae81434eee5a2 +size 14917 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-628/scheduler.pt b/qwen3/distillm_adaptive_srkl/checkpoint-628/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..96c0cabcd99be4945ffbd7897ecc5367ac4beb28 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-628/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb823a2c51d7fb42dfe3b693a7159784285dbf38c5a797e6e73292096123ab7a +size 1465 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-628/tokenizer.json b/qwen3/distillm_adaptive_srkl/checkpoint-628/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-628/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-628/tokenizer_config.json b/qwen3/distillm_adaptive_srkl/checkpoint-628/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-628/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-628/trainer_state.json b/qwen3/distillm_adaptive_srkl/checkpoint-628/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..05304d5a0e083d4de2be5ebf292e5d70529ed3ef --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-628/trainer_state.json @@ -0,0 +1,296 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 628, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "distillm_initial_generator_count": 1712.0, + "distillm_initial_generator_exact_match": 0.0, + "distillm_initial_generator_rouge1": 16.2149, + "distillm_initial_generator_rouge2": 3.2276, + "distillm_initial_generator_rougeL": 11.6725, + "distillm_initial_loss": 2.494271755218506, + "distillm_initial_runtime": 2946.4402, + "distillm_initial_samples_per_second": 1.159, + "distillm_initial_selector_accuracy": 52.278, + "distillm_initial_selector_count": 1712.0, + "distillm_initial_steps_per_second": 0.036, + "epoch": 0, + "step": 0 + }, + { + "epoch": 0.06382130035899482, + "grad_norm": 0.9302584528923035, + "learning_rate": 9.996390385474569e-05, + "loss": 0.375650691986084, + "step": 20 + }, + { + "epoch": 0.12764260071798963, + "grad_norm": 0.819491982460022, + "learning_rate": 9.984797509888031e-05, + "loss": 0.10328874588012696, + "step": 40 + }, + { + "epoch": 0.19146390107698444, + "grad_norm": 0.9121766686439514, + "learning_rate": 9.965229938050925e-05, + "loss": 0.06860063076019288, + "step": 60 + }, + { + "epoch": 0.25528520143597927, + "grad_norm": 0.3703226149082184, + "learning_rate": 9.937719005663432e-05, + "loss": 0.055023026466369626, + "step": 80 + }, + { + "epoch": 0.3191065017949741, + "grad_norm": 0.6126606464385986, + "learning_rate": 9.902308769000724e-05, + "loss": 0.03998597264289856, + "step": 100 + }, + { + "epoch": 0.3829278021539689, + "grad_norm": 1.0489412546157837, + "learning_rate": 9.859055934360789e-05, + "loss": 0.03420835733413696, + "step": 120 + }, + { + "epoch": 0.4467491025129637, + "grad_norm": 0.3606373965740204, + "learning_rate": 9.808029767254437e-05, + "loss": 0.03716553449630737, + "step": 140 + }, + { + "epoch": 0.5105704028719585, + "grad_norm": 1.0114212036132812, + "learning_rate": 9.749311981482817e-05, + "loss": 0.02128121256828308, + "step": 160 + }, + { + "epoch": 0.5743917032309533, + "grad_norm": 0.22879505157470703, + "learning_rate": 9.682996608280154e-05, + "loss": 0.016371789574623107, + "step": 180 + }, + { + "epoch": 0.6382130035899481, + "grad_norm": 0.40647953748703003, + "learning_rate": 9.609189845731207e-05, + "loss": 0.02666616439819336, + "step": 200 + }, + { + "epoch": 0.702034303948943, + "grad_norm": 0.38679832220077515, + "learning_rate": 9.52800988870462e-05, + "loss": 0.022025266289711, + "step": 220 + }, + { + "epoch": 0.7658556043079378, + "grad_norm": 0.2795315384864807, + "learning_rate": 9.439586739574513e-05, + "loss": 0.00786939337849617, + "step": 240 + }, + { + "epoch": 0.8296769046669326, + "grad_norm": 0.3553146421909332, + "learning_rate": 9.344062000033414e-05, + "loss": 0.010387377440929412, + "step": 260 + }, + { + "epoch": 0.8934982050259274, + "grad_norm": 0.20546464622020721, + "learning_rate": 9.241588644329919e-05, + "loss": 0.009576073288917542, + "step": 280 + }, + { + "epoch": 0.9573195053849222, + "grad_norm": 0.2525307536125183, + "learning_rate": 9.132330774294252e-05, + "loss": 0.009539780020713807, + "step": 300 + }, + { + "epoch": 1.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 85.3972, + "eval_generator_rouge1": 99.3064, + "eval_generator_rouge2": 98.9501, + "eval_generator_rougeL": 98.7596, + "eval_loss": 0.013466483913362026, + "eval_runtime": 592.176, + "eval_samples_per_second": 5.765, + "eval_selector_accuracy": 89.9533, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.181, + "step": 314 + }, + { + "epoch": 1.0191463901076985, + "grad_norm": 0.7174072265625, + "learning_rate": 9.01646335654399e-05, + "loss": 0.017302539944648743, + "step": 320 + }, + { + "epoch": 1.0829676904666932, + "grad_norm": 0.11603154987096786, + "learning_rate": 8.894171942290817e-05, + "loss": 0.008357083797454834, + "step": 340 + }, + { + "epoch": 1.146788990825688, + "grad_norm": 0.19480980932712555, + "learning_rate": 8.765652370197003e-05, + "loss": 0.005870707705616951, + "step": 360 + }, + { + "epoch": 1.210610291184683, + "grad_norm": 0.16763336956501007, + "learning_rate": 8.631110452757467e-05, + "loss": 0.0035324640572071075, + "step": 380 + }, + { + "epoch": 1.2744315915436777, + "grad_norm": 0.03674647584557533, + "learning_rate": 8.490761646709642e-05, + "loss": 0.005342463403940201, + "step": 400 + }, + { + "epoch": 1.3382528919026724, + "grad_norm": 0.0998690128326416, + "learning_rate": 8.344830707998953e-05, + "loss": 0.006832928955554962, + "step": 420 + }, + { + "epoch": 1.4020741922616673, + "grad_norm": 0.13103976845741272, + "learning_rate": 8.193551331852458e-05, + "loss": 0.005545094236731529, + "step": 440 + }, + { + "epoch": 1.4658954926206622, + "grad_norm": 0.4100915491580963, + "learning_rate": 8.037165778537033e-05, + "loss": 0.007910171896219254, + "step": 460 + }, + { + "epoch": 1.5297167929796571, + "grad_norm": 0.14175599813461304, + "learning_rate": 7.875924485401427e-05, + "loss": 0.0031586486846208574, + "step": 480 + }, + { + "epoch": 1.5935380933386518, + "grad_norm": 0.19752922654151917, + "learning_rate": 7.710085665823444e-05, + "loss": 0.004655975103378296, + "step": 500 + }, + { + "epoch": 1.6573593936976465, + "grad_norm": 0.03805746138095856, + "learning_rate": 7.539914895704521e-05, + "loss": 0.0038994356989860536, + "step": 520 + }, + { + "epoch": 1.7211806940566414, + "grad_norm": 0.7704943418502808, + "learning_rate": 7.365684688173908e-05, + "loss": 0.008619165420532227, + "step": 540 + }, + { + "epoch": 1.7850019944156363, + "grad_norm": 0.10960758477449417, + "learning_rate": 7.18767405718347e-05, + "loss": 0.004270360246300698, + "step": 560 + }, + { + "epoch": 1.848823294774631, + "grad_norm": 0.5830378532409668, + "learning_rate": 7.006168070692036e-05, + "loss": 0.010602302849292755, + "step": 580 + }, + { + "epoch": 1.9126445951336257, + "grad_norm": 0.1323094367980957, + "learning_rate": 6.821457394154786e-05, + "loss": 0.004179984703660012, + "step": 600 + }, + { + "epoch": 1.9764658954926206, + "grad_norm": 0.06579075008630753, + "learning_rate": 6.633837825048768e-05, + "loss": 0.0037369213998317717, + "step": 620 + }, + { + "epoch": 2.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 94.9766, + "eval_generator_rouge1": 99.8874, + "eval_generator_rouge2": 99.7935, + "eval_generator_rougeL": 99.6469, + "eval_loss": 0.005270988680422306, + "eval_runtime": 592.8427, + "eval_samples_per_second": 5.759, + "eval_selector_accuracy": 97.7804, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.18, + "step": 628 + } + ], + "logging_steps": 20, + "max_steps": 1570, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.346351770520781e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-628/training_args.bin b/qwen3/distillm_adaptive_srkl/checkpoint-628/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c03c7a9fc6ee30bf7a43346e8a9ee99e30a6079d --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-628/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9796a6c4dfa7bcfc7531b6a3163315134b78052b383dd8ed82be57a88f7087f9 +size 7185 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-628/zero_to_fp32.py b/qwen3/distillm_adaptive_srkl/checkpoint-628/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-628/zero_to_fp32.py @@ -0,0 +1,790 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY, + AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY, + AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def _has_autoep_zero3_partitioned_metadata(state_dict): + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY) + if autoep_layers is None: + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY) + if not isinstance(autoep_layers, list): + return False + return any( + isinstance(entry, dict) + and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT + for entry in autoep_layers) + + +def _raise_if_autoep_zero3_partitioned_state(state_dict): + if _has_autoep_zero3_partitioned_metadata(state_dict): + raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. " + "AutoEP expert parameters are partitioned over expert replica groups, so " + "global data-parallel consolidation would produce incomplete expert tensors. " + "Use ds_to_universal.py for expert-aware conversion.") + + +def _raise_if_autoep_zero3_partitioned_checkpoint(model_files): + for file in model_files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints + # before the expensive optimizer-shard load below. + model_files = get_model_state_files(ds_checkpoint_dir) + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-942/README.md b/qwen3/distillm_adaptive_srkl/checkpoint-942/README.md new file mode 100644 index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-942/README.md @@ -0,0 +1,208 @@ +--- +base_model: Qwen/Qwen3-0.6B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-0.6B +- llama-factory +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-942/adapter_config.json b/qwen3/distillm_adaptive_srkl/checkpoint-942/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa3291dd38d42da741ab7ac66ac0e7b712287fa0 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-942/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-0.6B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-942/adapter_model.safetensors b/qwen3/distillm_adaptive_srkl/checkpoint-942/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6d075d07c7628092b827bea0c30e4c851e7de998 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-942/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62d68d1744980f21397d36acb842d7c6b96a2e7a233a9e23ad5915989762a434 +size 40422560 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-942/chat_template.jinja b/qwen3/distillm_adaptive_srkl/checkpoint-942/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-942/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-942/distillm_state_rank0.pt b/qwen3/distillm_adaptive_srkl/checkpoint-942/distillm_state_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..4c4026c6e95aa9d01c436ecad3c6f0b27f11f91d --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-942/distillm_state_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be37e9f05d7d7943ae5068691d49e7fe146b0249ac3bd306b8a17830441b0c7c +size 9467 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-942/distillm_state_rank1.pt b/qwen3/distillm_adaptive_srkl/checkpoint-942/distillm_state_rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..15516d1a1e440bac4d70959dfa9a603927bd3bd4 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-942/distillm_state_rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5284c6faa8372751c674221b6912744948fcb78359dd17daa1518f8c1985ddf +size 9339 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/distillm_adaptive_srkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a69b235ca49594e69bd3de045be2f5d31627d3c --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5044f4a767c07e95cdaddbd84a05a5b395420a059e18e5b57eab626a7ff3daef +size 121135557 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/distillm_adaptive_srkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e501e15171b1a663e5f0346f199a3049d03f435 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d0cd54b26c2d58022367d18a8221d243800abfd1234cb22025bf8c7737276bc +size 121135749 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt b/qwen3/distillm_adaptive_srkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8864af0bf1a6e33afe5df87fe2fad4438f03b05e --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25b2abf91d62b675a1ba819c121c0b1aa085d161f7f635fa611d896f254fccba +size 351841765 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-942/latest b/qwen3/distillm_adaptive_srkl/checkpoint-942/latest new file mode 100644 index 0000000000000000000000000000000000000000..4e9a68e4460aca593c480611a486d2a4cd73ef42 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-942/latest @@ -0,0 +1 @@ +global_step942 \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-942/rng_state_0.pth b/qwen3/distillm_adaptive_srkl/checkpoint-942/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..124f83eb5533923b6ebc0062c6fd6c667ec3f8a6 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-942/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24ddce849298abe49718ebdf5a6aa2c94fe01f3424a9c8dfb42c30b2d36dc436 +size 14917 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-942/rng_state_1.pth b/qwen3/distillm_adaptive_srkl/checkpoint-942/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..f2dcc751c0a3cb8c901a6bb2dbcaed0c99fa2415 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-942/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0002144da2d2f4193b9805b17e48f38cd81a91bb7ef035f31e273a56f4016522 +size 14917 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-942/scheduler.pt b/qwen3/distillm_adaptive_srkl/checkpoint-942/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b1f8af0480bf83ab728abfeff11bc3288214349 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-942/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8076da6f897951c5bdf8649e4fba596e8c49819927746017442ffd85f40b18a3 +size 1465 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-942/tokenizer.json b/qwen3/distillm_adaptive_srkl/checkpoint-942/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-942/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-942/tokenizer_config.json b/qwen3/distillm_adaptive_srkl/checkpoint-942/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-942/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-942/trainer_state.json b/qwen3/distillm_adaptive_srkl/checkpoint-942/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d7a7ffc5973fe80152c52923a0ca54608a513c96 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-942/trainer_state.json @@ -0,0 +1,423 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 942, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "distillm_initial_generator_count": 1712.0, + "distillm_initial_generator_exact_match": 0.0, + "distillm_initial_generator_rouge1": 16.2149, + "distillm_initial_generator_rouge2": 3.2276, + "distillm_initial_generator_rougeL": 11.6725, + "distillm_initial_loss": 2.494271755218506, + "distillm_initial_runtime": 2946.4402, + "distillm_initial_samples_per_second": 1.159, + "distillm_initial_selector_accuracy": 52.278, + "distillm_initial_selector_count": 1712.0, + "distillm_initial_steps_per_second": 0.036, + "epoch": 0, + "step": 0 + }, + { + "epoch": 0.06382130035899482, + "grad_norm": 0.9302584528923035, + "learning_rate": 9.996390385474569e-05, + "loss": 0.375650691986084, + "step": 20 + }, + { + "epoch": 0.12764260071798963, + "grad_norm": 0.819491982460022, + "learning_rate": 9.984797509888031e-05, + "loss": 0.10328874588012696, + "step": 40 + }, + { + "epoch": 0.19146390107698444, + "grad_norm": 0.9121766686439514, + "learning_rate": 9.965229938050925e-05, + "loss": 0.06860063076019288, + "step": 60 + }, + { + "epoch": 0.25528520143597927, + "grad_norm": 0.3703226149082184, + "learning_rate": 9.937719005663432e-05, + "loss": 0.055023026466369626, + "step": 80 + }, + { + "epoch": 0.3191065017949741, + "grad_norm": 0.6126606464385986, + "learning_rate": 9.902308769000724e-05, + "loss": 0.03998597264289856, + "step": 100 + }, + { + "epoch": 0.3829278021539689, + "grad_norm": 1.0489412546157837, + "learning_rate": 9.859055934360789e-05, + "loss": 0.03420835733413696, + "step": 120 + }, + { + "epoch": 0.4467491025129637, + "grad_norm": 0.3606373965740204, + "learning_rate": 9.808029767254437e-05, + "loss": 0.03716553449630737, + "step": 140 + }, + { + "epoch": 0.5105704028719585, + "grad_norm": 1.0114212036132812, + "learning_rate": 9.749311981482817e-05, + "loss": 0.02128121256828308, + "step": 160 + }, + { + "epoch": 0.5743917032309533, + "grad_norm": 0.22879505157470703, + "learning_rate": 9.682996608280154e-05, + "loss": 0.016371789574623107, + "step": 180 + }, + { + "epoch": 0.6382130035899481, + "grad_norm": 0.40647953748703003, + "learning_rate": 9.609189845731207e-05, + "loss": 0.02666616439819336, + "step": 200 + }, + { + "epoch": 0.702034303948943, + "grad_norm": 0.38679832220077515, + "learning_rate": 9.52800988870462e-05, + "loss": 0.022025266289711, + "step": 220 + }, + { + "epoch": 0.7658556043079378, + "grad_norm": 0.2795315384864807, + "learning_rate": 9.439586739574513e-05, + "loss": 0.00786939337849617, + "step": 240 + }, + { + "epoch": 0.8296769046669326, + "grad_norm": 0.3553146421909332, + "learning_rate": 9.344062000033414e-05, + "loss": 0.010387377440929412, + "step": 260 + }, + { + "epoch": 0.8934982050259274, + "grad_norm": 0.20546464622020721, + "learning_rate": 9.241588644329919e-05, + "loss": 0.009576073288917542, + "step": 280 + }, + { + "epoch": 0.9573195053849222, + "grad_norm": 0.2525307536125183, + "learning_rate": 9.132330774294252e-05, + "loss": 0.009539780020713807, + "step": 300 + }, + { + "epoch": 1.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 85.3972, + "eval_generator_rouge1": 99.3064, + "eval_generator_rouge2": 98.9501, + "eval_generator_rougeL": 98.7596, + "eval_loss": 0.013466483913362026, + "eval_runtime": 592.176, + "eval_samples_per_second": 5.765, + "eval_selector_accuracy": 89.9533, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.181, + "step": 314 + }, + { + "epoch": 1.0191463901076985, + "grad_norm": 0.7174072265625, + "learning_rate": 9.01646335654399e-05, + "loss": 0.017302539944648743, + "step": 320 + }, + { + "epoch": 1.0829676904666932, + "grad_norm": 0.11603154987096786, + "learning_rate": 8.894171942290817e-05, + "loss": 0.008357083797454834, + "step": 340 + }, + { + "epoch": 1.146788990825688, + "grad_norm": 0.19480980932712555, + "learning_rate": 8.765652370197003e-05, + "loss": 0.005870707705616951, + "step": 360 + }, + { + "epoch": 1.210610291184683, + "grad_norm": 0.16763336956501007, + "learning_rate": 8.631110452757467e-05, + "loss": 0.0035324640572071075, + "step": 380 + }, + { + "epoch": 1.2744315915436777, + "grad_norm": 0.03674647584557533, + "learning_rate": 8.490761646709642e-05, + "loss": 0.005342463403940201, + "step": 400 + }, + { + "epoch": 1.3382528919026724, + "grad_norm": 0.0998690128326416, + "learning_rate": 8.344830707998953e-05, + "loss": 0.006832928955554962, + "step": 420 + }, + { + "epoch": 1.4020741922616673, + "grad_norm": 0.13103976845741272, + "learning_rate": 8.193551331852458e-05, + "loss": 0.005545094236731529, + "step": 440 + }, + { + "epoch": 1.4658954926206622, + "grad_norm": 0.4100915491580963, + "learning_rate": 8.037165778537033e-05, + "loss": 0.007910171896219254, + "step": 460 + }, + { + "epoch": 1.5297167929796571, + "grad_norm": 0.14175599813461304, + "learning_rate": 7.875924485401427e-05, + "loss": 0.0031586486846208574, + "step": 480 + }, + { + "epoch": 1.5935380933386518, + "grad_norm": 0.19752922654151917, + "learning_rate": 7.710085665823444e-05, + "loss": 0.004655975103378296, + "step": 500 + }, + { + "epoch": 1.6573593936976465, + "grad_norm": 0.03805746138095856, + "learning_rate": 7.539914895704521e-05, + "loss": 0.0038994356989860536, + "step": 520 + }, + { + "epoch": 1.7211806940566414, + "grad_norm": 0.7704943418502808, + "learning_rate": 7.365684688173908e-05, + "loss": 0.008619165420532227, + "step": 540 + }, + { + "epoch": 1.7850019944156363, + "grad_norm": 0.10960758477449417, + "learning_rate": 7.18767405718347e-05, + "loss": 0.004270360246300698, + "step": 560 + }, + { + "epoch": 1.848823294774631, + "grad_norm": 0.5830378532409668, + "learning_rate": 7.006168070692036e-05, + "loss": 0.010602302849292755, + "step": 580 + }, + { + "epoch": 1.9126445951336257, + "grad_norm": 0.1323094367980957, + "learning_rate": 6.821457394154786e-05, + "loss": 0.004179984703660012, + "step": 600 + }, + { + "epoch": 1.9764658954926206, + "grad_norm": 0.06579075008630753, + "learning_rate": 6.633837825048768e-05, + "loss": 0.0037369213998317717, + "step": 620 + }, + { + "epoch": 2.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 94.9766, + "eval_generator_rouge1": 99.8874, + "eval_generator_rouge2": 99.7935, + "eval_generator_rougeL": 99.6469, + "eval_loss": 0.005270988680422306, + "eval_runtime": 592.8427, + "eval_samples_per_second": 5.759, + "eval_selector_accuracy": 97.7804, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.18, + "step": 628 + }, + { + "epoch": 2.038292780215397, + "grad_norm": 0.08518785238265991, + "learning_rate": 6.443609819179926e-05, + "loss": 0.001967618428170681, + "step": 640 + }, + { + "epoch": 2.102114080574392, + "grad_norm": 0.1549246907234192, + "learning_rate": 6.251078009530256e-05, + "loss": 0.0023793572559952735, + "step": 660 + }, + { + "epoch": 2.1659353809333863, + "grad_norm": 0.08672250062227249, + "learning_rate": 6.0565507184155677e-05, + "loss": 0.004786616936326027, + "step": 680 + }, + { + "epoch": 2.2297566812923812, + "grad_norm": 0.03346117213368416, + "learning_rate": 5.860339463735136e-05, + "loss": 0.002478102594614029, + "step": 700 + }, + { + "epoch": 2.293577981651376, + "grad_norm": 0.015912503004074097, + "learning_rate": 5.662758460103895e-05, + "loss": 0.002031809091567993, + "step": 720 + }, + { + "epoch": 2.357399282010371, + "grad_norm": 0.03547865152359009, + "learning_rate": 5.464124115666095e-05, + "loss": 0.004523729532957077, + "step": 740 + }, + { + "epoch": 2.421220582369366, + "grad_norm": 0.007262590806931257, + "learning_rate": 5.264754525396212e-05, + "loss": 0.0013824908062815666, + "step": 760 + }, + { + "epoch": 2.4850418827283605, + "grad_norm": 0.024668537080287933, + "learning_rate": 5.06496896169857e-05, + "loss": 0.0008345138281583786, + "step": 780 + }, + { + "epoch": 2.5488631830873554, + "grad_norm": 0.07706174999475479, + "learning_rate": 4.865087363121384e-05, + "loss": 0.0049762561917305, + "step": 800 + }, + { + "epoch": 2.6126844834463503, + "grad_norm": 0.25065284967422485, + "learning_rate": 4.6654298220040673e-05, + "loss": 0.00360473096370697, + "step": 820 + }, + { + "epoch": 2.6765057838053448, + "grad_norm": 0.0715489313006401, + "learning_rate": 4.466316071878229e-05, + "loss": 0.001436175312846899, + "step": 840 + }, + { + "epoch": 2.74032708416434, + "grad_norm": 0.116939976811409, + "learning_rate": 4.2680649754432913e-05, + "loss": 0.0016343716531991958, + "step": 860 + }, + { + "epoch": 2.8041483845233346, + "grad_norm": 0.01825389452278614, + "learning_rate": 4.070994013936646e-05, + "loss": 0.0012953002005815506, + "step": 880 + }, + { + "epoch": 2.8679696848823295, + "grad_norm": 0.10613884776830673, + "learning_rate": 3.8754187787161155e-05, + "loss": 0.000981786847114563, + "step": 900 + }, + { + "epoch": 2.9317909852413244, + "grad_norm": 0.00612761452794075, + "learning_rate": 3.681652465868875e-05, + "loss": 0.0015057422220706939, + "step": 920 + }, + { + "epoch": 2.995612285600319, + "grad_norm": 0.10040487349033356, + "learning_rate": 3.490005374656184e-05, + "loss": 0.0025246324017643928, + "step": 940 + }, + { + "epoch": 3.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 95.6192, + "eval_generator_rouge1": 99.9324, + "eval_generator_rouge2": 99.8567, + "eval_generator_rougeL": 99.7224, + "eval_loss": 0.004592898301780224, + "eval_runtime": 596.6943, + "eval_samples_per_second": 5.722, + "eval_selector_accuracy": 98.014, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.179, + "step": 942 + } + ], + "logging_steps": 20, + "max_steps": 1570, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.524157743353037e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-942/training_args.bin b/qwen3/distillm_adaptive_srkl/checkpoint-942/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c03c7a9fc6ee30bf7a43346e8a9ee99e30a6079d --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-942/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9796a6c4dfa7bcfc7531b6a3163315134b78052b383dd8ed82be57a88f7087f9 +size 7185 diff --git a/qwen3/distillm_adaptive_srkl/checkpoint-942/zero_to_fp32.py b/qwen3/distillm_adaptive_srkl/checkpoint-942/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/checkpoint-942/zero_to_fp32.py @@ -0,0 +1,790 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY, + AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY, + AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def _has_autoep_zero3_partitioned_metadata(state_dict): + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY) + if autoep_layers is None: + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY) + if not isinstance(autoep_layers, list): + return False + return any( + isinstance(entry, dict) + and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT + for entry in autoep_layers) + + +def _raise_if_autoep_zero3_partitioned_state(state_dict): + if _has_autoep_zero3_partitioned_metadata(state_dict): + raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. " + "AutoEP expert parameters are partitioned over expert replica groups, so " + "global data-parallel consolidation would produce incomplete expert tensors. " + "Use ds_to_universal.py for expert-aware conversion.") + + +def _raise_if_autoep_zero3_partitioned_checkpoint(model_files): + for file in model_files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints + # before the expensive optimizer-shard load below. + model_files = get_model_state_files(ds_checkpoint_dir) + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/qwen3/distillm_adaptive_srkl/distillm_state_rank0.pt b/qwen3/distillm_adaptive_srkl/distillm_state_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..becca175688219a9b4a422f423edefb47590217d --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/distillm_state_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c45fea5636ba58d552a49ba5a651afe08fe7e86b64bd29d82359e1ffb3df111f +size 9403 diff --git a/qwen3/distillm_adaptive_srkl/distillm_state_rank1.pt b/qwen3/distillm_adaptive_srkl/distillm_state_rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d7c319e37a117a78ed636f3109afc6ef3c8fd98 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/distillm_state_rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34b6ffca878fd86b6133b9eb70d94fee805ff00ef1d33571b54dc0de3a307df3 +size 9339 diff --git a/qwen3/distillm_adaptive_srkl/eval_results.json b/qwen3/distillm_adaptive_srkl/eval_results.json new file mode 100644 index 0000000000000000000000000000000000000000..04f52dcd425b7700d862b868903b434baab35fc2 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/eval_results.json @@ -0,0 +1,19 @@ +{ + "epoch": 5.0, + "eval_distillm_threshold": 0.0, + "eval_distillm_threshold_changed": 0.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 96.9042, + "eval_generator_rouge1": 99.9492, + "eval_generator_rouge2": 99.899, + "eval_generator_rougeL": 99.7905, + "eval_loss": 0.003477960592135787, + "eval_rollout_dataset_steps": 12535, + "eval_rollout_fresh_steps": 0, + "eval_rollout_replay_steps": 0, + "eval_runtime": 580.143, + "eval_samples_per_second": 5.885, + "eval_selector_accuracy": 98.7734, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.184 +} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/tokenizer.json b/qwen3/distillm_adaptive_srkl/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/qwen3/distillm_adaptive_srkl/tokenizer_config.json b/qwen3/distillm_adaptive_srkl/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen3/distillm_adaptive_srkl/train_results.json b/qwen3/distillm_adaptive_srkl/train_results.json new file mode 100644 index 0000000000000000000000000000000000000000..df16d8734d0feb4a9f000f67960ea7e352417efd --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 5.0, + "total_flos": 5.871444840310374e+16, + "train_loss": 0.013134213612907251, + "train_runtime": 10459.448, + "train_samples_per_second": 4.793, + "train_steps_per_second": 0.15 +} \ No newline at end of file diff --git a/qwen3/distillm_adaptive_srkl/trainer_state.json b/qwen3/distillm_adaptive_srkl/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a80332a87e4244552f625cdb11857ea4943fe013 --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/trainer_state.json @@ -0,0 +1,679 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 1570, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "distillm_initial_generator_count": 1712.0, + "distillm_initial_generator_exact_match": 0.0, + "distillm_initial_generator_rouge1": 16.2149, + "distillm_initial_generator_rouge2": 3.2276, + "distillm_initial_generator_rougeL": 11.6725, + "distillm_initial_loss": 2.494271755218506, + "distillm_initial_runtime": 2946.4402, + "distillm_initial_samples_per_second": 1.159, + "distillm_initial_selector_accuracy": 52.278, + "distillm_initial_selector_count": 1712.0, + "distillm_initial_steps_per_second": 0.036, + "epoch": 0, + "step": 0 + }, + { + "epoch": 0.06382130035899482, + "grad_norm": 0.9302584528923035, + "learning_rate": 9.996390385474569e-05, + "loss": 0.375650691986084, + "step": 20 + }, + { + "epoch": 0.12764260071798963, + "grad_norm": 0.819491982460022, + "learning_rate": 9.984797509888031e-05, + "loss": 0.10328874588012696, + "step": 40 + }, + { + "epoch": 0.19146390107698444, + "grad_norm": 0.9121766686439514, + "learning_rate": 9.965229938050925e-05, + "loss": 0.06860063076019288, + "step": 60 + }, + { + "epoch": 0.25528520143597927, + "grad_norm": 0.3703226149082184, + "learning_rate": 9.937719005663432e-05, + "loss": 0.055023026466369626, + "step": 80 + }, + { + "epoch": 0.3191065017949741, + "grad_norm": 0.6126606464385986, + "learning_rate": 9.902308769000724e-05, + "loss": 0.03998597264289856, + "step": 100 + }, + { + "epoch": 0.3829278021539689, + "grad_norm": 1.0489412546157837, + "learning_rate": 9.859055934360789e-05, + "loss": 0.03420835733413696, + "step": 120 + }, + { + "epoch": 0.4467491025129637, + "grad_norm": 0.3606373965740204, + "learning_rate": 9.808029767254437e-05, + "loss": 0.03716553449630737, + "step": 140 + }, + { + "epoch": 0.5105704028719585, + "grad_norm": 1.0114212036132812, + "learning_rate": 9.749311981482817e-05, + "loss": 0.02128121256828308, + "step": 160 + }, + { + "epoch": 0.5743917032309533, + "grad_norm": 0.22879505157470703, + "learning_rate": 9.682996608280154e-05, + "loss": 0.016371789574623107, + "step": 180 + }, + { + "epoch": 0.6382130035899481, + "grad_norm": 0.40647953748703003, + "learning_rate": 9.609189845731207e-05, + "loss": 0.02666616439819336, + "step": 200 + }, + { + "epoch": 0.702034303948943, + "grad_norm": 0.38679832220077515, + "learning_rate": 9.52800988870462e-05, + "loss": 0.022025266289711, + "step": 220 + }, + { + "epoch": 0.7658556043079378, + "grad_norm": 0.2795315384864807, + "learning_rate": 9.439586739574513e-05, + "loss": 0.00786939337849617, + "step": 240 + }, + { + "epoch": 0.8296769046669326, + "grad_norm": 0.3553146421909332, + "learning_rate": 9.344062000033414e-05, + "loss": 0.010387377440929412, + "step": 260 + }, + { + "epoch": 0.8934982050259274, + "grad_norm": 0.20546464622020721, + "learning_rate": 9.241588644329919e-05, + "loss": 0.009576073288917542, + "step": 280 + }, + { + "epoch": 0.9573195053849222, + "grad_norm": 0.2525307536125183, + "learning_rate": 9.132330774294252e-05, + "loss": 0.009539780020713807, + "step": 300 + }, + { + "epoch": 1.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 85.3972, + "eval_generator_rouge1": 99.3064, + "eval_generator_rouge2": 98.9501, + "eval_generator_rougeL": 98.7596, + "eval_loss": 0.013466483913362026, + "eval_runtime": 592.176, + "eval_samples_per_second": 5.765, + "eval_selector_accuracy": 89.9533, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.181, + "step": 314 + }, + { + "epoch": 1.0191463901076985, + "grad_norm": 0.7174072265625, + "learning_rate": 9.01646335654399e-05, + "loss": 0.017302539944648743, + "step": 320 + }, + { + "epoch": 1.0829676904666932, + "grad_norm": 0.11603154987096786, + "learning_rate": 8.894171942290817e-05, + "loss": 0.008357083797454834, + "step": 340 + }, + { + "epoch": 1.146788990825688, + "grad_norm": 0.19480980932712555, + "learning_rate": 8.765652370197003e-05, + "loss": 0.005870707705616951, + "step": 360 + }, + { + "epoch": 1.210610291184683, + "grad_norm": 0.16763336956501007, + "learning_rate": 8.631110452757467e-05, + "loss": 0.0035324640572071075, + "step": 380 + }, + { + "epoch": 1.2744315915436777, + "grad_norm": 0.03674647584557533, + "learning_rate": 8.490761646709642e-05, + "loss": 0.005342463403940201, + "step": 400 + }, + { + "epoch": 1.3382528919026724, + "grad_norm": 0.0998690128326416, + "learning_rate": 8.344830707998953e-05, + "loss": 0.006832928955554962, + "step": 420 + }, + { + "epoch": 1.4020741922616673, + "grad_norm": 0.13103976845741272, + "learning_rate": 8.193551331852458e-05, + "loss": 0.005545094236731529, + "step": 440 + }, + { + "epoch": 1.4658954926206622, + "grad_norm": 0.4100915491580963, + "learning_rate": 8.037165778537033e-05, + "loss": 0.007910171896219254, + "step": 460 + }, + { + "epoch": 1.5297167929796571, + "grad_norm": 0.14175599813461304, + "learning_rate": 7.875924485401427e-05, + "loss": 0.0031586486846208574, + "step": 480 + }, + { + "epoch": 1.5935380933386518, + "grad_norm": 0.19752922654151917, + "learning_rate": 7.710085665823444e-05, + "loss": 0.004655975103378296, + "step": 500 + }, + { + "epoch": 1.6573593936976465, + "grad_norm": 0.03805746138095856, + "learning_rate": 7.539914895704521e-05, + "loss": 0.0038994356989860536, + "step": 520 + }, + { + "epoch": 1.7211806940566414, + "grad_norm": 0.7704943418502808, + "learning_rate": 7.365684688173908e-05, + "loss": 0.008619165420532227, + "step": 540 + }, + { + "epoch": 1.7850019944156363, + "grad_norm": 0.10960758477449417, + "learning_rate": 7.18767405718347e-05, + "loss": 0.004270360246300698, + "step": 560 + }, + { + "epoch": 1.848823294774631, + "grad_norm": 0.5830378532409668, + "learning_rate": 7.006168070692036e-05, + "loss": 0.010602302849292755, + "step": 580 + }, + { + "epoch": 1.9126445951336257, + "grad_norm": 0.1323094367980957, + "learning_rate": 6.821457394154786e-05, + "loss": 0.004179984703660012, + "step": 600 + }, + { + "epoch": 1.9764658954926206, + "grad_norm": 0.06579075008630753, + "learning_rate": 6.633837825048768e-05, + "loss": 0.0037369213998317717, + "step": 620 + }, + { + "epoch": 2.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 94.9766, + "eval_generator_rouge1": 99.8874, + "eval_generator_rouge2": 99.7935, + "eval_generator_rougeL": 99.6469, + "eval_loss": 0.005270988680422306, + "eval_runtime": 592.8427, + "eval_samples_per_second": 5.759, + "eval_selector_accuracy": 97.7804, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.18, + "step": 628 + }, + { + "epoch": 2.038292780215397, + "grad_norm": 0.08518785238265991, + "learning_rate": 6.443609819179926e-05, + "loss": 0.001967618428170681, + "step": 640 + }, + { + "epoch": 2.102114080574392, + "grad_norm": 0.1549246907234192, + "learning_rate": 6.251078009530256e-05, + "loss": 0.0023793572559952735, + "step": 660 + }, + { + "epoch": 2.1659353809333863, + "grad_norm": 0.08672250062227249, + "learning_rate": 6.0565507184155677e-05, + "loss": 0.004786616936326027, + "step": 680 + }, + { + "epoch": 2.2297566812923812, + "grad_norm": 0.03346117213368416, + "learning_rate": 5.860339463735136e-05, + "loss": 0.002478102594614029, + "step": 700 + }, + { + "epoch": 2.293577981651376, + "grad_norm": 0.015912503004074097, + "learning_rate": 5.662758460103895e-05, + "loss": 0.002031809091567993, + "step": 720 + }, + { + "epoch": 2.357399282010371, + "grad_norm": 0.03547865152359009, + "learning_rate": 5.464124115666095e-05, + "loss": 0.004523729532957077, + "step": 740 + }, + { + "epoch": 2.421220582369366, + "grad_norm": 0.007262590806931257, + "learning_rate": 5.264754525396212e-05, + "loss": 0.0013824908062815666, + "step": 760 + }, + { + "epoch": 2.4850418827283605, + "grad_norm": 0.024668537080287933, + "learning_rate": 5.06496896169857e-05, + "loss": 0.0008345138281583786, + "step": 780 + }, + { + "epoch": 2.5488631830873554, + "grad_norm": 0.07706174999475479, + "learning_rate": 4.865087363121384e-05, + "loss": 0.0049762561917305, + "step": 800 + }, + { + "epoch": 2.6126844834463503, + "grad_norm": 0.25065284967422485, + "learning_rate": 4.6654298220040673e-05, + "loss": 0.00360473096370697, + "step": 820 + }, + { + "epoch": 2.6765057838053448, + "grad_norm": 0.0715489313006401, + "learning_rate": 4.466316071878229e-05, + "loss": 0.001436175312846899, + "step": 840 + }, + { + "epoch": 2.74032708416434, + "grad_norm": 0.116939976811409, + "learning_rate": 4.2680649754432913e-05, + "loss": 0.0016343716531991958, + "step": 860 + }, + { + "epoch": 2.8041483845233346, + "grad_norm": 0.01825389452278614, + "learning_rate": 4.070994013936646e-05, + "loss": 0.0012953002005815506, + "step": 880 + }, + { + "epoch": 2.8679696848823295, + "grad_norm": 0.10613884776830673, + "learning_rate": 3.8754187787161155e-05, + "loss": 0.000981786847114563, + "step": 900 + }, + { + "epoch": 2.9317909852413244, + "grad_norm": 0.00612761452794075, + "learning_rate": 3.681652465868875e-05, + "loss": 0.0015057422220706939, + "step": 920 + }, + { + "epoch": 2.995612285600319, + "grad_norm": 0.10040487349033356, + "learning_rate": 3.490005374656184e-05, + "loss": 0.0025246324017643928, + "step": 940 + }, + { + "epoch": 3.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 95.6192, + "eval_generator_rouge1": 99.9324, + "eval_generator_rouge2": 99.8567, + "eval_generator_rougeL": 99.7224, + "eval_loss": 0.004592898301780224, + "eval_runtime": 596.6943, + "eval_samples_per_second": 5.722, + "eval_selector_accuracy": 98.014, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.179, + "step": 942 + }, + { + "epoch": 3.057439170323095, + "grad_norm": 0.015967005863785744, + "learning_rate": 3.3007844105971336e-05, + "loss": 0.004021823406219482, + "step": 960 + }, + { + "epoch": 3.12126047068209, + "grad_norm": 0.001990415621548891, + "learning_rate": 3.114292593987149e-05, + "loss": 0.002897568792104721, + "step": 980 + }, + { + "epoch": 3.185081771041085, + "grad_norm": 0.007704610005021095, + "learning_rate": 2.9308285746383408e-05, + "loss": 0.00075986348092556, + "step": 1000 + }, + { + "epoch": 3.24890307140008, + "grad_norm": 0.026389382779598236, + "learning_rate": 2.7506861536187867e-05, + "loss": 0.0032131548970937727, + "step": 1020 + }, + { + "epoch": 3.3127243717590744, + "grad_norm": 0.03544643148779869, + "learning_rate": 2.5741538127566396e-05, + "loss": 0.0007816605269908905, + "step": 1040 + }, + { + "epoch": 3.3765456721180693, + "grad_norm": 0.06217798963189125, + "learning_rate": 2.4015142526625115e-05, + "loss": 0.0025276828557252882, + "step": 1060 + }, + { + "epoch": 3.4403669724770642, + "grad_norm": 0.08190753310918808, + "learning_rate": 2.233043940009967e-05, + "loss": 0.0021929899230599403, + "step": 1080 + }, + { + "epoch": 3.504188272836059, + "grad_norm": 0.006671294569969177, + "learning_rate": 2.0690126647990974e-05, + "loss": 0.0009399710223078727, + "step": 1100 + }, + { + "epoch": 3.568009573195054, + "grad_norm": 0.028819965198636055, + "learning_rate": 1.9096831083121828e-05, + "loss": 0.0007333569228649139, + "step": 1120 + }, + { + "epoch": 3.6318308735540485, + "grad_norm": 0.10018011927604675, + "learning_rate": 1.7553104224533324e-05, + "loss": 0.0007428646087646485, + "step": 1140 + }, + { + "epoch": 3.6956521739130435, + "grad_norm": 1.5251667499542236, + "learning_rate": 1.606141821145735e-05, + "loss": 0.0033368520438671113, + "step": 1160 + }, + { + "epoch": 3.7594734742720384, + "grad_norm": 0.006973282899707556, + "learning_rate": 1.4624161844408763e-05, + "loss": 0.0012252738699316979, + "step": 1180 + }, + { + "epoch": 3.823294774631033, + "grad_norm": 0.005893409717828035, + "learning_rate": 1.324363675973698e-05, + "loss": 0.0025427818298339845, + "step": 1200 + }, + { + "epoch": 3.8871160749900278, + "grad_norm": 0.055641546845436096, + "learning_rate": 1.1922053743763052e-05, + "loss": 0.0033231116831302643, + "step": 1220 + }, + { + "epoch": 3.9509373753490227, + "grad_norm": 0.011760273016989231, + "learning_rate": 1.0661529192404973e-05, + "loss": 0.0006555643863976002, + "step": 1240 + }, + { + "epoch": 4.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 96.2617, + "eval_generator_rouge1": 99.9463, + "eval_generator_rouge2": 99.8925, + "eval_generator_rougeL": 99.7351, + "eval_loss": 0.00355384754948318, + "eval_runtime": 597.0533, + "eval_samples_per_second": 5.718, + "eval_selector_accuracy": 98.1893, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.179, + "step": 1256 + }, + { + "epoch": 4.0127642600717985, + "grad_norm": 0.35947123169898987, + "learning_rate": 9.464081721960592e-06, + "loss": 0.002213069796562195, + "step": 1260 + }, + { + "epoch": 4.076585560430794, + "grad_norm": 0.11643707007169724, + "learning_rate": 8.331628936475818e-06, + "loss": 0.0006028454750776291, + "step": 1280 + }, + { + "epoch": 4.140406860789788, + "grad_norm": 0.010412893258035183, + "learning_rate": 7.2659843568748385e-06, + "loss": 0.0008279280737042427, + "step": 1300 + }, + { + "epoch": 4.204228161148784, + "grad_norm": 0.008442528545856476, + "learning_rate": 6.268854516770008e-06, + "loss": 0.0014621550217270851, + "step": 1320 + }, + { + "epoch": 4.268049461507778, + "grad_norm": 0.029130427166819572, + "learning_rate": 5.341836229602331e-06, + "loss": 0.0010409696958959102, + "step": 1340 + }, + { + "epoch": 4.331870761866773, + "grad_norm": 0.3795068562030792, + "learning_rate": 4.4864140314888216e-06, + "loss": 0.0025711465626955033, + "step": 1360 + }, + { + "epoch": 4.395692062225768, + "grad_norm": 0.007296374533325434, + "learning_rate": 3.703957803871932e-06, + "loss": 0.000880935974419117, + "step": 1380 + }, + { + "epoch": 4.4595133625847625, + "grad_norm": 0.04088542237877846, + "learning_rate": 2.9957205797780495e-06, + "loss": 0.0011038152500987054, + "step": 1400 + }, + { + "epoch": 4.523334662943758, + "grad_norm": 0.03055841661989689, + "learning_rate": 2.362836537198199e-06, + "loss": 0.0015731073915958404, + "step": 1420 + }, + { + "epoch": 4.587155963302752, + "grad_norm": 0.016010979190468788, + "learning_rate": 1.8063191828043703e-06, + "loss": 0.0008220190182328224, + "step": 1440 + }, + { + "epoch": 4.650977263661747, + "grad_norm": 0.02590778097510338, + "learning_rate": 1.3270597289100437e-06, + "loss": 0.00046561416238546374, + "step": 1460 + }, + { + "epoch": 4.714798564020742, + "grad_norm": 0.00261318520642817, + "learning_rate": 9.258256662741205e-07, + "loss": 0.00044544278644025325, + "step": 1480 + }, + { + "epoch": 4.778619864379737, + "grad_norm": 0.010697306133806705, + "learning_rate": 6.032595350337341e-07, + "loss": 0.0008411692455410957, + "step": 1500 + }, + { + "epoch": 4.842441164738732, + "grad_norm": 0.011842342093586922, + "learning_rate": 3.5987789573421356e-07, + "loss": 0.003439590334892273, + "step": 1520 + }, + { + "epoch": 4.9062624650977265, + "grad_norm": 0.7014644742012024, + "learning_rate": 1.9607050210401192e-07, + "loss": 0.0016547713428735733, + "step": 1540 + }, + { + "epoch": 4.970083765456721, + "grad_norm": 0.03197485953569412, + "learning_rate": 1.1209967689925832e-07, + "loss": 0.0011944442056119442, + "step": 1560 + }, + { + "epoch": 5.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 96.5537, + "eval_generator_rouge1": 99.9447, + "eval_generator_rouge2": 99.8878, + "eval_generator_rougeL": 99.764, + "eval_loss": 0.003477960592135787, + "eval_runtime": 593.3647, + "eval_samples_per_second": 5.754, + "eval_selector_accuracy": 98.6565, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.18, + "step": 1570 + }, + { + "epoch": 5.0, + "step": 1570, + "total_flos": 5.871444840310374e+16, + "train_loss": 0.013134213612907251, + "train_runtime": 10459.448, + "train_samples_per_second": 4.793, + "train_steps_per_second": 0.15 + } + ], + "logging_steps": 20, + "max_steps": 1570, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 5.871444840310374e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/qwen3/distillm_adaptive_srkl/training_args.bin b/qwen3/distillm_adaptive_srkl/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c03c7a9fc6ee30bf7a43346e8a9ee99e30a6079d --- /dev/null +++ b/qwen3/distillm_adaptive_srkl/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9796a6c4dfa7bcfc7531b6a3163315134b78052b383dd8ed82be57a88f7087f9 +size 7185 diff --git a/qwen3/fdd_sfkl/README.md b/qwen3/fdd_sfkl/README.md new file mode 100644 index 0000000000000000000000000000000000000000..057a1f91cfa62cba6e87c40f30bc0bf41bee1ad6 --- /dev/null +++ b/qwen3/fdd_sfkl/README.md @@ -0,0 +1,79 @@ +--- +library_name: peft +license: other +base_model: Qwen/Qwen3-0.6B +tags: +- base_model:adapter:Qwen/Qwen3-0.6B +- llama-factory +- lora +- transformers +pipeline_tag: text-generation +model-index: +- name: fdd_sfkl + results: [] +--- + + + +# fdd_sfkl + +This model is a fine-tuned version of [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) on the cypher_prepared_train dataset. +It achieves the following results on the evaluation set: +- Loss: 0.0038 +- Selector Count: 1712.0 +- Generator Count: 1712.0 +- Selector Accuracy: 98.7734 +- Generator Exact Match: 96.4369 +- Generator Rouge1: 99.9258 +- Generator Rouge2: 99.8607 +- Generator Rougel: 99.7595 + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 0.0001 +- train_batch_size: 2 +- eval_batch_size: 16 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 2 +- gradient_accumulation_steps: 8 +- total_train_batch_size: 32 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine_with_min_lr +- num_epochs: 5 + +### Training results + +| Training Loss | Epoch | Step | Validation Loss | Selector Count | Generator Count | Selector Accuracy | Generator Exact Match | Generator Rouge1 | Generator Rouge2 | Generator Rougel | +|:-------------:|:-----:|:----:|:---------------:|:--------------:|:---------------:|:-----------------:|:---------------------:|:----------------:|:----------------:|:----------------:| +| 0.1008 | 1.0 | 314 | 0.0116 | 1712.0 | 1712.0 | 93.3995 | 87.8505 | 99.5556 | 99.2105 | 99.1217 | +| 0.0668 | 2.0 | 628 | 0.0070 | 1712.0 | 1712.0 | 98.0724 | 92.6402 | 99.741 | 99.5504 | 99.4854 | +| 0.0585 | 3.0 | 942 | 0.0055 | 1712.0 | 1712.0 | 96.9042 | 94.6846 | 99.8316 | 99.7174 | 99.6293 | +| 0.0490 | 4.0 | 1256 | 0.0038 | 1712.0 | 1712.0 | 98.5397 | 95.8528 | 99.9437 | 99.8882 | 99.7233 | +| 0.0490 | 5.0 | 1570 | 0.0038 | 1712.0 | 1712.0 | 98.8318 | 96.3785 | 99.9468 | 99.8886 | 99.7652 | + + +### Framework versions + +- PEFT 0.18.1 +- Transformers 5.8.0 +- Pytorch 2.9.1+cu128 +- Datasets 4.0.0 +- Tokenizers 0.22.2 \ No newline at end of file diff --git a/qwen3/fdd_sfkl/adapter_config.json b/qwen3/fdd_sfkl/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..09417a2e76df9aef95a5236d9a1aed2554e49cf2 --- /dev/null +++ b/qwen3/fdd_sfkl/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-0.6B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "q_proj", + "o_proj", + "down_proj", + "gate_proj", + "k_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/qwen3/fdd_sfkl/adapter_model.safetensors b/qwen3/fdd_sfkl/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..50556795650467d454d7fc7264461db029d298ad --- /dev/null +++ b/qwen3/fdd_sfkl/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3f0d9b21adf90b537c43ddb87d2dff137658e10aa83b08da7afb20e75f34950 +size 40422560 diff --git a/qwen3/fdd_sfkl/all_results.json b/qwen3/fdd_sfkl/all_results.json new file mode 100644 index 0000000000000000000000000000000000000000..a1182c87c69b30720f4e7f36417dc75aac490bf9 --- /dev/null +++ b/qwen3/fdd_sfkl/all_results.json @@ -0,0 +1,19 @@ +{ + "epoch": 5.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 96.4369, + "eval_generator_rouge1": 99.9258, + "eval_generator_rouge2": 99.8607, + "eval_generator_rougeL": 99.7595, + "eval_loss": 0.0038434057496488094, + "eval_runtime": 599.4736, + "eval_samples_per_second": 5.695, + "eval_selector_accuracy": 98.7734, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.178, + "total_flos": 5.871444840310374e+16, + "train_loss": 0.09919491964540664, + "train_runtime": 8002.0141, + "train_samples_per_second": 6.265, + "train_steps_per_second": 0.196 +} \ No newline at end of file diff --git a/qwen3/fdd_sfkl/chat_template.jinja b/qwen3/fdd_sfkl/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/qwen3/fdd_sfkl/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-1256/README.md b/qwen3/fdd_sfkl/checkpoint-1256/README.md new file mode 100644 index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1256/README.md @@ -0,0 +1,208 @@ +--- +base_model: Qwen/Qwen3-0.6B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-0.6B +- llama-factory +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-1256/adapter_config.json b/qwen3/fdd_sfkl/checkpoint-1256/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..09417a2e76df9aef95a5236d9a1aed2554e49cf2 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1256/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-0.6B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "q_proj", + "o_proj", + "down_proj", + "gate_proj", + "k_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-1256/adapter_model.safetensors b/qwen3/fdd_sfkl/checkpoint-1256/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cfc74fb04b766c228dca983091ef9cef34e28e45 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1256/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc2e4db8a30b879ad8504e84520a7a73c8e03aa2a0625f86c4ff5fb6fc0aa712 +size 40422560 diff --git a/qwen3/fdd_sfkl/checkpoint-1256/chat_template.jinja b/qwen3/fdd_sfkl/checkpoint-1256/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1256/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/fdd_sfkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50bafe2f1704926de7583f95508cfd8d86d4ed12 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7996645000a5e39c13c7d409b1bc3718022c0d049795c7051316017258fc85af +size 121135557 diff --git a/qwen3/fdd_sfkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/fdd_sfkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..420eae17bda7c349bd41ab2d2bd190b019ab8316 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2666007307ba7edd6b33f4e51f929fac30f1e673cc7e0b0089b35a178153b9bd +size 121135749 diff --git a/qwen3/fdd_sfkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt b/qwen3/fdd_sfkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99f80eabfa39f12a3ffdf4c2356a7b649cf8dd91 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d222c8ff2cf2f7c7d631b38476591fbf7e65f2de43e809f3ee972a6c18820952 +size 351841765 diff --git a/qwen3/fdd_sfkl/checkpoint-1256/latest b/qwen3/fdd_sfkl/checkpoint-1256/latest new file mode 100644 index 0000000000000000000000000000000000000000..74abb3d23ea83ebdac59918752db5d1782834ca7 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1256/latest @@ -0,0 +1 @@ +global_step1256 \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-1256/rng_state_0.pth b/qwen3/fdd_sfkl/checkpoint-1256/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..d1f809c6f91bda08507c87dca2f5a7ccfca4c76c --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1256/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:855ea034f1185a0dc4bcef9d40676d41474b1ffe4250bb5eff6b731ecb1faac7 +size 14917 diff --git a/qwen3/fdd_sfkl/checkpoint-1256/rng_state_1.pth b/qwen3/fdd_sfkl/checkpoint-1256/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..eda7639ddfe96cf198833f63e347d816c1aa02ae --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1256/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b1d8a276f822e24e3bb84b3165caf2e632488e28289c19b1a3ac75d58079d7a +size 14917 diff --git a/qwen3/fdd_sfkl/checkpoint-1256/scheduler.pt b/qwen3/fdd_sfkl/checkpoint-1256/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..40e863d07cfed22cd912014eb3ba3fd2b16b173f --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1256/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1775297f14e75f7463ae51983a66296f81f4b2e2e5d7234f1a6d295abfa4cf23 +size 1465 diff --git a/qwen3/fdd_sfkl/checkpoint-1256/tokenizer.json b/qwen3/fdd_sfkl/checkpoint-1256/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1256/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/qwen3/fdd_sfkl/checkpoint-1256/tokenizer_config.json b/qwen3/fdd_sfkl/checkpoint-1256/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1256/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen3/fdd_sfkl/checkpoint-1256/trainer_state.json b/qwen3/fdd_sfkl/checkpoint-1256/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..4627a0d4462f7dfdb036734b3dd494ada257648b --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1256/trainer_state.json @@ -0,0 +1,528 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1256, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.06382130035899482, + "grad_norm": 2.7527661323547363, + "learning_rate": 9.996390385474569e-05, + "loss": 1.4406415939331054, + "step": 20 + }, + { + "epoch": 0.12764260071798963, + "grad_norm": 2.376922607421875, + "learning_rate": 9.984797509888031e-05, + "loss": 0.4754016876220703, + "step": 40 + }, + { + "epoch": 0.19146390107698444, + "grad_norm": 2.429856300354004, + "learning_rate": 9.965229938050925e-05, + "loss": 0.37995381355285646, + "step": 60 + }, + { + "epoch": 0.25528520143597927, + "grad_norm": 1.2270593643188477, + "learning_rate": 9.937719005663432e-05, + "loss": 0.25868964195251465, + "step": 80 + }, + { + "epoch": 0.3191065017949741, + "grad_norm": 0.913340151309967, + "learning_rate": 9.902308769000724e-05, + "loss": 0.20970418453216552, + "step": 100 + }, + { + "epoch": 0.3829278021539689, + "grad_norm": 1.929979920387268, + "learning_rate": 9.859055934360789e-05, + "loss": 0.17272943258285522, + "step": 120 + }, + { + "epoch": 0.4467491025129637, + "grad_norm": 1.0000314712524414, + "learning_rate": 9.808029767254437e-05, + "loss": 0.16882457733154296, + "step": 140 + }, + { + "epoch": 0.5105704028719585, + "grad_norm": 1.1901942491531372, + "learning_rate": 9.749311981482817e-05, + "loss": 0.13708232641220092, + "step": 160 + }, + { + "epoch": 0.5743917032309533, + "grad_norm": 0.9724851250648499, + "learning_rate": 9.682996608280154e-05, + "loss": 0.12325812578201294, + "step": 180 + }, + { + "epoch": 0.6382130035899481, + "grad_norm": 1.5030133724212646, + "learning_rate": 9.609189845731207e-05, + "loss": 0.1345768690109253, + "step": 200 + }, + { + "epoch": 0.702034303948943, + "grad_norm": 0.5464316010475159, + "learning_rate": 9.52800988870462e-05, + "loss": 0.11802985668182372, + "step": 220 + }, + { + "epoch": 0.7658556043079378, + "grad_norm": 1.006695032119751, + "learning_rate": 9.439586739574513e-05, + "loss": 0.11421098709106445, + "step": 240 + }, + { + "epoch": 0.8296769046669326, + "grad_norm": 0.5092183947563171, + "learning_rate": 9.344062000033414e-05, + "loss": 0.10922021865844726, + "step": 260 + }, + { + "epoch": 0.8934982050259274, + "grad_norm": 0.8316523432731628, + "learning_rate": 9.241588644329919e-05, + "loss": 0.10600426197052001, + "step": 280 + }, + { + "epoch": 0.9573195053849222, + "grad_norm": 1.5138731002807617, + "learning_rate": 9.132330774294252e-05, + "loss": 0.1008190631866455, + "step": 300 + }, + { + "epoch": 1.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 87.8505, + "eval_generator_rouge1": 99.5556, + "eval_generator_rouge2": 99.2105, + "eval_generator_rougeL": 99.1217, + "eval_loss": 0.011629586108028889, + "eval_runtime": 615.4826, + "eval_samples_per_second": 5.547, + "eval_selector_accuracy": 93.3995, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.174, + "step": 314 + }, + { + "epoch": 1.0191463901076985, + "grad_norm": 0.41843533515930176, + "learning_rate": 9.01646335654399e-05, + "loss": 0.09401558637619019, + "step": 320 + }, + { + "epoch": 1.0829676904666932, + "grad_norm": 0.47839081287384033, + "learning_rate": 8.894171942290817e-05, + "loss": 0.0862088918685913, + "step": 340 + }, + { + "epoch": 1.146788990825688, + "grad_norm": 0.349435031414032, + "learning_rate": 8.765652370197003e-05, + "loss": 0.08293926119804382, + "step": 360 + }, + { + "epoch": 1.210610291184683, + "grad_norm": 0.5593855977058411, + "learning_rate": 8.631110452757467e-05, + "loss": 0.0761639952659607, + "step": 380 + }, + { + "epoch": 1.2744315915436777, + "grad_norm": 0.4683215320110321, + "learning_rate": 8.490761646709642e-05, + "loss": 0.0819424033164978, + "step": 400 + }, + { + "epoch": 1.3382528919026724, + "grad_norm": 0.3691541850566864, + "learning_rate": 8.344830707998953e-05, + "loss": 0.08254330158233643, + "step": 420 + }, + { + "epoch": 1.4020741922616673, + "grad_norm": 0.3268161416053772, + "learning_rate": 8.193551331852458e-05, + "loss": 0.07523589730262756, + "step": 440 + }, + { + "epoch": 1.4658954926206622, + "grad_norm": 0.9531517624855042, + "learning_rate": 8.037165778537033e-05, + "loss": 0.07261439561843872, + "step": 460 + }, + { + "epoch": 1.5297167929796571, + "grad_norm": 0.33297064900398254, + "learning_rate": 7.875924485401427e-05, + "loss": 0.07137778997421265, + "step": 480 + }, + { + "epoch": 1.5935380933386518, + "grad_norm": 0.5712356567382812, + "learning_rate": 7.710085665823444e-05, + "loss": 0.07516297101974487, + "step": 500 + }, + { + "epoch": 1.6573593936976465, + "grad_norm": 0.27220338582992554, + "learning_rate": 7.539914895704521e-05, + "loss": 0.0703214168548584, + "step": 520 + }, + { + "epoch": 1.7211806940566414, + "grad_norm": 1.4434664249420166, + "learning_rate": 7.365684688173908e-05, + "loss": 0.08012890815734863, + "step": 540 + }, + { + "epoch": 1.7850019944156363, + "grad_norm": 0.32184913754463196, + "learning_rate": 7.18767405718347e-05, + "loss": 0.06634219884872436, + "step": 560 + }, + { + "epoch": 1.848823294774631, + "grad_norm": 1.6946690082550049, + "learning_rate": 7.006168070692036e-05, + "loss": 0.07123274803161621, + "step": 580 + }, + { + "epoch": 1.9126445951336257, + "grad_norm": 1.2815985679626465, + "learning_rate": 6.821457394154786e-05, + "loss": 0.06565064191818237, + "step": 600 + }, + { + "epoch": 1.9764658954926206, + "grad_norm": 0.5232482552528381, + "learning_rate": 6.633837825048768e-05, + "loss": 0.06681714057922364, + "step": 620 + }, + { + "epoch": 2.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 92.6402, + "eval_generator_rouge1": 99.741, + "eval_generator_rouge2": 99.5504, + "eval_generator_rougeL": 99.4854, + "eval_loss": 0.007036722265183926, + "eval_runtime": 630.2117, + "eval_samples_per_second": 5.417, + "eval_selector_accuracy": 98.0724, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.17, + "step": 628 + }, + { + "epoch": 2.038292780215397, + "grad_norm": 0.2939830422401428, + "learning_rate": 6.443609819179926e-05, + "loss": 0.06403520107269287, + "step": 640 + }, + { + "epoch": 2.102114080574392, + "grad_norm": 0.32704657316207886, + "learning_rate": 6.251078009530256e-05, + "loss": 0.060550224781036374, + "step": 660 + }, + { + "epoch": 2.1659353809333863, + "grad_norm": 0.42840492725372314, + "learning_rate": 6.0565507184155677e-05, + "loss": 0.060464733839035036, + "step": 680 + }, + { + "epoch": 2.2297566812923812, + "grad_norm": 0.6081668734550476, + "learning_rate": 5.860339463735136e-05, + "loss": 0.05875073671340943, + "step": 700 + }, + { + "epoch": 2.293577981651376, + "grad_norm": 0.28151988983154297, + "learning_rate": 5.662758460103895e-05, + "loss": 0.05764824151992798, + "step": 720 + }, + { + "epoch": 2.357399282010371, + "grad_norm": 0.2843102514743805, + "learning_rate": 5.464124115666095e-05, + "loss": 0.05847916603088379, + "step": 740 + }, + { + "epoch": 2.421220582369366, + "grad_norm": 0.34520450234413147, + "learning_rate": 5.264754525396212e-05, + "loss": 0.056367015838623045, + "step": 760 + }, + { + "epoch": 2.4850418827283605, + "grad_norm": 0.45693477988243103, + "learning_rate": 5.06496896169857e-05, + "loss": 0.055580759048461915, + "step": 780 + }, + { + "epoch": 2.5488631830873554, + "grad_norm": 0.3360002338886261, + "learning_rate": 4.865087363121384e-05, + "loss": 0.061715197563171384, + "step": 800 + }, + { + "epoch": 2.6126844834463503, + "grad_norm": 0.5039923787117004, + "learning_rate": 4.6654298220040673e-05, + "loss": 0.05754677057266235, + "step": 820 + }, + { + "epoch": 2.6765057838053448, + "grad_norm": 0.29794442653656006, + "learning_rate": 4.466316071878229e-05, + "loss": 0.055103600025177, + "step": 840 + }, + { + "epoch": 2.74032708416434, + "grad_norm": 0.21690921485424042, + "learning_rate": 4.2680649754432913e-05, + "loss": 0.054493296146392825, + "step": 860 + }, + { + "epoch": 2.8041483845233346, + "grad_norm": 0.28939780592918396, + "learning_rate": 4.070994013936646e-05, + "loss": 0.05614350438117981, + "step": 880 + }, + { + "epoch": 2.8679696848823295, + "grad_norm": 0.5370737910270691, + "learning_rate": 3.8754187787161155e-05, + "loss": 0.05749887228012085, + "step": 900 + }, + { + "epoch": 2.9317909852413244, + "grad_norm": 0.21074210107326508, + "learning_rate": 3.681652465868875e-05, + "loss": 0.05852361917495728, + "step": 920 + }, + { + "epoch": 2.995612285600319, + "grad_norm": 0.4107605516910553, + "learning_rate": 3.490005374656184e-05, + "loss": 0.05852218866348267, + "step": 940 + }, + { + "epoch": 3.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 94.6846, + "eval_generator_rouge1": 99.8316, + "eval_generator_rouge2": 99.7174, + "eval_generator_rougeL": 99.6293, + "eval_loss": 0.005526242312043905, + "eval_runtime": 613.3531, + "eval_samples_per_second": 5.566, + "eval_selector_accuracy": 96.9042, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.174, + "step": 942 + }, + { + "epoch": 3.057439170323095, + "grad_norm": 0.21888521313667297, + "learning_rate": 3.3007844105971336e-05, + "loss": 0.05516682863235474, + "step": 960 + }, + { + "epoch": 3.12126047068209, + "grad_norm": 0.2247946858406067, + "learning_rate": 3.114292593987149e-05, + "loss": 0.05510848164558411, + "step": 980 + }, + { + "epoch": 3.185081771041085, + "grad_norm": 0.18932947516441345, + "learning_rate": 2.9308285746383408e-05, + "loss": 0.05192863345146179, + "step": 1000 + }, + { + "epoch": 3.24890307140008, + "grad_norm": 0.18617865443229675, + "learning_rate": 2.7506861536187867e-05, + "loss": 0.05452244281768799, + "step": 1020 + }, + { + "epoch": 3.3127243717590744, + "grad_norm": 0.21938033401966095, + "learning_rate": 2.5741538127566396e-05, + "loss": 0.051268589496612546, + "step": 1040 + }, + { + "epoch": 3.3765456721180693, + "grad_norm": 0.2056664675474167, + "learning_rate": 2.4015142526625115e-05, + "loss": 0.05224469304084778, + "step": 1060 + }, + { + "epoch": 3.4403669724770642, + "grad_norm": 0.2431439608335495, + "learning_rate": 2.233043940009967e-05, + "loss": 0.05370842218399048, + "step": 1080 + }, + { + "epoch": 3.504188272836059, + "grad_norm": 0.1797763705253601, + "learning_rate": 2.0690126647990974e-05, + "loss": 0.05102568864822388, + "step": 1100 + }, + { + "epoch": 3.568009573195054, + "grad_norm": 0.20713362097740173, + "learning_rate": 1.9096831083121828e-05, + "loss": 0.05096147060394287, + "step": 1120 + }, + { + "epoch": 3.6318308735540485, + "grad_norm": 0.26381605863571167, + "learning_rate": 1.7553104224533324e-05, + "loss": 0.050066190958023074, + "step": 1140 + }, + { + "epoch": 3.6956521739130435, + "grad_norm": 0.2653278708457947, + "learning_rate": 1.606141821145735e-05, + "loss": 0.050461554527282716, + "step": 1160 + }, + { + "epoch": 3.7594734742720384, + "grad_norm": 0.2132464498281479, + "learning_rate": 1.4624161844408763e-05, + "loss": 0.0523165225982666, + "step": 1180 + }, + { + "epoch": 3.823294774631033, + "grad_norm": 0.19349783658981323, + "learning_rate": 1.324363675973698e-05, + "loss": 0.04992191791534424, + "step": 1200 + }, + { + "epoch": 3.8871160749900278, + "grad_norm": 0.240342378616333, + "learning_rate": 1.1922053743763052e-05, + "loss": 0.05000295639038086, + "step": 1220 + }, + { + "epoch": 3.9509373753490227, + "grad_norm": 0.19015061855316162, + "learning_rate": 1.0661529192404973e-05, + "loss": 0.04904879331588745, + "step": 1240 + }, + { + "epoch": 4.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 95.8528, + "eval_generator_rouge1": 99.9437, + "eval_generator_rouge2": 99.8882, + "eval_generator_rougeL": 99.7233, + "eval_loss": 0.003825092688202858, + "eval_runtime": 603.7734, + "eval_samples_per_second": 5.654, + "eval_selector_accuracy": 98.5397, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.177, + "step": 1256 + } + ], + "logging_steps": 20, + "max_steps": 1570, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.697643190059008e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/qwen3/fdd_sfkl/checkpoint-1256/training_args.bin b/qwen3/fdd_sfkl/checkpoint-1256/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..4aaaca662465ff2e3933154ec90d4097299421a9 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1256/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18f98e7767ecd74653cd41dec83bbbe276858948c803da4b684e0aaf5a1cc1f7 +size 7185 diff --git a/qwen3/fdd_sfkl/checkpoint-1256/zero_to_fp32.py b/qwen3/fdd_sfkl/checkpoint-1256/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1256/zero_to_fp32.py @@ -0,0 +1,790 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY, + AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY, + AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def _has_autoep_zero3_partitioned_metadata(state_dict): + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY) + if autoep_layers is None: + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY) + if not isinstance(autoep_layers, list): + return False + return any( + isinstance(entry, dict) + and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT + for entry in autoep_layers) + + +def _raise_if_autoep_zero3_partitioned_state(state_dict): + if _has_autoep_zero3_partitioned_metadata(state_dict): + raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. " + "AutoEP expert parameters are partitioned over expert replica groups, so " + "global data-parallel consolidation would produce incomplete expert tensors. " + "Use ds_to_universal.py for expert-aware conversion.") + + +def _raise_if_autoep_zero3_partitioned_checkpoint(model_files): + for file in model_files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints + # before the expensive optimizer-shard load below. + model_files = get_model_state_files(ds_checkpoint_dir) + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/qwen3/fdd_sfkl/checkpoint-1570/README.md b/qwen3/fdd_sfkl/checkpoint-1570/README.md new file mode 100644 index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1570/README.md @@ -0,0 +1,208 @@ +--- +base_model: Qwen/Qwen3-0.6B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-0.6B +- llama-factory +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-1570/adapter_config.json b/qwen3/fdd_sfkl/checkpoint-1570/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..09417a2e76df9aef95a5236d9a1aed2554e49cf2 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1570/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-0.6B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "q_proj", + "o_proj", + "down_proj", + "gate_proj", + "k_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-1570/adapter_model.safetensors b/qwen3/fdd_sfkl/checkpoint-1570/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..50556795650467d454d7fc7264461db029d298ad --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1570/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3f0d9b21adf90b537c43ddb87d2dff137658e10aa83b08da7afb20e75f34950 +size 40422560 diff --git a/qwen3/fdd_sfkl/checkpoint-1570/chat_template.jinja b/qwen3/fdd_sfkl/checkpoint-1570/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1570/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/fdd_sfkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f0bfd2809f683ee028be76d7022f39f2aebd80bc --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49c6594a9821f5ae25502b77316571288a55e7a678c344058b00a62cd0e4aaba +size 121135557 diff --git a/qwen3/fdd_sfkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/fdd_sfkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7febbfb75f6dc851b07e306c894618a69367650c --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1570/global_step1570/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3dbcfcdb8ac808c2ca7f038de2cbf77c9385346941a2de53010f60533c08af1 +size 121135749 diff --git a/qwen3/fdd_sfkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt b/qwen3/fdd_sfkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0fc04246294ff2e8f57a3c985f9fc8d72728f36a --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1570/global_step1570/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a55278c39805f6a8048ff6746de29a140f721ecd26b11fbbf09df8d6a9f3190a +size 351841765 diff --git a/qwen3/fdd_sfkl/checkpoint-1570/latest b/qwen3/fdd_sfkl/checkpoint-1570/latest new file mode 100644 index 0000000000000000000000000000000000000000..18c2fcff58fa6daa0a2e7c3a015552a69085778b --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1570/latest @@ -0,0 +1 @@ +global_step1570 \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-1570/rng_state_0.pth b/qwen3/fdd_sfkl/checkpoint-1570/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..cf424aae5981c58cf15f8431bcbc55d04d4cd749 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1570/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71157745b9c6f5ff4bfb29f9ae30b0df6b1f004f724ec092815a32ee75a9d6f8 +size 14917 diff --git a/qwen3/fdd_sfkl/checkpoint-1570/rng_state_1.pth b/qwen3/fdd_sfkl/checkpoint-1570/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..a85b0e39f3685ca41401f703561817a469bb6a53 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1570/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f74455ec3b5eca2303cfd033e06bf301c40a94725fbe0af006b244b460899f1f +size 14917 diff --git a/qwen3/fdd_sfkl/checkpoint-1570/scheduler.pt b/qwen3/fdd_sfkl/checkpoint-1570/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..e22954463492d8a070ae427c95418df61893c54c --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1570/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6332fac17ae99a07597512c705658da2fd8429d1c39bb776b30e107c2cb7aae +size 1465 diff --git a/qwen3/fdd_sfkl/checkpoint-1570/tokenizer.json b/qwen3/fdd_sfkl/checkpoint-1570/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1570/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/qwen3/fdd_sfkl/checkpoint-1570/tokenizer_config.json b/qwen3/fdd_sfkl/checkpoint-1570/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1570/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen3/fdd_sfkl/checkpoint-1570/trainer_state.json b/qwen3/fdd_sfkl/checkpoint-1570/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c194ead5d9d436e2acf091a74c4d57a2a9a947c4 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1570/trainer_state.json @@ -0,0 +1,655 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 1570, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.06382130035899482, + "grad_norm": 2.7527661323547363, + "learning_rate": 9.996390385474569e-05, + "loss": 1.4406415939331054, + "step": 20 + }, + { + "epoch": 0.12764260071798963, + "grad_norm": 2.376922607421875, + "learning_rate": 9.984797509888031e-05, + "loss": 0.4754016876220703, + "step": 40 + }, + { + "epoch": 0.19146390107698444, + "grad_norm": 2.429856300354004, + "learning_rate": 9.965229938050925e-05, + "loss": 0.37995381355285646, + "step": 60 + }, + { + "epoch": 0.25528520143597927, + "grad_norm": 1.2270593643188477, + "learning_rate": 9.937719005663432e-05, + "loss": 0.25868964195251465, + "step": 80 + }, + { + "epoch": 0.3191065017949741, + "grad_norm": 0.913340151309967, + "learning_rate": 9.902308769000724e-05, + "loss": 0.20970418453216552, + "step": 100 + }, + { + "epoch": 0.3829278021539689, + "grad_norm": 1.929979920387268, + "learning_rate": 9.859055934360789e-05, + "loss": 0.17272943258285522, + "step": 120 + }, + { + "epoch": 0.4467491025129637, + "grad_norm": 1.0000314712524414, + "learning_rate": 9.808029767254437e-05, + "loss": 0.16882457733154296, + "step": 140 + }, + { + "epoch": 0.5105704028719585, + "grad_norm": 1.1901942491531372, + "learning_rate": 9.749311981482817e-05, + "loss": 0.13708232641220092, + "step": 160 + }, + { + "epoch": 0.5743917032309533, + "grad_norm": 0.9724851250648499, + "learning_rate": 9.682996608280154e-05, + "loss": 0.12325812578201294, + "step": 180 + }, + { + "epoch": 0.6382130035899481, + "grad_norm": 1.5030133724212646, + "learning_rate": 9.609189845731207e-05, + "loss": 0.1345768690109253, + "step": 200 + }, + { + "epoch": 0.702034303948943, + "grad_norm": 0.5464316010475159, + "learning_rate": 9.52800988870462e-05, + "loss": 0.11802985668182372, + "step": 220 + }, + { + "epoch": 0.7658556043079378, + "grad_norm": 1.006695032119751, + "learning_rate": 9.439586739574513e-05, + "loss": 0.11421098709106445, + "step": 240 + }, + { + "epoch": 0.8296769046669326, + "grad_norm": 0.5092183947563171, + "learning_rate": 9.344062000033414e-05, + "loss": 0.10922021865844726, + "step": 260 + }, + { + "epoch": 0.8934982050259274, + "grad_norm": 0.8316523432731628, + "learning_rate": 9.241588644329919e-05, + "loss": 0.10600426197052001, + "step": 280 + }, + { + "epoch": 0.9573195053849222, + "grad_norm": 1.5138731002807617, + "learning_rate": 9.132330774294252e-05, + "loss": 0.1008190631866455, + "step": 300 + }, + { + "epoch": 1.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 87.8505, + "eval_generator_rouge1": 99.5556, + "eval_generator_rouge2": 99.2105, + "eval_generator_rougeL": 99.1217, + "eval_loss": 0.011629586108028889, + "eval_runtime": 615.4826, + "eval_samples_per_second": 5.547, + "eval_selector_accuracy": 93.3995, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.174, + "step": 314 + }, + { + "epoch": 1.0191463901076985, + "grad_norm": 0.41843533515930176, + "learning_rate": 9.01646335654399e-05, + "loss": 0.09401558637619019, + "step": 320 + }, + { + "epoch": 1.0829676904666932, + "grad_norm": 0.47839081287384033, + "learning_rate": 8.894171942290817e-05, + "loss": 0.0862088918685913, + "step": 340 + }, + { + "epoch": 1.146788990825688, + "grad_norm": 0.349435031414032, + "learning_rate": 8.765652370197003e-05, + "loss": 0.08293926119804382, + "step": 360 + }, + { + "epoch": 1.210610291184683, + "grad_norm": 0.5593855977058411, + "learning_rate": 8.631110452757467e-05, + "loss": 0.0761639952659607, + "step": 380 + }, + { + "epoch": 1.2744315915436777, + "grad_norm": 0.4683215320110321, + "learning_rate": 8.490761646709642e-05, + "loss": 0.0819424033164978, + "step": 400 + }, + { + "epoch": 1.3382528919026724, + "grad_norm": 0.3691541850566864, + "learning_rate": 8.344830707998953e-05, + "loss": 0.08254330158233643, + "step": 420 + }, + { + "epoch": 1.4020741922616673, + "grad_norm": 0.3268161416053772, + "learning_rate": 8.193551331852458e-05, + "loss": 0.07523589730262756, + "step": 440 + }, + { + "epoch": 1.4658954926206622, + "grad_norm": 0.9531517624855042, + "learning_rate": 8.037165778537033e-05, + "loss": 0.07261439561843872, + "step": 460 + }, + { + "epoch": 1.5297167929796571, + "grad_norm": 0.33297064900398254, + "learning_rate": 7.875924485401427e-05, + "loss": 0.07137778997421265, + "step": 480 + }, + { + "epoch": 1.5935380933386518, + "grad_norm": 0.5712356567382812, + "learning_rate": 7.710085665823444e-05, + "loss": 0.07516297101974487, + "step": 500 + }, + { + "epoch": 1.6573593936976465, + "grad_norm": 0.27220338582992554, + "learning_rate": 7.539914895704521e-05, + "loss": 0.0703214168548584, + "step": 520 + }, + { + "epoch": 1.7211806940566414, + "grad_norm": 1.4434664249420166, + "learning_rate": 7.365684688173908e-05, + "loss": 0.08012890815734863, + "step": 540 + }, + { + "epoch": 1.7850019944156363, + "grad_norm": 0.32184913754463196, + "learning_rate": 7.18767405718347e-05, + "loss": 0.06634219884872436, + "step": 560 + }, + { + "epoch": 1.848823294774631, + "grad_norm": 1.6946690082550049, + "learning_rate": 7.006168070692036e-05, + "loss": 0.07123274803161621, + "step": 580 + }, + { + "epoch": 1.9126445951336257, + "grad_norm": 1.2815985679626465, + "learning_rate": 6.821457394154786e-05, + "loss": 0.06565064191818237, + "step": 600 + }, + { + "epoch": 1.9764658954926206, + "grad_norm": 0.5232482552528381, + "learning_rate": 6.633837825048768e-05, + "loss": 0.06681714057922364, + "step": 620 + }, + { + "epoch": 2.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 92.6402, + "eval_generator_rouge1": 99.741, + "eval_generator_rouge2": 99.5504, + "eval_generator_rougeL": 99.4854, + "eval_loss": 0.007036722265183926, + "eval_runtime": 630.2117, + "eval_samples_per_second": 5.417, + "eval_selector_accuracy": 98.0724, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.17, + "step": 628 + }, + { + "epoch": 2.038292780215397, + "grad_norm": 0.2939830422401428, + "learning_rate": 6.443609819179926e-05, + "loss": 0.06403520107269287, + "step": 640 + }, + { + "epoch": 2.102114080574392, + "grad_norm": 0.32704657316207886, + "learning_rate": 6.251078009530256e-05, + "loss": 0.060550224781036374, + "step": 660 + }, + { + "epoch": 2.1659353809333863, + "grad_norm": 0.42840492725372314, + "learning_rate": 6.0565507184155677e-05, + "loss": 0.060464733839035036, + "step": 680 + }, + { + "epoch": 2.2297566812923812, + "grad_norm": 0.6081668734550476, + "learning_rate": 5.860339463735136e-05, + "loss": 0.05875073671340943, + "step": 700 + }, + { + "epoch": 2.293577981651376, + "grad_norm": 0.28151988983154297, + "learning_rate": 5.662758460103895e-05, + "loss": 0.05764824151992798, + "step": 720 + }, + { + "epoch": 2.357399282010371, + "grad_norm": 0.2843102514743805, + "learning_rate": 5.464124115666095e-05, + "loss": 0.05847916603088379, + "step": 740 + }, + { + "epoch": 2.421220582369366, + "grad_norm": 0.34520450234413147, + "learning_rate": 5.264754525396212e-05, + "loss": 0.056367015838623045, + "step": 760 + }, + { + "epoch": 2.4850418827283605, + "grad_norm": 0.45693477988243103, + "learning_rate": 5.06496896169857e-05, + "loss": 0.055580759048461915, + "step": 780 + }, + { + "epoch": 2.5488631830873554, + "grad_norm": 0.3360002338886261, + "learning_rate": 4.865087363121384e-05, + "loss": 0.061715197563171384, + "step": 800 + }, + { + "epoch": 2.6126844834463503, + "grad_norm": 0.5039923787117004, + "learning_rate": 4.6654298220040673e-05, + "loss": 0.05754677057266235, + "step": 820 + }, + { + "epoch": 2.6765057838053448, + "grad_norm": 0.29794442653656006, + "learning_rate": 4.466316071878229e-05, + "loss": 0.055103600025177, + "step": 840 + }, + { + "epoch": 2.74032708416434, + "grad_norm": 0.21690921485424042, + "learning_rate": 4.2680649754432913e-05, + "loss": 0.054493296146392825, + "step": 860 + }, + { + "epoch": 2.8041483845233346, + "grad_norm": 0.28939780592918396, + "learning_rate": 4.070994013936646e-05, + "loss": 0.05614350438117981, + "step": 880 + }, + { + "epoch": 2.8679696848823295, + "grad_norm": 0.5370737910270691, + "learning_rate": 3.8754187787161155e-05, + "loss": 0.05749887228012085, + "step": 900 + }, + { + "epoch": 2.9317909852413244, + "grad_norm": 0.21074210107326508, + "learning_rate": 3.681652465868875e-05, + "loss": 0.05852361917495728, + "step": 920 + }, + { + "epoch": 2.995612285600319, + "grad_norm": 0.4107605516910553, + "learning_rate": 3.490005374656184e-05, + "loss": 0.05852218866348267, + "step": 940 + }, + { + "epoch": 3.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 94.6846, + "eval_generator_rouge1": 99.8316, + "eval_generator_rouge2": 99.7174, + "eval_generator_rougeL": 99.6293, + "eval_loss": 0.005526242312043905, + "eval_runtime": 613.3531, + "eval_samples_per_second": 5.566, + "eval_selector_accuracy": 96.9042, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.174, + "step": 942 + }, + { + "epoch": 3.057439170323095, + "grad_norm": 0.21888521313667297, + "learning_rate": 3.3007844105971336e-05, + "loss": 0.05516682863235474, + "step": 960 + }, + { + "epoch": 3.12126047068209, + "grad_norm": 0.2247946858406067, + "learning_rate": 3.114292593987149e-05, + "loss": 0.05510848164558411, + "step": 980 + }, + { + "epoch": 3.185081771041085, + "grad_norm": 0.18932947516441345, + "learning_rate": 2.9308285746383408e-05, + "loss": 0.05192863345146179, + "step": 1000 + }, + { + "epoch": 3.24890307140008, + "grad_norm": 0.18617865443229675, + "learning_rate": 2.7506861536187867e-05, + "loss": 0.05452244281768799, + "step": 1020 + }, + { + "epoch": 3.3127243717590744, + "grad_norm": 0.21938033401966095, + "learning_rate": 2.5741538127566396e-05, + "loss": 0.051268589496612546, + "step": 1040 + }, + { + "epoch": 3.3765456721180693, + "grad_norm": 0.2056664675474167, + "learning_rate": 2.4015142526625115e-05, + "loss": 0.05224469304084778, + "step": 1060 + }, + { + "epoch": 3.4403669724770642, + "grad_norm": 0.2431439608335495, + "learning_rate": 2.233043940009967e-05, + "loss": 0.05370842218399048, + "step": 1080 + }, + { + "epoch": 3.504188272836059, + "grad_norm": 0.1797763705253601, + "learning_rate": 2.0690126647990974e-05, + "loss": 0.05102568864822388, + "step": 1100 + }, + { + "epoch": 3.568009573195054, + "grad_norm": 0.20713362097740173, + "learning_rate": 1.9096831083121828e-05, + "loss": 0.05096147060394287, + "step": 1120 + }, + { + "epoch": 3.6318308735540485, + "grad_norm": 0.26381605863571167, + "learning_rate": 1.7553104224533324e-05, + "loss": 0.050066190958023074, + "step": 1140 + }, + { + "epoch": 3.6956521739130435, + "grad_norm": 0.2653278708457947, + "learning_rate": 1.606141821145735e-05, + "loss": 0.050461554527282716, + "step": 1160 + }, + { + "epoch": 3.7594734742720384, + "grad_norm": 0.2132464498281479, + "learning_rate": 1.4624161844408763e-05, + "loss": 0.0523165225982666, + "step": 1180 + }, + { + "epoch": 3.823294774631033, + "grad_norm": 0.19349783658981323, + "learning_rate": 1.324363675973698e-05, + "loss": 0.04992191791534424, + "step": 1200 + }, + { + "epoch": 3.8871160749900278, + "grad_norm": 0.240342378616333, + "learning_rate": 1.1922053743763052e-05, + "loss": 0.05000295639038086, + "step": 1220 + }, + { + "epoch": 3.9509373753490227, + "grad_norm": 0.19015061855316162, + "learning_rate": 1.0661529192404973e-05, + "loss": 0.04904879331588745, + "step": 1240 + }, + { + "epoch": 4.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 95.8528, + "eval_generator_rouge1": 99.9437, + "eval_generator_rouge2": 99.8882, + "eval_generator_rougeL": 99.7233, + "eval_loss": 0.003825092688202858, + "eval_runtime": 603.7734, + "eval_samples_per_second": 5.654, + "eval_selector_accuracy": 98.5397, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.177, + "step": 1256 + }, + { + "epoch": 4.0127642600717985, + "grad_norm": 0.19705015420913696, + "learning_rate": 9.464081721960592e-06, + "loss": 0.050104594230651854, + "step": 1260 + }, + { + "epoch": 4.076585560430794, + "grad_norm": 0.33697429299354553, + "learning_rate": 8.331628936475818e-06, + "loss": 0.04852663278579712, + "step": 1280 + }, + { + "epoch": 4.140406860789788, + "grad_norm": 0.21710842847824097, + "learning_rate": 7.2659843568748385e-06, + "loss": 0.048302799463272095, + "step": 1300 + }, + { + "epoch": 4.204228161148784, + "grad_norm": 0.16647987067699432, + "learning_rate": 6.268854516770008e-06, + "loss": 0.04852384328842163, + "step": 1320 + }, + { + "epoch": 4.268049461507778, + "grad_norm": 0.1737622171640396, + "learning_rate": 5.341836229602331e-06, + "loss": 0.04865268170833588, + "step": 1340 + }, + { + "epoch": 4.331870761866773, + "grad_norm": 0.19361254572868347, + "learning_rate": 4.4864140314888216e-06, + "loss": 0.04774414896965027, + "step": 1360 + }, + { + "epoch": 4.395692062225768, + "grad_norm": 0.18790625035762787, + "learning_rate": 3.703957803871932e-06, + "loss": 0.04869887530803681, + "step": 1380 + }, + { + "epoch": 4.4595133625847625, + "grad_norm": 0.16145937144756317, + "learning_rate": 2.9957205797780495e-06, + "loss": 0.049949395656585696, + "step": 1400 + }, + { + "epoch": 4.523334662943758, + "grad_norm": 0.31126755475997925, + "learning_rate": 2.362836537198199e-06, + "loss": 0.049376997351646426, + "step": 1420 + }, + { + "epoch": 4.587155963302752, + "grad_norm": 0.17530952394008636, + "learning_rate": 1.8063191828043703e-06, + "loss": 0.04881583452224732, + "step": 1440 + }, + { + "epoch": 4.650977263661747, + "grad_norm": 0.21170786023139954, + "learning_rate": 1.3270597289100437e-06, + "loss": 0.04805063009262085, + "step": 1460 + }, + { + "epoch": 4.714798564020742, + "grad_norm": 0.156987264752388, + "learning_rate": 9.258256662741205e-07, + "loss": 0.047132036089897154, + "step": 1480 + }, + { + "epoch": 4.778619864379737, + "grad_norm": 0.1759834736585617, + "learning_rate": 6.032595350337341e-07, + "loss": 0.048763877153396605, + "step": 1500 + }, + { + "epoch": 4.842441164738732, + "grad_norm": 0.16100358963012695, + "learning_rate": 3.5987789573421356e-07, + "loss": 0.05374665260314941, + "step": 1520 + }, + { + "epoch": 4.9062624650977265, + "grad_norm": 0.21420489251613617, + "learning_rate": 1.9607050210401192e-07, + "loss": 0.049358141422271726, + "step": 1540 + }, + { + "epoch": 4.970083765456721, + "grad_norm": 0.1897514909505844, + "learning_rate": 1.1209967689925832e-07, + "loss": 0.049035769701004026, + "step": 1560 + }, + { + "epoch": 5.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 96.3785, + "eval_generator_rouge1": 99.9468, + "eval_generator_rouge2": 99.8886, + "eval_generator_rougeL": 99.7652, + "eval_loss": 0.0038434057496488094, + "eval_runtime": 612.3424, + "eval_samples_per_second": 5.575, + "eval_selector_accuracy": 98.8318, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.175, + "step": 1570 + } + ], + "logging_steps": 20, + "max_steps": 1570, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 5.871444840310374e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/qwen3/fdd_sfkl/checkpoint-1570/training_args.bin b/qwen3/fdd_sfkl/checkpoint-1570/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..4aaaca662465ff2e3933154ec90d4097299421a9 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1570/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18f98e7767ecd74653cd41dec83bbbe276858948c803da4b684e0aaf5a1cc1f7 +size 7185 diff --git a/qwen3/fdd_sfkl/checkpoint-1570/zero_to_fp32.py b/qwen3/fdd_sfkl/checkpoint-1570/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-1570/zero_to_fp32.py @@ -0,0 +1,790 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY, + AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY, + AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def _has_autoep_zero3_partitioned_metadata(state_dict): + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY) + if autoep_layers is None: + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY) + if not isinstance(autoep_layers, list): + return False + return any( + isinstance(entry, dict) + and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT + for entry in autoep_layers) + + +def _raise_if_autoep_zero3_partitioned_state(state_dict): + if _has_autoep_zero3_partitioned_metadata(state_dict): + raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. " + "AutoEP expert parameters are partitioned over expert replica groups, so " + "global data-parallel consolidation would produce incomplete expert tensors. " + "Use ds_to_universal.py for expert-aware conversion.") + + +def _raise_if_autoep_zero3_partitioned_checkpoint(model_files): + for file in model_files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints + # before the expensive optimizer-shard load below. + model_files = get_model_state_files(ds_checkpoint_dir) + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/qwen3/fdd_sfkl/checkpoint-314/README.md b/qwen3/fdd_sfkl/checkpoint-314/README.md new file mode 100644 index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-314/README.md @@ -0,0 +1,208 @@ +--- +base_model: Qwen/Qwen3-0.6B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-0.6B +- llama-factory +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-314/adapter_config.json b/qwen3/fdd_sfkl/checkpoint-314/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..09417a2e76df9aef95a5236d9a1aed2554e49cf2 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-314/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-0.6B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "q_proj", + "o_proj", + "down_proj", + "gate_proj", + "k_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-314/adapter_model.safetensors b/qwen3/fdd_sfkl/checkpoint-314/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..29331a65ff6d6192c857563facbee84d48b0f514 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-314/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e98f41a038f691e29d715b3f4ce58e5c2d35e5fae30ffb9bac3eb7800951577a +size 40422560 diff --git a/qwen3/fdd_sfkl/checkpoint-314/chat_template.jinja b/qwen3/fdd_sfkl/checkpoint-314/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-314/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/fdd_sfkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cb7147a03f14fed429f54eed34719a332cb5c8e8 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-314/global_step314/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd4c91c01a4d9e98dfe6ab9b4996c2e57c16d612288419d0d39463d8b9549c12 +size 121135557 diff --git a/qwen3/fdd_sfkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/fdd_sfkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d55682e7711b939b0716ed8d18ce317704bf347d --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-314/global_step314/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f364f85ff843b0e00c7e1f14e94a595b024a2707965ab1a299fe09f9eaa48794 +size 121135749 diff --git a/qwen3/fdd_sfkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt b/qwen3/fdd_sfkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..04469e12f7f352c3cba7286461d9040e1ea059f7 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-314/global_step314/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4900fd807ec0adc8821c616c2ccea5c771a3550269c0cd48a2e6d47a16f52481 +size 351841765 diff --git a/qwen3/fdd_sfkl/checkpoint-314/latest b/qwen3/fdd_sfkl/checkpoint-314/latest new file mode 100644 index 0000000000000000000000000000000000000000..a9786513b1a514db2c3a9da7408844635663d55d --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-314/latest @@ -0,0 +1 @@ +global_step314 \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-314/rng_state_0.pth b/qwen3/fdd_sfkl/checkpoint-314/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..50c1843947936add2f9b96bd79adb4e89f555392 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-314/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a789c9b61412b33aaeaab412761a1a1cd4dae58bc0d166961a04608f87d5381 +size 14917 diff --git a/qwen3/fdd_sfkl/checkpoint-314/rng_state_1.pth b/qwen3/fdd_sfkl/checkpoint-314/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..e6b3866405f7d6e36a98f5a5acc91d5accdb9b90 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-314/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:864041536244bc384b9303e58fdc1562eb2e97b9aa8048b17d562a1dfaf65a05 +size 14917 diff --git a/qwen3/fdd_sfkl/checkpoint-314/scheduler.pt b/qwen3/fdd_sfkl/checkpoint-314/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..a3cb460802cda03bf0833a98a6f1428a3fd4f585 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-314/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17e254b5642bb75983fd5105bec79b35376cfbc99d29702287668b8767d46149 +size 1465 diff --git a/qwen3/fdd_sfkl/checkpoint-314/tokenizer.json b/qwen3/fdd_sfkl/checkpoint-314/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-314/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/qwen3/fdd_sfkl/checkpoint-314/tokenizer_config.json b/qwen3/fdd_sfkl/checkpoint-314/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-314/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen3/fdd_sfkl/checkpoint-314/trainer_state.json b/qwen3/fdd_sfkl/checkpoint-314/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9cae3b2190adfc312b6d4e29616116441603dd46 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-314/trainer_state.json @@ -0,0 +1,154 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 314, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.06382130035899482, + "grad_norm": 2.7527661323547363, + "learning_rate": 9.996390385474569e-05, + "loss": 1.4406415939331054, + "step": 20 + }, + { + "epoch": 0.12764260071798963, + "grad_norm": 2.376922607421875, + "learning_rate": 9.984797509888031e-05, + "loss": 0.4754016876220703, + "step": 40 + }, + { + "epoch": 0.19146390107698444, + "grad_norm": 2.429856300354004, + "learning_rate": 9.965229938050925e-05, + "loss": 0.37995381355285646, + "step": 60 + }, + { + "epoch": 0.25528520143597927, + "grad_norm": 1.2270593643188477, + "learning_rate": 9.937719005663432e-05, + "loss": 0.25868964195251465, + "step": 80 + }, + { + "epoch": 0.3191065017949741, + "grad_norm": 0.913340151309967, + "learning_rate": 9.902308769000724e-05, + "loss": 0.20970418453216552, + "step": 100 + }, + { + "epoch": 0.3829278021539689, + "grad_norm": 1.929979920387268, + "learning_rate": 9.859055934360789e-05, + "loss": 0.17272943258285522, + "step": 120 + }, + { + "epoch": 0.4467491025129637, + "grad_norm": 1.0000314712524414, + "learning_rate": 9.808029767254437e-05, + "loss": 0.16882457733154296, + "step": 140 + }, + { + "epoch": 0.5105704028719585, + "grad_norm": 1.1901942491531372, + "learning_rate": 9.749311981482817e-05, + "loss": 0.13708232641220092, + "step": 160 + }, + { + "epoch": 0.5743917032309533, + "grad_norm": 0.9724851250648499, + "learning_rate": 9.682996608280154e-05, + "loss": 0.12325812578201294, + "step": 180 + }, + { + "epoch": 0.6382130035899481, + "grad_norm": 1.5030133724212646, + "learning_rate": 9.609189845731207e-05, + "loss": 0.1345768690109253, + "step": 200 + }, + { + "epoch": 0.702034303948943, + "grad_norm": 0.5464316010475159, + "learning_rate": 9.52800988870462e-05, + "loss": 0.11802985668182372, + "step": 220 + }, + { + "epoch": 0.7658556043079378, + "grad_norm": 1.006695032119751, + "learning_rate": 9.439586739574513e-05, + "loss": 0.11421098709106445, + "step": 240 + }, + { + "epoch": 0.8296769046669326, + "grad_norm": 0.5092183947563171, + "learning_rate": 9.344062000033414e-05, + "loss": 0.10922021865844726, + "step": 260 + }, + { + "epoch": 0.8934982050259274, + "grad_norm": 0.8316523432731628, + "learning_rate": 9.241588644329919e-05, + "loss": 0.10600426197052001, + "step": 280 + }, + { + "epoch": 0.9573195053849222, + "grad_norm": 1.5138731002807617, + "learning_rate": 9.132330774294252e-05, + "loss": 0.1008190631866455, + "step": 300 + }, + { + "epoch": 1.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 87.8505, + "eval_generator_rouge1": 99.5556, + "eval_generator_rouge2": 99.2105, + "eval_generator_rougeL": 99.1217, + "eval_loss": 0.011629586108028889, + "eval_runtime": 615.4826, + "eval_samples_per_second": 5.547, + "eval_selector_accuracy": 93.3995, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.174, + "step": 314 + } + ], + "logging_steps": 20, + "max_steps": 1570, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.172842003562496e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/qwen3/fdd_sfkl/checkpoint-314/training_args.bin b/qwen3/fdd_sfkl/checkpoint-314/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..4aaaca662465ff2e3933154ec90d4097299421a9 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-314/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18f98e7767ecd74653cd41dec83bbbe276858948c803da4b684e0aaf5a1cc1f7 +size 7185 diff --git a/qwen3/fdd_sfkl/checkpoint-314/zero_to_fp32.py b/qwen3/fdd_sfkl/checkpoint-314/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-314/zero_to_fp32.py @@ -0,0 +1,790 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY, + AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY, + AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def _has_autoep_zero3_partitioned_metadata(state_dict): + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY) + if autoep_layers is None: + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY) + if not isinstance(autoep_layers, list): + return False + return any( + isinstance(entry, dict) + and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT + for entry in autoep_layers) + + +def _raise_if_autoep_zero3_partitioned_state(state_dict): + if _has_autoep_zero3_partitioned_metadata(state_dict): + raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. " + "AutoEP expert parameters are partitioned over expert replica groups, so " + "global data-parallel consolidation would produce incomplete expert tensors. " + "Use ds_to_universal.py for expert-aware conversion.") + + +def _raise_if_autoep_zero3_partitioned_checkpoint(model_files): + for file in model_files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints + # before the expensive optimizer-shard load below. + model_files = get_model_state_files(ds_checkpoint_dir) + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/qwen3/fdd_sfkl/checkpoint-628/README.md b/qwen3/fdd_sfkl/checkpoint-628/README.md new file mode 100644 index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-628/README.md @@ -0,0 +1,208 @@ +--- +base_model: Qwen/Qwen3-0.6B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-0.6B +- llama-factory +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-628/adapter_config.json b/qwen3/fdd_sfkl/checkpoint-628/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..09417a2e76df9aef95a5236d9a1aed2554e49cf2 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-628/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-0.6B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "q_proj", + "o_proj", + "down_proj", + "gate_proj", + "k_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-628/adapter_model.safetensors b/qwen3/fdd_sfkl/checkpoint-628/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8c0af8f8113d1314d908cd5024ea9e63dfc2b42e --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-628/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:653c807e074697e3518a96a87819d5e7977543b86f0a2ba9aa2bfd1dba9a4878 +size 40422560 diff --git a/qwen3/fdd_sfkl/checkpoint-628/chat_template.jinja b/qwen3/fdd_sfkl/checkpoint-628/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-628/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/fdd_sfkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec09b47219cc207b6f48085f18d53290dc901d59 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-628/global_step628/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d31c5ff79276a0869e60fc434b74f4e562850fc390aa2c73de9334a0026417f8 +size 121135557 diff --git a/qwen3/fdd_sfkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/fdd_sfkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e90cd100385d1d8dde303a34b0dc97e92ec5a4b6 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-628/global_step628/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:315d07add3cf055b98e38f9a04ad40f04e4ba5913e56c22877437a3e80e212fb +size 121135749 diff --git a/qwen3/fdd_sfkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt b/qwen3/fdd_sfkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..959553f95382a87832713609482b4be87001ff1b --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-628/global_step628/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdb5e187997d45f3c97677141a8a6c0427a27ebd8ca65ff03b369b93c1e562bc +size 351841765 diff --git a/qwen3/fdd_sfkl/checkpoint-628/latest b/qwen3/fdd_sfkl/checkpoint-628/latest new file mode 100644 index 0000000000000000000000000000000000000000..af055c3db05dcb2af541005f22308dbd946aba65 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-628/latest @@ -0,0 +1 @@ +global_step628 \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-628/rng_state_0.pth b/qwen3/fdd_sfkl/checkpoint-628/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..812f8ed18bbc16863c6934c556e92da3ca03c639 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-628/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c426c2eedb428548a4b66cb4e9cf850eefbfc552b086823e484a5f2d13f287dd +size 14917 diff --git a/qwen3/fdd_sfkl/checkpoint-628/rng_state_1.pth b/qwen3/fdd_sfkl/checkpoint-628/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..0a29f40a8e4d15aca83ea14451e3b6c9aadc10e1 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-628/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:601bafc91f356efde154ebb89c382e38cca26fbc4782772887bc12b3062afccd +size 14917 diff --git a/qwen3/fdd_sfkl/checkpoint-628/scheduler.pt b/qwen3/fdd_sfkl/checkpoint-628/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..96c0cabcd99be4945ffbd7897ecc5367ac4beb28 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-628/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb823a2c51d7fb42dfe3b693a7159784285dbf38c5a797e6e73292096123ab7a +size 1465 diff --git a/qwen3/fdd_sfkl/checkpoint-628/tokenizer.json b/qwen3/fdd_sfkl/checkpoint-628/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-628/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/qwen3/fdd_sfkl/checkpoint-628/tokenizer_config.json b/qwen3/fdd_sfkl/checkpoint-628/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-628/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen3/fdd_sfkl/checkpoint-628/trainer_state.json b/qwen3/fdd_sfkl/checkpoint-628/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..2ffc5276fa879dbea3a0ef9e9a99034f852ea066 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-628/trainer_state.json @@ -0,0 +1,281 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 628, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.06382130035899482, + "grad_norm": 2.7527661323547363, + "learning_rate": 9.996390385474569e-05, + "loss": 1.4406415939331054, + "step": 20 + }, + { + "epoch": 0.12764260071798963, + "grad_norm": 2.376922607421875, + "learning_rate": 9.984797509888031e-05, + "loss": 0.4754016876220703, + "step": 40 + }, + { + "epoch": 0.19146390107698444, + "grad_norm": 2.429856300354004, + "learning_rate": 9.965229938050925e-05, + "loss": 0.37995381355285646, + "step": 60 + }, + { + "epoch": 0.25528520143597927, + "grad_norm": 1.2270593643188477, + "learning_rate": 9.937719005663432e-05, + "loss": 0.25868964195251465, + "step": 80 + }, + { + "epoch": 0.3191065017949741, + "grad_norm": 0.913340151309967, + "learning_rate": 9.902308769000724e-05, + "loss": 0.20970418453216552, + "step": 100 + }, + { + "epoch": 0.3829278021539689, + "grad_norm": 1.929979920387268, + "learning_rate": 9.859055934360789e-05, + "loss": 0.17272943258285522, + "step": 120 + }, + { + "epoch": 0.4467491025129637, + "grad_norm": 1.0000314712524414, + "learning_rate": 9.808029767254437e-05, + "loss": 0.16882457733154296, + "step": 140 + }, + { + "epoch": 0.5105704028719585, + "grad_norm": 1.1901942491531372, + "learning_rate": 9.749311981482817e-05, + "loss": 0.13708232641220092, + "step": 160 + }, + { + "epoch": 0.5743917032309533, + "grad_norm": 0.9724851250648499, + "learning_rate": 9.682996608280154e-05, + "loss": 0.12325812578201294, + "step": 180 + }, + { + "epoch": 0.6382130035899481, + "grad_norm": 1.5030133724212646, + "learning_rate": 9.609189845731207e-05, + "loss": 0.1345768690109253, + "step": 200 + }, + { + "epoch": 0.702034303948943, + "grad_norm": 0.5464316010475159, + "learning_rate": 9.52800988870462e-05, + "loss": 0.11802985668182372, + "step": 220 + }, + { + "epoch": 0.7658556043079378, + "grad_norm": 1.006695032119751, + "learning_rate": 9.439586739574513e-05, + "loss": 0.11421098709106445, + "step": 240 + }, + { + "epoch": 0.8296769046669326, + "grad_norm": 0.5092183947563171, + "learning_rate": 9.344062000033414e-05, + "loss": 0.10922021865844726, + "step": 260 + }, + { + "epoch": 0.8934982050259274, + "grad_norm": 0.8316523432731628, + "learning_rate": 9.241588644329919e-05, + "loss": 0.10600426197052001, + "step": 280 + }, + { + "epoch": 0.9573195053849222, + "grad_norm": 1.5138731002807617, + "learning_rate": 9.132330774294252e-05, + "loss": 0.1008190631866455, + "step": 300 + }, + { + "epoch": 1.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 87.8505, + "eval_generator_rouge1": 99.5556, + "eval_generator_rouge2": 99.2105, + "eval_generator_rougeL": 99.1217, + "eval_loss": 0.011629586108028889, + "eval_runtime": 615.4826, + "eval_samples_per_second": 5.547, + "eval_selector_accuracy": 93.3995, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.174, + "step": 314 + }, + { + "epoch": 1.0191463901076985, + "grad_norm": 0.41843533515930176, + "learning_rate": 9.01646335654399e-05, + "loss": 0.09401558637619019, + "step": 320 + }, + { + "epoch": 1.0829676904666932, + "grad_norm": 0.47839081287384033, + "learning_rate": 8.894171942290817e-05, + "loss": 0.0862088918685913, + "step": 340 + }, + { + "epoch": 1.146788990825688, + "grad_norm": 0.349435031414032, + "learning_rate": 8.765652370197003e-05, + "loss": 0.08293926119804382, + "step": 360 + }, + { + "epoch": 1.210610291184683, + "grad_norm": 0.5593855977058411, + "learning_rate": 8.631110452757467e-05, + "loss": 0.0761639952659607, + "step": 380 + }, + { + "epoch": 1.2744315915436777, + "grad_norm": 0.4683215320110321, + "learning_rate": 8.490761646709642e-05, + "loss": 0.0819424033164978, + "step": 400 + }, + { + "epoch": 1.3382528919026724, + "grad_norm": 0.3691541850566864, + "learning_rate": 8.344830707998953e-05, + "loss": 0.08254330158233643, + "step": 420 + }, + { + "epoch": 1.4020741922616673, + "grad_norm": 0.3268161416053772, + "learning_rate": 8.193551331852458e-05, + "loss": 0.07523589730262756, + "step": 440 + }, + { + "epoch": 1.4658954926206622, + "grad_norm": 0.9531517624855042, + "learning_rate": 8.037165778537033e-05, + "loss": 0.07261439561843872, + "step": 460 + }, + { + "epoch": 1.5297167929796571, + "grad_norm": 0.33297064900398254, + "learning_rate": 7.875924485401427e-05, + "loss": 0.07137778997421265, + "step": 480 + }, + { + "epoch": 1.5935380933386518, + "grad_norm": 0.5712356567382812, + "learning_rate": 7.710085665823444e-05, + "loss": 0.07516297101974487, + "step": 500 + }, + { + "epoch": 1.6573593936976465, + "grad_norm": 0.27220338582992554, + "learning_rate": 7.539914895704521e-05, + "loss": 0.0703214168548584, + "step": 520 + }, + { + "epoch": 1.7211806940566414, + "grad_norm": 1.4434664249420166, + "learning_rate": 7.365684688173908e-05, + "loss": 0.08012890815734863, + "step": 540 + }, + { + "epoch": 1.7850019944156363, + "grad_norm": 0.32184913754463196, + "learning_rate": 7.18767405718347e-05, + "loss": 0.06634219884872436, + "step": 560 + }, + { + "epoch": 1.848823294774631, + "grad_norm": 1.6946690082550049, + "learning_rate": 7.006168070692036e-05, + "loss": 0.07123274803161621, + "step": 580 + }, + { + "epoch": 1.9126445951336257, + "grad_norm": 1.2815985679626465, + "learning_rate": 6.821457394154786e-05, + "loss": 0.06565064191818237, + "step": 600 + }, + { + "epoch": 1.9764658954926206, + "grad_norm": 0.5232482552528381, + "learning_rate": 6.633837825048768e-05, + "loss": 0.06681714057922364, + "step": 620 + }, + { + "epoch": 2.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 92.6402, + "eval_generator_rouge1": 99.741, + "eval_generator_rouge2": 99.5504, + "eval_generator_rougeL": 99.4854, + "eval_loss": 0.007036722265183926, + "eval_runtime": 630.2117, + "eval_samples_per_second": 5.417, + "eval_selector_accuracy": 98.0724, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.17, + "step": 628 + } + ], + "logging_steps": 20, + "max_steps": 1570, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.346351770520781e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/qwen3/fdd_sfkl/checkpoint-628/training_args.bin b/qwen3/fdd_sfkl/checkpoint-628/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..4aaaca662465ff2e3933154ec90d4097299421a9 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-628/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18f98e7767ecd74653cd41dec83bbbe276858948c803da4b684e0aaf5a1cc1f7 +size 7185 diff --git a/qwen3/fdd_sfkl/checkpoint-628/zero_to_fp32.py b/qwen3/fdd_sfkl/checkpoint-628/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-628/zero_to_fp32.py @@ -0,0 +1,790 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY, + AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY, + AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def _has_autoep_zero3_partitioned_metadata(state_dict): + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY) + if autoep_layers is None: + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY) + if not isinstance(autoep_layers, list): + return False + return any( + isinstance(entry, dict) + and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT + for entry in autoep_layers) + + +def _raise_if_autoep_zero3_partitioned_state(state_dict): + if _has_autoep_zero3_partitioned_metadata(state_dict): + raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. " + "AutoEP expert parameters are partitioned over expert replica groups, so " + "global data-parallel consolidation would produce incomplete expert tensors. " + "Use ds_to_universal.py for expert-aware conversion.") + + +def _raise_if_autoep_zero3_partitioned_checkpoint(model_files): + for file in model_files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints + # before the expensive optimizer-shard load below. + model_files = get_model_state_files(ds_checkpoint_dir) + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/qwen3/fdd_sfkl/checkpoint-942/README.md b/qwen3/fdd_sfkl/checkpoint-942/README.md new file mode 100644 index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-942/README.md @@ -0,0 +1,208 @@ +--- +base_model: Qwen/Qwen3-0.6B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-0.6B +- llama-factory +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-942/adapter_config.json b/qwen3/fdd_sfkl/checkpoint-942/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..09417a2e76df9aef95a5236d9a1aed2554e49cf2 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-942/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-0.6B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "q_proj", + "o_proj", + "down_proj", + "gate_proj", + "k_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-942/adapter_model.safetensors b/qwen3/fdd_sfkl/checkpoint-942/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1c1671a912ec4f0175b86b30dbd1dc6c4773ad70 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-942/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:781ca85d5088171a17676e92da57c430c2a7b5d2569fa6666ef544b8a9b94868 +size 40422560 diff --git a/qwen3/fdd_sfkl/checkpoint-942/chat_template.jinja b/qwen3/fdd_sfkl/checkpoint-942/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-942/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/fdd_sfkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..019a47612533ecfb381d2b437c5a98eb244a5f25 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-942/global_step942/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38ac58e977bf637d3116f6477d90c80ebcf9ae2b505a0a9ccec14050bdbb4832 +size 121135557 diff --git a/qwen3/fdd_sfkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/fdd_sfkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d63d0be0bb9d940707ecde7663e48563d449647 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-942/global_step942/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b0a0d8fa439046e693114caa854facaab4af696e81d04612b09920a258c2ed4 +size 121135749 diff --git a/qwen3/fdd_sfkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt b/qwen3/fdd_sfkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..86286310c1c7ed10ee37f14beaf06d6fe52aacb4 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-942/global_step942/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9807174ff9f42db965d678e08426caf70641769d6a0f1983ecabc31951aa8785 +size 351841765 diff --git a/qwen3/fdd_sfkl/checkpoint-942/latest b/qwen3/fdd_sfkl/checkpoint-942/latest new file mode 100644 index 0000000000000000000000000000000000000000..4e9a68e4460aca593c480611a486d2a4cd73ef42 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-942/latest @@ -0,0 +1 @@ +global_step942 \ No newline at end of file diff --git a/qwen3/fdd_sfkl/checkpoint-942/rng_state_0.pth b/qwen3/fdd_sfkl/checkpoint-942/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..86874024860284bddbf1b9f839a8d5b7cec79095 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-942/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0cd5aa62e8f3925cc578618b73373efa5fe64cefd1d609b22115e719dae5a0d +size 14917 diff --git a/qwen3/fdd_sfkl/checkpoint-942/rng_state_1.pth b/qwen3/fdd_sfkl/checkpoint-942/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..e22e58b953d0bbb12760f4544fd3108ad7f9ca2e --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-942/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c861ed598789e27cb4bdb019c6adba3b9370dfbd788510377a7feec88e5d1be7 +size 14917 diff --git a/qwen3/fdd_sfkl/checkpoint-942/scheduler.pt b/qwen3/fdd_sfkl/checkpoint-942/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b1f8af0480bf83ab728abfeff11bc3288214349 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-942/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8076da6f897951c5bdf8649e4fba596e8c49819927746017442ffd85f40b18a3 +size 1465 diff --git a/qwen3/fdd_sfkl/checkpoint-942/tokenizer.json b/qwen3/fdd_sfkl/checkpoint-942/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-942/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/qwen3/fdd_sfkl/checkpoint-942/tokenizer_config.json b/qwen3/fdd_sfkl/checkpoint-942/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-942/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen3/fdd_sfkl/checkpoint-942/trainer_state.json b/qwen3/fdd_sfkl/checkpoint-942/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9e67dfb9307928d19c0edfdf9566107ab0b3259a --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-942/trainer_state.json @@ -0,0 +1,408 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 942, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.06382130035899482, + "grad_norm": 2.7527661323547363, + "learning_rate": 9.996390385474569e-05, + "loss": 1.4406415939331054, + "step": 20 + }, + { + "epoch": 0.12764260071798963, + "grad_norm": 2.376922607421875, + "learning_rate": 9.984797509888031e-05, + "loss": 0.4754016876220703, + "step": 40 + }, + { + "epoch": 0.19146390107698444, + "grad_norm": 2.429856300354004, + "learning_rate": 9.965229938050925e-05, + "loss": 0.37995381355285646, + "step": 60 + }, + { + "epoch": 0.25528520143597927, + "grad_norm": 1.2270593643188477, + "learning_rate": 9.937719005663432e-05, + "loss": 0.25868964195251465, + "step": 80 + }, + { + "epoch": 0.3191065017949741, + "grad_norm": 0.913340151309967, + "learning_rate": 9.902308769000724e-05, + "loss": 0.20970418453216552, + "step": 100 + }, + { + "epoch": 0.3829278021539689, + "grad_norm": 1.929979920387268, + "learning_rate": 9.859055934360789e-05, + "loss": 0.17272943258285522, + "step": 120 + }, + { + "epoch": 0.4467491025129637, + "grad_norm": 1.0000314712524414, + "learning_rate": 9.808029767254437e-05, + "loss": 0.16882457733154296, + "step": 140 + }, + { + "epoch": 0.5105704028719585, + "grad_norm": 1.1901942491531372, + "learning_rate": 9.749311981482817e-05, + "loss": 0.13708232641220092, + "step": 160 + }, + { + "epoch": 0.5743917032309533, + "grad_norm": 0.9724851250648499, + "learning_rate": 9.682996608280154e-05, + "loss": 0.12325812578201294, + "step": 180 + }, + { + "epoch": 0.6382130035899481, + "grad_norm": 1.5030133724212646, + "learning_rate": 9.609189845731207e-05, + "loss": 0.1345768690109253, + "step": 200 + }, + { + "epoch": 0.702034303948943, + "grad_norm": 0.5464316010475159, + "learning_rate": 9.52800988870462e-05, + "loss": 0.11802985668182372, + "step": 220 + }, + { + "epoch": 0.7658556043079378, + "grad_norm": 1.006695032119751, + "learning_rate": 9.439586739574513e-05, + "loss": 0.11421098709106445, + "step": 240 + }, + { + "epoch": 0.8296769046669326, + "grad_norm": 0.5092183947563171, + "learning_rate": 9.344062000033414e-05, + "loss": 0.10922021865844726, + "step": 260 + }, + { + "epoch": 0.8934982050259274, + "grad_norm": 0.8316523432731628, + "learning_rate": 9.241588644329919e-05, + "loss": 0.10600426197052001, + "step": 280 + }, + { + "epoch": 0.9573195053849222, + "grad_norm": 1.5138731002807617, + "learning_rate": 9.132330774294252e-05, + "loss": 0.1008190631866455, + "step": 300 + }, + { + "epoch": 1.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 87.8505, + "eval_generator_rouge1": 99.5556, + "eval_generator_rouge2": 99.2105, + "eval_generator_rougeL": 99.1217, + "eval_loss": 0.011629586108028889, + "eval_runtime": 615.4826, + "eval_samples_per_second": 5.547, + "eval_selector_accuracy": 93.3995, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.174, + "step": 314 + }, + { + "epoch": 1.0191463901076985, + "grad_norm": 0.41843533515930176, + "learning_rate": 9.01646335654399e-05, + "loss": 0.09401558637619019, + "step": 320 + }, + { + "epoch": 1.0829676904666932, + "grad_norm": 0.47839081287384033, + "learning_rate": 8.894171942290817e-05, + "loss": 0.0862088918685913, + "step": 340 + }, + { + "epoch": 1.146788990825688, + "grad_norm": 0.349435031414032, + "learning_rate": 8.765652370197003e-05, + "loss": 0.08293926119804382, + "step": 360 + }, + { + "epoch": 1.210610291184683, + "grad_norm": 0.5593855977058411, + "learning_rate": 8.631110452757467e-05, + "loss": 0.0761639952659607, + "step": 380 + }, + { + "epoch": 1.2744315915436777, + "grad_norm": 0.4683215320110321, + "learning_rate": 8.490761646709642e-05, + "loss": 0.0819424033164978, + "step": 400 + }, + { + "epoch": 1.3382528919026724, + "grad_norm": 0.3691541850566864, + "learning_rate": 8.344830707998953e-05, + "loss": 0.08254330158233643, + "step": 420 + }, + { + "epoch": 1.4020741922616673, + "grad_norm": 0.3268161416053772, + "learning_rate": 8.193551331852458e-05, + "loss": 0.07523589730262756, + "step": 440 + }, + { + "epoch": 1.4658954926206622, + "grad_norm": 0.9531517624855042, + "learning_rate": 8.037165778537033e-05, + "loss": 0.07261439561843872, + "step": 460 + }, + { + "epoch": 1.5297167929796571, + "grad_norm": 0.33297064900398254, + "learning_rate": 7.875924485401427e-05, + "loss": 0.07137778997421265, + "step": 480 + }, + { + "epoch": 1.5935380933386518, + "grad_norm": 0.5712356567382812, + "learning_rate": 7.710085665823444e-05, + "loss": 0.07516297101974487, + "step": 500 + }, + { + "epoch": 1.6573593936976465, + "grad_norm": 0.27220338582992554, + "learning_rate": 7.539914895704521e-05, + "loss": 0.0703214168548584, + "step": 520 + }, + { + "epoch": 1.7211806940566414, + "grad_norm": 1.4434664249420166, + "learning_rate": 7.365684688173908e-05, + "loss": 0.08012890815734863, + "step": 540 + }, + { + "epoch": 1.7850019944156363, + "grad_norm": 0.32184913754463196, + "learning_rate": 7.18767405718347e-05, + "loss": 0.06634219884872436, + "step": 560 + }, + { + "epoch": 1.848823294774631, + "grad_norm": 1.6946690082550049, + "learning_rate": 7.006168070692036e-05, + "loss": 0.07123274803161621, + "step": 580 + }, + { + "epoch": 1.9126445951336257, + "grad_norm": 1.2815985679626465, + "learning_rate": 6.821457394154786e-05, + "loss": 0.06565064191818237, + "step": 600 + }, + { + "epoch": 1.9764658954926206, + "grad_norm": 0.5232482552528381, + "learning_rate": 6.633837825048768e-05, + "loss": 0.06681714057922364, + "step": 620 + }, + { + "epoch": 2.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 92.6402, + "eval_generator_rouge1": 99.741, + "eval_generator_rouge2": 99.5504, + "eval_generator_rougeL": 99.4854, + "eval_loss": 0.007036722265183926, + "eval_runtime": 630.2117, + "eval_samples_per_second": 5.417, + "eval_selector_accuracy": 98.0724, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.17, + "step": 628 + }, + { + "epoch": 2.038292780215397, + "grad_norm": 0.2939830422401428, + "learning_rate": 6.443609819179926e-05, + "loss": 0.06403520107269287, + "step": 640 + }, + { + "epoch": 2.102114080574392, + "grad_norm": 0.32704657316207886, + "learning_rate": 6.251078009530256e-05, + "loss": 0.060550224781036374, + "step": 660 + }, + { + "epoch": 2.1659353809333863, + "grad_norm": 0.42840492725372314, + "learning_rate": 6.0565507184155677e-05, + "loss": 0.060464733839035036, + "step": 680 + }, + { + "epoch": 2.2297566812923812, + "grad_norm": 0.6081668734550476, + "learning_rate": 5.860339463735136e-05, + "loss": 0.05875073671340943, + "step": 700 + }, + { + "epoch": 2.293577981651376, + "grad_norm": 0.28151988983154297, + "learning_rate": 5.662758460103895e-05, + "loss": 0.05764824151992798, + "step": 720 + }, + { + "epoch": 2.357399282010371, + "grad_norm": 0.2843102514743805, + "learning_rate": 5.464124115666095e-05, + "loss": 0.05847916603088379, + "step": 740 + }, + { + "epoch": 2.421220582369366, + "grad_norm": 0.34520450234413147, + "learning_rate": 5.264754525396212e-05, + "loss": 0.056367015838623045, + "step": 760 + }, + { + "epoch": 2.4850418827283605, + "grad_norm": 0.45693477988243103, + "learning_rate": 5.06496896169857e-05, + "loss": 0.055580759048461915, + "step": 780 + }, + { + "epoch": 2.5488631830873554, + "grad_norm": 0.3360002338886261, + "learning_rate": 4.865087363121384e-05, + "loss": 0.061715197563171384, + "step": 800 + }, + { + "epoch": 2.6126844834463503, + "grad_norm": 0.5039923787117004, + "learning_rate": 4.6654298220040673e-05, + "loss": 0.05754677057266235, + "step": 820 + }, + { + "epoch": 2.6765057838053448, + "grad_norm": 0.29794442653656006, + "learning_rate": 4.466316071878229e-05, + "loss": 0.055103600025177, + "step": 840 + }, + { + "epoch": 2.74032708416434, + "grad_norm": 0.21690921485424042, + "learning_rate": 4.2680649754432913e-05, + "loss": 0.054493296146392825, + "step": 860 + }, + { + "epoch": 2.8041483845233346, + "grad_norm": 0.28939780592918396, + "learning_rate": 4.070994013936646e-05, + "loss": 0.05614350438117981, + "step": 880 + }, + { + "epoch": 2.8679696848823295, + "grad_norm": 0.5370737910270691, + "learning_rate": 3.8754187787161155e-05, + "loss": 0.05749887228012085, + "step": 900 + }, + { + "epoch": 2.9317909852413244, + "grad_norm": 0.21074210107326508, + "learning_rate": 3.681652465868875e-05, + "loss": 0.05852361917495728, + "step": 920 + }, + { + "epoch": 2.995612285600319, + "grad_norm": 0.4107605516910553, + "learning_rate": 3.490005374656184e-05, + "loss": 0.05852218866348267, + "step": 940 + }, + { + "epoch": 3.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 94.6846, + "eval_generator_rouge1": 99.8316, + "eval_generator_rouge2": 99.7174, + "eval_generator_rougeL": 99.6293, + "eval_loss": 0.005526242312043905, + "eval_runtime": 613.3531, + "eval_samples_per_second": 5.566, + "eval_selector_accuracy": 96.9042, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.174, + "step": 942 + } + ], + "logging_steps": 20, + "max_steps": 1570, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.524157743353037e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/qwen3/fdd_sfkl/checkpoint-942/training_args.bin b/qwen3/fdd_sfkl/checkpoint-942/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..4aaaca662465ff2e3933154ec90d4097299421a9 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-942/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18f98e7767ecd74653cd41dec83bbbe276858948c803da4b684e0aaf5a1cc1f7 +size 7185 diff --git a/qwen3/fdd_sfkl/checkpoint-942/zero_to_fp32.py b/qwen3/fdd_sfkl/checkpoint-942/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138 --- /dev/null +++ b/qwen3/fdd_sfkl/checkpoint-942/zero_to_fp32.py @@ -0,0 +1,790 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY, + AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY, + AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def _has_autoep_zero3_partitioned_metadata(state_dict): + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY) + if autoep_layers is None: + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY) + if not isinstance(autoep_layers, list): + return False + return any( + isinstance(entry, dict) + and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT + for entry in autoep_layers) + + +def _raise_if_autoep_zero3_partitioned_state(state_dict): + if _has_autoep_zero3_partitioned_metadata(state_dict): + raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. " + "AutoEP expert parameters are partitioned over expert replica groups, so " + "global data-parallel consolidation would produce incomplete expert tensors. " + "Use ds_to_universal.py for expert-aware conversion.") + + +def _raise_if_autoep_zero3_partitioned_checkpoint(model_files): + for file in model_files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints + # before the expensive optimizer-shard load below. + model_files = get_model_state_files(ds_checkpoint_dir) + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/qwen3/fdd_sfkl/eval_results.json b/qwen3/fdd_sfkl/eval_results.json new file mode 100644 index 0000000000000000000000000000000000000000..ff00f6278cfac7a9a13a0aa8826e1e637f819952 --- /dev/null +++ b/qwen3/fdd_sfkl/eval_results.json @@ -0,0 +1,14 @@ +{ + "epoch": 5.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 96.4369, + "eval_generator_rouge1": 99.9258, + "eval_generator_rouge2": 99.8607, + "eval_generator_rougeL": 99.7595, + "eval_loss": 0.0038434057496488094, + "eval_runtime": 599.4736, + "eval_samples_per_second": 5.695, + "eval_selector_accuracy": 98.7734, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.178 +} \ No newline at end of file diff --git a/qwen3/fdd_sfkl/tokenizer.json b/qwen3/fdd_sfkl/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/qwen3/fdd_sfkl/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/qwen3/fdd_sfkl/tokenizer_config.json b/qwen3/fdd_sfkl/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c --- /dev/null +++ b/qwen3/fdd_sfkl/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen3/fdd_sfkl/train_results.json b/qwen3/fdd_sfkl/train_results.json new file mode 100644 index 0000000000000000000000000000000000000000..63705852ed8df728f0bbc323b4eb613ab8f92e64 --- /dev/null +++ b/qwen3/fdd_sfkl/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 5.0, + "total_flos": 5.871444840310374e+16, + "train_loss": 0.09919491964540664, + "train_runtime": 8002.0141, + "train_samples_per_second": 6.265, + "train_steps_per_second": 0.196 +} \ No newline at end of file diff --git a/qwen3/fdd_sfkl/trainer_state.json b/qwen3/fdd_sfkl/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..23cc7cb24b9829e7c5e5b8ffca16d6d58f005436 --- /dev/null +++ b/qwen3/fdd_sfkl/trainer_state.json @@ -0,0 +1,664 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 1570, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.06382130035899482, + "grad_norm": 2.7527661323547363, + "learning_rate": 9.996390385474569e-05, + "loss": 1.4406415939331054, + "step": 20 + }, + { + "epoch": 0.12764260071798963, + "grad_norm": 2.376922607421875, + "learning_rate": 9.984797509888031e-05, + "loss": 0.4754016876220703, + "step": 40 + }, + { + "epoch": 0.19146390107698444, + "grad_norm": 2.429856300354004, + "learning_rate": 9.965229938050925e-05, + "loss": 0.37995381355285646, + "step": 60 + }, + { + "epoch": 0.25528520143597927, + "grad_norm": 1.2270593643188477, + "learning_rate": 9.937719005663432e-05, + "loss": 0.25868964195251465, + "step": 80 + }, + { + "epoch": 0.3191065017949741, + "grad_norm": 0.913340151309967, + "learning_rate": 9.902308769000724e-05, + "loss": 0.20970418453216552, + "step": 100 + }, + { + "epoch": 0.3829278021539689, + "grad_norm": 1.929979920387268, + "learning_rate": 9.859055934360789e-05, + "loss": 0.17272943258285522, + "step": 120 + }, + { + "epoch": 0.4467491025129637, + "grad_norm": 1.0000314712524414, + "learning_rate": 9.808029767254437e-05, + "loss": 0.16882457733154296, + "step": 140 + }, + { + "epoch": 0.5105704028719585, + "grad_norm": 1.1901942491531372, + "learning_rate": 9.749311981482817e-05, + "loss": 0.13708232641220092, + "step": 160 + }, + { + "epoch": 0.5743917032309533, + "grad_norm": 0.9724851250648499, + "learning_rate": 9.682996608280154e-05, + "loss": 0.12325812578201294, + "step": 180 + }, + { + "epoch": 0.6382130035899481, + "grad_norm": 1.5030133724212646, + "learning_rate": 9.609189845731207e-05, + "loss": 0.1345768690109253, + "step": 200 + }, + { + "epoch": 0.702034303948943, + "grad_norm": 0.5464316010475159, + "learning_rate": 9.52800988870462e-05, + "loss": 0.11802985668182372, + "step": 220 + }, + { + "epoch": 0.7658556043079378, + "grad_norm": 1.006695032119751, + "learning_rate": 9.439586739574513e-05, + "loss": 0.11421098709106445, + "step": 240 + }, + { + "epoch": 0.8296769046669326, + "grad_norm": 0.5092183947563171, + "learning_rate": 9.344062000033414e-05, + "loss": 0.10922021865844726, + "step": 260 + }, + { + "epoch": 0.8934982050259274, + "grad_norm": 0.8316523432731628, + "learning_rate": 9.241588644329919e-05, + "loss": 0.10600426197052001, + "step": 280 + }, + { + "epoch": 0.9573195053849222, + "grad_norm": 1.5138731002807617, + "learning_rate": 9.132330774294252e-05, + "loss": 0.1008190631866455, + "step": 300 + }, + { + "epoch": 1.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 87.8505, + "eval_generator_rouge1": 99.5556, + "eval_generator_rouge2": 99.2105, + "eval_generator_rougeL": 99.1217, + "eval_loss": 0.011629586108028889, + "eval_runtime": 615.4826, + "eval_samples_per_second": 5.547, + "eval_selector_accuracy": 93.3995, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.174, + "step": 314 + }, + { + "epoch": 1.0191463901076985, + "grad_norm": 0.41843533515930176, + "learning_rate": 9.01646335654399e-05, + "loss": 0.09401558637619019, + "step": 320 + }, + { + "epoch": 1.0829676904666932, + "grad_norm": 0.47839081287384033, + "learning_rate": 8.894171942290817e-05, + "loss": 0.0862088918685913, + "step": 340 + }, + { + "epoch": 1.146788990825688, + "grad_norm": 0.349435031414032, + "learning_rate": 8.765652370197003e-05, + "loss": 0.08293926119804382, + "step": 360 + }, + { + "epoch": 1.210610291184683, + "grad_norm": 0.5593855977058411, + "learning_rate": 8.631110452757467e-05, + "loss": 0.0761639952659607, + "step": 380 + }, + { + "epoch": 1.2744315915436777, + "grad_norm": 0.4683215320110321, + "learning_rate": 8.490761646709642e-05, + "loss": 0.0819424033164978, + "step": 400 + }, + { + "epoch": 1.3382528919026724, + "grad_norm": 0.3691541850566864, + "learning_rate": 8.344830707998953e-05, + "loss": 0.08254330158233643, + "step": 420 + }, + { + "epoch": 1.4020741922616673, + "grad_norm": 0.3268161416053772, + "learning_rate": 8.193551331852458e-05, + "loss": 0.07523589730262756, + "step": 440 + }, + { + "epoch": 1.4658954926206622, + "grad_norm": 0.9531517624855042, + "learning_rate": 8.037165778537033e-05, + "loss": 0.07261439561843872, + "step": 460 + }, + { + "epoch": 1.5297167929796571, + "grad_norm": 0.33297064900398254, + "learning_rate": 7.875924485401427e-05, + "loss": 0.07137778997421265, + "step": 480 + }, + { + "epoch": 1.5935380933386518, + "grad_norm": 0.5712356567382812, + "learning_rate": 7.710085665823444e-05, + "loss": 0.07516297101974487, + "step": 500 + }, + { + "epoch": 1.6573593936976465, + "grad_norm": 0.27220338582992554, + "learning_rate": 7.539914895704521e-05, + "loss": 0.0703214168548584, + "step": 520 + }, + { + "epoch": 1.7211806940566414, + "grad_norm": 1.4434664249420166, + "learning_rate": 7.365684688173908e-05, + "loss": 0.08012890815734863, + "step": 540 + }, + { + "epoch": 1.7850019944156363, + "grad_norm": 0.32184913754463196, + "learning_rate": 7.18767405718347e-05, + "loss": 0.06634219884872436, + "step": 560 + }, + { + "epoch": 1.848823294774631, + "grad_norm": 1.6946690082550049, + "learning_rate": 7.006168070692036e-05, + "loss": 0.07123274803161621, + "step": 580 + }, + { + "epoch": 1.9126445951336257, + "grad_norm": 1.2815985679626465, + "learning_rate": 6.821457394154786e-05, + "loss": 0.06565064191818237, + "step": 600 + }, + { + "epoch": 1.9764658954926206, + "grad_norm": 0.5232482552528381, + "learning_rate": 6.633837825048768e-05, + "loss": 0.06681714057922364, + "step": 620 + }, + { + "epoch": 2.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 92.6402, + "eval_generator_rouge1": 99.741, + "eval_generator_rouge2": 99.5504, + "eval_generator_rougeL": 99.4854, + "eval_loss": 0.007036722265183926, + "eval_runtime": 630.2117, + "eval_samples_per_second": 5.417, + "eval_selector_accuracy": 98.0724, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.17, + "step": 628 + }, + { + "epoch": 2.038292780215397, + "grad_norm": 0.2939830422401428, + "learning_rate": 6.443609819179926e-05, + "loss": 0.06403520107269287, + "step": 640 + }, + { + "epoch": 2.102114080574392, + "grad_norm": 0.32704657316207886, + "learning_rate": 6.251078009530256e-05, + "loss": 0.060550224781036374, + "step": 660 + }, + { + "epoch": 2.1659353809333863, + "grad_norm": 0.42840492725372314, + "learning_rate": 6.0565507184155677e-05, + "loss": 0.060464733839035036, + "step": 680 + }, + { + "epoch": 2.2297566812923812, + "grad_norm": 0.6081668734550476, + "learning_rate": 5.860339463735136e-05, + "loss": 0.05875073671340943, + "step": 700 + }, + { + "epoch": 2.293577981651376, + "grad_norm": 0.28151988983154297, + "learning_rate": 5.662758460103895e-05, + "loss": 0.05764824151992798, + "step": 720 + }, + { + "epoch": 2.357399282010371, + "grad_norm": 0.2843102514743805, + "learning_rate": 5.464124115666095e-05, + "loss": 0.05847916603088379, + "step": 740 + }, + { + "epoch": 2.421220582369366, + "grad_norm": 0.34520450234413147, + "learning_rate": 5.264754525396212e-05, + "loss": 0.056367015838623045, + "step": 760 + }, + { + "epoch": 2.4850418827283605, + "grad_norm": 0.45693477988243103, + "learning_rate": 5.06496896169857e-05, + "loss": 0.055580759048461915, + "step": 780 + }, + { + "epoch": 2.5488631830873554, + "grad_norm": 0.3360002338886261, + "learning_rate": 4.865087363121384e-05, + "loss": 0.061715197563171384, + "step": 800 + }, + { + "epoch": 2.6126844834463503, + "grad_norm": 0.5039923787117004, + "learning_rate": 4.6654298220040673e-05, + "loss": 0.05754677057266235, + "step": 820 + }, + { + "epoch": 2.6765057838053448, + "grad_norm": 0.29794442653656006, + "learning_rate": 4.466316071878229e-05, + "loss": 0.055103600025177, + "step": 840 + }, + { + "epoch": 2.74032708416434, + "grad_norm": 0.21690921485424042, + "learning_rate": 4.2680649754432913e-05, + "loss": 0.054493296146392825, + "step": 860 + }, + { + "epoch": 2.8041483845233346, + "grad_norm": 0.28939780592918396, + "learning_rate": 4.070994013936646e-05, + "loss": 0.05614350438117981, + "step": 880 + }, + { + "epoch": 2.8679696848823295, + "grad_norm": 0.5370737910270691, + "learning_rate": 3.8754187787161155e-05, + "loss": 0.05749887228012085, + "step": 900 + }, + { + "epoch": 2.9317909852413244, + "grad_norm": 0.21074210107326508, + "learning_rate": 3.681652465868875e-05, + "loss": 0.05852361917495728, + "step": 920 + }, + { + "epoch": 2.995612285600319, + "grad_norm": 0.4107605516910553, + "learning_rate": 3.490005374656184e-05, + "loss": 0.05852218866348267, + "step": 940 + }, + { + "epoch": 3.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 94.6846, + "eval_generator_rouge1": 99.8316, + "eval_generator_rouge2": 99.7174, + "eval_generator_rougeL": 99.6293, + "eval_loss": 0.005526242312043905, + "eval_runtime": 613.3531, + "eval_samples_per_second": 5.566, + "eval_selector_accuracy": 96.9042, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.174, + "step": 942 + }, + { + "epoch": 3.057439170323095, + "grad_norm": 0.21888521313667297, + "learning_rate": 3.3007844105971336e-05, + "loss": 0.05516682863235474, + "step": 960 + }, + { + "epoch": 3.12126047068209, + "grad_norm": 0.2247946858406067, + "learning_rate": 3.114292593987149e-05, + "loss": 0.05510848164558411, + "step": 980 + }, + { + "epoch": 3.185081771041085, + "grad_norm": 0.18932947516441345, + "learning_rate": 2.9308285746383408e-05, + "loss": 0.05192863345146179, + "step": 1000 + }, + { + "epoch": 3.24890307140008, + "grad_norm": 0.18617865443229675, + "learning_rate": 2.7506861536187867e-05, + "loss": 0.05452244281768799, + "step": 1020 + }, + { + "epoch": 3.3127243717590744, + "grad_norm": 0.21938033401966095, + "learning_rate": 2.5741538127566396e-05, + "loss": 0.051268589496612546, + "step": 1040 + }, + { + "epoch": 3.3765456721180693, + "grad_norm": 0.2056664675474167, + "learning_rate": 2.4015142526625115e-05, + "loss": 0.05224469304084778, + "step": 1060 + }, + { + "epoch": 3.4403669724770642, + "grad_norm": 0.2431439608335495, + "learning_rate": 2.233043940009967e-05, + "loss": 0.05370842218399048, + "step": 1080 + }, + { + "epoch": 3.504188272836059, + "grad_norm": 0.1797763705253601, + "learning_rate": 2.0690126647990974e-05, + "loss": 0.05102568864822388, + "step": 1100 + }, + { + "epoch": 3.568009573195054, + "grad_norm": 0.20713362097740173, + "learning_rate": 1.9096831083121828e-05, + "loss": 0.05096147060394287, + "step": 1120 + }, + { + "epoch": 3.6318308735540485, + "grad_norm": 0.26381605863571167, + "learning_rate": 1.7553104224533324e-05, + "loss": 0.050066190958023074, + "step": 1140 + }, + { + "epoch": 3.6956521739130435, + "grad_norm": 0.2653278708457947, + "learning_rate": 1.606141821145735e-05, + "loss": 0.050461554527282716, + "step": 1160 + }, + { + "epoch": 3.7594734742720384, + "grad_norm": 0.2132464498281479, + "learning_rate": 1.4624161844408763e-05, + "loss": 0.0523165225982666, + "step": 1180 + }, + { + "epoch": 3.823294774631033, + "grad_norm": 0.19349783658981323, + "learning_rate": 1.324363675973698e-05, + "loss": 0.04992191791534424, + "step": 1200 + }, + { + "epoch": 3.8871160749900278, + "grad_norm": 0.240342378616333, + "learning_rate": 1.1922053743763052e-05, + "loss": 0.05000295639038086, + "step": 1220 + }, + { + "epoch": 3.9509373753490227, + "grad_norm": 0.19015061855316162, + "learning_rate": 1.0661529192404973e-05, + "loss": 0.04904879331588745, + "step": 1240 + }, + { + "epoch": 4.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 95.8528, + "eval_generator_rouge1": 99.9437, + "eval_generator_rouge2": 99.8882, + "eval_generator_rougeL": 99.7233, + "eval_loss": 0.003825092688202858, + "eval_runtime": 603.7734, + "eval_samples_per_second": 5.654, + "eval_selector_accuracy": 98.5397, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.177, + "step": 1256 + }, + { + "epoch": 4.0127642600717985, + "grad_norm": 0.19705015420913696, + "learning_rate": 9.464081721960592e-06, + "loss": 0.050104594230651854, + "step": 1260 + }, + { + "epoch": 4.076585560430794, + "grad_norm": 0.33697429299354553, + "learning_rate": 8.331628936475818e-06, + "loss": 0.04852663278579712, + "step": 1280 + }, + { + "epoch": 4.140406860789788, + "grad_norm": 0.21710842847824097, + "learning_rate": 7.2659843568748385e-06, + "loss": 0.048302799463272095, + "step": 1300 + }, + { + "epoch": 4.204228161148784, + "grad_norm": 0.16647987067699432, + "learning_rate": 6.268854516770008e-06, + "loss": 0.04852384328842163, + "step": 1320 + }, + { + "epoch": 4.268049461507778, + "grad_norm": 0.1737622171640396, + "learning_rate": 5.341836229602331e-06, + "loss": 0.04865268170833588, + "step": 1340 + }, + { + "epoch": 4.331870761866773, + "grad_norm": 0.19361254572868347, + "learning_rate": 4.4864140314888216e-06, + "loss": 0.04774414896965027, + "step": 1360 + }, + { + "epoch": 4.395692062225768, + "grad_norm": 0.18790625035762787, + "learning_rate": 3.703957803871932e-06, + "loss": 0.04869887530803681, + "step": 1380 + }, + { + "epoch": 4.4595133625847625, + "grad_norm": 0.16145937144756317, + "learning_rate": 2.9957205797780495e-06, + "loss": 0.049949395656585696, + "step": 1400 + }, + { + "epoch": 4.523334662943758, + "grad_norm": 0.31126755475997925, + "learning_rate": 2.362836537198199e-06, + "loss": 0.049376997351646426, + "step": 1420 + }, + { + "epoch": 4.587155963302752, + "grad_norm": 0.17530952394008636, + "learning_rate": 1.8063191828043703e-06, + "loss": 0.04881583452224732, + "step": 1440 + }, + { + "epoch": 4.650977263661747, + "grad_norm": 0.21170786023139954, + "learning_rate": 1.3270597289100437e-06, + "loss": 0.04805063009262085, + "step": 1460 + }, + { + "epoch": 4.714798564020742, + "grad_norm": 0.156987264752388, + "learning_rate": 9.258256662741205e-07, + "loss": 0.047132036089897154, + "step": 1480 + }, + { + "epoch": 4.778619864379737, + "grad_norm": 0.1759834736585617, + "learning_rate": 6.032595350337341e-07, + "loss": 0.048763877153396605, + "step": 1500 + }, + { + "epoch": 4.842441164738732, + "grad_norm": 0.16100358963012695, + "learning_rate": 3.5987789573421356e-07, + "loss": 0.05374665260314941, + "step": 1520 + }, + { + "epoch": 4.9062624650977265, + "grad_norm": 0.21420489251613617, + "learning_rate": 1.9607050210401192e-07, + "loss": 0.049358141422271726, + "step": 1540 + }, + { + "epoch": 4.970083765456721, + "grad_norm": 0.1897514909505844, + "learning_rate": 1.1209967689925832e-07, + "loss": 0.049035769701004026, + "step": 1560 + }, + { + "epoch": 5.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 96.3785, + "eval_generator_rouge1": 99.9468, + "eval_generator_rouge2": 99.8886, + "eval_generator_rougeL": 99.7652, + "eval_loss": 0.0038434057496488094, + "eval_runtime": 612.3424, + "eval_samples_per_second": 5.575, + "eval_selector_accuracy": 98.8318, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.175, + "step": 1570 + }, + { + "epoch": 5.0, + "step": 1570, + "total_flos": 5.871444840310374e+16, + "train_loss": 0.09919491964540664, + "train_runtime": 8002.0141, + "train_samples_per_second": 6.265, + "train_steps_per_second": 0.196 + } + ], + "logging_steps": 20, + "max_steps": 1570, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 5.871444840310374e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/qwen3/fdd_sfkl/training_args.bin b/qwen3/fdd_sfkl/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..4aaaca662465ff2e3933154ec90d4097299421a9 --- /dev/null +++ b/qwen3/fdd_sfkl/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18f98e7767ecd74653cd41dec83bbbe276858948c803da4b684e0aaf5a1cc1f7 +size 7185 diff --git a/qwen3/fdd_srkl/README.md b/qwen3/fdd_srkl/README.md new file mode 100644 index 0000000000000000000000000000000000000000..78fa0a15e70bfa969ec74562cbc5a476095a6417 --- /dev/null +++ b/qwen3/fdd_srkl/README.md @@ -0,0 +1,79 @@ +--- +library_name: peft +license: other +base_model: Qwen/Qwen3-0.6B +tags: +- base_model:adapter:Qwen/Qwen3-0.6B +- llama-factory +- lora +- transformers +pipeline_tag: text-generation +model-index: +- name: fdd_srkl + results: [] +--- + + + +# fdd_srkl + +This model is a fine-tuned version of [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) on the cypher_prepared_train dataset. +It achieves the following results on the evaluation set: +- Loss: 0.0041 +- Selector Count: 1712.0 +- Generator Count: 1712.0 +- Selector Accuracy: 98.7734 +- Generator Exact Match: 96.6121 +- Generator Rouge1: 99.8663 +- Generator Rouge2: 99.793 +- Generator Rougel: 99.724 + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 0.0001 +- train_batch_size: 2 +- eval_batch_size: 16 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 2 +- gradient_accumulation_steps: 8 +- total_train_batch_size: 32 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine_with_min_lr +- num_epochs: 5 + +### Training results + +| Training Loss | Epoch | Step | Validation Loss | Selector Count | Generator Count | Selector Accuracy | Generator Exact Match | Generator Rouge1 | Generator Rouge2 | Generator Rougel | +|:-------------:|:-----:|:----:|:---------------:|:--------------:|:---------------:|:-----------------:|:---------------------:|:----------------:|:----------------:|:----------------:| +| 0.1054 | 1.0 | 314 | 0.0091 | 1712.0 | 1712.0 | 96.6706 | 87.9089 | 99.464 | 99.1899 | 99.0517 | +| 0.0649 | 2.0 | 628 | 0.0065 | 1712.0 | 1712.0 | 97.3715 | 94.743 | 99.8867 | 99.7804 | 99.6736 | +| 0.0567 | 3.0 | 942 | 0.0057 | 1712.0 | 1712.0 | 98.4813 | 95.0935 | 99.8705 | 99.7886 | 99.6664 | +| 0.0499 | 4.0 | 1256 | 0.0042 | 1712.0 | 1712.0 | 98.5397 | 96.3201 | 99.9029 | 99.8325 | 99.7566 | +| 0.0492 | 5.0 | 1570 | 0.0041 | 1712.0 | 1712.0 | 98.8318 | 96.7874 | 99.9018 | 99.8253 | 99.7687 | + + +### Framework versions + +- PEFT 0.18.1 +- Transformers 5.8.0 +- Pytorch 2.9.1+cu128 +- Datasets 4.0.0 +- Tokenizers 0.22.2 \ No newline at end of file diff --git a/qwen3/fdd_srkl/adapter_config.json b/qwen3/fdd_srkl/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b3d78327098bdde7084c1184d3568542c0fc45b4 --- /dev/null +++ b/qwen3/fdd_srkl/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-0.6B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "q_proj", + "v_proj", + "o_proj", + "up_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/qwen3/fdd_srkl/adapter_model.safetensors b/qwen3/fdd_srkl/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..aaf004a5282d6eff939542541b017b669c2028af --- /dev/null +++ b/qwen3/fdd_srkl/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99fc0546ccfaf4ac4d14c8f45aee81602af1b5a2055a86913c4740cf4b7ef42c +size 40422560 diff --git a/qwen3/fdd_srkl/all_results.json b/qwen3/fdd_srkl/all_results.json new file mode 100644 index 0000000000000000000000000000000000000000..a0c6e120a0678f306f32f4914009418441792ff5 --- /dev/null +++ b/qwen3/fdd_srkl/all_results.json @@ -0,0 +1,19 @@ +{ + "epoch": 5.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 96.6121, + "eval_generator_rouge1": 99.8663, + "eval_generator_rouge2": 99.793, + "eval_generator_rougeL": 99.724, + "eval_loss": 0.0041342261247336864, + "eval_runtime": 595.4628, + "eval_samples_per_second": 5.733, + "eval_selector_accuracy": 98.7734, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.18, + "total_flos": 5.871444840310374e+16, + "train_loss": 0.10070288321774476, + "train_runtime": 8259.7035, + "train_samples_per_second": 6.07, + "train_steps_per_second": 0.19 +} \ No newline at end of file diff --git a/qwen3/fdd_srkl/chat_template.jinja b/qwen3/fdd_srkl/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/qwen3/fdd_srkl/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/qwen3/fdd_srkl/checkpoint-1256/README.md b/qwen3/fdd_srkl/checkpoint-1256/README.md new file mode 100644 index 0000000000000000000000000000000000000000..776f98d43659dd923b8c3e5649c5ae66d1f99f3e --- /dev/null +++ b/qwen3/fdd_srkl/checkpoint-1256/README.md @@ -0,0 +1,208 @@ +--- +base_model: Qwen/Qwen3-0.6B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3-0.6B +- llama-factory +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/qwen3/fdd_srkl/checkpoint-1256/adapter_config.json b/qwen3/fdd_srkl/checkpoint-1256/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b3d78327098bdde7084c1184d3568542c0fc45b4 --- /dev/null +++ b/qwen3/fdd_srkl/checkpoint-1256/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-0.6B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.1, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "q_proj", + "v_proj", + "o_proj", + "up_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/qwen3/fdd_srkl/checkpoint-1256/adapter_model.safetensors b/qwen3/fdd_srkl/checkpoint-1256/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e7930472e23896c949d0bb7b1282784a85e2fc66 --- /dev/null +++ b/qwen3/fdd_srkl/checkpoint-1256/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90401ea91feb1094bfe8e19177a5d35d413782e2499bf55041333f7fe0618265 +size 40422560 diff --git a/qwen3/fdd_srkl/checkpoint-1256/chat_template.jinja b/qwen3/fdd_srkl/checkpoint-1256/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..01be9b307daa2d425f7c168c9fb145a286e0afb4 --- /dev/null +++ b/qwen3/fdd_srkl/checkpoint-1256/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/qwen3/fdd_srkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/qwen3/fdd_srkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..62c5679a541a6bff30a2cf96750fa8e15a88b152 --- /dev/null +++ b/qwen3/fdd_srkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:299ca9fdb9d8d4937e41ae65083646d4f0af4078d24817d8908732f20a96e637 +size 121135557 diff --git a/qwen3/fdd_srkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/qwen3/fdd_srkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1e2c9b82281257351999f626c2971d8b1169857 --- /dev/null +++ b/qwen3/fdd_srkl/checkpoint-1256/global_step1256/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbf035b649393a8bb3a8fb9393e32e7fa0f79ab07659ad31c3e850cdd7d4e571 +size 121135749 diff --git a/qwen3/fdd_srkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt b/qwen3/fdd_srkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6fcea1d7d28fb077482f8cfc89500ff46feec53 --- /dev/null +++ b/qwen3/fdd_srkl/checkpoint-1256/global_step1256/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c10c1d9df4a070556411670ae2c5441343a8ab8146c11e62f2db60fee35f84b +size 351841765 diff --git a/qwen3/fdd_srkl/checkpoint-1256/latest b/qwen3/fdd_srkl/checkpoint-1256/latest new file mode 100644 index 0000000000000000000000000000000000000000..74abb3d23ea83ebdac59918752db5d1782834ca7 --- /dev/null +++ b/qwen3/fdd_srkl/checkpoint-1256/latest @@ -0,0 +1 @@ +global_step1256 \ No newline at end of file diff --git a/qwen3/fdd_srkl/checkpoint-1256/rng_state_0.pth b/qwen3/fdd_srkl/checkpoint-1256/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..c660ea1fcee12e9a54c99e7d9f7e74197d51b717 --- /dev/null +++ b/qwen3/fdd_srkl/checkpoint-1256/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f7b1ea77c2c27c89f0e4eb320691d92d9d6acf42cd6cfd5c9e30d29bd3fcb62 +size 14917 diff --git a/qwen3/fdd_srkl/checkpoint-1256/rng_state_1.pth b/qwen3/fdd_srkl/checkpoint-1256/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..7f08cd63199181edc8160029106c70585c8f7482 --- /dev/null +++ b/qwen3/fdd_srkl/checkpoint-1256/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3973ed1ef0317ab8d01b25086ace082d4bcbf40f8fb9ca4f76953913cbe78ebf +size 14917 diff --git a/qwen3/fdd_srkl/checkpoint-1256/scheduler.pt b/qwen3/fdd_srkl/checkpoint-1256/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..40e863d07cfed22cd912014eb3ba3fd2b16b173f --- /dev/null +++ b/qwen3/fdd_srkl/checkpoint-1256/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1775297f14e75f7463ae51983a66296f81f4b2e2e5d7234f1a6d295abfa4cf23 +size 1465 diff --git a/qwen3/fdd_srkl/checkpoint-1256/tokenizer.json b/qwen3/fdd_srkl/checkpoint-1256/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/qwen3/fdd_srkl/checkpoint-1256/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/qwen3/fdd_srkl/checkpoint-1256/tokenizer_config.json b/qwen3/fdd_srkl/checkpoint-1256/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6623da03457d79d2cd4af9557e9f31407cdb835c --- /dev/null +++ b/qwen3/fdd_srkl/checkpoint-1256/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/qwen3/fdd_srkl/checkpoint-1256/trainer_state.json b/qwen3/fdd_srkl/checkpoint-1256/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c68e8cbe2229abc97b063442f6bbe312298bdce3 --- /dev/null +++ b/qwen3/fdd_srkl/checkpoint-1256/trainer_state.json @@ -0,0 +1,528 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1256, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.06382130035899482, + "grad_norm": 2.4362361431121826, + "learning_rate": 9.996390385474569e-05, + "loss": 1.4636783599853516, + "step": 20 + }, + { + "epoch": 0.12764260071798963, + "grad_norm": 2.9437320232391357, + "learning_rate": 9.984797509888031e-05, + "loss": 0.5306460380554199, + "step": 40 + }, + { + "epoch": 0.19146390107698444, + "grad_norm": 1.3007458448410034, + "learning_rate": 9.965229938050925e-05, + "loss": 0.34657702445983884, + "step": 60 + }, + { + "epoch": 0.25528520143597927, + "grad_norm": 1.569557547569275, + "learning_rate": 9.937719005663432e-05, + "loss": 0.25293736457824706, + "step": 80 + }, + { + "epoch": 0.3191065017949741, + "grad_norm": 0.8431988954544067, + "learning_rate": 9.902308769000724e-05, + "loss": 0.208502197265625, + "step": 100 + }, + { + "epoch": 0.3829278021539689, + "grad_norm": 2.0384669303894043, + "learning_rate": 9.859055934360789e-05, + "loss": 0.16615729331970214, + "step": 120 + }, + { + "epoch": 0.4467491025129637, + "grad_norm": 1.341955542564392, + "learning_rate": 9.808029767254437e-05, + "loss": 0.14722728729248047, + "step": 140 + }, + { + "epoch": 0.5105704028719585, + "grad_norm": 1.9276875257492065, + "learning_rate": 9.749311981482817e-05, + "loss": 0.14779376983642578, + "step": 160 + }, + { + "epoch": 0.5743917032309533, + "grad_norm": 1.3371649980545044, + "learning_rate": 9.682996608280154e-05, + "loss": 0.13147571086883544, + "step": 180 + }, + { + "epoch": 0.6382130035899481, + "grad_norm": 1.0184792280197144, + "learning_rate": 9.609189845731207e-05, + "loss": 0.14180898666381836, + "step": 200 + }, + { + "epoch": 0.702034303948943, + "grad_norm": 0.5344496369361877, + "learning_rate": 9.52800988870462e-05, + "loss": 0.12216286659240723, + "step": 220 + }, + { + "epoch": 0.7658556043079378, + "grad_norm": 3.0611512660980225, + "learning_rate": 9.439586739574513e-05, + "loss": 0.11602286100387574, + "step": 240 + }, + { + "epoch": 0.8296769046669326, + "grad_norm": 0.5453760027885437, + "learning_rate": 9.344062000033414e-05, + "loss": 0.11221114397048951, + "step": 260 + }, + { + "epoch": 0.8934982050259274, + "grad_norm": 0.4289671778678894, + "learning_rate": 9.241588644329919e-05, + "loss": 0.11079610586166382, + "step": 280 + }, + { + "epoch": 0.9573195053849222, + "grad_norm": 0.4820654094219208, + "learning_rate": 9.132330774294252e-05, + "loss": 0.10537848472595215, + "step": 300 + }, + { + "epoch": 1.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 87.9089, + "eval_generator_rouge1": 99.464, + "eval_generator_rouge2": 99.1899, + "eval_generator_rougeL": 99.0517, + "eval_loss": 0.009097261354327202, + "eval_runtime": 606.4571, + "eval_samples_per_second": 5.629, + "eval_selector_accuracy": 96.6706, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.176, + "step": 314 + }, + { + "epoch": 1.0191463901076985, + "grad_norm": 3.0870883464813232, + "learning_rate": 9.01646335654399e-05, + "loss": 0.09972293376922607, + "step": 320 + }, + { + "epoch": 1.0829676904666932, + "grad_norm": 0.7636368870735168, + "learning_rate": 8.894171942290817e-05, + "loss": 0.08893377780914306, + "step": 340 + }, + { + "epoch": 1.146788990825688, + "grad_norm": 0.4203242063522339, + "learning_rate": 8.765652370197003e-05, + "loss": 0.08573378920555115, + "step": 360 + }, + { + "epoch": 1.210610291184683, + "grad_norm": 0.371820867061615, + "learning_rate": 8.631110452757467e-05, + "loss": 0.07898830771446227, + "step": 380 + }, + { + "epoch": 1.2744315915436777, + "grad_norm": 0.676804780960083, + "learning_rate": 8.490761646709642e-05, + "loss": 0.08897324800491332, + "step": 400 + }, + { + "epoch": 1.3382528919026724, + "grad_norm": 1.0131059885025024, + "learning_rate": 8.344830707998953e-05, + "loss": 0.08602790236473083, + "step": 420 + }, + { + "epoch": 1.4020741922616673, + "grad_norm": 0.4733346104621887, + "learning_rate": 8.193551331852458e-05, + "loss": 0.08213704824447632, + "step": 440 + }, + { + "epoch": 1.4658954926206622, + "grad_norm": 0.9082027673721313, + "learning_rate": 8.037165778537033e-05, + "loss": 0.0822598934173584, + "step": 460 + }, + { + "epoch": 1.5297167929796571, + "grad_norm": 0.3575717508792877, + "learning_rate": 7.875924485401427e-05, + "loss": 0.0718361496925354, + "step": 480 + }, + { + "epoch": 1.5935380933386518, + "grad_norm": 0.3527005612850189, + "learning_rate": 7.710085665823444e-05, + "loss": 0.07293112277984619, + "step": 500 + }, + { + "epoch": 1.6573593936976465, + "grad_norm": 0.3359999358654022, + "learning_rate": 7.539914895704521e-05, + "loss": 0.07311422824859619, + "step": 520 + }, + { + "epoch": 1.7211806940566414, + "grad_norm": 0.3345245122909546, + "learning_rate": 7.365684688173908e-05, + "loss": 0.07376066446304322, + "step": 540 + }, + { + "epoch": 1.7850019944156363, + "grad_norm": 0.3286590874195099, + "learning_rate": 7.18767405718347e-05, + "loss": 0.06656086444854736, + "step": 560 + }, + { + "epoch": 1.848823294774631, + "grad_norm": 0.41298067569732666, + "learning_rate": 7.006168070692036e-05, + "loss": 0.07345538735389709, + "step": 580 + }, + { + "epoch": 1.9126445951336257, + "grad_norm": 0.2776452600955963, + "learning_rate": 6.821457394154786e-05, + "loss": 0.06613650321960449, + "step": 600 + }, + { + "epoch": 1.9764658954926206, + "grad_norm": 0.27869823575019836, + "learning_rate": 6.633837825048768e-05, + "loss": 0.06492968797683715, + "step": 620 + }, + { + "epoch": 2.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 94.743, + "eval_generator_rouge1": 99.8867, + "eval_generator_rouge2": 99.7804, + "eval_generator_rougeL": 99.6736, + "eval_loss": 0.006523935589939356, + "eval_runtime": 602.2662, + "eval_samples_per_second": 5.669, + "eval_selector_accuracy": 97.3715, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.178, + "step": 628 + }, + { + "epoch": 2.038292780215397, + "grad_norm": 0.5264983177185059, + "learning_rate": 6.443609819179926e-05, + "loss": 0.06454238891601563, + "step": 640 + }, + { + "epoch": 2.102114080574392, + "grad_norm": 0.7168541550636292, + "learning_rate": 6.251078009530256e-05, + "loss": 0.06528459787368775, + "step": 660 + }, + { + "epoch": 2.1659353809333863, + "grad_norm": 0.4798216223716736, + "learning_rate": 6.0565507184155677e-05, + "loss": 0.06159168481826782, + "step": 680 + }, + { + "epoch": 2.2297566812923812, + "grad_norm": 0.5430735349655151, + "learning_rate": 5.860339463735136e-05, + "loss": 0.06343585252761841, + "step": 700 + }, + { + "epoch": 2.293577981651376, + "grad_norm": 0.2429123818874359, + "learning_rate": 5.662758460103895e-05, + "loss": 0.06073571443557739, + "step": 720 + }, + { + "epoch": 2.357399282010371, + "grad_norm": 0.2920048236846924, + "learning_rate": 5.464124115666095e-05, + "loss": 0.05954910516738891, + "step": 740 + }, + { + "epoch": 2.421220582369366, + "grad_norm": 0.19888697564601898, + "learning_rate": 5.264754525396212e-05, + "loss": 0.05689072012901306, + "step": 760 + }, + { + "epoch": 2.4850418827283605, + "grad_norm": 0.3021155893802643, + "learning_rate": 5.06496896169857e-05, + "loss": 0.0560643196105957, + "step": 780 + }, + { + "epoch": 2.5488631830873554, + "grad_norm": 0.32512202858924866, + "learning_rate": 4.865087363121384e-05, + "loss": 0.0600568413734436, + "step": 800 + }, + { + "epoch": 2.6126844834463503, + "grad_norm": 0.53708815574646, + "learning_rate": 4.6654298220040673e-05, + "loss": 0.05846037864685059, + "step": 820 + }, + { + "epoch": 2.6765057838053448, + "grad_norm": 0.3181110620498657, + "learning_rate": 4.466316071878229e-05, + "loss": 0.05601181983947754, + "step": 840 + }, + { + "epoch": 2.74032708416434, + "grad_norm": 0.22879371047019958, + "learning_rate": 4.2680649754432913e-05, + "loss": 0.05564823150634766, + "step": 860 + }, + { + "epoch": 2.8041483845233346, + "grad_norm": 0.24975994229316711, + "learning_rate": 4.070994013936646e-05, + "loss": 0.05476831197738648, + "step": 880 + }, + { + "epoch": 2.8679696848823295, + "grad_norm": 0.3306073546409607, + "learning_rate": 3.8754187787161155e-05, + "loss": 0.05817422866821289, + "step": 900 + }, + { + "epoch": 2.9317909852413244, + "grad_norm": 0.1665683090686798, + "learning_rate": 3.681652465868875e-05, + "loss": 0.05767209529876709, + "step": 920 + }, + { + "epoch": 2.995612285600319, + "grad_norm": 0.3881584703922272, + "learning_rate": 3.490005374656184e-05, + "loss": 0.05669119954109192, + "step": 940 + }, + { + "epoch": 3.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 95.0935, + "eval_generator_rouge1": 99.8705, + "eval_generator_rouge2": 99.7886, + "eval_generator_rougeL": 99.6664, + "eval_loss": 0.005655109416693449, + "eval_runtime": 604.2347, + "eval_samples_per_second": 5.65, + "eval_selector_accuracy": 98.4813, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.177, + "step": 942 + }, + { + "epoch": 3.057439170323095, + "grad_norm": 0.2090013325214386, + "learning_rate": 3.3007844105971336e-05, + "loss": 0.05627270340919495, + "step": 960 + }, + { + "epoch": 3.12126047068209, + "grad_norm": 0.2156447023153305, + "learning_rate": 3.114292593987149e-05, + "loss": 0.05334089994430542, + "step": 980 + }, + { + "epoch": 3.185081771041085, + "grad_norm": 0.1974238008260727, + "learning_rate": 2.9308285746383408e-05, + "loss": 0.051919639110565186, + "step": 1000 + }, + { + "epoch": 3.24890307140008, + "grad_norm": 0.18332256376743317, + "learning_rate": 2.7506861536187867e-05, + "loss": 0.05656099319458008, + "step": 1020 + }, + { + "epoch": 3.3127243717590744, + "grad_norm": 0.7237991094589233, + "learning_rate": 2.5741538127566396e-05, + "loss": 0.05259966850280762, + "step": 1040 + }, + { + "epoch": 3.3765456721180693, + "grad_norm": 0.2267737239599228, + "learning_rate": 2.4015142526625115e-05, + "loss": 0.05074975490570068, + "step": 1060 + }, + { + "epoch": 3.4403669724770642, + "grad_norm": 0.20692221820354462, + "learning_rate": 2.233043940009967e-05, + "loss": 0.054996371269226074, + "step": 1080 + }, + { + "epoch": 3.504188272836059, + "grad_norm": 0.15545181930065155, + "learning_rate": 2.0690126647990974e-05, + "loss": 0.05093409419059754, + "step": 1100 + }, + { + "epoch": 3.568009573195054, + "grad_norm": 0.1771135926246643, + "learning_rate": 1.9096831083121828e-05, + "loss": 0.05136837959289551, + "step": 1120 + }, + { + "epoch": 3.6318308735540485, + "grad_norm": 0.24874085187911987, + "learning_rate": 1.7553104224533324e-05, + "loss": 0.05098183155059814, + "step": 1140 + }, + { + "epoch": 3.6956521739130435, + "grad_norm": 0.17957162857055664, + "learning_rate": 1.606141821145735e-05, + "loss": 0.050333261489868164, + "step": 1160 + }, + { + "epoch": 3.7594734742720384, + "grad_norm": 0.1954505741596222, + "learning_rate": 1.4624161844408763e-05, + "loss": 0.05178115367889404, + "step": 1180 + }, + { + "epoch": 3.823294774631033, + "grad_norm": 0.19122923910617828, + "learning_rate": 1.324363675973698e-05, + "loss": 0.050999945402145384, + "step": 1200 + }, + { + "epoch": 3.8871160749900278, + "grad_norm": 0.26517826318740845, + "learning_rate": 1.1922053743763052e-05, + "loss": 0.0516182005405426, + "step": 1220 + }, + { + "epoch": 3.9509373753490227, + "grad_norm": 0.1767585426568985, + "learning_rate": 1.0661529192404973e-05, + "loss": 0.04989046454429626, + "step": 1240 + }, + { + "epoch": 4.0, + "eval_generator_count": 1712.0, + "eval_generator_exact_match": 96.3201, + "eval_generator_rouge1": 99.9029, + "eval_generator_rouge2": 99.8325, + "eval_generator_rougeL": 99.7566, + "eval_loss": 0.0041970605961978436, + "eval_runtime": 743.2973, + "eval_samples_per_second": 4.593, + "eval_selector_accuracy": 98.5397, + "eval_selector_count": 1712.0, + "eval_steps_per_second": 0.144, + "step": 1256 + } + ], + "logging_steps": 20, + "max_steps": 1570, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.697643190059008e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/qwen3/fdd_srkl/checkpoint-1256/training_args.bin b/qwen3/fdd_srkl/checkpoint-1256/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..063309915a275e078e6f834ef9c384b1cb1c965e --- /dev/null +++ b/qwen3/fdd_srkl/checkpoint-1256/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb0e76106b10de1344a3493c5f04dde38b30a16fde01bc76445c98b578802418 +size 7185 diff --git a/qwen3/fdd_srkl/checkpoint-1256/zero_to_fp32.py b/qwen3/fdd_srkl/checkpoint-1256/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..f40f1f0acbbff30e8e8d29906b56aae4e8b8a138 --- /dev/null +++ b/qwen3/fdd_srkl/checkpoint-1256/zero_to_fp32.py @@ -0,0 +1,790 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS, AUTOEP_LAYERS_KEY, + AUTOEP_LAYERS_KEY_LEGACY, AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY, + AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def _has_autoep_zero3_partitioned_metadata(state_dict): + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY) + if autoep_layers is None: + autoep_layers = state_dict.get(AUTOEP_LAYERS_KEY_LEGACY) + if not isinstance(autoep_layers, list): + return False + return any( + isinstance(entry, dict) + and entry.get(AUTOEP_ZERO3_EXPERT_STATE_FORMAT_KEY) == AUTOEP_ZERO3_PARTITIONED_EXPERT_STATE_FORMAT + for entry in autoep_layers) + + +def _raise_if_autoep_zero3_partitioned_state(state_dict): + if _has_autoep_zero3_partitioned_metadata(state_dict): + raise NotImplementedError("zero_to_fp32 does not support AutoEP ZeRO-3 partition-native checkpoints. " + "AutoEP expert parameters are partitioned over expert replica groups, so " + "global data-parallel consolidation would produce incomplete expert tensors. " + "Use ds_to_universal.py for expert-aware conversion.") + + +def _raise_if_autoep_zero3_partitioned_checkpoint(model_files): + for file in model_files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + _raise_if_autoep_zero3_partitioned_state(state_dict) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + # parse_model_states rejects AutoEP ZeRO-3 partition-native checkpoints + # before the expensive optimizer-shard load below. + model_files = get_model_state_files(ds_checkpoint_dir) + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters)