Add files using upload-large-folder tool

.gitattributes

@@ -50,3 +50,5 @@ pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/wandb/run-20260425_215604-jhqe4qjw/run-jhq
 generation_bench_results/runs.partial.jsonl filter=lfs diff=lfs merge=lfs -text
 generation_bench_results/results.json filter=lfs diff=lfs merge=lfs -text
 pythia_1b_rerun/pythia_1b_lr_2e-5/wandb/run-20260523_224742-l9w4bbvz/run-l9w4bbvz.wandb filter=lfs diff=lfs merge=lfs -text
+routing_rerun/N_8.0/wandb/run-20260524_111710-wj0poul7/run-wj0poul7.wandb filter=lfs diff=lfs merge=lfs -text
+routing_rerun/N_2.5/wandb/run-20260523_172819-hcp9axm4/run-hcp9axm4.wandb filter=lfs diff=lfs merge=lfs -text

routing_rerun/N_2.5/checkpoints/checkpoint_latest.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e8ca0ae6d9550a5a319ff24ca12a22fee8abcf486f2bb54a3cbed6f29b5b1f9
+size 3315169346

routing_rerun/N_2.5/routing_weights/routing_step_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:132bc38ff57f9db6c9ca06a66251bf38dc4231f85bd01101af2cf1df371b5db7
+size 13633736

routing_rerun/N_2.5/routing_weights/routing_step_12000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:680036bb3ff6ba5bc6cc1838021853aab9c93ded0c7992ca3ab89db3daa08988
+size 13633768

routing_rerun/N_2.5/routing_weights/routing_step_14061.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4cdcb761c7aceb7f0f38fdefc64eba4d883acc47f0c552c73fad6c5f4e2b816
+size 13633768

routing_rerun/N_2.5/routing_weights/routing_step_3000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4766cbd387f6634bd39c36fcf57f610274ed26fb1206c117f07986816886fafe
+size 13633760

routing_rerun/N_2.5/routing_weights/routing_step_6000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c174f875f37a059f3100953a39ac1870d0d6471ee10f2da1eb51f3300fd2b76
+size 13633760

routing_rerun/N_2.5/routing_weights/routing_step_9000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50bed38b07b22441e81d7d7cc6a9e5cdaea9014155705cd4a6c402b988b88522
+size 13633760

routing_rerun/N_2.5/wandb/run-20260523_172819-hcp9axm4/run-hcp9axm4.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7482ea30a6fbfc224f3c50aa03dfccb5a75a941579789493220869dedf2593c
+size 4539998

routing_rerun/N_4.0/checkpoints/checkpoint_step_12000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bd471e8cf6a32a257ae86c8d35c18634ac4a74a7e4bd0cceb306b627cc36aea
+size 3315170726

routing_rerun/N_4.0/checkpoints/checkpoint_step_3000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb89c708577dbc19b40f78b4ccd048eaae957c32fe0e7918249fc2a1b83f0e58
+size 3315170381

routing_rerun/N_4.0/checkpoints/checkpoint_step_6000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec93559bf5cae0838e60e2d61b9d3bfb3ebbf00edf5e07d63b631ae0ff8a765e
+size 3315170381

routing_rerun/N_4.0/checkpoints/checkpoint_step_9000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:473e94c9aace1258e3dfb8e71670d96646ae006a680965337c1ca9d869d0b47f
+size 3315170381

routing_rerun/N_4.0/wandb/run-20260524_000855-j30yot8x/files/code/routing_evolution_exp/train.py

@@ -0,0 +1,259 @@
+"""
+Routing Evolution Experiment: H-Net fine-tune с разными load_balancing_N.
+
+Цель: изучить как routing module (W_q, W_k) эволюционирует при разном давлении сжатия.
+
+Использование:
+    # Запуск с конкретным N
+    accelerate launch --config_file accelerate_config.yaml \
+        train.py training.load_balancing_N=6.0 tracking.run_name=routing_N6.0
+
+    # Запуск всех N через run_all.sh
+    bash run_all.sh
+"""
+
+import math
+import os
+from pathlib import Path
+
+import torch
+import hydra
+from hydra.core.hydra_config import HydraConfig
+from omegaconf import DictConfig, OmegaConf
+from accelerate import Accelerator
+from accelerate.utils import set_seed as accelerate_set_seed
+
+# HNet imports
+from hnet.load_utils import load_from_pretrained, load_from_config
+from hnet.utils.tokenizers import ByteTokenizer
+from hnet.utils.train import group_params
+
+# Ensure repo root is on sys.path (needed when running from subdirectory)
+import sys
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+# Shared training library
+from training_lib.utils import log_message
+from training_lib.checkpointing import save_checkpoint, load_checkpoint
+from training_lib.schedulers import get_lr_scheduler
+from training_lib.tracking import init_tracking, finish_tracking
+from training_lib.hnet.train_loop import train_epoch
+from training_lib.hnet.data import create_dataloaders
+
+
+def save_routing_weights(model, step, output_dir):
+    """Сохраняет только routing W_q/W_k для быстрого анализа weight evolution."""
+    routing_dir = output_dir / "routing_weights"
+    routing_dir.mkdir(parents=True, exist_ok=True)
+
+    state = model.state_dict()
+    routing_keys = [k for k in state if "routing_module" in k and "proj_layer" in k]
+    routing_weights = {k: state[k].cpu() for k in routing_keys}
+    torch.save(routing_weights, routing_dir / f"routing_step_{step}.pt")
+
+
+def on_checkpoint_callback(model, step, output_dir):
+    """Callback для train_epoch: сохраняет routing weights при каждом checkpoint.
+
+    Модель уже unwrapped в train_loop перед вызовом callback.
+    """
+    save_routing_weights(model, step, output_dir)
+
+
+@hydra.main(version_base=None, config_path="configs", config_name="config")
+def main(cfg: DictConfig):
+    """Тренировка H-Net с фиксированным числом шагов для анализа routing evolution."""
+
+    # === Accelerator Setup ===
+    mixed_precision = "bf16" if cfg.training.use_amp else "no"
+
+    accelerator = Accelerator(
+        mixed_precision=mixed_precision,
+        gradient_accumulation_steps=cfg.training.gradient_accumulation_steps,
+    )
+
+    accelerate_set_seed(cfg.seed)
+
+    if cfg.paths.output_dir is None:
+        cfg.paths.output_dir = HydraConfig.get().runtime.output_dir
+
+    OmegaConf.resolve(cfg)
+
+    log_message(
+        f"CUDA_VISIBLE_DEVICES: {os.environ.get('CUDA_VISIBLE_DEVICES', 'not set')}",
+        cfg,
+        accelerator,
+    )
+    log_message(f"Number of processes: {accelerator.num_processes}", cfg, accelerator)
+    log_message(f"Mixed precision: {mixed_precision}", cfg, accelerator)
+
+    log_message("=" * 60, cfg, accelerator)
+    log_message(
+        f"Routing Evolution Experiment | N={cfg.training.load_balancing_N}",
+        cfg,
+        accelerator,
+    )
+    log_message("=" * 60, cfg, accelerator)
+    log_message(f"Config:\n{OmegaConf.to_yaml(cfg)}", cfg, accelerator)
+
+    # === Experiment Tracking Init ===
+    init_tracking(cfg, accelerator)
+
+    # === Tokenizer ===
+    tokenizer = ByteTokenizer()
+
+    # === Model ===
+    log_message("Loading model...", cfg, accelerator)
+    if cfg.model.checkpoint_path:
+        model = load_from_pretrained(
+            model_path=cfg.model.checkpoint_path,
+            model_config_path=cfg.model.config_path,
+        )
+        log_message(f"Loaded pretrained: {cfg.model.checkpoint_path}", cfg, accelerator)
+    else:
+        model = load_from_config(
+            model_config_path=cfg.model.config_path,
+            device="cpu",
+        )
+        model.init_weights()
+        log_message("Initialized from scratch", cfg, accelerator)
+
+    model.train()
+
+    lr_multiplier = list(cfg.training.lr_multiplier)
+    model.apply_lr_multiplier(lr_multiplier)
+    log_message(f"Applied LR multipliers: {lr_multiplier}", cfg, accelerator)
+
+    if cfg.training.warmup_model:
+        log_message("Warming up model...", cfg, accelerator)
+        model = model.to(accelerator.device)
+        model.warmup(verbose=accelerator.is_main_process)
+
+    # Save initial routing weights (step 0)
+    if accelerator.is_main_process:
+        output_dir = Path(cfg.paths.output_dir)
+        save_routing_weights(model, 0, output_dir)
+
+    total_params = sum(p.numel() for p in model.parameters())
+    trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
+    log_message(f"Total params: {total_params:,}", cfg, accelerator)
+    log_message(f"Trainable params: {trainable_params:,}", cfg, accelerator)
+
+    # === Data ===
+    log_message("Creating dataloaders...", cfg, accelerator)
+    dataloaders = create_dataloaders(cfg, tokenizer)
+
+    train_dataloader = dataloaders["train"]
+    val_dataloader = dataloaders.get("validation", None)
+
+    log_message(
+        f"Train dataset size: {len(train_dataloader.dataset)} "
+        f"(max_train_samples={cfg.data.max_train_samples}) | "
+        f"Epochs: {cfg.training.epochs}",
+        cfg,
+        accelerator,
+    )
+
+    # === Optimizer ===
+    param_groups = group_params(model)
+
+    for group in param_groups:
+        if "lr" not in group:
+            group["lr"] = cfg.training.lr
+        else:
+            group["lr"] = cfg.training.lr * group.get("lr_multiplier", 1.0)
+        if "weight_decay" not in group:
+            group["weight_decay"] = cfg.training.weight_decay
+
+    optimizer = torch.optim.AdamW(
+        param_groups,
+        lr=cfg.training.lr,
+        betas=tuple(cfg.training.betas),
+        eps=cfg.training.eps,
+    )
+
+    # === Scheduler ===
+    steps_per_epoch = math.ceil(len(train_dataloader) / accelerator.num_processes)
+    max_steps = (
+        cfg.training.epochs
+        * steps_per_epoch
+        // cfg.training.gradient_accumulation_steps
+    )
+    scheduler = get_lr_scheduler(optimizer, cfg, max_steps)
+
+    log_message(
+        f"Max steps: {max_steps}, Steps per epoch: {steps_per_epoch}",
+        cfg,
+        accelerator,
+    )
+
+    # === Accelerate Prepare ===
+    if val_dataloader is not None:
+        model, optimizer, train_dataloader, val_dataloader, scheduler = (
+            accelerator.prepare(
+                model, optimizer, train_dataloader, val_dataloader, scheduler
+            )
+        )
+    else:
+        model, optimizer, train_dataloader, scheduler = accelerator.prepare(
+            model, optimizer, train_dataloader, scheduler
+        )
+
+    # === Training Loop ===
+    log_message("Starting training...", cfg, accelerator)
+
+    global_step = 0
+    best_val_loss = float("inf")
+    epoch = 0
+
+    num_epochs = cfg.training.epochs
+
+    try:
+        for epoch in range(1, num_epochs + 1):
+            log_message(f"\n{'=' * 60}", cfg, accelerator)
+            log_message(
+                f"EPOCH {epoch}/{num_epochs} (step {global_step})", cfg, accelerator
+            )
+            log_message(f"{'=' * 60}", cfg, accelerator)
+
+            global_step, best_val_loss = train_epoch(
+                model=model,
+                dataloader=train_dataloader,
+                optimizer=optimizer,
+                scheduler=scheduler,
+                cfg=cfg,
+                epoch=epoch,
+                global_step=global_step,
+                accelerator=accelerator,
+                val_dataloader=val_dataloader,
+                best_val_loss=best_val_loss,
+                max_steps=max_steps,
+                on_checkpoint=on_checkpoint_callback,
+            )
+
+    except KeyboardInterrupt:
+        log_message("Training interrupted by user", cfg, accelerator)
+        save_checkpoint(
+            model, optimizer, scheduler, global_step, epoch, cfg, accelerator,
+            model_only=True,
+        )
+
+    # === Final Save ===
+    log_message("\nTraining completed!", cfg, accelerator)
+
+    if accelerator.is_main_process:
+        final_model_path = Path(cfg.paths.output_dir) / "model_final.pt"
+        unwrapped_model = accelerator.unwrap_model(model)
+        torch.save(unwrapped_model.state_dict(), final_model_path)
+        log_message(f"Final model: {final_model_path}", cfg, accelerator)
+
+        # Save final routing weights
+        save_routing_weights(unwrapped_model, global_step, Path(cfg.paths.output_dir))
+
+    accelerator.wait_for_everyone()
+    accelerator.end_training()
+    finish_tracking()
+
+
+if __name__ == "__main__":
+    main()

routing_rerun/N_4.0/wandb/run-20260524_000855-j30yot8x/files/config.yaml

@@ -0,0 +1,134 @@
+_wandb:
+    value:
+        cli_version: 0.24.0
+        code_path: code/routing_evolution_exp/train.py
+        e:
+            z2zlg43ve4muflqdo00pdy9eowuzib74:
+                args:
+                    - training.load_balancing_N=4.0
+                    - tracking.run_name=routing_N4.0
+                codePath: routing_evolution_exp/train.py
+                codePathLocal: train.py
+                cpu_count: 36
+                cpu_count_logical: 72
+                cudaVersion: "13.2"
+                disk:
+                    /:
+                        total: "234075717632"
+                        used: "19258163200"
+                email: nikita@local.ru
+                executable: /venv/bytellm/bin/python
+                git:
+                    commit: 379d14a9668cada0997ccd4dfc2cc9ecb683c623
+                    remote: https://github.com/naryst/byte-llms-code.git
+                gpu: NVIDIA A100 80GB PCIe
+                gpu_count: 1
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100 80GB PCIe
+                      uuid: GPU-ec9764c4-224a-9508-002f-de7762d78498
+                host: 1a023b226280
+                memory:
+                    total: "404274495488"
+                os: Linux-6.8.0-117-generic-x86_64-with-glibc2.39
+                program: /workspace/byte-llms-code/routing_evolution_exp/train.py
+                python: CPython 3.12.0
+                root: outputs/2026-05-24/N_4.0
+                startedAt: "2026-05-24T00:08:55.906418Z"
+                writerId: z2zlg43ve4muflqdo00pdy9eowuzib74
+        m: []
+        python_version: 3.12.0
+        t:
+            "1":
+                - 1
+                - 11
+                - 49
+                - 50
+                - 51
+                - 71
+                - 105
+            "2":
+                - 1
+                - 11
+                - 49
+                - 50
+                - 51
+                - 71
+                - 105
+            "3":
+                - 2
+                - 13
+                - 16
+                - 61
+            "4": 3.12.0
+            "5": 0.24.0
+            "6": 4.57.6
+            "12": 0.24.0
+            "13": linux-x86_64
+data:
+    value:
+        max_context_len: 4096
+        max_target_len: 256
+        max_train_samples: 150000
+        max_val_samples: null
+        num_workers: 0
+        path: /workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+        pin_memory: true
+device:
+    value: cuda
+logging:
+    value:
+        eval_interval: 1000
+        log_interval: 10
+        model_only_checkpoints: true
+        save_every_epoch: false
+        save_interval: 3000
+model:
+    value:
+        checkpoint_path: /workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt
+        config_path: /workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json
+paths:
+    value:
+        output_dir: outputs/2026-05-24/N_4.0
+seed:
+    value: 42
+tracking:
+    value:
+        backend: wandb
+        base_url: https://wandb.platun0v.ru
+        enabled: true
+        entity: null
+        local_dir: outputs/2026-05-24/N_4.0
+        project: routing-evolution
+        run_name: routing_N4.0
+training:
+    value:
+        batch_size: 8
+        betas:
+            - 0.9
+            - 0.95
+        decay_ratio: 0.2
+        epochs: 3
+        eps: 1e-08
+        eval_batch_size: 24
+        gradient_accumulation_steps: 4
+        load_balancing_N: 4
+        load_balancing_weight: 0.5
+        lr: 0.0001
+        lr_multiplier:
+            - 2
+            - 1.5
+            - 1
+        lr_scheduler: wsd
+        max_grad_norm: 1
+        max_steps: null
+        min_lr_ratio: 0.1
+        resume: false
+        resume_checkpoint: null
+        use_amp: true
+        warmup_model: true
+        warmup_ratio: 0.1
+        warmup_steps: 100
+        weight_decay: 0.1

routing_rerun/N_4.0/wandb/run-20260524_000855-j30yot8x/files/requirements.txt

@@ -0,0 +1,245 @@
+setuptools==78.1.1
+wheel==0.45.1
+pip==25.2
+webencodings==0.5.1
+triton==3.2.0
+pytz==2025.2
+pydub==0.25.1
+pure_eval==0.2.3
+ptyprocess==0.7.0
+nvidia-ml-py==13.590.48
+nvidia-cusparselt-cu12==0.6.2
+mpmath==1.3.0
+ipython-genutils==0.2.0
+fastjsonschema==2.21.2
+brotli==1.2.0
+antlr4-python3-runtime==4.9.3
+xxhash==3.6.0
+widgetsnbextension==4.0.14
+websocket-client==1.9.0
+webcolors==24.11.1
+wcwidth==0.2.14
+urllib3==2.5.0
+uri-template==1.3.0
+tzdata==2025.2
+typing_extensions==4.15.0
+types-python-dateutil==2.9.0.20251008
+traitlets==5.14.3
+tqdm==4.67.1
+tornado==6.5.2
+tomlkit==0.13.3
+tinycss2==1.4.0
+tabulate==0.9.0
+sympy==1.13.1
+soupsieve==2.8
+sniffio==1.3.1
+smmap==5.0.2
+six==1.17.0
+shellingham==1.5.4
+Send2Trash==1.8.3
+semantic-version==2.10.0
+safetensors==0.6.2
+rpds-py==0.27.1
+rfc3986-validator==0.1.1
+regex==2025.9.18
+pyzmq==27.1.0
+PyYAML==6.0.3
+python-multipart==0.0.22
+python-json-logger==4.0.0
+python-dotenv==1.2.1
+pyparsing==3.2.5
+PyJWT==2.8.0
+Pygments==2.19.2
+pycparser==2.23
+pyarrow==22.0.0
+psutil==7.1.0
+protobuf==6.33.4
+propcache==0.4.1
+prometheus_client==0.23.1
+portalocker==3.2.0
+platformdirs==4.5.0
+pillow==11.3.0
+pexpect==4.9.0
+pathspec==1.0.4
+parso==0.8.5
+pandocfilters==1.5.1
+packaging==25.0
+orjson==3.11.6
+opt_einsum==3.4.0
+nvidia-nvtx-cu12==12.4.127
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nccl-cu12==2.21.5
+nvidia-curand-cu12==10.3.5.147
+nvidia-cufile-cu12==1.13.1.3
+nvidia-cufft-cu12==11.2.1.3
+nvidia-cuda-runtime-cu12==12.4.127
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cublas-cu12==12.4.5.8
+numpy==2.3.3
+ninja==1.13.0
+networkx==3.5
+nest-asyncio==1.6.0
+narwhals==2.15.0
+mypy_extensions==1.1.0
+multidict==6.7.0
+mistune==3.1.4
+mdurl==0.1.2
+MarkupSafe==3.0.3
+lxml==6.0.2
+librt==0.8.0
+lark==1.3.0
+kiwisolver==1.4.9
+jupyterlab_widgets==3.0.15
+jupyterlab_pygments==0.3.0
+jsonpointer==3.0.0
+json5==0.12.1
+itsdangerous==2.2.0
+idna==3.10
+hf-xet==1.1.10
+h11==0.16.0
+groovy==0.1.2
+fsspec==2025.9.0
+frozenlist==1.8.0
+fqdn==1.5.1
+fonttools==4.60.1
+filelock==3.19.1
+ffmpy==1.0.0
+executing==2.2.1
+einops==0.8.1
+dill==0.4.0
+defusedxml==0.7.1
+decorator==5.2.1
+debugpy==1.8.17
+dacite==1.9.2
+cycler==0.12.1
+comm==0.2.3
+colorama==0.4.6
+click==8.3.1
+charset-normalizer==3.4.3
+certifi==2025.10.5
+bleach==6.2.0
+babel==2.17.0
+attrs==25.4.0
+async-lru==2.0.5
+asttokens==3.0.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+aiofiles==24.1.0
+yarl==1.22.0
+uvicorn==0.40.0
+typing-inspection==0.4.2
+terminado==0.18.1
+stack-data==0.6.3
+sentry-sdk==2.50.0
+scipy==1.17.0
+sacrebleu==2.6.0
+rfc3987-syntax==1.1.0
+rfc3339-validator==0.1.4
+requests==2.32.5
+reportlab==4.4.9
+referencing==0.36.2
+python-dateutil==2.9.0.post0
+pydantic_core==2.41.5
+prompt_toolkit==3.0.52
+plotly==6.5.2
+pathlib2==2.3.7.post1
+orderedmultidict==1.0.2
+optree==0.17.0
+omegaconf==2.3.0
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cudnn-cu12==9.1.0.70
+mypy==1.19.1
+multiprocess==0.70.16
+matplotlib-inline==0.1.7
+markdown-it-py==4.0.0
+jupyter_core==5.8.1
+Jinja2==3.1.6
+jedi==0.19.2
+ipython_pygments_lexers==1.1.1
+httpcore==1.0.9
+gitdb==4.0.12
+ftfy==6.3.1
+contourpy==1.3.3
+cffi==2.0.0
+beautifulsoup4==4.14.2
+anyio==4.11.0
+aiosignal==1.4.0
+starlette==0.50.0
+rich==14.2.0
+pydantic==2.12.5
+pandas==2.3.3
+nvidia-cusolver-cu12==11.6.1.9
+matplotlib==3.10.7
+jupyter_server_terminals==0.5.3
+jupyter_client==8.6.3
+jsonschema-specifications==2025.9.1
+ipython==9.6.0
+hydra-core==1.3.2
+huggingface-hub==0.35.3
+httpx==0.28.1
+GitPython==3.1.46
+furl==2.1.4
+cryptography==46.0.4
+arrow==1.3.0
+argon2-cffi-bindings==25.1.0
+aiohttp==3.13.1
+wandb==0.24.0
+typer==0.21.1
+torch==2.6.0
+tokenizers==0.22.1
+seaborn==0.13.2
+safehttpx==0.1.7
+jsonschema==4.25.1
+joypy==0.2.6
+isoduration==20.11.0
+ipywidgets==8.1.7
+ipykernel==6.30.1
+gradio_client==2.0.3
+fastapi==0.128.0
+Authlib==1.6.6
+argon2-cffi==25.1.0
+transformers==4.57.6
+nbformat==5.10.4
+mlstm_kernels==2.0.2
+jupyter-console==6.6.3
+gradio==6.5.1
+datasets==4.3.0
+clearml==1.16.4
+accelerate==1.10.1
+xlstm==2.0.4
+nbclient==0.10.2
+jupyter-events==0.12.0
+trackio==0.15.0
+nbconvert==7.16.6
+jupyter_server==2.17.0
+notebook_shim==0.2.4
+jupyterlab_server==2.27.3
+jupyter-lsp==2.3.0
+nbclassic==1.3.3
+jupyterlab==4.4.9
+notebook==7.4.7
+jupyter_contrib_core==0.4.2
+jupyter==1.1.1
+jupyter_nbextensions_configurator==0.6.4
+causal-conv1d==1.5.0.post8
+flash_attn==2.7.4.post1
+mamba-ssm==2.2.4
+hnet==0.0.1
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+inflect==7.3.1
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+more-itertools==10.3.0
+packaging==24.2
+platformdirs==4.2.2
+tomli==2.0.1
+typeguard==4.3.0
+typing_extensions==4.12.2
+wheel==0.45.1
+zipp==3.19.2

routing_rerun/N_4.0/wandb/run-20260524_000855-j30yot8x/files/wandb-metadata.json

@@ -0,0 +1,44 @@
+{
+  "os": "Linux-6.8.0-117-generic-x86_64-with-glibc2.39",
+  "python": "CPython 3.12.0",
+  "startedAt": "2026-05-24T00:08:55.906418Z",
+  "args": [
+    "training.load_balancing_N=4.0",
+    "tracking.run_name=routing_N4.0"
+  ],
+  "program": "/workspace/byte-llms-code/routing_evolution_exp/train.py",
+  "codePath": "routing_evolution_exp/train.py",
+  "codePathLocal": "train.py",
+  "git": {
+    "remote": "https://github.com/naryst/byte-llms-code.git",
+    "commit": "379d14a9668cada0997ccd4dfc2cc9ecb683c623"
+  },
+  "email": "nikita@local.ru",
+  "root": "outputs/2026-05-24/N_4.0",
+  "host": "1a023b226280",
+  "executable": "/venv/bytellm/bin/python",
+  "cpu_count": 36,
+  "cpu_count_logical": 72,
+  "gpu": "NVIDIA A100 80GB PCIe",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "234075717632",
+      "used": "19258163200"
+    }
+  },
+  "memory": {
+    "total": "404274495488"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100 80GB PCIe",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-ec9764c4-224a-9508-002f-de7762d78498"
+    }
+  ],
+  "cudaVersion": "13.2",
+  "writerId": "z2zlg43ve4muflqdo00pdy9eowuzib74"
+}

routing_rerun/N_4.0/wandb/run-20260524_000855-j30yot8x/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/chunk_len_stage1":3.7689440605218008,"epoch/time":6862.479820966721,"epoch/chunk_len_stage1":3.768936003925935,"train/loss_avg":0.6704201395001311,"epoch/lb_loss":1.0050465959919677,"val/loss":0.8624915001269956,"epoch/hard_boundary_ratio_stage1":0.2669156306836737,"train/lb_loss":1.005639910697937,"train/step_time":1.1793010473251342,"train/loss":0.7162025570869446,"epoch/hard_boundary_ratio_stage0":0.3051249442133256,"_wandb":{"runtime":20591},"train/lm_loss":0.2354137897491455,"_timestamp":1.7796019244511673e+09,"epoch/chunk_len_stage0":3.2934883223119016,"_runtime":20591,"train/chunk_len_stage0":3.293471209458221,"train/lr":1.0000112334056358e-05,"epoch/lm_loss":0.16870175194540082,"epoch/soft_boundary_ratio_stage1":0.2276137836633942,"train/soft_boundary_ratio_stage0":0.27827713203132676,"val/lb_loss":1.003298749773718,"best/val_loss":0.8111355487821693,"epoch/soft_boundary_ratio_stage0":0.2782757019771054,"train/soft_boundary_ratio_stage1":0.22761410650356756,"train/epoch":3,"val/time":310.03714394569397,"train/hard_boundary_ratio_stage0":0.3051268035055855,"epoch/loss":0.670409958929631,"_step":14060,"train/hard_boundary_ratio_stage1":0.26691539681754184,"val/lm_loss":0.36084212524862774,"val/perplexity":1.435598828320708,"best/val_perplexity":1.3604376124872215,"best/step":4000}

routing_rerun/N_4.0/wandb/run-20260524_000855-j30yot8x/logs/debug-core.log

@@ -0,0 +1,16 @@
+{"time":"2026-05-24T00:08:56.021099149Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp4j1iv4zy/port-72724.txt","pid":72724,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-05-24T00:08:56.022013237Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":72724}
+{"time":"2026-05-24T00:08:56.022018469Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-72724-72749-3844324162/socket","Net":"unix"}}
+{"time":"2026-05-24T00:08:56.207582876Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-05-24T00:08:56.233767735Z","level":"INFO","msg":"handleInformInit: received","streamId":"j30yot8x","id":"1(@)"}
+{"time":"2026-05-24T00:08:56.701835812Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"j30yot8x","id":"1(@)"}
+{"time":"2026-05-24T05:52:10.06971293Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"j30yot8x","id":"1(@)"}
+{"time":"2026-05-24T05:52:10.071358673Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"j30yot8x","id":"1(@)"}
+{"time":"2026-05-24T05:52:10.093318012Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-05-24T05:52:10.093355414Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-05-24T05:52:10.09337123Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-05-24T05:52:10.093383808Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-05-24T05:52:10.093478832Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-72724-72749-3844324162/socket","Net":"unix"}}
+{"time":"2026-05-24T05:52:10.093492062Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-05-24T05:52:10.093546291Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-05-24T05:52:10.093563501Z","level":"INFO","msg":"server is closed"}

routing_rerun/N_4.0/wandb/run-20260524_000855-j30yot8x/logs/debug-internal.log

@@ -0,0 +1,14 @@
+{"time":"2026-05-24T00:08:56.234024021Z","level":"INFO","msg":"stream: starting","core version":"0.24.0"}
+{"time":"2026-05-24T00:08:56.701391294Z","level":"INFO","msg":"stream: created new stream","id":"j30yot8x"}
+{"time":"2026-05-24T00:08:56.701517641Z","level":"INFO","msg":"handler: started","stream_id":"j30yot8x"}
+{"time":"2026-05-24T00:08:56.701810168Z","level":"INFO","msg":"stream: started","id":"j30yot8x"}
+{"time":"2026-05-24T00:08:56.701847254Z","level":"INFO","msg":"sender: started","stream_id":"j30yot8x"}
+{"time":"2026-05-24T00:08:56.701851256Z","level":"INFO","msg":"writer: started","stream_id":"j30yot8x"}
+{"time":"2026-05-24T00:08:56.8332083Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2026-05-24T01:16:01.045615831Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://wandb.platun0v.ru/files/nikita/routing-evolution/j30yot8x/file_stream","body":"Bad Gateway"}
+{"time":"2026-05-24T05:52:09.949995651Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-05-24T05:52:10.063200014Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-05-24T05:52:10.069795312Z","level":"INFO","msg":"stream: closing","id":"j30yot8x"}
+{"time":"2026-05-24T05:52:10.069837927Z","level":"INFO","msg":"handler: closed","stream_id":"j30yot8x"}
+{"time":"2026-05-24T05:52:10.070129075Z","level":"INFO","msg":"sender: closed","stream_id":"j30yot8x"}
+{"time":"2026-05-24T05:52:10.070155803Z","level":"INFO","msg":"stream: closed","id":"j30yot8x"}

routing_rerun/N_6.0/.hydra/config.yaml

@@ -0,0 +1,57 @@
+model:
+  config_path: ${oc.env:PROJECT_ROOT}/hnet_project/configs/hnet_2stage_XL_code.json
+  checkpoint_path: ${oc.env:PROJECT_ROOT}/hnet_project/checkpoints/hnet_2stage_XL_code.pt
+training:
+  epochs: 3
+  max_steps: null
+  batch_size: 8
+  eval_batch_size: 24
+  gradient_accumulation_steps: 4
+  lr: 0.0001
+  weight_decay: 0.1
+  betas:
+  - 0.9
+  - 0.95
+  eps: 1.0e-08
+  lr_scheduler: wsd
+  warmup_ratio: 0.1
+  decay_ratio: 0.2
+  warmup_steps: 100
+  min_lr_ratio: 0.1
+  lr_multiplier:
+  - 2.0
+  - 1.5
+  - 1.0
+  load_balancing_weight: 0.5
+  load_balancing_N: 6.0
+  max_grad_norm: 1.0
+  use_amp: true
+  resume: false
+  resume_checkpoint: null
+  warmup_model: true
+data:
+  path: ${oc.env:PROJECT_ROOT}/code_completion_exp/datasets/data_V4_full
+  max_context_len: 4096
+  max_target_len: 256
+  num_workers: 0
+  pin_memory: true
+  max_train_samples: 150000
+  max_val_samples: null
+logging:
+  log_interval: 10
+  save_interval: 3000
+  eval_interval: 1000
+  save_every_epoch: false
+  model_only_checkpoints: true
+tracking:
+  enabled: true
+  backend: wandb
+  project: routing-evolution
+  run_name: routing_N6.0
+  entity: null
+  base_url: https://wandb.platun0v.ru
+  local_dir: ${paths.output_dir}
+paths:
+  output_dir: outputs/${now:%Y-%m-%d}/N_${training.load_balancing_N}
+seed: 42
+device: cuda

routing_rerun/N_6.0/.hydra/hydra.yaml

@@ -0,0 +1,162 @@
+hydra:
+  run:
+    dir: ${paths.output_dir}
+  sweep:
+    dir: outputs/multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - training.load_balancing_N=6.0
+    - tracking.run_name=routing_N6.0
+  job:
+    name: train
+    chdir: false
+    override_dirname: tracking.run_name=routing_N6.0,training.load_balancing_N=6.0
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /workspace/byte-llms-code/routing_evolution_exp
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /workspace/byte-llms-code/routing_evolution_exp/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /workspace/byte-llms-code/routing_evolution_exp/outputs/2026-05-24/N_6.0
+    choices:
+      paths: default
+      tracking: default
+      logging: default
+      data: default
+      training: default
+      model: hnet_xl_code
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

routing_rerun/N_6.0/.hydra/overrides.yaml

@@ -0,0 +1,2 @@
+- training.load_balancing_N=6.0
+- tracking.run_name=routing_N6.0

routing_rerun/N_6.0/checkpoints/checkpoint_latest.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9343a25637fd3c00c26704d7773c184e2c6fd1c695bb11d898f6484db6a4585
+size 3315169346

routing_rerun/N_6.0/checkpoints/checkpoint_step_12000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0994a16edf205e510663d239bbd034caab8d2850f01ed41311785361443dfc45
+size 3315170726

routing_rerun/N_6.0/checkpoints/checkpoint_step_3000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30b3ba319f811d546e404e6053d6fef6728d22b744e77b028d0cba49413defbf
+size 3315170381

routing_rerun/N_6.0/checkpoints/checkpoint_step_6000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22343bc626d98a030d846288d49e4f870af093244bf992be3004a31555077630
+size 3315170381

routing_rerun/N_6.0/checkpoints/checkpoint_step_9000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d243021548005fe67971d07b5f52e1f6183d289de40963d77bf5ab59adb80f9
+size 3315170381

routing_rerun/N_6.0/eval_results/eval_config.yaml

@@ -0,0 +1,29 @@
+model:
+  config_path: /workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json
+  checkpoint_path: /workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt
+data:
+  path: /workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+  max_context_len: 4096
+  max_target_len: 256
+  num_workers: 0
+  pin_memory: true
+  max_train_samples: 150000
+  max_val_samples: null
+evaluation:
+  batch_size: 16
+  max_samples: 2000
+  compute_bpb: true
+  bleu_tokenize: none
+  use_amp: true
+  save_predictions: true
+generation:
+  max_length: 256
+  temperature: 0.1
+  top_k: 0
+  top_p: 1.0
+paths:
+  run_dir: outputs/2026-05-24/N_6.0
+  eval_initial: true
+  eval_final: true
+seed: 42
+device: cuda

routing_rerun/N_6.0/eval_results/metrics_model_best.txt

@@ -0,0 +1,17 @@
+Checkpoint: model_best (outputs/2026-05-24/N_6.0/model_best.pt)
+================================================================================
+
+exact_match: 0.2945
+token_accuracy: 0.4274717137100246
+bleu: 13.724561822061911
+bpb: 1.3975490092372271
+num_samples: 2000
+gen_wall_time_s: 312.76023558113957
+gen_samples_per_s: 6.394674809870895
+gen_time_per_sample_ms: 156.38011779056978
+gen_chars_per_s: 172.20539529241827
+gen_batch_mean_ms: 2502.0818846491165
+gen_batch_p50_ms: 2298.0513839866035
+gen_batch_p95_ms: 4876.716210239101
+gen_batch_max_ms: 7006.153669033665
+gen_num_batches: 125

routing_rerun/N_6.0/eval_results/metrics_model_final.txt

@@ -0,0 +1,17 @@
+Checkpoint: model_final (outputs/2026-05-24/N_6.0/model_final.pt)
+================================================================================
+
+exact_match: 0.266
+token_accuracy: 0.405445433723113
+bleu: 12.143190186967047
+bpb: 1.4800909646911653
+num_samples: 2000
+gen_wall_time_s: 339.75642288831295
+gen_samples_per_s: 5.886570099242697
+gen_time_per_sample_ms: 169.87821144415648
+gen_chars_per_s: 169.45375016184988
+gen_batch_mean_ms: 2718.0513831065036
+gen_batch_p50_ms: 2349.4825870147906
+gen_batch_p95_ms: 5625.178886170033
+gen_batch_max_ms: 5939.189083001111
+gen_num_batches: 125

routing_rerun/N_6.0/eval_results/metrics_pretrained.txt

@@ -0,0 +1,17 @@
+Checkpoint: pretrained (/workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt)
+================================================================================
+
+exact_match: 0.0
+token_accuracy: 0.46647837842492296
+bleu: 1.0987264273338606
+bpb: 1.8771313252548079
+num_samples: 2000
+gen_wall_time_s: 1851.4914190099225
+gen_samples_per_s: 1.0802102453542517
+gen_time_per_sample_ms: 925.7457095049613
+gen_chars_per_s: 275.8138626821598
+gen_batch_mean_ms: 14811.93135207938
+gen_batch_p50_ms: 15092.08012802992
+gen_batch_p95_ms: 15535.188522632234
+gen_batch_max_ms: 36298.368007002864
+gen_num_batches: 125

routing_rerun/N_6.0/eval_results/summary.txt

@@ -0,0 +1,8 @@
+EVALUATION SUMMARY
+=============================================================================
+
+Checkpoint            Exact Match    Token Acc     BLEU        BPB  ms/sample
+-----------------------------------------------------------------------------
+pretrained                  0.00%       46.65%     1.10     1.8771      925.7
+model_best                 29.45%       42.75%    13.72     1.3975      156.4
+model_final                26.60%       40.54%    12.14     1.4801      169.9

routing_rerun/N_6.0/wandb/debug-internal.log

@@ -0,0 +1,14 @@
+{"time":"2026-05-24T05:52:22.647937006Z","level":"INFO","msg":"stream: starting","core version":"0.24.0"}
+{"time":"2026-05-24T05:52:23.102485815Z","level":"INFO","msg":"stream: created new stream","id":"tjovd8fs"}
+{"time":"2026-05-24T05:52:23.102632811Z","level":"INFO","msg":"handler: started","stream_id":"tjovd8fs"}
+{"time":"2026-05-24T05:52:23.102884167Z","level":"INFO","msg":"stream: started","id":"tjovd8fs"}
+{"time":"2026-05-24T05:52:23.102920051Z","level":"INFO","msg":"sender: started","stream_id":"tjovd8fs"}
+{"time":"2026-05-24T05:52:23.102924565Z","level":"INFO","msg":"writer: started","stream_id":"tjovd8fs"}
+{"time":"2026-05-24T05:52:23.243158587Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2026-05-24T06:57:38.830216521Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://wandb.platun0v.ru/files/nikita/routing-evolution/tjovd8fs/file_stream","body":"Bad Gateway"}
+{"time":"2026-05-24T11:16:57.561181762Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-05-24T11:16:57.667532298Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-05-24T11:16:57.673785539Z","level":"INFO","msg":"stream: closing","id":"tjovd8fs"}
+{"time":"2026-05-24T11:16:57.673813908Z","level":"INFO","msg":"handler: closed","stream_id":"tjovd8fs"}
+{"time":"2026-05-24T11:16:57.67401224Z","level":"INFO","msg":"sender: closed","stream_id":"tjovd8fs"}
+{"time":"2026-05-24T11:16:57.674029503Z","level":"INFO","msg":"stream: closed","id":"tjovd8fs"}

routing_rerun/N_6.0/wandb/debug.log

@@ -0,0 +1,24 @@
+2026-05-24 05:52:22,321 INFO    MainThread:73212 [wandb_setup.py:_flush():81] Current SDK version is 0.24.0
+2026-05-24 05:52:22,321 INFO    MainThread:73212 [wandb_setup.py:_flush():81] Configure stats pid to 73212
+2026-05-24 05:52:22,321 INFO    MainThread:73212 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-24 05:52:22,321 INFO    MainThread:73212 [wandb_init.py:setup_run_log_directory():717] Logging user logs to outputs/2026-05-24/N_6.0/wandb/run-20260524_055222-tjovd8fs/logs/debug.log
+2026-05-24 05:52:22,321 INFO    MainThread:73212 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to outputs/2026-05-24/N_6.0/wandb/run-20260524_055222-tjovd8fs/logs/debug-internal.log
+2026-05-24 05:52:22,321 INFO    MainThread:73212 [wandb_init.py:init():844] calling init triggers
+2026-05-24 05:52:22,321 INFO    MainThread:73212 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'model': {'config_path': '/workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json', 'checkpoint_path': '/workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt'}, 'training': {'epochs': 3, 'max_steps': None, 'batch_size': 8, 'eval_batch_size': 24, 'gradient_accumulation_steps': 4, 'lr': 0.0001, 'weight_decay': 0.1, 'betas': [0.9, 0.95], 'eps': 1e-08, 'lr_scheduler': 'wsd', 'warmup_ratio': 0.1, 'decay_ratio': 0.2, 'warmup_steps': 100, 'min_lr_ratio': 0.1, 'lr_multiplier': [2.0, 1.5, 1.0], 'load_balancing_weight': 0.5, 'load_balancing_N': 6.0, 'max_grad_norm': 1.0, 'use_amp': True, 'resume': False, 'resume_checkpoint': None, 'warmup_model': True}, 'data': {'path': '/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full', 'max_context_len': 4096, 'max_target_len': 256, 'num_workers': 0, 'pin_memory': True, 'max_train_samples': 150000, 'max_val_samples': None}, 'logging': {'log_interval': 10, 'save_interval': 3000, 'eval_interval': 1000, 'save_every_epoch': False, 'model_only_checkpoints': True}, 'tracking': {'enabled': True, 'backend': 'wandb', 'project': 'routing-evolution', 'run_name': 'routing_N6.0', 'entity': None, 'base_url': 'https://wandb.platun0v.ru', 'local_dir': 'outputs/2026-05-24/N_6.0'}, 'paths': {'output_dir': 'outputs/2026-05-24/N_6.0'}, 'seed': 42, 'device': 'cuda', '_wandb': {'code_path': 'code/routing_evolution_exp/train.py'}}
+2026-05-24 05:52:22,321 INFO    MainThread:73212 [wandb_init.py:init():892] starting backend
+2026-05-24 05:52:22,622 INFO    MainThread:73212 [wandb_init.py:init():895] sending inform_init request
+2026-05-24 05:52:22,644 INFO    MainThread:73212 [wandb_init.py:init():903] backend started and connected
+2026-05-24 05:52:22,650 INFO    MainThread:73212 [wandb_init.py:init():973] updated telemetry
+2026-05-24 05:52:22,682 INFO    MainThread:73212 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-05-24 05:52:23,241 INFO    MainThread:73212 [wandb_init.py:init():1044] starting run threads in backend
+2026-05-24 05:52:23,492 INFO    MainThread:73212 [wandb_run.py:_console_start():2529] atexit reg
+2026-05-24 05:52:23,492 INFO    MainThread:73212 [wandb_run.py:_redirect():2377] redirect: wrap_raw
+2026-05-24 05:52:23,492 INFO    MainThread:73212 [wandb_run.py:_redirect():2446] Wrapping output streams.
+2026-05-24 05:52:23,492 INFO    MainThread:73212 [wandb_run.py:_redirect():2469] Redirects installed.
+2026-05-24 05:52:23,497 INFO    MainThread:73212 [wandb_init.py:init():1084] run started, returning control to user process
+2026-05-24 11:16:56,449 INFO    MainThread:73212 [wandb_run.py:_finish():2295] finishing run nikita/routing-evolution/tjovd8fs
+2026-05-24 11:16:56,449 INFO    MainThread:73212 [wandb_run.py:_atexit_cleanup():2494] got exitcode: 0
+2026-05-24 11:16:56,449 INFO    MainThread:73212 [wandb_run.py:_restore():2476] restore
+2026-05-24 11:16:56,449 INFO    MainThread:73212 [wandb_run.py:_restore():2482] restore done
+2026-05-24 11:16:57,672 INFO    MainThread:73212 [wandb_run.py:_footer_sync_info():3870] logging synced files

routing_rerun/N_6.0/wandb/run-20260524_055222-tjovd8fs/files/code/routing_evolution_exp/train.py

@@ -0,0 +1,259 @@
+"""
+Routing Evolution Experiment: H-Net fine-tune с разными load_balancing_N.
+
+Цель: изучить как routing module (W_q, W_k) эволюционирует при разном давлении сжатия.
+
+Использование:
+    # Запуск с конкретным N
+    accelerate launch --config_file accelerate_config.yaml \
+        train.py training.load_balancing_N=6.0 tracking.run_name=routing_N6.0
+
+    # Запуск всех N через run_all.sh
+    bash run_all.sh
+"""
+
+import math
+import os
+from pathlib import Path
+
+import torch
+import hydra
+from hydra.core.hydra_config import HydraConfig
+from omegaconf import DictConfig, OmegaConf
+from accelerate import Accelerator
+from accelerate.utils import set_seed as accelerate_set_seed
+
+# HNet imports
+from hnet.load_utils import load_from_pretrained, load_from_config
+from hnet.utils.tokenizers import ByteTokenizer
+from hnet.utils.train import group_params
+
+# Ensure repo root is on sys.path (needed when running from subdirectory)
+import sys
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+# Shared training library
+from training_lib.utils import log_message
+from training_lib.checkpointing import save_checkpoint, load_checkpoint
+from training_lib.schedulers import get_lr_scheduler
+from training_lib.tracking import init_tracking, finish_tracking
+from training_lib.hnet.train_loop import train_epoch
+from training_lib.hnet.data import create_dataloaders
+
+
+def save_routing_weights(model, step, output_dir):
+    """Сохраняет только routing W_q/W_k для быстрого анализа weight evolution."""
+    routing_dir = output_dir / "routing_weights"
+    routing_dir.mkdir(parents=True, exist_ok=True)
+
+    state = model.state_dict()
+    routing_keys = [k for k in state if "routing_module" in k and "proj_layer" in k]
+    routing_weights = {k: state[k].cpu() for k in routing_keys}
+    torch.save(routing_weights, routing_dir / f"routing_step_{step}.pt")
+
+
+def on_checkpoint_callback(model, step, output_dir):
+    """Callback для train_epoch: сохраняет routing weights при каждом checkpoint.
+
+    Модель уже unwrapped в train_loop перед вызовом callback.
+    """
+    save_routing_weights(model, step, output_dir)
+
+
+@hydra.main(version_base=None, config_path="configs", config_name="config")
+def main(cfg: DictConfig):
+    """Тренировка H-Net с фиксированным числом шагов для анализа routing evolution."""
+
+    # === Accelerator Setup ===
+    mixed_precision = "bf16" if cfg.training.use_amp else "no"
+
+    accelerator = Accelerator(
+        mixed_precision=mixed_precision,
+        gradient_accumulation_steps=cfg.training.gradient_accumulation_steps,
+    )
+
+    accelerate_set_seed(cfg.seed)
+
+    if cfg.paths.output_dir is None:
+        cfg.paths.output_dir = HydraConfig.get().runtime.output_dir
+
+    OmegaConf.resolve(cfg)
+
+    log_message(
+        f"CUDA_VISIBLE_DEVICES: {os.environ.get('CUDA_VISIBLE_DEVICES', 'not set')}",
+        cfg,
+        accelerator,
+    )
+    log_message(f"Number of processes: {accelerator.num_processes}", cfg, accelerator)
+    log_message(f"Mixed precision: {mixed_precision}", cfg, accelerator)
+
+    log_message("=" * 60, cfg, accelerator)
+    log_message(
+        f"Routing Evolution Experiment | N={cfg.training.load_balancing_N}",
+        cfg,
+        accelerator,
+    )
+    log_message("=" * 60, cfg, accelerator)
+    log_message(f"Config:\n{OmegaConf.to_yaml(cfg)}", cfg, accelerator)
+
+    # === Experiment Tracking Init ===
+    init_tracking(cfg, accelerator)
+
+    # === Tokenizer ===
+    tokenizer = ByteTokenizer()
+
+    # === Model ===
+    log_message("Loading model...", cfg, accelerator)
+    if cfg.model.checkpoint_path:
+        model = load_from_pretrained(
+            model_path=cfg.model.checkpoint_path,
+            model_config_path=cfg.model.config_path,
+        )
+        log_message(f"Loaded pretrained: {cfg.model.checkpoint_path}", cfg, accelerator)
+    else:
+        model = load_from_config(
+            model_config_path=cfg.model.config_path,
+            device="cpu",
+        )
+        model.init_weights()
+        log_message("Initialized from scratch", cfg, accelerator)
+
+    model.train()
+
+    lr_multiplier = list(cfg.training.lr_multiplier)
+    model.apply_lr_multiplier(lr_multiplier)
+    log_message(f"Applied LR multipliers: {lr_multiplier}", cfg, accelerator)
+
+    if cfg.training.warmup_model:
+        log_message("Warming up model...", cfg, accelerator)
+        model = model.to(accelerator.device)
+        model.warmup(verbose=accelerator.is_main_process)
+
+    # Save initial routing weights (step 0)
+    if accelerator.is_main_process:
+        output_dir = Path(cfg.paths.output_dir)
+        save_routing_weights(model, 0, output_dir)
+
+    total_params = sum(p.numel() for p in model.parameters())
+    trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
+    log_message(f"Total params: {total_params:,}", cfg, accelerator)
+    log_message(f"Trainable params: {trainable_params:,}", cfg, accelerator)
+
+    # === Data ===
+    log_message("Creating dataloaders...", cfg, accelerator)
+    dataloaders = create_dataloaders(cfg, tokenizer)
+
+    train_dataloader = dataloaders["train"]
+    val_dataloader = dataloaders.get("validation", None)
+
+    log_message(
+        f"Train dataset size: {len(train_dataloader.dataset)} "
+        f"(max_train_samples={cfg.data.max_train_samples}) | "
+        f"Epochs: {cfg.training.epochs}",
+        cfg,
+        accelerator,
+    )
+
+    # === Optimizer ===
+    param_groups = group_params(model)
+
+    for group in param_groups:
+        if "lr" not in group:
+            group["lr"] = cfg.training.lr
+        else:
+            group["lr"] = cfg.training.lr * group.get("lr_multiplier", 1.0)
+        if "weight_decay" not in group:
+            group["weight_decay"] = cfg.training.weight_decay
+
+    optimizer = torch.optim.AdamW(
+        param_groups,
+        lr=cfg.training.lr,
+        betas=tuple(cfg.training.betas),
+        eps=cfg.training.eps,
+    )
+
+    # === Scheduler ===
+    steps_per_epoch = math.ceil(len(train_dataloader) / accelerator.num_processes)
+    max_steps = (
+        cfg.training.epochs
+        * steps_per_epoch
+        // cfg.training.gradient_accumulation_steps
+    )
+    scheduler = get_lr_scheduler(optimizer, cfg, max_steps)
+
+    log_message(
+        f"Max steps: {max_steps}, Steps per epoch: {steps_per_epoch}",
+        cfg,
+        accelerator,
+    )
+
+    # === Accelerate Prepare ===
+    if val_dataloader is not None:
+        model, optimizer, train_dataloader, val_dataloader, scheduler = (
+            accelerator.prepare(
+                model, optimizer, train_dataloader, val_dataloader, scheduler
+            )
+        )
+    else:
+        model, optimizer, train_dataloader, scheduler = accelerator.prepare(
+            model, optimizer, train_dataloader, scheduler
+        )
+
+    # === Training Loop ===
+    log_message("Starting training...", cfg, accelerator)
+
+    global_step = 0
+    best_val_loss = float("inf")
+    epoch = 0
+
+    num_epochs = cfg.training.epochs
+
+    try:
+        for epoch in range(1, num_epochs + 1):
+            log_message(f"\n{'=' * 60}", cfg, accelerator)
+            log_message(
+                f"EPOCH {epoch}/{num_epochs} (step {global_step})", cfg, accelerator
+            )
+            log_message(f"{'=' * 60}", cfg, accelerator)
+
+            global_step, best_val_loss = train_epoch(
+                model=model,
+                dataloader=train_dataloader,
+                optimizer=optimizer,
+                scheduler=scheduler,
+                cfg=cfg,
+                epoch=epoch,
+                global_step=global_step,
+                accelerator=accelerator,
+                val_dataloader=val_dataloader,
+                best_val_loss=best_val_loss,
+                max_steps=max_steps,
+                on_checkpoint=on_checkpoint_callback,
+            )
+
+    except KeyboardInterrupt:
+        log_message("Training interrupted by user", cfg, accelerator)
+        save_checkpoint(
+            model, optimizer, scheduler, global_step, epoch, cfg, accelerator,
+            model_only=True,
+        )
+
+    # === Final Save ===
+    log_message("\nTraining completed!", cfg, accelerator)
+
+    if accelerator.is_main_process:
+        final_model_path = Path(cfg.paths.output_dir) / "model_final.pt"
+        unwrapped_model = accelerator.unwrap_model(model)
+        torch.save(unwrapped_model.state_dict(), final_model_path)
+        log_message(f"Final model: {final_model_path}", cfg, accelerator)
+
+        # Save final routing weights
+        save_routing_weights(unwrapped_model, global_step, Path(cfg.paths.output_dir))
+
+    accelerator.wait_for_everyone()
+    accelerator.end_training()
+    finish_tracking()
+
+
+if __name__ == "__main__":
+    main()

routing_rerun/N_6.0/wandb/run-20260524_055222-tjovd8fs/files/config.yaml

@@ -0,0 +1,134 @@
+_wandb:
+    value:
+        cli_version: 0.24.0
+        code_path: code/routing_evolution_exp/train.py
+        e:
+            qnhiyfqvhqw9ifnmw9ecehnx1sss1ocb:
+                args:
+                    - training.load_balancing_N=6.0
+                    - tracking.run_name=routing_N6.0
+                codePath: routing_evolution_exp/train.py
+                codePathLocal: train.py
+                cpu_count: 36
+                cpu_count_logical: 72
+                cudaVersion: "13.2"
+                disk:
+                    /:
+                        total: "234075717632"
+                        used: "19262914560"
+                email: nikita@local.ru
+                executable: /venv/bytellm/bin/python
+                git:
+                    commit: 379d14a9668cada0997ccd4dfc2cc9ecb683c623
+                    remote: https://github.com/naryst/byte-llms-code.git
+                gpu: NVIDIA A100 80GB PCIe
+                gpu_count: 1
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100 80GB PCIe
+                      uuid: GPU-ec9764c4-224a-9508-002f-de7762d78498
+                host: 1a023b226280
+                memory:
+                    total: "404274495488"
+                os: Linux-6.8.0-117-generic-x86_64-with-glibc2.39
+                program: /workspace/byte-llms-code/routing_evolution_exp/train.py
+                python: CPython 3.12.0
+                root: outputs/2026-05-24/N_6.0
+                startedAt: "2026-05-24T05:52:22.319248Z"
+                writerId: qnhiyfqvhqw9ifnmw9ecehnx1sss1ocb
+        m: []
+        python_version: 3.12.0
+        t:
+            "1":
+                - 1
+                - 11
+                - 49
+                - 50
+                - 51
+                - 71
+                - 105
+            "2":
+                - 1
+                - 11
+                - 49
+                - 50
+                - 51
+                - 71
+                - 105
+            "3":
+                - 2
+                - 13
+                - 16
+                - 61
+            "4": 3.12.0
+            "5": 0.24.0
+            "6": 4.57.6
+            "12": 0.24.0
+            "13": linux-x86_64
+data:
+    value:
+        max_context_len: 4096
+        max_target_len: 256
+        max_train_samples: 150000
+        max_val_samples: null
+        num_workers: 0
+        path: /workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+        pin_memory: true
+device:
+    value: cuda
+logging:
+    value:
+        eval_interval: 1000
+        log_interval: 10
+        model_only_checkpoints: true
+        save_every_epoch: false
+        save_interval: 3000
+model:
+    value:
+        checkpoint_path: /workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt
+        config_path: /workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json
+paths:
+    value:
+        output_dir: outputs/2026-05-24/N_6.0
+seed:
+    value: 42
+tracking:
+    value:
+        backend: wandb
+        base_url: https://wandb.platun0v.ru
+        enabled: true
+        entity: null
+        local_dir: outputs/2026-05-24/N_6.0
+        project: routing-evolution
+        run_name: routing_N6.0
+training:
+    value:
+        batch_size: 8
+        betas:
+            - 0.9
+            - 0.95
+        decay_ratio: 0.2
+        epochs: 3
+        eps: 1e-08
+        eval_batch_size: 24
+        gradient_accumulation_steps: 4
+        load_balancing_N: 6
+        load_balancing_weight: 0.5
+        lr: 0.0001
+        lr_multiplier:
+            - 2
+            - 1.5
+            - 1
+        lr_scheduler: wsd
+        max_grad_norm: 1
+        max_steps: null
+        min_lr_ratio: 0.1
+        resume: false
+        resume_checkpoint: null
+        use_amp: true
+        warmup_model: true
+        warmup_ratio: 0.1
+        warmup_steps: 100
+        weight_decay: 0.1

routing_rerun/N_6.0/wandb/run-20260524_055222-tjovd8fs/files/requirements.txt

@@ -0,0 +1,245 @@
+setuptools==78.1.1
+wheel==0.45.1
+pip==25.2
+webencodings==0.5.1
+triton==3.2.0
+pytz==2025.2
+pydub==0.25.1
+pure_eval==0.2.3
+ptyprocess==0.7.0
+nvidia-ml-py==13.590.48
+nvidia-cusparselt-cu12==0.6.2
+mpmath==1.3.0
+ipython-genutils==0.2.0
+fastjsonschema==2.21.2
+brotli==1.2.0
+antlr4-python3-runtime==4.9.3
+xxhash==3.6.0
+widgetsnbextension==4.0.14
+websocket-client==1.9.0
+webcolors==24.11.1
+wcwidth==0.2.14
+urllib3==2.5.0
+uri-template==1.3.0
+tzdata==2025.2
+typing_extensions==4.15.0
+types-python-dateutil==2.9.0.20251008
+traitlets==5.14.3
+tqdm==4.67.1
+tornado==6.5.2
+tomlkit==0.13.3
+tinycss2==1.4.0
+tabulate==0.9.0
+sympy==1.13.1
+soupsieve==2.8
+sniffio==1.3.1
+smmap==5.0.2
+six==1.17.0
+shellingham==1.5.4
+Send2Trash==1.8.3
+semantic-version==2.10.0
+safetensors==0.6.2
+rpds-py==0.27.1
+rfc3986-validator==0.1.1
+regex==2025.9.18
+pyzmq==27.1.0
+PyYAML==6.0.3
+python-multipart==0.0.22
+python-json-logger==4.0.0
+python-dotenv==1.2.1
+pyparsing==3.2.5
+PyJWT==2.8.0
+Pygments==2.19.2
+pycparser==2.23
+pyarrow==22.0.0
+psutil==7.1.0
+protobuf==6.33.4
+propcache==0.4.1
+prometheus_client==0.23.1
+portalocker==3.2.0
+platformdirs==4.5.0
+pillow==11.3.0
+pexpect==4.9.0
+pathspec==1.0.4
+parso==0.8.5
+pandocfilters==1.5.1
+packaging==25.0
+orjson==3.11.6
+opt_einsum==3.4.0
+nvidia-nvtx-cu12==12.4.127
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nccl-cu12==2.21.5
+nvidia-curand-cu12==10.3.5.147
+nvidia-cufile-cu12==1.13.1.3
+nvidia-cufft-cu12==11.2.1.3
+nvidia-cuda-runtime-cu12==12.4.127
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cublas-cu12==12.4.5.8
+numpy==2.3.3
+ninja==1.13.0
+networkx==3.5
+nest-asyncio==1.6.0
+narwhals==2.15.0
+mypy_extensions==1.1.0
+multidict==6.7.0
+mistune==3.1.4
+mdurl==0.1.2
+MarkupSafe==3.0.3
+lxml==6.0.2
+librt==0.8.0
+lark==1.3.0
+kiwisolver==1.4.9
+jupyterlab_widgets==3.0.15
+jupyterlab_pygments==0.3.0
+jsonpointer==3.0.0
+json5==0.12.1
+itsdangerous==2.2.0
+idna==3.10
+hf-xet==1.1.10
+h11==0.16.0
+groovy==0.1.2
+fsspec==2025.9.0
+frozenlist==1.8.0
+fqdn==1.5.1
+fonttools==4.60.1
+filelock==3.19.1
+ffmpy==1.0.0
+executing==2.2.1
+einops==0.8.1
+dill==0.4.0
+defusedxml==0.7.1
+decorator==5.2.1
+debugpy==1.8.17
+dacite==1.9.2
+cycler==0.12.1
+comm==0.2.3
+colorama==0.4.6
+click==8.3.1
+charset-normalizer==3.4.3
+certifi==2025.10.5
+bleach==6.2.0
+babel==2.17.0
+attrs==25.4.0
+async-lru==2.0.5
+asttokens==3.0.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+aiofiles==24.1.0
+yarl==1.22.0
+uvicorn==0.40.0
+typing-inspection==0.4.2
+terminado==0.18.1
+stack-data==0.6.3
+sentry-sdk==2.50.0
+scipy==1.17.0
+sacrebleu==2.6.0
+rfc3987-syntax==1.1.0
+rfc3339-validator==0.1.4
+requests==2.32.5
+reportlab==4.4.9
+referencing==0.36.2
+python-dateutil==2.9.0.post0
+pydantic_core==2.41.5
+prompt_toolkit==3.0.52
+plotly==6.5.2
+pathlib2==2.3.7.post1
+orderedmultidict==1.0.2
+optree==0.17.0
+omegaconf==2.3.0
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cudnn-cu12==9.1.0.70
+mypy==1.19.1
+multiprocess==0.70.16
+matplotlib-inline==0.1.7
+markdown-it-py==4.0.0
+jupyter_core==5.8.1
+Jinja2==3.1.6
+jedi==0.19.2
+ipython_pygments_lexers==1.1.1
+httpcore==1.0.9
+gitdb==4.0.12
+ftfy==6.3.1
+contourpy==1.3.3
+cffi==2.0.0
+beautifulsoup4==4.14.2
+anyio==4.11.0
+aiosignal==1.4.0
+starlette==0.50.0
+rich==14.2.0
+pydantic==2.12.5
+pandas==2.3.3
+nvidia-cusolver-cu12==11.6.1.9
+matplotlib==3.10.7
+jupyter_server_terminals==0.5.3
+jupyter_client==8.6.3
+jsonschema-specifications==2025.9.1
+ipython==9.6.0
+hydra-core==1.3.2
+huggingface-hub==0.35.3
+httpx==0.28.1
+GitPython==3.1.46
+furl==2.1.4
+cryptography==46.0.4
+arrow==1.3.0
+argon2-cffi-bindings==25.1.0
+aiohttp==3.13.1
+wandb==0.24.0
+typer==0.21.1
+torch==2.6.0
+tokenizers==0.22.1
+seaborn==0.13.2
+safehttpx==0.1.7
+jsonschema==4.25.1
+joypy==0.2.6
+isoduration==20.11.0
+ipywidgets==8.1.7
+ipykernel==6.30.1
+gradio_client==2.0.3
+fastapi==0.128.0
+Authlib==1.6.6
+argon2-cffi==25.1.0
+transformers==4.57.6
+nbformat==5.10.4
+mlstm_kernels==2.0.2
+jupyter-console==6.6.3
+gradio==6.5.1
+datasets==4.3.0
+clearml==1.16.4
+accelerate==1.10.1
+xlstm==2.0.4
+nbclient==0.10.2
+jupyter-events==0.12.0
+trackio==0.15.0
+nbconvert==7.16.6
+jupyter_server==2.17.0
+notebook_shim==0.2.4
+jupyterlab_server==2.27.3
+jupyter-lsp==2.3.0
+nbclassic==1.3.3
+jupyterlab==4.4.9
+notebook==7.4.7
+jupyter_contrib_core==0.4.2
+jupyter==1.1.1
+jupyter_nbextensions_configurator==0.6.4
+causal-conv1d==1.5.0.post8
+flash_attn==2.7.4.post1
+mamba-ssm==2.2.4
+hnet==0.0.1
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+inflect==7.3.1
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+more-itertools==10.3.0
+packaging==24.2
+platformdirs==4.2.2
+tomli==2.0.1
+typeguard==4.3.0
+typing_extensions==4.12.2
+wheel==0.45.1
+zipp==3.19.2

routing_rerun/N_6.0/wandb/run-20260524_055222-tjovd8fs/files/wandb-metadata.json

@@ -0,0 +1,44 @@
+{
+  "os": "Linux-6.8.0-117-generic-x86_64-with-glibc2.39",
+  "python": "CPython 3.12.0",
+  "startedAt": "2026-05-24T05:52:22.319248Z",
+  "args": [
+    "training.load_balancing_N=6.0",
+    "tracking.run_name=routing_N6.0"
+  ],
+  "program": "/workspace/byte-llms-code/routing_evolution_exp/train.py",
+  "codePath": "routing_evolution_exp/train.py",
+  "codePathLocal": "train.py",
+  "git": {
+    "remote": "https://github.com/naryst/byte-llms-code.git",
+    "commit": "379d14a9668cada0997ccd4dfc2cc9ecb683c623"
+  },
+  "email": "nikita@local.ru",
+  "root": "outputs/2026-05-24/N_6.0",
+  "host": "1a023b226280",
+  "executable": "/venv/bytellm/bin/python",
+  "cpu_count": 36,
+  "cpu_count_logical": 72,
+  "gpu": "NVIDIA A100 80GB PCIe",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "234075717632",
+      "used": "19262914560"
+    }
+  },
+  "memory": {
+    "total": "404274495488"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100 80GB PCIe",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-ec9764c4-224a-9508-002f-de7762d78498"
+    }
+  ],
+  "cudaVersion": "13.2",
+  "writerId": "qnhiyfqvhqw9ifnmw9ecehnx1sss1ocb"
+}

routing_rerun/N_6.0/wandb/run-20260524_055222-tjovd8fs/files/wandb-summary.json

@@ -0,0 +1 @@
+{"epoch/lb_loss":1.016017396045133,"epoch/soft_boundary_ratio_stage1":0.16652716106240512,"train/lr":1.0000112334056358e-05,"epoch/chunk_len_stage0":4.092642846045787,"_timestamp":1.7796214120613477e+09,"epoch/chunk_len_stage1":5.219640202211922,"best/val_loss":0.8618771416678089,"train/soft_boundary_ratio_stage0":0.21564617116651916,"train/lm_loss":0.3704099953174591,"epoch/soft_boundary_ratio_stage0":0.21564443765435157,"val/lm_loss":0.3920209985206226,"val/loss":0.8986752326315003,"_runtime":19473,"train/soft_boundary_ratio_stage1":0.1665283167775648,"val/perplexity":1.4808691752566727,"train/epoch":3,"epoch/hard_boundary_ratio_stage0":0.24555311473595748,"train/chunk_len_stage1":5.219573927960947,"train/chunk_len_stage0":4.0926066759484145,"train/loss_avg":0.7787240958505088,"epoch/lm_loss":0.2719557210024115,"val/lb_loss":1.0133084676307698,"train/lb_loss":1.0087056159973145,"best/val_perplexity":1.4006408556239227,"_step":14060,"train/hard_boundary_ratio_stage1":0.19387243849367575,"train/step_time":1.1521595954895019,"best/step":3000,"epoch/time":6433.849749088287,"epoch/loss":0.7787093779814124,"train/loss":0.8548774272203445,"epoch/hard_boundary_ratio_stage1":0.19386965479498253,"val/time":259.69529390335083,"_wandb":{"runtime":19473},"train/hard_boundary_ratio_stage0":0.24555544699565662}

routing_rerun/N_6.0/wandb/run-20260524_055222-tjovd8fs/logs/debug-core.log

@@ -0,0 +1,16 @@
+{"time":"2026-05-24T05:52:22.436010727Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpfkqmajrf/port-73212.txt","pid":73212,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-05-24T05:52:22.43670458Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":73212}
+{"time":"2026-05-24T05:52:22.43671213Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-73212-73222-3923105823/socket","Net":"unix"}}
+{"time":"2026-05-24T05:52:22.62222821Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-05-24T05:52:22.647727658Z","level":"INFO","msg":"handleInformInit: received","streamId":"tjovd8fs","id":"1(@)"}
+{"time":"2026-05-24T05:52:23.102908806Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"tjovd8fs","id":"1(@)"}
+{"time":"2026-05-24T11:16:57.673719508Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"tjovd8fs","id":"1(@)"}
+{"time":"2026-05-24T11:16:57.691122109Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"tjovd8fs","id":"1(@)"}
+{"time":"2026-05-24T11:16:57.694742298Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-05-24T11:16:57.694797689Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-05-24T11:16:57.694815699Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-05-24T11:16:57.694817168Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-05-24T11:16:57.694913326Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-05-24T11:16:57.69492678Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-05-24T11:16:57.694957606Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-73212-73222-3923105823/socket","Net":"unix"}}
+{"time":"2026-05-24T11:16:57.69501297Z","level":"INFO","msg":"server is closed"}

routing_rerun/N_6.0/wandb/run-20260524_055222-tjovd8fs/logs/debug-internal.log

@@ -0,0 +1,14 @@
+{"time":"2026-05-24T05:52:22.647937006Z","level":"INFO","msg":"stream: starting","core version":"0.24.0"}
+{"time":"2026-05-24T05:52:23.102485815Z","level":"INFO","msg":"stream: created new stream","id":"tjovd8fs"}
+{"time":"2026-05-24T05:52:23.102632811Z","level":"INFO","msg":"handler: started","stream_id":"tjovd8fs"}
+{"time":"2026-05-24T05:52:23.102884167Z","level":"INFO","msg":"stream: started","id":"tjovd8fs"}
+{"time":"2026-05-24T05:52:23.102920051Z","level":"INFO","msg":"sender: started","stream_id":"tjovd8fs"}
+{"time":"2026-05-24T05:52:23.102924565Z","level":"INFO","msg":"writer: started","stream_id":"tjovd8fs"}
+{"time":"2026-05-24T05:52:23.243158587Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2026-05-24T06:57:38.830216521Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://wandb.platun0v.ru/files/nikita/routing-evolution/tjovd8fs/file_stream","body":"Bad Gateway"}
+{"time":"2026-05-24T11:16:57.561181762Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-05-24T11:16:57.667532298Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-05-24T11:16:57.673785539Z","level":"INFO","msg":"stream: closing","id":"tjovd8fs"}
+{"time":"2026-05-24T11:16:57.673813908Z","level":"INFO","msg":"handler: closed","stream_id":"tjovd8fs"}
+{"time":"2026-05-24T11:16:57.67401224Z","level":"INFO","msg":"sender: closed","stream_id":"tjovd8fs"}
+{"time":"2026-05-24T11:16:57.674029503Z","level":"INFO","msg":"stream: closed","id":"tjovd8fs"}

routing_rerun/N_6.0/wandb/run-20260524_055222-tjovd8fs/logs/debug.log

@@ -0,0 +1,24 @@
+2026-05-24 05:52:22,321 INFO    MainThread:73212 [wandb_setup.py:_flush():81] Current SDK version is 0.24.0
+2026-05-24 05:52:22,321 INFO    MainThread:73212 [wandb_setup.py:_flush():81] Configure stats pid to 73212
+2026-05-24 05:52:22,321 INFO    MainThread:73212 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-24 05:52:22,321 INFO    MainThread:73212 [wandb_init.py:setup_run_log_directory():717] Logging user logs to outputs/2026-05-24/N_6.0/wandb/run-20260524_055222-tjovd8fs/logs/debug.log
+2026-05-24 05:52:22,321 INFO    MainThread:73212 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to outputs/2026-05-24/N_6.0/wandb/run-20260524_055222-tjovd8fs/logs/debug-internal.log
+2026-05-24 05:52:22,321 INFO    MainThread:73212 [wandb_init.py:init():844] calling init triggers
+2026-05-24 05:52:22,321 INFO    MainThread:73212 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'model': {'config_path': '/workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json', 'checkpoint_path': '/workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt'}, 'training': {'epochs': 3, 'max_steps': None, 'batch_size': 8, 'eval_batch_size': 24, 'gradient_accumulation_steps': 4, 'lr': 0.0001, 'weight_decay': 0.1, 'betas': [0.9, 0.95], 'eps': 1e-08, 'lr_scheduler': 'wsd', 'warmup_ratio': 0.1, 'decay_ratio': 0.2, 'warmup_steps': 100, 'min_lr_ratio': 0.1, 'lr_multiplier': [2.0, 1.5, 1.0], 'load_balancing_weight': 0.5, 'load_balancing_N': 6.0, 'max_grad_norm': 1.0, 'use_amp': True, 'resume': False, 'resume_checkpoint': None, 'warmup_model': True}, 'data': {'path': '/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full', 'max_context_len': 4096, 'max_target_len': 256, 'num_workers': 0, 'pin_memory': True, 'max_train_samples': 150000, 'max_val_samples': None}, 'logging': {'log_interval': 10, 'save_interval': 3000, 'eval_interval': 1000, 'save_every_epoch': False, 'model_only_checkpoints': True}, 'tracking': {'enabled': True, 'backend': 'wandb', 'project': 'routing-evolution', 'run_name': 'routing_N6.0', 'entity': None, 'base_url': 'https://wandb.platun0v.ru', 'local_dir': 'outputs/2026-05-24/N_6.0'}, 'paths': {'output_dir': 'outputs/2026-05-24/N_6.0'}, 'seed': 42, 'device': 'cuda', '_wandb': {'code_path': 'code/routing_evolution_exp/train.py'}}
+2026-05-24 05:52:22,321 INFO    MainThread:73212 [wandb_init.py:init():892] starting backend
+2026-05-24 05:52:22,622 INFO    MainThread:73212 [wandb_init.py:init():895] sending inform_init request
+2026-05-24 05:52:22,644 INFO    MainThread:73212 [wandb_init.py:init():903] backend started and connected
+2026-05-24 05:52:22,650 INFO    MainThread:73212 [wandb_init.py:init():973] updated telemetry
+2026-05-24 05:52:22,682 INFO    MainThread:73212 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-05-24 05:52:23,241 INFO    MainThread:73212 [wandb_init.py:init():1044] starting run threads in backend
+2026-05-24 05:52:23,492 INFO    MainThread:73212 [wandb_run.py:_console_start():2529] atexit reg
+2026-05-24 05:52:23,492 INFO    MainThread:73212 [wandb_run.py:_redirect():2377] redirect: wrap_raw
+2026-05-24 05:52:23,492 INFO    MainThread:73212 [wandb_run.py:_redirect():2446] Wrapping output streams.
+2026-05-24 05:52:23,492 INFO    MainThread:73212 [wandb_run.py:_redirect():2469] Redirects installed.
+2026-05-24 05:52:23,497 INFO    MainThread:73212 [wandb_init.py:init():1084] run started, returning control to user process
+2026-05-24 11:16:56,449 INFO    MainThread:73212 [wandb_run.py:_finish():2295] finishing run nikita/routing-evolution/tjovd8fs
+2026-05-24 11:16:56,449 INFO    MainThread:73212 [wandb_run.py:_atexit_cleanup():2494] got exitcode: 0
+2026-05-24 11:16:56,449 INFO    MainThread:73212 [wandb_run.py:_restore():2476] restore
+2026-05-24 11:16:56,449 INFO    MainThread:73212 [wandb_run.py:_restore():2482] restore done
+2026-05-24 11:16:57,672 INFO    MainThread:73212 [wandb_run.py:_footer_sync_info():3870] logging synced files

routing_rerun/N_8.0/.hydra/config.yaml

@@ -0,0 +1,57 @@
+model:
+  config_path: ${oc.env:PROJECT_ROOT}/hnet_project/configs/hnet_2stage_XL_code.json
+  checkpoint_path: ${oc.env:PROJECT_ROOT}/hnet_project/checkpoints/hnet_2stage_XL_code.pt
+training:
+  epochs: 3
+  max_steps: null
+  batch_size: 8
+  eval_batch_size: 24
+  gradient_accumulation_steps: 4
+  lr: 0.0001
+  weight_decay: 0.1
+  betas:
+  - 0.9
+  - 0.95
+  eps: 1.0e-08
+  lr_scheduler: wsd
+  warmup_ratio: 0.1
+  decay_ratio: 0.2
+  warmup_steps: 100
+  min_lr_ratio: 0.1
+  lr_multiplier:
+  - 2.0
+  - 1.5
+  - 1.0
+  load_balancing_weight: 0.5
+  load_balancing_N: 8.0
+  max_grad_norm: 1.0
+  use_amp: true
+  resume: false
+  resume_checkpoint: null
+  warmup_model: true
+data:
+  path: ${oc.env:PROJECT_ROOT}/code_completion_exp/datasets/data_V4_full
+  max_context_len: 4096
+  max_target_len: 256
+  num_workers: 0
+  pin_memory: true
+  max_train_samples: 150000
+  max_val_samples: null
+logging:
+  log_interval: 10
+  save_interval: 3000
+  eval_interval: 1000
+  save_every_epoch: false
+  model_only_checkpoints: true
+tracking:
+  enabled: true
+  backend: wandb
+  project: routing-evolution
+  run_name: routing_N8.0
+  entity: null
+  base_url: https://wandb.platun0v.ru
+  local_dir: ${paths.output_dir}
+paths:
+  output_dir: outputs/${now:%Y-%m-%d}/N_${training.load_balancing_N}
+seed: 42
+device: cuda

routing_rerun/N_8.0/.hydra/hydra.yaml

@@ -0,0 +1,162 @@
+hydra:
+  run:
+    dir: ${paths.output_dir}
+  sweep:
+    dir: outputs/multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - training.load_balancing_N=8.0
+    - tracking.run_name=routing_N8.0
+  job:
+    name: train
+    chdir: false
+    override_dirname: tracking.run_name=routing_N8.0,training.load_balancing_N=8.0
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /workspace/byte-llms-code/routing_evolution_exp
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /workspace/byte-llms-code/routing_evolution_exp/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /workspace/byte-llms-code/routing_evolution_exp/outputs/2026-05-24/N_8.0
+    choices:
+      paths: default
+      tracking: default
+      logging: default
+      data: default
+      training: default
+      model: hnet_xl_code
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false