periodic: post_bench_pipeline.log (0.0MB)
Browse files- runs/post_bench_pipeline.log +44 -0
runs/post_bench_pipeline.log
CHANGED
|
@@ -134,3 +134,47 @@ adapter] max_new_tokens=384
|
|
| 134 |
[trace] err id=train:pair_pos:Ast:MORN1:chr1_2312786_2313286:919: HTTPError: HTTP Error 429: Too Many Requests
|
| 135 |
[trace] err id=train:pair_pos:Ast:CALML6:chr1_1763650_1764150:722: HTTPError: HTTP Error 429: Too Many Requests
|
| 136 |
[trace] err id=train:pair_pos:Ast:MORN1:chr1_2322822_2323322:923: HTTPError: HTTP Error 429: Too Many Requests
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
[trace] err id=train:pair_pos:Ast:MORN1:chr1_2312786_2313286:919: HTTPError: HTTP Error 429: Too Many Requests
|
| 135 |
[trace] err id=train:pair_pos:Ast:CALML6:chr1_1763650_1764150:722: HTTPError: HTTP Error 429: Too Many Requests
|
| 136 |
[trace] err id=train:pair_pos:Ast:MORN1:chr1_2322822_2323322:923: HTTPError: HTTP Error 429: Too Many Requests
|
| 137 |
+
[trace] err id=train:pair_pos:Ast:SKI:chr1_2322822_2323322:893: HTTPError: HTTP Error 429: Too Many Requests
|
| 138 |
+
[trace] err id=train:pair_pos:Ast:SKI:chr1_2342900_2343400:895: HTTPError: HTTP Error 429: Too Many Requests
|
| 139 |
+
[trace] err id=train:pair_pos:Ast:SKI:chr1_2407052_2407552:897: HTTPError: HTTP Error 429: Too Many Requests
|
| 140 |
+
[trace] err id=train:pair_pos:Ast:SKI:chr1_2432242_2432742:899: HTTPError: HTTP Error 429: Too Many Requests
|
| 141 |
+
[trace] err id=train:pair_pos:Ast:SKI:chr1_2464965_2465465:901: HTTPError: HTTP Error 429: Too Many Requests
|
| 142 |
+
[trace] err id=train:pair_pos:Ast:MORN1:chr1_2200658_2201158:903: HTTPError: HTTP Error 429: Too Many Requests
|
| 143 |
+
[trace] err id=train:pair_pos:Ast:MORN1:chr1_2214850_2215350:905: HTTPError: HTTP Error 429: Too Many Requests
|
| 144 |
+
[2026-04-28 12:18:18 UTC] post-bench pipeline v4 starting
|
| 145 |
+
[12:18:18] STAGE 0: genqual on T1+T3 zs predictions
|
| 146 |
+
skip: no /workspace/dnathinker/runs/exp_t1_grid_separatedQA_20260426_h100_vllm_full/zs_raw/predictions.jsonl
|
| 147 |
+
skip: no /workspace/dnathinker/runs/exp_t3_grid_separatedQA_20260426_h100_vllm_full/zs_raw/predictions.jsonl
|
| 148 |
+
skip: no /workspace/dnathinker/runs/exp_t1_grid_separatedQA_20260426_h100_vllm_full/zs_enriched/predictions.jsonl
|
| 149 |
+
skip: no /workspace/dnathinker/runs/exp_t3_grid_separatedQA_20260426_h100_vllm_full/zs_enriched/predictions.jsonl
|
| 150 |
+
[12:18:18] STAGE 0c: T3 oracle-based eval (objective satisfaction)
|
| 151 |
+
skip: no /workspace/dnathinker/runs/exp_t3_grid_separatedQA_20260426_h100_vllm_full/zs_raw/predictions.jsonl
|
| 152 |
+
skip: no /workspace/dnathinker/runs/exp_t3_grid_separatedQA_20260426_h100_vllm_full/zs_enriched/predictions.jsonl
|
| 153 |
+
[12:18:18] STAGE 0b: early HF push
|
| 154 |
+
[12:18:19] STAGE 1 train data: /workspace/dnathinker/data/prod_samples_merged/train.enhancer_generation.strat7c.n35k.merged_v2_frozen_20260428T112500Z.jsonl
|
| 155 |
+
skip: exp_t1_fusion_sft_20260427_h100/final exists
|
| 156 |
+
[12:18:19] PREDICT-FUSION exp_t1_fusion_sft_20260427_h100 T1/raw
|
| 157 |
+
[score_adapter] max_new_tokens=384
|
| 158 |
+
[score_adapter] batch_size=256
|
| 159 |
+
[score_adapter] inference script = scripts/predict_fusion_proper.py
|
| 160 |
+
[predict-fusion-proper] task=enhancer_generation prompt=raw mode=embed
|
| 161 |
+
Traceback (most recent call last):
|
| 162 |
+
File "/workspace/biomodel_reasoning_calling_study2/regureasoner_loop/scripts/predict_fusion_proper.py", line 349, in <module>
|
| 163 |
+
raise SystemExit(main())
|
| 164 |
+
^^^^^^
|
| 165 |
+
File "/workspace/biomodel_reasoning_calling_study2/regureasoner_loop/scripts/predict_fusion_proper.py", line 85, in main
|
| 166 |
+
from regureasoner.models.one_shot_fusion import (
|
| 167 |
+
ModuleNotFoundError: No module named 'regureasoner'
|
| 168 |
+
[score_adapter] predict_fusion_proper rc=1 — falling back to legacy predict_fusion.py
|
| 169 |
+
[predict-fusion] task=enhancer_generation prompt=raw
|
| 170 |
+
[predict-fusion] building OneShotFusionLM (llm=Qwen/Qwen3.5-2B, dna=ntv3-650m)
|
| 171 |
+
[transformers] `torch_dtype` is deprecated! Use `dtype` instead!
|
| 172 |
+
|
| 173 |
+
2026-04-28 12:18:40 [INFO] regureasoner.models.dna_encoder: Loading NTv3 DNA encoder from local snapshot /workspace/dnathinker/ntv3_local/generative
|
| 174 |
+
[transformers] You are using a model of type `ntv3_posttrained` to instantiate a model of type `ntv3_conditioned`. This may be expected if you are loading a checkpoint that shares a subset of the architecture (e.g., loading a `sam2_video` checkpoint into `Sam2Model`), but is otherwise not supported and can yield errors. Please verify that the checkpoint is compatible with the model you are instantiating.
|
| 175 |
+
[transformers] You are using a model of type `ntv3_posttrained` to instantiate a model of type `ntv3_conditioned`. This may be expected if you are loading a checkpoint that shares a subset of the architecture (e.g., loading a `sam2_video` checkpoint into `Sam2Model`), but is otherwise not supported and can yield errors. Please verify that the checkpoint is compatible with the model you are instantiating.
|
| 176 |
+
[predict-fusion] loading state_dict: /workspace/dnathinker/runs/exp_t1_fusion_sft_20260427_h100/final/pytorch_model.bin
|
| 177 |
+
[predict-fusion] loaded; missing=0 unexpected=24
|
| 178 |
+
[trace] err id=train:pair_pos:Ast:MORN1:chr1_2230140_2230640:907: HTTPError: HTTP Error 429: Too Many Requests
|
| 179 |
+
[predict-fusion] 372210 eval examples (resume_skipped=0 of 372210 total)
|
| 180 |
+
[predict-fusion] early-stop tokens = [29, 248046]
|