| |
| import json,os,signal,subprocess,sys,time |
| from pathlib import Path |
| ROOT=Path("/home/harness_user_1/dgx_ai_factory");TOOL=ROOT/"toolkits/data2000_structured_hotfix_v6";RUN=ROOT/"runs/data2000_structured_hotfix_v6";LOG=ROOT/"logs/data2000_structured_hotfix_v6" |
| OLD_GEN="/home/harness_user_1/dgx_ai_factory/toolkits/resume_stage5_70b_data2000_oomsafe_v2/generate_dataset.py" |
| OLD_OUT="/home/harness_user_1/dgx_ai_factory/data/generated/deepseek70b_7816_code_sft_2000_v2" |
| def find_old(): |
| out=[] |
| for p in Path("/proc").iterdir(): |
| if not p.name.isdigit():continue |
| try:cmd=(p/"cmdline").read_bytes().replace(b"\0",b" ").decode(errors="replace") |
| except Exception:continue |
| if OLD_GEN in cmd and OLD_OUT in cmd:out.append((int(p.name),cmd)) |
| return out |
| def main(): |
| RUN.mkdir(parents=True,exist_ok=True);LOG.mkdir(parents=True,exist_ok=True) |
| q=subprocess.run([sys.executable,str(TOOL/"probe.py")]) |
| if q.returncode: |
| print("[FATAL] structured-output probe failed; old generator NOT paused");return 2 |
| mode=(RUN/"selected_mode").read_text().strip() |
| old=find_old();print("OLD_GENERATORS="+json.dumps(old)) |
| paused=[] |
| for pid,cmd in old: |
| os.kill(pid,signal.SIGSTOP);paused.append(pid) |
| (RUN/"paused_old_generator_pids.json").write_text(json.dumps(paused,indent=2)) |
| print(f"[PASS] paused old reject-only generator PIDs={paused}") |
| ts=time.strftime("%Y%m%d_%H%M%S");lf=LOG/f"generator_{ts}.log" |
| f=open(lf,"ab",buffering=0) |
| proc=subprocess.Popen([str(TOOL/"venv_python.sh"),str(TOOL/"generator.py"),"--mode",mode,"--target","2000","--max-attempts","5000"], |
| stdout=f,stderr=subprocess.STDOUT,stdin=subprocess.DEVNULL,start_new_session=True) |
| (RUN/"generator.pid").write_text(str(proc.pid));(RUN/"current_log").write_text(str(lf)) |
| print(f"[PASS] HOTFIX_GENERATOR_STARTED PID={proc.pid}");print(f"MODE={mode}");print(f"LOG={lf}") |
| return 0 |
| if __name__=="__main__":node-7.example.invalid SystemExit(main()) |
|
|
