Buckets:
| from __future__ import annotations | |
| import json | |
| import os | |
| import subprocess | |
| from pathlib import Path | |
| import pytest | |
| from cluster_jobs.launch import execute_submission | |
| from cluster_jobs.resubmit import resubmit_run | |
| from cluster_jobs.snapshot import ( | |
| SnapshotError, | |
| SnapshotVerificationError, | |
| SourceChangedError, | |
| create_snapshot, | |
| verify_submission, | |
| ) | |
| from cluster_jobs.submit import main as submit_main | |
| from cluster_jobs.submit import resolve_account, submit_job | |
| def _project(tmp_path: Path) -> tuple[Path, Path]: | |
| project = tmp_path / "project" | |
| (project / "OURS/src/pkg").mkdir(parents=True) | |
| (project / "OURS/config").mkdir() | |
| (project / "experiments/src/cluster_jobs").mkdir(parents=True) | |
| (project / "OURS/src/pkg/__init__.py").write_text("VALUE = 1\n") | |
| (project / "OURS/config/train.yaml").write_text("epochs: 1\n") | |
| (project / "experiments/src/run.py").write_text("print('old')\n") | |
| (project / "experiments/src/cluster_jobs/launch.py").write_text("# copied\n") | |
| env = tmp_path / "env" | |
| (env / "conda-meta").mkdir(parents=True) | |
| (env / "bin").mkdir() | |
| (env / "conda-meta/history").write_text("created\n") | |
| (env / "bin/python").write_text("fake interpreter\n") | |
| os.chmod(env / "bin/python", 0o755) | |
| subprocess.run(["git", "init", "-q"], cwd=project, check=True) | |
| subprocess.run(["git", "add", "."], cwd=project, check=True) | |
| subprocess.run( | |
| [ | |
| "git", | |
| "-c", | |
| "user.name=Test", | |
| "-c", | |
| "user.email=test@example.invalid", | |
| "commit", | |
| "-qm", | |
| "initial", | |
| ], | |
| cwd=project, | |
| check=True, | |
| ) | |
| return project, env | |
| def _snapshot(tmp_path: Path, **overrides: object): | |
| project, env = _project(tmp_path) | |
| kwargs = { | |
| "project_root": project, | |
| "run_dir": tmp_path / "run", | |
| "env_prefix": env, | |
| "argv": ["python", "experiments/src/run.py", "safe argument"], | |
| "sbatch_args": ["--time=00:10:00"], | |
| } | |
| kwargs.update(overrides) | |
| return project, env, create_snapshot(**kwargs) | |
| def test_snapshot_copies_dirty_and_untracked_files(tmp_path: Path) -> None: | |
| project, env = _project(tmp_path) | |
| tracked = project / "experiments/src/run.py" | |
| tracked.write_text("print('dirty')\n") | |
| untracked = project / "OURS/src/pkg/new.py" | |
| untracked.write_text("NEW = True\n") | |
| result = create_snapshot( | |
| project_root=project, | |
| run_dir=tmp_path / "run", | |
| env_prefix=env, | |
| argv=["python", "experiments/src/run.py"], | |
| ) | |
| assert (result.source_dir / "experiments/src/run.py").read_text() == "print('dirty')\n" | |
| assert (result.source_dir / "OURS/src/pkg/new.py").read_text() == "NEW = True\n" | |
| git_state = json.loads((result.submission_dir / "git.json").read_text()) | |
| assert git_state["is_dirty"] is True | |
| manifest = json.loads((result.submission_dir / "manifest.json").read_text()) | |
| assert manifest["source_hash_before"] == manifest["source_hash_after"] | |
| assert manifest["source_hash_after"] == manifest["copied_hash"] | |
| assert not ( | |
| (result.source_dir / "OURS/src/pkg/new.py").stat().st_mode & 0o222 | |
| ) | |
| assert not result.source_dir.stat().st_mode & 0o222 | |
| assert not (result.source_dir / "OURS/src/pkg").stat().st_mode & 0o222 | |
| assert (result.submission_dir / "READY").is_file() | |
| job_script = (result.submission_dir / "job.sbatch").read_text() | |
| assert "#!/bin/bash" in job_script | |
| assert "#SBATCH --export=NONE" in job_script | |
| def test_bagel_source_profile_freezes_only_bagel_engine_and_orchestration( | |
| tmp_path: Path, | |
| ) -> None: | |
| project, env = _project(tmp_path) | |
| (project / "OURS-Bagel/src/bagel_mmcot").mkdir(parents=True) | |
| (project / "OURS-Bagel/src/bagel_mmcot/__init__.py").write_text("VALUE = 2\n") | |
| (project / "projects/ThinkMorph-edited/modeling").mkdir(parents=True) | |
| (project / "projects/ThinkMorph-edited/modeling/__init__.py").write_text( | |
| "ENGINE = True\n" | |
| ) | |
| result = create_snapshot( | |
| project_root=project, | |
| run_dir=tmp_path / "bagel-run", | |
| env_prefix=env, | |
| argv=["python", "-m", "bagel_mmcot"], | |
| source_profile="bagel", | |
| ) | |
| manifest = json.loads((result.submission_dir / "manifest.json").read_text()) | |
| assert manifest["roots"] == [ | |
| "OURS-Bagel", | |
| "projects/ThinkMorph-edited", | |
| "experiments/src", | |
| ] | |
| assert not (result.source_dir / "OURS").exists() | |
| job_script = (result.submission_dir / "job.sbatch").read_text() | |
| assert str(result.source_dir / "OURS-Bagel/src") in job_script | |
| captured = {} | |
| def fake_exec(file, argv, env): | |
| captured.update(file=file, argv=argv, env=env) | |
| execute_submission(result.submission_dir, exec_fn=fake_exec) | |
| assert captured["env"]["PYTHONPATH"].split(os.pathsep) == [ | |
| str(result.source_dir / "OURS-Bagel/src"), | |
| str(result.source_dir / "projects/ThinkMorph-edited"), | |
| str(result.source_dir / "experiments/src"), | |
| str(result.source_dir), | |
| ] | |
| def test_latentum_source_profile_freezes_framework_engine_and_orchestration( | |
| tmp_path: Path, | |
| ) -> None: | |
| project, env = _project(tmp_path) | |
| (project / "OURS-LatentUM/src/latentum_mmcot").mkdir(parents=True) | |
| (project / "OURS-LatentUM/src/latentum_mmcot/__init__.py").write_text( | |
| "VALUE = 3\n" | |
| ) | |
| (project / "projects/LatentUM/model").mkdir(parents=True) | |
| (project / "projects/LatentUM/model/__init__.py").write_text( | |
| "ENGINE = True\n" | |
| ) | |
| result = create_snapshot( | |
| project_root=project, | |
| run_dir=tmp_path / "latentum-run", | |
| env_prefix=env, | |
| argv=["python", "-m", "latentum_mmcot"], | |
| source_profile="latentum", | |
| ) | |
| manifest = json.loads((result.submission_dir / "manifest.json").read_text()) | |
| assert manifest["roots"] == [ | |
| "OURS-LatentUM", | |
| "projects/LatentUM", | |
| "experiments/src", | |
| ] | |
| assert not (result.source_dir / "OURS").exists() | |
| captured = {} | |
| def fake_exec(file, argv, env): | |
| captured.update(file=file, argv=argv, env=env) | |
| execute_submission(result.submission_dir, exec_fn=fake_exec) | |
| assert captured["env"]["PYTHONPATH"].split(os.pathsep) == [ | |
| str(result.source_dir / "OURS-LatentUM/src"), | |
| str(result.source_dir / "projects/LatentUM"), | |
| str(result.source_dir / "experiments/src"), | |
| str(result.source_dir), | |
| ] | |
| def test_source_mutation_aborts_without_publishing_run(tmp_path: Path) -> None: | |
| project, env = _project(tmp_path) | |
| output = tmp_path / "run" | |
| with pytest.raises(SourceChangedError): | |
| create_snapshot( | |
| project_root=project, | |
| run_dir=output, | |
| env_prefix=env, | |
| argv=["python", "experiments/src/run.py"], | |
| mutation_hook=lambda: (project / "experiments/src/run.py").write_text( | |
| "print('changed during copy')\n" | |
| ), | |
| ) | |
| assert not output.exists() | |
| assert not list(tmp_path.glob(".run.submission-*")) | |
| def test_manifest_tamper_is_rejected(tmp_path: Path) -> None: | |
| _, _, result = _snapshot(tmp_path) | |
| copied = result.source_dir / "experiments/src/run.py" | |
| copied.chmod(0o644) | |
| copied.write_text("tampered\n") | |
| with pytest.raises(SnapshotVerificationError, match="source differs"): | |
| verify_submission(result.submission_dir) | |
| def test_environment_drift_is_rejected(tmp_path: Path) -> None: | |
| _, env, result = _snapshot(tmp_path) | |
| (env / "conda-meta/history").write_text("changed\n") | |
| with pytest.raises(SnapshotVerificationError, match="environment fingerprint"): | |
| verify_submission(result.submission_dir, verify_environment=True) | |
| def test_exact_argv_exec_never_builds_a_shell_command( | |
| tmp_path: Path, monkeypatch: pytest.MonkeyPatch | |
| ) -> None: | |
| monkeypatch.setenv("SLURM_JOB_ID", "789") | |
| attack = "; touch should-not-exist" | |
| project, env, result = _snapshot( | |
| tmp_path, | |
| argv=["python", "experiments/src/run.py", attack], | |
| ) | |
| (env / "conda-meta/history").write_text("changed after submission\n") | |
| captured: dict[str, object] = {} | |
| def fake_exec(file: str, argv: list[str], env: dict[str, str]) -> None: | |
| captured.update(file=file, argv=argv, env=env) | |
| execute_submission(result.submission_dir, exec_fn=fake_exec) | |
| executed = captured["argv"] | |
| assert isinstance(executed, list) | |
| assert executed[-1] == attack | |
| assert executed[1] == str(result.source_dir / "experiments/src/run.py") | |
| assert captured["file"].endswith("/bin/python") | |
| assert captured["env"]["PYTHONPATH"].split(os.pathsep) == [ | |
| str(result.source_dir / "OURS/src"), | |
| str(result.source_dir / "experiments/src"), | |
| str(result.source_dir), | |
| ] | |
| manifest = json.loads((result.submission_dir / "manifest.json").read_text()) | |
| assert captured["env"]["SPATIAL_MMCOT_SOURCE_ROOT"] == str(result.source_dir) | |
| assert captured["env"]["SPATIAL_MMCOT_SOURCE_HASH"] == manifest["copied_hash"] | |
| attestation = json.loads( | |
| (result.submission_dir / "runtime-verification-789.json").read_text() | |
| ) | |
| assert attestation["source_hash"] == manifest["copied_hash"] | |
| assert not (project / "should-not-exist").exists() | |
| def test_config_tokens_are_mapped_to_snapshot_and_secrets_are_not_dumped( | |
| tmp_path: Path, monkeypatch: pytest.MonkeyPatch | |
| ) -> None: | |
| monkeypatch.setenv("CLUSTER_JOB_TEST_SECRET", "never-write-this-value") | |
| _, _, result = _snapshot( | |
| tmp_path, | |
| argv=[ | |
| "torchrun", | |
| "experiments/src/run.py", | |
| "--config", | |
| "OURS/config/train.yaml", | |
| "--other-config=OURS/config/train.yaml", | |
| ], | |
| ) | |
| argv = json.loads((result.submission_dir / "argv.json").read_text())["exec"] | |
| config = str(result.source_dir / "OURS/config/train.yaml") | |
| assert argv == [ | |
| "torchrun", | |
| str(result.source_dir / "experiments/src/run.py"), | |
| "--config", | |
| config, | |
| f"--other-config={config}", | |
| ] | |
| metadata = "\n".join( | |
| path.read_text(errors="ignore") | |
| for path in result.submission_dir.iterdir() | |
| if path.is_file() | |
| ) | |
| assert "never-write-this-value" not in metadata | |
| def test_project_local_script_requires_explicit_snapshot_extra(tmp_path: Path) -> None: | |
| project, env = _project(tmp_path) | |
| orchestration = project / "orchestration/run.sh" | |
| orchestration.parent.mkdir() | |
| orchestration.write_text("#!/bin/bash\n") | |
| with pytest.raises(SnapshotError, match="outside the source snapshot"): | |
| create_snapshot( | |
| project_root=project, | |
| run_dir=tmp_path / "rejected", | |
| env_prefix=env, | |
| argv=["bash", str(orchestration)], | |
| ) | |
| accepted = create_snapshot( | |
| project_root=project, | |
| run_dir=tmp_path / "accepted", | |
| env_prefix=env, | |
| argv=["bash", str(orchestration)], | |
| extra_paths=["orchestration/run.sh"], | |
| ) | |
| argv = json.loads((accepted.submission_dir / "argv.json").read_text())["exec"] | |
| assert argv[1] == str(accepted.source_dir / "orchestration/run.sh") | |
| def test_external_script_requires_a_declared_input(tmp_path: Path) -> None: | |
| project, env = _project(tmp_path) | |
| script = tmp_path / "external.sh" | |
| script.write_text("#!/bin/bash\n") | |
| with pytest.raises(SnapshotError, match="not declared as an input"): | |
| create_snapshot( | |
| project_root=project, | |
| run_dir=tmp_path / "rejected", | |
| env_prefix=env, | |
| argv=["bash", str(script)], | |
| ) | |
| accepted = create_snapshot( | |
| project_root=project, | |
| run_dir=tmp_path / "accepted", | |
| env_prefix=env, | |
| argv=["bash", str(script)], | |
| inputs=[script], | |
| ) | |
| argv = json.loads((accepted.submission_dir / "argv.json").read_text())["exec"] | |
| assert argv[1] == str(script) | |
| def test_relative_inputs_resolve_from_project_root(tmp_path: Path) -> None: | |
| project, env = _project(tmp_path) | |
| (project / "data").mkdir() | |
| (project / "data/selection.json").write_text("[]\n") | |
| result = create_snapshot( | |
| project_root=project, | |
| run_dir=tmp_path / "run", | |
| env_prefix=env, | |
| argv=["python", "experiments/src/run.py"], | |
| inputs=["data/selection.json"], | |
| ) | |
| inputs = json.loads((result.submission_dir / "inputs.json").read_text()) | |
| assert inputs[0]["path"] == str(project / "data/selection.json") | |
| def test_snapshot_excludes_credentials_and_generated_trees(tmp_path: Path) -> None: | |
| project, env = _project(tmp_path) | |
| (project / "OURS/.env.local").write_text("SECRET=value\n") | |
| (project / "OURS/API_KEY").write_text("sk-secret\n") | |
| (project / "OURS/private.pem").write_text("credential\n") | |
| (project / "external").mkdir() | |
| (project / "external/API_KEY").write_text("sk-external-secret\n") | |
| (project / "external/module.py").write_text("VALUE = 1\n") | |
| (project / "OURS/checkpoints").mkdir() | |
| (project / "OURS/checkpoints/state.pt").write_text("generated\n") | |
| result = create_snapshot( | |
| project_root=project, | |
| run_dir=tmp_path / "run", | |
| env_prefix=env, | |
| argv=["python", "experiments/src/run.py"], | |
| extra_paths=["external"], | |
| ) | |
| assert not (result.source_dir / "OURS/.env.local").exists() | |
| assert not (result.source_dir / "OURS/API_KEY").exists() | |
| assert not (result.source_dir / "OURS/private.pem").exists() | |
| assert not (result.source_dir / "external/API_KEY").exists() | |
| assert (result.source_dir / "external/module.py").exists() | |
| assert not (result.source_dir / "OURS/checkpoints").exists() | |
| def test_declared_input_symlinks_fail_closed(tmp_path: Path) -> None: | |
| project, env = _project(tmp_path) | |
| target = project / "target.json" | |
| target.write_text("{}\n") | |
| alias = project / "alias.json" | |
| alias.symlink_to(target) | |
| with pytest.raises(SnapshotError, match="input cannot traverse a symlink"): | |
| create_snapshot( | |
| project_root=project, | |
| run_dir=tmp_path / "run", | |
| env_prefix=env, | |
| argv=["python", "experiments/src/run.py"], | |
| inputs=[alias], | |
| ) | |
| target_dir = project / "target-dir" | |
| target_dir.mkdir() | |
| (target_dir / "input.json").write_text("{}\n") | |
| alias_dir = project / "alias-dir" | |
| alias_dir.symlink_to(target_dir, target_is_directory=True) | |
| with pytest.raises(SnapshotError, match="input cannot traverse a symlink"): | |
| create_snapshot( | |
| project_root=project, | |
| run_dir=tmp_path / "ancestor-run", | |
| env_prefix=env, | |
| argv=["python", "experiments/src/run.py"], | |
| inputs=[alias_dir / "input.json"], | |
| ) | |
| def test_existing_directory_and_escaping_symlink_are_rejected(tmp_path: Path) -> None: | |
| project, env = _project(tmp_path) | |
| output = tmp_path / "run" | |
| output.mkdir() | |
| marker = output / "keep" | |
| marker.write_text("mine") | |
| with pytest.raises(FileExistsError): | |
| create_snapshot( | |
| project_root=project, | |
| run_dir=output, | |
| env_prefix=env, | |
| argv=["python", "-m", "pkg"], | |
| ) | |
| assert marker.read_text() == "mine" | |
| output.rmdir() if not any(output.iterdir()) else None | |
| marker.unlink() | |
| output.rmdir() | |
| outside = tmp_path / "outside.py" | |
| outside.write_text("SECRET = True\n") | |
| (project / "OURS/src/escape.py").symlink_to(outside) | |
| with pytest.raises(SnapshotError, match="symlink escapes"): | |
| create_snapshot( | |
| project_root=project, | |
| run_dir=output, | |
| env_prefix=env, | |
| argv=["python", "-m", "pkg"], | |
| ) | |
| def test_caller_cannot_override_sanitized_slurm_environment(tmp_path: Path) -> None: | |
| project, env = _project(tmp_path) | |
| with pytest.raises(SnapshotError, match="owns account"): | |
| create_snapshot( | |
| project_root=project, | |
| run_dir=tmp_path / "run", | |
| env_prefix=env, | |
| argv=["python", "-m", "pkg"], | |
| sbatch_args=["--export=ALL"], | |
| ) | |
| with pytest.raises(SnapshotError, match="owns account"): | |
| create_snapshot( | |
| project_root=project, | |
| run_dir=tmp_path / "run-output", | |
| env_prefix=env, | |
| argv=["python", "-m", "pkg"], | |
| sbatch_args=["--output=/tmp/escaped.out"], | |
| ) | |
| for argument in ("--account=wrong", "--export-file=/tmp/env", "--get-user-env"): | |
| with pytest.raises(SnapshotError, match="owns account"): | |
| create_snapshot( | |
| project_root=project, | |
| run_dir=tmp_path / ("run-" + argument.split("=")[0][2:]), | |
| env_prefix=env, | |
| argv=["python", "-m", "pkg"], | |
| sbatch_args=[argument], | |
| ) | |
| def test_submit_commands_are_argument_vectors(tmp_path: Path) -> None: | |
| calls: list[list[str]] = [] | |
| def runner(command, **kwargs): | |
| calls.append(list(command)) | |
| stdout = "project-a\n" if "account" in command else "Submitted batch job 123\n" | |
| return subprocess.CompletedProcess(command, 0, stdout=stdout, stderr="") | |
| account = resolve_account("delta", runner=runner) | |
| job_id = submit_job(tmp_path / "job.sbatch", host="delta", account=account, runner=runner) | |
| assert account == "project-a" | |
| assert job_id == "123" | |
| assert calls == [ | |
| ["ssh", "--", "delta", "bash", "-ic", "account"], | |
| ["ssh", "--", "delta", "sbatch", "--account=project-a", str(tmp_path / "job.sbatch")], | |
| ] | |
| def test_deferred_submit_prepares_resumable_run_without_sbatch(tmp_path: Path) -> None: | |
| project, env = _project(tmp_path) | |
| run = tmp_path / "deferred" | |
| assert submit_main( | |
| [ | |
| "--project-root", str(project), | |
| "--output-dir", str(run), | |
| "--env-prefix", str(env), | |
| "--resumable", | |
| "--defer", | |
| "--", | |
| "python", "experiments/src/run.py", | |
| ] | |
| ) == 0 | |
| record = json.loads((run / "_submission/job_id.json").read_text()) | |
| assert record == {"schema": 1, "status": "deferred", "host": "local"} | |
| def test_interactive_submit_requires_deferred_siq_handoff(tmp_path: Path) -> None: | |
| project, env = _project(tmp_path) | |
| with pytest.raises(SystemExit, match="interactive partitions require"): | |
| submit_main( | |
| [ | |
| "--project-root", str(project), | |
| "--output-dir", str(tmp_path / "run"), | |
| "--env-prefix", str(env), | |
| "--resumable", | |
| "--sbatch-arg=--partition=ghx4-interactive", | |
| "--", | |
| "python", "experiments/src/run.py", | |
| ] | |
| ) | |
| def test_deferred_siq_snapshot_rejects_ephemeral_slurm_dependency(tmp_path: Path) -> None: | |
| project, env = _project(tmp_path) | |
| with pytest.raises(SystemExit, match="ephemeral Slurm job dependencies"): | |
| submit_main( | |
| [ | |
| "--project-root", str(project), | |
| "--output-dir", str(tmp_path / "run"), | |
| "--env-prefix", str(env), | |
| "--resumable", | |
| "--defer", | |
| "--sbatch-arg=--partition=ghx4-interactive", | |
| "--sbatch-arg=--dependency=afterok:123", | |
| "--", | |
| "python", "experiments/src/run.py", | |
| ] | |
| ) | |
| def test_submit_error_includes_scheduler_stderr(tmp_path: Path) -> None: | |
| def runner(command, **kwargs): | |
| raise subprocess.CalledProcessError( | |
| 1, command, output="", stderr="invalid account/partition" | |
| ) | |
| with pytest.raises(SnapshotError, match="invalid account/partition"): | |
| submit_job( | |
| tmp_path / "job.sbatch", | |
| host="local", | |
| account="project-a", | |
| runner=runner, | |
| ) | |
| def test_resubmit_reuses_frozen_job_with_scheduler_overrides(tmp_path: Path) -> None: | |
| _, env, result = _snapshot(tmp_path, resumable=True) | |
| (env / "conda-meta/history").write_text("changed after submission\n") | |
| calls: list[list[str]] = [] | |
| def runner(command, **kwargs): | |
| calls.append(list(command)) | |
| stdout = "project-a\n" if "account" in command else "Submitted batch job 456\n" | |
| return subprocess.CompletedProcess(command, 0, stdout=stdout, stderr="") | |
| job_id = resubmit_run( | |
| result.run_dir, | |
| host="local", | |
| sbatch_args=["--time=01:00:00"], | |
| runner=runner, | |
| ) | |
| assert job_id == "456" | |
| assert calls[-1] == [ | |
| "sbatch", | |
| "--account=project-a", | |
| "--time=01:00:00", | |
| str(result.job_script), | |
| ] | |
| history = [ | |
| json.loads(line) | |
| for line in (result.submission_dir / "resubmissions.jsonl").read_text().splitlines() | |
| ] | |
| assert history[0]["job_id"] == "456" | |
| def test_changed_source_continuation_records_parent_run(tmp_path: Path) -> None: | |
| project, env, parent = _snapshot(tmp_path / "parent") | |
| child = create_snapshot( | |
| project_root=project, | |
| run_dir=tmp_path / "child", | |
| env_prefix=env, | |
| argv=["python", "experiments/src/run.py"], | |
| parent_run=parent.run_dir, | |
| ) | |
| submission = json.loads((child.submission_dir / "submission.json").read_text()) | |
| manifest = json.loads((parent.submission_dir / "manifest.json").read_text()) | |
| assert submission["parent_run"] == { | |
| "run_dir": str(parent.run_dir), | |
| "source_hash": manifest["copied_hash"], | |
| } | |
| def test_one_shot_snapshot_cannot_be_resubmitted(tmp_path: Path) -> None: | |
| _, _, result = _snapshot(tmp_path) | |
| with pytest.raises(SnapshotError, match="not declared resumable"): | |
| resubmit_run(result.run_dir, host="local") | |
| def test_resubmit_rejects_an_active_attempt(tmp_path: Path) -> None: | |
| _, _, result = _snapshot(tmp_path, resumable=True) | |
| (result.submission_dir / "job_id.json").write_text( | |
| json.dumps( | |
| { | |
| "status": "submitted", | |
| "job_id": "123", | |
| "host": "local", | |
| } | |
| ) | |
| + "\n" | |
| ) | |
| def runner(command, **kwargs): | |
| assert command[0] == "squeue" | |
| return subprocess.CompletedProcess(command, 0, stdout="123\n", stderr="") | |
| with pytest.raises(SnapshotError, match="active Slurm attempts"): | |
| resubmit_run(result.run_dir, host="local", runner=runner) | |
| def test_resubmit_treats_purged_slurm_job_as_inactive(tmp_path: Path) -> None: | |
| _, _, result = _snapshot(tmp_path, resumable=True) | |
| (result.submission_dir / "job_id.json").write_text( | |
| json.dumps({"status": "submitted", "job_id": "123", "host": "local"}) | |
| + "\n" | |
| ) | |
| def runner(command, **kwargs): | |
| if command[0] == "squeue": | |
| raise subprocess.CalledProcessError( | |
| 1, | |
| command, | |
| output="", | |
| stderr="slurm_load_jobs error: Invalid job id specified\n", | |
| ) | |
| stdout = "project-a\n" if "account" in command else "Submitted batch job 456\n" | |
| return subprocess.CompletedProcess(command, 0, stdout=stdout, stderr="") | |
| assert resubmit_run(result.run_dir, host="local", runner=runner) == "456" | |
Xet Storage Details
- Size:
- 23.2 kB
- Xet hash:
- 7c6c61b1d2eebcafc26155535a4ab816df9932961dd4e57ae1ad2139732ca049
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.