| #!/usr/bin/env bash |
| set -euo pipefail |
|
|
| campaign_tag="${1:?usage: scripts/spark_campaign.sh CAMPAIGN_TAG [CONFIG]}" |
| config_path="${2:-configs/main.yaml}" |
| project_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" |
| cd "$project_dir" |
|
|
| python_bin="${BGC_PYTHON:?set BGC_PYTHON to a CUDA-enabled Python executable}" |
| dependency_dir="${BGC_EXTRA_PYTHONPATH:-$project_dir/.deps}" |
| export PYTHONPATH="$dependency_dir:$project_dir/src${PYTHONPATH:+:$PYTHONPATH}" |
|
|
| run_bgc() { |
| "$python_bin" -c 'from bgc_retrieval.cli import main; main()' "$@" |
| } |
|
|
| run_step() { |
| local name="$1" |
| shift |
| local started=$SECONDS |
| printf '[%s] START %s\n' "$(date --iso-8601=seconds)" "$name" |
| "$@" |
| printf '[%s] DONE %s (%ss)\n' "$(date --iso-8601=seconds)" "$name" "$((SECONDS - started))" |
| } |
|
|
| "$python_bin" -c 'import torch; assert torch.cuda.is_available(), "CUDA GPU is required"; print(torch.__version__, torch.cuda.get_device_name(0))' |
| printf 'CONFIG %s\n' "$config_path" |
|
|
| if [[ ! -f data/manifests/legacy_manifest.json ]]; then |
| run_step audit run_bgc audit --config "$config_path" |
| fi |
| if [[ ! -f data/manifests/silver_split.csv ]]; then |
| run_step splits run_bgc build-splits --config "$config_path" |
| fi |
| if [[ ! -f data/external/processed/external_atlas.csv ]]; then |
| run_step external-preparation "$python_bin" scripts/prepare_external_benchmark.py |
| fi |
| if [[ ! -f data/external/processed/external_esm2.h5 ]]; then |
| run_step external-esm2 "$python_bin" scripts/embed_external_proteins.py --resume |
| fi |
|
|
| run_step phase1 run_bgc train --config "$config_path" --stage phase1 \ |
| --run-id "${campaign_tag}-phase1" |
| run_step phase2-main run_bgc train --config "$config_path" --stage phase2 \ |
| --run-id "${campaign_tag}-main" \ |
| --phase1-checkpoint "artifacts/${campaign_tag}-phase1/phase1_best.pt" |
| run_step phase2-no-phase1 run_bgc train --config "$config_path" --stage phase2 \ |
| --run-id "${campaign_tag}-no-phase1" |
| run_step internal-main run_bgc evaluate --config "$config_path" \ |
| --checkpoint "artifacts/${campaign_tag}-main/phase2_best.pt" \ |
| --run-id "${campaign_tag}-main-evaluation" |
| run_step internal-no-phase1 run_bgc evaluate --config "$config_path" \ |
| --checkpoint "artifacts/${campaign_tag}-no-phase1/phase2_best.pt" \ |
| --run-id "${campaign_tag}-no-phase1-evaluation" |
| run_step external-main "$python_bin" scripts/evaluate_external.py --config "$config_path" \ |
| --checkpoint "artifacts/${campaign_tag}-main/phase2_best.pt" \ |
| --run-id "${campaign_tag}-main-external" \ |
| --bootstrap-samples "${EXTERNAL_BOOTSTRAP_SAMPLES:-1000}" |
| run_step external-no-phase1 "$python_bin" scripts/evaluate_external.py --config "$config_path" \ |
| --checkpoint "artifacts/${campaign_tag}-no-phase1/phase2_best.pt" \ |
| --run-id "${campaign_tag}-no-phase1-external" \ |
| --bootstrap-samples "${EXTERNAL_BOOTSTRAP_SAMPLES:-1000}" |
|
|
| printf '[%s] CAMPAIGN COMPLETE: %s\n' "$(date --iso-8601=seconds)" "$campaign_tag" |
|
|
