File size: 2,233 Bytes
21b75e3 b032961 21b75e3 b032961 21b75e3 b032961 21b75e3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 | #!/bin/bash
set -euo pipefail
CORPUS_NAME="${1:?usage: $0 <corpus-name>}"
REPO_URL="${REPO_URL:-https://github.com/vedatonuryilmaz/dev_genopix.git}"
REPO_BRANCH="${REPO_BRANCH:-ml-intern}"
DATA_DATASET="${DATA_DATASET:-vedatonuryilmaz/te-variant-simulator-v1}"
REFERENCE_DATASET="${REFERENCE_DATASET:-vedatonuryilmaz/te-seqdata-v1}"
REFERENCE_REVISION="${REFERENCE_REVISION:-f107d46c5a61087eb3fb19b4e3e75fdef0b74fdd}"
REFERENCE_FILE="${REFERENCE_FILE:-train/te_seqdata.parquet}"
REFERENCE_PARQUET="${REFERENCE_PARQUET:-data/raw/reference/${REFERENCE_FILE}}"
CORPUS_KIND="${CORPUS_KIND:-controlled}"
N_LOCI="${N_LOCI:-64}"
COVERAGE="${COVERAGE:-50}"
READ_LENGTH="${READ_LENGTH:-150}"
ERROR_RATE="${ERROR_RATE:-0.001}"
BP_PER_TOKEN="${BP_PER_TOKEN:-24}"
VARIANT_TYPES="${VARIANT_TYPES:-snp}"
if [ "${DEEPGENOPIX_ALREADY_CLONED:-0}" = "1" ]; then
echo "using existing DeepGenopix checkout at $(pwd)"
else
git clone --branch "$REPO_BRANCH" --depth 1 "$REPO_URL" deepgenopix
cd deepgenopix
fi
uv sync --extra hf --extra quant
if [ ! -f "$REFERENCE_PARQUET" ]; then
mkdir -p "$(dirname "$REFERENCE_PARQUET")"
uv run python -c "
from huggingface_hub import hf_hub_download
hf_hub_download(
repo_id='$REFERENCE_DATASET',
repo_type='dataset',
revision='$REFERENCE_REVISION',
filename='$REFERENCE_FILE',
local_dir='data/raw/reference',
)
"
fi
OUTPUT_DIR="data/output/simulator/$CORPUS_KIND/$CORPUS_NAME"
mkdir -p "$OUTPUT_DIR"
uv run python -c "
from pathlib import Path
from deepgenopix.variant.simulate import SimulatorConfig, materialize_corpus
variant_types = tuple(item.strip() for item in '$VARIANT_TYPES'.split(',') if item.strip())
cfg = SimulatorConfig(
coverage=int('$COVERAGE'),
read_length=int('$READ_LENGTH'),
error_rate=float('$ERROR_RATE'),
bp_per_token=int('$BP_PER_TOKEN'),
variant_types=variant_types,
)
paths = materialize_corpus(
Path('$REFERENCE_PARQUET'),
cfg,
Path('$OUTPUT_DIR'),
n_loci=int('$N_LOCI'),
)
print(paths)
"
uv run python scripts/hf_artifacts.py push-dataset-folder \
--repo "$DATA_DATASET" \
--source "$OUTPUT_DIR" \
--path-in-repo "$CORPUS_KIND/$CORPUS_NAME" \
--message "variant simulator corpus $CORPUS_KIND/$CORPUS_NAME"
|