| #!/bin/bash |
| |
| |
|
|
| CLUSTER_CONFIG='./cluster.json' |
| cat <<EOT > ${CLUSTER_CONFIG} |
| { |
| "worker": { |
| "gpu": 100, |
| "cpu": 1200, |
| "memory": 50000 |
| } |
| } |
| EOT |
|
|
| WORKER_COUNT=8 |
|
|
| args="--resume" |
| nebulactl run mdl --engine=xdl \ |
| --refresh \ |
| --ignore=data/*,.git/* \ |
| --queue=pailitao_algo_high_2 \ |
| --worker_count=$WORKER_COUNT \ |
| --odps_project=pailitao_algo_high_2 \ |
| --entry="./cot_gen/generate_cot.py" \ |
| --oss_access_id=LTAI5tMZb5HHH6HHgAExAyiL \ |
| --oss_access_key=6MVP8JnTE01t4TLUHhPoFV0swZNTrb \ |
| --oss_bucket=pailitao-ai \ |
| --oss_endpoint=oss-cn-zhangjiakou-internal.aliyuncs.com \ |
| --algo_name=pytorch2100 \ |
| --user_params="$args" \ |
| --job_name=shaixuan-cot-gen-qwen36-27b \ |
| --file.cluster_file=$CLUSTER_CONFIG \ |
| --force \ |
| --max_failover_times=2 \ |
| --requirements_file_name="./requirements.txt" |
|
|
| rm ${CLUSTER_CONFIG} |
|
|
