2024-04-21 16:37:32 +00:00
|
|
|
#!/bin/bash
|
|
|
|
|
|
2024-05-07 09:50:27 +00:00
|
|
|
NPROC_PER_NODE=4
|
2024-05-14 15:32:53 +00:00
|
|
|
NNODES=1
|
|
|
|
|
RANK=0
|
|
|
|
|
MASTER_ADDR=127.0.0.1
|
|
|
|
|
MASTER_PORT=29500
|
2024-05-07 09:50:27 +00:00
|
|
|
|
2024-05-14 15:32:53 +00:00
|
|
|
CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun \
|
2024-05-07 09:50:27 +00:00
|
|
|
--nproc_per_node $NPROC_PER_NODE \
|
2024-05-14 15:32:53 +00:00
|
|
|
--nnodes $NNODES \
|
|
|
|
|
--node_rank $RANK \
|
|
|
|
|
--master_addr $MASTER_ADDR \
|
|
|
|
|
--master_port $MASTER_PORT \
|
2024-05-06 14:51:02 +00:00
|
|
|
src/train.py examples/lora_multi_gpu/llama3_lora_sft_ds.yaml
|
