NVIDIA MPS(Multi-Process Service) に対応させた sbatch.sh です

ロジック的には各ノードで 「nvidia-cuda-mps-control -d」を起動させ、その中でrelion計算を動かして、計算が終わったらストップファイル(MPS_STOP_FILE)を作らせて、

       while [ ! -e "$MPS_STOP_FILE" ]; do
         sleep 1
       done

で拾って、 「nvidia-cuda-mps-control -d」の停止作業に入る。そんな感じです。

っでその「sbatch.sh」

#!/bin/bash
#SBATCH --ntasks=XXXmpinodesXXX                  # Number of MPI procs
#SBATCH --cpus-per-task=XXXthreadsXXX            # Number of threads
#SBATCH --partition=XXXqueueXXX                  # Queue name
#SBATCH --nodes=XXXnodesXXX                      # CEIL( XXXcoresXXX / XXXdedicatedXXX ) 必要なcore数を1台当たりのcore数で割って切り上げ. ノード数
#SBATCH --error=XXXerrfileXXX
#SBATCH --output=XXXoutfileXXX
#SBATCH --job-name=XXXnameXXX
 
cd $SLURM_SUBMIT_DIR
 
### Environment Modules
. /etc/profile.d/modules.sh
module load mpi/openmpi-x86_64
module load relion/XXXversionXXX
 
### requested resources
cat <<_EOF_>XXXoutfileXXX
----------------------------------------------
start  `date`
----------------------------------------------
Queue name          XXXqueueXXX
  required cores    XXXcoresXXX  :( XXXmpinodesXXX[mpi] * XXXthreadsXXX[threads] )
  nodes             XXXnodesXXX  :CEIL( XXXcoresXXX / XXXdedicatedXXX ) :( ${SLURM_JOB_NODELIST} )
_EOF_
 
### check command
cmd_=$(/bin/cat <<_EOF_
XXXcommandXXX
_EOF_
)
cmd=$(echo "$cmd_" | sed -z 's/\n/\&\&/g')
run=$(echo $cmd|awk '{print $1}')
 
### OpenMPI
export OMPI_MCA_pml=ob1
export OMPI_MCA_btl=self,vader,tcp
 
### NVIDIA MPS
export CUDA_MPS_PIPE_DIRECTORY=/tmp/nvidia-mps-${SLURM_JOB_ID}
export CUDA_MPS_LOG_DIRECTORY=/tmp/nvidia-mps-log-${SLURM_JOB_ID}
 
# STOP file must be on a filesystem shared by all compute nodes
export MPS_STOP_FILE=${SLURM_SUBMIT_DIR}/.nvidia-mps-stop-${SLURM_JOB_ID}
 
rm -f "$MPS_STOP_FILE"
cat <<_EOF_>>XXXoutfileXXX
----------------------------------------------
NVIDIA MPS start
----------------------------------------------
_EOF_
 
### Start one MPS control daemon on each allocated node.
### --overlap is required because this step remains alive while RELION runs.
 
srun --overlap \
     --nodes=${SLURM_NNODES} \
     --ntasks=${SLURM_NNODES} \
     --ntasks-per-node=1 \
     --cpus-per-task=1 \
     /bin/bash -c '
       mkdir -p "$CUDA_MPS_PIPE_DIRECTORY" "$CUDA_MPS_LOG_DIRECTORY"
       echo "[$(hostname)] starting MPS"
       echo "[$(hostname)] CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES"
       nvidia-cuda-mps-control -d
       while [ ! -e "$MPS_STOP_FILE" ]; do
         sleep 1
       done
       echo "[$(hostname)] stopping MPS"
       echo quit | nvidia-cuda-mps-control
       rm -rf "$CUDA_MPS_PIPE_DIRECTORY" "$CUDA_MPS_LOG_DIRECTORY"
    ' >>XXXoutfileXXX 2>>XXXerrfileXXX &
MPS_STEP_PID=$!
 
### Give the MPS control daemons time to start
sleep 2
 
### Run RELION
SECONDS=0
command="{ { srun --overlap --mpi=pmix_v3 ${cmd:0:-2} | tee -a XXXoutfileXXX ; } 3>&2 2>&1 1>&3 | tee -a XXXerrfileXXX ; } 3>&2 2>&1 1>&3"
eval $command
RELION_RC=$?
 
### Stop MPS on all allocated nodes
touch "$MPS_STOP_FILE"
 
### Wait until the MPS srun step has completely finished
wait "$MPS_STEP_PID"
rm -f "$MPS_STOP_FILE"
 
cat <<_EOF_>>XXXoutfileXXX
----------------------------------------------
use relion: `readlink -f $run`
end time: `date '+%y%m%d %H:%M:%S'`  (spend  `printf "%02d:%02d:%02d\n" $(($SECONDS/3600)) $((($SECONDS%3600)/60)) $(($SECONDS%60))`)
RELION exit code: ${RELION_RC}
----------------------------------------------
done
_EOF_
 
exit "$RELION_RC"
最新の60件
2026-11-17 2026-11-16 2026-11-14 2026-11-12 2026-11-02 2026-11-01 2026-10-30 2026-10-28 2026-10-26 2026-10-21 2026-10-20 2026-10-18 2026-10-12 2026-10-08 2026-10-07 2026-10-06 2026-10-04 2026-10-02 2026-10-01 2026-09-30 2026-09-28 2026-09-27 2026-09-26 2026-09-22 2026-09-17 2026-09-11 2026-09-06 2026-08-27 2026-08-24 2026-08-20 2026-08-19 2026-08-17 2026-08-16 2026-08-15 2026-08-13 2026-08-12 2026-08-11 2026-08-09 2026-07-30 2026-07-25 2026-07-24 2026-07-20

edit


トップ   編集 差分 履歴 添付 複製 名前変更 リロード   新規 一覧 検索 最終更新   ヘルプ   最終更新のRSS
Last-modified: 2026-10-02 (金) 14:15:15