NVIDIA MPS(Multi-Process Service) に対応させた sbatch.sh です
ロジック的には各ノードで 「nvidia-cuda-mps-control -d」を起動させ、その中でrelion計算を動かして、計算が終わったらストップファイル(MPS_STOP_FILE)を作らせて、
while [ ! -e "$MPS_STOP_FILE" ]; do
sleep 1
doneで拾って、 「nvidia-cuda-mps-control -d」の停止作業に入る。そんな感じです。
っでその「sbatch.sh」
#!/bin/bash
#SBATCH --ntasks=XXXmpinodesXXX # Number of MPI procs
#SBATCH --cpus-per-task=XXXthreadsXXX # Number of threads
#SBATCH --partition=XXXqueueXXX # Queue name
#SBATCH --nodes=XXXnodesXXX # CEIL( XXXcoresXXX / XXXdedicatedXXX ) 必要なcore数を1台当たりのcore数で割って切り上げ. ノード数
#SBATCH --error=XXXerrfileXXX
#SBATCH --output=XXXoutfileXXX
#SBATCH --job-name=XXXnameXXX
cd $SLURM_SUBMIT_DIR
### Environment Modules
. /etc/profile.d/modules.sh
module load mpi/openmpi-x86_64
module load relion/XXXversionXXX
### requested resources
cat <<_EOF_>XXXoutfileXXX
----------------------------------------------
start `date`
----------------------------------------------
Queue name XXXqueueXXX
required cores XXXcoresXXX :( XXXmpinodesXXX[mpi] * XXXthreadsXXX[threads] )
nodes XXXnodesXXX :CEIL( XXXcoresXXX / XXXdedicatedXXX ) :( ${SLURM_JOB_NODELIST} )
_EOF_
### check command
cmd_=$(/bin/cat <<_EOF_
XXXcommandXXX
_EOF_
)
cmd=$(echo "$cmd_" | sed -z 's/\n/\&\&/g')
run=$(echo $cmd|awk '{print $1}')
### OpenMPI
export OMPI_MCA_pml=ob1
export OMPI_MCA_btl=self,vader,tcp
### NVIDIA MPS
export CUDA_MPS_PIPE_DIRECTORY=/tmp/nvidia-mps-${SLURM_JOB_ID}
export CUDA_MPS_LOG_DIRECTORY=/tmp/nvidia-mps-log-${SLURM_JOB_ID}
# STOP file must be on a filesystem shared by all compute nodes
export MPS_STOP_FILE=${SLURM_SUBMIT_DIR}/.nvidia-mps-stop-${SLURM_JOB_ID}
rm -f "$MPS_STOP_FILE"
cat <<_EOF_>>XXXoutfileXXX
----------------------------------------------
NVIDIA MPS start
----------------------------------------------
_EOF_
### Start one MPS control daemon on each allocated node.
### --overlap is required because this step remains alive while RELION runs.
srun --overlap \
--nodes=${SLURM_NNODES} \
--ntasks=${SLURM_NNODES} \
--ntasks-per-node=1 \
--cpus-per-task=1 \
/bin/bash -c '
mkdir -p "$CUDA_MPS_PIPE_DIRECTORY" "$CUDA_MPS_LOG_DIRECTORY"
echo "[$(hostname)] starting MPS"
echo "[$(hostname)] CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES"
nvidia-cuda-mps-control -d
while [ ! -e "$MPS_STOP_FILE" ]; do
sleep 1
done
echo "[$(hostname)] stopping MPS"
echo quit | nvidia-cuda-mps-control
rm -rf "$CUDA_MPS_PIPE_DIRECTORY" "$CUDA_MPS_LOG_DIRECTORY"
' >>XXXoutfileXXX 2>>XXXerrfileXXX &
MPS_STEP_PID=$!
### Give the MPS control daemons time to start
sleep 2
### Run RELION
SECONDS=0
command="{ { srun --overlap --mpi=pmix_v3 ${cmd:0:-2} | tee -a XXXoutfileXXX ; } 3>&2 2>&1 1>&3 | tee -a XXXerrfileXXX ; } 3>&2 2>&1 1>&3"
eval $command
RELION_RC=$?
### Stop MPS on all allocated nodes
touch "$MPS_STOP_FILE"
### Wait until the MPS srun step has completely finished
wait "$MPS_STEP_PID"
rm -f "$MPS_STOP_FILE"
cat <<_EOF_>>XXXoutfileXXX
----------------------------------------------
use relion: `readlink -f $run`
end time: `date '+%y%m%d %H:%M:%S'` (spend `printf "%02d:%02d:%02d\n" $(($SECONDS/3600)) $((($SECONDS%3600)/60)) $(($SECONDS%60))`)
RELION exit code: ${RELION_RC}
----------------------------------------------
done
_EOF_
exit "$RELION_RC"