Relion/slurm

#!/bin/bash
#SBATCH --partition=XXXqueueXXX
#SBATCH --ntasks=XXXmpinodesXXX
#SBATCH --cpus-per-task=XXXthreadsXXX
#SBATCH --error=XXXerrfileXXX
#SBATCH --output=XXXoutfileXXX
#SBATCH --job-name=XXXnameXXX
#SBATCH --gres=gpu:4
#################################
 
cd $SLURM_SUBMIT_DIR
 
### Environment Modules
. /etc/profile.d/modules.sh
module load mpi/openmpi-x86_64
module load relion
 
### requested resources
cat <<_EOF_>XXXoutfileXXX
----------------------------------------------
start  `date`
----------------------------------------------
Queue name                         XXXqueueXXX
used nodes                         ${SLURM_JOB_NODELIST}
  +--NumberOfProcess[MPI procs]    XXXmpinodesXXX
     +-- NumberOfThreads[threads]  XXXthreadsXXX
_EOF_
 
### check command
cmd_=$(/bin/cat <<_EOF_
XXXcommandXXX
_EOF_
)
cmd=$(echo "$cmd_" | sed -z 's/\n/\&\&/g')
run=$(echo $cmd|awk '{print $1}')
 
### mpirun option
opt=" --display-map --bind-to none --mca plm_rsh_agent rsh --map-by node --oversubscribe "
 
### check GPU
np=XXXmpinodesXXX
localnodefile=1
if [ ${run##*/} =  "relion_refine_mpi" ]; then
   if [[ $cmd = *--gpu* ]]; then
      np=$(expr $np + 1)
   fi
fi
 
### Run
SECONDS=0
command="{ { mpirun $opt --host ${SLURMD_NODENAME} -np $np ${cmd:0:-2} | tee -a XXXoutfileXXX ; } 3>&2 2>&1 1>&3 | tee -a XXXerrfileXXX ; } 3>&2 2>&1 1>&3"
 
eval $command
 
cat <<_EOF_>>XXXoutfileXXX
----------------------------------------------
use relion: `readlink -f $run`
end time: `date '+%y%m%d %H:%M:%S'`  (spend  `printf "%02d:%02d:%02d\n" $(($SECONDS/3600)) $((($SECONDS%3600)/60)) $(($SECONDS%60))`)
----------------------------------------------
done
_EOF_