slurm

Torque:
   queue名称 + node(マシン数) + ppn(processors per node)
 
Slurm:
   queue名称 + ntasks(MPI数)

[root@slurm ~]# id munge
uid=5001(munge) gid=5001(munge) groups=5001(munge)
 
[root@slurm ~]# dnf install munge

[root@slurm ~]# /usr/sbin/create-munge-key
Generating a pseudo-random key using /dev/urandom completed.
[root@slurm ~]# ls -l /etc/munge/
total 4
-r--------. 1 munge munge 1024 Dec 18 01:52 munge.key
[root@slurm ~]# systemctl enable munge.service --now
Created symlink /etc/systemd/system/multi-user.target.wants/munge.service → /usr/lib/systemd/system/munge.service.
[root@slurm ~]#

[root@slurm ~]# ssh -l root n1 'apt update && apt install munge -y'       *ubuntu20.04ですが、rootを許可してsshのroot接続も許可してます
 
[root@slurm ~]# scp /etc/munge/munge.key root@n1:/etc/munge/munge.key
[root@slurm ~]# ssh -l root n1 'chown munge. /etc/munge/munge.key'

[illya@slurm-client ~]$ sinfo -N -l
Sun Dec 25 20:56:52 2022
NODELIST   NODES PARTITION       STATE CPUS    S:C:T MEMORY TMP_DISK WEIGHT AVAIL_FE REASON
n1             1    workq*        idle 1       1:1:1  16001        0      1   (null) none
n2             1    workq*        idle 2       1:2:1  15732        0      1   (null) none
n3             1    workq*        idle 4       1:4:1  16000        0      1   (null) none
[illya@slurm-client ~]$

[illya@slurm-client ~]$ vi a.sh
#!/bin/bash
hostname
echo "sleeping..."
sleep 10
echo "done!"
 
[illya@slurm-client ~]$ sbatch a.sh    <-- ジョブファイル「a.sh」を投入
Submitted batch job 41
 
[illya@slurm-client ~]$ squeue         <-- 確認
             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)
                41     workq     a.sh    illya  R       0:01      1 n1
(結果確認)
[illya@slurm-client ~]$ cat slurm-41.out
n1
sleeping...
done!
[illya@slurm-client ~]$

[illya@slurm-client ~]$ srun --pty bash
illya@n1:~$ 
illya@n1:~$ sinfo
PARTITION AVAIL  TIMELIMIT  NODES  STATE NODELIST
workq*       up   infinite      1  alloc n1
workq*       up   infinite      2   idle n[2-3]
illya@n1:~$ exit
 
(n3を指定して)
[illya@slurm-client ~]$ srun --nodelist=n3 --pty bash
illya@n3:~$ sinfo
PARTITION AVAIL  TIMELIMIT  NODES  STATE NODELIST
workq*       up   infinite      1    mix n3               <-- mixなのでまだ余裕がある
workq*       up   infinite      2   idle n[1-2]
illya@n3:~$ exit
 
(4coreを使うと宣言)
[illya@slurm-client ~]$ srun  --ntasks=4 --pty bash
illya@n3:~$ sinfo
PARTITION AVAIL  TIMELIMIT  NODES  STATE NODELIST
workq*       up   infinite      1  alloc n3               <-- allocとなって全て払い下げられた.
workq*       up   infinite      2   idle n[1-2]
illya@n3:~$ exit
[illya@slurm-client ~]$

[illya@client ~]$ vi a.sh
#SBATCH -J test_run           # --job-name=test_run
#SBATCH -o output_%j.txt      # --output=output_test_run.txt
#SBATCH -e errors_%j.txt      # --error=errors_test_run.txt
#SBATCH -p batch              # いわゆる使用するキュー名
#SBATCH -n 4                  # --ntasks=4 このジョブで4プロセスを実行する
#SBATCH -c 2                  # --cpus-per-task=2 １プロセス当たり２core使用する
 
mpirun mpirun.exe
[illya@client ~]$
[illya@client ~]$ sbatch a.sh    #これで実行される

[illya@slurm-client ~]$ sinfo -N -o "%N %G"
NODELIST GRES
n1 gpu:1(S:0)
n2 gpu:1(S:0)
n3 (null)
[illya@slurm-client ~]$
[illya@slurm-client ~]$ srun --gres=gpu:1  --pty nvidia-smi -L
GPU 0: NVIDIA RTX A2000 (UUID: GPU-23cc3ee7-31d3-a068-2f61-5aa00052d084)
 
[illya@slurm-client ~]$ srun --gres=gpu:1 -w n1 --pty bash
illya@n1:~$ nvidia-smi -L
GPU 0: NVIDIA RTX A2000 (UUID: GPU-23cc3ee7-31d3-a068-2f61-5aa00052d084)
illya@n1:~$ exit
exit
[illya@slurm-client ~]$ srun --gres=gpu:1 -w n2 --pty bash
[illya@n2 ~]$ nvidia-smi -L
GPU 0: NVIDIA T600 (UUID: GPU-01226552-df75-5989-c2f3-3550cdc3427d)
[illya@n2 ~]$ exit
exit
[illya@slurm-client ~]$

[illya@slurm-client ~]$ srun --gres=gpu:1 -w s --pty bash
[illya@s ~]$ nvidia-smi -L
GPU 0: NVIDIA GeForce GT 710 (UUID: GPU-f0753e20-06f7-695a-f325-b5b6342393ba)
[illya@s ~]$ exit
exit
[illya@slurm-client ~]$
[illya@slurm-client ~]$ srun --gres=gpu:2 -w s --pty bash
[illya@s ~]$ nvidia-smi -L
GPU 0: NVIDIA GeForce GT 710 (UUID: GPU-f0753e20-06f7-695a-f325-b5b6342393ba)
GPU 1: NVIDIA GeForce GT 710 (UUID: GPU-34c49ab7-b79d-e341-93e2-1659e9bd9e57)
[illya@s ~]$ exit
exit
[illya@slurm-client ~]$

[illya@slurm-client ~]$ srun --gres=gpu:1 --pty nvidia-smi -L
slurmstepd: error: no gpu utilization TRES! This should never happen
GPU 0: NVIDIA GeForce GT 710 (UUID: GPU-f0753e20-06f7-695a-f325-b5b6342393ba)
 
[illya@slurm-client ~]$