cryoSPARC/qsub

[saber@cryosparc ~]$ cryosparcm cluster example
Writing example cluster_info.json and cluster_script.sh to current dir
Unknown cluster type. Supported templates are:
  pbs
  slurm
  gridengine
Any cluster scheduler is supported, but you may have to write your own custom submission script.
 
[saber@cryosparc ~]$

[saber@cryosparc ~]$ cryosparcm cluster example pbs
Writing example cluster_info.json and cluster_script.sh to current dir
[saber@cryosparc ~]$

[saber@cryosparc ~]$ cat cluster_info.json
{
    "name" : "pbscluster",
    "worker_bin_path" : "/path/to/cryosparc_worker/bin/cryosparcw",
    "cache_path" : "/path/to/local/SSD/on/cluster/nodes"
    "send_cmd_tpl" : "ssh loginnode {{ command } }",
    "qsub_cmd_tpl" : "qsub {{ script_path_abs } }",
    "qstat_cmd_tpl" : "qstat -as {{ cluster_job_id } }",
    "qdel_cmd_tpl" : "qdel {{ cluster_job_id } }",
    "qinfo_cmd_tpl" : "qstat -q",
    "transfer_cmd_tpl" : "scp {{ src_path } } loginnode:{{ dest_path } }"
}
[saber@cryosparc ~]$

[cryosparc@c ~]$ cat cluster_script.sh
[saber@cryosparc ~]$ cat cluster_script.sh
#!/usr/bin/env bash
#### cryoSPARC cluster submission script template for PBS
## Available variables:
## {{ run_cmd } }            - the complete command string to run the job
## {{ num_cpu } }            - the number of CPUs needed
## {{ num_gpu } }            - the number of GPUs needed.
##                            Note: the code will use this many GPUs starting from dev id 0
##                                  the cluster scheduler or this script have the responsibility
##                                  of setting CUDA_VISIBLE_DEVICES so that the job code ends up
##                                  using the correct cluster-allocated GPUs.
## {{ ram_gb } }             - the amount of RAM needed in GB
## {{ job_dir_abs } }        - absolute path to the job directory
## {{ project_dir_abs } }    - absolute path to the project dir
## {{ job_log_path_abs } }   - absolute path to the log file for the job
## {{ worker_bin_path } }    - absolute path to the cryosparc worker command
## {{ run_args } }           - arguments to be passed to cryosparcw run
## {{ project_uid } }        - uid of the project
## {{ job_uid } }            - uid of the job
## {{ job_creator } }        - name of the user that created the job (may contain spaces)
## {{ cryosparc_username } } - cryosparc username of the user that created the job (usually an email)
##
## What follows is a simple PBS script:
 
#PBS -N cryosparc_{{ project_uid } }_{{ job_uid } }
#PBS -l select=1:ncpus={{ num_cpu } }:ngpus={{ num_gpu } }:mem={{ (ram_gb*1000)|int } }mb:gputype=P100
#PBS -o {{ job_dir_abs } }
#PBS -e {{ job_dir_abs } }
 
available_devs=""
for devidx in $(seq 1 16);
do
    if [[ -z $(nvidia-smi -i $devidx --query-compute-apps=pid --format=csv,noheader) ]] ; then
        if [[ -z "$available_devs" ]] ; then
            available_devs=$devidx
        else
            available_devs=$available_devs,$devidx
        fi
    fi
done
export CUDA_VISIBLE_DEVICES=$available_devs
 
{{ run_cmd } }
 
[saber@cryosparc ~]$

[saber@cryosparc ~]$ cryosparcm cluster connect
 :
Successfully added cluster lane pbscluster
 
(確認は下記コマンドで)
[saber@cryosparc ~]$ cryosparcm cli "get_scheduler_lanes()"
 
(削除はcluster_info.jsonのnameを使います)
[saber@cryosparc ~]$ cryosparcm cluster remove pbscluster

qsub -q workq -l select=1:ncpus=8:host=s -I
 「1ノードでcore数が8のhost名がsなノードをインターラクティヴモードで接続」

[saber@cryosparc ~]$ cryosparcm cluster connect
 :
Successfully added cluster lane run-on-s
[saber@cryosparc ~]$

cryosparcm cli "get_scheduler_lanes()"

cryosparcm cli "remove_scheduler_lane('em-g1-a01')"

list=(${CUDA_VISIBLE_DEVICES//,/ })
n=0
available_devs=""
for i in ${list[@]} ; do
   if [ $n -eq 0 ]; then
     available_devs=$n
   else
     available_devs=$available_devs,$n
   fi
   n=$(( $n + 1 ))
done
export CUDA_VISIBLE_DEVICES=$available_devs