![[PukiWiki]](image/picc.png "[PukiWiki]")
Alphafoldは docker にて稼働させてますが、もうひとつのコンテナ技術 Singularity で行う場合のお話
本家様 alphafold_singularity https://github.com/prehensilecode/alphafold_singularity
ここではこの alphafold_singularity を構築してみる.
Alphafoldにはdocker向けのレシピのDockerfileがある. 同様にalphafold_singularity のgitにはSingularity向けのレシピ def ファイルがある
これら両者を比べたのが下記で、左は alphafld のDokerfile、右は alphafold_singularity のSingularity.def です
alphafold(20250127,alphafold/docker/Dockerfile,6350ddd) alphafold_singularity(20250127,alphafold_singularity/Singularity.def,86dba48)
# Copyright 2021 DeepMind Technologies Limited Bootstrap: docker
# From: nvidia/cuda:11.1.1-cudnn8-runtime-ubuntu18.04
# Licensed under the Apache License, Version 2.0 (the "License"); Stage: spython-base
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at %files
# . /app/alphafold
# http://www.apache.org/licenses/LICENSE-2.0 %post
# # Copyright 2023 David Chin
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, # This file is part of alphafold_singularity.
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and # alphafold_singularity is free software: you can redistribute it and/or
# limitations under the License. # modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation, either version 3 of the
ARG CUDA=12.2.2 # License, or (at your option) any later version.
FROM nvidia/cuda:${CUDA}-cudnn8-runtime-ubuntu20.04 #
# FROM directive resets ARGS, so we specify again (the value is retained if # alphafold_singularity is distributed in the hope that it will be
# previously set). # useful, but WITHOUT ANY WARRANTY; without even the implied warranty
ARG CUDA # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
# Use bash to support string substitution. #
SHELL ["/bin/bash", "-o", "pipefail", "-c"] # You should have received a copy of the GNU General Public License
# along with alphafold_singularity. If not, see <https://www.gnu.org/licenses/>.
ARG DEBIAN_FRONTEND=noninteractive
RUN apt-get update --quiet \
&& apt-get install --no-install-recommends --yes --quiet \ # FROM directive resets ARGS, so we specify again (the value is retained if
build-essential \ # previously set).
cmake \
cuda-command-line-tools-$(cut -f1,2 -d- <<< ${CUDA//./-}) \ apt-get update \
git \ && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
hmmer \ build-essential \
kalign \ cmake \
tzdata \ cuda-command-line-tools-11-1 \
wget \ git \
&& rm -rf /var/lib/apt/lists/* \ hmmer \
&& apt-get autoremove --yes \ kalign \
&& apt-get clean tzdata \
wget \
# Compile HHsuite from source. && rm -rf /var/lib/apt/lists/* \
RUN git clone --branch v3.3.0 --single-branch https://github.com/soedinglab/hh-suite.git /tmp/hh-suite \ && apt-get autoremove -y \
&& mkdir /tmp/hh-suite/build \ && apt-get clean
&& pushd /tmp/hh-suite/build \
&& cmake -DCMAKE_INSTALL_PREFIX=/opt/hhsuite .. \ # Compile HHsuite from source.
&& make --jobs 4 && make install \ /bin/rm -rf /tmp/hh-suite \
&& ln -s /opt/hhsuite/bin/* /usr/bin \ && git clone --branch v3.3.0 https://github.com/soedinglab/hh-suite.git /tmp/hh-suite \
&& popd \ && mkdir /tmp/hh-suite/build \
&& rm -rf /tmp/hh-suite && cd /tmp/hh-suite/build \
&& cmake -DCMAKE_INSTALL_PREFIX=/opt/hhsuite .. \
# Install Miniconda package manager. && make -j 4 && make install \
RUN wget -q -P /tmp \ && ln -s /opt/hhsuite/bin/* /usr/bin \
https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ && cd / \
&& bash /tmp/Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda \ && /bin/rm -rf /tmp/hh-suite
&& rm /tmp/Miniconda3-latest-Linux-x86_64.sh
# Install Miniconda package manager.
# Install Conda packages. wget -q -P /tmp \
ENV PATH="/opt/conda/bin:$PATH" https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
ENV LD_LIBRARY_PATH="/opt/conda/lib:$LD_LIBRARY_PATH" && bash /tmp/Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda \
RUN conda install --quiet --yes conda==24.11.1 pip python=3.11 \ && rm /tmp/Miniconda3-latest-Linux-x86_64.sh
&& conda install --quiet --yes --channel nvidia cuda=${CUDA_VERSION} \
&& conda install --quiet --yes --channel conda-forge openmm=8.0.0 pdbfixer \ # Install conda packages.
&& conda clean --all --force-pkgs-dirs --yes PATH="/opt/conda/bin:/usr/local/cuda-11.1/bin:$PATH"
conda install -qy conda==23.5.2 \
COPY . /app/alphafold && conda install -y -c conda-forge \
RUN wget -q -P /app/alphafold/alphafold/common/ \ openmm=7.7.0 \
https://git.scicore.unibas.ch/schwede/openstructure/-/raw/7102c63615b64735c4941278d92b554ec94415f8(略 cudatoolkit==11.1.1 \
pdbfixer \
# Install pip packages. pip \
RUN pip3 install --upgrade pip --no-cache-dir \ python=3.10 \
&& pip3 install -r /app/alphafold/requirements.txt --no-cache-dir \ && conda clean --all --force-pkgs-dirs --yes
&& pip3 install --upgrade --no-cache-dir \
jax==0.4.26 \ ### /bin/cp -r . /app/alphafold
jaxlib==0.4.26+cuda12.cudnn89 \
-f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html wget -q -P /app/alphafold/alphafold/common/ \
https://git.scicore.unibas.ch/schwede/openstructure/-/raw/7102c63615b64735c4941278d92b554ec94415f8/modules/mol/alg/src/stereo_chemical_props.txt
# Add SETUID bit to the ldconfig binary so that non-root users can run it.
RUN chmod u+s /sbin/ldconfig.real # Install pip packages.
# N.B. The URL specifies the list of jaxlib releases.
# Currently needed to avoid undefined_symbol error. pip3 install --upgrade pip --no-cache-dir \
RUN ln -sf /usr/lib/x86_64-linux-gnu/libffi.so.7 /opt/conda/lib/libffi.so.7 && pip3 install -r /app/alphafold/requirements.txt --no-cache-dir \
&& pip3 install --upgrade --no-cache-dir \
# We need to run `ldconfig` first to ensure GPUs are visible, due to some quirk jax==0.3.25 \
# with Debian. See https://github.com/NVIDIA/nvidia-docker/issues/1399 for jaxlib==0.3.25+cuda11.cudnn805 \
# details. -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
# ENTRYPOINT does not support easily running multiple commands, so instead we
# write a shell script to wrap them up. # Add SETUID bit to the ldconfig binary so that non-root users can run it.
WORKDIR /app/alphafold chmod u+s /sbin/ldconfig.real
RUN echo $'#!/bin/bash\n\
ldconfig\n\ ### SETUID bit does not matter: Apptainer does not allow suid commands
python /app/alphafold/run_alphafold.py "$@"' > /app/run_alphafold.sh \ ### Workaround below is to use /mnt/out/ld.so.cache for the ld cache file
&& chmod +x /app/run_alphafold.sh
ENTRYPOINT ["/app/run_alphafold.sh"] %environment
export PATH="/opt/conda/bin:/usr/local/cuda-11.1/bin:$PATH"
%runscript
cd /app/alphafold
ldconfig -C /mnt/output/ld.so.cache
exec python /app/alphafold/run_alphafold.py "$@"
# %startscript
# cd /app/alphafold
# exec python /app/alphafold/run_alphafold.py "$@"alphafold側が cuda-12.2 の ubuntu20.04 ベースになった模様.
Singularity側を下記のように修正しました
|
環境はこんな感じ
[root@rockylinux ~]# cat /etc/redhat-release
Rocky Linux release 8.7 (Green Obsidian)
[root@rockylinux ~]# cat /proc/driver/nvidia/version
NVRM version: NVIDIA UNIX x86_64 Kernel Module 515.105.01 Mon Feb 27 12:49:44 UTC 2023
GCC version: gcc version 8.5.0 20210514 (Red Hat 8.5.0-15) (GCC)
[root@rockylinux ~]#Alphafoldとalphafold_singularityを準備します
[root@rockylinux ~]# cd /apps/
[root@rockylinux apps]# git clone https://github.com/google-deepmind/alphafold
[root@rockylinux apps]# cd alphafold
[root@rockylinux alphafold]# git clone https://github.com/prehensilecode/alphafold_singularity
[root@rockylinux alphafold]# mv alphafold_singularity singularity
[root@rockylinux alphafold]# ls -lF
total 92
drwxr-xr-x 2 root root 23 Feb 16 01:09 afdb
drwxr-xr-x 7 root root 112 Feb 16 01:09 alphafold
-rw-r--r-- 1 root root 973 Feb 16 01:09 CONTRIBUTING.md
drwxr-xr-x 2 root root 69 Feb 16 01:09 docker
drwxr-xr-x 2 root root 68 Feb 16 01:09 docs
drwxr-xr-x 2 root root 54 Feb 16 01:09 imgs
-rw-r--r-- 1 root root 11358 Feb 16 01:09 LICENSE
drwxr-xr-x 2 root root 29 Feb 16 01:09 notebooks
-rw-r--r-- 1 root root 35017 Feb 16 01:09 README.md
-rw-r--r-- 1 root root 209 Feb 16 01:09 requirements.txt
-rw-r--r-- 1 root root 22575 Feb 16 01:09 run_alphafold.py
-rw-r--r-- 1 root root 4284 Feb 16 01:09 run_alphafold_test.py
drwxr-xr-x 2 root root 315 Feb 17 13:08 scripts
-rw-r--r-- 1 root root 2131 Feb 16 01:09 setup.py
drwxr-xr-x 3 root root 197 Feb 17 20:45 singularity
[root@rockylinux alphafold]#ここからはpyenv/anaconda経由で実行環境を作ります
もしtopazらで既に anaconda 環境があるのならそれを使います
git clone https://github.com/yyuu/pyenv.git /apps/pyenv
export PYENV_ROOT=/apps/pyenv
export PATH=$PYENV_ROOT/bin:$PATH
pyenv install anaconda3-2024.10-1
pyenv global anaconda3-2024.10-1
source /apps/pyenv/versions/anaconda3-2024.10-1/etc/profile.d/conda.sh
conda update --all
(既にanaconda環境があるなら)
source /apps/pyenv/versions/anaconda3-2024.10-1/etc/profile.d/conda.sh作成する環境は alphafold_singularity としてます
[root@rockylinux alphafold]# conda create -n alphafold_singularity pip -y
[root@rockylinux alphafold]# conda activate alphafold_singularity
(alphafold_singularity) [root@rockylinux alphafold]#この状態で構築を進める
(alphafold_singularity) [root@rockylinux alphafold]# which pip
/apps/pyenv/versions/anaconda3-2023.09-0/envs/alphafold_singularity/bin/pip
(alphafold_singularity) [root@rockylinux alphafold]# cat singularity/requirements.txt
(略
absl-py==1.0.0
spython==0.3.0
(alphafold_singularity) [root@rockylinux alphafold]# pip install -r singularity/requirements.txt次に Singularity パッケージをインストールして、
(alphafold_singularity) [root@rockylinux alphafold]# dnf install epel-release -y
(alphafold_singularity) [root@rockylinux alphafold]# dnf install apptainer apptainer-suidAlphafoldのSingularity コンテナイメージを作成します.
(alphafold_singularity) [root@rockylinux alphafold]# singularity build alphafold.sif singularity/Singularity.def
(alphafold_singularity) [root@rockylinux alphafold]# ls -lh alphafold.sif
-rwxr-xr-x 1 root root 4.6G Feb 17 21:29 alphafold.sif
(alphafold_singularity) [root@rockylinux alphafold]# conda deactivate
[root@rockylinux alphafold]#4.6GBほどのサイズになりました.
ここでエラーが発生するなら alphafold/docker/Dockerfile と alphafold_singularity/Singularity.def を見比べて
alphafold/docker/Dockerfileが優先になるように singularity/Singularity.def を修正します.
alphafold と alphafold_singularity は互いに連携しているわけではないので. あとalphafold_singularityに書かれた方法は一見するとバージョン番号で統制が取れるっぽいが失敗する. 目で見比べて修正した方が早い
次に計算に使用するスクリプト「/apps/alphafold/singularity/run_singularity.py」
「/apps/alphafold/singularity」にPATHを入れて「run_singularity.py」を実行すれば計算が出来るようにしたいので
行頭にcondaのpythonを入れる.
|
これで実行環境は完成. っで次は恒例の EnvironmentModules.
いつものように「/apps/modulefiles/alphafold_singularity」として記載
[root@rockylinux ~]# vi /apps/modulefiles/alphafold_singularity
#%Module1.0
#
set root /apps/pyenv/versions/anaconda3-2023.09-0/envs/alphafold_singularity
set af /apps/alphafold
prepend-path PATH $af/singularity:$root/bin
setenv ALPHAFOLD_DIR $af
[root@rockylinux ~]#[illya@rockylinux ~]$ module use --append /apps/modulefiles/
[illya@rockylinux ~]$ module load alphafold_singularity
[illya@rockylinux ~]$ run_singularity.py
/apps/alphafold/alphafold.sif
FATAL Flags parsing error:
flag --data_dir=None: Flag --data_dir must have a value other than None.
flag --fasta_paths=None: Flag --fasta_paths must have a value other than None.
flag --max_template_date=None: Flag --max_template_date must have a value other than None.
Pass --helpshort or --helpfull to see help on flags.
[saber@rockylinux ~]$
[illya@rockylinux ~]$ vi query.fasta
>dummy_sequence
GWSTELEKHREELKEFLKKEGITNVEIRIDNGRLEVRVEGGTERLKRFLEELRQKLEKKGYTVDIKIE
[illya@rockylinux ~]$ mkdir out
[illya@rockylinux ~]$ run_singularity.py --fasta_paths=query.fasta --max_template_date=2020-05-14 --data_dir=/AlphaFold --db_preset=reduced_dbs --output_dir=out
:
:
[illya@rockylinux ~]$ ls -lF ./out
total 24
-rw-r--r-- 1 illya illya 20104 Feb 17 21:47 ld.so.cache
drwxr-xr-x 3 illya illya 4096 Feb 17 22:16 query/
[illya@rockylinux ~]$
結果は $HOME/out に置かれる. --output_dir=. とすると入力ファイルのファイル名を使ったフォルダが用意されます. query.fasta --> 「query」フォルダが作られる
次にdimerでテスト
[illya@rockylinux ~]$ vi dimer.fasta
>XP_009313165.1.1
MRAAFAEARAALAEGEVPVGCVLVPVDASCAANAQLAADDDDDENKSKGSSNSNNSKKNDAVERLIAARG
RNATNREHHALAHAEFVAVEALLRELAANGQQRPASLAGYVLYVVVEPCIMCAAMLLYNRVQKVFFGCGN
PRFGGNGTVLAVHTAAGCSAPGYESSGGHRADEAVALLQEFYRHENTNAPGHKRRRKCECLNN
>XP_009313165.1.2
MRAAFAEARAALAEGEVPVGCVLVPVDASCAANAQLAADDDDDENKSKGSSNSNNSKKNDAVERLIAARG
RNATNREHHALAHAEFVAVEALLRELAANGQQRPASLAGYVLYVVVEPCIMCAAMLLYNRVQKVFFGCGN
PRFGGNGTVLAVHTAAGCSAPGYESSGGHRADEAVALLQEFYRHENTNAPGHKRRRKCECLNN
[illya@rockylinux ~]$ run_singularity.py --fasta_paths=dimer.fasta --max_template_date=2020-05-14 --data_dir=/AlphaFold \
--db_preset=reduced_dbs --model_preset=multimer --output_dir=.
[illya@rockylinux ~]$ ls -lFd dimer*
drwxr-xr-x 3 illya illya 12288 Feb 17 23:19 dimer/
-rw-r--r-- 1 illya illya 448 Feb 17 22:19 dimer.fasta
[illya@rockylinux ~]$ ls -l dimer/ | head -n 5
total 2586692
-rw-r--r-- 1 illya illya 5595 Feb 17 22:43 confidence_model_1_multimer_v3_pred_0.json
-rw-r--r-- 1 illya illya 5598 Feb 17 22:44 confidence_model_1_multimer_v3_pred_1.json
-rw-r--r-- 1 illya illya 5598 Feb 17 22:45 confidence_model_1_multimer_v3_pred_2.json
-rw-r--r-- 1 illya illya 5604 Feb 17 22:47 confidence_model_1_multimer_v3_pred_3.json
[illya@rockylinux ~]$Singularityイメージファイルならクラスターマシンとかでも比較的容易に使えます.
hmmer と hh-suite を oneAPI で最適化してみた.
事前にoffline向けの oneAPI base-toolkit と hpc-toollit を取得して
Bootstrap: docker
From: nvidia/cuda:12.2.2-cudnn8-runtime-ubuntu20.04
Stage: spython-base
%files
. /app/alphafold
%post
# Copyright 2023 David Chin
#
# This file is part of alphafold_singularity.
#
# alphafold_singularity is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# alphafold_singularity is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with alphafold_singularity. If not, see <https://www.gnu.org/licenses/>.
# FROM directive resets ARGS, so we specify again (the value is retained if
# previously set).
bash /app/alphafold/intel-oneapi-base-toolkit-2025.0.1.46_offline.sh -a -s --eula accept --intel-sw-improvement-program-consent decline \
--components intel.oneapi.lin.dpl:intel.oneapi.lin.tbb.devel:intel.oneapi.lin.dpcpp-cpp-compiler:intel.oneapi.lin.dpl:intel.oneapi.lin.ipp.devel:intel.oneapi.lin.mkl.devel:intel.oneapi.lin.tbb.devel
bash /app/alphafold/intel-oneapi-hpc-toolkit-2025.0.1.47_offline.sh -a -s --eula accept --intel-sw-improvement-program-consent decline \
--components intel.oneapi.lin.mpi.devel:intel.oneapi.lin.ifort-compiler
rm -f /app/alphafold/intel-oneapi-base-toolkit-2025.0.1.46_offline.sh
rm -f /app/alphafold/intel-oneapi-hpc-toolkit-2025.0.1.47_offline.sh
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
build-essential \
cmake \
cuda-command-line-tools-12-2 \
git \
kalign \
tzdata \
autoconf \
wget \
&& rm -rf /var/lib/apt/lists/* \
&& apt-get autoremove -y \
&& apt-get clean
PATH="/opt/intel/oneapi/compiler/2025.0/bin:$PATH"
LD_LIBRARY_PATH="/opt/intel/oneapi/mkl/2025.0/lib:/opt/intel/oneapi/umf/0.9/lib/:/opt/intel/oneapi/compiler/2025.0/opt/compiler/lib:/opt/intel/oneapi/compiler/2025.0/lib:/opt/intel/oneapi/tbb/2022.0/lib:$LD_LIBRARY_PATH"
# Compile HHsuite from source.
/bin/rm -rf /tmp/hh-suite \
&& git clone https://github.com/IntelLabs/hh-suite.git /tmp/hh-suite \
&& mkdir /tmp/hh-suite/build \
&& cd /tmp/hh-suite/build \
&& cmake -DCMAKE_INSTALL_PREFIX=/opt/hhsuite -DCMAKE_CXX_COMPILER="icpx" -DCMAKE_CXX_FLAGS_RELEASE="-O3 -march=native" .. \
&& make -j 4 && make install \
&& ln -s /opt/hhsuite/bin/* /usr/bin \
&& cd / \
&& /bin/rm -rf /tmp/hh-suite
/bin/rm -rf /tmp/hmmer \
&& git clone --recursive https://github.com/IntelLabs/hmmer.git /tmp/hmmer \
&& cd /tmp/hmmer \
&& cp easel_makefile.in easel/Makefile.in \
&& cd easel && autoconf && ./configure --prefix=/opt/hmmer && cd .. \
&& autoconf && CC=gcc CFLAGS="-O3 -march=native -fPIC" ./configure --prefix=/opt/hmmer \
&& make -j 4 && make install \
&& ln -s /opt/hmmer/bin/* /usr/bin \
&& cd / \
&& /bin/rm -rf /tmp/hmmer
# Install Miniconda package manager.
wget -q -P /tmp \
https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
&& bash /tmp/Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda \
&& rm /tmp/Miniconda3-latest-Linux-x86_64.sh
# Install conda packages.
PATH="/opt/conda/bin:/usr/local/cuda-12.2/bin:$PATH"
conda install -qy conda=24 pip python=3.11 \
&& conda install -y -c conda-forge -c nvidia \
openmm=8.0.0 \
cuda=12.2.2 \
pdbfixer \
&& conda clean --all --force-pkgs-dirs --yes
### /bin/cp -r . /app/alphafold
wget -q -P /app/alphafold/alphafold/common/ \
https://git.scicore.unibas.ch/schwede/openstructure/-/raw/7102c63615b64735c4941278d92b554ec94415f8/modules/mol/alg/src/stereo_chemical_props.txt
# Install pip packages.
# N.B. The URL specifies the list of jaxlib releases.
pip3 install --upgrade pip --no-cache-dir \
&& pip3 install -r /app/alphafold/requirements.txt --no-cache-dir \
&& pip3 install --upgrade --no-cache-dir \
jax==0.4.26 \
jaxlib==0.4.26+cuda12.cudnn89 \
-f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
# Add SETUID bit to the ldconfig binary so that non-root users can run it.
chmod u+s /sbin/ldconfig.real
### SETUID bit does not matter: Apptainer does not allow suid commands
### Workaround below is to use /mnt/out/ld.so.cache for the ld cache file
%environment
export PATH="/opt/conda/bin:/usr/local/cuda-12.2/bin:$PATH"
export LD_LIBRARY_PATH="/opt/intel/oneapi/compiler/2025.0/lib/:$LD_LIBRARY_PATH"
%runscript
cd /app/alphafold
ldconfig -C /mnt/output/ld.so.cache
exec python /app/alphafold/run_alphafold.py "$@"
# %startscript
# cd /app/alphafold
# exec python /app/alphafold/run_alphafold.py "$@"
