![[PukiWiki]](image/picc.png "[PukiWiki]")
Alphafoldは docker にて稼働させてますが、もうひとつのコンテナ技術 Singularity で行う場合のお話
本家様 alphafold_singularity https://github.com/prehensilecode/alphafold_singularity
ここではこの alphafold_singularity を構築してみる.
Alphafoldにはdocker向けのレシピのDockerfileがある. 同様にalphafold_singularity のgitにはSingularity向けのレシピ def ファイルがある
これら両者を比べたのが下記で、左は alphafld のDokerfile、右は alphafold_singularity のSingularity.def です
alphafold(20250826,alphafold/docker/Dockerfile,c095a657ade1a927bdfb424cee860c6a56ee0d7c) alphafold_singularity(20250826,alphafold_singularity/Singularity.def,2c31c9a3b509360ed0d51519cf8455c7875686d7)
# Copyright 2021 DeepMind Technologies Limited Bootstrap: docker
# From: nvidia/cuda:11.1.1-cudnn8-runtime-ubuntu18.04
# Licensed under the Apache License, Version 2.0 (the "License"); Stage: spython-base
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at %files
# . /app/alphafold
# http://www.apache.org/licenses/LICENSE-2.0 %post
# # Copyright 2023 David Chin
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, # This file is part of alphafold_singularity.
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and # alphafold_singularity is free software: you can redistribute it and/or
# limitations under the License. # modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation, either version 3 of the
ARG CUDA=12.2.2 # License, or (at your option) any later version.
FROM nvidia/cuda:${CUDA}-cudnn8-runtime-ubuntu20.04 #
# FROM directive resets ARGS, so we specify again (the value is retained if # alphafold_singularity is distributed in the hope that it will be
# previously set). # useful, but WITHOUT ANY WARRANTY; without even the implied warranty
ARG CUDA # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
# Use bash to support string substitution. #
SHELL ["/bin/bash", "-o", "pipefail", "-c"] # You should have received a copy of the GNU General Public License
# along with alphafold_singularity. If not, see <https://www.gnu.org/licenses/>.
ARG DEBIAN_FRONTEND=noninteractive
RUN apt-get update --quiet \
&& apt-get install --no-install-recommends --yes --quiet \ # FROM directive resets ARGS, so we specify again (the value is retained if
build-essential \ # previously set).
cmake \
cuda-command-line-tools-$(cut -f1,2 -d- <<< ${CUDA//./-}) \ apt-get update \
git \ && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
hmmer \ build-essential \
kalign \ cmake \
tzdata \ cuda-command-line-tools-11-1 \
wget \ git \
&& rm -rf /var/lib/apt/lists/* \ hmmer \
&& apt-get autoremove --yes \ kalign \
&& apt-get clean tzdata \
wget \
# Compile HHsuite from source. && rm -rf /var/lib/apt/lists/* \
RUN git clone --branch v3.3.0 --single-branch https://github.com/soedinglab/hh-suite.git /tmp/hh-suite \ && apt-get autoremove -y \
&& mkdir /tmp/hh-suite/build \ && apt-get clean
&& pushd /tmp/hh-suite/build \
&& cmake -DCMAKE_INSTALL_PREFIX=/opt/hhsuite .. \ # Compile HHsuite from source.
&& make -j && make install \ /bin/rm -rf /tmp/hh-suite \
&& ln -s /opt/hhsuite/bin/* /usr/bin \ && git clone --branch v3.3.0 https://github.com/soedinglab/hh-suite.git /tmp/hh-suite \
&& popd \ && mkdir /tmp/hh-suite/build \
&& rm -rf /tmp/hh-suite && cd /tmp/hh-suite/build \
&& cmake -DCMAKE_INSTALL_PREFIX=/opt/hhsuite .. \
# Install Miniconda package manager. && make -j 4 && make install \
RUN wget -q -P /tmp \ && ln -s /opt/hhsuite/bin/* /usr/bin \
https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ && cd / \
&& bash /tmp/Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda \ && /bin/rm -rf /tmp/hh-suite
&& rm /tmp/Miniconda3-latest-Linux-x86_64.sh
# Install Miniconda package manager.
# Install Conda packages. wget -q -P /tmp \
ENV PATH="/opt/conda/bin:$PATH" https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
ENV LD_LIBRARY_PATH="/opt/conda/lib:$LD_LIBRARY_PATH" && bash /tmp/Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda \
ENV CONDA_PLUGINS_AUTO_ACCEPT_TOS="yes" && rm /tmp/Miniconda3-latest-Linux-x86_64.sh
RUN conda install --quiet --yes conda==24.11.1 pip python=3.11 \
&& conda install --quiet --yes --channel nvidia cuda=${CUDA_VERSION} \ # Install conda packages.
&& conda install --quiet --yes --channel conda-forge openmm=8.0.0 pdbfixer \ PATH="/opt/conda/bin:/usr/local/cuda-11.1/bin:$PATH"
&& conda clean --all --force-pkgs-dirs --yes conda install -qy conda==23.5.2 \
&& conda install -y -c conda-forge \
COPY . /app/alphafold openmm=7.7.0 \
RUN wget -q -P /app/alphafold/alphafold/common/ \ cudatoolkit==11.1.1 \
https://git.scicore.unibas.ch/schwede/openstructure/-/raw/7102c63615b64735c4941278d92b554ec94415f8(略 pdbfixer \
pip \
# Install pip packages. python=3.10 \
RUN pip3 install --upgrade pip --no-cache-dir \ && conda clean --all --force-pkgs-dirs --yes
&& pip3 install -r /app/alphafold/requirements.txt --no-cache-dir \
&& pip3 install --upgrade --no-cache-dir \ ### /bin/cp -r . /app/alphafold
jax==0.4.26 \
jaxlib==0.4.26+cuda12.cudnn89 \ wget -q -P /app/alphafold/alphafold/common/ \
-f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html https://git.scicore.unibas.ch/schwede/openstructure/-/raw/7102c63615b64735c4941278d92b554ec94415f8/modules/mol/alg/src/stereo_chemical_props.txt
# Add SETUID bit to the ldconfig binary so that non-root users can run it. # Install pip packages.
RUN chmod u+s /sbin/ldconfig.real # N.B. The URL specifies the list of jaxlib releases.
pip3 install --upgrade pip --no-cache-dir \
# Currently needed to avoid undefined_symbol error. && pip3 install -r /app/alphafold/requirements.txt --no-cache-dir \
RUN ln -sf /usr/lib/x86_64-linux-gnu/libffi.so.7 /opt/conda/lib/libffi.so.7 && pip3 install --upgrade --no-cache-dir \
jax==0.3.25 \
# We need to run `ldconfig` first to ensure GPUs are visible, due to some quirk jaxlib==0.3.25+cuda11.cudnn805 \
# with Debian. See https://github.com/NVIDIA/nvidia-docker/issues/1399 for -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
# details.
# ENTRYPOINT does not support easily running multiple commands, so instead we # Add SETUID bit to the ldconfig binary so that non-root users can run it.
# write a shell script to wrap them up. chmod u+s /sbin/ldconfig.real
WORKDIR /app/alphafold
RUN echo $'#!/bin/bash\n\ ### SETUID bit does not matter: Apptainer does not allow suid commands
ldconfig\n\ ### Workaround below is to use /mnt/out/ld.so.cache for the ld cache file
python /app/alphafold/run_alphafold.py "$@"' > /app/run_alphafold.sh \
&& chmod +x /app/run_alphafold.sh %environment
ENTRYPOINT ["/app/run_alphafold.sh"] export PATH="/opt/conda/bin:/usr/local/cuda-11.1/bin:$PATH"
%runscript
cd /app/alphafold
ldconfig -C /mnt/output/ld.so.cache
exec python /app/alphafold/run_alphafold.py "$@"
# %startscript
# cd /app/alphafold
# exec python /app/alphafold/run_alphafold.py "$@"
alphafold側が cuda-12.2.2 の ubuntu20.04 ベースになった模様.
Singularity側を下記のように修正しました. 「Miniconda3-latest-Linux-x86_64.sh」を特定のversionに変更してます. python3.11対応となる「Miniconda3-py311_23.5.0-3-Linux-x86_64.sh」にしました.
加えて「conda config --add channels defaults」と「pip3 install --upgrade pipenv」の追記をしてます.
|
環境はこんな感じ
[root@rockylinux9 ~]# cat /etc/redhat-release
Rocky Linux release 9.6 (Blue Onyx)
[root@rockylinux9 ~]# cat /proc/driver/nvidia/version
NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 570.181 Release Build (dvs-builder@U22-I3-AF02-20-5) Wed Jul 30 18:41:07 UTC 2025
GCC version: gcc version 11.5.0 20240719 (Red Hat 11.5.0-5) (GCC)
[root@rockylinux9 ~]#Alphafoldとalphafold_singularityを準備します
[root@rockylinux9 ~]# cd /apps/
[root@rockylinux9 apps]# git clone https://github.com/google-deepmind/alphafold
[root@rockylinux9 apps]# cd alphafold
[root@rockylinux9 alphafold]# git clone https://github.com/prehensilecode/alphafold_singularity
[root@rockylinux9 alphafold]# mv alphafold_singularity singularity
[root@rockylinux9 alphafold]# ls -lF
total 100
drwxr-xr-x. 2 root root 23 Aug 26 19:53 afdb/
drwxr-xr-x. 7 root root 112 Aug 26 19:53 alphafold/
-rw-r--r--. 1 root root 936 Aug 26 19:53 conftest.py
-rw-r--r--. 1 root root 973 Aug 26 19:53 CONTRIBUTING.md
drwxr-xr-x. 2 root root 69 Aug 26 19:53 docker/
drwxr-xr-x. 2 root root 68 Aug 26 19:53 docs/
drwxr-xr-x. 2 root root 54 Aug 26 19:53 imgs/
-rw-r--r--. 1 root root 11358 Aug 26 19:53 LICENSE
drwxr-xr-x. 2 root root 29 Aug 26 19:53 notebooks/
-rw-r--r--. 1 root root 35433 Aug 26 19:53 README.md
-rw-r--r--. 1 root root 225 Aug 26 19:53 requirements.txt
-rw-r--r--. 1 root root 22638 Aug 26 19:53 run_alphafold.py
-rw-r--r--. 1 root root 4266 Aug 26 19:53 run_alphafold_test.py
drwxr-xr-x. 2 root root 4096 Aug 26 19:53 scripts/
drwxr-xr-x. 2 root root 43 Aug 26 19:53 server/
-rw-r--r--. 1 root root 2039 Aug 26 19:53 setup.py
drwxr-xr-x. 3 root root 169 Aug 26 19:54 singularity/
[root@rockylinux9 alphafold]#ここからはpyenv/anaconda経由で実行環境を作ります. ここでは anaconda の代わりに miniforge3 を使います
git clone https://github.com/yyuu/pyenv.git /apps/pyenv
export PYENV_ROOT=/apps/pyenv
export PATH=$PYENV_ROOT/bin:$PATH
pyenv install miniforge3-25.3.1-0
pyenv global miniforge3-25.3.1-0
source /apps/pyenv/versions/miniforge3-25.3.1-0/etc/profile.d/conda.sh
(既にanaconda環境があるなら)
source /apps/pyenv/versions/miniforge3-25.3.1-0/etc/profile.d/conda.sh作成する環境は alphafold_singularity としてます
[root@rockylinux9 alphafold]# conda create -n alphafold_singularity pip -y
:
:
[root@rockylinux9 alphafold]# conda activate alphafold_singularity
(alphafold_singularity) [root@rockylinux9 alphafold]#この状態で構築を進める
(alphafold_singularity) [root@rockylinux9 alphafold]# which pip
/apps/pyenv/versions/miniforge3-25.3.1-0/envs/alphafold_singularity/bin/pip
(alphafold_singularity) [root@rockylinux9 alphafold]# cat singularity/requirements.txt
(略
absl-py==1.0.0
spython==0.3.0
(alphafold_singularity) [root@rockylinux9 alphafold]# pip install -r singularity/requirements.txt次に Singularity パッケージをインストールして、
(alphafold_singularity) [root@rockylinux9 alphafold]# dnf install epel-release -y
(alphafold_singularity) [root@rockylinux9 alphafold]# dnf install apptainer apptainer-suidAlphafoldのSingularity コンテナイメージを作成します.
(alphafold_singularity) [root@rockylinux9 alphafold]# singularity build alphafold.sif singularity/Singularity.def
(alphafold_singularity) [root@rockylinux alphafold]# ls -lh alphafold.sif
-rwxr-xr-x 1 root root 4.6G Feb 17 21:29 alphafold.sif
(alphafold_singularity) [root@rockylinux alphafold]# conda deactivate
[root@rockylinux alphafold]#4.6GBほどのサイズになりました.
ここでエラーが発生するなら alphafold/docker/Dockerfile と alphafold_singularity/Singularity.def を見比べて
alphafold/docker/Dockerfileが優先になるように singularity/Singularity.def を修正します.
alphafold と alphafold_singularity は互いに連携しているわけではないので. あとalphafold_singularityに書かれた方法は一見するとバージョン番号で統制が取れるっぽいが失敗する. 目で見比べて修正した方が早い
次に計算に使用するスクリプト「/apps/alphafold/singularity/run_singularity.py」
「/apps/alphafold/singularity」にPATHを入れて「run_singularity.py」を実行すれば計算が出来るようにしたいので
行頭にcondaのpythonを入れる.
|
これで実行環境は完成. っで次は恒例の EnvironmentModules.
いつものように「/apps/modulefiles/alphafold_singularity」として記載
[root@rockylinux ~]# vi /apps/modulefiles/alphafold_singularity
#%Module1.0
#
set root /apps/pyenv/versions/anaconda3-2023.09-0/envs/alphafold_singularity
set af /apps/alphafold
prepend-path PATH $af/singularity:$root/bin
setenv ALPHAFOLD_DIR $af
[root@rockylinux ~]#[illya@rockylinux ~]$ module use --append /apps/modulefiles/
[illya@rockylinux ~]$ module load alphafold_singularity
[illya@rockylinux ~]$ run_singularity.py
/apps/alphafold/alphafold.sif
FATAL Flags parsing error:
flag --data_dir=None: Flag --data_dir must have a value other than None.
flag --fasta_paths=None: Flag --fasta_paths must have a value other than None.
flag --max_template_date=None: Flag --max_template_date must have a value other than None.
Pass --helpshort or --helpfull to see help on flags.
[saber@rockylinux ~]$
[illya@rockylinux ~]$ vi query.fasta
>dummy_sequence
GWSTELEKHREELKEFLKKEGITNVEIRIDNGRLEVRVEGGTERLKRFLEELRQKLEKKGYTVDIKIE
[illya@rockylinux ~]$ mkdir out
[illya@rockylinux ~]$ run_singularity.py --fasta_paths=query.fasta --max_template_date=2020-05-14 --data_dir=/AlphaFold --db_preset=reduced_dbs --output_dir=out
:
:
[illya@rockylinux ~]$ ls -lF ./out
total 24
-rw-r--r-- 1 illya illya 20104 Feb 17 21:47 ld.so.cache
drwxr-xr-x 3 illya illya 4096 Feb 17 22:16 query/
[illya@rockylinux ~]$
結果は $HOME/out に置かれる. --output_dir=. とすると入力ファイルのファイル名を使ったフォルダが用意されます. query.fasta --> 「query」フォルダが作られる
次にdimerでテスト
[illya@rockylinux ~]$ vi dimer.fasta
>XP_009313165.1.1
MRAAFAEARAALAEGEVPVGCVLVPVDASCAANAQLAADDDDDENKSKGSSNSNNSKKNDAVERLIAARG
RNATNREHHALAHAEFVAVEALLRELAANGQQRPASLAGYVLYVVVEPCIMCAAMLLYNRVQKVFFGCGN
PRFGGNGTVLAVHTAAGCSAPGYESSGGHRADEAVALLQEFYRHENTNAPGHKRRRKCECLNN
>XP_009313165.1.2
MRAAFAEARAALAEGEVPVGCVLVPVDASCAANAQLAADDDDDENKSKGSSNSNNSKKNDAVERLIAARG
RNATNREHHALAHAEFVAVEALLRELAANGQQRPASLAGYVLYVVVEPCIMCAAMLLYNRVQKVFFGCGN
PRFGGNGTVLAVHTAAGCSAPGYESSGGHRADEAVALLQEFYRHENTNAPGHKRRRKCECLNN
[illya@rockylinux ~]$ run_singularity.py --fasta_paths=dimer.fasta --max_template_date=2020-05-14 --data_dir=/AlphaFold \
--db_preset=reduced_dbs --model_preset=multimer --output_dir=.
[illya@rockylinux ~]$ ls -lFd dimer*
drwxr-xr-x 3 illya illya 12288 Feb 17 23:19 dimer/
-rw-r--r-- 1 illya illya 448 Feb 17 22:19 dimer.fasta
[illya@rockylinux ~]$ ls -l dimer/ | head -n 5
total 2586692
-rw-r--r-- 1 illya illya 5595 Feb 17 22:43 confidence_model_1_multimer_v3_pred_0.json
-rw-r--r-- 1 illya illya 5598 Feb 17 22:44 confidence_model_1_multimer_v3_pred_1.json
-rw-r--r-- 1 illya illya 5598 Feb 17 22:45 confidence_model_1_multimer_v3_pred_2.json
-rw-r--r-- 1 illya illya 5604 Feb 17 22:47 confidence_model_1_multimer_v3_pred_3.json
[illya@rockylinux ~]$Singularityイメージファイルならクラスターマシンとかでも比較的容易に使えます.
hmmer と hh-suite を oneAPI で最適化してみた.
事前にoffline向けの oneAPI base-toolkit と hpc-toollit を取得して
Bootstrap: docker
From: nvidia/cuda:12.2.2-cudnn8-runtime-ubuntu20.04
Stage: spython-base
%files
. /app/alphafold
%post
# Copyright 2023 David Chin
#
# This file is part of alphafold_singularity.
#
# alphafold_singularity is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# alphafold_singularity is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with alphafold_singularity. If not, see <https://www.gnu.org/licenses/>.
# FROM directive resets ARGS, so we specify again (the value is retained if
# previously set).
bash /app/alphafold/intel-oneapi-base-toolkit-2025.0.1.46_offline.sh -a -s --eula accept --intel-sw-improvement-program-consent decline \
--components intel.oneapi.lin.dpl:intel.oneapi.lin.tbb.devel:intel.oneapi.lin.dpcpp-cpp-compiler:intel.oneapi.lin.dpl:intel.oneapi.lin.ipp.devel:intel.oneapi.lin.mkl.devel:intel.oneapi.lin.tbb.devel
bash /app/alphafold/intel-oneapi-hpc-toolkit-2025.0.1.47_offline.sh -a -s --eula accept --intel-sw-improvement-program-consent decline \
--components intel.oneapi.lin.mpi.devel:intel.oneapi.lin.ifort-compiler
rm -f /app/alphafold/intel-oneapi-base-toolkit-2025.0.1.46_offline.sh
rm -f /app/alphafold/intel-oneapi-hpc-toolkit-2025.0.1.47_offline.sh
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
build-essential \
cmake \
cuda-command-line-tools-12-2 \
git \
kalign \
tzdata \
autoconf \
wget \
&& rm -rf /var/lib/apt/lists/* \
&& apt-get autoremove -y \
&& apt-get clean
PATH="/opt/intel/oneapi/compiler/2025.0/bin:$PATH"
LD_LIBRARY_PATH="/opt/intel/oneapi/mkl/2025.0/lib:/opt/intel/oneapi/umf/0.9/lib/:/opt/intel/oneapi/compiler/2025.0/opt/compiler/lib:/opt/intel/oneapi/compiler/2025.0/lib:/opt/intel/oneapi/tbb/2022.0/lib:$LD_LIBRARY_PATH"
# Compile HHsuite from source.
/bin/rm -rf /tmp/hh-suite \
&& git clone https://github.com/IntelLabs/hh-suite.git /tmp/hh-suite \
&& mkdir /tmp/hh-suite/build \
&& cd /tmp/hh-suite/build \
&& cmake -DCMAKE_INSTALL_PREFIX=/opt/hhsuite -DCMAKE_CXX_COMPILER="icpx" -DCMAKE_CXX_FLAGS_RELEASE="-O3 -march=native" .. \
&& make -j 4 && make install \
&& ln -s /opt/hhsuite/bin/* /usr/bin \
&& cd / \
&& /bin/rm -rf /tmp/hh-suite
/bin/rm -rf /tmp/hmmer \
&& git clone --recursive https://github.com/IntelLabs/hmmer.git /tmp/hmmer \
&& cd /tmp/hmmer \
&& cp easel_makefile.in easel/Makefile.in \
&& cd easel && autoconf && ./configure --prefix=/opt/hmmer && cd .. \
&& autoconf && CC=gcc CFLAGS="-O3 -march=native -fPIC" ./configure --prefix=/opt/hmmer \
&& make -j 4 && make install \
&& ln -s /opt/hmmer/bin/* /usr/bin \
&& cd / \
&& /bin/rm -rf /tmp/hmmer
# Install Miniconda package manager.
wget -q -P /tmp \
https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
&& bash /tmp/Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda \
&& rm /tmp/Miniconda3-latest-Linux-x86_64.sh
# Install conda packages.
PATH="/opt/conda/bin:/usr/local/cuda-12.2/bin:$PATH"
conda install -qy conda=24 pip python=3.11 \
&& conda install -y -c conda-forge -c nvidia \
openmm=8.0.0 \
cuda=12.2.2 \
pdbfixer \
&& conda clean --all --force-pkgs-dirs --yes
### /bin/cp -r . /app/alphafold
wget -q -P /app/alphafold/alphafold/common/ \
https://git.scicore.unibas.ch/schwede/openstructure/-/raw/7102c63615b64735c4941278d92b554ec94415f8/modules/mol/alg/src/stereo_chemical_props.txt
# Install pip packages.
# N.B. The URL specifies the list of jaxlib releases.
pip3 install --upgrade pip --no-cache-dir \
&& pip3 install -r /app/alphafold/requirements.txt --no-cache-dir \
&& pip3 install --upgrade --no-cache-dir \
jax==0.4.26 \
jaxlib==0.4.26+cuda12.cudnn89 \
-f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
# Add SETUID bit to the ldconfig binary so that non-root users can run it.
chmod u+s /sbin/ldconfig.real
### SETUID bit does not matter: Apptainer does not allow suid commands
### Workaround below is to use /mnt/out/ld.so.cache for the ld cache file
%environment
export PATH="/opt/conda/bin:/usr/local/cuda-12.2/bin:$PATH"
export LD_LIBRARY_PATH="/opt/intel/oneapi/compiler/2025.0/lib/:$LD_LIBRARY_PATH"
%runscript
cd /app/alphafold
ldconfig -C /mnt/output/ld.so.cache
exec python /app/alphafold/run_alphafold.py "$@"
# %startscript
# cd /app/alphafold
# exec python /app/alphafold/run_alphafold.py "$@"
