![[PukiWiki]](image/picc.png "[PukiWiki]")
取り合えず intel謹製のコンパイラをインストールします oneAPI
relionは(スレッドが有効な)mpi対応でして、mpiに 「openmpi」 か 「intel謹製のintel mpi」のどちらも選べる.
コンパイルするにあたって下記の組み合わせを考える必要がある
ただ組み合わせによっては出来ないものもある. 例えばコンパイラに[oneAPI]を選んでacceralationに[GPU]を選ぶと
GPUのcudaライブラリのversionによって利用可能な[oneAPI]が制限されます。
https://docs.nvidia.com/cuda/archive/12.8.1/cuda-installation-guide-linux/index.html
とかを纏めると
| cuda version | Distribution | GCC | Clang | NVHPC | XLC | ArmC/C++ | ICC |
| 13.0.0 | x86_64 | 6.x - 15.x | 7.x - 20.x | 24.9-25.5 | No | No | (null) |
| 12.8.1 | x86_64 | 6.x - 14.x | 7.x - 19.x | 24.9 | No | No | 2021.7 |
| 12.4.1 | x86_64 | 6.x - 13.2 | 7.x - 17.0 | 23.x | No | No | 2021.7 |
| 12.2.2 | x86_64 | 6.x - 12.2 | 7.x - 16.0 | 23.x | No | No | 2021.7 |
| 11.8.0 | Rockylinux9.0 x86_64 | 11.2.1 | 14.0 | 22.3 | No | No | 2021 |
| 11.6.2 | RHEL8 x86_64 | 11 | 12 | 21.7 | No | No | 2021 |
oneAPIのclang versionは
[root@rockylinux9 ~]# icx -x c /dev/null -dM -E | grep clang_version
#define __clang_version__ "21.0.0git (icx 2025.2.0.20250605)"
[root@rockylinux9 ~]#と表示されて、clangは 21.0.0 となる. これではoneAPI 2025.2 と cuda 12.8.1 の組み合わせでは使えないとなる.
多少古いversionはoneAPIから入手可能ですが、古いものiccが使えるバージョンはサポートに問い合わせって感じみたい.
別途取り寄せた oneAPI 2025.0 をloadしてみたら
[root@rockylinux9 ~]# icx -x c /dev/null -dM -E | grep clang_version
#define __clang_version__ "19.0.0git (icx 2025.0.4.20241205)"
[root@rockylinux9 ~]#と一見このversionでよさげに見えるが、「icx --version」では clang の情報がない
[root@rockylinux9 ~]# icx --version
Intel(R) oneAPI DPC++/C++ Compiler 2025.0.4 (2025.0.4.20241205)
Target: x86_64-unknown-linux-gnu
Thread model: posix
InstalledDir: /opt/intel/oneapi/compiler/2025.0/bin/compiler
Configuration file: /opt/intel/oneapi/compiler/2025.0/bin/compiler/../icx.cfg
[root@rockylinux9 ~]#微妙だけど、これなら一応 cuda 12.8 にも使えそうかな
まずは確認
[root@rockylinux9 ~]# module use /apps/modulefiles
[root@rockylinux9 ~]# module load compiler/2025.0.4
[root@rockylinux9 ~]# module load cuda/12.8
[root@rockylinux9 ~]# module load mkl/2025.0
[root@rockylinux9 ~]# cat /etc/redhat-release
Rocky Linux release 9.6 (Blue Onyx)
[root@rockylinux9 ~]# lscpu | grep avx2
Flags: ... hle avx2 smep ...
[root@rockylinux9 ~]# cat /proc/driver/nvidia/version
NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 570.181 Release Build (dvs-builder@U22-I3-AF02-20-5) Wed Jul 30 18:41:07 UTC 2025 <--「570.181」はcuda 12.8 対応品
GCC version: gcc version 11.5.0 20240719 (Red Hat 11.5.0-5) (GCC)
[root@rockylinux9 ~]# nvcc --version
nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2025 NVIDIA Corporation
Built on Fri_Feb_21_20:23:50_PST_2025
Cuda compilation tools, release 12.8, V12.8.93
Build cuda_12.8.r12.8/compiler.35583870_0
[root@rockylinux9 ~]# icx -x c /dev/null -dM -E | grep clang_version
#define __clang_version__ "19.0.0git (icx 2025.0.4.20241205)"
[root@rockylinux9 ~]#[root@rockylinux9 ~]# cd /apps/src/relion/
[root@rockylinux9 relion]# git branch
master
ver4.0
* ver5.0
[root@rockylinux9 relion]# mkdir 50i
[root@rockylinux9 relion]# cd 50i/
[root@rockylinux9 50i]#
[root@rockylinux9 50i]# which mpirun
/opt/intel/oneapi/mpi/2021.16/bin/mpirun
[root@rockylinux9 50i]#下記 cmake を実施してコンパイル準備を行います
cmake .. -DCMAKE_INSTALL_PREFIX=/apps/relion-5.0.0-oneAPI-impi-cuda \
-DCUDA_ARCH=86 \
-DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-12.8 \
-DPYTHON_EXE_PATH=/apps/pyenv/versions/anaconda3-2025.06-1/envs/relion-5.0/bin/python \
-DTORCH_HOME_PATH=/apps/relion-torch \
-DMKLFFT=ON \
-DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DMPI_C_COMPILER=mpiicx -DMPI_CXX_COMPILER=mpiicpx \
-DCMAKE_C_FLAGS="-O3 -qopenmp-simd -xCORE-AVX2 -qopt-zmm-usage=high -qoverride-limits " \
-DCMAKE_CXX_FLAGS="-O3 -qopenmp-simd -xCORE-AVX2 -qopt-zmm-usage=high -qoverride-limits " \
-DCUDA_NVCC_FLAGS="--allow-unsupported-compiler"結局 nvcc で icx の確認が取れないとのメッセージが出たので「-DCUDA_NVCC_FLAGS="--allow-unsupported-compiler"」を入れてます
っでコンパイルとインストール
[root@rockylinux9 50i]# make ; make installっで確認
[saber@rockylinux9 ~]$ module load compiler-rt/2025.0.4
[saber@rockylinux9 ~]$ module load mpi/2021.16
[saber@rockylinux9 ~]$ module load mkl/2025.0
[saber@rockylinux9 ~]$ ldd /apps/relion-5.0.0-oneAPI-impi-cuda/bin/relion_refine_mpi
linux-vdso.so.1 (0x00007fff1c387000)
libimf.so => /opt/intel/oneapi/compiler/2025.0/lib/libimf.so (0x00007f3ad8800000) <-- oneAPI
libcufft.so.11 => /usr/local/cuda-12.8/lib64/libcufft.so.11 (0x00007f3ac7600000) <-- cuda
libmpicxx.so.12 => /opt/intel/oneapi/mpi/2021.16/lib/libmpicxx.so.12 (0x00007f3ac7200000) <-- oneAPI
libmpifort.so.12 => /opt/intel/oneapi/mpi/2021.16/lib/libmpifort.so.12 (0x00007f3ac6e00000) <-- oneAPI
libmpi.so.12 => /opt/intel/oneapi/mpi/2021.16/lib/libmpi.so.12 (0x00007f3abcc00000) <-- oneAPI
libtiff.so.5 => /lib64/libtiff.so.5 (0x00007f3ad8d02000)
libcurand.so.10 => /usr/local/cuda-12.8/lib64/libcurand.so.10 (0x00007f3ab4000000) <-- cuda
libpng16.so.16 => /lib64/libpng16.so.16 (0x00007f3ad8ccb000)
libjpeg.so.62 => /lib64/libjpeg.so.62 (0x00007f3ad8c4a000)
libmkl_intel_lp64.so.2 => /opt/intel/oneapi/mkl/2025.0/lib/libmkl_intel_lp64.so.2 (0x00007f3ab3000000) <-- oneAPI
libmkl_intel_thread.so.2 => /opt/intel/oneapi/mkl/2025.0/lib/libmkl_intel_thread.so.2 (0x00007f3ab0c00000) <-- oneAPI
libmkl_core.so.2 => /opt/intel/oneapi/mkl/2025.0/lib/libmkl_core.so.2 (0x00007f3aacc00000) <-- oneAPI
libstdc++.so.6 => /lib64/libstdc++.so.6 (0x00007f3aac800000)
libm.so.6 => /lib64/libm.so.6 (0x00007f3ac7525000)
libgcc_s.so.1 => /lib64/libgcc_s.so.1 (0x00007f3ad8c2e000)
libiomp5.so => /opt/intel/oneapi/compiler/2025.0/lib/libiomp5.so (0x00007f3aac400000) <-- oneAPI
libc.so.6 => /lib64/libc.so.6 (0x00007f3aac000000)
/lib64/ld-linux-x86-64.so.2 (0x00007f3ad8d9e000)
libintlc.so.5 => /opt/intel/oneapi/compiler/2025.0/lib/libintlc.so.5 (0x00007f3ac74c3000) <-- oneAPI
libdl.so.2 => /lib64/libdl.so.2 (0x00007f3ad8c27000)
libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f3ad8c22000)
librt.so.1 => /lib64/librt.so.1 (0x00007f3ad8c1d000)
libwebp.so.7 => /lib64/libwebp.so.7 (0x00007f3ac7456000)
libzstd.so.1 => /lib64/libzstd.so.1 (0x00007f3ac6d49000)
libjbig.so.2.1 => /lib64/libjbig.so.2.1 (0x00007f3ad8c0d000)
libz.so.1 => /lib64/libz.so.1 (0x00007f3ad87e6000)
[saber@rockylinux9 ~]$「-DALTCPU=ON」を入れていないのでoneAPIでコンパイルしても GPU acceralationです。
environment-modules
「/apps/modulefiles/relion/5.0.0-oneAPI-impi-cuda」
#%Module1.0
module load compiler-rt/2025.0.4
module load mpi/2021.16
module load mkl/2025.0
set RELION /apps/relion-5.0.0-oneAPI-impi-cuda
prepend-path PATH $RELION/bin
setenv RELION_CTFFIND_EXECUTABLE /apps/ctffind-4.1.14/ctffind
setenv RELION_MOTIONCOR2_EXECUTABLE /apps/MotionCor2/MotionCor2_1.6.4_Cuda118_Mar312023
setenv RELION_GCTF_EXECUTABLE /apps/GCTF_Gautomatch_Cu10.1/GCTF_v1.18_sm30-75_cu10.1
setenv RELION_RESMAP_EXECUTABLE /apps/ResMap/ResMap-1.1.4-linux64
setenv RELION_PDFVIEWER_EXECUTABLE evince
##setenv RELION_MPIRUN "mpirun --mca mtl psm2 --mca btl ^ofi"
setenv RELION_EXTERNAL_RECONSTRUCT_EXECUTABLE /apps/SIDESPLITTER/sidesplitter_wrapper.sh
setenv SIDESPLITTER /apps/SIDESPLITTER/sidesplitterまずは oneAPI でコンパイルされた opnempi を用意します。参照 oneAPI#bcedde40
[root@rockylinux9 ~]# module use /apps/modulefiles
[root@rockylinux9 ~]# module load compiler/2025.0.4
[root@rockylinux9 ~]# module load cuda/12.8
[root@rockylinux9 ~]# module load mkl/2025.0
[root@rockylinux9 ~]# module load mpi/openmpi-5.0.8_icx-2025.2.0
[root@rockylinux9 ~]# which mpirun
/apps/openmpi-5.0.8_icx-2025.2.0/bin/mpirun
[root@rockylinux9 ~]# cd /apps/src/relion/50i
[root@rockylinux9 50i]# rm -rf ./*っで cmake を回す.
cmake .. -DCMAKE_INSTALL_PREFIX=/apps/relion-5.0.0-oneAPI-ompi-cuda \
-DCUDA_ARCH=86 \
-DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-12.8 \
-DPYTHON_EXE_PATH=/apps/pyenv/versions/anaconda3-2025.06-1/envs/relion-5.0/bin/python \
-DTORCH_HOME_PATH=/apps/relion-torch \
-DMKLFFT=ON \
-DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DMPI_C_COMPILER=mpicc -DMPI_CXX_COMPILER=mpicxx \
-DCMAKE_C_FLAGS="-O3 -qopenmp-simd -xCORE-AVX2 -qopt-zmm-usage=high -qoverride-limits " \
-DCMAKE_CXX_FLAGS="-O3 -qopenmp-simd -xCORE-AVX2 -qopt-zmm-usage=high -qoverride-limits " \
-DCUDA_NVCC_FLAGS="--allow-unsupported-compiler"「-DCMAKE_INSTALL_PREFIX」と「-DMPI_C_COMPILER」「-DMPI_CXX_COMPILER」の値が異なります.
そしてコンパイルとインストール
[root@rockylinux9 50i]# make ; make install確認
[saber@rockylinux9 ~]$ module pu
[saber@rockylinux9 ~]$ module use /apps/modulefiles/
[saber@rockylinux9 ~]$ module load compiler-rt/2025.0.4
[saber@rockylinux9 ~]$ module load mkl/2025.0
[saber@rockylinux9 ~]$ module load mpi/openmpi-5.0.8_icx-2025.2.0
[saber@rockylinux9 ~]$ which mpirun
/apps/openmpi-5.0.8_icx-2025.2.0/bin/mpirun
[saber@rockylinux9 ~]$
[saber@rockylinux9 ~]$ ldd /apps/relion-5.0.0-oneAPI-ompi-cuda/bin/relion_refine_mpi
linux-vdso.so.1 (0x00007ffc6bfc5000)
libimf.so => /opt/intel/oneapi/compiler/2025.0/lib/libimf.so (0x00007f6c3f800000) <-- oneAPI
libcufft.so.11 => /usr/local/cuda-12.8/lib64/libcufft.so.11 (0x00007f6c2e600000) <-- cuda
libmpi.so.40 => /apps/openmpi-5.0.8_icx-2025.2.0/lib/libmpi.so.40 (0x00007f6c2e200000) <-- openmpi
libtiff.so.5 => /lib64/libtiff.so.5 (0x00007f6c2e177000)
libcurand.so.10 => /usr/local/cuda-12.8/lib64/libcurand.so.10 (0x00007f6c25600000) <-- cuda
libpng16.so.16 => /lib64/libpng16.so.16 (0x00007f6c2e5c9000)
libjpeg.so.62 => /lib64/libjpeg.so.62 (0x00007f6c2e0f6000)
libmkl_intel_lp64.so.2 => /opt/intel/oneapi/mkl/2025.0/lib/libmkl_intel_lp64.so.2 (0x00007f6c24600000) <-- oneAPI
libmkl_intel_thread.so.2 => /opt/intel/oneapi/mkl/2025.0/lib/libmkl_intel_thread.so.2 (0x00007f6c22200000) <-- oneAPI
libmkl_core.so.2 => /opt/intel/oneapi/mkl/2025.0/lib/libmkl_core.so.2 (0x00007f6c1e200000) <-- oneAPI
libstdc++.so.6 => /lib64/libstdc++.so.6 (0x00007f6c1de00000)
libm.so.6 => /lib64/libm.so.6 (0x00007f6c25525000)
libgcc_s.so.1 => /lib64/libgcc_s.so.1 (0x00007f6c3f7d5000)
libiomp5.so => /opt/intel/oneapi/compiler/2025.0/lib/libiomp5.so (0x00007f6c1da00000) <-- oneAPI
libc.so.6 => /lib64/libc.so.6 (0x00007f6c1d600000)
/lib64/ld-linux-x86-64.so.2 (0x00007f6c3fc10000)
libintlc.so.5 => /opt/intel/oneapi/compiler/2025.0/lib/libintlc.so.5 (0x00007f6c2459e000) <-- oneAPI
libdl.so.2 => /lib64/libdl.so.2 (0x00007f6c3f7d0000)
libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f6c3f7c9000)
librt.so.1 => /lib64/librt.so.1 (0x00007f6c2e5c4000)
libopen-pal.so.80 => /apps/openmpi-5.0.8_icx-2025.2.0/lib/libopen-pal.so.80 (0x00007f6c1e112000) <-- openmpi
libpmix.so.2 => /apps/openmpi-5.0.8_icx-2025.2.0/lib/libpmix.so.2 (0x00007f6c1d200000) <-- openmpi
libevent_core-2.1.so.7 => /apps/openmpi-5.0.8_icx-2025.2.0/lib/libevent_core-2.1.so.7 (0x00007f6c2e0c2000) <-- openmpi
libevent_pthreads-2.1.so.7 => /apps/openmpi-5.0.8_icx-2025.2.0/lib/libevent_pthreads-2.1.so.7 (0x00007f6c2e5bf000) <-- openmpi
libhwloc.so.15 => /apps/openmpi-5.0.8_icx-2025.2.0/lib/libhwloc.so.15 (0x00007f6c2453c000) <-- openmpi
libsvml.so => /opt/intel/oneapi/compiler/2025.0/lib/libsvml.so (0x00007f6c1ba00000) <-- oneAPI
libirng.so => /opt/intel/oneapi/compiler/2025.0/lib/libirng.so (0x00007f6c1d907000) <-- oneAPI
libwebp.so.7 => /lib64/libwebp.so.7 (0x00007f6c1e0a5000)
libzstd.so.1 => /lib64/libzstd.so.1 (0x00007f6c1d850000)
libjbig.so.2.1 => /lib64/libjbig.so.2.1 (0x00007f6c2e5b1000)
libz.so.1 => /lib64/libz.so.1 (0x00007f6c2e0a8000)
[saber@rockylinux9 ~]$environment-modules
「/apps/modulefiles/relion/5.0.0-oneAPI-ompi-cuda」
#%Module1.0
module load compiler-rt/2025.0.4
module load mkl/2025.0
module load mpi/openmpi-5.0.8_icx-2025.2.0
set RELION /apps/relion-5.0.0-oneAPI-ompi-cuda
prepend-path PATH $RELION/bin
setenv RELION_CTFFIND_EXECUTABLE /apps/ctffind-4.1.14/ctffind
setenv RELION_MOTIONCOR2_EXECUTABLE /apps/MotionCor2/MotionCor2_1.6.4_Cuda118_Mar312023
setenv RELION_GCTF_EXECUTABLE /apps/GCTF_Gautomatch_Cu10.1/GCTF_v1.18_sm30-75_cu10.1
setenv RELION_RESMAP_EXECUTABLE /apps/ResMap/ResMap-1.1.4-linux64
setenv RELION_PDFVIEWER_EXECUTABLE evince
#setenv RELION_MPIRUN "mpirun --mca mtl psm2 --mca btl ^ofi"
setenv RELION_EXTERNAL_RECONSTRUCT_EXECUTABLE /apps/SIDESPLITTER/sidesplitter_wrapper.sh
setenv SIDESPLITTER /apps/SIDESPLITTER/sidesplitter一応「動く.」
「使える」かは不明. ベンチとれるほどのリソースがない.
