Compilation
On Pitagora, appropriate kokkos is not yet built. Therefore, we should install it with orb5x's cmake (that's normal on most machines).
Also, the system built parallel HDF5 does not have C++ API (h5c++ is missing). I have built parallel HDF5 for C++ with spack so that we could use it.
module load spack
spack find -lv hdf5
spack load /u6vqglp
module load cmake
module load openmpi/4.1.6--gcc--12.3.0
module load hdf5/1.14.3--openmpi--4.1.6--gcc--12.3.0
module load fftw/3.3.10--openmpi--4.1.6--gcc--12.3.0
module load openblas/0.3.26--gcc--12.3.0
module load cuda/12.6

cmake /path/to/root/repo \
-DKokkos_ENABLE_CUDA=ON \
-DKokkos_ARCH_HOPPER90=ON \
-DORB5X_BUILD_KOKKOS_FROM_SOURCE=ON
The make command is the same as usual
Job Script:
Here's an example with 2 GPUs.
#!/bin/bash
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=2
#SBATCH --cpus-per-task=1
#SBATCH --time=00:10:00
#SBATCH --job-name=ORB5X
#SBATCH --account=FUPB2_TSVVG
#SBATCH --gres=gpu:2
#SBATCH --mem=480G
#SBATCH --partition=boost_fua_prod
#SBATCH --qos=normal
module purge
module load spack
spack find -lv hdf5
spack load /u6vqglp
module load cmake
module load openmpi/4.1.6--gcc--12.3.0
module load hdf5/1.14.3--openmpi--4.1.6--gcc--12.3.0
module load fftw/3.3.10--openmpi--4.1.6--gcc--12.3.0
module load openblas/0.3.26--gcc--12.3.0
module load cuda/12.6
export OMP_NUM_THREADS="${SLURM_CPUS_PER_TASK:-1}"
export OMP_PLACES=cores
export OMP_PROC_BIND=close
mpirun -np "${SLURM_NTASKS}" ./orb5x ITG.in --kokkos-map-device-id-by=mpi_rank \
> orb5x.out 2> orb5x.err
Merlin 7
For H100
module load gcc/13.4.0 openmpi/5.0.8 hdf5/1.14.6 ftw/3.3.10 cmake/3.25.2
For GH200 Grace Hopper
salloc --partition=gh-daily --clusters=gmerlin7 --time=08:00:00 --ntasks=4 --nodes=1 --gpus=1 --mem=40000 $SHELL
ssh <allocated_gpu>
module purge
module use Spack unstable
module load gcc/13.2.0 openmpi/5.0.7-z3y6-GH200-gpu hdf5/1.14.3-wspx fftw/3.3.10-jctz cmake/3.31.4-u2nm openblas/0.3.29-d3m2
sample job script:
#!/bin/bash
#SBATCH --job-name=orb5x
#SBATCH --error=orb5x.err
#SBATCH --output=orb5x.out
#SBATCH --time=00:15:00
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=2
#SBATCH --mem-per-cpu=4G
#SBATCH --cpus-per-task=8
#SBATCH --cluster=gmerlin7
#SBATCH --partition=gh-hourly
#SBATCH --gpus=2
module purge
module use Spack unstable
module load gcc/13.2.0 openmpi/5.0.7-z3y6-GH200-gpu hdf5/1.14.3-wspx fftw/3.3.10-jctz cmake/3.31.4-u2nm openblas/0.3.29-d3m2
srun ./orb5x ITG.in --kokkos-map-device-id-by=mpi_rank
Daint GH200
Daint uses CSCS uenv images for the programming environment instead of a global Spack module tree. Pull the required Grace Hopper image once:
uenv image pull prgenv-gnu-openmpi/26.3:v1@daint%gh200
Then configure ORB5X from the build directory. The explicit FFTW and MPI library paths are needed so that CMake's sanity checks and final link line use the libraries from the uenv image.
cd ~/ORB5X/build_gh
uenv run --view=default prgenv-gnu-openmpi/26.3:v1 -- cmake .. \
-DCMAKE_C_COMPILER=/user-environment/env/default/bin/mpicc \
-DCMAKE_CXX_COMPILER=/user-environment/env/default/bin/mpicxx \
-DKokkos_ENABLE_CUDA=ON \
-DKokkos_ARCH_HOPPER90=ON \
-DFFTW3_INCLUDE_DIRS=/user-environment/env/default/include \
-DFFTW3_LIBRARIES=/user-environment/env/default/lib/libfftw3.so \
-DMPI_C_LIBRARIES=/user-environment/env/default/lib/libmpi.so \
-DMPI_CXX_LIBRARIES=/user-environment/env/default/lib/libmpi.so
Build with:
uenv run --view=default prgenv-gnu-openmpi/26.3:v1 -- make -j4
The resulting executable is orb5x in the build directory. For runs with more than one GPU, pass Kokkos the MPI-rank device mapping option:
srun ./orb5x ITG.in --kokkos-map-device-id-by=mpi_rank
Sample job script:
#!/bin/bash
#SBATCH --job-name=orb5x
#SBATCH --error=orb5x_%j.err
#SBATCH --output=orb5x_%j.out
#SBATCH --time=00:30:00
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=4
#SBATCH --mem=0
#SBATCH --cpus-per-task=1
#SBATCH --gpus-per-task=1
#SBATCH --gpus=4
#SBATCH --uenv=prgenv-gnu-openmpi/26.3:v1
#SBATCH --view=default
#SBATCH --account=<account>
set -euo pipefail
ulimit -c unlimited
export OMP_PROC_BIND=spread
export OMP_PLACES=threads
export OMP_NUM_THREADS="${SLURM_CPUS_PER_TASK:-1}"
srun ./orb5x ITG.in --kokkos-map-device-id-by=mpi_rank
The --uenv and --view directives are required at runtime. Without them, the executable can fail to start because CUDA and other shared libraries from the build environment, for example libcudart.so.13, are not visible on the compute nodes.
Discoverer+
For details, see: https://docs.discoverer.bg
e.g of compilation script:
#!/bin/bash
#SBATCH --partition=common # Partition
#SBATCH --job-name=comp-orb5x # Job name
#SBATCH --time=01:00:00 # WallTime - set it accordingly
#SBATCH --account=ehpc-ben-2026b07-016
#SBATCH --qos=2cpu-single-host
#SBATCH --nodes=1 # Single node
#SBATCH --ntasks=2 # Single task
#SBATCH --cpus-per-task=1 # Number of parallel compilation threads
#SBATCH --mem=32G # Memory for compilation
#SBATCH -o compile.%j.out # STDOUT
#SBATCH -e compile.%j.err # STDERR
module purge || exit
module load cmake || exit
module load gcc/14/14.2.0 || exit
module load openmpi/gcc/4/4.1.8 || exit
module load fftw/3/3.3.10
module load lapack/3/3.12.1
#module load hdf5/gcc/openmpi/1/1.14/1.14.6
module load nvidia/cuda/12/12.8 || exit
cd ${SLURM_SUBMIT_DIR}
export MAKEFLAGS="-j2"
export CMAKE_BUILD_PARALLEL_LEVEL=2
cmake .. \
-DKokkos_ENABLE_CUDA=ON \
-DKokkos_ARCH_HOPPER90=ON \
-DORB5X_BUILD_KOKKOS_FROM_SOURCE=ON \
-DORB5X_BUILD_HDF5_FROM_SOURCE=ON
make -j2 || exit
e.g. of job script for running simulations:
#!/bin/bash
#SBATCH --partition=common
#SBATCH --job-name=orb5x_ITPA_g4
#SBATCH --time=00:30:00
#SBATCH --account=ehpc-ben-2026b07-016
#SBATCH --qos=ehpc-ben-2026b07-016
#SBATCH --nodes=1
#SBATCH --ntasks=4
#SBATCH --ntasks-per-node=4
#SBATCH --cpus-per-task=4
#SBATCH --gres=gpu:4
#SBATCH --mem=128G
#SBATCH --output=run.%j.out
#SBATCH --error=run.%j.err
set -euo pipefail
module purge
module load cmake
module load gcc/14/14.2.0
module load openmpi/gcc/4/4.1.8
module load fftw/3/3.3.10
module load lapack/3/3.12.1
module load nvidia/cuda/12/12.8
ulimit -c unlimited
export HDF5_USE_FILE_LOCKING=FALSE
export OMP_PROC_BIND=spread
export OMP_PLACES=threads
export OMP_NUM_THREADS="${SLURM_CPUS_PER_TASK:-1}"
MPI_RANKS="${SLURM_NTASKS}"
srun --mpi=pmix \
--ntasks="${MPI_RANKS}" \
--ntasks-per-node="4" \
--gpus-per-task=1 \
"/home/msadr/ORB5X/build_h100/orb5x" "run_itpa.in" --kokkos-map-device-id-by=mpi_rank
LUMI-G
Compilation:
module load LUMI/25.03 partition/G cpeAMD rocm buildtools/25.03 cray-fftw
export PATH=/opt/cray/pe/mpich/8.1.32/ofi/amd/6.0/bin:$PATH # cray-mpich's mpicc/mpicxx
mkdir -p build_amd && cd build_amd
cmake .. \
-DCMAKE_C_COMPILER=mpicc \
-DCMAKE_CXX_COMPILER=mpicxx \
-DKokkos_ENABLE_SERIAL=ON \
-DKokkos_ENABLE_HIP=ON \
-DKokkos_ARCH_AMD_GFX90A=ON \
-DORB5X_BUILD_KOKKOS_FROM_SOURCE=ON \
-DORB5X_BUILD_HDF5_FROM_SOURCE=ON \
-DLAPACK_LIBRARIES="/opt/cray/pe/libsci/25.03.0/AMD/6.0/x86_64/lib/libsci_amd_mpi.so;/opt/cray/pe/libsci/25.03.0/AMD/6.0/x86_64/lib/libsci_amd.so"
make -j8
Job script for simulation:
#!/bin/bash
# ---
# sbatch settings for LUMI-G
# ---
#SBATCH --job-name=orb5x-test
#SBATCH --partition=standard-g
#SBATCH --account=project_465003190
#SBATCH --time=00:30:00
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=8
#SBATCH --gpus-per-node=8
##SBATCH --cpus-per-task=7
##SBATCH --exclusive
#SBATCH --output=orb5x-test.out
#SBATCH --error=orb5x-test.error
module load LUMI/25.03 partition/G cpeAMD rocm buildtools/25.03 cray-fftw
export PATH=/opt/cray/pe/mpich/8.1.32/ofi/amd/6.0/bin:$PATH # cray-mpich's mpicc/mpicxx
export OMP_PROC_BIND=spread
export OMP_PLACES=threads
export OMP_NUM_THREADS=6
export MPICH_GPU_SUPPORT_ENABLED=1
#CPU_BIND="mask_cpu:fe000000000000,fe00000000000000"
#CPU_BIND="${CPU_BIND},fe0000,fe000000"
#CPU_BIND="${CPU_BIND},fe,fe00"
#CPU_BIND="${CPU_BIND},fe00000000,fe0000000000"
CPU_BIND="map_cpu:49,57,17,25,1,9,33,41"
srun --cpu-bind=${CPU_BIND} ./orb5x itpa.in
1.17.0