ELPA installation guide

Below we provide examples of compilation process in on selected systems

Installation on Summit
- Notes by Victor Wen-zhe Yu:
- Installation instructions
Installation on Piz Daint
Installation on Cygnus
Installation on Tsubame
Installation on Eagle
Installation on LUMI

Installation on Summit

Notes by Victor Wen-zhe Yu:

Installation instructions

For compilation use interactive session:

bsub -Is -W 0:15 -nnodes 1 -P nph139 $SHELL

Next execute:

module load gcc/9.1.0
module load autoconf
module load automake
module load cuda 
module load essl
module load netlib-lapack
module load netlib-scalapack
module load python/3.7.0

git clone https://gitlab.mpcdf.mpg.de/elpa/elpa.git

# go to ELPA distribution directory
cd elpa
./autogen.sh
# if autogen.sh fails rerun
# libtoolize, aclocal, autoheader

mkdir build && cd build

../configure FC=mpif90 CC=mpicc FCFLAGS="-O2 -mcpu=power9 -ffree-line-length-none" CFLAGS="-O2 -mcpu=power9" CPP="cpp -E" LDFLAGS="-L${OLCF_NETLIB_SCALAPACK_ROOT}/lib -lscalapack -L${OLCF_ESSL_ROOT}/lib64 -lessl -L${OLCF_NETLIB_LAPACK_ROOT}/lib64 -llapack" --disable-openmp --enable-gpu --with-cuda-path=${OLCF_CUDA_ROOT} --with-GPU-compute-capability=sm_70 --disable-sse-assembly --disable-sse --disable-avx --disable-avx2 --disable-avx512 --enable-c-tests=no --prefix="/ccs/home/gabrielw/proj-shared/share/elpa"

make -j 8

# Before you install lib, check if CUDA support works correctly
jsrun -n 6 -a 1 -c 1 -g 1 ./validate_complex_double_eigenvectors_2stage_default_kernel_gpu_random 1000 1000 16  
jsrun -n 6 -a 1 -c 1 -g 1 ./validate_real_double_eigenvectors_2stage_default_kernel_gpu_random 1000 1000 16

# If tests are ok, install to the final destination
make install

Installation on Piz Daint

# Clone latest version
git clone https://gitlab.mpcdf.mpg.de/elpa/elpa.git

# Load modules
module load daint-gpu
module swap PrgEnv-cray PrgEnv-gnu/6.0.9
module swap gcc gcc/9.3.0
module load cudatoolkit/11.0.2_3.33-7.0.2.1_3.1__g1ba0366
module load craype-accel-nvidia60
module list

./autogen.sh

# go to ELPA distribution directory

mkdir build && cd build

../configure --prefix="/project/pr91/share" FC=ftn CC=cc FCFLAGS="-O2 -ffree-line-length-none" CFLAGS="-O2" -disable-avx512 --disable-openmp --enable-static --enable-gpu --with-cuda-path=${CRAY_CUDATOOLKIT_DIR} --with-GPU-compute-capability=sm_60 --enable-c-tests=no

make 

make install

Installation on Cygnus

module load intel/19.0.5
module load mkl/19.0.5
module load cuda/10.2
module load intmpi/19.0.5

# go to ELPA distribution directory

mkdir build && cd build

../configure --prefix="/work/TDSLDA20/share/elpa" --enable-static CC=mpiicc FC=mpiifort CPP=cpp FCPP=cpp  LIBS="-lmkl_scalapack_lp64 -lmkl_blacs_intelmpi_lp64 -lpthread -lm -ldl -mkl=cluster"  FCFLAGS="-mkl=cluster -O3 -xAVX2" CFLAGS="-mkl=cluster -qopenmp -O3 -xAVX2" --enable-gpu --with-cuda-path=${CUDA_PATH} --with-GPU-compute-capability=sm_60

make 

make install

Installation on Tsubame

module load cuda/10.2.89 
module load intel/19.1.0.166 
module load intel-mpi/19.6.166

# go to ELPA distribution directory
mkdir && cd
git clone https://gitlab.mpcdf.mpg.de/elpa/elpa && ./autogen.sh && cd ..
mkdir build && cd build
../configure --prefix="/gs/hs1/hp190063/share/elpa" --disable-openmp --enable-static CC=mpiicc FC=mpiifort LIBS="-lmkl_scalapack_ilp64 -lmkl_blacs_intelmpi_ilp64 -lpthread -lm -ldl  -DMKL_ILP64 -mkl=cluster"  FCFLAGS="-DMKL_ILP64 -mkl=cluster -O3 -xAVX2" CFLAGS="-DMKL_ILP64 -mkl=cluster -O3 -xAVX2" --enable-gpu --with-cuda-path=${CUDA_PATH} --with-GPU-compute-capability=sm_60

make 

make install

Installation on Eagle

module load impi/2020.4.912
module load mkl/2020.0.4
module load cuda/11.2.1_460.32.03

# get the latest version of elpa
git clone https://gitlab.mpcdf.mpg.de/elpa/elpa.git

# go to elpa lib
cd elpa

# prepare for compilation
libtoolize
aclocal
autoheader
./autogen.sh

# prepare for building
mkdir build && cd build

../configure --prefix="/home/users/gabrielw/grant_518/project_data/share/elpa" --enable-static CXX=mpiicpc CC=mpiicc FC=mpiifort CPP=cpp FCPP=cpp  LIBS="-lmkl_scalapack_lp64 -lmkl_blacs_intelmpi_lp64 -lpthread -lm -ldl -mkl=cluster"  FCFLAGS="-mkl=cluster -O3 -xAVX2" CFLAGS="-mkl=cluster -qopenmp -O3 -xAVX2" --enable-gpu --with-cuda-path=${CUDA_PATH} --with-GPU-compute-capability=sm_70

# build
make

# install
make install

Installation on LUMI

module load PrgEnv-gnu/8.4.0
module load LUMI/23.09  partition/G
module load rocm/5.6.1
module load buildtools/23.09

#install blis
git clone https://github.com/flame/blis.git
cd blis
./configure --prefix=/project/project_465000708/share/libs amd64
make
make install

#install libflame
git clone https://github.com/flame/libflame.git
cd libflame
./configure --prefix=/project/project_465000708/share/libs amd64
make
make install

#build ELPA
git clone https://gitlab.mpcdf.mpg.de/elpa/elpa.git
cd elpa
git checkout master_pre_stage
# prepare for compilation
libtoolize
aclocal
autoheader
./autogen.sh
# building process in separate folder
mkdir build && cd build
../configure --prefix=/project/project_465000708/share/libs CPP="gcc -E" CC=cc CXX=hipcc FC=ftn CXXFLAGS="-DROCBLAS_V3 -D__HIP_PLATFORM_AMD__ --offload-arch=gfx90a -g -O3 -std=c++17" CFLAGS="-g -O3 -std=c++17" LIBS="-lamdhip64 -fPIC" FCFLAGS="-g -O3"  --enable-option-checking=fatal --with-mpi=yes --disable-sse --disable-sse-assembly --disable-avx --disable-avx2 --disable-avx512 --enable-amd-gpu --enable-single-precision --enable-gpu-streams=amd --enable-hipcub --disable-cpp-tests --with-rocsolver
make
make install