| ... | ... | @@ -186,10 +186,142 @@ E_pairext: NEW= 0 OLD= 0 DIFF= 0 CONVS |
|
|
|
# WRITING CHECKPOINT FILE `test_checkpoint.dat`
|
|
|
|
# TIMING rt_tot= 0.51: rt_zheev= 0.03[ 5.56%] rt_dens= 0.00[ 0.72%] rt_pot= 0.00[ 0.14%] rt_me= 0.00[ 0.06%] rt_redistrib= 0.00[ 0.31%] rt_other= 0.48[93.21%]
|
|
|
|
# ALGORITHM CONVERGED!
|
|
|
|
|
|
|
|
```
|
|
|
|
|
|
|
|
# Step 5: Setting templates for `td` codes
|
|
|
|
The same operations were applied to `td-project-template` and `td-testcase-uniform` folders. The following files were modified to the form as presented below.
|
|
|
|
|
|
|
|
### env.sh
|
|
|
|
```bash
|
|
|
|
# source env.sh
|
|
|
|
|
|
|
|
module load openmpi/3.1.4_gcc620
|
|
|
|
module unload cudatoolkit
|
|
|
|
module load cuda/11.2.1_460.32.03
|
|
|
|
module list
|
|
|
|
cat $WSLDA/VERSION.h
|
|
|
|
```
|
|
|
|
|
|
|
|
### Makefile
|
|
|
|
```bash
|
|
|
|
# LOAD MODULES BEFORE COMPILATION
|
|
|
|
#
|
|
|
|
# module load openmpi/3.1.4_gcc620
|
|
|
|
# module unload cudatoolkit
|
|
|
|
# module load cuda/11.2.1_460.32.03
|
|
|
|
|
|
|
|
# COMPILERS
|
|
|
|
CXX=mpicc
|
|
|
|
NVCC=nvcc
|
|
|
|
PYTHON=python
|
|
|
|
|
|
|
|
# DIRECTORY SETTINGS (must end with /)
|
|
|
|
# W-SLDA instalation folder (for read only)
|
|
|
|
WSLDADIR=/mnt/storage_2/project_data/grant_518/wslda/
|
|
|
|
# folder where temporarry files will be stored (will be created automatically)
|
|
|
|
OBJDIR=./obj/
|
|
|
|
# folder where executable binary will be placed (will be created automatically)
|
|
|
|
BINDIR=./
|
|
|
|
|
|
|
|
# COMPILER FLAGS & LIBRARIES
|
|
|
|
# C compiler flags
|
|
|
|
CFLAGS= -std=gnu99 -O3
|
|
|
|
# CUDA compiler flags
|
|
|
|
NVCCFLAGS = -arch sm_60 -O3
|
|
|
|
# prescription how to link with: FFTW, CUDA
|
|
|
|
LIBS=-lcudart -lcufft -lfftw3 -lm
|
|
|
|
|
|
|
|
# ----- DO NOT MODIFY -----
|
|
|
|
include $(WSLDADIR)hpc-engine/mk.td
|
|
|
|
```
|
|
|
|
|
|
|
|
### job.sh
|
|
|
|
```bash
|
|
|
|
#!/bin/bash -l
|
|
|
|
#SBATCH -J test # name that will be displayed in queue system
|
|
|
|
#SBATCH --output="test.out" # stdout will be saved here
|
|
|
|
#SBATCH -N 1 # here set number of nodes
|
|
|
|
#SBATCH -n 32 # must be N*32
|
|
|
|
#SBATCH --mem-per-cpu=2GB
|
|
|
|
#SBATCH --gpus-per-node=8 # use 8 if you want to use Altair nodes
|
|
|
|
#SBATCH --time=24:00:00 # here set walltime hh:mm:ss
|
|
|
|
#SBATCH --mail-type=BEGIN,END,FAIL # notifications for job started, done & fail
|
|
|
|
#SBATCH --mail-user=your@email # send-to address
|
|
|
|
#SBATCH -p tesla
|
|
|
|
|
|
|
|
## ------ QUEUE SYSTEM ------
|
|
|
|
## For submission use:
|
|
|
|
## sbatch job.sh
|
|
|
|
## For checking status use:
|
|
|
|
## squeue
|
|
|
|
## To kill job use:
|
|
|
|
## scancel pid
|
|
|
|
##
|
|
|
|
## ------ COMPUTATION -------
|
|
|
|
## For computation it is recomended to use location:
|
|
|
|
## cd ~/grant_518/scratch/
|
|
|
|
|
|
|
|
cd $SLURM_SUBMIT_DIR
|
|
|
|
source ./env.sh
|
|
|
|
|
|
|
|
## NOTE: YOU MUST ASSURE: -np=N*gpus-per-node
|
|
|
|
## -npernode=gpus-per-node
|
|
|
|
mpirun -np 8 -npernode 8 ./td-wslda-2d input.txt
|
|
|
|
```
|
|
|
|
|
|
|
|
### input.txt
|
|
|
|
The `job.sh` script is configured to send the code to standard GPU nodes, with `gpus-per-node=8`. Thus, one needs to set the correct value of `gpuspernode` in template of input file.
|
|
|
|
```bash
|
|
|
|
gpuspernode 8 # must be equal gpus-per-node provided in the job script, 8 for Altair nodes
|
|
|
|
```
|
|
|
|
|
|
|
|
## Testing `td` codes
|
|
|
|
For testing, it is recommended to use `td-testcase-uniform`.
|
|
|
|
```
|
|
|
|
# Use scratch area for tests
|
|
|
|
gabrielw@eagle:~$ cd grant_518/scratch/
|
|
|
|
gabrielw@eagle:~/grant_518/scratch$ cp -r $WSLDA/td-testcase-uniform .
|
|
|
|
gabrielw@eagle:~/grant_518/scratch$ cd td-testcase-uniform/
|
|
|
|
gabrielw@e1296:/mnt/storage_2/scratch/grant_518/td-testcase-uniform$ ls
|
|
|
|
env.sh input.txt job.sh logger.h Makefile predefines.h problem-definition.h README.txt
|
|
|
|
|
|
|
|
# Compile codes
|
|
|
|
gabrielw@eagle:~/grant_518/scratch/td-testcase-uniform$ srun --pty -n 1 /bin/bash
|
|
|
|
gabrielw@e1296:/mnt/storage_2/scratch/grant_518/td-testcase-uniform$ source env.sh
|
|
|
|
openmpi/3.1.4_gcc620 load complete.
|
|
|
|
cuda/11.2.1_460.32.03 load complete.
|
|
|
|
Currently Loaded Modulefiles:
|
|
|
|
1) gmp/5.1.3(default) 2) mpfr/3.1.2(default) 3) libmpc/1.0.1(default) 4) gcc/6.2.0 5) openmpi/3.1.4_gcc620 6) cuda/11.2.1_460.32.03
|
|
|
|
#define CODE "W-SLDA-TOOLKIT"
|
|
|
|
#define VERSION "2021.09.05"
|
|
|
|
|
|
|
|
gabrielw@e1296:~/grant_518/scratch/td-testcase-uniform$ make
|
|
|
|
mkdir -p ./obj/
|
|
|
|
mkdir -p ./
|
|
|
|
mpicc -std=gnu99 -O3 -I/mnt/storage_2/project_data/grant_518/wslda/hpc-engine/ -I/mnt/storage_2/project_data/grant_518/wslda/lib/wdata/c/ -I/mnt/storage_2/project_data/grant_518/wslda/lib/wderiv/c/ -I/mnt/storage_2/project_data/grant_518/wslda/lib/winterp/c/ -I./ -DTDWSLDA -DCODEDIM=1 -c -o obj/ccpca.o /mnt/storage_2/project_data/grant_518/wslda/hpc-engine/ccpca.c
|
|
|
|
# ...
|
|
|
|
gabrielw@e1296:~/grant_518/scratch/td-testcase-uniform$ exit
|
|
|
|
|
|
|
|
# submit the job and wait till it executes
|
|
|
|
gabrielw@eagle:~/grant_518/scratch/td-testcase-uniform$ sbatch job.sh
|
|
|
|
Submitted batch job 19731598
|
|
|
|
|
|
|
|
# check is files were produced...
|
|
|
|
gabrielw@eagle:~/grant_518/scratch/td-testcase-uniform$ ls
|
|
|
|
env.sh logger.h predefines.h td-wslda-1d test test_input.txt test_logger.h test_problem-definition.h test.stdout
|
|
|
|
input.txt Makefile problem-definition.h td-wslda-2d test_check.stamp test_j_a.wdat test.out test_rho_a.wdat test.wlog
|
|
|
|
job.sh obj README.txt td-wslda-3d test_delta.wdat test_j_b.wdat test_predefines.h test_rho_b.wdat test.wtxt
|
|
|
|
|
|
|
|
TODO |
|
|
\ No newline at end of file |
|
|
|
# ... and if the code was producing results - all numbers should be the same if everything went ok.
|
|
|
|
gabrielw@eagle:~/grant_518/scratch/td-testcase-uniform$ tail test.stdout
|
|
|
|
0.3500 17.00000061 17.99999904 34.99999965 0.43917132 1.91518974 -0.52650127 -0.94951715 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.25
|
|
|
|
0.7000 17.00000061 17.99999904 34.99999965 0.43917132 1.91518974 -0.52650127 -0.94951715 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.21
|
|
|
|
1.0500 17.00000061 17.99999904 34.99999965 0.43917132 1.91518974 -0.52650127 -0.94951715 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.21
|
|
|
|
1.4000 17.00000061 17.99999904 34.99999965 0.43917132 1.91518974 -0.52650127 -0.94951716 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.21
|
|
|
|
1.7500 17.00000061 17.99999904 34.99999965 0.43917132 1.91518974 -0.52650127 -0.94951716 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.21
|
|
|
|
2.1000 17.00000061 17.99999904 34.99999965 0.43917132 1.91518974 -0.52650127 -0.94951716 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.21
|
|
|
|
2.4500 17.00000061 17.99999904 34.99999965 0.43917132 1.91518974 -0.52650127 -0.94951716 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.21
|
|
|
|
2.8000 17.00000061 17.99999904 34.99999965 0.43917132 1.91518974 -0.52650127 -0.94951716 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.21
|
|
|
|
3.1500 17.00000061 17.99999904 34.99999965 0.43917132 1.91518974 -0.52650127 -0.94951716 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.21
|
|
|
|
3.5000 17.00000061 17.99999904 34.99999965 0.43917132 1.91518974 -0.52650127 -0.94951716 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.21
|
|
|
|
``` |
|
|
\ No newline at end of file |
