From 81ed72bf92c29bbbc5c375fbfab9bf6f6c694879 Mon Sep 17 00:00:00 2001 From: chn Date: Tue, 20 Feb 2024 18:22:38 +0800 Subject: [PATCH] localPackages.vasp-gpu: init --- local/pkgs/default.nix | 3 +- local/pkgs/nvhpc-sdk/default.nix | 26 ------- local/pkgs/nvhpc/default.nix | 42 +++++++++++ local/pkgs/vasp-gpu/default.nix | 39 ++++++++++ local/pkgs/vasp-gpu/makefile.include | 109 +++++++++++++++++++++++++++ 5 files changed, 192 insertions(+), 27 deletions(-) delete mode 100644 local/pkgs/nvhpc-sdk/default.nix create mode 100644 local/pkgs/nvhpc/default.nix create mode 100644 local/pkgs/vasp-gpu/default.nix create mode 100644 local/pkgs/vasp-gpu/makefile.include diff --git a/local/pkgs/default.nix b/local/pkgs/default.nix index 20461a1d..d0b34fdf 100644 --- a/local/pkgs/default.nix +++ b/local/pkgs/default.nix @@ -49,6 +49,7 @@ fluent-kde = callPackage ./fluent-kde { src = topInputs.fluent-kde; }; blurred-wallpaper = callPackage ./blurred-wallpaper { src = topInputs.blurred-wallpaper; }; slate = callPackage ./slate { src = topInputs.slate; }; - nvhpc-sdk = callPackage ./nvhpc-sdk {}; + nvhpc = callPackage ./nvhpc {}; lmod = callPackage ./lmod { src = topInputs.lmod; }; + vasp-gpu = callPackage ./vasp-gpu { inherit nvhpc lmod;}; } diff --git a/local/pkgs/nvhpc-sdk/default.nix b/local/pkgs/nvhpc-sdk/default.nix deleted file mode 100644 index 28c7cbc9..00000000 --- a/local/pkgs/nvhpc-sdk/default.nix +++ /dev/null @@ -1,26 +0,0 @@ -{ - stdenvNoCC, fetchurl, gcc, gfortran -}: -stdenvNoCC.mkDerivation rec -{ - pname = "nvhpc-sdk"; - version = "24.1"; - src = let versions = builtins.splitVersion version; in fetchurl - { - url = "https://developer.download.nvidia.com/hpc-sdk/${version}/" - + "nvhpc_20${builtins.elemAt versions 0}_${builtins.concatStringsSep "" versions}" - + "_Linux_x86_64_cuda_multi.tar.gz"; - sha256 = "1n0x1x7ywvr3623ylvrjagayn44mbvfas3c3062p7y3asmgjx697"; - }; - BuildInputs = [ gfortran gfortran.cc gcc ]; - propagatedBuildInputs = BuildInputs; - postUnpack = ''patchShebangs . ''; - dontBuild = true; - NVHPC_SILENT = "true"; - NVHPC_INSTALL_TYPE = "single"; - installPhase = - '' - export NVHPC_INSTALL_DIR=$out/share/nvhpc-sdk - ./install - ''; -} diff --git a/local/pkgs/nvhpc/default.nix b/local/pkgs/nvhpc/default.nix new file mode 100644 index 00000000..e26ec1ff --- /dev/null +++ b/local/pkgs/nvhpc/default.nix @@ -0,0 +1,42 @@ +{ + stdenvNoCC, fetchurl, + gcc, gfortran, autoPatchelfHook, + flock, glibc, coreutils, util-linux, iconv +}: +stdenvNoCC.mkDerivation rec +{ + pname = "nvhpc"; + version = "24.1"; + src = let versions = builtins.splitVersion version; in fetchurl + { + url = "https://developer.download.nvidia.com/hpc-sdk/${version}/" + + "nvhpc_20${builtins.elemAt versions 0}_${builtins.concatStringsSep "" versions}" + + "_Linux_x86_64_cuda_multi.tar.gz"; + sha256 = "1n0x1x7ywvr3623ylvrjagayn44mbvfas3c3062p7y3asmgjx697"; + }; + BuildInputs = [ gfortran gfortran.cc gcc ]; + propagatedBuildInputs = BuildInputs; + nativeBuildInputs = + [ + autoPatchelfHook flock + glibc.bin # ldconfig + coreutils # chmod + util-linux # rev + ]; + postUnpack = ''patchShebangs .''; + dontBuild = true; + dontStrip = true; + NVHPC_SILENT = "true"; + NVHPC_INSTALL_TYPE = "single"; + installPhase = + '' + export NVHPC_INSTALL_DIR=$out/share/nvhpc + ldconfig -C $NIX_BUILD_TOP/ld.so.cache + sed -i 's|/bin/chmod|chmod|g' install_components/install + sed -i 's|/sbin/ldconfig|ldconfig -C $NIX_BUILD_TOP/ld.so.cache|g' install_components/install + sed -i 's|/usr/lib/x86_64-linux-gnu|${iconv.out}/lib|g' \ + install_components/Linux_x86_64/${version}/compilers/bin/makelocalrc + ./install + ''; + autoPatchelfIgnoreMissingDeps = [ "*" ]; +} diff --git a/local/pkgs/vasp-gpu/default.nix b/local/pkgs/vasp-gpu/default.nix new file mode 100644 index 00000000..aff5670b --- /dev/null +++ b/local/pkgs/vasp-gpu/default.nix @@ -0,0 +1,39 @@ +{ + stdenvNoCC, requireFile, + nvhpc, rsync, mkl, lmod, bash, which, + glibc_multi +}: +stdenvNoCC.mkDerivation rec +{ + pname = "vasp"; + version = "6.4.0"; + # nix-store --query --hash $(nix store add-path ./vasp-6.4.0) + src = requireFile + { + name = "${pname}-${version}"; + sha256 = "189i1l5q33ynmps93p2mwqf5fx7p4l50sls1krqlv8ls14s3m71f"; + hashMode = "recursive"; + message = "Source file not found."; + }; + makeFlags = "DEPS=1"; + enableParallelBuilding = true; + buildInputs = [ nvhpc mkl glibc_multi ]; + nativeBuildInputs = [ rsync bash which ]; + MKLROOT = mkl; + configurePhase = + '' + cp ${./makefile.include} makefile.include + . ${lmod}/lmod/lmod/init/bash + module use ${nvhpc}/share/nvhpc/modulefiles + module load nvhpc + # chmod +w makefile.include + # echo "CPP_OPTIONS += -Duse_shmem -Dshmem_bcast_buffer -Dshmem_rproj" >> makefile.include + # echo "OBJECTS_LIB += getshmem.o" >> makefile.include + mkdir -p bin + ''; + installPhase = + '' + mkdir -p $out + cp -r bin $out + ''; +} diff --git a/local/pkgs/vasp-gpu/makefile.include b/local/pkgs/vasp-gpu/makefile.include new file mode 100644 index 00000000..7dde3049 --- /dev/null +++ b/local/pkgs/vasp-gpu/makefile.include @@ -0,0 +1,109 @@ +# Default precompiler options +CPP_OPTIONS = -DHOST=\"LinuxNV\" \ + -DMPI -DMPI_INPLACE -DMPI_BLOCK=8000 -Duse_collective \ + -DscaLAPACK \ + -DCACHE_SIZE=4000 \ + -Davoidalloc \ + -Dvasp6 \ + -Duse_bse_te \ + -Dtbdyn \ + -Dqd_emulate \ + -Dfock_dblbuf \ + -D_OPENMP \ + -D_OPENACC \ + -DUSENCCL -DUSENCCLP2P + +CPP = nvfortran -Mpreprocess -Mfree -Mextend -E $(CPP_OPTIONS) $*$(FUFFIX) > $*$(SUFFIX) + +# N.B.: you might need to change the cuda-version here +# to one that comes with your NVIDIA-HPC SDK +FC = mpif90 -acc -gpu=cc89,cuda12.3 -mp +FCL = mpif90 -acc -gpu=cc89,cuda12.3 -mp -c++libs + +FREE = -Mfree + +FFLAGS = -Mbackslash -Mlarge_arrays + +OFLAG = -fast + +DEBUG = -Mfree -O0 -traceback + +OBJECTS = fftmpiw.o fftmpi_map.o fftw3d.o fft3dlib.o + +LLIBS = -cudalib=cublas,cusolver,cufft,nccl -cuda + +# Redefine the standard list of O1 and O2 objects +SOURCE_O1 := pade_fit.o minimax_dependence.o +SOURCE_O2 := pead.o + +# For what used to be vasp.5.lib +CPP_LIB = $(CPP) +FC_LIB = nvfortran +CC_LIB = nvc -w +CFLAGS_LIB = -O +FFLAGS_LIB = -O1 -Mfixed +FREE_LIB = $(FREE) + +OBJECTS_LIB = linpack_double.o + +# For the parser library +CXX_PARS = nvc++ --no_warnings + +## +## Customize as of this point! Of course you may change the preceding +## part of this file as well if you like, but it should rarely be +## necessary ... +## +# When compiling on the target machine itself , change this to the +# relevant target when cross-compiling for another architecture +VASP_TARGET_CPU ?= -tp=zen4 +FFLAGS += $(VASP_TARGET_CPU) + +# Specify your NV HPC-SDK installation (mandatory) +#... first try to set it automatically +NVROOT =$(shell which nvfortran | awk -F /compilers/bin/nvfortran '{ print $$1 }') + +# If the above fails, then NVROOT needs to be set manually +#NVHPC ?= /opt/nvidia/hpc_sdk +#NVVERSION = 21.11 +#NVROOT = $(NVHPC)/Linux_x86_64/$(NVVERSION) + +## Improves performance when using NV HPC-SDK >=21.11 and CUDA >11.2 +#OFLAG_IN = -fast -Mwarperf +#SOURCE_IN := nonlr.o + +# Software emulation of quadruple precsion (mandatory) +QD ?= $(NVROOT)/compilers/extras/qd +LLIBS += -L$(QD)/lib -lqdmod -lqd +INCS += -I$(QD)/include/qd + +# Intel MKL for FFTW, BLAS, LAPACK, and scaLAPACK +MKLROOT ?= /path/to/your/mkl/installation +LLIBS_MKL = -Mmkl -L$(MKLROOT)/lib/intel64 -lmkl_scalapack_lp64 -lmkl_blacs_openmpi_lp64 +INCS += -I$(MKLROOT)/include/fftw + +# Use a separate scaLAPACK installation (optional but recommended in combination with OpenMPI) +# Comment out the two lines below if you want to use scaLAPACK from MKL instead +# SCALAPACK_ROOT ?= /path/to/your/scalapack/installation +# LLIBS_MKL = -L$(SCALAPACK_ROOT)/lib -lscalapack -Mmkl + +LLIBS += $(LLIBS_MKL) + +# HDF5-support (optional but strongly recommended) +#CPP_OPTIONS+= -DVASP_HDF5 +#HDF5_ROOT ?= /path/to/your/hdf5/installation +#LLIBS += -L$(HDF5_ROOT)/lib -lhdf5_fortran +#INCS += -I$(HDF5_ROOT)/include + +# For the VASP-2-Wannier90 interface (optional) +#CPP_OPTIONS += -DVASP2WANNIER90 +#WANNIER90_ROOT ?= /path/to/your/wannier90/installation +#LLIBS += -L$(WANNIER90_ROOT)/lib -lwannier + +# For the fftlib library (hardly any benefit for the OpenACC GPU port, especially in combination with MKL's FFTs) +#CPP_OPTIONS+= -Dsysv +#FCL += fftlib.o +#CXX_FFTLIB = nvc++ -mp --no_warnings -std=c++11 -DFFTLIB_USE_MKL -DFFTLIB_THREADSAFE +#INCS_FFTLIB = -I./include -I$(MKLROOT)/include/fftw +#LIBS += fftlib +#LLIBS += -ldl