From bbc8071fa4113f556a53f554e23db7cb76e45d81 Mon Sep 17 00:00:00 2001 From: chn Date: Thu, 14 Mar 2024 22:47:47 +0800 Subject: [PATCH] localPackages.vasp.amd: init --- local/pkgs/aocl/default.nix | 7 +- local/pkgs/default.nix | 10 ++- local/pkgs/hdf5-aocc/default.nix | 9 ++- local/pkgs/hdf5-oneapi/default.nix | 4 - local/pkgs/openmpi-aocc/default.nix | 45 +++++++++++ local/pkgs/vasp/amd/default.nix | 38 ++++----- local/pkgs/vasp/amd/makefile.include-6.3.1 | 91 ++++++++++++++++++++++ local/pkgs/vasp/amd/makefile.include-6.4.0 | 91 ++++++++++++++++++++++ 8 files changed, 264 insertions(+), 31 deletions(-) create mode 100644 local/pkgs/openmpi-aocc/default.nix create mode 100644 local/pkgs/vasp/amd/makefile.include-6.3.1 create mode 100644 local/pkgs/vasp/amd/makefile.include-6.4.0 diff --git a/local/pkgs/aocl/default.nix b/local/pkgs/aocl/default.nix index 7119b43b..37141e4a 100644 --- a/local/pkgs/aocl/default.nix +++ b/local/pkgs/aocl/default.nix @@ -1,6 +1,6 @@ { stdenv, fetchurl }: -stdenv.mkDerivation +stdenv.mkDerivation rec { pname = "aocl"; version = "4.2.0"; @@ -12,7 +12,10 @@ stdenv.mkDerivation dontBuild = true; installPhase = '' - bash ./install.sh -t $out/share + installDir=$(mktemp -d) + bash ./install.sh -t $installDir + mkdir -p $out + cp -r $installDir/${version}/aocc/lib_LP64 $out/lib ''; dontFixup = true; } diff --git a/local/pkgs/default.nix b/local/pkgs/default.nix index 7a3380b4..0b3c04a3 100644 --- a/local/pkgs/default.nix +++ b/local/pkgs/default.nix @@ -62,6 +62,13 @@ hdf5 = hdf5-oneapi.override { oneapi = oneapi."2024.0"; }; wannier90 = callPackage "${topInputs.nixpkgs-unstable}/pkgs/by-name/wa/wannier90/package.nix" {}; }; + amd = callPackage ./vasp/amd + { + inherit aocc aocl; + hdf5 = hdf5-aocc; + openmpi = openmpi-aocc; + wannier90 = callPackage "${topInputs.nixpkgs-unstable}/pkgs/by-name/wa/wannier90/package.nix" {}; + }; }; hdf5-nvhpc = callPackage ./hdf5-nvhpc { inherit lmod; inherit (hdf5) src; nvhpc = nvhpc."24.1"; }; hdf5-oneapi = callPackage ./hdf5-oneapi { inherit lmod; inherit (hdf5) src; oneapi = oneapi."2024.0"; }; @@ -69,5 +76,6 @@ mumax = callPackage ./mumax { src = topInputs.mumax; }; aocc = callPackage ./aocc {}; aocl = callPackage ./aocl {}; - hdf5-aocc = callPackage ./hdf5-aocc { inherit (hdf5) src; inherit aocc; }; + hdf5-aocc = callPackage ./hdf5-aocc { inherit (hdf5) src; inherit aocc; openmpi = openmpi-aocc; }; + openmpi-aocc = callPackage ./openmpi-aocc { inherit aocc; }; } diff --git a/local/pkgs/hdf5-aocc/default.nix b/local/pkgs/hdf5-aocc/default.nix index 6ce0aea3..e91f8603 100644 --- a/local/pkgs/hdf5-aocc/default.nix +++ b/local/pkgs/hdf5-aocc/default.nix @@ -1,13 +1,13 @@ { buildFHSEnv, writeScript, stdenvNoCC, src, - aocc, cmake + aocc, cmake, openmpi }: let buildEnv = buildFHSEnv { name = "buildEnv"; - targetPkgs = pkgs: with pkgs; [ zlib aocc gcc.cc gcc.cc.lib mpi glibc.dev binutils.bintools ]; + targetPkgs = pkgs: with pkgs; [ zlib aocc gcc.cc gcc.cc.lib glibc.dev binutils.bintools openmpi pkg-config ]; extraBwrapArgs = [ "--bind" "$out" "$out" ]; }; buildScript = writeScript "build" @@ -15,7 +15,7 @@ let mkdir build cd build cmake -DCMAKE_INSTALL_PREFIX=$out -DHDF5_INSTALL_CMAKE_DIR=$out/lib/cmake \ - -DHDF5_BUILD_FORTRAN=ON -DHDF5_ENABLE_PARALLEL=ON -DBUILD_SHARED_LIBS=OFF .. + -DHDF5_BUILD_FORTRAN=ON -DHDF5_ENABLE_PARALLEL=ON -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF .. make -j$NIX_BUILD_CORES make install ''; @@ -32,6 +32,9 @@ in stdenvNoCC.mkDerivation OMPI_CC = "clang"; OMPI_CXX = "clang++"; OMPI_FC = "flang"; + # CFLAGS = "-march=${stdenvNoCC.hostPlatform.gcc.arch} -O2"; + # CXXFLAGS = "-march=${stdenvNoCC.hostPlatform.gcc.arch} -O2"; + # FCFLAGS = "-march=${stdenvNoCC.hostPlatform.gcc.arch} -O2"; buildPhase = '' mkdir -p $out diff --git a/local/pkgs/hdf5-oneapi/default.nix b/local/pkgs/hdf5-oneapi/default.nix index 39d7cf17..bfbbe470 100644 --- a/local/pkgs/hdf5-oneapi/default.nix +++ b/local/pkgs/hdf5-oneapi/default.nix @@ -33,10 +33,6 @@ in stdenvNoCC.mkDerivation dontConfigure = true; enableParallelBuilding = true; nativeBuildInputs = [ cmake ]; - # somehow make it build failed, currently remove it - # CFLAGS = "-x${oneapiArch}"; - # CXXFLAGS = "-x${oneapiArch}"; - # FFLAGS = "-x${oneapiArch}"; I_MPI_CC = "icx"; I_MPI_CXX = "icpx"; I_MPI_FC = "ifx"; diff --git a/local/pkgs/openmpi-aocc/default.nix b/local/pkgs/openmpi-aocc/default.nix new file mode 100644 index 00000000..b3c51ec2 --- /dev/null +++ b/local/pkgs/openmpi-aocc/default.nix @@ -0,0 +1,45 @@ +{ + lib, buildFHSEnv, writeScript, stdenvNoCC, + openmpi, + aocc, cmake, libnl, pmix, libpsm2, libfabric, zlib, numactl, ucx, ucc, libevent, hwloc, rdma-core, perl +}: +let + buildEnv = buildFHSEnv + { + name = "buildEnv"; + targetPkgs = pkgs: with pkgs; [ zlib aocc gcc.cc gcc.cc.lib glibc.dev binutils.bintools libnl numactl ucx ucc libevent hwloc rdma-core libpsm2 libfabric perl ]; + extraBwrapArgs = [ "--bind" "$out" "$out" ]; + }; + buildScript = writeScript "build" + '' + ./configure --prefix=$out --disable-mca-dso + make -j$NIX_BUILD_CORES + make install + ''; +in stdenvNoCC.mkDerivation +{ + name = "openmpi-aocc"; + inherit (openmpi) src postPatch; + dontConfigure = true; + CC = "clang"; + CXX = "clang++"; + FC = "flang"; + OMPI_CC = "clang"; + OMPI_CXX = "clang++"; + OMPI_FC = "flang"; + CFLAGS = "-march=${stdenvNoCC.hostPlatform.gcc.arch} -O2"; + CXXFLAGS = "-march=${stdenvNoCC.hostPlatform.gcc.arch} -O2"; + FCFLAGS = "-march=${stdenvNoCC.hostPlatform.gcc.arch} -O2"; + enableParallelBuilding = true; + buildPhase = + '' + runHook preBuild + mkdir -p $out + ${buildEnv}/bin/buildEnv ${buildScript} + runHook postBuild + ''; + postBuild = with openmpi; postInstall + postFixup; + dontInstall = true; + dontFixup = true; + requiredSystemFeatures = [ "gccarch-exact-${stdenvNoCC.hostPlatform.gcc.arch}" "big-parallel" ]; +} diff --git a/local/pkgs/vasp/amd/default.nix b/local/pkgs/vasp/amd/default.nix index 2299590b..c32501f3 100644 --- a/local/pkgs/vasp/amd/default.nix +++ b/local/pkgs/vasp/amd/default.nix @@ -1,33 +1,27 @@ { buildFHSEnv, writeScript, stdenvNoCC, requireFile, substituteAll, - aocc, rsync, which, hdf5, wannier90 + aocc, rsync, which, hdf5, wannier90, aocl, openmpi }: let sources = import ../source.nix { inherit requireFile; }; buildEnv = buildFHSEnv { name = "buildEnv"; - targetPkgs = pkgs: with pkgs; [ zlib ]; + targetPkgs = pkgs: with pkgs; [ zlib aocc aocl openmpi gcc.cc gcc.cc.lib glibc.dev binutils.bintools ]; }; buildScript = writeScript "build" '' - . ${lmod}/share/lmod/lmod/init/bash - module use ${nvhpc}/share/nvhpc/modulefiles - module load nvhpc mkdir -p bin - make DEPS=1 -j$NIX_BUILD_CORES + make DEPS=1 -j$NIX_BUILD_CORES std ''; include = version: substituteAll { src = ./makefile.include-${version}; - cudaCapabilities = builtins.concatStringsSep "," (builtins.map - (cap: "cc${builtins.replaceStrings ["."] [""] cap}") - cudaCapabilities); - inherit nvhpcArch; + gccArch = stdenvNoCC.hostPlatform.gcc.arch; }; vasp = version: stdenvNoCC.mkDerivation rec { - pname = "vasp-nvidia"; + pname = "vasp-amd"; inherit version; src = sources.${version}; configurePhase = @@ -35,12 +29,18 @@ let cp ${include version} makefile.include cp ${../constr_cell_relax.F} src/constr_cell_relax.F ''; - enableParallelBuilding = true; - buildInputs = [ mkl hdf5 wannier90 ]; - nativeBuildInputs = [ gfortran rsync which ]; - MKLROOT = mkl; + enableParallelBuilding = false; + buildInputs = [ hdf5 wannier90 ]; + nativeBuildInputs = [ rsync which ]; + AMDBLIS_ROOT = aocl; + AMDLIBFLAME_ROOT = aocl; + AMDSCALAPACK_ROOT = aocl; + AMDFFTW_ROOT = aocl; HDF5_ROOT = hdf5; WANNIER90_ROOT = wannier90; + OMPI_CC = "clang"; + OMPI_CXX = "clang++"; + OMPI_FC = "flang"; buildPhase = "${buildEnv}/bin/buildEnv ${buildScript}"; installPhase = '' @@ -52,10 +52,6 @@ let }; startScript = version: writeScript "vasp-nvidia-${version}" '' - . ${lmod}/share/lmod/lmod/init/bash - module use ${nvhpc}/share/nvhpc/modulefiles - module load nvhpc - # if SLURM_CPUS_PER_TASK is set, use it to set OMP_NUM_THREADS if [ -n "''${SLURM_CPUS_PER_TASK-}" ]; then export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK @@ -65,8 +61,8 @@ let ''; runEnv = version: buildFHSEnv { - name = "vasp-nvidia-${version}"; - targetPkgs = pkgs: with pkgs; [ zlib (vasp version) ]; + name = "vasp-amd-${version}"; + targetPkgs = pkgs: with pkgs; [ zlib (vasp version) aocc aocl ]; runScript = startScript version; }; in builtins.mapAttrs (version: _: runEnv version) sources diff --git a/local/pkgs/vasp/amd/makefile.include-6.3.1 b/local/pkgs/vasp/amd/makefile.include-6.3.1 new file mode 100644 index 00000000..c4044744 --- /dev/null +++ b/local/pkgs/vasp/amd/makefile.include-6.3.1 @@ -0,0 +1,91 @@ +# Default precompiler options +CPP_OPTIONS = -DHOST=\"LinuxGNU\" \ + -DMPI -DMPI_BLOCK=8000 -Duse_collective \ + -DscaLAPACK \ + -DCACHE_SIZE=4000 \ + -Davoidalloc \ + -Dvasp6 \ + -Duse_bse_te \ + -Dtbdyn \ + -Dfock_dblbuf \ + -D_OPENMP -Duse_shmem -Dshmem_bcast_buffer -Dshmem_rproj + +CPP = flang -E -C -w $*$(FUFFIX) >$*$(SUFFIX) $(CPP_OPTIONS) + +FC = mpif90 -fopenmp +FCL = mpif90 -fopenmp + +FREE = -ffree-form -ffree-line-length-none + +FFLAGS = -w -fno-fortran-main -Mbackslash + +OFLAG = -O2 +OFLAG_IN = $(OFLAG) +DEBUG = -O0 + +OBJECTS = fftmpiw.o fftmpi_map.o fftw3d.o fft3dlib.o +OBJECTS_O1 += fftw3d.o fftmpi.o fftmpiw.o +OBJECTS_O2 += fft3dlib.o + +# For what used to be vasp.5.lib +CPP_LIB = $(CPP) +FC_LIB = $(FC) +CC_LIB = clang +CFLAGS_LIB = -O +FFLAGS_LIB = -O1 +FREE_LIB = $(FREE) + +OBJECTS_LIB = linpack_double.o getshmem.o + +# For the parser library +CXX_PARS = clang++ +LLIBS = -lstdc++ + +## +## Customize as of this point! Of course you may change the preceding +## part of this file as well if you like, but it should rarely be +## necessary ... +## + +# When compiling on the target machine itself, change this to the +# relevant target when cross-compiling for another architecture +VASP_TARGET_CPU ?= -march=@gccArch@ +FFLAGS += $(VASP_TARGET_CPU) + +# BLAS (mandatory) +AMDBLIS_ROOT ?= /path/to/your/amdblis/installation +BLAS = -L${AMDBLIS_ROOT}/lib -lblis-mt + +# LAPACK (mandatory) +AMDLIBFLAME_ROOT ?= /path/to/your/amdlibflame/installation +LAPACK = -L${AMDLIBFLAME_ROOT}/lib -lflame + +# scaLAPACK (mandatory) +AMDSCALAPACK_ROOT ?= /path/to/your/amdscalapack/installation +SCALAPACK = -L${AMDSCALAPACK_ROOT}/lib -lscalapack + +LLIBS += $(SCALAPACK) $(LAPACK) $(BLAS) + +# FFTW (mandatory) +AMDFFTW_ROOT ?= /path/to/your/amdfftw/installation +LLIBS += -L$(AMDFFTW_ROOT)/lib -lfftw3 -lfftw3_omp +INCS += -I$(AMDFFTW_ROOT)/include + +# HDF5-support (optional but strongly recommended) +CPP_OPTIONS+= -DVASP_HDF5 +HDF5_ROOT ?= /path/to/your/hdf5/installation +LLIBS += -L$(HDF5_ROOT)/lib -lhdf5_fortran +INCS += -I$(HDF5_ROOT)/include + +# For the VASP-2-Wannier90 interface (optional) +CPP_OPTIONS += -DVASP2WANNIER90 +WANNIER90_ROOT ?= /path/to/your/wannier90/installation +LLIBS += -L$(WANNIER90_ROOT)/lib -lwannier + +# For the fftlib library (recommended) +#CPP_OPTIONS+= -Dsysv +#FCL += fftlib.o +#CXX_FFTLIB = clang++ -fopenmp -std=c++11 -DFFTLIB_THREADSAFE +#INCS_FFTLIB = -I./include -I$(AMDFFTW_ROOT)/include +#LIBS += fftlib +#LLIBS += -ldl diff --git a/local/pkgs/vasp/amd/makefile.include-6.4.0 b/local/pkgs/vasp/amd/makefile.include-6.4.0 new file mode 100644 index 00000000..00b6ce44 --- /dev/null +++ b/local/pkgs/vasp/amd/makefile.include-6.4.0 @@ -0,0 +1,91 @@ +# Default precompiler options +CPP_OPTIONS = -DHOST=\"LinuxGNU\" \ + -DMPI -DMPI_BLOCK=8000 -Duse_collective \ + -DscaLAPACK \ + -DCACHE_SIZE=4000 \ + -Davoidalloc \ + -Dvasp6 \ + -Duse_bse_te \ + -Dtbdyn \ + -Dfock_dblbuf \ + -D_OPENMP -Duse_shmem -Dshmem_bcast_buffer -Dshmem_rproj + +CPP = flang -E -ffree-form -C -w $*$(FUFFIX) >$*$(SUFFIX) $(CPP_OPTIONS) + +FC = mpif90 -fopenmp +FCL = mpif90 -fopenmp + +FREE = -ffree-form -ffree-line-length-none + +FFLAGS = -w -fno-fortran-main -Mbackslash + +OFLAG = -O2 +OFLAG_IN = $(OFLAG) +DEBUG = -O0 + +OBJECTS = fftmpiw.o fftmpi_map.o fftw3d.o fft3dlib.o +OBJECTS_O1 += fftw3d.o fftmpi.o fftmpiw.o +OBJECTS_O2 += fft3dlib.o + +# For what used to be vasp.5.lib +CPP_LIB = $(CPP) +FC_LIB = $(FC) +CC_LIB = clang +CFLAGS_LIB = -O +FFLAGS_LIB = -O1 +FREE_LIB = $(FREE) + +OBJECTS_LIB = linpack_double.o getshmem.o + +# For the parser library +CXX_PARS = clang++ +LLIBS = -lstdc++ + +## +## Customize as of this point! Of course you may change the preceding +## part of this file as well if you like, but it should rarely be +## necessary ... +## + +# When compiling on the target machine itself, change this to the +# relevant target when cross-compiling for another architecture +VASP_TARGET_CPU ?= -march=@gccArch@ +FFLAGS += $(VASP_TARGET_CPU) + +# BLAS (mandatory) +AMDBLIS_ROOT ?= /path/to/your/amdblis/installation +BLAS = -L${AMDBLIS_ROOT}/lib -lblis-mt + +# LAPACK (mandatory) +AMDLIBFLAME_ROOT ?= /path/to/your/amdlibflame/installation +LAPACK = -L${AMDLIBFLAME_ROOT}/lib -lflame + +# scaLAPACK (mandatory) +AMDSCALAPACK_ROOT ?= /path/to/your/amdscalapack/installation +SCALAPACK = -L${AMDSCALAPACK_ROOT}/lib -lscalapack + +LLIBS += $(SCALAPACK) $(LAPACK) $(BLAS) + +# FFTW (mandatory) +AMDFFTW_ROOT ?= /path/to/your/amdfftw/installation +LLIBS += -L$(AMDFFTW_ROOT)/lib -lfftw3 -lfftw3_omp +INCS += -I$(AMDFFTW_ROOT)/include + +# HDF5-support (optional but strongly recommended) +CPP_OPTIONS+= -DVASP_HDF5 +HDF5_ROOT ?= /path/to/your/hdf5/installation +LLIBS += -L$(HDF5_ROOT)/lib -lhdf5_fortran +INCS += -I$(HDF5_ROOT)/include + +# For the VASP-2-Wannier90 interface (optional) +CPP_OPTIONS += -DVASP2WANNIER90 +WANNIER90_ROOT ?= /path/to/your/wannier90/installation +LLIBS += -L$(WANNIER90_ROOT)/lib -lwannier + +# For the fftlib library (recommended) +#CPP_OPTIONS+= -Dsysv +#FCL += fftlib.o +#CXX_FFTLIB = clang++ -fopenmp -std=c++11 -DFFTLIB_THREADSAFE +#INCS_FFTLIB = -I./include -I$(AMDFFTW_ROOT)/include +#LIBS += fftlib +#LLIBS += -ldl