From caa44d5ddba76f558c01eba5bf5509aeea600eaf Mon Sep 17 00:00:00 2001 From: chn Date: Tue, 17 Dec 2024 20:06:05 +0800 Subject: [PATCH] debug --- packages/default.nix | 5 +- packages/nvhpcStdenv.nix | 25 ++++-- packages/vasp/nvidia/default.nix | 124 ++++++++------------------ packages/vasp/nvidia/makefile.include | 6 +- 4 files changed, 61 insertions(+), 99 deletions(-) diff --git a/packages/default.nix b/packages/default.nix index f3f96bb2..7b157721 100644 --- a/packages/default.nix +++ b/packages/default.nix @@ -40,8 +40,8 @@ inputs: rec }; nvidia = inputs.pkgs.callPackage ./vasp/nvidia { - inherit lmod nvhpc vtst src; - hdf5 = hdf5-nvhpc; + inherit (nvhpcPackages) stdenv hdf5 mpi qd; + inherit vtst src; wannier90 = inputs.pkgs.wannier90.overrideAttrs { buildFlags = [ "dynlib" ]; }; }; intel = inputs.pkgs.callPackage ./vasp/intel @@ -113,6 +113,7 @@ inputs: rec hdf5 = inputs.pkgs.hdf5.override { inherit (final) stdenv; cppSupport = false; fortranSupport = true; enableShared = false; enableStatic = true; }; qd = final.callPackage ./qd.nix { src = inputs.topInputs.qd; }; + mpi = inputs.pkgs.openmpi.override { inherit (final) stdenv; enableSubstitute = false; }; }); gccFull = inputs.pkgs.symlinkJoin { diff --git a/packages/nvhpcStdenv.nix b/packages/nvhpcStdenv.nix index 2202a06d..cbfcfb61 100644 --- a/packages/nvhpcStdenv.nix +++ b/packages/nvhpcStdenv.nix @@ -1,13 +1,13 @@ { - src, stdenv, autoPatchelfHook, wrapCCWith, writeText, addAttrsToDerivation, config, overrideCC, - gcc, glibc_multi, libz, zstd, libxml2, flock, numactl, ncurses, dpkg + src, stdenv, autoPatchelfHook, wrapCCWith, writeText, addAttrsToDerivation, config, overrideCC, symlinkJoin, + gcc, glibc_multi, libz, zstd, libxml2, flock, numactl, ncurses, dpkg, cudaPackages, openssl, gmp, openssl_1_1 }: let nvhpc = stdenv.mkDerivation { pname = "nvhpc"; inherit (src) src version; - buildInputs = [ libz libxml2 zstd numactl ncurses ]; + buildInputs = [ libz libxml2 zstd numactl ncurses openssl gmp openssl_1_1 ]; nativeBuildInputs = [ autoPatchelfHook dpkg flock ]; langFortran = true; dontConfigure = true; @@ -19,13 +19,14 @@ let # NVHPC use very complex mechanism to identify the location of compilers, headers, etc. # we should keep the original structure mkdir -p $out/opt/nvidia/hpc_sdk/Linux_x86_64/${src.version}/ - cp -r opt/nvidia/hpc_sdk/Linux_x86_64/${src.version}/compilers $out/opt/nvidia/hpc_sdk/Linux_x86_64/${src.version} + cp -r opt/nvidia/hpc_sdk/Linux_x86_64/${src.version}/{compilers,cuda} \ + $out/opt/nvidia/hpc_sdk/Linux_x86_64/${src.version} ''; postFixup = '' sed -i '/makelocalrc executed by/d' $out/opt/nvidia/hpc_sdk/Linux_x86_64/${src.version}/compilers/bin/makelocalrc $out/opt/nvidia/hpc_sdk/Linux_x86_64/${src.version}/compilers/bin/makelocalrc \ - $out/opt/nvidia/hpc_sdk/Linux_x86_64/${src.version}/compilers/bin -x -no-cuda + $out/opt/nvidia/hpc_sdk/Linux_x86_64/${src.version}/compilers/bin -x ln -s $out/opt/nvidia/hpc_sdk/Linux_x86_64/${src.version}/compilers/bin $out ''; }; @@ -35,8 +36,12 @@ let set DEFLIBDIR=${glibc_multi}/lib; set DEFSTDOBJDIR=${glibc_multi}/lib; ''; + # do not set -gpu=cuda12.4 since this only switch the cuda version installed with NVHPC cudaCapability = builtins.concatStringsSep "," - (builtins.map (cap: "cc${builtins.replaceStrings ["."] [""] cap}") config.cudaCapabilities); + ( + (builtins.map (cap: "cc${builtins.replaceStrings ["."] [""] cap}") config.cudaCapabilities) + ++ [ "cuda12.6" ] + ); wrapper = (wrapCCWith { cc = nvhpc; @@ -47,7 +52,13 @@ let echo "-tp=${config.nvhpcArch}" >> $out/nix-support/cc-cflags-before echo "-gpu=${cudaCapability}" >> $out/nix-support/cc-cflags-before - echo "-noswitcherror -#" >> $out/nix-support/cc-cflags + echo "-noswitcherror" >> $out/nix-support/cc-cflags + + echo 'export "PATH=${gcc}/bin:$PATH"' >> $out/nix-support/cc-wrapper-hook + + # print verbose output for debugging + echo "-v" >> $out/nix-support/cc-cflags + # echo "-#" >> $out/nix-support/cc-cflags # echo "" > $out/nix-support/add-hardening.sh diff --git a/packages/vasp/nvidia/default.nix b/packages/vasp/nvidia/default.nix index 4939b96b..3cafacbb 100644 --- a/packages/vasp/nvidia/default.nix +++ b/packages/vasp/nvidia/default.nix @@ -1,92 +1,44 @@ { - buildFHSEnv, writeScript, stdenvNoCC, substituteAll, symlinkJoin, src, - config, cudaCapabilities ? config.cudaCapabilities, nvhpcArch ? config.nvhpcArch or "px", - nvhpc, lmod, mkl, gfortran, rsync, which, hdf5, wannier90, zlib, vtst + stdenv, src, writeShellScriptBin, + rsync, which, wannier90, hdf5, vtst, mpi, mkl, qd }: -let - buildEnv = buildFHSEnv { name = "buildEnv"; targetPkgs = _: [ zlib ]; }; - buildScript = writeScript "build" +let vasp = stdenv.mkDerivation +{ + name = "vasp-nvidia"; + inherit src; + # patches = [ ../vtst.patch ]; + configurePhase = '' - . ${lmod}/share/lmod/lmod/init/bash - module use ${nvhpc}/share/nvhpc/modulefiles - module load nvhpc - make DEPS=1 -j$NIX_BUILD_CORES + cp ${./makefile.include} makefile.include + chmod +w makefile.include + cp ${../constr_cell_relax.F} src/constr_cell_relax.F + # cp -r ${vtst}/* src + chmod -R +w src ''; - include = substituteAll - { - src = ./makefile.include; - cudaCapabilities = builtins.concatStringsSep "," (builtins.map - (cap: "cc${builtins.replaceStrings ["."] [""] cap}") - cudaCapabilities); - inherit nvhpcArch; - }; - vasp = stdenvNoCC.mkDerivation - { - name = "vasp-nvidia"; - inherit src; - # patches = [ ../vtst.patch ]; - configurePhase = - '' - cp ${include} makefile.include - chmod +w makefile.include - cp ${../constr_cell_relax.F} src/constr_cell_relax.F - # cp -r ${vtst}/* src - chmod -R +w src - ''; - enableParallelBuilding = true; - buildInputs = [ mkl hdf5 wannier90 ]; - nativeBuildInputs = [ gfortran rsync which ]; - MKLROOT = mkl; - HDF5_ROOT = hdf5; - WANNIER90_ROOT = wannier90; - buildPhase = "${buildEnv}/bin/buildEnv ${buildScript}"; - installPhase = - '' - mkdir -p $out/bin - for i in std gam ncl; do cp bin/vasp_$i $out/bin/vasp-$i; done - mkdir $out/src - ln -s ${src} $out/src/vasp - ln -s ${vtst} $out/src/vtst - ''; - dontFixup = true; - requiredSystemFeatures = [ "gccarch-exact-${stdenvNoCC.hostPlatform.gcc.arch}" "big-parallel" ]; - }; - startScript = variant: writeScript "vasp-nvidia" + buildInputs = [ hdf5 wannier90 mkl ]; + nativeBuildInputs = [ rsync which mpi ]; + installPhase = '' - . ${lmod}/share/lmod/lmod/init/bash - module use ${nvhpc}/share/nvhpc/modulefiles - module load nvhpc - - # if OMP_NUM_THREADS is not set, set it according to SLURM_CPUS_PER_TASK or to 1 - if [ -z "''${OMP_NUM_THREADS-}" ]; then - if [ -n "''${SLURM_CPUS_PER_TASK-}" ]; then - OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK - else - OMP_NUM_THREADS=1 - fi - fi - export OMP_NUM_THREADS - - ${ - if variant == "env" then ''exec "$@"'' - else - '' - if [ -n "''${SLURM_JOB_ID-}" ]; then - exec mpirun --bind-to none ${vasp}/bin/vasp-${variant} - else - exec mpirun -np 1 ${vasp}/bin/vasp-${variant} - fi - '' - } + mkdir -p $out/bin + for i in std gam ncl; do cp bin/vasp_$i $out/bin/vasp-$i; done + mkdir $out/src + ln -s ${src} $out/src/vasp + ln -s ${vtst} $out/src/vtst ''; - runEnv = variant: buildFHSEnv - { - name = "vasp-nvidia${if variant == "" then "" else "-${variant}"}"; - targetPkgs = _: [ zlib vasp ]; - runScript = startScript (if variant == "" then "std" else variant); - }; -in symlinkJoin - { - name = "vasp-nvidia"; - paths = builtins.map (variant: runEnv variant) [ "" "env" "std" "gam" "ncl" ]; - } + + # enable parallel build + enableParallelBuilding = true; + DEPS = "1"; + + # vasp directly include headers under ${mkl}/include/fftw + MKLROOT = mkl; + QD = qd; + + # tell openmpi use ifx + # OMPI_F90 = "ifx"; +}; +in writeShellScriptBin "vasp-nvidia" +'' + export PATH=${vasp}/bin:${mpi}/bin''${PATH:+:$PATH} + exec "$@" +'' diff --git a/packages/vasp/nvidia/makefile.include b/packages/vasp/nvidia/makefile.include index c78e254a..d3a8c64e 100644 --- a/packages/vasp/nvidia/makefile.include +++ b/packages/vasp/nvidia/makefile.include @@ -17,8 +17,8 @@ CPP = nvfortran -Mpreprocess -Mfree -Mextend -E $(CPP_OPTIONS) $*$(FUFFI # N.B.: you might need to change the cuda-version here # to one that comes with your NVIDIA-HPC SDK -FC = mpif90 -acc -gpu=@cudaCapabilities@ -mp -FCL = mpif90 -acc -gpu=@cudaCapabilities@ -mp -c++libs +FC = mpif90 -acc -mp +FCL = mpif90 -acc -mp -c++libs FREE = -Mfree @@ -56,8 +56,6 @@ CXX_PARS = nvc++ --no_warnings ## # When compiling on the target machine itself , change this to the # relevant target when cross-compiling for another architecture -VASP_TARGET_CPU ?= -tp=@nvhpcArch@ -FFLAGS += $(VASP_TARGET_CPU) # Specify your NV HPC-SDK installation (mandatory) #... first try to set it automatically