From bad48d324a5bd44707508927073efea2ca2de744 Mon Sep 17 00:00:00 2001 From: chn Date: Mon, 6 Jan 2025 16:50:41 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9A=82=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- flake/src.nix | 5 ++- packages/default.nix | 6 +-- packages/nvhpcStdenv.nix | 75 +++++++++++++++++--------------- packages/vasp/nvidia/default.nix | 30 ++++++------- 4 files changed, 61 insertions(+), 55 deletions(-) diff --git a/flake/src.nix b/flake/src.nix index ce9a53bc..35ea93e9 100644 --- a/flake/src.nix +++ b/flake/src.nix @@ -5,10 +5,11 @@ { src = pkgs.fetchurl { - url = "https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64/nvhpc-24-11_24.11-0_amd64.deb"; - sha256 = "0xzfgdz7s8kzxmcm3k6n9nqd0isfzj23nxfq0y3ca9f046gp4zp3"; + url = "https://developer.download.nvidia.com/hpc-sdk/24.11/nvhpc_2024_2411_Linux_x86_64_cuda_12.6.tar.gz"; + sha256 = "080rb89p2z98b75wqssvp3s8x6b5n0556d0zskh3cfapcb08lh1r"; }; version = "24.11"; + cudaVersion = "12.6"; }; iso = pkgs.fetchurl { diff --git a/packages/default.nix b/packages/default.nix index 3e5de0c6..4cef359c 100644 --- a/packages/default.nix +++ b/packages/default.nix @@ -40,7 +40,7 @@ inputs: rec }; nvidia = inputs.pkgs.callPackage ./vasp/nvidia { - inherit (nvhpcPackages) stdenv hdf5 mpi qd; + inherit (nvhpcPackages) stdenv hdf5; inherit vtst src; wannier90 = inputs.pkgs.wannier90.overrideAttrs { buildFlags = [ "dynlib" ]; }; }; @@ -97,15 +97,13 @@ inputs: rec spectroscopy = inputs.pkgs.callPackage ./spectroscopy.nix { src = inputs.topInputs.spectroscopy; }; mirism = inputs.pkgs.callPackage ./mirism { inherit biu; stdenv = inputs.pkgs.clang18Stdenv; }; vaspberry = inputs.pkgs.callPackage ./vaspberry.nix { src = inputs.topInputs.vaspberry; }; - nvhpcStdenv = inputs.pkgs.callPackage ./nvhpcStdenv.nix { src = inputs.topInputs.self.src.nvhpc; gcc = gccFull; }; + nvhpcStdenv = inputs.pkgs.callPackage ./nvhpcStdenv.nix { src = inputs.topInputs.self.src.nvhpc; }; nvhpcPackages = inputs.pkgs.lib.makeScope inputs.pkgs.newScope (final: { stdenv = nvhpcStdenv; fmt = (inputs.pkgs.fmt.override { inherit (final) stdenv; }).overrideAttrs { doCheck = false; }; hdf5 = inputs.pkgs.hdf5.override { inherit (final) stdenv; cppSupport = false; fortranSupport = true; enableShared = false; enableStatic = true; }; - qd = final.callPackage ./qd.nix { src = inputs.topInputs.qd; }; - mpi = inputs.pkgs.openmpi.override { inherit (final) stdenv; enableSubstitute = false; }; }); gccFull = inputs.pkgs.symlinkJoin { diff --git a/packages/nvhpcStdenv.nix b/packages/nvhpcStdenv.nix index d0e4d315..5baa3f48 100644 --- a/packages/nvhpcStdenv.nix +++ b/packages/nvhpcStdenv.nix @@ -1,74 +1,81 @@ { src, stdenv, autoPatchelfHook, wrapCCWith, writeText, addAttrsToDerivation, config, overrideCC, symlinkJoin, - gcc, glibc_multi, libz, zstd, libxml2, flock, numactl, ncurses, dpkg, cudaPackages, openssl, gmp, openssl_1_1, - libxcrypt-legacy, libfabric, rdma-core + gcc, glibc, libz, zstd, libxml2, flock, numactl, ncurses, dpkg, cudaPackages, openssl, gmp, openssl_1_1, + libxcrypt-legacy, libfabric, rdma-core, gfortran, xorg, makeSetupHook, writeScript, bash, overrideInStdenv }: let nvhpc = stdenv.mkDerivation { pname = "nvhpc"; inherit (src) src version; - buildInputs = [ libz libxml2 zstd numactl ncurses openssl gmp openssl_1_1 libxcrypt-legacy libfabric rdma-core ]; + buildInputs = + [ + libz libxml2 zstd numactl ncurses openssl gmp openssl_1_1 libxcrypt-legacy libfabric rdma-core xorg.libpciaccess + ]; nativeBuildInputs = [ autoPatchelfHook dpkg flock ]; langFortran = true; dontConfigure = true; dontBuild = true; - unpackPhase = ''dpkg-deb -x $src .''; installPhase = '' - # install component - # NVHPC use very complex mechanism to identify the location of compilers, headers, etc. - # we should keep the original structure mkdir -p $out - cp -r opt $out - rm -rf $out/opt/nvidia/hpc_sdk/Linux_x86_64/${src.version}/{examples,profilers,comm_libs/${src.cudaVersion}/hpcx/hpcx-*/ompi/tests,cuda/12.6/bin/cuda-gdb-python3.10-tui} - mkdir -p $out/nix-support - > $out/nix-support/setup-hook << EOF - addNvhpcEnv() { - addToSearchPath PATH $out/opt/nvidia/hpc_sdk/Linux_x86_64/${src.version}/compilers/bin - addToSearchPath PATH $out/opt/nvidia/hpc_sdk/Linux_x86_64/${src.version}/comm_libs/mpi/bin - } - addEnvHooks "$hostOffset" addHarepath - EOF - ''; - autoPatchelfIgnoreMissingDeps = [ "libgdrapi.so.2" ]; - postFixup = - '' - sed -i '/makelocalrc executed by/d' $out/opt/nvidia/hpc_sdk/Linux_x86_64/${src.version}/compilers/bin/makelocalrc - $out/opt/nvidia/hpc_sdk/Linux_x86_64/${src.version}/compilers/bin/makelocalrc \ - $out/opt/nvidia/hpc_sdk/Linux_x86_64/${src.version}/compilers/bin -x + sed -i 's|/bin/chmod|chmod|g' install_components/install + sed -i 's|/sbin/ldconfig|ldconfig|g' install_components/install + patchShebangs install_components/Linux_x86_64/${src.version}/compilers/bin/makelocalrc + sed -i '/makelocalrc executed by/d' install_components/Linux_x86_64/${src.version}/compilers/bin/makelocalrc + + NVHPC_SILENT=true NVHPC_INSTALL_DIR=$out NVHPC_INSTALL_TYPE=single ./install_components/install + + addAutoPatchelfSearchPath $out/Linux_x86_64/${src.version}/cuda/${src.cudaVersion}/targets/x86_64-linux/lib/stubs + addAutoPatchelfSearchPath $out/Linux_x86_64/${src.version}/compilers/lib + + rm -rf $out/Linux_x86_64/${src.version}/cuda/${src.cudaVersion}/bin/cuda-gdb-python*-tui + rm -rf $out/Linux_x86_64/${src.version}/profilers + rm -rf $out/Linux_x86_64/${src.version}/comm_libs/${src.cudaVersion}/hpcx/hpcx-*/ompi/tests ''; + autoPatchelfIgnoreMissingDeps = [ "libgdrapi.so.2" "libxpmem.so.0" "libnvidia-ml.so.1" ]; }; # fix /usr/lib/crt1.o impure path used in link customLocalrc = writeText "localrc" '' - set DEFLIBDIR=${glibc_multi}/lib; - set DEFSTDOBJDIR=${glibc_multi}/lib; + set DEFLIBDIR=${glibc}/lib; + set DEFSTDOBJDIR=${glibc}/lib; ''; # do not set -gpu=cuda12.4 since this only switch the cuda version installed with NVHPC cudaCapability = builtins.concatStringsSep "," ( (builtins.map (cap: "cc${builtins.replaceStrings ["."] [""] cap}") config.cudaCapabilities) - ++ [ "cuda12.6" ] + ++ [ "cuda${src.cudaVersion}" ] ); + env = makeSetupHook { name = "nvhpc-env"; } (writeScript "nvhpc-env" + '' + addNvhpcEnv() { + addToSearchPath PATH ${nvhpc}/Linux_x86_64/${src.version}/compilers/bin + addToSearchPath PATH ${nvhpc}/Linux_x86_64/${src.version}/comm_libs/mpi/bin + addToSearchPath PATH ${gcc.cc}/bin + export NVLOCALRC=${customLocalrc} + } + addEnvHooks "$hostOffset" addNvhpcEnv + ''); wrapper = (wrapCCWith { cc = nvhpc; extraBuildCommands = '' - echo "-L${gcc}/lib" >> $out/nix-support/cc-ldflags + # make -lgomp find libgomp.so provided by nvhpc instead of gcc + echo "-L${nvhpc}/Linux_x86_64/${src.version}/compilers/lib" >> $out/nix-support/cc-ldflags + + # provide libgcc_s.so but not libgomp.so + echo "-L${gcc.cc.libgcc}/lib" >> $out/nix-support/cc-ldflags echo "-tp=${config.nvhpcArch}" >> $out/nix-support/cc-cflags-before echo "-gpu=${cudaCapability}" >> $out/nix-support/cc-cflags-before echo "-noswitcherror" >> $out/nix-support/cc-cflags - echo 'export "PATH=${gcc}/bin:$PATH"' >> $out/nix-support/cc-wrapper-hook - # print verbose output for debugging - echo "-v" >> $out/nix-support/cc-cflags - # echo "-#" >> $out/nix-support/cc-cflags + # echo "-v" >> $out/nix-support/cc-cflags # echo "" > $out/nix-support/add-hardening.sh @@ -77,7 +84,7 @@ let sed -i 's/-idirafter/-I/g' $out/nix-support/libc-cflags for i in nvc nvc++ nvcc nvfortran; do - wrap $i $wrapper $ccPath/$i + wrap $i $wrapper ${nvhpc}/Linux_x86_64/${nvhpc.version}/compilers/bin/$i done ''; }).overrideAttrs (prev: { installPhase = prev.installPhase + @@ -86,4 +93,4 @@ let export named_cxx=nvc++ export named_fc=nvfortran '';}); -in addAttrsToDerivation { NVLOCALRC = customLocalrc; } (overrideCC stdenv wrapper) +in overrideInStdenv (overrideCC stdenv wrapper) [ env ] diff --git a/packages/vasp/nvidia/default.nix b/packages/vasp/nvidia/default.nix index 3cafacbb..c638ea6f 100644 --- a/packages/vasp/nvidia/default.nix +++ b/packages/vasp/nvidia/default.nix @@ -1,6 +1,6 @@ { stdenv, src, writeShellScriptBin, - rsync, which, wannier90, hdf5, vtst, mpi, mkl, qd + rsync, which, wannier90, hdf5, vtst, mkl }: let vasp = stdenv.mkDerivation { @@ -16,7 +16,7 @@ let vasp = stdenv.mkDerivation chmod -R +w src ''; buildInputs = [ hdf5 wannier90 mkl ]; - nativeBuildInputs = [ rsync which mpi ]; + nativeBuildInputs = [ rsync which ]; installPhase = '' mkdir -p $out/bin @@ -28,17 +28,17 @@ let vasp = stdenv.mkDerivation # enable parallel build enableParallelBuilding = true; - DEPS = "1"; - - # vasp directly include headers under ${mkl}/include/fftw - MKLROOT = mkl; - QD = qd; - - # tell openmpi use ifx - # OMPI_F90 = "ifx"; + env = + { + DEPS = "1"; + # vasp directly include headers under ${mkl}/include/fftw + MKLROOT = mkl; + QD = "${stdenv.cc.cc}/Linux_x86_64/${stdenv.cc.cc.version}/compilers/extras/qd"; + }; }; -in writeShellScriptBin "vasp-nvidia" -'' - export PATH=${vasp}/bin:${mpi}/bin''${PATH:+:$PATH} - exec "$@" -'' +in vasp +# in writeShellScriptBin "vasp-nvidia" +# '' +# export PATH=${vasp}/bin:''${PATH:+:$PATH} +# exec "$@" +# ''