packages.vasp.intel: run without fhs

devices/srv1/default.nix

@@ -63,6 +63,7 @@ inputs:
             old = [ "srv1-node1" "srv1-node2" "srv1-node3" ];
           };
           tui = { cpuMpiThreads = 8; cpuOpenmpThreads = 10; };
+          setupFirewall = true;
         };
       };
       user.users = [ "chn" ];

flake.lock

@@ -60,17 +60,17 @@
         ]
       },
       "locked": {
-        "lastModified": 1713974364,
-        "narHash": "sha256-ilZTVWSaNP1ibhQIIRXE+q9Lj2XOH+F9W3Co4QyY1eU=",
+        "lastModified": 1726983918,
+        "narHash": "sha256-eRGl50mhbJRT9Mk2NbePr6yzxzZRygndx4O7kObVa3g=",
         "ref": "refs/heads/master",
-        "rev": "de89197a4a7b162db7df9d41c9d07759d87c5709",
-        "revCount": 937,
+        "rev": "7370c1d1ef2421608e1355368361af92777a2c6c",
+        "revCount": 940,
         "type": "git",
-        "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git"
+        "url": "https://git.chn.moe/chn/bscpkgs.git"
       },
       "original": {
         "type": "git",
-        "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git"
+        "url": "https://git.chn.moe/chn/bscpkgs.git"
       }
     },
     "catppuccin": {

flake.nix

@@ -37,7 +37,7 @@
     };
     gricad = { url = "github:Gricad/nur-packages"; flake = false; };
     catppuccin.url = "github:catppuccin/nix";
-    bscpkgs = { url = "git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git"; inputs.nixpkgs.follows = "nixpkgs"; };
+    bscpkgs = { url = "git+https://git.chn.moe/chn/bscpkgs.git"; inputs.nixpkgs.follows = "nixpkgs"; };
     poetry2nix = { url = "github:CHN-beta/poetry2nix"; inputs.nixpkgs.follows = "nixpkgs"; };
 
     misskey = { url = "git+https://github.com/CHN-beta/misskey?submodules=1"; flake = false; };

modules/services/slurm.nix

@@ -29,6 +29,8 @@ inputs:
       cpuOpenmpThreads = mkOption { type = types.ints.unsigned; default = 1; };
       gpus = mkOption { type = types.nullOr (types.attrsOf types.ints.unsigned); default = null; };
     };
+    # 是否打开防火墙相应端口，对于多节点部署需要打开
+    setupFirewall = mkOption { type = types.bool; default = false; };
   };
   config = let inherit (inputs.config.nixos.services) slurm; in inputs.lib.mkIf slurm.enable (inputs.lib.mkMerge
   [
@@ -107,6 +109,7 @@ inputs:
             SchedulerParameters=enable_user_top
 
             SlurmdDebug=debug2
+            DebugFlags=NO_CONF_HASH
 
             # automatically resume node after drain
             ReturnToService=2
@@ -139,6 +142,9 @@ inputs:
         sopsFile = "${builtins.dirOf inputs.config.sops.defaultSopsFile}/munge.key";
         owner = inputs.config.systemd.services.munged.serviceConfig.User;
       };
+      networking.firewall =
+        let config = inputs.lib.mkIf slurm.setupFirewall [ 6818 ];
+        in { allowedTCPPorts = config; allowedUDPPorts = config; };
     }
     # master 配置
     (inputs.lib.mkIf (slurm.master == inputs.config.nixos.system.networking.hostname)
@@ -182,6 +188,9 @@ inputs:
         CpuOpenmpThreads = slurm.tui.cpuOpenmpThreads;
         GpuIds = slurm.tui.gpus;
       };
+      networking.firewall =
+        let config = inputs.lib.mkIf slurm.setupFirewall [ 6817 ];
+        in { allowedTCPPorts = config; allowedUDPPorts = config; };
     })
   ]);
 }

modules/system/nixpkgs.nix

@@ -46,7 +46,9 @@ inputs:
           }
           // (if nixpkgs.march == null then {} else
           {
-            oneapiArch = let match = { znver3 = "CORE-AVX2"; znver4 = "CORE-AVX512"; };
+            # TODO: change znver4 after update oneapi
+            # TODO: test znver3 do use AVX
+            oneapiArch = let match = { znver3 = "CORE-AVX2"; znver4 = "core-avx2"; };
               in match.${nixpkgs.march} or nixpkgs.march;
             nvhpcArch = nixpkgs.march;
             # contentAddressedByDefault = true;

packages/default.nix

@@ -52,9 +52,24 @@ inputs: rec
     nvidia = inputs.pkgs.callPackage ./vasp/nvidia
       { inherit lmod nvhpc vtst src; hdf5 = hdf5-nvhpc; };
     intel = inputs.pkgs.callPackage ./vasp/intel
-      { inherit lmod oneapi vtst src; hdf5 = hdf5-oneapi; };
+    {
+      inherit vtst src;
+      inherit (inputs.pkgs.intelPackages_2023) stdenv;
+      mpi = inputs.pkgs.intelPackages_2023.intel-mpi;
+      hdf5 = hdf5-oneapi;
+    };
     hdf5-nvhpc = inputs.pkgs.callPackage ./vasp/hdf5-nvhpc { inherit lmod nvhpc; inherit (inputs.pkgs.hdf5) src; };
-    hdf5-oneapi = inputs.pkgs.callPackage ./vasp/hdf5-oneapi { inherit lmod oneapi; inherit (inputs.pkgs.hdf5) src; };
+    hdf5-oneapi = inputs.pkgs.hdf5.override
+    {
+      inherit (inputs.pkgs.intelPackages_2023) stdenv;
+      cppSupport = false;
+      fortranSupport = true;
+      # mpiSupport = true;
+      enableShared = false;
+      enableStatic = true;
+      # fortran = builtins.toFile "empty" "";
+      # mpi = builtins.toFile "empty" "";
+    };
     vtst = (inputs.pkgs.callPackage ./vasp/vtst.nix {});
     vtstscripts = inputs.pkgs.callPackage ./vasp/vtstscripts.nix {};
   };

packages/vasp/hdf5-oneapi/default.nix

@@ -1,48 +0,0 @@
-{
-  buildFHSEnv, writeScript, stdenvNoCC, symlinkJoin,
-  src,
-  oneapi, lmod, cmake, gcc, glibc, binutils,
-  config, oneapiArch ? config.oneapiArch or "SSE3"
-}:
-let
-  gccFull = symlinkJoin { name = "gcc"; paths = [ gcc gcc.cc gcc.cc.lib glibc.dev binutils.bintools ]; };
-  buildEnv = buildFHSEnv
-  {
-    name = "buildEnv";
-    targetPkgs = pkgs: with pkgs; [ zlib (writeTextDir "etc/release" "") gccFull ];
-    extraBwrapArgs = [ "--bind" "$out" "$out" ];
-  };
-  buildScript = writeScript "build"
-  ''
-    . ${lmod}/share/lmod/lmod/init/bash
-    module use ${oneapi}/share/intel/modulefiles
-    module load tbb compiler-rt oclfpga # dependencies
-    module load mpi mkl compiler
-    mkdir build
-    cd build
-    cmake -DCMAKE_INSTALL_PREFIX=$out -DHDF5_INSTALL_CMAKE_DIR=$out/lib/cmake \
-      -DHDF5_BUILD_FORTRAN=ON -DHDF5_ENABLE_PARALLEL=ON -DBUILD_SHARED_LIBS=OFF -DBUILD_STATIC_LIBS=OFF \
-      -DBUILD_TESTING=OFF ..
-    make -j$NIX_BUILD_CORES
-    make install
-  '';
-in stdenvNoCC.mkDerivation
-{
-  name = "hdf5-oneapi";
-  inherit src;
-  dontConfigure = true;
-  enableParallelBuilding = true;
-  nativeBuildInputs = [ cmake ];
-  I_MPI_CC = "icx";
-  I_MPI_CXX = "icpx";
-  I_MPI_FC = "ifx";
-  I_MPI_F90 = "ifx";
-  buildPhase =
-  ''
-    mkdir -p $out
-    ${buildEnv}/bin/buildEnv ${buildScript}
-  '';
-  dontInstall = true;
-  dontFixup = true;
-  requiredSystemFeatures = [ "gccarch-exact-${stdenvNoCC.hostPlatform.gcc.arch}" "big-parallel" ];
-}

packages/vasp/intel/default.nix

@@ -1,107 +1,35 @@
 {
-  buildFHSEnv, writeScript, stdenvNoCC, substituteAll, symlinkJoin, writeTextDir, src,
-  config, oneapiArch ? config.oneapiArch or "SSE3",
-  oneapi, gcc, glibc, lmod, rsync, which, wannier90, binutils, hdf5, zlib, vtst
-}:
-let
-  buildEnv = buildFHSEnv
-  {
-    name = "buildEnv";
-    # make "module load mpi" success
-    targetPkgs = _: [ zlib (writeTextDir "etc/release" "") gccFull ];
-  };
-  buildScript = writeScript "build"
-  ''
-    . ${lmod}/share/lmod/lmod/init/bash
-    module use ${oneapi}/share/intel/modulefiles
-    module load tbb compiler-rt oclfpga # dependencies
-    module load mpi mkl compiler
-    make DEPS=1 -j$NIX_BUILD_CORES
-  '';
-  include = substituteAll { src = ./makefile.include; inherit oneapiArch; };
-  gccFull = symlinkJoin { name = "gcc"; paths = [ gcc gcc.cc gcc.cc.lib glibc.dev binutils.bintools ]; };
-  vasp = stdenvNoCC.mkDerivation
-  {
-    name = "vasp-intel";
-    inherit src;
-    # patches = [ ../vtst.patch ];
-    configurePhase =
-    ''
-      cp ${include} makefile.include
-      chmod +w makefile.include
-      cp ${../constr_cell_relax.F} src/constr_cell_relax.F
-      # cp -r ${vtst}/* src
-      chmod -R +w src
-    '';
-    nativeBuildInputs = [ rsync which ];
-    HDF5_ROOT = hdf5;
-    WANNIER90_ROOT = wannier90;
-    buildPhase = "${buildEnv}/bin/buildEnv ${buildScript}";
-    installPhase =
-    ''
-      mkdir -p $out/bin
-      for i in std gam ncl; do cp bin/vasp_$i $out/bin/vasp-$i; done
-      mkdir $out/src
-      ln -s ${src} $out/src/vasp
-      ln -s ${vtst} $out/src/vtst
-    '';
-    dontFixup = true;
-    requiredSystemFeatures = [ "gccarch-exact-${stdenvNoCC.hostPlatform.gcc.arch}" "big-parallel" ];
-  };
-  startScript = variant: writeScript "vasp-intel"
-  ''
-    . ${lmod}/share/lmod/lmod/init/bash
-    module use ${oneapi}/share/intel/modulefiles
-    module load tbb compiler-rt oclfpga # dependencies
-    module load mpi mkl compiler
-
-    # if OMP_NUM_THREADS is not set, set it according to SLURM_CPUS_PER_TASK or to 1
-    if [ -z "''${OMP_NUM_THREADS-}" ]; then
-      if [ -n "''${SLURM_CPUS_PER_TASK-}" ]; then
-        OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
-      else
-        OMP_NUM_THREADS=1
-      fi
-    fi
-    export OMP_NUM_THREADS
-
-    # if I_MPI_PIN_PROCESSOR_LIST is not set, set it to allcores
-    if [ -z "''${I_MPI_PIN_PROCESSOR_LIST-}" ]; then
-      I_MPI_PIN_PROCESSOR_LIST=allcores
-    fi
-    export I_MPI_PIN_PROCESSOR_LIST
-
-    # set I_MPI_PIN I_MPI_PIN_DOMAIN I_MPI_DEBUG if not set
-    export I_MPI_PIN=''${I_MPI_PIN-yes}
-    export I_MPI_PIN_DOMAIN=''${I_MPI_PIN_DOMAIN-omp}
-    export I_MPI_DEBUG=''${I_MPI_DEBUG-4}
-
-    # fork to bootstrap, do not use srun, causing it could not find proper ld
-    export I_MPI_HYDRA_BOOTSTRAP=''${I_MPI_HYDRA_BOOTSTRAP-fork}
-
-    # set OMP_STACKSIZE if not set
-    export OMP_STACKSIZE=''${OMP_STACKSIZE-512M}
-
-    ${
-      if variant == "env" then ''exec "$@"''
-      else
-      ''
-        if [ -n "''${SLURM_JOB_ID-}" ]; then
-          exec mpirun -n $SLURM_NTASKS ${vasp}/bin/vasp-${variant}
-        else
-          exec mpirun -n 1 ${vasp}/bin/vasp-${variant}
-        fi
-      ''
-    }
-  '';
-  runEnv = variant: buildFHSEnv
-  {
-    name = "vasp-intel${if variant == "" then "" else "-${variant}"}";
-    targetPkgs = _: [ zlib vasp (writeTextDir "etc/release" "") gccFull ];
-    runScript = startScript (if variant == "" then "std" else variant);
-  };
-in symlinkJoin
+  stdenv, src, rsync, which, wannier90, hdf5, vtst, mpi, mkl
+}: stdenv.mkDerivation
 {
   name = "vasp-intel";
-  paths = builtins.map (variant: runEnv variant) [ "" "env" "std" "gam" "ncl" ];
+  inherit src;
+  # patches = [ ../vtst.patch ];
+  configurePhase =
+  ''
+    cp ${./makefile.include} makefile.include
+    chmod +w makefile.include
+    cp ${../constr_cell_relax.F} src/constr_cell_relax.F
+    # cp -r ${vtst}/* src
+    chmod -R +w src
+  '';
+  buildInputs = [ hdf5 wannier90 mkl ];
+  nativeBuildInputs = [ rsync which mpi ];
+  installPhase =
+  ''
+    mkdir -p $out/bin
+    for i in std gam ncl; do cp bin/vasp_$i $out/bin/vasp-$i; done
+    mkdir $out/src
+    ln -s ${src} $out/src/vasp
+    ln -s ${vtst} $out/src/vtst
+  '';
+
+  # NIX_DEBUG = "7";
+
+  # enable parallel build
+  enableParallelBuilding = true;
+  DEPS = "1";
+
+  # vasp directly include headers under ${mkl}/include/fftw
+  MKLROOT = mkl;
 }

packages/vasp/intel/makefile.include

@@ -41,35 +41,12 @@ OBJECTS_LIB = linpack_double.o getshmem.o
 CXX_PARS    = icpx
 LLIBS       = -lstdc++
 
-##
-## Customize as of this point! Of course you may change the preceding
-## part of this file as well if you like, but it should rarely be
-## necessary ...
-##
-
-# When compiling on the target machine itself, change this to the
-# relevant target when cross-compiling for another architecture
-VASP_TARGET_CPU ?= -x@oneapiArch@
-FFLAGS     += $(VASP_TARGET_CPU)
- 
-# Intel MKL (FFTW, BLAS, LAPACK, and scaLAPACK)
-# (Note: for Intel Parallel Studio's MKL use -mkl instead of -qmkl)
 FCL        += -qmkl
-MKLROOT    ?= /path/to/your/mkl/installation 
-LLIBS      += -L$(MKLROOT)/lib/intel64 -lmkl_scalapack_lp64 -lmkl_blacs_intelmpi_lp64
+LLIBS      += -lmkl_scalapack_lp64 -lmkl_blacs_intelmpi_lp64
 INCS        =-I$(MKLROOT)/include/fftw
 
-# HDF5-support (optional but strongly recommended)
 CPP_OPTIONS+= -DVASP_HDF5
-LLIBS      += -L$(HDF5_ROOT)/lib -lhdf5_fortran
-INCS       += -I$(HDF5_ROOT)/include
+LLIBS      += -lhdf5_fortran
 
-# For the VASP-2-Wannier90 interface (optional)
 CPP_OPTIONS    += -DVASP2WANNIER90
-LLIBS          += -L$(WANNIER90_ROOT)/lib -lwannier
-
-# For the fftlib library (hardly any benefit in combination with MKL's FFTs)
-#FCL         = mpiifort fftlib.o -qmkl
-#CXX_FFTLIB  = icpc -qopenmp -std=c++11 -DFFTLIB_USE_MKL -DFFTLIB_THREADSAFE
-#INCS_FFTLIB = -I./include -I$(MKLROOT)/include/fftw
-#LIBS       += fftlib
+LLIBS          += -lwannier