devices.srv1: fix network

devices/srv1/node1/default.nix

@@ -7,12 +7,10 @@ inputs:
       system =
       {
         nixpkgs.march = "broadwell";
-        networking.networkd.static =
-        {
-          eno1 = { ip = "192.168.1.11"; mask = 24; gateway = "192.168.1.1"; };
-          eno2 = { ip = "192.168.178.2"; mask = 24; gateway = "192.168.178.1"; dns = "192.168.178.1"; };
-        };
+        networking.networkd.static.eno2 =
+          { ip = "192.168.178.2"; mask = 24; gateway = "192.168.178.1"; dns = "192.168.178.1"; };
         cluster.nodeType = "worker";
+        fileSystems.mount.nfs."192.168.178.1:/home" = "/home";
       };
       services.beesd.instances.root = { device = "/"; hashTableSizeMB = 256; threads = 4; };
       packages.packages._prebuildPackages =
@@ -20,31 +18,10 @@ inputs:
     };
     specialisation.no-share-home.configuration =
     {
-      nixos =
-      {
-        services.slurm.enable = inputs.lib.mkForce false;
-        system.cluster.nodeType = inputs.lib.mkForce "master";
-      };
+      nixos.system.fileSystems.mount.nfs = inputs.lib.mkForce null;
       system.nixos.tags = [ "no-share-home" ];
     };
-    fileSystems = inputs.lib.mkIf (inputs.config.nixos.system.cluster.nodeType == "worker")
-    {
-      "/home" =
-      {
-        device = "192.168.178.1:/home";
-        fsType = "nfs";
-        neededForBoot = true;
-      };
-    };
-    boot.initrd.network.enable = true; 
     boot.initrd.systemd.network.networks."10-eno2" = inputs.config.systemd.network.networks."10-eno2";
-    boot.initrd.systemd.extraBin =
-    {
-      "ifconfig" = "${inputs.pkgs.nettools}/bin/ifconfig";
-      "mount.nfs" = "${inputs.pkgs.nfs-utils}/bin/mount.nfs";
-      "mount.nfs4" = "${inputs.pkgs.nfs-utils}/bin/mount.nfs4";
-    };
-    services.rpcbind.enable = true;
     # make slurm sub process to be able to communicate with the master
     networking.firewall.trustedInterfaces = [ "eno2" ];
   };

devices/srv1/node2/default.nix

@@ -7,43 +7,28 @@ inputs:
       system =
       {
         nixpkgs.march = "broadwell";
-        networking.networkd.static.eno2 =
-          { ip = "192.168.178.3"; mask = 24; gateway = "192.168.178.1"; dns = "192.168.178.1"; };
+        networking.networkd.static =
+        {
+          eno1 = { ip = "192.168.1.12"; mask = 24; gateway = "192.168.1.1"; dns = "192.168.1.1"; };
+          eno2 = { ip = "192.168.178.3"; mask = 24; };
+        };
         cluster.nodeType = "worker";
-        initrd.sshd.enable = true;
-        nix.remote.slave.enable = true;
+        fileSystems.mount.nfs."192.168.178.1:/home" = "/home";
+      };
+      services =
+      {
+        xray.client.enable = true;
+        beesd.instances.root = { device = "/"; hashTableSizeMB = 256; threads = 4; };
       };
-      services.beesd.instances.root = { device = "/"; hashTableSizeMB = 256; threads = 4; };
       packages.packages._prebuildPackages =
         [ inputs.topInputs.self.nixosConfigurations.srv1-node0.config.system.build.toplevel ];
     };
     specialisation.no-share-home.configuration =
     {
-      nixos =
-      {
-        services.slurm.enable = inputs.lib.mkForce false;
-        system.cluster.nodeType = inputs.lib.mkForce "master";
-      };
+      nixos.system.fileSystems.mount.nfs = inputs.lib.mkForce null;
       system.nixos.tags = [ "no-share-home" ];
     };
-    fileSystems = inputs.lib.mkIf (inputs.config.nixos.system.cluster.nodeType == "worker")
-    {
-      "/home" =
-      {
-        device = "192.168.178.1:/home";
-        fsType = "nfs";
-        neededForBoot = true;
-      };
-    };
-    boot.initrd.network.enable = true; 
     boot.initrd.systemd.network.networks."10-eno2" = inputs.config.systemd.network.networks."10-eno2";
-    boot.initrd.systemd.extraBin =
-    {
-      "ifconfig" = "${inputs.pkgs.nettools}/bin/ifconfig";
-      "mount.nfs" = "${inputs.pkgs.nfs-utils}/bin/mount.nfs";
-      "mount.nfs4" = "${inputs.pkgs.nfs-utils}/bin/mount.nfs4";
-    };
-    services.rpcbind.enable = true;
     # make slurm sub process to be able to communicate with the master
     networking.firewall.trustedInterfaces = [ "eno2" ];
   };

devices/srv1/node2/secrets/default.yaml

@@ -1,4 +1,5 @@
-hello: ENC[AES256_GCM,data:/WGwXDnQio1BwD/zPoURTjVzTasWICOA7CBsgT5DbYIkKLt5DxzogeYWpiqjVg==,iv:BY82U/y9V8PYtn3Bre+nabGBcVgFbppIQZb7GhgY62I=,tag:JFqrezoWNJ8ZACCKQ43n5g==,type:str]
+xray-client:
+    uuid: ENC[AES256_GCM,data:U+unsiKt9vNo/EXEpLHR0Ny3DxQEwx7a40KmwZDZki7RQEuM,iv:7w90HNM5lfh2VY20AcUEVdu5X2uxqXxR0hARncmMR60=,tag:xIbKc+9SF5LP/tY/XoGYxA==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -23,8 +24,8 @@ sops:
             MVU1UW9lWFJnSTE2aC9ZL0huYURUK3MK5U4cLWRMm+FFo8ATE/OoAcHzYHFMpOtV
             Q5kbq5PDMdp4qvoM3T4kLsB34oU55HjFvac0pilOhNRrz4xRMQgvoQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-09-20T05:30:52Z"
-    mac: ENC[AES256_GCM,data:nSrkKUo4yB57aetzdJ1sjSKcm5STQ6jfMhvY4/tXft2P9zRYigSP4PkZj7z+knxcIx9sFdA86h8X45oUjxaAa5xDJpgmvC/EEKxm5rZtVTxYYYdy40W72qThVuKUasWpYrrGZbZEbTu3Dad1yfJTilwofRtxoo1Nmj5lMvw+HRo=,iv:UvBSF5GLEj+hTZksrIV3Ow+HQ/xjqwCUuwqkdz8g0Qg=,tag:U5wJPhmeevB2i2GBgMGBFQ==,type:str]
+    lastmodified: "2024-09-26T04:24:23Z"
+    mac: ENC[AES256_GCM,data:GS9TPomEy3rHaanqMWkqGV9NX8lsYMnujxhuWTnJxdgfCP8scOEo2O+ATlmxmcmKquXlq+zo4LF6RCzF65eSvpSKcIDQRc0yDgBQ5+M8gXv1lk0WNdicpHJsEk8j/ostJTwEkV5QDyp3+J6lRNtFMQnrZ7+UxpgQwK7DaP8mnF4=,iv:1BEb2Xr8jQO6M19bC+jlGGSI0aT1MEgEoYwLuCT1T9U=,tag:Be7BQsjWq7PZBrgsrH/cjA==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.9.0

devices/srv1/node3/default.nix

@@ -10,8 +10,7 @@ inputs:
         networking.networkd.static.eno2 =
           { ip = "192.168.178.4"; mask = 24; gateway = "192.168.178.1"; dns = "192.168.178.1"; };
         cluster.nodeType = "worker";
-        initrd.sshd.enable = true;
-        nix.remote.slave.enable = true;
+        fileSystems.mount.nfs."192.168.178.1:/home" = "/home";
       };
       services.beesd.instances.root = { device = "/"; hashTableSizeMB = 256; threads = 4; };
       packages.packages._prebuildPackages =
@@ -19,31 +18,10 @@ inputs:
     };
     specialisation.no-share-home.configuration =
     {
-      nixos =
-      {
-        services.slurm.enable = inputs.lib.mkForce false;
-        system.cluster.nodeType = inputs.lib.mkForce "master";
-      };
+      nixos.system.fileSystems.mount.nfs = inputs.lib.mkForce null;
       system.nixos.tags = [ "no-share-home" ];
     };
-    fileSystems = inputs.lib.mkIf (inputs.config.nixos.system.cluster.nodeType == "worker")
-    {
-      "/home" =
-      {
-        device = "192.168.178.1:/home";
-        fsType = "nfs";
-        neededForBoot = true;
-      };
-    };
-    boot.initrd.network.enable = true; 
     boot.initrd.systemd.network.networks."10-eno2" = inputs.config.systemd.network.networks."10-eno2";
-    boot.initrd.systemd.extraBin =
-    {
-      "ifconfig" = "${inputs.pkgs.nettools}/bin/ifconfig";
-      "mount.nfs" = "${inputs.pkgs.nfs-utils}/bin/mount.nfs";
-      "mount.nfs4" = "${inputs.pkgs.nfs-utils}/bin/mount.nfs4";
-    };
-    services.rpcbind.enable = true;
     # make slurm sub process to be able to communicate with the master
     networking.firewall.trustedInterfaces = [ "eno2" ];
   };

devices/vps6/default.nix

@@ -36,7 +36,7 @@ inputs:
       {
         snapper.enable = true;
         sshd = {};
-        xray.server = { serverName = "vps6.xserver.chn.moe"; userNumber = 21; };
+        xray.server = { serverName = "vps6.xserver.chn.moe"; userNumber = 22; };
         frpServer = { enable = true; serverName = "frp.chn.moe"; };
         nginx =
         {

devices/vps6/secrets.yaml

@@ -46,6 +46,8 @@ xray-server:
         user19: ENC[AES256_GCM,data:+Mh15DR9xvFAwks86iuHEA9FpObKWTSuVOEzUDpBUS/h0hOz,iv:zYIkic2bibvwCBpomnJ9465mda1rbm3RERBZY9twXuc=,tag:bwdL6DAGgkGYhYFI2C4A+A==,type:str]
         #ENC[AES256_GCM,data:1g2gohLbiixMes8=,iv:E3HA6cAdv3BdLMcrrcWW4Zsc2KLtW7L8Xrk9Z57l49o=,tag:rZ7W9ckf7lzJ23u5zwQiwg==,type:comment]
         user20: ENC[AES256_GCM,data:3UbVnn9oMRc0zZR46tWxwM9VFOvMOYm690csUomEVBcS3xPm,iv:KHuPXttLAFr7WT/qa/UYLY8GRsPWYZPyKNmdUh4iFQQ=,tag:jN8rQ0Gv+qnhwOWGH+CwlA==,type:str]
+        #ENC[AES256_GCM,data:GzxXsTbEvdHV7A0=,iv:uxUG4hnYEsmJtnqbEwamwhtLt3UClt7ktmkGyAFdxsc=,tag:sF8YQ2cejAezI3Bbp9qKIw==,type:comment]
+        user21: ENC[AES256_GCM,data:hgDJ11crZaWcKrc+ZDQklXwpnvt/sMbARkx3sLZfQGZqQZeA,iv:2Re+hdJuT5yg/qTymfpN+KdU3criOmwuqqg+SHb8iAo=,tag:s16N6u5cRDaoWxnrCkamuw==,type:str]
     private-key: ENC[AES256_GCM,data:ts/LRGFAsYqvGvkvlxUI42IW1a8cGsSkpZhMDd3QVceRKvhPb1SRDaXoSw==,iv:6xX9xFIFUNlLBZ6CPBOz9JbHpvC4+QG9ZaCZcWdl12c=,tag:DYIa+QTV8vyl1l7OKKykTw==,type:str]
 nginx:
     #ENC[AES256_GCM,data:85LrqdTMIhSa,iv:mIQPYz8VPd5AxeMCQEdTGMD0Iqa5QEAa5+8JVFaj3JM=,tag:TcZd7S3WRPpEV9lHI1fzbw==,type:comment]
@@ -89,8 +91,8 @@ sops:
             ZXFTU3ZCaW1pTVh0RUJzdDdGdHlPYTgK2mlgcX2kEc8+2UDdBnhUm6IIuh8V6agW
             ooxH9OEPXUVI/4JcDo4v8ZUhAyU1ehLH0Ef7PJCChOZe2KZmWSNbhA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-09-15T03:48:21Z"
-    mac: ENC[AES256_GCM,data:kZDIr2NHVew+BsreAoYNAcP/3i2A7U1RGIiA6qok1EsXLcunO+vfjIZl3L/0CEEH4+u6PEXQ51atzufqOGpoq9XqorBMRDEdlhitZZIUZm8Cji9BJxixeTUQ+KmFEbdw8H1XDIPWOQJCmTUbkOElMzHO4BNtTpdjE4u1IZ0bUiY=,iv:cCjln5wrScDz7A5/OHVoAj671VtkBmK7H0pnpKLsjD8=,tag:rohzoGp2V1sS03W0z5hM/A==,type:str]
+    lastmodified: "2024-09-26T04:24:17Z"
+    mac: ENC[AES256_GCM,data:AXhLmyZWGD6KvMkyHqmCERE6eNE3pD5Pa/9mRBWZe4hiXL4mKTzCn5C/ODGQ1ZeQjDdP+awjJRvLRjMiYFhVlU8rKpg/f2G1gDr4cIbr61sCdzXKX8wFW0G7bJWxxpAC4X59+u9EJ3sNcyf7bJrMdkTzTYpgXh29mtl2bprcdJQ=,iv:pK4hYexcWng3GwOmWGqgyMsmATnXgcwR3NH4UxCwpvE=,tag:zpv64JWoXc5cDCukDuW51g==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.9.0

modules/system/fileSystems/default.nix

@@ -32,124 +32,112 @@ inputs:
       default = null;
     };
   };
-  config =
-    let
-      inherit (builtins) listToAttrs map concatLists concatStringsSep;
-      inherit (inputs.lib) mkMerge mkIf;
-      inherit (inputs.localLib) attrsToList;
-      inherit (inputs.config.nixos.system) fileSystems;
-    in mkMerge
-    [
-      # mount.vfat
-      {
-        fileSystems = listToAttrs (map
-          (device:
-          {
-            name = device.value;
-            value = { device = device.name; fsType = "vfat"; neededForBoot = true; options = [ "noatime" ]; };
-          })
-          (attrsToList fileSystems.mount.vfat));
-      }
-      # mount.btrfs
-      # Disable CoW for VM image and database: sudo chattr +C images
-      # resize btrfs:
-      # sudo btrfs filesystem resize -50G /nix
-      # sudo cryptsetup status root
-      # sudo cryptsetup -b 3787456512 resize root
-      # sudo cfdisk /dev/nvme1n1p3
-      {
-        fileSystems = listToAttrs (concatLists (map
-          (
-            device: map
-              (
-                subvol:
-                {
-                  name = subvol.value;
-                  value =
-                  {
-                    device = device.name;
-                    fsType = "btrfs";
-                    # zstd:15 cause sound stuttering
-                    # test on e20dae7d8b317f95718b5f4175bd4246c09735de mathematica ~15G
-                    # zstd:15 5m33s 7.16G
-                    # zstd:8 54s 7.32G
-                    # zstd:3 17s 7.52G
-                    options = [ "compress-force=zstd" "subvol=${subvol.name}" "acl" "noatime" ];
-                    neededForBoot = true;
-                  };
-                }
-              )
-              (attrsToList device.value)
-          )
-          (attrsToList fileSystems.mount.btrfs)));
-      }
-      # mdadm
-      (
-        mkIf (fileSystems.mdadm != null)
-          { boot.initrd.services.swraid = { enable = true; mdadmConf = fileSystems.mdadm; }; }
-      )
-      # swap
-      { swapDevices = map (device: { device = device; }) fileSystems.swap; }
-      # resume
-      (
-        mkIf (fileSystems.resume != null) { boot =
-        (
-          if builtins.typeOf fileSystems.resume == "string" then
-            { resumeDevice = fileSystems.resume; }
-          else
-          {
-            resumeDevice = fileSystems.resume.device;
-            kernelModules = [ "resume_offset=${builtins.toString fileSystems.resume.offset}" ];
-          }
-        );}
-      )
-      # rollingRootfs
-      (
-        mkIf (fileSystems.rollingRootfs != null)
+  config = let inherit (inputs.config.nixos.system) fileSystems; in inputs.lib.mkMerge
+  [
+    # mount.vfat
+    {
+      fileSystems = builtins.listToAttrs (builtins.map
+        (device:
         {
-          boot.initrd.systemd =
-          {
-            extraBin =
+          name = device.value;
+          value = { device = device.name; fsType = "vfat"; neededForBoot = true; options = [ "noatime" ]; };
+        })
+        (inputs.localLib.attrsToList fileSystems.mount.vfat));
+    }
+    # mount.btrfs
+    # Disable CoW for VM image and database: sudo chattr +C images
+    # resize btrfs:
+    # sudo btrfs filesystem resize -50G /nix
+    # sudo cryptsetup status root
+    # sudo cryptsetup -b 3787456512 resize root
+    # sudo cfdisk /dev/nvme1n1p3
+    {
+      fileSystems =  builtins.listToAttrs (builtins.concatLists (builtins.map
+        (device: builtins.map
+          (
+            subvol:
             {
-              grep = "${inputs.pkgs.gnugrep}/bin/grep";
-              awk = "${inputs.pkgs.gawk}/bin/awk";
-              chattr = "${inputs.pkgs.e2fsprogs}/bin/chattr";
-              lsmod = "${inputs.pkgs.kmod}/bin/lsmod";
-            };
-            services.roll-rootfs =
-            {
-              wantedBy = [ "initrd.target" ];
-              after = [ "cryptsetup.target" "systemd-hibernate-resume.service" ];
-              before = [ "local-fs-pre.target" "sysroot.mount" ];
-              unitConfig.DefaultDependencies = false;
-              serviceConfig.Type = "oneshot";
-              script =
-                let
-                  inherit (fileSystems.rollingRootfs) device path waitDevices;
-                  waitDevice = concatStringsSep "\n" (builtins.map
-                    (device: "while ! [ -e ${device} ]; do sleep 1; done") (waitDevices ++ [ device ]));
-                in
-                ''
-                  while ! lsmod | grep -q btrfs; do sleep 1; done
-                  ${waitDevice}
-                  mount ${device} /mnt -m
-                  if [ -f /mnt${path}/current/.timestamp ]
-                  then
-                    timestamp=$(cat /mnt${path}/current/.timestamp)
-                    subvolid=$(btrfs subvolume show /mnt${path}/current | grep 'Subvolume ID:' | awk '{print $NF}')
-                    mv /mnt${path}/current /mnt${path}/$timestamp-$subvolid
-                    btrfs property set -ts /mnt${path}/$timestamp-$subvolid ro true
-                  fi
-                  btrfs subvolume create /mnt${path}/current
-                  chattr +C /mnt${path}/current
-                  echo $(date '+%Y%m%d%H%M%S') > /mnt${path}/current/.timestamp
-                  umount /mnt
-                '';
-            };
-          };
-        }
-      )
-    ];
+              name = subvol.value;
+              value =
+              {
+                device = device.name;
+                fsType = "btrfs";
+                # zstd:15 cause sound stuttering
+                # test on e20dae7d8b317f95718b5f4175bd4246c09735de mathematica ~15G
+                # zstd:15 5m33s 7.16G
+                # zstd:8 54s 7.32G
+                # zstd:3 17s 7.52G
+                options = [ "compress-force=zstd" "subvol=${subvol.name}" "acl" "noatime" ];
+                neededForBoot = true;
+              };
+            }
+          )
+          (inputs.localLib.attrsToList device.value)
+        )
+        (inputs.localLib.attrsToList fileSystems.mount.btrfs)));
+    }
+    # mdadm
+    (inputs.lib.mkIf (fileSystems.mdadm != null)
+      { boot.initrd.services.swraid = { enable = true; mdadmConf = fileSystems.mdadm; }; }
+    )
+    # swap
+    { swapDevices = builtins.map (device: { device = device; }) fileSystems.swap; }
+    # resume
+    (inputs.lib.mkIf (fileSystems.resume != null) { boot =
+    (
+      if builtins.typeOf fileSystems.resume == "string" then
+        { resumeDevice = fileSystems.resume; }
+      else
+      {
+        resumeDevice = fileSystems.resume.device;
+        kernelModules = [ "resume_offset=${builtins.toString fileSystems.resume.offset}" ];
+      }
+    );})
+    # rollingRootfs
+    (inputs.lib.mkIf (fileSystems.rollingRootfs != null)
+    {
+      boot.initrd.systemd =
+      {
+        extraBin =
+        {
+          grep = "${inputs.pkgs.gnugrep}/bin/grep";
+          awk = "${inputs.pkgs.gawk}/bin/awk";
+          chattr = "${inputs.pkgs.e2fsprogs}/bin/chattr";
+          lsmod = "${inputs.pkgs.kmod}/bin/lsmod";
+        };
+        services.roll-rootfs =
+        {
+          wantedBy = [ "initrd.target" ];
+          after = [ "cryptsetup.target" "systemd-hibernate-resume.service" ];
+          before = [ "local-fs-pre.target" "sysroot.mount" ];
+          unitConfig.DefaultDependencies = false;
+          serviceConfig.Type = "oneshot";
+          script =
+            let
+              inherit (fileSystems.rollingRootfs) device path waitDevices;
+              waitDevice = builtins.concatStringsSep "\n" (builtins.map
+                (device: "while ! [ -e ${device} ]; do sleep 1; done") (waitDevices ++ [ device ]));
+            in
+            ''
+              while ! lsmod | grep -q btrfs; do sleep 1; done
+              ${waitDevice}
+              mount ${device} /mnt -m
+              if [ -f /mnt${path}/current/.timestamp ]
+              then
+                timestamp=$(cat /mnt${path}/current/.timestamp)
+                subvolid=$(btrfs subvolume show /mnt${path}/current | grep 'Subvolume ID:' | awk '{print $NF}')
+                mv /mnt${path}/current /mnt${path}/$timestamp-$subvolid
+                btrfs property set -ts /mnt${path}/$timestamp-$subvolid ro true
+              fi
+              btrfs subvolume create /mnt${path}/current
+              chattr +C /mnt${path}/current
+              echo $(date '+%Y%m%d%H%M%S') > /mnt${path}/current/.timestamp
+              umount /mnt
+            '';
+        };
+      };
+    })
+  ];
 }
 
 

modules/system/fileSystems/nfs.nix

@@ -0,0 +1,29 @@
+inputs:
+{
+  imports = inputs.localLib.findModules ./.;
+  options.nixos.system.fileSystems.mount.nfs = let inherit (inputs.lib) mkOption types; in mkOption
+  {
+    type = types.nullOr (types.attrsOf types.nonEmptyStr); default = null;
+  };
+  config = let inherit (inputs.config.nixos.system.fileSystems.mount) nfs; in inputs.lib.mkIf (nfs != null)
+  {
+    fileSystems = builtins.listToAttrs (builtins.map
+      (device:
+      {
+        name = device.value;
+        value = { device = device.name; fsType = "nfs"; neededForBoot = true; };
+      })
+      (inputs.localLib.attrsToList nfs));
+    boot.initrd =
+    {
+      network.enable = true;
+      systemd.extraBin =
+      {
+        "ifconfig" = "${inputs.pkgs.nettools}/bin/ifconfig";
+        "mount.nfs" = "${inputs.pkgs.nfs-utils}/bin/mount.nfs";
+        "mount.nfs4" = "${inputs.pkgs.nfs-utils}/bin/mount.nfs4";
+      };
+    };
+    services.rpcbind.enable = true;
+  };
+}