From 8faa50a4272283d9bfd773a2a26ad1a1136fe9ee Mon Sep 17 00:00:00 2001 From: chn Date: Thu, 26 Sep 2024 11:23:07 +0800 Subject: [PATCH 01/11] modules.services.nfs: init --- devices/srv1/node0/default.nix | 31 +++---------------------------- devices/srv1/node1/default.nix | 15 +++++---------- devices/srv1/node2/default.nix | 13 +++++-------- devices/srv1/node3/default.nix | 13 +++++-------- modules/services/nfs.nix | 29 +++++++++++++++++++++++++++++ 5 files changed, 47 insertions(+), 54 deletions(-) create mode 100644 modules/services/nfs.nix diff --git a/devices/srv1/node0/default.nix b/devices/srv1/node0/default.nix index 79e269a0..73cb98af 100644 --- a/devices/srv1/node0/default.nix +++ b/devices/srv1/node0/default.nix @@ -6,7 +6,6 @@ inputs: { system = { - nix = { marches = [ "cascadelake" "broadwell" ]; remote.slave.enable = true; }; nixpkgs.march = "cascadelake"; networking.networkd.static = { @@ -17,11 +16,7 @@ inputs: }; services = { - xray.client = - { - enable = true; - dnsmasq.extraInterfaces = [ "eno146" ]; - }; + xray.client = { enable = true; dnsmasq.extraInterfaces = [ "eno146" ]; }; beesd.instances.root = { device = "/"; hashTableSizeMB = 512; threads = 4; }; wireguard = { @@ -30,33 +25,13 @@ inputs: publicKey = "Br+ou+t9M9kMrnNnhTvaZi2oNFRygzebA1NqcHWADWM="; wireguardIp = "192.168.83.9"; }; + nfs = { root = "/"; exports = "/home"; accessLimit = "192.168.178.0/24"; }; }; packages.packages._prebuildPackages = [ inputs.topInputs.self.nixosConfigurations.srv1-node1.pkgs.localPackages.vasp.intel ]; }; - services.nfs.server = - { - enable = true; - exports = - '' - / 192.168.178.0/24(rw,no_root_squash,fsid=0,sync,crossmnt) - /home 192.168.178.0/24(rw,no_root_squash,sync,crossmnt) - ''; - }; - networking = - { - firewall.allowedTCPPorts = [ 2049 ]; - }; + # allow other machine access network by this machine systemd.network.networks."10-eno146".networkConfig.IPMasquerade = "both"; - services.rpcbind.enable = true; - fileSystems = - { - "/nix/share/home" = - { - device = "/home"; - options = [ "rbind" ]; - }; - }; # without this, tproxy does not work # TODO: why? networking.firewall.trustedInterfaces = [ "eno146" ]; diff --git a/devices/srv1/node1/default.nix b/devices/srv1/node1/default.nix index 1d574c90..ca876c90 100644 --- a/devices/srv1/node1/default.nix +++ b/devices/srv1/node1/default.nix @@ -13,24 +13,19 @@ inputs: eno2 = { ip = "192.168.178.2"; mask = 24; gateway = "192.168.178.1"; dns = "192.168.178.1"; }; }; cluster.nodeType = "worker"; - initrd.sshd.enable = true; - nix.remote.slave.enable = true; }; services.beesd.instances.root = { device = "/"; hashTableSizeMB = 256; threads = 4; }; packages.packages._prebuildPackages = [ inputs.topInputs.self.nixosConfigurations.srv1-node0.config.system.build.toplevel ]; }; - specialisation = + specialisation.no-share-home.configuration = { - no-share-home.configuration = + nixos = { - nixos = - { - services.slurm.enable = inputs.lib.mkForce false; - system.cluster.nodeType = inputs.lib.mkForce "master"; - }; - system.nixos.tags = [ "no-share-home" ]; + services.slurm.enable = inputs.lib.mkForce false; + system.cluster.nodeType = inputs.lib.mkForce "master"; }; + system.nixos.tags = [ "no-share-home" ]; }; fileSystems = inputs.lib.mkIf (inputs.config.nixos.system.cluster.nodeType == "worker") { diff --git a/devices/srv1/node2/default.nix b/devices/srv1/node2/default.nix index 025e431c..4ceaa442 100644 --- a/devices/srv1/node2/default.nix +++ b/devices/srv1/node2/default.nix @@ -17,17 +17,14 @@ inputs: packages.packages._prebuildPackages = [ inputs.topInputs.self.nixosConfigurations.srv1-node0.config.system.build.toplevel ]; }; - specialisation = + specialisation.no-share-home.configuration = { - no-share-home.configuration = + nixos = { - nixos = - { - services.slurm.enable = inputs.lib.mkForce false; - system.cluster.nodeType = inputs.lib.mkForce "master"; - }; - system.nixos.tags = [ "no-share-home" ]; + services.slurm.enable = inputs.lib.mkForce false; + system.cluster.nodeType = inputs.lib.mkForce "master"; }; + system.nixos.tags = [ "no-share-home" ]; }; fileSystems = inputs.lib.mkIf (inputs.config.nixos.system.cluster.nodeType == "worker") { diff --git a/devices/srv1/node3/default.nix b/devices/srv1/node3/default.nix index 63b9253c..59a75784 100644 --- a/devices/srv1/node3/default.nix +++ b/devices/srv1/node3/default.nix @@ -17,17 +17,14 @@ inputs: packages.packages._prebuildPackages = [ inputs.topInputs.self.nixosConfigurations.srv1-node0.config.system.build.toplevel ]; }; - specialisation = + specialisation.no-share-home.configuration = { - no-share-home.configuration = + nixos = { - nixos = - { - services.slurm.enable = inputs.lib.mkForce false; - system.cluster.nodeType = inputs.lib.mkForce "master"; - }; - system.nixos.tags = [ "no-share-home" ]; + services.slurm.enable = inputs.lib.mkForce false; + system.cluster.nodeType = inputs.lib.mkForce "master"; }; + system.nixos.tags = [ "no-share-home" ]; }; fileSystems = inputs.lib.mkIf (inputs.config.nixos.system.cluster.nodeType == "worker") { diff --git a/modules/services/nfs.nix b/modules/services/nfs.nix new file mode 100644 index 00000000..53b09ea7 --- /dev/null +++ b/modules/services/nfs.nix @@ -0,0 +1,29 @@ +inputs: +{ + options.nixos.services.nfs = let inherit (inputs.lib) mkOption types; in mkOption + { + type = types.nullOr (types.submodule { options = + { + root = mkOption { type = types.nonEmptyStr; }; + exports = mkOption { type = types.listOf types.nonEmptyStr; }; + accessLimit = mkOption { type = types.nonEmptyStr; }; + };}); + default = null; + }; + config = let inherit (inputs.config.nixos.services) nfs; in inputs.lib.mkIf (nfs != null) + { + services = + { + rpcbind.enable = true; + nfs.server = + { + enable = true; + exports = "${nfs.root} ${nfs.accessLimit}(rw,no_root_squash,fsid=0,sync,crossmnt)" + + builtins.concatStringsSep "\n" (builtins.map + (export: "${export} ${nfs.accessLimit}(rw,no_root_squash,sync,crossmnt)") + nfs.exports); + }; + }; + networking.firewall.allowedTCPPorts = [ 2049 ]; + }; +} From bab49afeb33c389dfd9c48b48e6950a3bf339efa Mon Sep 17 00:00:00 2001 From: chn Date: Thu, 26 Sep 2024 11:46:06 +0800 Subject: [PATCH 02/11] modules.system.fileSystems.luks: split --- devices/nas/default.nix | 2 +- devices/pc/default.nix | 4 +- devices/surface/default.nix | 2 +- devices/vps4/default.nix | 2 +- devices/vps6/default.nix | 2 +- devices/vps7/default.nix | 2 +- modules/system/fileSystems/default.nix | 117 +----------------- modules/system/fileSystems/luks/default.nix | 80 ++++++++++++ modules/system/fileSystems/{ => luks}/nas.key | Bin .../system/fileSystems/{ => luks}/vps4.key | Bin .../system/fileSystems/{ => luks}/vps6.key | Bin .../system/fileSystems/{ => luks}/vps7.key | Bin modules/user/chn/default.nix | 5 +- 13 files changed, 91 insertions(+), 125 deletions(-) create mode 100644 modules/system/fileSystems/luks/default.nix rename modules/system/fileSystems/{ => luks}/nas.key (100%) rename modules/system/fileSystems/{ => luks}/vps4.key (100%) rename modules/system/fileSystems/{ => luks}/vps6.key (100%) rename modules/system/fileSystems/{ => luks}/vps7.key (100%) diff --git a/devices/nas/default.nix b/devices/nas/default.nix index 791a5747..56bf9f46 100644 --- a/devices/nas/default.nix +++ b/devices/nas/default.nix @@ -24,7 +24,7 @@ inputs: }; }; }; - decrypt.manual = + luks.manual = { enable = true; devices = diff --git a/devices/pc/default.nix b/devices/pc/default.nix index 6768ed6e..848b5996 100644 --- a/devices/pc/default.nix +++ b/devices/pc/default.nix @@ -13,7 +13,7 @@ inputs: vfat."/dev/disk/by-uuid/7A60-4232" = "/boot"; btrfs."/dev/mapper/root1" = { "/nix" = "/nix"; "/nix/rootfs/current" = "/"; }; }; - decrypt.auto = + luks.auto = { "/dev/disk/by-uuid/4c73288c-bcd8-4a7e-b683-693f9eed2d81" = { mapper = "root1"; ssd = true; }; "/dev/disk/by-uuid/4be45329-a054-4c20-8965-8c5b7ee6b35d" = @@ -41,7 +41,7 @@ inputs: # SERIALIZE SGX WAITPKG WIDEKL XSAVE XSAVEOPT "alderlake" ]; - remote.master = { enable = true; hosts = [ "xmupc1" "xmupc2" "srv1-node0" "srv1-node1" ]; }; + remote.master = { enable = true; hosts = [ "xmupc1" "xmupc2" ]; }; githubToken.enable = true; }; nixpkgs = diff --git a/devices/surface/default.nix b/devices/surface/default.nix index 4817e1c6..534da119 100644 --- a/devices/surface/default.nix +++ b/devices/surface/default.nix @@ -14,7 +14,7 @@ inputs: vfat."/dev/disk/by-uuid/4596-D670" = "/boot"; btrfs."/dev/mapper/root1" = { "/nix" = "/nix"; "/nix/rootfs/current" = "/"; }; }; - decrypt.auto = + luks.auto = { "/dev/disk/by-uuid/eda0042b-ffd5-47d1-b828-4cf99d744c9f" = { mapper = "root1"; ssd = true; }; "/dev/disk/by-uuid/41d83848-f3dd-4b2f-946f-de1d2ae1cbd4" = { mapper = "swap"; ssd = true; }; diff --git a/devices/vps4/default.nix b/devices/vps4/default.nix index 18658a25..b528b0ef 100644 --- a/devices/vps4/default.nix +++ b/devices/vps4/default.nix @@ -16,7 +16,7 @@ inputs: "/dev/mapper/root" = { "/nix" = "/nix"; "/nix/rootfs/current" = "/"; }; }; }; - decrypt.manual = + luks.manual = { enable = true; devices."/dev/disk/by-uuid/bf7646f9-496c-484e-ada0-30335da57068" = { mapper = "root"; ssd = true; }; diff --git a/devices/vps6/default.nix b/devices/vps6/default.nix index 2529d42b..d382494f 100644 --- a/devices/vps6/default.nix +++ b/devices/vps6/default.nix @@ -16,7 +16,7 @@ inputs: "/dev/mapper/root" = { "/nix" = "/nix"; "/nix/rootfs/current" = "/"; }; }; }; - decrypt.manual = + luks.manual = { enable = true; devices."/dev/disk/by-uuid/4f8aca22-9ec6-4fad-b21a-fd9d8d0514e8" = { mapper = "root"; ssd = true; }; diff --git a/devices/vps7/default.nix b/devices/vps7/default.nix index a3da33e1..1597bf53 100644 --- a/devices/vps7/default.nix +++ b/devices/vps7/default.nix @@ -16,7 +16,7 @@ inputs: "/dev/mapper/root" = { "/nix" = "/nix"; "/nix/rootfs/current" = "/"; }; }; }; - decrypt.manual = + luks.manual = { enable = true; devices."/dev/disk/by-uuid/db48c8de-bcf7-43ae-a977-60c4f390d5c4" = { mapper = "root"; ssd = true; }; diff --git a/modules/system/fileSystems/default.nix b/modules/system/fileSystems/default.nix index 4622cf0b..fc803dde 100644 --- a/modules/system/fileSystems/default.nix +++ b/modules/system/fileSystems/default.nix @@ -1,5 +1,6 @@ inputs: { + imports = inputs.localLib.findModules ./.; options.nixos.system.fileSystems = let inherit (inputs.lib) mkOption types; in { mount = @@ -9,41 +10,6 @@ inputs: # device.subvol = mountPoint; btrfs = mkOption { type = types.attrsOf (types.attrsOf types.nonEmptyStr); default = {}; }; }; - decrypt = - { - auto = mkOption - { - type = types.attrsOf (types.submodule - { - options = - { - mapper = mkOption { type = types.nonEmptyStr; }; - ssd = mkOption { type = types.bool; default = false; }; - before = mkOption { type = types.nullOr (types.listOf types.nonEmptyStr); default = null; }; - }; - }); - default = {}; - }; - manual = - { - enable = mkOption { type = types.bool; default = false; }; - devices = mkOption - { - type = types.attrsOf (types.submodule - { - options = - { - mapper = mkOption { type = types.nonEmptyStr; }; - ssd = mkOption { type = types.bool; default = false; }; - }; - }); - default = {}; - }; - keyFile = mkOption - { type = types.path; default = ./. + "/${inputs.config.nixos.system.networking.hostname}.key"; }; - delayedMount = mkOption { type = types.listOf types.nonEmptyStr; default = []; }; - }; - }; # generate using: sudo mdadm --examine --scan mdadm = mkOption { type = types.nullOr types.lines; default = null; }; swap = mkOption { type = types.listOf types.nonEmptyStr; default = []; }; @@ -117,87 +83,6 @@ inputs: ) (attrsToList fileSystems.mount.btrfs))); } - # decrypt.auto - ( - mkIf (fileSystems.decrypt.auto != null) - { - boot.initrd = - { - luks.devices = (listToAttrs (map - ( - device: - { - name = device.value.mapper; - value = - { - device = device.name; - allowDiscards = device.value.ssd; - bypassWorkqueues = device.value.ssd; - crypttabExtraOpts = [ "fido2-device=auto" "x-initrd.attach" ]; - }; - } - ) - (attrsToList fileSystems.decrypt.auto))); - systemd.services = - let - createService = device: - { - name = "systemd-cryptsetup@${device.value.mapper}"; - value = - { - before = map (device: "systemd-cryptsetup@${device}.service") device.value.before; - overrideStrategy = "asDropin"; - }; - }; - in - listToAttrs (map createService - (builtins.filter (device: device.value.before != null) (attrsToList fileSystems.decrypt.auto))); - }; - } - ) - # decrypt.manual - ( - mkIf (fileSystems.decrypt.manual.enable) - { - boot.initrd = - { - luks.forceLuksSupportInInitrd = true; - systemd = - { - extraBin = - { - cryptsetup = "${inputs.pkgs.cryptsetup.bin}/bin/cryptsetup"; - usbip = "${inputs.config.boot.kernelPackages.usbip}/bin/usbip"; - sed = "${inputs.pkgs.gnused}/bin/sed"; - awk = "${inputs.pkgs.gawk}/bin/awk"; - decrypt = inputs.pkgs.writeShellScript "decrypt" - '' - modprobe vhci-hcd - busid=$(usbip list -r 127.0.0.1 | head -n4 | tail -n1 | awk '{print $1}' | sed 's/://') - usbip attach -r 127.0.0.1 -b $busid - ${concatStringsSep "\n" (map - (device: ''systemd-cryptsetup attach ${device.value.mapper} ${device.name} "" fido2-device=auto'' - + (if device.value.ssd then ",discard" else "")) - (attrsToList fileSystems.decrypt.manual.devices))} - ''; - }; - services.wait-manual-decrypt = - { - wantedBy = [ "initrd-root-fs.target" ]; - before = [ "roll-rootfs.service" ]; - unitConfig.DefaultDependencies = false; - serviceConfig.Type = "oneshot"; - script = concatStringsSep "\n" (map - (device: "while [ ! -e /dev/mapper/${device.value.mapper} ]; do sleep 1; done") - (attrsToList fileSystems.decrypt.manual.devices)); - }; - }; - }; - fileSystems = listToAttrs (map - (mount: { name = mount; value.options = [ "x-systemd.device-timeout=48h" ]; }) - fileSystems.decrypt.manual.delayedMount); - } - ) # mdadm ( mkIf (fileSystems.mdadm != null) diff --git a/modules/system/fileSystems/luks/default.nix b/modules/system/fileSystems/luks/default.nix new file mode 100644 index 00000000..8826ef84 --- /dev/null +++ b/modules/system/fileSystems/luks/default.nix @@ -0,0 +1,80 @@ +inputs: +{ + options.nixos.system.fileSystems.luks = let inherit (inputs.lib) mkOption types; in + { + auto = mkOption + { + type = types.attrsOf (types.submodule { options = + { + mapper = mkOption { type = types.nonEmptyStr; }; + ssd = mkOption { type = types.bool; default = false; }; + before = mkOption { type = types.nullOr (types.listOf types.nonEmptyStr); default = null; }; + };}); + default = {}; + }; + manual = + { + enable = mkOption { type = types.bool; default = false; }; + devices = mkOption + { + type = types.attrsOf (types.submodule { options = + { + mapper = mkOption { type = types.nonEmptyStr; }; + ssd = mkOption { type = types.bool; default = false; }; + };}); + default = {}; + }; + delayedMount = mkOption { type = types.listOf types.nonEmptyStr; default = []; }; + }; + }; + config = let inherit (inputs.config.nixos.system.fileSystems) luks; in inputs.lib.mkMerge + [ + (inputs.lib.mkIf (luks.auto != null) { boot.initrd = + { + luks.devices = (builtins.listToAttrs (builtins.map + (device: + { + name = device.value.mapper; + value = + { + device = device.name; + allowDiscards = device.value.ssd; + bypassWorkqueues = device.value.ssd; + crypttabExtraOpts = [ "fido2-device=auto" "x-initrd.attach" ]; + }; + }) + (inputs.localLib.attrsToList luks.auto))); + systemd.services = builtins.listToAttrs (builtins.map + (device: + { + name = "systemd-cryptsetup@${device.value.mapper}"; + value = + { + before = map (device: "systemd-cryptsetup@${device}.service") device.value.before; + overrideStrategy = "asDropin"; + }; + }) + (builtins.filter (device: device.value.before != null) (inputs.localLib.attrsToList luks.auto))); + };}) + (inputs.lib.mkIf luks.manual.enable + { + boot.initrd = + { + luks.forceLuksSupportInInitrd = true; + systemd.services.wait-manual-decrypt = + { + wantedBy = [ "initrd-root-fs.target" ]; + before = [ "roll-rootfs.service" ]; + unitConfig.DefaultDependencies = false; + serviceConfig.Type = "oneshot"; + script = builtins.concatStringsSep "\n" (builtins.map + (device: "while [ ! -e /dev/mapper/${device.value.mapper} ]; do sleep 1; done") + (inputs.localLib.attrsToList luks.manual.devices)); + }; + }; + fileSystems = builtins.listToAttrs (builtins.map + (mount: { name = mount; value.options = [ "x-systemd.device-timeout=48h" ]; }) + luks.manual.delayedMount); + }) + ]; +} diff --git a/modules/system/fileSystems/nas.key b/modules/system/fileSystems/luks/nas.key similarity index 100% rename from modules/system/fileSystems/nas.key rename to modules/system/fileSystems/luks/nas.key diff --git a/modules/system/fileSystems/vps4.key b/modules/system/fileSystems/luks/vps4.key similarity index 100% rename from modules/system/fileSystems/vps4.key rename to modules/system/fileSystems/luks/vps4.key diff --git a/modules/system/fileSystems/vps6.key b/modules/system/fileSystems/luks/vps6.key similarity index 100% rename from modules/system/fileSystems/vps6.key rename to modules/system/fileSystems/luks/vps6.key diff --git a/modules/system/fileSystems/vps7.key b/modules/system/fileSystems/luks/vps7.key similarity index 100% rename from modules/system/fileSystems/vps7.key rename to modules/system/fileSystems/luks/vps7.key diff --git a/modules/user/chn/default.nix b/modules/user/chn/default.nix index f17fb0ec..3321e08c 100644 --- a/modules/user/chn/default.nix +++ b/modules/user/chn/default.nix @@ -56,7 +56,7 @@ inputs: (system: { name = system.config.nixos.system.networking.hostname; - value = system.config.nixos.system.fileSystems.decrypt.manual; + value = system.config.nixos.system.fileSystems.luks.manual; }) (builtins.attrValues inputs.topInputs.self.nixosConfigurations)); cat = "${inputs.pkgs.coreutils}/bin/cat"; @@ -68,7 +68,8 @@ inputs: (builtins.map (system: builtins.concatStringsSep "\n" [ "decrypt-${system.name}() {" - " key=$(${cat} ${system.value.keyFile} | ${gpg} --decrypt)" + " key=$(${cat} ${inputs.topInputs.self}/modules/system/fileSystems/luks/${system.name}.key \\" + " | ${gpg} --decrypt)" (builtins.concatStringsSep "\n" (builtins.map (device: " echo $key | ${ssh} root@initrd.${system.name}.chn.moe cryptsetup luksOpen " + (if device.value.ssd then "--allow-discards " else "") From 515b6eedc16e280039758fefcf51d49bef2cf93e Mon Sep 17 00:00:00 2001 From: chn Date: Thu, 26 Sep 2024 12:00:26 +0800 Subject: [PATCH 03/11] =?UTF-8?q?modules.system.fileSystems:=20=E6=95=B4?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/system/fileSystems/default.nix | 220 ++++++++++++------------- 1 file changed, 104 insertions(+), 116 deletions(-) diff --git a/modules/system/fileSystems/default.nix b/modules/system/fileSystems/default.nix index fc803dde..b79bd9ce 100644 --- a/modules/system/fileSystems/default.nix +++ b/modules/system/fileSystems/default.nix @@ -32,124 +32,112 @@ inputs: default = null; }; }; - config = - let - inherit (builtins) listToAttrs map concatLists concatStringsSep; - inherit (inputs.lib) mkMerge mkIf; - inherit (inputs.localLib) attrsToList; - inherit (inputs.config.nixos.system) fileSystems; - in mkMerge - [ - # mount.vfat - { - fileSystems = listToAttrs (map - (device: - { - name = device.value; - value = { device = device.name; fsType = "vfat"; neededForBoot = true; options = [ "noatime" ]; }; - }) - (attrsToList fileSystems.mount.vfat)); - } - # mount.btrfs - # Disable CoW for VM image and database: sudo chattr +C images - # resize btrfs: - # sudo btrfs filesystem resize -50G /nix - # sudo cryptsetup status root - # sudo cryptsetup -b 3787456512 resize root - # sudo cfdisk /dev/nvme1n1p3 - { - fileSystems = listToAttrs (concatLists (map - ( - device: map - ( - subvol: - { - name = subvol.value; - value = - { - device = device.name; - fsType = "btrfs"; - # zstd:15 cause sound stuttering - # test on e20dae7d8b317f95718b5f4175bd4246c09735de mathematica ~15G - # zstd:15 5m33s 7.16G - # zstd:8 54s 7.32G - # zstd:3 17s 7.52G - options = [ "compress-force=zstd" "subvol=${subvol.name}" "acl" "noatime" ]; - neededForBoot = true; - }; - } - ) - (attrsToList device.value) - ) - (attrsToList fileSystems.mount.btrfs))); - } - # mdadm - ( - mkIf (fileSystems.mdadm != null) - { boot.initrd.services.swraid = { enable = true; mdadmConf = fileSystems.mdadm; }; } - ) - # swap - { swapDevices = map (device: { device = device; }) fileSystems.swap; } - # resume - ( - mkIf (fileSystems.resume != null) { boot = - ( - if builtins.typeOf fileSystems.resume == "string" then - { resumeDevice = fileSystems.resume; } - else - { - resumeDevice = fileSystems.resume.device; - kernelModules = [ "resume_offset=${builtins.toString fileSystems.resume.offset}" ]; - } - );} - ) - # rollingRootfs - ( - mkIf (fileSystems.rollingRootfs != null) + config = let inherit (inputs.config.nixos.system) fileSystems; in inputs.lib.mkMerge + [ + # mount.vfat + { + fileSystems = builtins.listToAttrs (builtins.map + (device: { - boot.initrd.systemd = - { - extraBin = + name = device.value; + value = { device = device.name; fsType = "vfat"; neededForBoot = true; options = [ "noatime" ]; }; + }) + (inputs.localLib.attrsToList fileSystems.mount.vfat)); + } + # mount.btrfs + # Disable CoW for VM image and database: sudo chattr +C images + # resize btrfs: + # sudo btrfs filesystem resize -50G /nix + # sudo cryptsetup status root + # sudo cryptsetup -b 3787456512 resize root + # sudo cfdisk /dev/nvme1n1p3 + { + fileSystems = builtins.listToAttrs (builtins.concatLists (builtins.map + (device: builtins.map + ( + subvol: { - grep = "${inputs.pkgs.gnugrep}/bin/grep"; - awk = "${inputs.pkgs.gawk}/bin/awk"; - chattr = "${inputs.pkgs.e2fsprogs}/bin/chattr"; - lsmod = "${inputs.pkgs.kmod}/bin/lsmod"; - }; - services.roll-rootfs = - { - wantedBy = [ "initrd.target" ]; - after = [ "cryptsetup.target" "systemd-hibernate-resume.service" ]; - before = [ "local-fs-pre.target" "sysroot.mount" ]; - unitConfig.DefaultDependencies = false; - serviceConfig.Type = "oneshot"; - script = - let - inherit (fileSystems.rollingRootfs) device path waitDevices; - waitDevice = concatStringsSep "\n" (builtins.map - (device: "while ! [ -e ${device} ]; do sleep 1; done") (waitDevices ++ [ device ])); - in - '' - while ! lsmod | grep -q btrfs; do sleep 1; done - ${waitDevice} - mount ${device} /mnt -m - if [ -f /mnt${path}/current/.timestamp ] - then - timestamp=$(cat /mnt${path}/current/.timestamp) - subvolid=$(btrfs subvolume show /mnt${path}/current | grep 'Subvolume ID:' | awk '{print $NF}') - mv /mnt${path}/current /mnt${path}/$timestamp-$subvolid - btrfs property set -ts /mnt${path}/$timestamp-$subvolid ro true - fi - btrfs subvolume create /mnt${path}/current - chattr +C /mnt${path}/current - echo $(date '+%Y%m%d%H%M%S') > /mnt${path}/current/.timestamp - umount /mnt - ''; - }; - }; - } - ) - ]; + name = subvol.value; + value = + { + device = device.name; + fsType = "btrfs"; + # zstd:15 cause sound stuttering + # test on e20dae7d8b317f95718b5f4175bd4246c09735de mathematica ~15G + # zstd:15 5m33s 7.16G + # zstd:8 54s 7.32G + # zstd:3 17s 7.52G + options = [ "compress-force=zstd" "subvol=${subvol.name}" "acl" "noatime" ]; + neededForBoot = true; + }; + } + ) + (inputs.localLib.attrsToList device.value) + ) + (inputs.localLib.attrsToList fileSystems.mount.btrfs))); + } + # mdadm + (inputs.lib.mkIf (fileSystems.mdadm != null) + { boot.initrd.services.swraid = { enable = true; mdadmConf = fileSystems.mdadm; }; } + ) + # swap + { swapDevices = builtins.map (device: { device = device; }) fileSystems.swap; } + # resume + (inputs.lib.mkIf (fileSystems.resume != null) { boot = + ( + if builtins.typeOf fileSystems.resume == "string" then + { resumeDevice = fileSystems.resume; } + else + { + resumeDevice = fileSystems.resume.device; + kernelModules = [ "resume_offset=${builtins.toString fileSystems.resume.offset}" ]; + } + );}) + # rollingRootfs + (inputs.lib.mkIf (fileSystems.rollingRootfs != null) + { + boot.initrd.systemd = + { + extraBin = + { + grep = "${inputs.pkgs.gnugrep}/bin/grep"; + awk = "${inputs.pkgs.gawk}/bin/awk"; + chattr = "${inputs.pkgs.e2fsprogs}/bin/chattr"; + lsmod = "${inputs.pkgs.kmod}/bin/lsmod"; + }; + services.roll-rootfs = + { + wantedBy = [ "initrd.target" ]; + after = [ "cryptsetup.target" "systemd-hibernate-resume.service" ]; + before = [ "local-fs-pre.target" "sysroot.mount" ]; + unitConfig.DefaultDependencies = false; + serviceConfig.Type = "oneshot"; + script = + let + inherit (fileSystems.rollingRootfs) device path waitDevices; + waitDevice = builtins.concatStringsSep "\n" (builtins.map + (device: "while ! [ -e ${device} ]; do sleep 1; done") (waitDevices ++ [ device ])); + in + '' + while ! lsmod | grep -q btrfs; do sleep 1; done + ${waitDevice} + mount ${device} /mnt -m + if [ -f /mnt${path}/current/.timestamp ] + then + timestamp=$(cat /mnt${path}/current/.timestamp) + subvolid=$(btrfs subvolume show /mnt${path}/current | grep 'Subvolume ID:' | awk '{print $NF}') + mv /mnt${path}/current /mnt${path}/$timestamp-$subvolid + btrfs property set -ts /mnt${path}/$timestamp-$subvolid ro true + fi + btrfs subvolume create /mnt${path}/current + chattr +C /mnt${path}/current + echo $(date '+%Y%m%d%H%M%S') > /mnt${path}/current/.timestamp + umount /mnt + ''; + }; + }; + }) + ]; } From e72bec278ece0591915a63d93b10427faf7a8295 Mon Sep 17 00:00:00 2001 From: chn Date: Thu, 26 Sep 2024 12:16:15 +0800 Subject: [PATCH 04/11] modules.system.fileSystems.nfs: init --- devices/srv1/node1/default.nix | 24 ++---------------------- devices/srv1/node2/default.nix | 26 ++------------------------ devices/srv1/node3/default.nix | 26 ++------------------------ modules/system/fileSystems/nfs.nix | 29 +++++++++++++++++++++++++++++ 4 files changed, 35 insertions(+), 70 deletions(-) create mode 100644 modules/system/fileSystems/nfs.nix diff --git a/devices/srv1/node1/default.nix b/devices/srv1/node1/default.nix index ca876c90..5c7e17c8 100644 --- a/devices/srv1/node1/default.nix +++ b/devices/srv1/node1/default.nix @@ -13,6 +13,7 @@ inputs: eno2 = { ip = "192.168.178.2"; mask = 24; gateway = "192.168.178.1"; dns = "192.168.178.1"; }; }; cluster.nodeType = "worker"; + fileSystems.mount.nfs."192.168.178.1:/home" = "/home"; }; services.beesd.instances.root = { device = "/"; hashTableSizeMB = 256; threads = 4; }; packages.packages._prebuildPackages = @@ -20,31 +21,10 @@ inputs: }; specialisation.no-share-home.configuration = { - nixos = - { - services.slurm.enable = inputs.lib.mkForce false; - system.cluster.nodeType = inputs.lib.mkForce "master"; - }; + nixos.system.fileSystems.mount.nfs = inputs.lib.mkForce null; system.nixos.tags = [ "no-share-home" ]; }; - fileSystems = inputs.lib.mkIf (inputs.config.nixos.system.cluster.nodeType == "worker") - { - "/home" = - { - device = "192.168.178.1:/home"; - fsType = "nfs"; - neededForBoot = true; - }; - }; - boot.initrd.network.enable = true; boot.initrd.systemd.network.networks."10-eno2" = inputs.config.systemd.network.networks."10-eno2"; - boot.initrd.systemd.extraBin = - { - "ifconfig" = "${inputs.pkgs.nettools}/bin/ifconfig"; - "mount.nfs" = "${inputs.pkgs.nfs-utils}/bin/mount.nfs"; - "mount.nfs4" = "${inputs.pkgs.nfs-utils}/bin/mount.nfs4"; - }; - services.rpcbind.enable = true; # make slurm sub process to be able to communicate with the master networking.firewall.trustedInterfaces = [ "eno2" ]; }; diff --git a/devices/srv1/node2/default.nix b/devices/srv1/node2/default.nix index 4ceaa442..02174e03 100644 --- a/devices/srv1/node2/default.nix +++ b/devices/srv1/node2/default.nix @@ -10,8 +10,7 @@ inputs: networking.networkd.static.eno2 = { ip = "192.168.178.3"; mask = 24; gateway = "192.168.178.1"; dns = "192.168.178.1"; }; cluster.nodeType = "worker"; - initrd.sshd.enable = true; - nix.remote.slave.enable = true; + fileSystems.mount.nfs."192.168.178.1:/home" = "/home"; }; services.beesd.instances.root = { device = "/"; hashTableSizeMB = 256; threads = 4; }; packages.packages._prebuildPackages = @@ -19,31 +18,10 @@ inputs: }; specialisation.no-share-home.configuration = { - nixos = - { - services.slurm.enable = inputs.lib.mkForce false; - system.cluster.nodeType = inputs.lib.mkForce "master"; - }; + nixos.system.fileSystems.mount.nfs = inputs.lib.mkForce null; system.nixos.tags = [ "no-share-home" ]; }; - fileSystems = inputs.lib.mkIf (inputs.config.nixos.system.cluster.nodeType == "worker") - { - "/home" = - { - device = "192.168.178.1:/home"; - fsType = "nfs"; - neededForBoot = true; - }; - }; - boot.initrd.network.enable = true; boot.initrd.systemd.network.networks."10-eno2" = inputs.config.systemd.network.networks."10-eno2"; - boot.initrd.systemd.extraBin = - { - "ifconfig" = "${inputs.pkgs.nettools}/bin/ifconfig"; - "mount.nfs" = "${inputs.pkgs.nfs-utils}/bin/mount.nfs"; - "mount.nfs4" = "${inputs.pkgs.nfs-utils}/bin/mount.nfs4"; - }; - services.rpcbind.enable = true; # make slurm sub process to be able to communicate with the master networking.firewall.trustedInterfaces = [ "eno2" ]; }; diff --git a/devices/srv1/node3/default.nix b/devices/srv1/node3/default.nix index 59a75784..c182bb1f 100644 --- a/devices/srv1/node3/default.nix +++ b/devices/srv1/node3/default.nix @@ -10,8 +10,7 @@ inputs: networking.networkd.static.eno2 = { ip = "192.168.178.4"; mask = 24; gateway = "192.168.178.1"; dns = "192.168.178.1"; }; cluster.nodeType = "worker"; - initrd.sshd.enable = true; - nix.remote.slave.enable = true; + fileSystems.mount.nfs."192.168.178.1:/home" = "/home"; }; services.beesd.instances.root = { device = "/"; hashTableSizeMB = 256; threads = 4; }; packages.packages._prebuildPackages = @@ -19,31 +18,10 @@ inputs: }; specialisation.no-share-home.configuration = { - nixos = - { - services.slurm.enable = inputs.lib.mkForce false; - system.cluster.nodeType = inputs.lib.mkForce "master"; - }; + nixos.system.fileSystems.mount.nfs = inputs.lib.mkForce null; system.nixos.tags = [ "no-share-home" ]; }; - fileSystems = inputs.lib.mkIf (inputs.config.nixos.system.cluster.nodeType == "worker") - { - "/home" = - { - device = "192.168.178.1:/home"; - fsType = "nfs"; - neededForBoot = true; - }; - }; - boot.initrd.network.enable = true; boot.initrd.systemd.network.networks."10-eno2" = inputs.config.systemd.network.networks."10-eno2"; - boot.initrd.systemd.extraBin = - { - "ifconfig" = "${inputs.pkgs.nettools}/bin/ifconfig"; - "mount.nfs" = "${inputs.pkgs.nfs-utils}/bin/mount.nfs"; - "mount.nfs4" = "${inputs.pkgs.nfs-utils}/bin/mount.nfs4"; - }; - services.rpcbind.enable = true; # make slurm sub process to be able to communicate with the master networking.firewall.trustedInterfaces = [ "eno2" ]; }; diff --git a/modules/system/fileSystems/nfs.nix b/modules/system/fileSystems/nfs.nix new file mode 100644 index 00000000..3f90ad38 --- /dev/null +++ b/modules/system/fileSystems/nfs.nix @@ -0,0 +1,29 @@ +inputs: +{ + imports = inputs.localLib.findModules ./.; + options.nixos.system.fileSystems.mount.nfs = let inherit (inputs.lib) mkOption types; in mkOption + { + type = types.nullOr (types.attrsOf types.nonEmptyStr); default = null; + }; + config = let inherit (inputs.config.nixos.system.fileSystems.mount) nfs; in inputs.lib.mkIf (nfs != null) + { + fileSystems = builtins.listToAttrs (builtins.map + (device: + { + name = device.value; + value = { device = device.name; fsType = "nfs"; neededForBoot = true; }; + }) + (inputs.localLib.attrsToList nfs)); + boot.initrd = + { + network.enable = true; + systemd.extraBin = + { + "ifconfig" = "${inputs.pkgs.nettools}/bin/ifconfig"; + "mount.nfs" = "${inputs.pkgs.nfs-utils}/bin/mount.nfs"; + "mount.nfs4" = "${inputs.pkgs.nfs-utils}/bin/mount.nfs4"; + }; + }; + services.rpcbind.enable = true; + }; +} From e574916fd07677507973430996f02277f9320d80 Mon Sep 17 00:00:00 2001 From: chn Date: Thu, 26 Sep 2024 12:24:57 +0800 Subject: [PATCH 05/11] devices.srv1: fix network --- devices/srv1/node1/default.nix | 7 ++----- devices/srv1/node2/default.nix | 13 ++++++++++--- devices/srv1/node2/secrets/default.yaml | 7 ++++--- devices/vps6/default.nix | 2 +- devices/vps6/secrets.yaml | 6 ++++-- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/devices/srv1/node1/default.nix b/devices/srv1/node1/default.nix index 5c7e17c8..fa6f6ff2 100644 --- a/devices/srv1/node1/default.nix +++ b/devices/srv1/node1/default.nix @@ -7,11 +7,8 @@ inputs: system = { nixpkgs.march = "broadwell"; - networking.networkd.static = - { - eno1 = { ip = "192.168.1.11"; mask = 24; gateway = "192.168.1.1"; }; - eno2 = { ip = "192.168.178.2"; mask = 24; gateway = "192.168.178.1"; dns = "192.168.178.1"; }; - }; + networking.networkd.static.eno2 = + { ip = "192.168.178.2"; mask = 24; gateway = "192.168.178.1"; dns = "192.168.178.1"; }; cluster.nodeType = "worker"; fileSystems.mount.nfs."192.168.178.1:/home" = "/home"; }; diff --git a/devices/srv1/node2/default.nix b/devices/srv1/node2/default.nix index 02174e03..3dcfaf76 100644 --- a/devices/srv1/node2/default.nix +++ b/devices/srv1/node2/default.nix @@ -7,12 +7,19 @@ inputs: system = { nixpkgs.march = "broadwell"; - networking.networkd.static.eno2 = - { ip = "192.168.178.3"; mask = 24; gateway = "192.168.178.1"; dns = "192.168.178.1"; }; + networking.networkd.static = + { + eno1 = { ip = "192.168.1.12"; mask = 24; gateway = "192.168.1.1"; dns = "192.168.1.1"; }; + eno2 = { ip = "192.168.178.3"; mask = 24; }; + }; cluster.nodeType = "worker"; fileSystems.mount.nfs."192.168.178.1:/home" = "/home"; }; - services.beesd.instances.root = { device = "/"; hashTableSizeMB = 256; threads = 4; }; + services = + { + xray.client.enable = true; + beesd.instances.root = { device = "/"; hashTableSizeMB = 256; threads = 4; }; + }; packages.packages._prebuildPackages = [ inputs.topInputs.self.nixosConfigurations.srv1-node0.config.system.build.toplevel ]; }; diff --git a/devices/srv1/node2/secrets/default.yaml b/devices/srv1/node2/secrets/default.yaml index 5125ef2f..2b1f5f0b 100644 --- a/devices/srv1/node2/secrets/default.yaml +++ b/devices/srv1/node2/secrets/default.yaml @@ -1,4 +1,5 @@ -hello: ENC[AES256_GCM,data:/WGwXDnQio1BwD/zPoURTjVzTasWICOA7CBsgT5DbYIkKLt5DxzogeYWpiqjVg==,iv:BY82U/y9V8PYtn3Bre+nabGBcVgFbppIQZb7GhgY62I=,tag:JFqrezoWNJ8ZACCKQ43n5g==,type:str] +xray-client: + uuid: ENC[AES256_GCM,data:U+unsiKt9vNo/EXEpLHR0Ny3DxQEwx7a40KmwZDZki7RQEuM,iv:7w90HNM5lfh2VY20AcUEVdu5X2uxqXxR0hARncmMR60=,tag:xIbKc+9SF5LP/tY/XoGYxA==,type:str] sops: kms: [] gcp_kms: [] @@ -23,8 +24,8 @@ sops: MVU1UW9lWFJnSTE2aC9ZL0huYURUK3MK5U4cLWRMm+FFo8ATE/OoAcHzYHFMpOtV Q5kbq5PDMdp4qvoM3T4kLsB34oU55HjFvac0pilOhNRrz4xRMQgvoQ== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-09-20T05:30:52Z" - mac: ENC[AES256_GCM,data:nSrkKUo4yB57aetzdJ1sjSKcm5STQ6jfMhvY4/tXft2P9zRYigSP4PkZj7z+knxcIx9sFdA86h8X45oUjxaAa5xDJpgmvC/EEKxm5rZtVTxYYYdy40W72qThVuKUasWpYrrGZbZEbTu3Dad1yfJTilwofRtxoo1Nmj5lMvw+HRo=,iv:UvBSF5GLEj+hTZksrIV3Ow+HQ/xjqwCUuwqkdz8g0Qg=,tag:U5wJPhmeevB2i2GBgMGBFQ==,type:str] + lastmodified: "2024-09-26T04:24:23Z" + mac: ENC[AES256_GCM,data:GS9TPomEy3rHaanqMWkqGV9NX8lsYMnujxhuWTnJxdgfCP8scOEo2O+ATlmxmcmKquXlq+zo4LF6RCzF65eSvpSKcIDQRc0yDgBQ5+M8gXv1lk0WNdicpHJsEk8j/ostJTwEkV5QDyp3+J6lRNtFMQnrZ7+UxpgQwK7DaP8mnF4=,iv:1BEb2Xr8jQO6M19bC+jlGGSI0aT1MEgEoYwLuCT1T9U=,tag:Be7BQsjWq7PZBrgsrH/cjA==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.9.0 diff --git a/devices/vps6/default.nix b/devices/vps6/default.nix index d382494f..aae63be4 100644 --- a/devices/vps6/default.nix +++ b/devices/vps6/default.nix @@ -36,7 +36,7 @@ inputs: { snapper.enable = true; sshd = {}; - xray.server = { serverName = "vps6.xserver.chn.moe"; userNumber = 21; }; + xray.server = { serverName = "vps6.xserver.chn.moe"; userNumber = 22; }; frpServer = { enable = true; serverName = "frp.chn.moe"; }; nginx = { diff --git a/devices/vps6/secrets.yaml b/devices/vps6/secrets.yaml index bfcde7c1..8f7750b5 100644 --- a/devices/vps6/secrets.yaml +++ b/devices/vps6/secrets.yaml @@ -46,6 +46,8 @@ xray-server: user19: ENC[AES256_GCM,data:+Mh15DR9xvFAwks86iuHEA9FpObKWTSuVOEzUDpBUS/h0hOz,iv:zYIkic2bibvwCBpomnJ9465mda1rbm3RERBZY9twXuc=,tag:bwdL6DAGgkGYhYFI2C4A+A==,type:str] #ENC[AES256_GCM,data:1g2gohLbiixMes8=,iv:E3HA6cAdv3BdLMcrrcWW4Zsc2KLtW7L8Xrk9Z57l49o=,tag:rZ7W9ckf7lzJ23u5zwQiwg==,type:comment] user20: ENC[AES256_GCM,data:3UbVnn9oMRc0zZR46tWxwM9VFOvMOYm690csUomEVBcS3xPm,iv:KHuPXttLAFr7WT/qa/UYLY8GRsPWYZPyKNmdUh4iFQQ=,tag:jN8rQ0Gv+qnhwOWGH+CwlA==,type:str] + #ENC[AES256_GCM,data:GzxXsTbEvdHV7A0=,iv:uxUG4hnYEsmJtnqbEwamwhtLt3UClt7ktmkGyAFdxsc=,tag:sF8YQ2cejAezI3Bbp9qKIw==,type:comment] + user21: ENC[AES256_GCM,data:hgDJ11crZaWcKrc+ZDQklXwpnvt/sMbARkx3sLZfQGZqQZeA,iv:2Re+hdJuT5yg/qTymfpN+KdU3criOmwuqqg+SHb8iAo=,tag:s16N6u5cRDaoWxnrCkamuw==,type:str] private-key: ENC[AES256_GCM,data:ts/LRGFAsYqvGvkvlxUI42IW1a8cGsSkpZhMDd3QVceRKvhPb1SRDaXoSw==,iv:6xX9xFIFUNlLBZ6CPBOz9JbHpvC4+QG9ZaCZcWdl12c=,tag:DYIa+QTV8vyl1l7OKKykTw==,type:str] nginx: #ENC[AES256_GCM,data:85LrqdTMIhSa,iv:mIQPYz8VPd5AxeMCQEdTGMD0Iqa5QEAa5+8JVFaj3JM=,tag:TcZd7S3WRPpEV9lHI1fzbw==,type:comment] @@ -89,8 +91,8 @@ sops: ZXFTU3ZCaW1pTVh0RUJzdDdGdHlPYTgK2mlgcX2kEc8+2UDdBnhUm6IIuh8V6agW ooxH9OEPXUVI/4JcDo4v8ZUhAyU1ehLH0Ef7PJCChOZe2KZmWSNbhA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-09-15T03:48:21Z" - mac: ENC[AES256_GCM,data:kZDIr2NHVew+BsreAoYNAcP/3i2A7U1RGIiA6qok1EsXLcunO+vfjIZl3L/0CEEH4+u6PEXQ51atzufqOGpoq9XqorBMRDEdlhitZZIUZm8Cji9BJxixeTUQ+KmFEbdw8H1XDIPWOQJCmTUbkOElMzHO4BNtTpdjE4u1IZ0bUiY=,iv:cCjln5wrScDz7A5/OHVoAj671VtkBmK7H0pnpKLsjD8=,tag:rohzoGp2V1sS03W0z5hM/A==,type:str] + lastmodified: "2024-09-26T04:24:17Z" + mac: ENC[AES256_GCM,data:AXhLmyZWGD6KvMkyHqmCERE6eNE3pD5Pa/9mRBWZe4hiXL4mKTzCn5C/ODGQ1ZeQjDdP+awjJRvLRjMiYFhVlU8rKpg/f2G1gDr4cIbr61sCdzXKX8wFW0G7bJWxxpAC4X59+u9EJ3sNcyf7bJrMdkTzTYpgXh29mtl2bprcdJQ=,iv:pK4hYexcWng3GwOmWGqgyMsmATnXgcwR3NH4UxCwpvE=,tag:zpv64JWoXc5cDCukDuW51g==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.9.0 From 4fea6edcadcabd5c0dd8607afb5ab77e3323cb64 Mon Sep 17 00:00:00 2001 From: chn Date: Thu, 26 Sep 2024 12:41:26 +0800 Subject: [PATCH 06/11] fix build --- devices/srv1/node0/default.nix | 2 +- modules/system/fileSystems/nfs.nix | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/devices/srv1/node0/default.nix b/devices/srv1/node0/default.nix index 73cb98af..83bab04d 100644 --- a/devices/srv1/node0/default.nix +++ b/devices/srv1/node0/default.nix @@ -25,7 +25,7 @@ inputs: publicKey = "Br+ou+t9M9kMrnNnhTvaZi2oNFRygzebA1NqcHWADWM="; wireguardIp = "192.168.83.9"; }; - nfs = { root = "/"; exports = "/home"; accessLimit = "192.168.178.0/24"; }; + nfs = { root = "/"; exports = [ "/home" ]; accessLimit = "192.168.178.0/24"; }; }; packages.packages._prebuildPackages = [ inputs.topInputs.self.nixosConfigurations.srv1-node1.pkgs.localPackages.vasp.intel ]; diff --git a/modules/system/fileSystems/nfs.nix b/modules/system/fileSystems/nfs.nix index 3f90ad38..5fc52d45 100644 --- a/modules/system/fileSystems/nfs.nix +++ b/modules/system/fileSystems/nfs.nix @@ -1,6 +1,5 @@ inputs: { - imports = inputs.localLib.findModules ./.; options.nixos.system.fileSystems.mount.nfs = let inherit (inputs.lib) mkOption types; in mkOption { type = types.nullOr (types.attrsOf types.nonEmptyStr); default = null; From c80617c6c0f2f18fe2080887fed4905bcb0a6f1a Mon Sep 17 00:00:00 2001 From: chn Date: Thu, 26 Sep 2024 13:27:16 +0800 Subject: [PATCH 07/11] modules.services.nfs: fix --- modules/services/nfs.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/services/nfs.nix b/modules/services/nfs.nix index 53b09ea7..7be2c737 100644 --- a/modules/services/nfs.nix +++ b/modules/services/nfs.nix @@ -18,7 +18,7 @@ inputs: nfs.server = { enable = true; - exports = "${nfs.root} ${nfs.accessLimit}(rw,no_root_squash,fsid=0,sync,crossmnt)" + exports = "${nfs.root} ${nfs.accessLimit}(rw,no_root_squash,fsid=0,sync,crossmnt)\n" + builtins.concatStringsSep "\n" (builtins.map (export: "${export} ${nfs.accessLimit}(rw,no_root_squash,sync,crossmnt)") nfs.exports); From cf3882beccce802b768d40a160905f118860830c Mon Sep 17 00:00:00 2001 From: chn Date: Thu, 26 Sep 2024 13:51:57 +0800 Subject: [PATCH 08/11] modules.packages: fix prebuild-packages --- modules/packages/default.nix | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/modules/packages/default.nix b/modules/packages/default.nix index d1a45f6a..32f5c699 100644 --- a/modules/packages/default.nix +++ b/modules/packages/default.nix @@ -24,19 +24,9 @@ inputs: excludePythonPackages)) (builtins.concatLists (builtins.map (packageFunction: packageFunction pythonPackages) (_pythonPackages ++ extraPythonPackages))))) - (inputs.pkgs.callPackage ({ stdenv }: stdenv.mkDerivation - { - name = "prebuild-packages"; - propagateBuildInputs = inputs.lib.lists.subtractLists excludePrebuildPackages - (_prebuildPackages ++ extraPrebuildPackages); - phases = [ "installPhase" ]; - installPhase = - '' - runHook preInstall - mkdir -p $out - runHook postInstall - ''; - }) {}) + (inputs.pkgs.writeTextDir "share/prebuild-packages" + (builtins.concatStringsSep "\n" (builtins.map builtins.toString + (inputs.lib.lists.subtractLists excludePrebuildPackages (_prebuildPackages ++ extraPrebuildPackages))))) ]; }; } From 198fccc7bf504af9157d5b4c3b0dcc963c2f713a Mon Sep 17 00:00:00 2001 From: chn Date: Thu, 26 Sep 2024 15:23:59 +0800 Subject: [PATCH 09/11] packages.sbatch-tui: fix --- packages/sbatch-tui/src/main.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/sbatch-tui/src/main.cpp b/packages/sbatch-tui/src/main.cpp index 5b94dd21..55ee6bb2 100644 --- a/packages/sbatch-tui/src/main.cpp +++ b/packages/sbatch-tui/src/main.cpp @@ -138,10 +138,10 @@ int main() state.job_name, state.output_file, state.vasp_version_entries[state.vasp_version_selected] ); else state.submit_command = - "sbatch --ntasks={}\n--cpus-per-task={}\n--hint=nomultithread\n--job-name='{}'\n--output='{}'" - "\n--wrap=\"vasp-intel srun vasp-{}\""_f + "sbatch --ntasks={}\n--cpus-per-task={}\n--export=ALL,OMP_NUM_THREADS={}\n--hint=nomultithread\n--job-name='{}'\n" + "--output='{}'\n--wrap=\"vasp-intel srun --mpi=pmix vasp-{}\""_f ( - state.mpi_threads, state.openmp_threads, state.job_name, state.output_file, + state.mpi_threads, state.openmp_threads, state.openmp_threads, state.job_name, state.output_file, state.vasp_version_entries[state.vasp_version_selected] ); From 5944409604dad7973048780c9dec6423a64a0149 Mon Sep 17 00:00:00 2001 From: chn Date: Thu, 26 Sep 2024 15:31:59 +0800 Subject: [PATCH 10/11] devices.srv1: add users --- devices/srv1/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devices/srv1/default.nix b/devices/srv1/default.nix index 6c5a8d99..1ce123cd 100644 --- a/devices/srv1/default.nix +++ b/devices/srv1/default.nix @@ -66,7 +66,7 @@ inputs: setupFirewall = true; }; }; - user.users = [ "chn" ]; + user.users = [ "chn" "xll" "zem" "yjq" "gb" "wp" "hjp" "wm" ]; }; }; } From 7d27bad072240d3e47e563ea119b5504bc41a8bd Mon Sep 17 00:00:00 2001 From: chn Date: Thu, 26 Sep 2024 20:41:26 +0800 Subject: [PATCH 11/11] devices.srv1.node2: enable kvm --- devices/srv1/node2/default.nix | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/devices/srv1/node2/default.nix b/devices/srv1/node2/default.nix index 3dcfaf76..90a6a325 100644 --- a/devices/srv1/node2/default.nix +++ b/devices/srv1/node2/default.nix @@ -13,7 +13,12 @@ inputs: eno2 = { ip = "192.168.178.3"; mask = 24; }; }; cluster.nodeType = "worker"; - fileSystems.mount.nfs."192.168.178.1:/home" = "/home"; + fileSystems.mount = + { + nfs."192.168.178.1:/home" = "/home"; + btrfs."/dev/disk/by-partlabel/srv1-node2-nodatacow" = + { "/nix/nodatacow" = "/nix/nodatacow"; "/nix/backups" = "/nix/backups"; }; + }; }; services = { @@ -22,6 +27,7 @@ inputs: }; packages.packages._prebuildPackages = [ inputs.topInputs.self.nixosConfigurations.srv1-node0.config.system.build.toplevel ]; + virtualization.kvmHost = { enable = true; gui = true; }; }; specialisation.no-share-home.configuration = {