nixos/netboot: use config.system.boot.loader.initrdFile instead of hardcoding initrd, disable compression for initrd

As with many things, we have scenarios where we don't want to boot on a
disk / bootloader and also we don't want to boot directly.

Sometimes, we want to boot through an OptionROM of our NIC, e.g. netboot
scenarios or let the firmware decide something, e.g. UEFI PXE (or even
UEFI OptionROM!).

This is composed of:

- `directBoot.enable`: whether to direct boot or not
- `directBoot.initrd`: enable overriding the
  `config.system.build.initialRamdisk` defaults, useful for
  netbootRamdisk for example.

This makes it possible.

nixos/modules/installer/cd-dvd/channel.nix

@@ -56,7 +56,9 @@ in
           -i ${channelSources} --quiet --option build-use-substitutes false \
           ${optionalString config.boot.initrd.systemd.enable "--option sandbox false"} # There's an issue with pivot_root
         mkdir -m 0700 -p /root/.nix-defexpr
-        ln -s /nix/var/nix/profiles/per-user/root/channels /root/.nix-defexpr/channels
+        # We do not want to ship broken channels.
+        unlink /root/.nix-defexpr/channels
+        ln -sf /nix/var/nix/profiles/per-user/root/channels /root/.nix-defexpr/channels || fail
         mkdir -m 0755 -p /var/lib/nixos
         touch /var/lib/nixos/did-channel-init
       fi

nixos/modules/installer/netboot/netboot.nix

@@ -1,7 +1,7 @@
 # This module creates netboot media containing the given NixOS
 # configuration.
 
-{ config, lib, pkgs, ... }:
+{ options, config, lib, pkgs, ... }:
 
 with lib;
 
@@ -29,6 +29,34 @@ with lib;
           then []
           else [ pkgs.grub2 pkgs.syslinux ]);
 
+    # We only want to set those options in the context of
+    # the QEMU infrastructure.
+    virtualisation = lib.optionalAttrs (options ? virtualisation.directBoot) {
+      # By default, using netboot images in virtualized contexts
+      # should not create any disk image ideally, except if
+      # asked explicitly.
+      diskImage = mkDefault null;
+      # We do not want to mount the host Nix store in those situations.
+      mountHostNixStore = mkDefault false;
+      # We do not need the nix store image because:
+      # - either we boot through network and we have the squashfs image
+      # - either we direct boot, we have the squashfs image
+      useNixStoreImage = mkDefault false;
+      # Though, we still want a writable store through .rw-store
+      writableStore = mkDefault true;
+      # Ideally, we might not want to test the network / firmware.
+      directBoot = {
+        enable = mkDefault true;
+        # We need to use our netboot initrd which contains a copy of the Nix store.
+        initrd = "${config.system.build.netbootRamdisk}/${config.system.boot.loader.initrdFile}";
+      };
+      # We do not want to use the default filesystems.
+      useDefaultFilesystems = mkDefault false;
+      # Bump the default memory size as we are loading the whole initrd in RAM.
+      memorySize = lib.mkDefault 1536;
+    };
+
+
     fileSystems."/" = mkImageMediaOverride
       { fsType = "tmpfs";
         options = [ "mode=0755" ];
@@ -82,8 +110,7 @@ with lib;
 
     # Create the initrd
     system.build.netbootRamdisk = pkgs.makeInitrdNG {
-      inherit (config.boot.initrd) compressor;
-      prepend = [ "${config.system.build.initialRamdisk}/initrd" ];
+      prepend = [ "${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}" ];
 
       contents =
         [ { object = config.system.build.squashfsStore;
@@ -96,8 +123,8 @@ with lib;
       #!ipxe
       # Use the cmdline variable to allow the user to specify custom kernel params
       # when chainloading this script from other iPXE scripts like netboot.xyz
-      kernel ${pkgs.stdenv.hostPlatform.linux-kernel.target} init=${config.system.build.toplevel}/init initrd=initrd ${toString config.boot.kernelParams} ''${cmdline}
-      initrd initrd
+      kernel ${pkgs.stdenv.hostPlatform.linux-kernel.target} init=${config.system.build.toplevel}/init initrd=${config.system.boot.loader.initrdFile} ${toString config.boot.kernelParams} ''${cmdline}
+      initrd ${config.system.boot.loader.initrdFile}
       boot
     '';
 
@@ -111,7 +138,7 @@ with lib;
       fi
       SCRIPT_DIR=$( cd -- "$( dirname -- "''${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
       kexec --load ''${SCRIPT_DIR}/bzImage \
-        --initrd=''${SCRIPT_DIR}/initrd.gz \
+        --initrd=''${SCRIPT_DIR}/${config.system.boot.loader.initrdFile} \
         --command-line "init=${config.system.build.toplevel}/init ${toString config.boot.kernelParams}"
       kexec -e
     '';
@@ -119,8 +146,8 @@ with lib;
     # A tree containing initrd.gz, bzImage and a kexec-boot script.
     system.build.kexecTree = pkgs.linkFarm "kexec-tree" [
       {
-        name = "initrd.gz";
-        path = "${config.system.build.netbootRamdisk}/initrd";
+        name = "${config.system.boot.loader.initrdFile}";
+        path = "${config.system.build.netbootRamdisk}/${config.system.boot.loader.initrdFile}";
       }
       {
         name = "bzImage";

nixos/modules/virtualisation/qemu-vm.nix

@@ -258,13 +258,14 @@ let
         if cfg.useEFIBoot then "efi_bootloading_with_default_fs"
         else "legacy_bootloading_with_default_fs"
       else
-        "direct_boot_with_default_fs"
+        if cfg.directBoot.enable then "direct_boot_with_default_fs"
+        else "custom"
     else
       "custom";
   suggestedRootDevice = {
     "efi_bootloading_with_default_fs" = "${cfg.bootLoaderDevice}2";
     "legacy_bootloading_with_default_fs" = "${cfg.bootLoaderDevice}1";
-    "direct_boot_with_default_fs" = cfg.bootLoaderDevice;
+    "direct_boot_with_default_fs" = lookupDriveDeviceName "root" cfg.qemu.drives;
     # This will enforce a NixOS module type checking error
     # to ask explicitly the user to set a rootDevice.
     # As it will look like `rootDevice = lib.mkDefault null;` after
@@ -336,14 +337,14 @@ in
 
     virtualisation.bootLoaderDevice =
       mkOption {
-        type = types.path;
-        default = lookupDriveDeviceName "root" cfg.qemu.drives;
-        defaultText = literalExpression ''lookupDriveDeviceName "root" cfg.qemu.drives'';
+        type = types.nullOr types.path;
+        default = if cfg.useBootLoader then lookupDriveDeviceName "root" cfg.qemu.drives else null;
+        defaultText = literalExpression ''if cfg.useBootLoader then lookupDriveDeviceName "root" cfg.qemu.drives else null;'';
         example = "/dev/vda";
         description =
           lib.mdDoc ''
             The disk to be used for the boot filesystem.
-            By default, it is the same disk as the root filesystem.
+            By default, it is the same disk as the root filesystem if you use a bootloader, otherwise it's null.
           '';
         };
 
@@ -734,6 +735,39 @@ in
         '';
       };
 
+    virtualisation.directBoot = {
+      enable = mkOption {
+        type = types.bool;
+        default = !cfg.useBootLoader;
+        defaultText = "!cfg.useBootLoader";
+        description =
+          lib.mdDoc ''
+            If enabled, the virtual machine will direct boot the system
+            by passing the kernel, initrd and relevant parameters to QEMU.
+
+            If you want to use a bootloader, use `virtualisation.useBootLoader`,
+            if you want to test netboot, you might be interested into disabling this.
+
+            This is enabled by default if you don't enable bootloaders.
+            Read more about this feature: <https://qemu-project.gitlab.io/qemu/system/linuxboot.html>.
+          '';
+      };
+      initrd =
+        mkOption {
+          type = types.str;
+          default = "${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}";
+          defaultText = "\${config.system.build.initialRamdisk}/\${config.system.boot.loader.initrdFile}";
+          description =
+            lib.mdDoc ''
+              In direct boot situations, you may want to influence the initrd to load
+              to use your own customized payload.
+
+              This is useful if you want to test the netboot image without
+              testing the firmware or the loading part.
+            '';
+        };
+    };
+
     virtualisation.useBootLoader =
       mkOption {
         type = types.bool;
@@ -854,6 +888,28 @@ in
                 Invalid virtualisation.forwardPorts.<entry ${toString i}>.guest.address:
                   The address must be in the default VLAN (10.0.2.0/24).
               '';
+            }
+          { assertion = cfg.useBootLoader -> cfg.diskImage != null;
+            message =
+              ''
+                Currently, bootloaders cannot be used with a tmpfs disk image.
+                It would require some rework in the boot configuration mechanism
+                to detect the proper boot partition in UEFI scenarios for example.
+
+                If you are interested into this feature, please open an issue or open a pull request.
+              '';
+          }
+          { assertion = cfg.directBoot.initrd != options.virtualisation.directBoot.initrd.default -> cfg.directBoot.enable;
+            message =
+              ''
+                You changed the default of `virtualisation.directBoot.initrd` but you are not
+                using QEMU direct boot. This initrd will not be used in your current
+                boot configuration.
+
+                Either do not mutate `virtualisation.directBoot.initrd` or enable direct boot.
+
+                If you have a more advanced usecase, please open an issue or a pull request.
+              '';
           }
         ]));
 
@@ -875,13 +931,19 @@ in
           Otherwise, we recommend
 
             ${opt.writableStore} = false;
+            ''
+      ++ optional (cfg.directBoot.enable && cfg.useBootLoader)
+        ''
+          You enabled direct boot and a bootloader, QEMU will not boot your bootloader, rendering
+          `useBootLoader` useless. You might want to disable one of those options.
         '';
 
     # In UEFI boot, we use a EFI-only partition table layout, thus GRUB will fail when trying to install
     # legacy and UEFI. In order to avoid this, we have to put "nodev" to force UEFI-only installs.
     # Otherwise, we set the proper bootloader device for this.
     # FIXME: make a sense of this mess wrt to multiple ESP present in the system, probably use boot.efiSysMountpoint?
-    boot.loader.grub.device = mkVMOverride (if cfg.useEFIBoot then "nodev" else cfg.bootLoaderDevice);
+    boot.loader.grub.enable = cfg.useBootLoader;
+    boot.loader.grub.device = mkIf cfg.useBootLoader (mkVMOverride (if cfg.useEFIBoot then "nodev" else cfg.bootLoaderDevice));
     boot.loader.grub.gfxmodeBios = with cfg.resolution; "${toString x}x${toString y}";
     virtualisation.rootDevice = mkDefault suggestedRootDevice;
 
@@ -889,13 +951,13 @@ in
 
     boot.loader.supportsInitrdSecrets = mkIf (!cfg.useBootLoader) (mkVMOverride false);
 
-    boot.initrd.extraUtilsCommands = lib.mkIf (cfg.useDefaultFilesystems && !config.boot.initrd.systemd.enable)
+    boot.initrd.extraUtilsCommands = lib.mkIf (cfg.useDefaultFilesystems && !config.boot.initrd.systemd.enable && cfg.diskImage != null)
       ''
         # We need mke2fs in the initrd.
         copy_bin_and_libs ${pkgs.e2fsprogs}/bin/mke2fs
       '';
 
-    boot.initrd.postDeviceCommands = lib.mkIf (cfg.useDefaultFilesystems && !config.boot.initrd.systemd.enable)
+    boot.initrd.postDeviceCommands = lib.mkIf (cfg.useDefaultFilesystems && !config.boot.initrd.systemd.enable && cfg.diskImage != null)
       ''
         # If the disk image appears to be empty, run mke2fs to
         # initialise.
@@ -997,9 +1059,9 @@ in
         alphaNumericChars = lowerChars ++ upperChars ++ (map toString (range 0 9));
         # Replace all non-alphanumeric characters with underscores
         sanitizeShellIdent = s: concatMapStrings (c: if builtins.elem c alphaNumericChars then c else "_") (stringToCharacters s);
-      in mkIf (!cfg.useBootLoader) [
+      in mkIf cfg.directBoot.enable [
         "-kernel \${NIXPKGS_QEMU_KERNEL_${sanitizeShellIdent config.system.name}:-${config.system.build.toplevel}/kernel}"
-        "-initrd ${config.system.build.toplevel}/initrd"
+        "-initrd ${cfg.directBoot.initrd}"
         ''-append "$(cat ${config.system.build.toplevel}/kernel-params) init=${config.system.build.toplevel}/init regInfo=${regInfo}/registration ${consoles} $QEMU_KERNEL_PARAMS"''
       ])
       (mkIf cfg.useEFIBoot [
@@ -1034,7 +1096,7 @@ in
       }) cfg.emptyDiskImages)
     ];
 
-    fileSystems = mkVMOverride cfg.fileSystems;
+    fileSystems = lib.mapAttrs (n: v: mkVMOverride v) cfg.fileSystems;
 
     # Mount the host filesystem via 9P, and bind-mount the Nix store
     # of the host into our own filesystem.  We use mkVMOverride to

nixos/tests/boot.nix

@@ -60,9 +60,37 @@ let
       config = (import ../lib/eval-config.nix {
           inherit system;
           modules =
-            [ ../modules/installer/netboot/netboot.nix
+            [
+              ../modules/installer/netboot/netboot-minimal.nix
               ../modules/testing/test-instrumentation.nix
-              { key = "serial"; }
+              {
+                system.nixos.revision = mkForce "constant-nixos-revision";
+                documentation.enable = false;
+              }
+              {
+                nix.settings = {
+                  substituters = [ ];
+                  hashed-mirrors = null;
+                  connect-timeout = 1;
+                };
+
+                system.extraDependencies = with pkgs; [
+                  curl
+                  desktop-file-utils
+                  docbook5
+                  docbook_xsl_ns
+                  kmod.dev
+                  libarchive
+                  libarchive.dev
+                  libxml2.bin
+                  libxslt.bin
+                  python3Minimal
+                  shared-mime-info
+                  stdenv
+                  sudo
+                  xorg.lndir
+                ];
+              }
             ];
         }).config;
       ipxeBootDir = pkgs.symlinkJoin {
@@ -73,21 +101,56 @@ let
           config.system.build.netbootIpxeScript
         ];
       };
-      machineConfig = pythonDict ({
-        qemuBinary = qemu-common.qemuBinary pkgs.qemu_test;
-        qemuFlags = "-boot order=n -m 2000";
-        netBackendArgs = "tftp=${ipxeBootDir},bootfile=netboot.ipxe";
-      } // extraConfig);
     in
       makeTest {
-        name = "boot-netboot-" + name;
-        nodes = { };
-        testScript = ''
-            machine = create_machine(${machineConfig})
-            machine.start()
-            machine.wait_for_unit("multi-user.target")
-            machine.shutdown()
-          '';
+        name = "boot-netboot-${name}";
+        meta.maintainers = with pkgs.lib.maintainers; [ raitobezarius ];
+        nodes.machine = {
+          imports = [
+            extraConfig
+          ];
+
+          # We *NEVER* want to use a Nix store image with a netboot image.
+          virtualisation.useNixStoreImage = mkForce false;
+          # We want to use at least the netboot's image Nix store!
+          virtualisation.mountHostNixStore = mkForce false;
+          # 2GiB because we are loading from memory the binaries.
+          virtualisation.memorySize = 2048;
+          # We do not need `diskImage` here.
+          virtualisation.diskImage = null;
+          # We do not want to direct boot the NixOS system generated by this expression.
+          virtualisation.directBoot.enable = false;
+
+          virtualisation.qemu.options = [
+            # Network is the ONLY way to *boot*.
+            # No disk, no CD.
+            "-boot order=n,strict=true"
+          ];
+
+          # We do not want any networking in the guest, except for our TFTP stuff.
+          virtualisation.vlans = mkDefault [];
+          virtualisation.qemu.networkingOptions = [
+            # Simulate a TFTP server, user.0 is already taken by the existing networking system.
+            ''-netdev user,id=user.1,net=10.0.3.0/24,tftp-server-name="NixOS QEMU test built-in server",tftp=${ipxeBootDir},bootfile=netboot.ipxe,''${QEMU_NET_OPTS:+,$QEMU_NET_OPTS}''
+            "-device virtio-net-pci,netdev=user.1"
+          ];
+        };
+        testScript =
+        ''
+          machine.start()
+          machine.wait_for_unit("multi-user.target")
+
+          # For debugging purpose and sanity checks.
+          print(machine.succeed("lsblk"))
+          print(machine.succeed("mount"))
+
+          machine.succeed("nix store verify --no-trust -r --option experimental-features nix-command /run/current-system")
+          with subtest("Check whether the channel got installed correctly"):
+              machine.succeed("nix-instantiate --dry-run '<nixpkgs>' -A hello")
+              machine.succeed("nix-env --dry-run -iA nixos.procps")
+
+          machine.shutdown()
+        '';
       };
   uefiBinary = {
     x86_64-linux = "${pkgs.OVMF.fd}/FV/OVMF.fd";
@@ -104,10 +167,22 @@ in {
       bios = uefiBinary;
     };
 
+    directNetboot = makeTest {
+      name = "directboot-netboot";
+      nodes.machine = {
+        imports = [ ../modules/installer/netboot/netboot-minimal.nix ];
+      };
+      testScript = "machine.fail('stat /dev/vda')";
+    };
+
     uefiNetboot = makeNetbootTest "uefi" {
-      bios = uefiBinary;
+      virtualisation.useEFIBoot = true;
+      # TODO: an ideal test would be to try netbooting through another machine with DHCP and proper configuration.
+      # Disable romfile for iPXE in NIC, we want to use EDK2 network stack.
+      # virtualisation.qemu.networkingOptions = [ "-global virtio-net-pci.romfile=" ];
+
       # Custom ROM is needed for EFI PXE boot. I failed to understand exactly why, because QEMU should still use iPXE for EFI.
-      netFrontendArgs = "romfile=${pkgs.ipxe}/ipxe.efirom";
+      virtualisation.qemu.networkingOptions = [ "-global virtio-net-pci.romfile=${pkgs.ipxe}/ipxe.efirom" ];
     };
 } // optionalAttrs (pkgs.stdenv.hostPlatform.system == "x86_64-linux") {
     biosCdrom = makeBootTest "bios-cdrom" {