From 24e33a4d2e41fc1201034e0cd1a6bd5a642d94c5 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Mon, 14 Nov 2022 16:40:21 +0100 Subject: [PATCH 1/4] nixos/ec2: remove paravirtualization-specific code Paravirtualized EC2 instances haven't been supported since 2017. It's safe to remove this now. --- nixos/maintainers/scripts/ec2/amazon-image.nix | 9 ++------- nixos/modules/virtualisation/amazon-image.nix | 15 +++------------ nixos/modules/virtualisation/amazon-options.nix | 10 +++------- nixos/tests/ec2.nix | 2 -- 4 files changed, 8 insertions(+), 28 deletions(-) diff --git a/nixos/maintainers/scripts/ec2/amazon-image.nix b/nixos/maintainers/scripts/ec2/amazon-image.nix index e2a05a09d0c2..0db0f4d0dcca 100644 --- a/nixos/maintainers/scripts/ec2/amazon-image.nix +++ b/nixos/maintainers/scripts/ec2/amazon-image.nix @@ -43,7 +43,7 @@ in { sizeMB = mkOption { type = with types; either (enum [ "auto" ]) int; - default = if config.ec2.hvm then 2048 else 8192; + default = 2048; example = 8192; description = lib.mdDoc "The size in MB of the image"; }; @@ -60,9 +60,6 @@ in { '' { modulesPath, ... }: { imports = [ "''${modulesPath}/virtualisation/amazon-image.nix" ]; - ${optionalString config.ec2.hvm '' - ec2.hvm = true; - ''} ${optionalString config.ec2.efi '' ec2.efi = true; ''} @@ -129,9 +126,7 @@ in { pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package fsType = "ext4"; - partitionTableType = if config.ec2.efi then "efi" - else if config.ec2.hvm then "legacy+gpt" - else "none"; + partitionTableType = if config.ec2.efi then "efi" else "legacy+gpt"; diskSize = cfg.sizeMB; diff --git a/nixos/modules/virtualisation/amazon-image.nix b/nixos/modules/virtualisation/amazon-image.nix index 12fe6fa44479..9de863967e4f 100644 --- a/nixos/modules/virtualisation/amazon-image.nix +++ b/nixos/modules/virtualisation/amazon-image.nix @@ -31,18 +31,12 @@ in config = { assertions = [ - { assertion = cfg.hvm; - message = "Paravirtualized EC2 instances are no longer supported."; - } - { assertion = cfg.efi -> cfg.hvm; - message = "EC2 instances using EFI must be HVM instances."; - } { assertion = versionOlder config.boot.kernelPackages.kernel.version "5.17"; message = "ENA driver fails to build with kernel >= 5.17"; } ]; - boot.growPartition = cfg.hvm; + boot.growPartition = true; fileSystems."/" = mkIf (!cfg.zfs.enable) { device = "/dev/disk/by-label/nixos"; @@ -66,7 +60,7 @@ in ]; boot.initrd.kernelModules = [ "xen-blkfront" "xen-netfront" ]; boot.initrd.availableKernelModules = [ "ixgbevf" "ena" "nvme" ]; - boot.kernelParams = mkIf cfg.hvm [ "console=ttyS0,115200n8" "random.trust_cpu=on" ]; + boot.kernelParams = [ "console=ttyS0,115200n8" "random.trust_cpu=on" ]; # Prevent the nouveau kernel module from being loaded, as it # interferes with the nvidia/nvidia-uvm modules needed for CUDA. @@ -74,10 +68,7 @@ in # boot. boot.blacklistedKernelModules = [ "nouveau" "xen_fbfront" ]; - # Generate a GRUB menu. Amazon's pv-grub uses this to boot our kernel/initrd. - boot.loader.grub.version = if cfg.hvm then 2 else 1; - boot.loader.grub.device = if (cfg.hvm && !cfg.efi) then "/dev/xvda" else "nodev"; - boot.loader.grub.extraPerEntryConfig = mkIf (!cfg.hvm) "root (hd0)"; + boot.loader.grub.device = if cfg.efi then "nodev" else "/dev/xvda"; boot.loader.grub.efiSupport = cfg.efi; boot.loader.grub.efiInstallAsRemovable = cfg.efi; boot.loader.timeout = 1; diff --git a/nixos/modules/virtualisation/amazon-options.nix b/nixos/modules/virtualisation/amazon-options.nix index 227f3e433c10..915bbf9763db 100644 --- a/nixos/modules/virtualisation/amazon-options.nix +++ b/nixos/modules/virtualisation/amazon-options.nix @@ -2,6 +2,9 @@ let inherit (lib) literalExpression types; in { + imports = [ + (lib.mkRemovedOptionModule [ "ec2" "hvm" ] "Only HVM instances are supported, so specifying it is no longer necessary.") + ]; options = { ec2 = { zfs = { @@ -41,13 +44,6 @@ in { }); }; }; - hvm = lib.mkOption { - default = lib.versionAtLeast config.system.stateVersion "17.03"; - internal = true; - description = lib.mdDoc '' - Whether the EC2 instance is a HVM instance. - ''; - }; efi = lib.mkOption { default = pkgs.stdenv.hostPlatform.isAarch64; defaultText = literalExpression "pkgs.stdenv.hostPlatform.isAarch64"; diff --git a/nixos/tests/ec2.nix b/nixos/tests/ec2.nix index aa3c2b7051f6..e649761d029d 100644 --- a/nixos/tests/ec2.nix +++ b/nixos/tests/ec2.nix @@ -16,8 +16,6 @@ let ../modules/testing/test-instrumentation.nix ../modules/profiles/qemu-guest.nix { - ec2.hvm = true; - # Hack to make the partition resizing work in QEMU. boot.initrd.postDeviceCommands = mkBefore '' ln -s vda /dev/xvda From eddfcf8622f547676d36c42619b68de766a78a6d Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Mon, 14 Nov 2022 18:19:29 +0100 Subject: [PATCH 2/4] amazon-image: fetch metadata only in stage-2 This also removes automatic enablement/mounting of instance store swap devices and ext3 filesystems. This behaviour is strongly opinionated and shouldn't be enabled by default. The unionfs behaviour never took effect anyway, because the AMI manifest path only exists for instance store-backed AMIs, which have not been supported by nixpkgs since 84742e22934d697e0476fab5a6c8886723ff92ef (2019). --- .../from_md/release-notes/rl-2305.section.xml | 36 ++++++++- .../manual/release-notes/rl-2305.section.md | 7 ++ nixos/modules/virtualisation/amazon-image.nix | 81 +++---------------- nixos/modules/virtualisation/ec2-data.nix | 1 + 4 files changed, 54 insertions(+), 71 deletions(-) diff --git a/nixos/doc/manual/from_md/release-notes/rl-2305.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2305.section.xml index 930f6b2066e3..da765edcd08e 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2305.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2305.section.xml @@ -30,7 +30,7 @@
Backward Incompatibilities - + carnix and cratesIO has @@ -42,6 +42,40 @@ instead. + + + The EC2 image module no longer fetches instance metadata in + stage-1. This results in a significantly smaller initramfs, + since network drivers no longer need to be included, and + faster boots, since metadata fetching can happen in parallel + with startup of other services. This breaks services which + rely on metadata being present by the time stage-2 is entered. + Anything which reads EC2 metadata from + /etc/ec2-metadata should now have an + after dependency on + fetch-ec2-metadata.service + + + + + The EC2 image module previously detected and automatically + mounted ext3-formatted instance store devices and partitions + in stage-1 (initramfs), storing /tmp on the + first discovered device. This behaviour, which only catered to + very specific use cases and could not be disabled, has been + removed. Users relying on this should provide their own + implementation, and probably use ext4 and perform the mount in + stage-2. + + + + + The EC2 image module previously detected and activated + swap-formatted instance store devices and partitions in + stage-1 (initramfs). This behaviour has been removed. Users + relying on this should provide their own implementation. + +
diff --git a/nixos/doc/manual/release-notes/rl-2305.section.md b/nixos/doc/manual/release-notes/rl-2305.section.md index 0e8e1ad0b230..3cbf4621a787 100644 --- a/nixos/doc/manual/release-notes/rl-2305.section.md +++ b/nixos/doc/manual/release-notes/rl-2305.section.md @@ -22,6 +22,13 @@ In addition to numerous new and upgraded packages, this release has the followin - `carnix` and `cratesIO` has been removed due to being unmaintained, use alternatives such as [naersk](https://github.com/nix-community/naersk) and [crate2nix](https://github.com/kolloch/crate2nix) instead. +- The EC2 image module no longer fetches instance metadata in stage-1. This results in a significantly smaller initramfs, since network drivers no longer need to be included, and faster boots, since metadata fetching can happen in parallel with startup of other services. + This breaks services which rely on metadata being present by the time stage-2 is entered. Anything which reads EC2 metadata from `/etc/ec2-metadata` should now have an `after` dependency on `fetch-ec2-metadata.service` + +- The EC2 image module previously detected and automatically mounted ext3-formatted instance store devices and partitions in stage-1 (initramfs), storing `/tmp` on the first discovered device. This behaviour, which only catered to very specific use cases and could not be disabled, has been removed. Users relying on this should provide their own implementation, and probably use ext4 and perform the mount in stage-2. + +- The EC2 image module previously detected and activated swap-formatted instance store devices and partitions in stage-1 (initramfs). This behaviour has been removed. Users relying on this should provide their own implementation. + ## Other Notable Changes {#sec-release-23.05-notable-changes} diff --git a/nixos/modules/virtualisation/amazon-image.nix b/nixos/modules/virtualisation/amazon-image.nix index 9de863967e4f..91321fc3f49c 100644 --- a/nixos/modules/virtualisation/amazon-image.nix +++ b/nixos/modules/virtualisation/amazon-image.nix @@ -10,11 +10,6 @@ with lib; let cfg = config.ec2; - metadataFetcher = import ./ec2-metadata-fetcher.nix { - inherit (pkgs) curl; - targetRoot = "$targetRoot/"; - wgetExtraOptions = "-q"; - }; in { @@ -58,8 +53,8 @@ in boot.extraModulePackages = [ config.boot.kernelPackages.ena ]; - boot.initrd.kernelModules = [ "xen-blkfront" "xen-netfront" ]; - boot.initrd.availableKernelModules = [ "ixgbevf" "ena" "nvme" ]; + boot.initrd.kernelModules = [ "xen-blkfront" ]; + boot.initrd.availableKernelModules = [ "nvme" ]; boot.kernelParams = [ "console=ttyS0,115200n8" "random.trust_cpu=on" ]; # Prevent the nouveau kernel module from being loaded, as it @@ -78,67 +73,15 @@ in terminal_input console serial ''; - boot.initrd.network.enable = true; - - # Mount all formatted ephemeral disks and activate all swap devices. - # We cannot do this with the ‘fileSystems’ and ‘swapDevices’ options - # because the set of devices is dependent on the instance type - # (e.g. "m1.small" has one ephemeral filesystem and one swap device, - # while "m1.large" has two ephemeral filesystems and no swap - # devices). Also, put /tmp and /var on /disk0, since it has a lot - # more space than the root device. Similarly, "move" /nix to /disk0 - # by layering a unionfs-fuse mount on top of it so we have a lot more space for - # Nix operations. - boot.initrd.postMountCommands = - '' - ${metadataFetcher} - - diskNr=0 - diskForUnionfs= - for device in /dev/xvd[abcde]*; do - if [ "$device" = /dev/xvda -o "$device" = /dev/xvda1 ]; then continue; fi - fsType=$(blkid -o value -s TYPE "$device" || true) - if [ "$fsType" = swap ]; then - echo "activating swap device $device..." - swapon "$device" || true - elif [ "$fsType" = ext3 ]; then - mp="/disk$diskNr" - diskNr=$((diskNr + 1)) - if mountFS "$device" "$mp" "" ext3; then - if [ -z "$diskForUnionfs" ]; then diskForUnionfs="$mp"; fi - fi - else - echo "skipping unknown device type $device" - fi - done - - if [ -n "$diskForUnionfs" ]; then - mkdir -m 755 -p $targetRoot/$diskForUnionfs/root - - mkdir -m 1777 -p $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp - mount --bind $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp - - if [ "$(cat "$metaDir/ami-manifest-path")" != "(unknown)" ]; then - mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/var $targetRoot/var - mount --bind $targetRoot/$diskForUnionfs/root/var $targetRoot/var - - mkdir -p /unionfs-chroot/ro-nix - mount --rbind $targetRoot/nix /unionfs-chroot/ro-nix - - mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/nix - mkdir -p /unionfs-chroot/rw-nix - mount --rbind $targetRoot/$diskForUnionfs/root/nix /unionfs-chroot/rw-nix - - unionfs -o allow_other,cow,nonempty,chroot=/unionfs-chroot,max_files=32768 /rw-nix=RW:/ro-nix=RO $targetRoot/nix - fi - fi - ''; - - boot.initrd.extraUtilsCommands = - '' - # We need swapon in the initrd. - copy_bin_and_libs ${pkgs.util-linux}/sbin/swapon - ''; + systemd.services.fetch-ec2-metadata = { + wantedBy = [ "multi-user.target" ]; + path = [ pkgs.wget ]; + script = pkgs.callPackage ./ec2-metadata-fetcher.nix { + targetRoot = "/"; + wgetExtraOptions = ""; + }; + serviceConfig.Type = "oneshot"; + }; # Allow root logins only using the SSH key that the user specified # at instance creation time. @@ -157,8 +100,6 @@ in # Always include cryptsetup so that Charon can use it. environment.systemPackages = [ pkgs.cryptsetup ]; - boot.initrd.supportedFilesystems = [ "unionfs-fuse" ]; - # EC2 has its own NTP server provided by the hypervisor networking.timeServers = [ "169.254.169.123" ]; diff --git a/nixos/modules/virtualisation/ec2-data.nix b/nixos/modules/virtualisation/ec2-data.nix index 1b764e7e4d80..0cc6d9938e22 100644 --- a/nixos/modules/virtualisation/ec2-data.nix +++ b/nixos/modules/virtualisation/ec2-data.nix @@ -18,6 +18,7 @@ with lib; wantedBy = [ "multi-user.target" "sshd.service" ]; before = [ "sshd.service" ]; + after = ["fetch-ec2-metadata.service"]; path = [ pkgs.iproute2 ]; From 6fb582e0301da53d6d5a32d02b8ecbaf0f91c188 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Mon, 14 Nov 2022 19:01:39 +0100 Subject: [PATCH 3/4] ec2-metadata-fetcher: ignore failure when fetching metadata parts Instances without SSH keys configured will receive a 404 from the metadata server when attempting to fetch an SSH key. This is not an actual problem though, and shouldn't result in the service failing. If the metadata server cannot be reached, the script will fail at an earlier stage when attempting to get authentication data. --- nixos/modules/virtualisation/ec2-metadata-fetcher.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nixos/modules/virtualisation/ec2-metadata-fetcher.nix b/nixos/modules/virtualisation/ec2-metadata-fetcher.nix index 760f024f33fb..e78df12c961e 100644 --- a/nixos/modules/virtualisation/ec2-metadata-fetcher.nix +++ b/nixos/modules/virtualisation/ec2-metadata-fetcher.nix @@ -70,8 +70,8 @@ wget ${wgetExtraOptions} --header "X-aws-ec2-metadata-token: $IMDS_TOKEN" "$@"; } - wget_imds -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path - (umask 077 && wget_imds -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data) - wget_imds -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname - wget_imds -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key + wget_imds -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path || true + (umask 077 && wget_imds -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data || true) + wget_imds -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname || true + wget_imds -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key || true '' From 36ca2b495fc58d303d60af87f5e2568c5ed2f967 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Thu, 17 Nov 2022 19:44:52 +0100 Subject: [PATCH 4/4] nixos/ec2: use only curl in metadata fetcher, log to console We don't need both wget and curl, so let's use only curl (which is part of a minimal NixOS closure, unlike wget). Logging to the console is helpful for debugging. --- nixos/modules/virtualisation/amazon-image.nix | 9 +-- .../virtualisation/ec2-metadata-fetcher.nix | 77 ------------------- .../virtualisation/ec2-metadata-fetcher.sh | 67 ++++++++++++++++ 3 files changed, 71 insertions(+), 82 deletions(-) delete mode 100644 nixos/modules/virtualisation/ec2-metadata-fetcher.nix create mode 100644 nixos/modules/virtualisation/ec2-metadata-fetcher.sh diff --git a/nixos/modules/virtualisation/amazon-image.nix b/nixos/modules/virtualisation/amazon-image.nix index 91321fc3f49c..9751f5755f96 100644 --- a/nixos/modules/virtualisation/amazon-image.nix +++ b/nixos/modules/virtualisation/amazon-image.nix @@ -75,12 +75,11 @@ in systemd.services.fetch-ec2-metadata = { wantedBy = [ "multi-user.target" ]; - path = [ pkgs.wget ]; - script = pkgs.callPackage ./ec2-metadata-fetcher.nix { - targetRoot = "/"; - wgetExtraOptions = ""; - }; + after = ["network-online.target"]; + path = [ pkgs.curl ]; + script = builtins.readFile ./ec2-metadata-fetcher.sh; serviceConfig.Type = "oneshot"; + serviceConfig.StandardOutput = "journal+console"; }; # Allow root logins only using the SSH key that the user specified diff --git a/nixos/modules/virtualisation/ec2-metadata-fetcher.nix b/nixos/modules/virtualisation/ec2-metadata-fetcher.nix deleted file mode 100644 index e78df12c961e..000000000000 --- a/nixos/modules/virtualisation/ec2-metadata-fetcher.nix +++ /dev/null @@ -1,77 +0,0 @@ -{ curl, targetRoot, wgetExtraOptions }: -# Note: be very cautious about dependencies, each dependency grows -# the closure of the initrd. Ideally we would not even require curl, -# but there is no reasonable way to send an HTTP PUT request without -# it. Note: do not be fooled: the wget referenced in this script -# is busybox's wget, not the fully featured one with --method support. -# -# Make sure that every package you depend on here is already listed as -# a channel blocker for both the full-sized and small channels. -# Otherwise, we risk breaking user deploys in released channels. -# -# Also note: OpenStack's metadata service for its instances aims to be -# compatible with the EC2 IMDS. Where possible, try to keep the set of -# fetched metadata in sync with ./openstack-metadata-fetcher.nix . -'' - metaDir=${targetRoot}etc/ec2-metadata - mkdir -m 0755 -p "$metaDir" - rm -f "$metaDir/*" - - get_imds_token() { - # retry-delay of 1 selected to give the system a second to get going, - # but not add a lot to the bootup time - ${curl}/bin/curl \ - -v \ - --retry 3 \ - --retry-delay 1 \ - --fail \ - -X PUT \ - --connect-timeout 1 \ - -H "X-aws-ec2-metadata-token-ttl-seconds: 600" \ - http://169.254.169.254/latest/api/token - } - - preflight_imds_token() { - # retry-delay of 1 selected to give the system a second to get going, - # but not add a lot to the bootup time - ${curl}/bin/curl \ - -v \ - --retry 3 \ - --retry-delay 1 \ - --fail \ - --connect-timeout 1 \ - -H "X-aws-ec2-metadata-token: $IMDS_TOKEN" \ - http://169.254.169.254/1.0/meta-data/instance-id - } - - try=1 - while [ $try -le 3 ]; do - echo "(attempt $try/3) getting an EC2 instance metadata service v2 token..." - IMDS_TOKEN=$(get_imds_token) && break - try=$((try + 1)) - sleep 1 - done - - if [ "x$IMDS_TOKEN" == "x" ]; then - echo "failed to fetch an IMDS2v token." - fi - - try=1 - while [ $try -le 10 ]; do - echo "(attempt $try/10) validating the EC2 instance metadata service v2 token..." - preflight_imds_token && break - try=$((try + 1)) - sleep 1 - done - - echo "getting EC2 instance metadata..." - - wget_imds() { - wget ${wgetExtraOptions} --header "X-aws-ec2-metadata-token: $IMDS_TOKEN" "$@"; - } - - wget_imds -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path || true - (umask 077 && wget_imds -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data || true) - wget_imds -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname || true - wget_imds -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key || true -'' diff --git a/nixos/modules/virtualisation/ec2-metadata-fetcher.sh b/nixos/modules/virtualisation/ec2-metadata-fetcher.sh new file mode 100644 index 000000000000..9e204d45dbd8 --- /dev/null +++ b/nixos/modules/virtualisation/ec2-metadata-fetcher.sh @@ -0,0 +1,67 @@ +metaDir=/etc/ec2-metadata +mkdir -m 0755 -p "$metaDir" +rm -f "$metaDir/*" + +get_imds_token() { + # retry-delay of 1 selected to give the system a second to get going, + # but not add a lot to the bootup time + curl \ + --silent \ + --show-error \ + --retry 3 \ + --retry-delay 1 \ + --fail \ + -X PUT \ + --connect-timeout 1 \ + -H "X-aws-ec2-metadata-token-ttl-seconds: 600" \ + http://169.254.169.254/latest/api/token +} + +preflight_imds_token() { + # retry-delay of 1 selected to give the system a second to get going, + # but not add a lot to the bootup time + curl \ + --silent \ + --show-error \ + --retry 3 \ + --retry-delay 1 \ + --fail \ + --connect-timeout 1 \ + -H "X-aws-ec2-metadata-token: $IMDS_TOKEN" \ + -o /dev/null \ + http://169.254.169.254/1.0/meta-data/instance-id +} + +try=1 +while [ $try -le 3 ]; do + echo "(attempt $try/3) getting an EC2 instance metadata service v2 token..." + IMDS_TOKEN=$(get_imds_token) && break + try=$((try + 1)) + sleep 1 +done + +if [ "x$IMDS_TOKEN" == "x" ]; then + echo "failed to fetch an IMDS2v token." +fi + +try=1 +while [ $try -le 10 ]; do + echo "(attempt $try/10) validating the EC2 instance metadata service v2 token..." + preflight_imds_token && break + try=$((try + 1)) + sleep 1 +done + +echo "getting EC2 instance metadata..." + +get_imds() { + # Intentionally no --fail here, so that we proceed even if e.g. a + # 404 was returned (but we still fail if we can't reach the IMDS + # server). + curl --silent --show-error --header "X-aws-ec2-metadata-token: $IMDS_TOKEN" "$@" +} + +get_imds -o "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path +(umask 077 && get_imds -o "$metaDir/user-data" http://169.254.169.254/1.0/user-data) +get_imds -o "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname +get_imds -o "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key