Merge pull request #32248 from awakesecurity/parnell/fetchdocker

Support fetching docker images from V2 registries

nixos/release.nix

@@ -274,6 +274,7 @@ in rec {
   tests.hibernate = callTest tests/hibernate.nix {};
   tests.home-assistant = callTest tests/home-assistant.nix { };
   tests.hound = callTest tests/hound.nix {};
+  tests.hocker-fetchdocker = callTest tests/hocker-fetchdocker {};
   tests.i3wm = callTest tests/i3wm.nix {};
   tests.initrd-network-ssh = callTest tests/initrd-network-ssh {};
   tests.installer = callSubTests tests/installer.nix {};

nixos/tests/hocker-fetchdocker/default.nix

@@ -0,0 +1,15 @@
+import ../make-test.nix ({ pkgs, ...} : {
+  name = "test-hocker-fetchdocker";
+  meta = with pkgs.stdenv.lib.maintainers; {
+    maintainers = [ ixmatus ];
+  };
+
+  machine = import ./machine.nix;
+
+  testScript = ''
+    startAll;
+
+    $machine->waitForUnit("sockets.target");
+    $machine->waitUntilSucceeds("docker run registry-1.docker.io/v2/library/hello-world:latest");
+  '';
+})

nixos/tests/hocker-fetchdocker/hello-world-container.nix

@@ -0,0 +1,19 @@
+{ fetchDockerConfig, fetchDockerLayer, fetchdocker }:
+fetchdocker rec {
+    name = "hello-world";
+    registry = "https://registry-1.docker.io/v2/";
+    repository = "library";
+    imageName = "hello-world";
+    tag = "latest";
+    imageConfig = fetchDockerConfig {
+      inherit tag registry repository imageName;
+      sha256 = "1ivbd23hyindkahzfw4kahgzi6ibzz2ablmgsz6340vc6qr1gagj";
+    };
+    imageLayers = let
+      layer0 = fetchDockerLayer {
+        inherit registry repository imageName;
+        layerDigest = "ca4f61b1923c10e9eb81228bd46bee1dfba02b9c7dac1844527a734752688ede";
+        sha256 = "1plfd194fwvsa921ib3xkhms1yqxxrmx92r2h7myj41wjaqn2kya";
+      };
+      in [ layer0 ];
+  }

nixos/tests/hocker-fetchdocker/machine.nix

@@ -0,0 +1,26 @@
+{ config, pkgs, ... }:
+{ nixpkgs.config.packageOverrides = pkgs': {
+    hello-world-container = pkgs'.callPackage ./hello-world-container.nix { };
+  };
+
+  virtualisation.docker = {
+    enable  = true;
+    package = pkgs.docker;
+  };
+
+  systemd.services.docker-load-fetchdocker-image = {
+    description = "Docker load hello-world-container";
+    wantedBy    = [ "multi-user.target" ];
+    wants       = [ "docker.service" "local-fs.target" ];
+    after       = [ "docker.service" "local-fs.target" ];
+
+    script = ''
+      ${pkgs.hello-world-container}/compositeImage.sh | ${pkgs.docker}/bin/docker load
+    '';
+
+    serviceConfig = {
+      Type = "oneshot";
+    };
+  };
+}
+

pkgs/build-support/fetchdocker/credentials.nix

@@ -0,0 +1,38 @@
+# We provide three paths to get the credentials into the builder's
+# environment:
+#
+# 1. Via impureEnvVars. This method is difficult for multi-user Nix
+#    installations (but works very well for single-user Nix
+#    installations!) because it requires setting the environment
+#    variables on the nix-daemon which is either complicated or unsafe
+#    (i.e: configuring via Nix means the secrets will be persisted
+#    into the store)
+#
+# 2. If the DOCKER_CREDENTIALS key with a path to a credentials file
+#    is added to the NIX_PATH (usually via the '-I ' argument to most
+#    Nix tools) then an attempt will be made to read credentials from
+#    it. The semantics are simple, the file should contain two lines
+#    for the username and password based authentication:
+#
+# $ cat ./credentials-file.txt
+# DOCKER_USER=myusername
+# DOCKER_PASS=mypassword
+#
+#    ... and a single line for the token based authentication:
+#
+# $ cat ./credentials-file.txt
+# DOCKER_TOKEN=mytoken
+#
+# 3. A credential file at /etc/nix-docker-credentials.txt with the
+#    same format as the file described in #2 can also be used to
+#    communicate credentials to the builder. This is necessary for
+#    situations (like Hydra) where you cannot customize the NIX_PATH
+#    given to the nix-build invocation to provide it with the
+#    DOCKER_CREDENTIALS path
+let
+  pathParts =
+   (builtins.filter
+    ({path, prefix}: "DOCKER_CREDENTIALS" == prefix)
+    builtins.nixPath);
+in
+  if (pathParts != []) then (builtins.head pathParts).path else ""

pkgs/build-support/fetchdocker/default.nix

@@ -0,0 +1,61 @@
+{ stdenv, lib, coreutils, bash, gnutar, jq, writeText }:
+let
+  stripScheme =
+    builtins.replaceStrings [ "https://" "http://" ] [ "" "" ];
+  stripNixStore =
+    s: lib.removePrefix "/nix/store/" s;
+in
+{ name
+, registry         ? "https://registry-1.docker.io/v2/"
+, repository       ? "library"
+, imageName
+, tag
+, imageLayers
+, imageConfig
+, image            ? "${stripScheme registry}/${repository}/${imageName}:${tag}"
+}:
+
+# Make sure there are *no* slashes in the repository or container
+# names since we use these to make the output derivation name for the
+# nix-store path.
+assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters repository);
+assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters imageName);
+
+let
+  # Abuse `builtins.toPath` to collapse possible double slashes
+  repoTag0 = builtins.toString (builtins.toPath "/${stripScheme registry}/${repository}/${imageName}");
+  repoTag1 = lib.removePrefix "/" repoTag0;
+
+  layers = builtins.map stripNixStore imageLayers;
+
+  manifest =
+    writeText "manifest.json" (builtins.toJSON [
+      { Config   = stripNixStore imageConfig;
+        Layers   = layers;
+        RepoTags = [ "${repoTag1}:${tag}" ];
+      }]);
+
+  repositories =
+    writeText "repositories" (builtins.toJSON {
+      "${repoTag1}" = {
+        "${tag}" = lib.last layers;
+      };
+    });
+
+  imageFileStorePaths =
+    writeText "imageFileStorePaths.txt"
+      (lib.concatStringsSep "\n" ((lib.unique imageLayers) ++ [imageConfig]));
+in
+stdenv.mkDerivation {
+  builder     = ./fetchdocker-builder.sh;
+  buildInputs = [ coreutils ];
+  preferLocalBuild = true;
+
+  inherit name imageName repository tag;
+  inherit bash gnutar manifest repositories;
+  inherit imageFileStorePaths;
+
+  passthru = {
+    inherit image;
+  };
+}

pkgs/build-support/fetchdocker/fetchDockerConfig.nix

@@ -0,0 +1,13 @@
+pkgargs@{ stdenv, lib, haskellPackages, writeText, gawk }:
+let
+  generic-fetcher =
+    import ./generic-fetcher.nix pkgargs;
+in
+
+args@{ repository ? "library", imageName, tag, ... }:
+
+generic-fetcher ({
+  fetcher = "hocker-config";
+  name    = "${repository}_${imageName}_${tag}-config.json";
+  tag     = "unused";
+} // args)

pkgs/build-support/fetchdocker/fetchDockerLayer.nix

@@ -0,0 +1,13 @@
+pkgargs@{ stdenv, lib, haskellPackages, writeText, gawk }:
+let
+  generic-fetcher =
+    import ./generic-fetcher.nix pkgargs;
+in
+
+args@{ layerDigest, ... }:
+
+generic-fetcher ({
+  fetcher = "hocker-layer";
+  name    = "docker-layer-${layerDigest}.tar.gz";
+  tag     = "unused";
+} // args)

pkgs/build-support/fetchdocker/fetchdocker-builder.sh

@@ -0,0 +1,28 @@
+source "${stdenv}/setup"
+header "exporting ${repository}/${imageName} (tag: ${tag}) into ${out}"
+mkdir -p "${out}"
+
+cat <<EOF > "${out}/compositeImage.sh"
+#! ${bash}/bin/bash
+#
+# Create a tar archive of a docker image's layers, docker image config
+# json, manifest.json, and repositories json; this streams directly to
+# stdout and is intended to be used in concert with docker load, i.e:
+# 
+# ${out}/compositeImage.sh | docker load
+
+# The first character follow the 's' command for sed becomes the
+# delimiter sed will use; this makes the transformation regex easy to
+# read. We feed tar a file listing the files we want in the archive,
+# because the paths are absolute and docker load wants them flattened in
+# the archive, we need to transform all of the paths going in by
+# stripping everything *including* the last solidus so that we end up
+# with the basename of the path.
+${gnutar}/bin/tar \
+  --transform='s=.*/==' \
+  --transform="s=.*-manifest.json=manifest.json=" \
+  --transform="s=.*-repositories=repositories=" \
+  -c "${manifest}" "${repositories}" -T "${imageFileStorePaths}"
+EOF
+chmod +x "${out}/compositeImage.sh"
+stopNest

pkgs/build-support/fetchdocker/generic-fetcher.nix

@@ -0,0 +1,97 @@
+{ stdenv, lib, haskellPackages, writeText, gawk }:
+let
+  awk                   = "${gawk}/bin/awk";
+  dockerCredentialsFile = import ./credentials.nix;
+  stripScheme           =
+    builtins.replaceStrings [ "https://" "http://" ] [ "" "" ];
+in
+{ fetcher
+, name
+ , registry    ? "https://registry-1.docker.io/v2/"
+ , repository  ? "library"
+ , imageName
+ , sha256
+ , tag         ? ""
+ , layerDigest ? ""
+}:
+
+# There must be no slashes in the repository or container names since
+# we use these to make the output derivation name for the nix store
+# path
+assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters repository);
+assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters imageName);
+
+# Only allow hocker-config and hocker-layer as fetchers for now
+assert (builtins.elem fetcher ["hocker-config" "hocker-layer"]);
+
+# If layerDigest is non-empty then it must not have a 'sha256:' prefix!
+assert
+  (if layerDigest != ""
+   then !lib.hasPrefix "sha256:" layerDigest
+   else true);
+
+let
+  layerDigestFlag =
+    lib.optionalString (layerDigest != "") "--layer ${layerDigest}";
+in
+stdenv.mkDerivation {
+  inherit name;
+  builder = writeText "${fetcher}-builder.sh" ''
+    source "$stdenv/setup"
+    header "${fetcher} exporting to $out"
+
+    declare -A creds
+
+    # This is a hack for Hydra since we have no way of adding values
+    # to the NIX_PATH for Hydra jobsets!!
+    staticCredentialsFile="/etc/nix-docker-credentials.txt"
+    if [ ! -f "$dockerCredentialsFile" -a -f "$staticCredentialsFile" ]; then
+      echo "credentials file not set, falling back on static credentials file at: $staticCredentialsFile"
+      dockerCredentialsFile=$staticCredentialsFile
+    fi
+
+    if [ -f "$dockerCredentialsFile" ]; then
+      header "using credentials from $dockerCredentialsFile"
+
+      CREDSFILE=$(cat "$dockerCredentialsFile")
+      creds[token]=$(${awk} -F'=' '/DOCKER_TOKEN/ {print $2}' <<< "$CREDSFILE" | head -n1)
+
+      # Prefer DOCKER_TOKEN over the username and password
+      # authentication method
+      if [ -z "''${creds[token]}" ]; then
+        creds[user]=$(${awk} -F'=' '/DOCKER_USER/  {print $2}' <<< "$CREDSFILE" | head -n1)
+        creds[pass]=$(${awk} -F'=' '/DOCKER_PASS/  {print $2}' <<< "$CREDSFILE" | head -n1)
+      fi
+    fi
+
+    # These variables will be filled in first by the impureEnvVars, if
+    # those variables are empty then they will default to the
+    # credentials that may have been read in from the 'DOCKER_CREDENTIALS'
+    DOCKER_USER="''${DOCKER_USER:-''${creds[user]}}"
+    DOCKER_PASS="''${DOCKER_PASS:-''${creds[pass]}}"
+    DOCKER_TOKEN="''${DOCKER_TOKEN:-''${creds[token]}}"
+
+    ${fetcher} --out="$out" \
+      ''${registry:+--registry "$registry"} \
+      ''${DOCKER_USER:+--username "$DOCKER_USER"} \
+      ''${DOCKER_PASS:+--password "$DOCKER_PASS"} \
+      ''${DOCKER_TOKEN:+--token "$DOCKER_TOKEN"} \
+      ${layerDigestFlag} \
+      "${repository}/${imageName}" \
+      "${tag}"
+
+    stopNest
+  '';
+
+  buildInputs = [ haskellPackages.hocker ];
+
+  outputHashAlgo = "sha256";
+  outputHashMode = "flat";
+  outputHash = sha256;
+
+  preferLocalBuild = true;
+
+  impureEnvVars = [ "DOCKER_USER" "DOCKER_PASS" "DOCKER_TOKEN" ];
+
+  inherit registry dockerCredentialsFile;
+}

pkgs/top-level/all-packages.nix

@@ -160,6 +160,12 @@ with pkgs;
 
   fetchdarcs = callPackage ../build-support/fetchdarcs { };
 
+  fetchdocker = callPackage ../build-support/fetchdocker { };
+
+  fetchDockerConfig = callPackage ../build-support/fetchdocker/fetchDockerConfig.nix { };
+
+  fetchDockerLayer = callPackage ../build-support/fetchdocker/fetchDockerLayer.nix { };
+
   fetchfossil = callPackage ../build-support/fetchfossil { };
 
   fetchgit = callPackage ../build-support/fetchgit {