mirror of
https://github.com/NixOS/nixpkgs.git
synced 2024-10-30 22:21:26 +00:00
Merge pull request #115857 from lbpdt/feature/docker-tools-layered-base-image
dockerTools.buildLayeredImage: support fromImage
This commit is contained in:
commit
561cc81ee6
@ -111,6 +111,12 @@ Create a Docker image with many of the store paths being on their own layer to i
|
||||
|
||||
*Default:* the output path's hash
|
||||
|
||||
`fromImage` _optional_
|
||||
|
||||
: The repository tarball containing the base image. It must be a valid Docker image, such as one exported by `docker save`.
|
||||
|
||||
*Default:* `null`, which can be seen as equivalent to `FROM scratch` of a `Dockerfile`.
|
||||
|
||||
`contents` _optional_
|
||||
|
||||
: Top level paths in the container. Either a single derivation, or a list of derivations.
|
||||
|
@ -161,12 +161,18 @@ import ./make-test-python.nix ({ pkgs, ... }: {
|
||||
"docker run --rm ${examples.layered-image.imageName} cat extraCommands",
|
||||
)
|
||||
|
||||
with subtest("Ensure building an image on top of a layered Docker images work"):
|
||||
with subtest("Ensure images built on top of layered Docker images work"):
|
||||
docker.succeed(
|
||||
"docker load --input='${examples.layered-on-top}'",
|
||||
"docker run --rm ${examples.layered-on-top.imageName}",
|
||||
)
|
||||
|
||||
with subtest("Ensure layered images built on top of layered Docker images work"):
|
||||
docker.succeed(
|
||||
"docker load --input='${examples.layered-on-top-layered}'",
|
||||
"docker run --rm ${examples.layered-on-top-layered.imageName}",
|
||||
)
|
||||
|
||||
|
||||
def set_of_layers(image_name):
|
||||
return set(
|
||||
@ -205,6 +211,16 @@ import ./make-test-python.nix ({ pkgs, ... }: {
|
||||
assert "FROM_CHILD=true" in env, "envvars from the child should be preserved"
|
||||
assert "LAST_LAYER=child" in env, "envvars from the child should take priority"
|
||||
|
||||
with subtest("Ensure environment variables of layered images are correctly inherited"):
|
||||
docker.succeed(
|
||||
"docker load --input='${examples.environmentVariablesLayered}'"
|
||||
)
|
||||
out = docker.succeed("docker run --rm ${examples.environmentVariablesLayered.imageName} env")
|
||||
env = out.splitlines()
|
||||
assert "FROM_PARENT=true" in env, "envvars from the parent should be preserved"
|
||||
assert "FROM_CHILD=true" in env, "envvars from the child should be preserved"
|
||||
assert "LAST_LAYER=child" in env, "envvars from the child should take priority"
|
||||
|
||||
with subtest("Ensure image with only 2 layers can be loaded"):
|
||||
docker.succeed(
|
||||
"docker load --input='${examples.two-layered-image}'"
|
||||
@ -219,6 +235,18 @@ import ./make-test-python.nix ({ pkgs, ... }: {
|
||||
"docker run bulk-layer ls /bin/hello",
|
||||
)
|
||||
|
||||
with subtest(
|
||||
"Ensure the bulk layer with a base image respects the number of maxLayers"
|
||||
):
|
||||
docker.succeed(
|
||||
"docker load --input='${pkgs.dockerTools.examples.layered-bulk-layer}'",
|
||||
# Ensure the image runs correctly
|
||||
"docker run layered-bulk-layer ls /bin/hello",
|
||||
)
|
||||
|
||||
# Ensure the image has the correct number of layers
|
||||
assert len(set_of_layers("layered-bulk-layer")) == 4
|
||||
|
||||
with subtest("Ensure correct behavior when no store is needed"):
|
||||
# This check tests that buildLayeredImage can build images that don't need a store.
|
||||
docker.succeed(
|
||||
|
@ -729,6 +729,8 @@ rec {
|
||||
name,
|
||||
# Image tag, the Nix's output hash will be used if null
|
||||
tag ? null,
|
||||
# Parent image, to append to.
|
||||
fromImage ? null,
|
||||
# Files to put on the image (a nix store path or list of paths).
|
||||
contents ? [],
|
||||
# Docker config; e.g. what command to run on the container.
|
||||
@ -791,7 +793,7 @@ rec {
|
||||
unnecessaryDrvs = [ baseJson overallClosure ];
|
||||
|
||||
conf = runCommand "${baseName}-conf.json" {
|
||||
inherit maxLayers created;
|
||||
inherit fromImage maxLayers created;
|
||||
imageName = lib.toLower name;
|
||||
passthru.imageTag =
|
||||
if tag != null
|
||||
@ -821,6 +823,27 @@ rec {
|
||||
unnecessaryDrvs}
|
||||
}
|
||||
|
||||
# Compute the number of layers that are already used by a potential
|
||||
# 'fromImage' as well as the customization layer. Ensure that there is
|
||||
# still at least one layer available to store the image contents.
|
||||
usedLayers=0
|
||||
|
||||
# subtract number of base image layers
|
||||
if [[ -n "$fromImage" ]]; then
|
||||
(( usedLayers += $(tar -xOf "$fromImage" manifest.json | jq '.[0].Layers | length') ))
|
||||
fi
|
||||
|
||||
# one layer will be taken up by the customisation layer
|
||||
(( usedLayers += 1 ))
|
||||
|
||||
if ! (( $usedLayers < $maxLayers )); then
|
||||
echo >&2 "Error: usedLayers $usedLayers layers to store 'fromImage' and" \
|
||||
"'extraCommands', but only maxLayers=$maxLayers were" \
|
||||
"allowed. At least 1 layer is required to store contents."
|
||||
exit 1
|
||||
fi
|
||||
availableLayers=$(( maxLayers - usedLayers ))
|
||||
|
||||
# Create $maxLayers worth of Docker Layers, one layer per store path
|
||||
# unless there are more paths than $maxLayers. In that case, create
|
||||
# $maxLayers-1 for the most popular layers, and smush the remainaing
|
||||
@ -838,18 +861,20 @@ rec {
|
||||
| (.[:$maxLayers-1] | map([.])) + [ .[$maxLayers-1:] ]
|
||||
| map(select(length > 0))
|
||||
' \
|
||||
--argjson maxLayers "$(( maxLayers - 1 ))" # one layer will be taken up by the customisation layer
|
||||
--argjson maxLayers "$availableLayers"
|
||||
)"
|
||||
|
||||
cat ${baseJson} | jq '
|
||||
. + {
|
||||
"store_dir": $store_dir,
|
||||
"from_image": $from_image,
|
||||
"store_layers": $store_layers,
|
||||
"customisation_layer", $customisation_layer,
|
||||
"repo_tag": $repo_tag,
|
||||
"created": $created
|
||||
}
|
||||
' --arg store_dir "${storeDir}" \
|
||||
--argjson from_image ${if fromImage == null then "null" else "'\"${fromImage}\"'"} \
|
||||
--argjson store_layers "$store_layers" \
|
||||
--arg customisation_layer ${customisationLayer} \
|
||||
--arg repo_tag "$imageName:$imageTag" \
|
||||
|
@ -188,7 +188,25 @@ rec {
|
||||
};
|
||||
};
|
||||
|
||||
# 12. example of running something as root on top of a parent image
|
||||
# 12 Create a layered image on top of a layered image
|
||||
layered-on-top-layered = pkgs.dockerTools.buildLayeredImage {
|
||||
name = "layered-on-top-layered";
|
||||
tag = "latest";
|
||||
fromImage = layered-image;
|
||||
extraCommands = ''
|
||||
mkdir ./example-output
|
||||
chmod 777 ./example-output
|
||||
'';
|
||||
config = {
|
||||
Env = [ "PATH=${pkgs.coreutils}/bin/" ];
|
||||
WorkingDir = "/example-output";
|
||||
Cmd = [
|
||||
"${pkgs.bash}/bin/bash" "-c" "echo hello > foo; cat foo"
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
# 13. example of running something as root on top of a parent image
|
||||
# Regression test related to PR #52109
|
||||
runAsRootParentImage = buildImage {
|
||||
name = "runAsRootParentImage";
|
||||
@ -197,7 +215,7 @@ rec {
|
||||
fromImage = bash;
|
||||
};
|
||||
|
||||
# 13. example of 3 layers images This image is used to verify the
|
||||
# 14. example of 3 layers images This image is used to verify the
|
||||
# order of layers is correct.
|
||||
# It allows to validate
|
||||
# - the layer of parent are below
|
||||
@ -235,23 +253,23 @@ rec {
|
||||
'';
|
||||
};
|
||||
|
||||
# 14. Environment variable inheritance.
|
||||
# 15. Environment variable inheritance.
|
||||
# Child image should inherit parents environment variables,
|
||||
# optionally overriding them.
|
||||
environmentVariables = let
|
||||
parent = pkgs.dockerTools.buildImage {
|
||||
name = "parent";
|
||||
tag = "latest";
|
||||
config = {
|
||||
Env = [
|
||||
"FROM_PARENT=true"
|
||||
"LAST_LAYER=parent"
|
||||
];
|
||||
};
|
||||
environmentVariablesParent = pkgs.dockerTools.buildImage {
|
||||
name = "parent";
|
||||
tag = "latest";
|
||||
config = {
|
||||
Env = [
|
||||
"FROM_PARENT=true"
|
||||
"LAST_LAYER=parent"
|
||||
];
|
||||
};
|
||||
in pkgs.dockerTools.buildImage {
|
||||
};
|
||||
|
||||
environmentVariables = pkgs.dockerTools.buildImage {
|
||||
name = "child";
|
||||
fromImage = parent;
|
||||
fromImage = environmentVariablesParent;
|
||||
tag = "latest";
|
||||
contents = [ pkgs.coreutils ];
|
||||
config = {
|
||||
@ -262,14 +280,27 @@ rec {
|
||||
};
|
||||
};
|
||||
|
||||
# 15. Create another layered image, for comparing layers with image 10.
|
||||
environmentVariablesLayered = pkgs.dockerTools.buildLayeredImage {
|
||||
name = "child";
|
||||
fromImage = environmentVariablesParent;
|
||||
tag = "latest";
|
||||
contents = [ pkgs.coreutils ];
|
||||
config = {
|
||||
Env = [
|
||||
"FROM_CHILD=true"
|
||||
"LAST_LAYER=child"
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
# 16. Create another layered image, for comparing layers with image 10.
|
||||
another-layered-image = pkgs.dockerTools.buildLayeredImage {
|
||||
name = "another-layered-image";
|
||||
tag = "latest";
|
||||
config.Cmd = [ "${pkgs.hello}/bin/hello" ];
|
||||
};
|
||||
|
||||
# 16. Create a layered image with only 2 layers
|
||||
# 17. Create a layered image with only 2 layers
|
||||
two-layered-image = pkgs.dockerTools.buildLayeredImage {
|
||||
name = "two-layered-image";
|
||||
tag = "latest";
|
||||
@ -278,7 +309,7 @@ rec {
|
||||
maxLayers = 2;
|
||||
};
|
||||
|
||||
# 17. Create a layered image with more packages than max layers.
|
||||
# 18. Create a layered image with more packages than max layers.
|
||||
# coreutils and hello are part of the same layer
|
||||
bulk-layer = pkgs.dockerTools.buildLayeredImage {
|
||||
name = "bulk-layer";
|
||||
@ -289,7 +320,19 @@ rec {
|
||||
maxLayers = 2;
|
||||
};
|
||||
|
||||
# 18. Create a "layered" image without nix store layers. This is not
|
||||
# 19. Create a layered image with a base image and more packages than max
|
||||
# layers. coreutils and hello are part of the same layer
|
||||
layered-bulk-layer = pkgs.dockerTools.buildLayeredImage {
|
||||
name = "layered-bulk-layer";
|
||||
tag = "latest";
|
||||
fromImage = two-layered-image;
|
||||
contents = with pkgs; [
|
||||
coreutils hello
|
||||
];
|
||||
maxLayers = 4;
|
||||
};
|
||||
|
||||
# 20. Create a "layered" image without nix store layers. This is not
|
||||
# recommended, but can be useful for base images in rare cases.
|
||||
no-store-paths = pkgs.dockerTools.buildLayeredImage {
|
||||
name = "no-store-paths";
|
||||
@ -321,7 +364,7 @@ rec {
|
||||
};
|
||||
};
|
||||
|
||||
# 19. Support files in the store on buildLayeredImage
|
||||
# 21. Support files in the store on buildLayeredImage
|
||||
# See: https://github.com/NixOS/nixpkgs/pull/91084#issuecomment-653496223
|
||||
filesInStore = pkgs.dockerTools.buildLayeredImageWithNixDb {
|
||||
name = "file-in-store";
|
||||
@ -341,7 +384,7 @@ rec {
|
||||
};
|
||||
};
|
||||
|
||||
# 20. Ensure that setting created to now results in a date which
|
||||
# 22. Ensure that setting created to now results in a date which
|
||||
# isn't the epoch + 1 for layered images.
|
||||
unstableDateLayered = pkgs.dockerTools.buildLayeredImage {
|
||||
name = "unstable-date-layered";
|
||||
|
@ -33,6 +33,7 @@ function does all this.
|
||||
|
||||
import io
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
import hashlib
|
||||
@ -126,10 +127,85 @@ class ExtractChecksum:
|
||||
return (self._digest.hexdigest(), self._size)
|
||||
|
||||
|
||||
FromImage = namedtuple("FromImage", ["tar", "manifest_json", "image_json"])
|
||||
# Some metadata for a layer
|
||||
LayerInfo = namedtuple("LayerInfo", ["size", "checksum", "path", "paths"])
|
||||
|
||||
|
||||
def load_from_image(from_image_str):
|
||||
"""
|
||||
Loads the given base image, if any.
|
||||
|
||||
from_image_str: Path to the base image archive.
|
||||
|
||||
Returns: A 'FromImage' object with references to the loaded base image,
|
||||
or 'None' if no base image was provided.
|
||||
"""
|
||||
if from_image_str is None:
|
||||
return None
|
||||
|
||||
base_tar = tarfile.open(from_image_str)
|
||||
|
||||
manifest_json_tarinfo = base_tar.getmember("manifest.json")
|
||||
with base_tar.extractfile(manifest_json_tarinfo) as f:
|
||||
manifest_json = json.load(f)
|
||||
|
||||
image_json_tarinfo = base_tar.getmember(manifest_json[0]["Config"])
|
||||
with base_tar.extractfile(image_json_tarinfo) as f:
|
||||
image_json = json.load(f)
|
||||
|
||||
return FromImage(base_tar, manifest_json, image_json)
|
||||
|
||||
|
||||
def add_base_layers(tar, from_image):
|
||||
"""
|
||||
Adds the layers from the given base image to the final image.
|
||||
|
||||
tar: 'tarfile.TarFile' object for new layers to be added to.
|
||||
from_image: 'FromImage' object with references to the loaded base image.
|
||||
"""
|
||||
if from_image is None:
|
||||
print("No 'fromImage' provided", file=sys.stderr)
|
||||
return []
|
||||
|
||||
layers = from_image.manifest_json[0]["Layers"]
|
||||
checksums = from_image.image_json["rootfs"]["diff_ids"]
|
||||
layers_checksums = zip(layers, checksums)
|
||||
|
||||
for num, (layer, checksum) in enumerate(layers_checksums, start=1):
|
||||
layer_tarinfo = from_image.tar.getmember(layer)
|
||||
checksum = re.sub(r"^sha256:", "", checksum)
|
||||
|
||||
tar.addfile(layer_tarinfo, from_image.tar.extractfile(layer_tarinfo))
|
||||
path = layer_tarinfo.path
|
||||
size = layer_tarinfo.size
|
||||
|
||||
print("Adding base layer", num, "from", path, file=sys.stderr)
|
||||
yield LayerInfo(size=size, checksum=checksum, path=path, paths=[path])
|
||||
|
||||
from_image.tar.close()
|
||||
|
||||
|
||||
def overlay_base_config(from_image, final_config):
|
||||
"""
|
||||
Overlays the final image 'config' JSON on top of selected defaults from the
|
||||
base image 'config' JSON.
|
||||
|
||||
from_image: 'FromImage' object with references to the loaded base image.
|
||||
final_config: 'dict' object of the final image 'config' JSON.
|
||||
"""
|
||||
if from_image is None:
|
||||
return final_config
|
||||
|
||||
base_config = from_image.image_json["config"]
|
||||
|
||||
# Preserve environment from base image
|
||||
final_env = base_config.get("Env", []) + final_config.get("Env", [])
|
||||
if final_env:
|
||||
final_config["Env"] = final_env
|
||||
return final_config
|
||||
|
||||
|
||||
def add_layer_dir(tar, paths, store_dir, mtime):
|
||||
"""
|
||||
Appends given store paths to a TarFile object as a new layer.
|
||||
@ -248,17 +324,21 @@ def main():
|
||||
mtime = int(created.timestamp())
|
||||
store_dir = conf["store_dir"]
|
||||
|
||||
from_image = load_from_image(conf["from_image"])
|
||||
|
||||
with tarfile.open(mode="w|", fileobj=sys.stdout.buffer) as tar:
|
||||
layers = []
|
||||
for num, store_layer in enumerate(conf["store_layers"]):
|
||||
print(
|
||||
"Creating layer", num,
|
||||
"from paths:", store_layer,
|
||||
file=sys.stderr)
|
||||
layers.extend(add_base_layers(tar, from_image))
|
||||
|
||||
start = len(layers) + 1
|
||||
for num, store_layer in enumerate(conf["store_layers"], start=start):
|
||||
print("Creating layer", num, "from paths:", store_layer,
|
||||
file=sys.stderr)
|
||||
info = add_layer_dir(tar, store_layer, store_dir, mtime=mtime)
|
||||
layers.append(info)
|
||||
|
||||
print("Creating the customisation layer...", file=sys.stderr)
|
||||
print("Creating layer", len(layers) + 1, "with customisation...",
|
||||
file=sys.stderr)
|
||||
layers.append(
|
||||
add_customisation_layer(
|
||||
tar,
|
||||
@ -273,7 +353,7 @@ def main():
|
||||
"created": datetime.isoformat(created),
|
||||
"architecture": conf["architecture"],
|
||||
"os": "linux",
|
||||
"config": conf["config"],
|
||||
"config": overlay_base_config(from_image, conf["config"]),
|
||||
"rootfs": {
|
||||
"diff_ids": [f"sha256:{layer.checksum}" for layer in layers],
|
||||
"type": "layers",
|
||||
|
Loading…
Reference in New Issue
Block a user