Merge pull request #14018 from kampfschlaefer/feature/hostbridge_and_ipv6_for_containers

containers: hostbridge and IPv6
This commit is contained in:
Franz Pletz 2016-04-24 20:33:46 +02:00
commit 8cca66f774
6 changed files with 398 additions and 181 deletions

View File

@ -28,14 +28,23 @@ let
# Initialise the container side of the veth pair.
if [ "$PRIVATE_NETWORK" = 1 ]; then
ip link set host0 name eth0
ip link set dev eth0 up
if [ -n "$LOCAL_ADDRESS" ]; then
ip addr add $LOCAL_ADDRESS dev eth0
fi
if [ -n "$LOCAL_ADDRESS6" ]; then
ip -6 addr add $LOCAL_ADDRESS6 dev eth0
fi
if [ -n "$HOST_ADDRESS" ]; then
ip route add $HOST_ADDRESS dev eth0
ip route add default via $HOST_ADDRESS
fi
if [ -n "$LOCAL_ADDRESS" ]; then
ip addr add $LOCAL_ADDRESS dev eth0
if [ -n "$HOST_ADDRESS6" ]; then
ip -6 route add $HOST_ADDRESS6 dev eth0
ip -6 route add default via $HOST_ADDRESS6
fi
fi
@ -48,7 +57,7 @@ let
system = config.nixpkgs.system;
bindMountOpts = { name, config, ... }: {
options = {
mountPoint = mkOption {
example = "/mnt/usb";
@ -68,13 +77,13 @@ let
description = "Determine whether the mounted path will be accessed in read-only mode.";
};
};
config = {
mountPoint = mkDefault name;
};
};
mkBindFlag = d:
let flagPrefix = if d.isReadOnly then " --bind-ro=" else " --bind=";
mountstr = if d.hostPath != null then "${d.hostPath}:${d.mountPoint}" else "${d.mountPoint}";
@ -142,12 +151,33 @@ in
'';
};
hostBridge = mkOption {
type = types.nullOr types.string;
default = null;
example = "br0";
description = ''
Put the host-side of the veth-pair into the named bridge.
Only one of hostAddress* or hostBridge can be given.
'';
};
hostAddress = mkOption {
type = types.nullOr types.str;
default = null;
example = "10.231.136.1";
description = ''
The IPv4 address assigned to the host interface.
(Not used when hostBridge is set.)
'';
};
hostAddress6 = mkOption {
type = types.nullOr types.string;
default = null;
example = "fc00::1";
description = ''
The IPv6 address assigned to the host interface.
(Not used when hostBridge is set.)
'';
};
@ -161,6 +191,16 @@ in
'';
};
localAddress6 = mkOption {
type = types.nullOr types.string;
default = null;
example = "fc00::2";
description = ''
The IPv6 address assigned to <literal>eth0</literal>
in the container.
'';
};
interfaces = mkOption {
type = types.listOf types.string;
default = [];
@ -185,7 +225,7 @@ in
example = { "/home" = { hostPath = "/home/alice";
isReadOnly = false; };
};
description =
''
An extra list of directories that is bound to the container.
@ -238,154 +278,180 @@ in
};
config = mkIf (config.boot.enableContainers) {
config = mkIf (config.boot.enableContainers) (let
systemd.services."container@" =
{ description = "Container '%i'";
unit = {
description = "Container '%i'";
unitConfig.RequiresMountsFor = [ "/var/lib/containers/%i" ];
unitConfig.RequiresMountsFor = [ "/var/lib/containers/%i" ];
path = [ pkgs.iproute ];
path = [ pkgs.iproute ];
environment.INSTANCE = "%i";
environment.root = "/var/lib/containers/%i";
environment.INSTANCE = "%i";
environment.root = "/var/lib/containers/%i";
preStart =
''
# Clean up existing machined registration and interfaces.
machinectl terminate "$INSTANCE" 2> /dev/null || true
preStart =
''
# Clean up existing machined registration and interfaces.
machinectl terminate "$INSTANCE" 2> /dev/null || true
if [ "$PRIVATE_NETWORK" = 1 ]; then
ip link del dev "ve-$INSTANCE" 2> /dev/null || true
if [ "$PRIVATE_NETWORK" = 1 ]; then
ip link del dev "ve-$INSTANCE" 2> /dev/null || true
ip link del dev "vb-$INSTANCE" 2> /dev/null || true
fi
'';
script =
''
mkdir -p -m 0755 "$root/etc" "$root/var/lib"
mkdir -p -m 0700 "$root/var/lib/private" "$root/root" /run/containers
if ! [ -e "$root/etc/os-release" ]; then
touch "$root/etc/os-release"
fi
mkdir -p -m 0755 \
"/nix/var/nix/profiles/per-container/$INSTANCE" \
"/nix/var/nix/gcroots/per-container/$INSTANCE"
cp --remove-destination /etc/resolv.conf "$root/etc/resolv.conf"
if [ "$PRIVATE_NETWORK" = 1 ]; then
extraFlags+=" --network-veth"
if [ -n "$HOST_BRIDGE" ]; then
extraFlags+=" --network-bridge=$HOST_BRIDGE"
fi
fi
for iface in $INTERFACES; do
extraFlags+=" --network-interface=$iface"
done
if [ "$PRIVATE_NETWORK" = 1 ]; then
ip link del dev "ve-$INSTANCE" 2> /dev/null || true
for iface in $MACVLANS; do
extraFlags+=" --network-macvlan=$iface"
done
# If the host is 64-bit and the container is 32-bit, add a
# --personality flag.
${optionalString (config.nixpkgs.system == "x86_64-linux") ''
if [ "$(< ''${SYSTEM_PATH:-/nix/var/nix/profiles/per-container/$INSTANCE/system}/system)" = i686-linux ]; then
extraFlags+=" --personality=x86"
fi
'';
script =
''
mkdir -p -m 0755 "$root/etc" "$root/var/lib"
mkdir -p -m 0700 "$root/var/lib/private" "$root/root" /run/containers
if ! [ -e "$root/etc/os-release" ]; then
touch "$root/etc/os-release"
fi
mkdir -p -m 0755 \
"/nix/var/nix/profiles/per-container/$INSTANCE" \
"/nix/var/nix/gcroots/per-container/$INSTANCE"
cp --remove-destination /etc/resolv.conf "$root/etc/resolv.conf"
if [ "$PRIVATE_NETWORK" = 1 ]; then
extraFlags+=" --network-veth"
fi
for iface in $INTERFACES; do
extraFlags+=" --network-interface=$iface"
done
for iface in $MACVLANS; do
extraFlags+=" --network-macvlan=$iface"
done
# If the host is 64-bit and the container is 32-bit, add a
# --personality flag.
${optionalString (config.nixpkgs.system == "x86_64-linux") ''
if [ "$(< ''${SYSTEM_PATH:-/nix/var/nix/profiles/per-container/$INSTANCE/system}/system)" = i686-linux ]; then
extraFlags+=" --personality=x86"
fi
''}
''}
# Run systemd-nspawn without startup notification (we'll
# wait for the container systemd to signal readiness).
EXIT_ON_REBOOT=1 NOTIFY_SOCKET= \
exec ${config.systemd.package}/bin/systemd-nspawn \
--keep-unit \
-M "$INSTANCE" -D "$root" $extraFlags \
$EXTRA_NSPAWN_FLAGS \
--bind-ro=/nix/store \
--bind-ro=/nix/var/nix/db \
--bind-ro=/nix/var/nix/daemon-socket \
--bind=/run/systemd/notify:/var/lib/private/host-notify \
--bind="/nix/var/nix/profiles/per-container/$INSTANCE:/nix/var/nix/profiles" \
--bind="/nix/var/nix/gcroots/per-container/$INSTANCE:/nix/var/nix/gcroots" \
--setenv PRIVATE_NETWORK="$PRIVATE_NETWORK" \
--setenv HOST_ADDRESS="$HOST_ADDRESS" \
--setenv LOCAL_ADDRESS="$LOCAL_ADDRESS" \
--setenv PATH="$PATH" \
${containerInit} "''${SYSTEM_PATH:-/nix/var/nix/profiles/system}/init"
'';
# Run systemd-nspawn without startup notification (we'll
# wait for the container systemd to signal readiness).
EXIT_ON_REBOOT=1 NOTIFY_SOCKET= \
exec ${config.systemd.package}/bin/systemd-nspawn \
--keep-unit \
-M "$INSTANCE" -D "$root" $extraFlags \
$EXTRA_NSPAWN_FLAGS \
--bind-ro=/nix/store \
--bind-ro=/nix/var/nix/db \
--bind-ro=/nix/var/nix/daemon-socket \
--bind=/run/systemd/notify:/var/lib/private/host-notify \
--bind="/nix/var/nix/profiles/per-container/$INSTANCE:/nix/var/nix/profiles" \
--bind="/nix/var/nix/gcroots/per-container/$INSTANCE:/nix/var/nix/gcroots" \
--setenv PRIVATE_NETWORK="$PRIVATE_NETWORK" \
--setenv HOST_BRIDGE="$HOST_BRIDGE" \
--setenv HOST_ADDRESS="$HOST_ADDRESS" \
--setenv LOCAL_ADDRESS="$LOCAL_ADDRESS" \
--setenv HOST_ADDRESS6="$HOST_ADDRESS6" \
--setenv LOCAL_ADDRESS6="$LOCAL_ADDRESS6" \
--setenv PATH="$PATH" \
${containerInit} "''${SYSTEM_PATH:-/nix/var/nix/profiles/system}/init"
'';
postStart =
''
if [ "$PRIVATE_NETWORK" = 1 ]; then
postStart =
''
if [ "$PRIVATE_NETWORK" = 1 ]; then
if [ -z "$HOST_BRIDGE" ]; then
ifaceHost=ve-$INSTANCE
ip link set dev $ifaceHost up
if [ -n "$HOST_ADDRESS" ]; then
ip addr add $HOST_ADDRESS dev $ifaceHost
fi
if [ -n "$HOST_ADDRESS6" ]; then
ip -6 addr add $HOST_ADDRESS6 dev $ifaceHost
fi
if [ -n "$LOCAL_ADDRESS" ]; then
ip route add $LOCAL_ADDRESS dev $ifaceHost
fi
if [ -n "$LOCAL_ADDRESS6" ]; then
ip -6 route add $LOCAL_ADDRESS6 dev $ifaceHost
fi
fi
fi
# Get the leader PID so that we can signal it in
# preStop. We can't use machinectl there because D-Bus
# might be shutting down. FIXME: in systemd 219 we can
# just signal systemd-nspawn to do a clean shutdown.
machinectl show "$INSTANCE" | sed 's/Leader=\(.*\)/\1/;t;d' > "/run/containers/$INSTANCE.pid"
'';
# Get the leader PID so that we can signal it in
# preStop. We can't use machinectl there because D-Bus
# might be shutting down. FIXME: in systemd 219 we can
# just signal systemd-nspawn to do a clean shutdown.
machinectl show "$INSTANCE" | sed 's/Leader=\(.*\)/\1/;t;d' > "/run/containers/$INSTANCE.pid"
'';
preStop =
preStop =
''
pid="$(cat /run/containers/$INSTANCE.pid)"
if [ -n "$pid" ]; then
kill -RTMIN+4 "$pid"
fi
rm -f "/run/containers/$INSTANCE.pid"
'';
restartIfChanged = false;
serviceConfig = {
ExecReload = pkgs.writeScript "reload-container"
''
pid="$(cat /run/containers/$INSTANCE.pid)"
if [ -n "$pid" ]; then
kill -RTMIN+4 "$pid"
fi
rm -f "/run/containers/$INSTANCE.pid"
#! ${pkgs.stdenv.shell} -e
${nixos-container}/bin/nixos-container run "$INSTANCE" -- \
bash --login -c "''${SYSTEM_PATH:-/nix/var/nix/profiles/system}/bin/switch-to-configuration test"
'';
restartIfChanged = false;
#reloadIfChanged = true; # FIXME
SyslogIdentifier = "container %i";
serviceConfig = {
ExecReload = pkgs.writeScript "reload-container"
''
#! ${pkgs.stdenv.shell} -e
${nixos-container}/bin/nixos-container run "$INSTANCE" -- \
bash --login -c "''${SYSTEM_PATH:-/nix/var/nix/profiles/system}/bin/switch-to-configuration test"
'';
EnvironmentFile = "-/etc/containers/%i.conf";
SyslogIdentifier = "container %i";
Type = "notify";
EnvironmentFile = "-/etc/containers/%i.conf";
NotifyAccess = "all";
Type = "notify";
# Note that on reboot, systemd-nspawn returns 133, so this
# unit will be restarted. On poweroff, it returns 0, so the
# unit won't be restarted.
RestartForceExitStatus = "133";
SuccessExitStatus = "133";
NotifyAccess = "all";
Restart = "on-failure";
# Note that on reboot, systemd-nspawn returns 133, so this
# unit will be restarted. On poweroff, it returns 0, so the
# unit won't be restarted.
RestartForceExitStatus = "133";
SuccessExitStatus = "133";
Restart = "on-failure";
# Hack: we don't want to kill systemd-nspawn, since we call
# "machinectl poweroff" in preStop to shut down the
# container cleanly. But systemd requires sending a signal
# (at least if we want remaining processes to be killed
# after the timeout). So send an ignored signal.
KillMode = "mixed";
KillSignal = "WINCH";
};
# Hack: we don't want to kill systemd-nspawn, since we call
# "machinectl poweroff" in preStop to shut down the
# container cleanly. But systemd requires sending a signal
# (at least if we want remaining processes to be killed
# after the timeout). So send an ignored signal.
KillMode = "mixed";
KillSignal = "WINCH";
};
};
in {
systemd.services = listToAttrs (filter (x: x.value != null) (
# The generic container template used by imperative containers
[{ name = "container@"; value = unit; }]
# declarative containers
++ (mapAttrsToList (name: cfg: nameValuePair "container@${name}" (
if cfg.autoStart then
unit // {
wantedBy = [ "multi-user.target" ];
wants = [ "network.target" ];
after = [ "network.target" ];
restartTriggers = [ cfg.path ];
reloadIfChanged = true;
}
else null
)) config.containers)
));
# Generate a configuration file in /etc/containers for each
# container so that container@.target can get the container
@ -396,12 +462,21 @@ in
SYSTEM_PATH=${cfg.path}
${optionalString cfg.privateNetwork ''
PRIVATE_NETWORK=1
${optionalString (cfg.hostBridge != null) ''
HOST_BRIDGE=${cfg.hostBridge}
''}
${optionalString (cfg.hostAddress != null) ''
HOST_ADDRESS=${cfg.hostAddress}
''}
${optionalString (cfg.hostAddress6 != null) ''
HOST_ADDRESS6=${cfg.hostAddress6}
''}
${optionalString (cfg.localAddress != null) ''
LOCAL_ADDRESS=${cfg.localAddress}
''}
${optionalString (cfg.localAddress6 != null) ''
LOCAL_ADDRESS6=${cfg.localAddress6}
''}
''}
INTERFACES="${toString cfg.interfaces}"
${optionalString cfg.autoStart ''
@ -420,31 +495,5 @@ in
networking.dhcpcd.denyInterfaces = [ "ve-*" ];
environment.systemPackages = [ nixos-container ];
# Start containers at boot time.
systemd.services.all-containers =
{ description = "All Containers";
wantedBy = [ "multi-user.target" ];
unitConfig.ConditionDirectoryNotEmpty = "/etc/containers";
serviceConfig.Type = "oneshot";
script =
''
res=0
shopt -s nullglob
for i in /etc/containers/*.conf; do
AUTO_START=
source "$i"
if [ "$AUTO_START" = 1 ]; then
systemctl start "container@$(basename "$i" .conf).service" || res=1
fi
done
exit $res
''; # */
};
};
});
}

View File

@ -199,7 +199,10 @@ in rec {
tests.cadvisor = hydraJob (import tests/cadvisor.nix { system = "x86_64-linux"; });
tests.chromium = callSubTests tests/chromium.nix {};
tests.cjdns = callTest tests/cjdns.nix {};
tests.containers = callTest tests/containers.nix {};
tests.containers-ipv4 = callTest tests/containers-ipv4.nix {};
tests.containers-ipv6 = callTest tests/containers-ipv6.nix {};
tests.containers-bridge = callTest tests/containers-bridge.nix {};
tests.containers-imperative = callTest tests/containers-imperative.nix {};
tests.docker = hydraJob (import tests/docker.nix { system = "x86_64-linux"; });
tests.dockerRegistry = hydraJob (import tests/docker-registry.nix { system = "x86_64-linux"; });
tests.dnscrypt-proxy = callTest tests/dnscrypt-proxy.nix { system = "x86_64-linux"; };

View File

@ -0,0 +1,81 @@
# Test for NixOS' container support.
let
hostIp = "192.168.0.1";
containerIp = "192.168.0.100/24";
hostIp6 = "fc00::1";
containerIp6 = "fc00::2/7";
in
import ./make-test.nix ({ pkgs, ...} : {
name = "containers-bridge";
meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ aristid aszlig eelco chaoflow ];
};
machine =
{ config, pkgs, ... }:
{ imports = [ ../modules/installer/cd-dvd/channel.nix ];
virtualisation.writableStore = true;
virtualisation.memorySize = 768;
networking.bridges = {
br0 = {
interfaces = [];
};
};
networking.interfaces = {
br0 = {
ip4 = [{ address = hostIp; prefixLength = 24; }];
ip6 = [{ address = hostIp6; prefixLength = 7; }];
};
};
containers.webserver =
{
autoStart = true;
privateNetwork = true;
hostBridge = "br0";
localAddress = containerIp;
localAddress6 = containerIp6;
config =
{ services.httpd.enable = true;
services.httpd.adminAddr = "foo@example.org";
networking.firewall.allowedTCPPorts = [ 80 ];
networking.firewall.allowPing = true;
};
};
virtualisation.pathsInNixDB = [ pkgs.stdenv ];
};
testScript =
''
$machine->waitForUnit("default.target");
$machine->succeed("nixos-container list") =~ /webserver/ or die;
# Start the webserver container.
$machine->succeed("nixos-container status webserver") =~ /up/ or die;
"${containerIp}" =~ /([^\/]+)\/([0-9+])/;
my $ip = $1;
chomp $ip;
$machine->succeed("ping -n -c 1 $ip");
$machine->succeed("curl --fail http://$ip/ > /dev/null");
"${containerIp6}" =~ /([^\/]+)\/([0-9+])/;
my $ip6 = $1;
chomp $ip6;
$machine->succeed("ping6 -n -c 1 $ip6");
$machine->succeed("curl --fail http://[$ip6]/ > /dev/null");
# Stop the container.
$machine->succeed("nixos-container stop webserver");
$machine->fail("curl --fail --connect-timeout 2 http://$ip/ > /dev/null");
$machine->fail("curl --fail --connect-timeout 2 http://[$ip6]/ > /dev/null");
# Destroying a declarative container should fail.
$machine->fail("nixos-container destroy webserver");
'';
})

View File

@ -1,7 +1,7 @@
# Test for NixOS' container support.
import ./make-test.nix ({ pkgs, ...} : {
name = "containers";
name = "containers-imperative";
meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ aristid aszlig eelco chaoflow ];
};
@ -11,40 +11,11 @@ import ./make-test.nix ({ pkgs, ...} : {
{ imports = [ ../modules/installer/cd-dvd/channel.nix ];
virtualisation.writableStore = true;
virtualisation.memorySize = 768;
containers.webserver =
{ privateNetwork = true;
hostAddress = "10.231.136.1";
localAddress = "10.231.136.2";
config =
{ services.httpd.enable = true;
services.httpd.adminAddr = "foo@example.org";
networking.firewall.allowedTCPPorts = [ 80 ];
networking.firewall.allowPing = true;
};
};
virtualisation.pathsInNixDB = [ pkgs.stdenv ];
};
testScript =
''
$machine->succeed("nixos-container list") =~ /webserver/ or die;
# Start the webserver container.
$machine->succeed("nixos-container start webserver");
# Since "start" returns after the container has reached
# multi-user.target, we should now be able to access it.
my $ip = $machine->succeed("nixos-container show-ip webserver");
chomp $ip;
#$machine->succeed("ping -c1 $ip"); # FIXME
$machine->succeed("curl --fail http://$ip/ > /dev/null");
# Stop the container.
$machine->succeed("nixos-container stop webserver");
$machine->fail("curl --fail --connect-timeout 2 http://$ip/ > /dev/null");
# Make sure we have a NixOS tree (required by nixos-container create).
$machine->succeed("PAGER=cat nix-env -qa -A nixos.hello >&2");
@ -111,9 +82,6 @@ import ./make-test.nix ({ pkgs, ...} : {
# Ensure that the container path is gone
"test ! -e /var/lib/containers/$id1"
);
# Destroying a declarative container should fail.
$machine->fail("nixos-container destroy webserver");
'';
})

View File

@ -0,0 +1,55 @@
# Test for NixOS' container support.
import ./make-test.nix ({ pkgs, ...} : {
name = "containers-ipv4";
meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ aristid aszlig eelco chaoflow ];
};
machine =
{ config, pkgs, ... }:
{ imports = [ ../modules/installer/cd-dvd/channel.nix ];
virtualisation.writableStore = true;
virtualisation.memorySize = 768;
containers.webserver =
{ privateNetwork = true;
hostAddress = "10.231.136.1";
localAddress = "10.231.136.2";
config =
{ services.httpd.enable = true;
services.httpd.adminAddr = "foo@example.org";
networking.firewall.allowedTCPPorts = [ 80 ];
networking.firewall.allowPing = true;
};
};
virtualisation.pathsInNixDB = [ pkgs.stdenv ];
};
testScript =
''
$machine->succeed("nixos-container list") =~ /webserver/ or die;
# Start the webserver container.
$machine->succeed("nixos-container start webserver");
# wait two seconds for the container to start and the network to be up
sleep 2;
# Since "start" returns after the container has reached
# multi-user.target, we should now be able to access it.
my $ip = $machine->succeed("nixos-container show-ip webserver");
chomp $ip;
$machine->succeed("ping -n -c1 $ip");
$machine->succeed("curl --fail http://$ip/ > /dev/null");
# Stop the container.
$machine->succeed("nixos-container stop webserver");
$machine->fail("curl --fail --connect-timeout 2 http://$ip/ > /dev/null");
# Destroying a declarative container should fail.
$machine->fail("nixos-container destroy webserver");
'';
})

View File

@ -0,0 +1,61 @@
# Test for NixOS' container support.
let
hostIp = "fc00::2";
localIp = "fc00::1";
in
import ./make-test.nix ({ pkgs, ...} : {
name = "containers-ipv6";
meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ aristid aszlig eelco chaoflow ];
};
machine =
{ config, pkgs, ... }:
{ imports = [ ../modules/installer/cd-dvd/channel.nix ];
virtualisation.writableStore = true;
virtualisation.memorySize = 768;
containers.webserver =
{ privateNetwork = true;
hostAddress6 = hostIp;
localAddress6 = localIp;
config =
{ services.httpd.enable = true;
services.httpd.adminAddr = "foo@example.org";
networking.firewall.allowedTCPPorts = [ 80 ];
networking.firewall.allowPing = true;
};
};
virtualisation.pathsInNixDB = [ pkgs.stdenv ];
};
testScript =
''
$machine->waitForUnit("default.target");
$machine->succeed("nixos-container list") =~ /webserver/ or die;
# Start the webserver container.
$machine->succeed("nixos-container start webserver");
# wait two seconds for the container to start and the network to be up
sleep 2;
# Since "start" returns after the container has reached
# multi-user.target, we should now be able to access it.
my $ip = "${localIp}";
chomp $ip;
$machine->succeed("ping6 -n -c 1 $ip");
$machine->succeed("curl --fail http://[$ip]/ > /dev/null");
# Stop the container.
$machine->succeed("nixos-container stop webserver");
$machine->fail("curl --fail --connect-timeout 2 http://[$ip]/ > /dev/null");
# Destroying a declarative container should fail.
$machine->fail("nixos-container destroy webserver");
'';
})