build2: split out dependencies

Build2's dependency libraries and helper programs
bpkg/bdep are all built and installed separately.

Adds a build2 setup hook for packaging purposes,
based on those of cmake and ninja.

Two small patches are applied to build2's source:
one to remove store references from the stored config,
and one to help build2 detect library dirs from NIX_LDFLAGS.

pkgs/development/libraries/libbpkg/default.nix

@@ -0,0 +1,47 @@
+{ lib, stdenv
+, build2
+, fetchurl
+, libbutl
+, enableShared ? !stdenv.hostPlatform.isStatic
+, enableStatic ? !enableShared
+}:
+
+stdenv.mkDerivation rec {
+  pname = "libbpkg";
+  version = "0.13.0";
+  outputs = [ "out" "dev" "doc" ];
+
+  src = fetchurl {
+    url = "https://pkg.cppget.org/1/alpha/build2/libbpkg-${version}.tar.gz";
+    sha256 = "732849cdd5d773c589dd0ac220002fa41424784df10617adc4dea729faafb22b";
+  };
+
+  nativeBuildInputs = [
+    build2
+  ];
+  buildInputs = [
+    libbutl
+  ];
+
+  build2ConfigureFlags = [
+    "config.bin.lib=${build2.configSharedStatic enableShared enableStatic}"
+  ];
+
+  strictDeps = true;
+
+  doCheck = true;
+
+  meta = with lib; {
+    description = "build2 package dependency manager utility library";
+    longDescription = ''
+      This library defines the types and utilities for working with build2 packages.
+      In particular, it provides C++ classes as well as the parser and serializer
+      implementations that can be used to read, manipulate, and write package,
+      repository and signature manifests.
+    '';
+    homepage = "https://build2.org/";
+    changelog = "https://git.build2.org/cgit/libbpkg/log";
+    license = licenses.mit;
+    maintainers = with maintainers; [ r-burns ];
+  };
+}

pkgs/development/libraries/libbutl/default.nix

@@ -0,0 +1,54 @@
+{ lib, stdenv
+, build2
+, fetchurl
+, libuuid
+, enableShared ? !stdenv.hostPlatform.isStatic
+, enableStatic ? !enableShared
+}:
+
+stdenv.mkDerivation rec {
+  pname = "libbutl";
+  version = "0.13.0";
+
+  outputs = [ "out" "dev" "doc" ];
+
+  src = fetchurl {
+    url = "https://pkg.cppget.org/1/alpha/build2/libbutl-${version}.tar.gz";
+    sha256 = "d7944637ab4a17d3a299c04ff6f146e89b2a0f433ddd9d08d8632a25bae9c9cb";
+  };
+
+  nativeBuildInputs = [
+    build2
+  ];
+
+  strictDeps = true;
+
+  # Should be true for anything built with build2,
+  # but especially important when bootstrapping
+  disallowedReferences = [ build2 ];
+
+  postPatch = lib.optionalString stdenv.isLinux ''
+    substituteInPlace libbutl/uuid-linux.cxx \
+      --replace '"libuuid.so' '"${lib.getLib libuuid}/lib/libuuid.so'
+  '';
+
+  build2ConfigureFlags = [
+    "config.bin.lib=${build2.configSharedStatic enableShared enableStatic}"
+  ];
+
+  # tests broken with -DNDEBUG
+  # https://github.com/build2/libbutl/issues/4
+  # doCheck = true;
+
+  meta = with lib; {
+    description = "build2 utility library";
+    longDescription = ''
+      This library is a collection of utilities that are used throughout the
+      build2 toolchain.
+    '';
+    homepage = "https://build2.org/";
+    changelog = "https://git.build2.org/cgit/libbutl/log";
+    license = licenses.mit;
+    maintainers = with maintainers; [ r-burns ];
+  };
+}

pkgs/development/libraries/libodb-sqlite/default.nix

@@ -0,0 +1,56 @@
+{ lib, stdenv
+, build2
+, fetchurl
+, libodb
+, sqlite
+, enableShared ? !stdenv.hostPlatform.isStatic
+, enableStatic ? !enableShared
+}:
+stdenv.mkDerivation rec {
+  pname = "libodb-sqlite";
+  version = "2.5.0-b.19";
+
+  outputs = [ "out" "dev" "doc" ];
+
+  src = fetchurl {
+    url = "https://pkg.cppget.org/1/beta/odb/libodb-sqlite-${version}.tar.gz";
+    sha256 = "9443653bfc31d02d0d723f18072f6b04083d090e6580844e33c1e769db122494";
+  };
+
+  nativeBuildInputs = [
+    build2
+  ];
+  buildInputs = [
+    libodb
+  ];
+  propagatedBuildInputs = [
+    sqlite
+  ];
+
+  build2ConfigureFlags = [
+    "config.bin.lib=${build2.configSharedStatic enableShared enableStatic}"
+  ];
+
+  doCheck = true;
+
+  meta = with lib; {
+    description = "SQLite ODB runtime library";
+    longDescription = ''
+      ODB is an object-relational mapping (ORM) system for C++. It provides
+      tools, APIs, and library support that allow you to persist C++ objects
+      to a relational database (RDBMS) without having to deal with tables,
+      columns, or SQL and without manually writing any of the mapping code.
+      For more information see:
+
+      http://www.codesynthesis.com/products/odb/
+
+      This package contains the SQLite ODB runtime library. Every application
+      that includes code generated for the SQLite database will need to link
+      to this library.
+    '';
+    homepage = "https://www.codesynthesis.com/products/odb/";
+    changelog = "https://git.codesynthesis.com/cgit/odb/libodb-sqlite/tree/NEWS";
+    license = licenses.gpl2Only;
+    maintainers = with maintainers; [ r-burns ];
+  };
+}

pkgs/development/libraries/libodb/default.nix

@@ -0,0 +1,45 @@
+{ lib, stdenv
+, build2
+, fetchurl
+, enableShared ? !stdenv.hostPlatform.isStatic
+, enableStatic ? !enableShared
+}:
+stdenv.mkDerivation rec {
+  pname = "libodb";
+  version = "2.5.0-b.19";
+
+  outputs = [ "out" "dev" "doc" ];
+
+  src = fetchurl {
+    url = "https://pkg.cppget.org/1/beta/odb/libodb-${version}.tar.gz";
+    sha256 = "8180d9d40d8e74ed25b1712953f19379a29abdee3896ae98ba9ade35846adb39";
+  };
+
+  nativeBuildInputs = [ build2 ];
+  build2ConfigureFlags = [
+    "config.bin.lib=${build2.configSharedStatic enableShared enableStatic}"
+  ];
+
+  doCheck = true;
+
+  meta = with lib; {
+    description = "Common ODB runtime library";
+    longDescription = ''
+      ODB is an object-relational mapping (ORM) system for C++. It provides
+      tools, APIs, and library support that allow you to persist C++ objects
+      to a relational database (RDBMS) without having to deal with tables,
+      columns, or SQL and without manually writing any of the mapping code.
+      For more information see:
+
+      http://www.codesynthesis.com/products/odb/
+
+      This package contains the common ODB runtime library. Every application
+      that includes code generated by the ODB compiler will need to link to
+      this library.
+    '';
+    homepage = "https://www.codesynthesis.com/products/odb/";
+    changelog = "https://git.codesynthesis.com/cgit/odb/libodb/tree/NEWS";
+    license = licenses.gpl2Only;
+    maintainers = with maintainers; [ r-burns ];
+  };
+}

pkgs/development/tools/build-managers/build2/bdep.nix

@@ -0,0 +1,49 @@
+{ lib, stdenv
+, build2
+, fetchurl
+, libbpkg
+, libbutl
+, libodb
+, libodb-sqlite
+, enableShared ? !stdenv.hostPlatform.isStatic
+, enableStatic ? !enableShared
+}:
+
+stdenv.mkDerivation rec {
+  pname = "bdep";
+  version = "0.13.0";
+
+  outputs = [ "out" "doc" "man" ];
+  src = fetchurl {
+    url = "https://pkg.cppget.org/1/alpha/build2/bdep-${version}.tar.gz";
+    sha256 = "8e11b469d875d05c4eb7a228416b78a61c68a49310e5e41db78ed6d048f6ba2a";
+  };
+
+  strictDeps = true;
+  nativeBuildInputs = [
+    build2
+  ];
+  buildInputs = [
+    libbpkg
+    libbutl
+    libodb
+    libodb-sqlite
+  ];
+
+  build2ConfigureFlags = [
+    "config.bin.lib=${build2.configSharedStatic enableShared enableStatic}"
+  ];
+
+  meta = with lib; {
+    description = "build2 project dependency manager";
+    # https://build2.org/bdep/doc/bdep.xhtml
+    longDescription = ''
+      The build2 project dependency manager is used to manage the dependencies
+      of a project during development.
+    '';
+    homepage = "https://build2.org/";
+    changelog = "https://git.build2.org/cgit/bdep/tree/NEWS";
+    license = licenses.mit;
+    maintainers = with maintainers; [ r-burns ];
+  };
+}

pkgs/development/tools/build-managers/build2/bootstrap.nix

@@ -0,0 +1,44 @@
+{ lib, stdenv
+, fetchurl
+, pkgs
+, fixDarwinDylibNames
+}:
+stdenv.mkDerivation rec {
+  pname = "build2-bootstrap";
+  version = "0.13.0";
+  src = fetchurl {
+    url = "https://download.build2.org/${version}/build2-toolchain-${version}.tar.xz";
+    sha256 = "01hmr5y8aa28qchwy9ci8x5q746flwxmlxarmy4w9zay9nmvryms";
+  };
+  patches = [
+    # Pick up sysdirs from NIX_LDFLAGS
+    ./nix-ldflags-sysdirs.patch
+  ];
+
+  sourceRoot = "build2-toolchain-${version}/build2";
+  makefile = "bootstrap.gmake";
+  enableParallelBuilding = true;
+
+  setupHook = ./setup-hook.sh;
+
+  strictDeps = true;
+
+  propagatedBuildInputs = lib.optionals stdenv.targetPlatform.isDarwin [
+    fixDarwinDylibNames
+  ];
+
+  doCheck = true;
+  checkPhase = ''
+    runHook preCheck
+    build2/b-boot --version
+    runHook postCheck
+  '';
+
+  installPhase = ''
+    runHook preInstall
+    install -D build2/b-boot $out/bin/b
+    runHook postInstall
+  '';
+
+  inherit (pkgs.build2) passthru;
+}

pkgs/development/tools/build-managers/build2/bpkg.nix

@@ -0,0 +1,63 @@
+{ lib, stdenv
+, build2
+, fetchurl
+, git
+, libbpkg
+, libbutl
+, libodb
+, libodb-sqlite
+, openssl
+, enableShared ? !stdenv.hostPlatform.isStatic
+, enableStatic ? !enableShared
+}:
+
+stdenv.mkDerivation rec {
+  pname = "bpkg";
+  version = "0.13.0";
+
+  outputs = [ "out" "doc" "man" ];
+
+  src = fetchurl {
+    url = "https://pkg.cppget.org/1/alpha/build2/bpkg-${version}.tar.gz";
+    sha256 = "fec41e171c8ea7967bfc44850568cd624def544fd866c383bd413c5b4349e282";
+  };
+
+  strictDeps = true;
+  nativeBuildInputs = [
+    build2
+  ];
+  buildInputs = [
+    libbpkg
+    libbutl
+    libodb
+    libodb-sqlite
+  ];
+  checkInputs = [
+    git
+    openssl
+  ];
+
+  doCheck = !stdenv.isDarwin; # tests hang
+
+  # Failing test
+  postPatch = ''
+    rm tests/rep-create.testscript
+  '';
+
+  build2ConfigureFlags = [
+    "config.bin.lib=${build2.configSharedStatic enableShared enableStatic}"
+  ];
+
+  meta = with lib; {
+    description = "build2 package dependency manager";
+    # https://build2.org/bpkg/doc/bpkg.xhtml
+    longDescription = ''
+      The build2 package dependency manager is used to manipulate build
+      configurations, packages, and repositories.
+    '';
+    homepage = "https://build2.org/";
+    changelog = "https://git.build2.org/cgit/bpkg/tree/NEWS";
+    license = licenses.mit;
+    maintainers = with maintainers; [ r-burns ];
+  };
+}

pkgs/development/tools/build-managers/build2/default.nix

@@ -1,23 +1,86 @@
-{ stdenv, lib, fetchurl }:
-
+{ stdenv, lib
+, build2
+, fetchurl
+, fetchpatch
+, fixDarwinDylibNames
+, libbutl
+, libpkgconf
+, enableShared ? !stdenv.hostPlatform.isStatic
+, enableStatic ? !enableShared
+}:
+let
+  configSharedStatic = enableShared: enableStatic:
+    if enableShared && enableStatic then "both"
+    else if enableShared then "shared"
+    else if enableStatic then "static"
+    else throw "neither shared nor static libraries requested";
+in
 stdenv.mkDerivation rec {
   pname = "build2";
   version = "0.13.0";
 
+  outputs = [ "out" "dev" "doc" "man" ];
+
+  setupHook = ./setup-hook.sh;
+
   src = fetchurl {
-    url = "https://download.build2.org/${version}/build2-toolchain-${version}.tar.xz";
-    sha256 = "01hmr5y8aa28qchwy9ci8x5q746flwxmlxarmy4w9zay9nmvryms";
+    url = "https://pkg.cppget.org/1/alpha/build2/build2-${version}.tar.gz";
+    sha256 = "aff53a87c23534e0232b5cf746e0be4b2aaa840c3de4e668b98e382a3973c45e";
   };
 
-  dontConfigure = true;
-  dontInstall = true;
+  patches = [
+    # Remove any build/host config entries which refer to nix store paths
+    ./remove-config-store-paths.patch
+    # Pick up sysdirs from NIX_LDFLAGS
+    ./nix-ldflags-sysdirs.patch
+    # Fix stray '-l' linker flags in pkg-config files (remove in next release)
+    (fetchpatch {
+      url = "https://github.com/build2/build2/commit/d51892e33a0fe69e743e02d9620312133a7ac61d.patch";
+      sha256 = "0xzm084bxnfi8lqng0cwxvz8ylbfzk0didbr2wf385gssv4fva81";
+    })
+  ];
 
-  buildPhase = ''
-    runHook preBuild
-    ./build.sh --local --trust yes --install-dir "$out" "$CXX"
-    runHook postBuild
+  strictDeps = true;
+  nativeBuildInputs = [
+    build2
+  ];
+  disallowedReferences = [
+    build2
+    libbutl.dev
+    libpkgconf.dev
+  ];
+  buildInputs = [
+    libbutl
+    libpkgconf
+  ];
+
+  # Build2 uses @rpath on darwin
+  # https://github.com/build2/build2/issues/166
+  # N.B. this only adjusts the install_name after all libraries are installed;
+  # packages containing multiple interdependent libraries may have
+  # LC_LOAD_DYLIB entries containing @rpath, requiring manual fixup
+  propagatedBuildInputs = lib.optionals stdenv.targetPlatform.isDarwin [
+    fixDarwinDylibNames
+  ];
+
+  postPatch = ''
+    patchShebangs --build tests/bash/testscript
   '';
 
+  build2ConfigureFlags = [
+    "config.bin.lib=${configSharedStatic enableShared enableStatic}"
+    "config.cc.poptions+=-I${lib.getDev libpkgconf}/include/pkgconf"
+  ];
+
+  postInstall = lib.optionalString stdenv.isDarwin ''
+    install_name_tool -add_rpath "''${!outputLib}/lib" "''${!outputBin}/bin/b"
+  '';
+
+  passthru = {
+    bootstrap = build2;
+    inherit configSharedStatic;
+  };
+
   meta = with lib; {
     homepage = "https://www.build2.org/";
     description = "build2 build system";
@@ -34,6 +97,7 @@ stdenv.mkDerivation rec {
       at C/C++ projects as well as mixed-language projects involving
       one of these languages (see bash and rust modules, for example).
     '';
+    changelog = "https://git.build2.org/cgit/build2/tree/NEWS";
     platforms = platforms.all;
     maintainers = with maintainers; [ hiro98 r-burns ];
   };

pkgs/development/tools/build-managers/build2/nix-ldflags-sysdirs.patch

@@ -0,0 +1,42 @@
+diff --git a/libbuild2/cc/common.cxx b/libbuild2/cc/common.cxx
+index f848003c..0f14f9a5 100644
+--- a/libbuild2/cc/common.cxx
++++ b/libbuild2/cc/common.cxx
+@@ -966,6 +966,17 @@ namespace build2
+     void
+     msvc_extract_library_search_dirs (const strings&, dir_paths&); // msvc.cxx
+ 
++    static strings split (const string& s, const char delim) {
++      stringstream ss (s);
++      string item;
++      strings result;
++
++      while (getline (ss, item, delim)) {
++        result.push_back (item);
++      }
++      return result;
++    }
++
+     dir_paths common::
+     extract_library_search_dirs (const scope& bs) const
+     {
+@@ -987,8 +998,19 @@ namespace build2
+           msvc_extract_library_search_dirs (v, r);
+         else
+           gcc_extract_library_search_dirs (v, r);
++
+       };
+ 
++      // NIX_LDFLAGS are implicitly used when linking,
++      // so its -L flags effectively specify system dirs.
++      // However, they are only enabled when actually linking and are thus
++      // not detected by build2, so we need to manually pick them up here.
++      if (auto s = getenv ("NIX_LDFLAGS")) {
++        // TODO: do we need more robust args splitting here? e.g. shlex.split
++        auto args = split (s.value (), ' ');
++        gcc_extract_library_search_dirs (args, r);
++      }
++
+       // Note that the compiler mode options are in sys_lib_dirs.
+       //
+       if (auto l = bs[c_loptions]) extract (*l, c_loptions);

pkgs/development/tools/build-managers/build2/remove-config-store-paths.patch

@@ -0,0 +1,14 @@
+--- a/libbuild2/buildfile
++++ b/libbuild2/buildfile
+@@ -68,7 +68,11 @@ config/cxx{host-config}: config/in{host-config}
+   #
+   build2_config = $regex.replace_lines(                 \
++    $regex.replace_lines(                               \
+     $config.save(),                                     \
+     '^ *(#|config\.dist\.|config\.install\.chroot).*$', \
+     [null],                                             \
++    return_lines),                                      \
++    '^.*'$getenv(NIX_STORE)'/[a-z0-9]{32}-.*$',         \
++    [null],                                             \
+     return_lines)
+ 

pkgs/development/tools/build-managers/build2/setup-hook.sh

@@ -0,0 +1,85 @@
+build2ConfigurePhase() {
+    runHook preConfigure
+
+    local flagsArray=(
+        "config.c=$CC"
+        "config.cxx=$CXX"
+        "config.cc.coptions+=-O2"
+        "config.cc.poptions+=-DNDEBUG"
+        "config.install.root=$prefix"
+        "config.install.bin=${!outputBin}/bin"
+        "config.install.doc=${!outputDoc}/share/doc/${shareDocName}"
+        "config.install.exec_root=${!outputBin}"
+        "config.install.include=${!outputInclude}/include"
+        "config.install.lib=${!outputLib}/lib"
+        "config.install.libexec=${!outputLib}/libexec"
+        "config.install.man=${!outputDoc}/share/man"
+        "config.install.sbin=${!outputBin}/sbin"
+        "config.install.bin.mode=755"
+        $build2ConfigureFlags "${build2ConfigureFlagsArray[@]}"
+    )
+
+    echo 'configure flags' "${flagsArray[@]}"
+
+    b configure "${flagsArray[@]}"
+
+    runHook postConfigure
+}
+
+build2BuildPhase() {
+    runHook preBuild
+
+    local flagsArray=(
+        $build2BuildFlags "${build2BuildFlagsArray[@]}"
+    )
+
+    echo 'build flags' "${flagsArray[@]}"
+    b "${flagsArray[@]}"
+
+    runHook postBuild
+}
+
+build2CheckPhase() {
+    runHook preCheck
+
+    local flagsArray=(
+        $build2CheckFlags "${build2CheckFlags[@]}"
+    )
+
+    echo 'check flags' "${flagsArray[@]}"
+
+    b test ${build2Dir:-.} "${flagsArray[@]}"
+
+    runHook postCheck
+}
+
+build2InstallPhase() {
+    runHook preInstall
+
+    local flagsArray=(
+        $build2InstallFlags "${build2InstallFlagsArray[@]}"
+        ${installTargets:-}
+    )
+
+    echo 'install flags' "${flagsArray[@]}"
+    b install "${flagsArray[@]}"
+
+    runHook postInstall
+}
+
+if [ -z "${dontUseBuild2Configure-}" -a -z "${configurePhase-}" ]; then
+    setOutputFlags=
+    configurePhase=build2ConfigurePhase
+fi
+
+if [ -z "${dontUseBuild2Build-}" -a -z "${buildPhase-}" ]; then
+    buildPhase=build2BuildPhase
+fi
+
+if [ -z "${dontUseBuild2Check-}" -a -z "${checkPhase-}" ]; then
+    checkPhase=build2CheckPhase
+fi
+
+if [ -z "${dontUseBuild2Install-}" -a -z "${installPhase-}" ]; then
+    installPhase=build2InstallPhase
+fi

pkgs/top-level/all-packages.nix

@@ -13547,7 +13547,21 @@ with pkgs;
 
   buck = callPackage ../development/tools/build-managers/buck { };
 
-  build2 = callPackage ../development/tools/build-managers/build2 { };
+  build2 = callPackage ../development/tools/build-managers/build2 {
+    # Break cycle by using self-contained toolchain for bootstrapping
+    build2 = buildPackages.callPackage ../development/tools/build-managers/build2/bootstrap.nix { };
+  };
+
+  # Dependency of build2, must also break cycle for this
+  libbutl = callPackage ../development/libraries/libbutl {
+    build2 = build2.bootstrap;
+  };
+
+  libbpkg = callPackage ../development/libraries/libbpkg { };
+  libodb = callPackage ../development/libraries/libodb { };
+  libodb-sqlite = callPackage ../development/libraries/libodb-sqlite { };
+  bdep = callPackage ../development/tools/build-managers/build2/bdep.nix { };
+  bpkg = callPackage ../development/tools/build-managers/build2/bpkg.nix { };
 
   buildkite-agent = callPackage ../development/tools/continuous-integration/buildkite-agent { };
 
@@ -14402,6 +14416,7 @@ with pkgs;
     pkg-config = pkgconf-unwrapped;
     baseBinName = "pkgconf";
   };
+  libpkgconf = pkgconf-unwrapped;
 
   pkg-config-unwrapped = callPackage ../development/tools/misc/pkg-config { };
   pkg-config = callPackage ../build-support/pkg-config-wrapper {