From eaff0725def0247cd434d7d31a3db1ad7921b19c Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Thu, 14 Nov 2024 10:49:19 +0100 Subject: [PATCH] buildNimSbom: init a new package builder for Nim Ref: #327064 --- .github/labeler.yml | 1 + ci/OWNERS | 1 + doc/languages-frameworks/nim.section.md | 46 +++- doc/redirects.json | 13 +- .../manual/release-notes/rl-2411.section.md | 2 + pkgs/build-support/build-nim-sbom.nix | 205 ++++++++++++++++++ pkgs/top-level/all-packages.nix | 1 + 7 files changed, 261 insertions(+), 8 deletions(-) create mode 100644 pkgs/build-support/build-nim-sbom.nix diff --git a/.github/labeler.yml b/.github/labeler.yml index 4c8a048c763b..ff0ef5e6f1ef 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -328,6 +328,7 @@ - any-glob-to-any-file: - doc/languages-frameworks/nim.section.md - pkgs/build-support/build-nim-package.nix + - pkgs/build-support/build-nim-sbom.nix - pkgs/by-name/ni/nim* - pkgs/top-level/nim-overrides.nix diff --git a/ci/OWNERS b/ci/OWNERS index ed8f37d1df0b..ce8ceadd5657 100644 --- a/ci/OWNERS +++ b/ci/OWNERS @@ -361,6 +361,7 @@ pkgs/development/python-modules/buildcatrust/ @ajs124 @lukegb @mweinelt # nim /doc/languages-frameworks/nim.section.md @ehmry /pkgs/build-support/build-nim-package.nix @ehmry +/pkgs/build-support/build-nim-sbom.nix @ehmry /pkgs/top-level/nim-overrides.nix @ehmry # terraform providers diff --git a/doc/languages-frameworks/nim.section.md b/doc/languages-frameworks/nim.section.md index fd97746c6938..f0196c9d116f 100644 --- a/doc/languages-frameworks/nim.section.md +++ b/doc/languages-frameworks/nim.section.md @@ -1,7 +1,9 @@ -# Nim {#nim} +# Nim {#sec-language-nim} The Nim compiler and a builder function is available. -Nim programs are built using `buildNimPackage` and a lockfile containing Nim dependencies. +Nim programs are built using a lockfile and either `buildNimPackage` or `buildNimSbom`. + +## buildNimPackage {#buildNimPackage} The following example shows a Nim program that depends only on Nim libraries: ```nix @@ -15,7 +17,7 @@ buildNimPackage (finalAttrs: { owner = "inv2004"; repo = "ttop"; rev = "v${finalAttrs.version}"; - hash = "sha256-oPdaUqh6eN1X5kAYVvevOndkB/xnQng9QVLX9bu5P5E="; + hash = lib.fakeHash; }; lockFile = ./lock.json; @@ -26,7 +28,7 @@ buildNimPackage (finalAttrs: { }) ``` -## `buildNimPackage` parameters {#buildnimpackage-parameters} +### `buildNimPackage` parameters {#buildnimpackage-parameters} The `buildNimPackage` function takes an attrset of parameters that are passed on to `stdenv.mkDerivation`. @@ -41,7 +43,7 @@ The following parameters are specific to `buildNimPackage`: Use this to specify defines with arguments in the form of `-d:${name}=${value}`. * `nimDoc` ? false`: Build and install HTML documentation. -## Lockfiles {#nim-lockfiles} +### Lockfiles {#nim-lockfiles} Nim lockfiles are created with the `nim_lk` utility. Run `nim_lk` with the source directory as an argument and it will print a lockfile to stdout. ```sh @@ -50,9 +52,41 @@ $ nix build -f . ttop.src $ nix run -f . nim_lk ./result | jq --sort-keys > pkgs/by-name/tt/ttop/lock.json ``` +## buildNimSbom {#buildNimSbom} + +An alternative to `buildNimPackage` is `buildNimSbom` which builds packages from [CycloneDX SBOM](https://cyclonedx.org/) files. +`buildNimSbom` resolves Nim dependencies to [fixed-output derivations](https://nixos.org/manual/nix/stable/glossary#gloss-fixed-output-derivation) using the [nix:fod namespace](#sec-interop.cylonedx-fod). + +In the following minimal example only the source code checkout and a `buildInput` are specified. +The SBOM file provides metadata such as `pname` and `version` as well as the sources to Nim dependencies. +```nix +# pkgs/by-name/ni/nim_lk/package.nix +{ + lib, + buildNimSbom, + fetchFromSourcehut, + openssl, +}: + +buildNimSbom (finalAttrs: { + src = fetchFromSourcehut { + owner = "~ehmry"; + repo = "nim_lk"; + rev = finalAttrs.version; + hash = lib.fakeHash; + }; + buildInputs = [ openssl ]; +}) ./sbom.json +``` + +### Generating SBOMs {#generating-nim-sboms} + +The [nim_lk](https://git.sr.ht/~ehmry/nim_lk) utility can generate SBOMs from [Nimble](https://github.com/nim-lang/nimble) package metadata. +See the [nim_lk documentation](https://git.sr.ht/~ehmry/nim_lk#nimble-to-cyclonedx-sbom) for more information. + ## Overriding Nim packages {#nim-overrides} -The `buildNimPackage` function generates flags and additional build dependencies from the `lockFile` parameter passed to `buildNimPackage`. Using [`overrideAttrs`](#sec-pkg-overrideAttrs) on the final package will apply after this has already been generated, so this can't be used to override the `lockFile` in a package built with `buildNimPackage`. To be able to override parameters before flags and build dependencies are generated from the `lockFile`, use `overrideNimAttrs` instead with the same syntax as `overrideAttrs`: +The `buildNimPackage` and `buildNimSbom` functions generate flags and additional build dependencies from the `lockFile` parameter passed to `buildNimPackage`. Using [`overrideAttrs`](#sec-pkg-overrideAttrs) on the final package will apply after this has already been generated, so this can't be used to override the `lockFile` in a package built with `buildNimPackage`. To be able to override parameters before flags and build dependencies are generated from the `lockFile`, use `overrideNimAttrs` instead with the same syntax as `overrideAttrs`: ```nix pkgs.nitter.overrideNimAttrs { diff --git a/doc/redirects.json b/doc/redirects.json index 311ca1ff4af7..c43fcc152c13 100644 --- a/doc/redirects.json +++ b/doc/redirects.json @@ -3223,8 +3223,11 @@ "manifest-file-via-maven-plugin": [ "index.html#manifest-file-via-maven-plugin" ], - "nim": [ - "index.html#nim" + "sec-language-nim": [ + "index.html#sec-language-nim" + ], + "buildNimPackage": [ + "index.html#buildNimPackage" ], "buildnimpackage-parameters": [ "index.html#buildnimpackage-parameters" @@ -3232,6 +3235,12 @@ "nim-lockfiles": [ "index.html#nim-lockfiles" ], + "buildNimSbom": [ + "index.html#buildNimSbom" + ], + "generating-nim-sboms": [ + "index.html#generating-nim-sboms" + ], "nim-overrides": [ "index.html#nim-overrides" ], diff --git a/nixos/doc/manual/release-notes/rl-2411.section.md b/nixos/doc/manual/release-notes/rl-2411.section.md index 84c2f6fe344a..3c6f0ff26fb8 100644 --- a/nixos/doc/manual/release-notes/rl-2411.section.md +++ b/nixos/doc/manual/release-notes/rl-2411.section.md @@ -911,6 +911,8 @@ - `freecad` now supports addons and custom configuration in nix-way, which can be used by calling `freecad.customize`. +- `buildNimSbom` was added as an alternative to `buildNimPackage`. `buildNimSbom` uses [SBOMs](https://cyclonedx.org/) to generate packages whereas `buildNimPackage` uses a custom JSON lockfile format. + ## Detailed Migration Information {#sec-release-24.11-migration} ### `sound` options removal {#sec-release-24.11-migration-sound} diff --git a/pkgs/build-support/build-nim-sbom.nix b/pkgs/build-support/build-nim-sbom.nix new file mode 100644 index 000000000000..ef60ac5a0af8 --- /dev/null +++ b/pkgs/build-support/build-nim-sbom.nix @@ -0,0 +1,205 @@ +{ + lib, + stdenv, + fetchgit, + fetchzip, + runCommand, + xorg, + nim, + nimOverrides, +}: + +let + fetchers = { + fetchzip = + { url, sha256, ... }: + fetchzip { + name = "source"; + inherit url sha256; + }; + fetchgit = + { + fetchSubmodules ? false, + leaveDotGit ? false, + rev, + sha256, + url, + ... + }: + fetchgit { + inherit + fetchSubmodules + leaveDotGit + rev + sha256 + url + ; + }; + }; + + filterPropertiesToAttrs = + prefix: properties: + lib.pipe properties [ + (builtins.filter ({ name, ... }: (lib.strings.hasPrefix prefix name))) + (map ( + { name, value }: + { + name = lib.strings.removePrefix prefix name; + inherit value; + } + )) + builtins.listToAttrs + ]; + + buildNimCfg = + { backend, components, ... }: + let + componentSrcDirs = map ( + { properties, ... }: + let + fodProps = filterPropertiesToAttrs "nix:fod:" properties; + fod = fetchers.${fodProps.method} fodProps; + srcDir = fodProps.srcDir or ""; + in + if srcDir == "" then fod else "${fod}/${srcDir}" + ) components; + in + runCommand "nim.cfg" + { + outputs = [ + "out" + "src" + ]; + nativeBuildInputs = [ xorg.lndir ]; + } + '' + cat << EOF >> $out + backend:${backend} + path:"$src" + EOF + mkdir -p "$src" + ${lib.strings.concatMapStrings (d: '' + lndir "${d}" "$src" + '') componentSrcDirs} + ''; + + buildCommands = lib.attrsets.mapAttrsToList ( + output: input: '' + nim compile $nimFlags --out:${output} ${input} + '' + ); + + installCommands = lib.attrsets.mapAttrsToList ( + output: input: '' + install -Dt $out/bin ${output} + '' + ); + + applySbom = + sbom: + { + nimFlags ? [ ], + nimRelease ? true, + passthru ? { }, + ... + }@prevAttrs: + let + properties = # SBOM metadata.component.properties as an attrset. + lib.attrsets.recursiveUpdate (builtins.listToAttrs sbom.metadata.component.properties) + passthru.properties or { }; + + nimBin = # A mapping of Nim module file paths to names of programs. + lib.attrsets.recursiveUpdate (lib.pipe properties [ + (lib.attrsets.filterAttrs (name: value: lib.strings.hasPrefix "nim:bin:" name)) + (lib.attrsets.mapAttrs' ( + name: value: { + name = lib.strings.removePrefix "nim:bin:" name; + value = "${properties."nim:binDir" or (properties."nim:srcDir" or ".")}/${value}"; + } + )) + ]) passthru.nimBin or { }; + in + { + strictDeps = true; + + pname = prevAttrs.pname or sbom.metadata.component.name; + version = prevAttrs.version or sbom.metadata.component.version or null; + + nimFlags = + nimFlags + ++ (lib.optional nimRelease "-d:release") + ++ ( + let + srcDir = properties."nim:srcDir" or ""; + in + lib.optional (srcDir != "") "--path:${srcDir}" + ); + + configurePhase = + prevAttrs.configurePhase or '' + runHook preConfigure + echo "nim.cfg << $nimCfg" + cat $nimCfg >> nim.cfg + cat << EOF >> nim.cfg + nimcache:"$NIX_BUILD_TOP/nimcache" + parallelBuild:$NIX_BUILD_CORES + EOF + runHook postConfigure + ''; + + buildPhase = + prevAttrs.buildPhase or '' + runHook preBuild + ${lib.strings.concatLines (buildCommands nimBin)} + runHook postBuild + ''; + + installPhase = + prevAttrs.installPhase or '' + runHook preInstall + ${lib.strings.concatLines (installCommands nimBin)} + runHook postInstall + ''; + + nativeBuildInputs = (prevAttrs.nativeBuildInputs or [ ]) ++ [ nim ]; + + nimCfg = + prevAttrs.nimCfg or (buildNimCfg { + backend = prevAttrs.nimBackend or properties."nim:backend" or "c"; + inherit (sbom) components; + }); + + passthru = { + inherit sbom properties nimBin; + }; + }; + + applyOverrides = + prevAttrs: + builtins.foldl' ( + prevAttrs: + { name, ... }@component: + if (builtins.hasAttr name nimOverrides) then + let + result = nimOverrides.${name} component prevAttrs; + in + prevAttrs // (if builtins.isAttrs result then result else result { }) + else + prevAttrs + ) prevAttrs prevAttrs.passthru.sbom.components; + + compose = + callerArg: sbom: finalAttrs: + let + callerAttrs = if builtins.isAttrs callerArg then callerArg else callerArg finalAttrs; + sbomAttrs = callerAttrs // (applySbom sbom callerAttrs); + overrideAttrs = sbomAttrs // (applyOverrides sbomAttrs); + in + overrideAttrs; +in +callerArg: sbomArg: +let + sbom = if builtins.isAttrs sbomArg then sbomArg else builtins.fromJSON (builtins.readFile sbomArg); + overrideSbom = f: stdenv.mkDerivation (compose callerArg (sbom // (f sbom))); +in +(stdenv.mkDerivation (compose callerArg sbom)) // { inherit overrideSbom; } diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 63910bd6cd83..eecbd8c6e87d 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -6724,6 +6724,7 @@ with pkgs; nim-unwrapped-2 = nim-unwrapped-2_2; buildNimPackage = callPackage ../build-support/build-nim-package.nix { }; + buildNimSbom = callPackage ../build-support/build-nim-sbom.nix { }; nimOverrides = callPackage ./nim-overrides.nix { }; nextpnrWithGui = libsForQt5.callPackage ../by-name/ne/nextpnr/package.nix {