From 729e2822ffb2628cb7c2f5e4cabed0c70159c62e Mon Sep 17 00:00:00 2001 From: thomasjm Date: Tue, 2 Jul 2024 01:11:27 -0700 Subject: [PATCH] julia.withPackages: fix for overrides not in a registry --- .../julia-modules/package-closure.nix | 15 ++++- .../julia-modules/python/minimal_registry.py | 3 +- .../tests/julia-top-n/app/Main.hs | 64 +++++++++++++------ 3 files changed, 59 insertions(+), 23 deletions(-) diff --git a/pkgs/development/julia-modules/package-closure.nix b/pkgs/development/julia-modules/package-closure.nix index d1138394e993..bb0b08b33e41 100644 --- a/pkgs/development/julia-modules/package-closure.nix +++ b/pkgs/development/julia-modules/package-closure.nix @@ -42,8 +42,9 @@ let resolveCode1_8 = '' import Pkg.API: handle_package_input! - import Pkg.Types: PRESERVE_NONE, project_deps_resolve!, registry_resolve!, stdlib_resolve!, ensure_resolved + import Pkg.Types: PRESERVE_NONE, UUID, VersionSpec, project_deps_resolve!, registry_resolve!, stdlib_resolve!, ensure_resolved import Pkg.Operations: _resolve, assert_can_add, update_package_add + import TOML foreach(handle_package_input!, pkgs) @@ -54,6 +55,18 @@ let for pkg in pkgs if pkg.name in keys(overrides) pkg.path = overrides[pkg.name] + + # Try to read the UUID from $(pkg.path)/Project.toml. If successful, put the package into ctx.env.project.deps. + # This is necessary for the ensure_resolved call below to succeed, and will allow us to use an override even + # if it does not appear in the registry. + # See https://github.com/NixOS/nixpkgs/issues/279853 + project_toml = joinpath(pkg.path, "Project.toml") + if isfile(project_toml) + toml_data = TOML.parsefile(project_toml) + if haskey(toml_data, "uuid") + ctx.env.project.deps[pkg.name] = UUID(toml_data["uuid"]) + end + end end end diff --git a/pkgs/development/julia-modules/python/minimal_registry.py b/pkgs/development/julia-modules/python/minimal_registry.py index c9527f0ef809..c42aa2a9497d 100755 --- a/pkgs/development/julia-modules/python/minimal_registry.py +++ b/pkgs/development/julia-modules/python/minimal_registry.py @@ -56,7 +56,8 @@ for (uuid, versions) in uuid_to_versions.items(): # Write nothing in Compat.toml, because we've already resolved everything with open(out_path / path / Path("Deps.toml"), "w") as f: f.write('["%s"]\n' % info["version"]) - toml.dump(project["deps"], f) + if "deps" in project: + toml.dump(project["deps"], f) with open(out_path / path / Path("Versions.toml"), "w") as f: f.write('["%s"]\n' % info["version"]) f.write('git-tree-sha1 = "%s"\n' % info["treehash"]) diff --git a/pkgs/development/julia-modules/tests/julia-top-n/app/Main.hs b/pkgs/development/julia-modules/tests/julia-top-n/app/Main.hs index 55f5fd3cfeef..c8d935a608b8 100644 --- a/pkgs/development/julia-modules/tests/julia-top-n/app/Main.hs +++ b/pkgs/development/julia-modules/tests/julia-top-n/app/Main.hs @@ -5,6 +5,7 @@ {-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE QuasiQuotes #-} +{-# LANGUAGE RankNTypes #-} {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE ViewPatterns #-} @@ -19,7 +20,7 @@ import qualified Data.Aeson.KeyMap as HM import qualified Data.ByteString.Lazy.Char8 as BL8 import qualified Data.List as L import Data.String.Interpolate -import Data.Text as T +import Data.Text as T hiding (count) import qualified Data.Vector as V import qualified Data.Yaml as Yaml import GHC.Generics @@ -60,34 +61,55 @@ julia = Label main :: IO () main = do clo <- parseCommandLineArgs argsParser (return ()) - let Args {..} = optUserOptions clo + let args@(Args {..}) = optUserOptions clo namesAndCounts :: [NameAndCount] <- Yaml.decodeFileEither countFilePath >>= \case Left err -> throwIO $ userError ("Couldn't decode names and counts YAML file: " <> show err) Right x -> pure x - runSandwichWithCommandLineArgs' defaultOptions argsParser $ + runSandwichWithCommandLineArgs' defaultOptions argsParser $ do + miscTests args + describe ("Building environments for top " <> show topN <> " Julia packages") $ parallelN parallelism $ forM_ (L.take topN namesAndCounts) $ \(NameAndCount {..}) -> - introduce' (defaultNodeOptions { nodeOptionsVisibilityThreshold = 0 }) (T.unpack name) julia (newMVar Nothing) (const $ return ()) $ do - it "Builds" $ do - let cp = proc "nix" ["build", "--impure", "--no-link", "--json", "--expr" - , [i|with import ../../../../. {}; #{juliaAttr}.withPackages ["#{name}"]|] - ] - output <- readCreateProcessWithLogging cp "" - juliaPath <- case A.eitherDecode (BL8.pack output) of - Right (A.Array ((V.!? 0) -> Just (A.Object (aesonLookup "outputs" -> Just (A.Object (aesonLookup "out" -> Just (A.String t))))))) -> pure (JuliaPath ((T.unpack t) "bin" "julia")) - x -> expectationFailure ("Couldn't parse output: " <> show x) + testExpr args name [i|#{juliaAttr}.withPackages ["#{name}"]|] - getContext julia >>= flip modifyMVar_ (const $ return (Just juliaPath)) +miscTests :: Args -> SpecFree ctx IO () +miscTests args@(Args {..}) = describe "Misc tests" $ do + describe "works for a package outside the General registry" $ do + testExpr args "HelloWorld" [iii|(#{juliaAttr}.withPackages.override { + packageOverrides = { + "HelloWorld" = pkgs.fetchFromGitHub { + owner = "codedownio"; + repo = "HelloWorld.jl"; + rev = "9b41c55df76eb87830dd3bd0b5601ee2582a37c6"; + sha256 = "sha256-G+xpMRb0RopW/xWA8KCFF/S8wuHTQbpj0qwm9CihfSc="; + }; + }; + }) [ "HelloWorld" ]|] - it "Uses" $ do - getContext julia >>= readMVar >>= \case - Nothing -> expectationFailure "Build step failed." - Just (JuliaPath juliaPath) -> do - let cp = proc juliaPath ["-e", "using " <> T.unpack name] - createProcessWithLogging cp >>= waitForProcess >>= (`shouldBe` ExitSuccess) +-- * Low-level -aesonLookup :: Text -> HM.KeyMap v -> Maybe v -aesonLookup = HM.lookup . A.fromText +testExpr :: Args -> Text -> String -> SpecFree ctx IO () +testExpr _args name expr = do + introduce' (defaultNodeOptions { nodeOptionsVisibilityThreshold = 0 }) (T.unpack name) julia (newMVar Nothing) (const $ return ()) $ do + it "Builds" $ do + let cp = proc "nix" ["build", "--impure", "--no-link", "--json", "--expr", [i|with import ../../../../. {}; #{expr}|]] + output <- readCreateProcessWithLogging cp "" + juliaPath <- case A.eitherDecode (BL8.pack output) of + Right (A.Array ((V.!? 0) -> Just (A.Object (aesonLookup "outputs" -> Just (A.Object (aesonLookup "out" -> Just (A.String t))))))) -> pure (JuliaPath ((T.unpack t) "bin" "julia")) + x -> expectationFailure ("Couldn't parse output: " <> show x) + + getContext julia >>= flip modifyMVar_ (const $ return (Just juliaPath)) + + it "Uses" $ do + getContext julia >>= readMVar >>= \case + Nothing -> expectationFailure "Build step failed." + Just (JuliaPath juliaPath) -> do + let cp = proc juliaPath ["-e", "using " <> T.unpack name] + createProcessWithLogging cp >>= waitForProcess >>= (`shouldBe` ExitSuccess) + + where + aesonLookup :: Text -> HM.KeyMap v -> Maybe v + aesonLookup = HM.lookup . A.fromText