Merge pull request #324004 from codedownio/julia-withpackages-override-fix

julia.withPackages: fix for overrides not in a registry

pkgs/development/julia-modules/package-closure.nix

@@ -42,8 +42,9 @@ let
 
   resolveCode1_8 = ''
     import Pkg.API: handle_package_input!
-    import Pkg.Types: PRESERVE_NONE, project_deps_resolve!, registry_resolve!, stdlib_resolve!, ensure_resolved
+    import Pkg.Types: PRESERVE_NONE, UUID, VersionSpec, project_deps_resolve!, registry_resolve!, stdlib_resolve!, ensure_resolved
     import Pkg.Operations: _resolve, assert_can_add, update_package_add
+    import TOML
 
     foreach(handle_package_input!, pkgs)
 
@@ -54,6 +55,18 @@ let
     for pkg in pkgs
       if pkg.name in keys(overrides)
         pkg.path = overrides[pkg.name]
+
+        # Try to read the UUID from $(pkg.path)/Project.toml. If successful, put the package into ctx.env.project.deps.
+        # This is necessary for the ensure_resolved call below to succeed, and will allow us to use an override even
+        # if it does not appear in the registry.
+        # See https://github.com/NixOS/nixpkgs/issues/279853
+        project_toml = joinpath(pkg.path, "Project.toml")
+        if isfile(project_toml)
+          toml_data = TOML.parsefile(project_toml)
+          if haskey(toml_data, "uuid")
+            ctx.env.project.deps[pkg.name] = UUID(toml_data["uuid"])
+          end
+        end
       end
     end
 

pkgs/development/julia-modules/python/minimal_registry.py

@@ -56,7 +56,8 @@ for (uuid, versions) in uuid_to_versions.items():
       # Write nothing in Compat.toml, because we've already resolved everything
     with open(out_path / path / Path("Deps.toml"), "w") as f:
       f.write('["%s"]\n' % info["version"])
-      toml.dump(project["deps"], f)
+      if "deps" in project:
+        toml.dump(project["deps"], f)
     with open(out_path / path / Path("Versions.toml"), "w") as f:
       f.write('["%s"]\n' % info["version"])
       f.write('git-tree-sha1 = "%s"\n' % info["treehash"])

pkgs/development/julia-modules/tests/julia-top-n/app/Main.hs

@@ -5,6 +5,7 @@
 {-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE QuasiQuotes #-}
+{-# LANGUAGE RankNTypes #-}
 {-# LANGUAGE RecordWildCards #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE ViewPatterns #-}
@@ -19,7 +20,7 @@ import qualified Data.Aeson.KeyMap          as HM
 import qualified Data.ByteString.Lazy.Char8 as BL8
 import qualified Data.List as L
 import Data.String.Interpolate
-import Data.Text as T
+import Data.Text as T hiding (count)
 import qualified Data.Vector as V
 import qualified Data.Yaml as Yaml
 import GHC.Generics
@@ -60,34 +61,55 @@ julia = Label
 main :: IO ()
 main = do
   clo <- parseCommandLineArgs argsParser (return ())
-  let Args {..} = optUserOptions clo
+  let args@(Args {..}) = optUserOptions clo
 
   namesAndCounts :: [NameAndCount] <- Yaml.decodeFileEither countFilePath >>= \case
     Left err -> throwIO $ userError ("Couldn't decode names and counts YAML file: " <> show err)
     Right x -> pure x
 
-  runSandwichWithCommandLineArgs' defaultOptions argsParser $
+  runSandwichWithCommandLineArgs' defaultOptions argsParser $ do
+    miscTests args
+
     describe ("Building environments for top " <> show topN <> " Julia packages") $
       parallelN parallelism $
         forM_ (L.take topN namesAndCounts) $ \(NameAndCount {..}) ->
-          introduce' (defaultNodeOptions { nodeOptionsVisibilityThreshold = 0 }) (T.unpack name) julia (newMVar Nothing) (const $ return ()) $ do
+          testExpr args name [i|#{juliaAttr}.withPackages ["#{name}"]|]
-            it "Builds" $ do
-              let cp = proc "nix" ["build", "--impure", "--no-link", "--json", "--expr"
-                                  , [i|with import ../../../../. {}; #{juliaAttr}.withPackages ["#{name}"]|]
-                                  ]
-              output <- readCreateProcessWithLogging cp ""
-              juliaPath <- case A.eitherDecode (BL8.pack output) of
-                Right (A.Array ((V.!? 0) -> Just (A.Object (aesonLookup "outputs" -> Just (A.Object (aesonLookup "out" -> Just (A.String t))))))) -> pure (JuliaPath ((T.unpack t) </> "bin" </> "julia"))
-                x -> expectationFailure ("Couldn't parse output: " <> show x)
 
-              getContext julia >>= flip modifyMVar_ (const $ return (Just juliaPath))
+miscTests :: Args -> SpecFree ctx IO ()
+miscTests args@(Args {..}) = describe "Misc tests" $ do
+  describe "works for a package outside the General registry" $ do
+    testExpr args "HelloWorld" [iii|(#{juliaAttr}.withPackages.override {
+                                      packageOverrides = {
+                                        "HelloWorld" = pkgs.fetchFromGitHub {
+                                          owner = "codedownio";
+                                          repo = "HelloWorld.jl";
+                                          rev = "9b41c55df76eb87830dd3bd0b5601ee2582a37c6";
+                                          sha256 = "sha256-G+xpMRb0RopW/xWA8KCFF/S8wuHTQbpj0qwm9CihfSc=";
+                                        };
+                                      };
+                                    }) [ "HelloWorld" ]|]
 
-            it "Uses" $ do
+-- * Low-level
-              getContext julia >>= readMVar >>= \case
-                Nothing -> expectationFailure "Build step failed."
-                Just (JuliaPath juliaPath) -> do
-                  let cp = proc juliaPath ["-e", "using " <> T.unpack name]
-                  createProcessWithLogging cp >>= waitForProcess >>= (`shouldBe` ExitSuccess)
 
-aesonLookup :: Text -> HM.KeyMap v -> Maybe v
+testExpr :: Args -> Text -> String -> SpecFree ctx IO ()
-aesonLookup = HM.lookup . A.fromText
+testExpr _args name expr = do
+  introduce' (defaultNodeOptions { nodeOptionsVisibilityThreshold = 0 }) (T.unpack name) julia (newMVar Nothing) (const $ return ()) $ do
+    it "Builds" $ do
+      let cp = proc "nix" ["build", "--impure", "--no-link", "--json", "--expr", [i|with import ../../../../. {}; #{expr}|]]
+      output <- readCreateProcessWithLogging cp ""
+      juliaPath <- case A.eitherDecode (BL8.pack output) of
+        Right (A.Array ((V.!? 0) -> Just (A.Object (aesonLookup "outputs" -> Just (A.Object (aesonLookup "out" -> Just (A.String t))))))) -> pure (JuliaPath ((T.unpack t) </> "bin" </> "julia"))
+        x -> expectationFailure ("Couldn't parse output: " <> show x)
+
+      getContext julia >>= flip modifyMVar_ (const $ return (Just juliaPath))
+
+    it "Uses" $ do
+      getContext julia >>= readMVar >>= \case
+        Nothing -> expectationFailure "Build step failed."
+        Just (JuliaPath juliaPath) -> do
+          let cp = proc juliaPath ["-e", "using " <> T.unpack name]
+          createProcessWithLogging cp >>= waitForProcess >>= (`shouldBe` ExitSuccess)
+
+  where
+    aesonLookup :: Text -> HM.KeyMap v -> Maybe v
+    aesonLookup = HM.lookup . A.fromText