julia.withPackages: fix for overrides not in a registry

pkgs/development/julia-modules/package-closure.nix

@@ -42,8 +42,9 @@ let
 
   resolveCode1_8 = ''
     import Pkg.API: handle_package_input!
-    import Pkg.Types: PRESERVE_NONE, project_deps_resolve!, registry_resolve!, stdlib_resolve!, ensure_resolved
+    import Pkg.Types: PRESERVE_NONE, UUID, VersionSpec, project_deps_resolve!, registry_resolve!, stdlib_resolve!, ensure_resolved
     import Pkg.Operations: _resolve, assert_can_add, update_package_add
+    import TOML
 
     foreach(handle_package_input!, pkgs)
 
@@ -54,6 +55,18 @@ let
     for pkg in pkgs
       if pkg.name in keys(overrides)
         pkg.path = overrides[pkg.name]
+
+        # Try to read the UUID from $(pkg.path)/Project.toml. If successful, put the package into ctx.env.project.deps.
+        # This is necessary for the ensure_resolved call below to succeed, and will allow us to use an override even
+        # if it does not appear in the registry.
+        # See https://github.com/NixOS/nixpkgs/issues/279853
+        project_toml = joinpath(pkg.path, "Project.toml")
+        if isfile(project_toml)
+          toml_data = TOML.parsefile(project_toml)
+          if haskey(toml_data, "uuid")
+            ctx.env.project.deps[pkg.name] = UUID(toml_data["uuid"])
+          end
+        end
       end
     end
 

pkgs/development/julia-modules/python/minimal_registry.py

@@ -56,6 +56,7 @@ for (uuid, versions) in uuid_to_versions.items():
       # Write nothing in Compat.toml, because we've already resolved everything
     with open(out_path / path / Path("Deps.toml"), "w") as f:
       f.write('["%s"]\n' % info["version"])
+      if "deps" in project:
         toml.dump(project["deps"], f)
     with open(out_path / path / Path("Versions.toml"), "w") as f:
       f.write('["%s"]\n' % info["version"])

pkgs/development/julia-modules/tests/julia-top-n/app/Main.hs

@@ -5,6 +5,7 @@
 {-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE QuasiQuotes #-}
+{-# LANGUAGE RankNTypes #-}
 {-# LANGUAGE RecordWildCards #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE ViewPatterns #-}
@@ -19,7 +20,7 @@ import qualified Data.Aeson.KeyMap          as HM
 import qualified Data.ByteString.Lazy.Char8 as BL8
 import qualified Data.List as L
 import Data.String.Interpolate
-import Data.Text as T
+import Data.Text as T hiding (count)
 import qualified Data.Vector as V
 import qualified Data.Yaml as Yaml
 import GHC.Generics
@@ -60,21 +61,41 @@ julia = Label
 main :: IO ()
 main = do
   clo <- parseCommandLineArgs argsParser (return ())
-  let Args {..} = optUserOptions clo
+  let args@(Args {..}) = optUserOptions clo
 
   namesAndCounts :: [NameAndCount] <- Yaml.decodeFileEither countFilePath >>= \case
     Left err -> throwIO $ userError ("Couldn't decode names and counts YAML file: " <> show err)
     Right x -> pure x
 
-  runSandwichWithCommandLineArgs' defaultOptions argsParser $
+  runSandwichWithCommandLineArgs' defaultOptions argsParser $ do
+    miscTests args
+
     describe ("Building environments for top " <> show topN <> " Julia packages") $
       parallelN parallelism $
         forM_ (L.take topN namesAndCounts) $ \(NameAndCount {..}) ->
+          testExpr args name [i|#{juliaAttr}.withPackages ["#{name}"]|]
+
+miscTests :: Args -> SpecFree ctx IO ()
+miscTests args@(Args {..}) = describe "Misc tests" $ do
+  describe "works for a package outside the General registry" $ do
+    testExpr args "HelloWorld" [iii|(#{juliaAttr}.withPackages.override {
+                                      packageOverrides = {
+                                        "HelloWorld" = pkgs.fetchFromGitHub {
+                                          owner = "codedownio";
+                                          repo = "HelloWorld.jl";
+                                          rev = "9b41c55df76eb87830dd3bd0b5601ee2582a37c6";
+                                          sha256 = "sha256-G+xpMRb0RopW/xWA8KCFF/S8wuHTQbpj0qwm9CihfSc=";
+                                        };
+                                      };
+                                    }) [ "HelloWorld" ]|]
+
+-- * Low-level
+
+testExpr :: Args -> Text -> String -> SpecFree ctx IO ()
+testExpr _args name expr = do
   introduce' (defaultNodeOptions { nodeOptionsVisibilityThreshold = 0 }) (T.unpack name) julia (newMVar Nothing) (const $ return ()) $ do
     it "Builds" $ do
-              let cp = proc "nix" ["build", "--impure", "--no-link", "--json", "--expr"
-                                  , [i|with import ../../../../. {}; #{juliaAttr}.withPackages ["#{name}"]|]
-                                  ]
+      let cp = proc "nix" ["build", "--impure", "--no-link", "--json", "--expr", [i|with import ../../../../. {}; #{expr}|]]
       output <- readCreateProcessWithLogging cp ""
       juliaPath <- case A.eitherDecode (BL8.pack output) of
         Right (A.Array ((V.!? 0) -> Just (A.Object (aesonLookup "outputs" -> Just (A.Object (aesonLookup "out" -> Just (A.String t))))))) -> pure (JuliaPath ((T.unpack t) </> "bin" </> "julia"))
@@ -89,5 +110,6 @@ main = do
           let cp = proc juliaPath ["-e", "using " <> T.unpack name]
           createProcessWithLogging cp >>= waitForProcess >>= (`shouldBe` ExitSuccess)
 
-aesonLookup :: Text -> HM.KeyMap v -> Maybe v
-aesonLookup = HM.lookup . A.fromText
+  where
+    aesonLookup :: Text -> HM.KeyMap v -> Maybe v
+    aesonLookup = HM.lookup . A.fromText