From 0c5eac9c4550a6de2cd829d25e628f779e2a29c7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 31 Oct 2023 15:59:25 +0100 Subject: [PATCH] Git fetcher: Handle submodules for workdirs --- src/libfetchers/git-utils.cc | 83 +++++++++++-------- src/libfetchers/git-utils.hh | 27 ++++-- src/libfetchers/git.cc | 49 +++++++++-- tests/functional/flakes/flake-in-submodule.sh | 14 +++- 4 files changed, 119 insertions(+), 54 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 5e3e6dae4..5b14cfdb1 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -216,6 +216,43 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return toHash(*oid); } + std::vector parseSubmodules(const CanonPath & configFile) + { + GitConfig config; + if (git_config_open_ondisk(Setter(config), configFile.abs().c_str())) + throw Error("parsing .gitmodules file: %s", git_error_last()->message); + + ConfigIterator it; + if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$")) + throw Error("iterating over .gitmodules: %s", git_error_last()->message); + + std::map entries; + + while (true) { + git_config_entry * entry = nullptr; + if (auto err = git_config_next(&entry, it.get())) { + if (err == GIT_ITEROVER) break; + throw Error("iterating over .gitmodules: %s", git_error_last()->message); + } + entries.emplace(entry->name + 10, entry->value); + } + + std::vector result; + + for (auto & [key, value] : entries) { + if (!hasSuffix(key, ".path")) continue; + std::string key2(key, 0, key.size() - 5); + auto path = CanonPath(value); + result.push_back(Submodule { + .path = path, + .url = entries[key2 + ".url"], + .branch = entries[key2 + ".branch"], + }); + } + + return result; + } + WorkdirInfo getWorkdirInfo() override { WorkdirInfo info; @@ -246,6 +283,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback)) throw Error("getting working directory status: %s", git_error_last()->message); + /* Get submodule info. */ + auto modulesFile = path + ".gitmodules"; + if (pathExists(modulesFile.abs())) + info.submodules = parseSubmodules(modulesFile); + return info; } @@ -261,7 +303,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return std::nullopt; } - std::vector getSubmodules(const Hash & rev) override; + std::vector> getSubmodules(const Hash & rev) override; std::string resolveSubmoduleUrl(const std::string & url) override { @@ -521,7 +563,7 @@ ref GitRepoImpl::getAccessor(const Hash & rev) return make_ref(ref(shared_from_this()), rev); } -std::vector GitRepoImpl::getSubmodules(const Hash & rev) +std::vector> GitRepoImpl::getSubmodules(const Hash & rev) { /* Read the .gitmodules files from this revision. */ CanonPath modulesFile(".gitmodules"); @@ -529,44 +571,17 @@ std::vector GitRepoImpl::getSubmodules(const Hash & rev) auto accessor = getAccessor(rev); if (!accessor->pathExists(modulesFile)) return {}; - /* Parse it. */ + /* Parse it and get the revision of each submodule. */ auto configS = accessor->readFile(modulesFile); auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules"); writeFull(fdTemp.get(), configS); - GitConfig config; - if (git_config_open_ondisk(Setter(config), pathTemp.c_str())) - throw Error("parsing .gitmodules file: %s", git_error_last()->message); + std::vector> result; - ConfigIterator it; - if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$")) - throw Error("iterating over .gitmodules: %s", git_error_last()->message); - - std::map entries; - - while (true) { - git_config_entry * entry = nullptr; - if (auto err = git_config_next(&entry, it.get())) { - if (err == GIT_ITEROVER) break; - throw Error("iterating over .gitmodules: %s", git_error_last()->message); - } - entries.emplace(entry->name + 10, entry->value); - } - - std::vector result; - - for (auto & [key, value] : entries) { - if (!hasSuffix(key, ".path")) continue; - std::string key2(key, 0, key.size() - 5); - auto path = CanonPath(value); - auto rev = accessor.dynamic_pointer_cast()->getSubmoduleRev(path); - result.push_back(Submodule { - .path = path, - .url = entries[key2 + ".url"], - .branch = entries[key2 + ".branch"], - .rev = rev, - }); + for (auto & submodule : parseSubmodules(CanonPath(pathTemp))) { + auto rev = accessor.dynamic_pointer_cast()->getSubmoduleRev(submodule.path); + result.push_back({std::move(submodule), rev}); } return result; diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 55e7ef969..a425e5814 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -20,6 +20,16 @@ struct GitRepo /* Return the commit hash to which a ref points. */ virtual Hash resolveRef(std::string ref) = 0; + /** + * Info about a submodule. + */ + struct Submodule + { + CanonPath path; + std::string url; + std::string branch; + }; + struct WorkdirInfo { bool isDirty = false; @@ -31,6 +41,9 @@ struct GitRepo /* All files in the working directory that are unchanged, modified or added, but excluding deleted files. */ std::set files; + + /* The submodules listed in .gitmodules of this workdir. */ + std::vector submodules; }; virtual WorkdirInfo getWorkdirInfo() = 0; @@ -38,15 +51,11 @@ struct GitRepo /* Get the ref that HEAD points to. */ virtual std::optional getWorkdirRef() = 0; - struct Submodule - { - CanonPath path; - std::string url; - std::string branch; - Hash rev; - }; - - virtual std::vector getSubmodules(const Hash & rev) = 0; + /** + * Return the submodules of this repo at the indicated revision, + * along with the revision of each submodule. + */ + virtual std::vector> getSubmodules(const Hash & rev) = 0; virtual std::string resolveSubmoduleUrl(const std::string & url) = 0; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index a66a51cca..5471eb260 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -525,16 +525,16 @@ struct GitInputScheme : InputScheme if (repoInfo.submodules) { std::map> mounts; - for (auto & submodule : repo->getSubmodules(rev)) { + for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) { auto resolved = repo->resolveSubmoduleUrl(submodule.url); debug("Git submodule %s: %s %s %s -> %s", - submodule.path, submodule.url, submodule.branch, submodule.rev.gitRev(), resolved); + submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved); fetchers::Attrs attrs; attrs.insert_or_assign("type", "git"); attrs.insert_or_assign("url", resolved); if (submodule.branch != "") attrs.insert_or_assign("ref", submodule.branch); - attrs.insert_or_assign("rev", submodule.rev.gitRev()); + attrs.insert_or_assign("rev", submoduleRev.gitRev()); auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.scheme->getAccessor(store, submoduleInput); @@ -557,9 +557,45 @@ struct GitInputScheme : InputScheme } std::pair, Input> getAccessorFromWorkdir( + ref store, RepoInfo & repoInfo, Input && input) const { + if (repoInfo.submodules) + /* Create mountpoints for the submodules. */ + for (auto & submodule : repoInfo.workdirInfo.submodules) + repoInfo.workdirInfo.files.insert(submodule.path); + + ref accessor = + makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url)); + + /* If the repo has submodules, return a union input accessor + consisting of the accessor for the top-level repo and the + accessors for the submodule workdirs. */ + if (repoInfo.submodules && !repoInfo.workdirInfo.submodules.empty()) { + std::map> mounts; + + for (auto & submodule : repoInfo.workdirInfo.submodules) { + auto submodulePath = CanonPath(repoInfo.url) + submodule.path; + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "git"); + attrs.insert_or_assign("url", submodulePath.abs()); + auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); + auto [submoduleAccessor, submoduleInput2] = + submoduleInput.scheme->getAccessor(store, submoduleInput); + + /* If the submodule is dirty, mark this repo dirty as + well. */ + if (!submoduleInput2.getRev()) + repoInfo.workdirInfo.isDirty = true; + + mounts.insert_or_assign(submodule.path, submoduleAccessor); + } + + mounts.insert_or_assign(CanonPath::root, accessor); + accessor = makeUnionInputAccessor(std::move(mounts)); + } + if (!repoInfo.workdirInfo.isDirty) { if (auto ref = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef()) input.attrs.insert_or_assign("ref", *ref); @@ -588,10 +624,7 @@ struct GitInputScheme : InputScheme input.locked = true; // FIXME - return { - makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url)), - std::move(input) - }; + return {accessor, std::move(input)}; } std::pair, Input> getAccessor(ref store, const Input & _input) const override @@ -603,7 +636,7 @@ struct GitInputScheme : InputScheme if (input.getRef() || input.getRev() || !repoInfo.isLocal) return getAccessorFromCommit(store, repoInfo, std::move(input)); else - return getAccessorFromWorkdir(repoInfo, std::move(input)); + return getAccessorFromWorkdir(store, repoInfo, std::move(input)); } }; diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index 6e24a80c1..85a4d3389 100644 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -46,8 +46,16 @@ echo '"expression in root repo"' > $rootRepo/root.nix git -C $rootRepo add root.nix git -C $rootRepo commit -m "Add root.nix" -# FIXME +flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule + # Flake can live inside a submodule and can be accessed via ?dir=submodule -#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#sub ) = '"expression in submodule"' ]] +[[ $(nix eval --json $flakeref#sub ) = '"expression in submodule"' ]] + # The flake can access content outside of the submodule -#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#root ) = '"expression in root repo"' ]] +[[ $(nix eval --json $flakeref#root ) = '"expression in root repo"' ]] + +# Check that dirtying a submodule makes the entire thing dirty. +[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) != null ]] +echo '"foo"' > $rootRepo/submodule/sub.nix +[[ $(nix eval --json $flakeref#sub ) = '"foo"' ]] +[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) = null ]]