Merge pull request #11213 from NixOS/backport-11171-to-2.22-maintenance

[Backport 2.22-maintenance] Increase download buffer size and improve tarball import logging
This commit is contained in:
Eelco Dolstra 2024-07-29 16:20:36 +02:00 committed by GitHub
commit 184b10f3d6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 28 additions and 3 deletions

View File

@ -248,10 +248,15 @@ struct GitArchiveInputScheme : InputScheme
getFileTransfer()->download(std::move(req), sink); getFileTransfer()->download(std::move(req), sink);
}); });
auto act = std::make_unique<Activity>(*logger, lvlInfo, actUnknown,
fmt("unpacking '%s' into the Git cache", input.to_string()));
TarArchive archive { *source }; TarArchive archive { *source };
auto parseSink = getTarballCache()->getFileSystemObjectSink(); auto parseSink = getTarballCache()->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink); auto lastModified = unpackTarfileToSink(archive, *parseSink);
act.reset();
TarballInfo tarballInfo { TarballInfo tarballInfo {
.treeHash = parseSink->sync(), .treeHash = parseSink->sync(),
.lastModified = lastModified .lastModified = lastModified

View File

@ -155,6 +155,9 @@ DownloadTarballResult downloadTarball(
// TODO: fall back to cached value if download fails. // TODO: fall back to cached value if download fails.
auto act = std::make_unique<Activity>(*logger, lvlInfo, actUnknown,
fmt("unpacking '%s' into the Git cache", url));
AutoDelete cleanupTemp; AutoDelete cleanupTemp;
/* Note: if the download is cached, `importTarball()` will receive /* Note: if the download is cached, `importTarball()` will receive
@ -179,6 +182,8 @@ DownloadTarballResult downloadTarball(
auto parseSink = getTarballCache()->getFileSystemObjectSink(); auto parseSink = getTarballCache()->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink); auto lastModified = unpackTarfileToSink(archive, *parseSink);
act.reset();
auto res(_res->lock()); auto res(_res->lock());
Attrs infoAttrs; Attrs infoAttrs;

View File

@ -70,7 +70,10 @@ struct curlFileTransfer : public FileTransfer
curl_off_t writtenToSink = 0; curl_off_t writtenToSink = 0;
std::chrono::steady_clock::time_point startTime = std::chrono::steady_clock::now();
inline static const std::set<long> successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */}; inline static const std::set<long> successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */};
/* Get the HTTP status code, or 0 for other protocols. */ /* Get the HTTP status code, or 0 for other protocols. */
long getHTTPStatus() long getHTTPStatus()
{ {
@ -372,10 +375,14 @@ struct curlFileTransfer : public FileTransfer
void finish(CURLcode code) void finish(CURLcode code)
{ {
auto finishTime = std::chrono::steady_clock::now();
auto httpStatus = getHTTPStatus(); auto httpStatus = getHTTPStatus();
debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s",
request.verb(), request.uri, code, httpStatus, result.bodySize); request.verb(), request.uri, code, httpStatus, result.bodySize,
std::chrono::duration_cast<std::chrono::milliseconds>(finishTime - startTime).count() / 1000.0f
);
appendCurrentUrl(); appendCurrentUrl();
@ -845,8 +852,10 @@ void FileTransfer::download(
buffer). We don't wait forever to prevent stalling the buffer). We don't wait forever to prevent stalling the
download thread. (Hopefully sleeping will throttle the download thread. (Hopefully sleeping will throttle the
sender.) */ sender.) */
if (state->data.size() > 1024 * 1024) { if (state->data.size() > fileTransferSettings.downloadBufferSize) {
debug("download buffer is full; going to sleep"); debug("download buffer is full; going to sleep");
static bool haveWarned = false;
warnOnce(haveWarned, "download buffer is full; consider increasing the 'download-buffer-size' setting");
state.wait_for(state->request, std::chrono::seconds(10)); state.wait_for(state->request, std::chrono::seconds(10));
} }

View File

@ -45,6 +45,12 @@ struct FileTransferSettings : Config
Setting<unsigned int> tries{this, 5, "download-attempts", Setting<unsigned int> tries{this, 5, "download-attempts",
"How often Nix will attempt to download a file before giving up."}; "How often Nix will attempt to download a file before giving up."};
Setting<size_t> downloadBufferSize{this, 64 * 1024 * 1024, "download-buffer-size",
R"(
The size of Nix's internal download buffer during `curl` transfers. If data is
not processed quickly enough to exceed the size of this buffer, downloads may stall.
)"};
}; };
extern FileTransferSettings fileTransferSettings; extern FileTransferSettings fileTransferSettings;