From deb1d7ac1ed8475d8dd1d1accf045203bfb4ee58 Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Fri, 24 Mar 2023 14:27:12 +0100 Subject: [PATCH 1/2] ceph: 16.2.10 -> 17.2.5 Updates Ceph from the Pacific to the Quincy release tree. Upgrade to boost179 and python310, cleans up the python environment, builds with more system libraries. Disables parquet support for the radosgw, because I couldn't get it to build. Also properly enable the testsuite on ceph-common and generally unroll and sort lists for a better overview and history. Co-Authored-By: name_snrl --- .../ceph/0000-fix-SPDK-build-env.patch | 11 - pkgs/tools/filesystems/ceph/default.nix | 317 ++++++++++++------ 2 files changed, 214 insertions(+), 114 deletions(-) delete mode 100644 pkgs/tools/filesystems/ceph/0000-fix-SPDK-build-env.patch diff --git a/pkgs/tools/filesystems/ceph/0000-fix-SPDK-build-env.patch b/pkgs/tools/filesystems/ceph/0000-fix-SPDK-build-env.patch deleted file mode 100644 index a117408b000f..000000000000 --- a/pkgs/tools/filesystems/ceph/0000-fix-SPDK-build-env.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- a/cmake/modules/BuildSPDK.cmake -+++ b/cmake/modules/BuildSPDK.cmake -@@ -35,7 +35,7 @@ macro(build_spdk) - # unset $CFLAGS, otherwise it will interfere with how SPDK sets - # its include directory. - # unset $LDFLAGS, otherwise SPDK will fail to mock some functions. -- BUILD_COMMAND env -i PATH=$ENV{PATH} CC=${CMAKE_C_COMPILER} ${make_cmd} EXTRA_CFLAGS="${spdk_CFLAGS}" -+ BUILD_COMMAND env -i PATH=$ENV{PATH} CC=${CMAKE_C_COMPILER} ${make_cmd} EXTRA_CFLAGS="${spdk_CFLAGS}" C_OPT="-mssse3" - BUILD_IN_SOURCE 1 - INSTALL_COMMAND "true") - unset(make_cmd) diff --git a/pkgs/tools/filesystems/ceph/default.nix b/pkgs/tools/filesystems/ceph/default.nix index 15e5071f963a..34befedf18b7 100644 --- a/pkgs/tools/filesystems/ceph/default.nix +++ b/pkgs/tools/filesystems/ceph/default.nix @@ -1,44 +1,82 @@ -{ lib, stdenv, runCommand, fetchurl, fetchpatch +{ lib +, stdenv +, runCommand +, fetchurl +, fetchFromGitHub + +# Build time +, cmake , ensureNewerSourcesHook -, cmake, pkg-config -, which, git -, boost175, xz -, libxml2, zlib, lz4 -, openldap, lttng-ust -, babeltrace, gperf -, gtest -, cunit, snappy -, makeWrapper -, leveldb, oath-toolkit -, libnl, libcap_ng -, rdkafka -, nixosTests -, cryptsetup -, sqlite -, lua -, icu -, bzip2 -, doxygen -, graphviz , fmt -, python39 +, git +, makeWrapper +, pkg-config +, which + +# Tests +, nixosTests + +# Runtime dependencies +, arrow-cpp +, babeltrace +, boost179 +, bzip2 +, cryptsetup +, cimg +, cunit +, doxygen +, gperf +, graphviz +, gtest +, icu +, jsoncpp +, libcap_ng +, libnl +, libxml2 +, lttng-ust +, lua +, lz4 +, oath-toolkit +, openldap +, python310 +, rdkafka +, rocksdb +, snappy +, sqlite +, utf8proc +, zlib +, zstd # Optional Dependencies -, yasm ? null, fcgi ? null, expat ? null -, curl ? null, fuse ? null -, libedit ? null, libatomic_ops ? null +, curl ? null +, expat ? null +, fuse ? null +, libatomic_ops ? null +, libedit ? null , libs3 ? null +, yasm ? null # Mallocs -, jemalloc ? null, gperftools ? null +, gperftools ? null +, jemalloc ? null # Crypto Dependencies , cryptopp ? null -, nss ? null, nspr ? null +, nspr ? null +, nss ? null # Linux Only Dependencies -, linuxHeaders, util-linux, libuuid, udev, keyutils, rdma-core, rabbitmq-c -, libaio ? null, libxfs ? null, zfs ? null, liburing ? null +, linuxHeaders +, util-linux +, libuuid +, udev +, keyutils +, rdma-core +, rabbitmq-c +, libaio ? null +, libxfs ? null +, liburing ? null +, zfs ? null , ... }: @@ -49,7 +87,6 @@ let shouldUsePkg = pkg: if pkg != null && pkg.meta.available then pkg else null; optYasm = shouldUsePkg yasm; - optFcgi = shouldUsePkg fcgi; optExpat = shouldUsePkg expat; optCurl = shouldUsePkg curl; optFuse = shouldUsePkg fuse; @@ -68,8 +105,18 @@ let optLibxfs = shouldUsePkg libxfs; optZfs = shouldUsePkg zfs; - hasRadosgw = optFcgi != null && optExpat != null && optCurl != null && optLibedit != null; + # Downgrade rocksdb, 7.10 breaks ceph + rocksdb' = rocksdb.overrideAttrs (oldAttrs: { + version = "7.9.2"; + src = fetchFromGitHub { + owner = "facebook"; + repo = "rocksdb"; + rev = "refs/tags/v7.9.2"; + hash = "sha256-5P7IqJ14EZzDkbjaBvbix04ceGGdlWBuVFH/5dpD5VM="; + }; + }); + hasRadosgw = optExpat != null && optCurl != null && optLibedit != null; # Malloc implementation (can be jemalloc, tcmalloc or null) malloc = if optJemalloc != null then optJemalloc else optGperftools; @@ -92,20 +139,30 @@ let platforms = [ "x86_64-linux" "aarch64-linux" ]; }; - ceph-common = python.pkgs.buildPythonPackage rec{ + ceph-common = with python.pkgs; buildPythonPackage { pname = "ceph-common"; inherit src version; sourceRoot = "ceph-${version}/src/python-common"; - nativeCheckInputs = [ python.pkgs.pytest ]; - propagatedBuildInputs = with python.pkgs; [ pyyaml six ]; + propagatedBuildInputs = [ + pyyaml + ]; + + nativeCheckInputs = [ + pytestCheckHook + ]; + + disabledTests = [ + # requires network access + "test_valid_addr" + ]; meta = getMeta "Ceph common module for code shared by manager modules"; }; - # Boost 1.75 is not compatible with Python 3.10 - python = python39.override { + # Watch out for python <> boost compatibility + python = python310.override { packageOverrides = self: super: { sqlalchemy = super.sqlalchemy.overridePythonAttrs (oldAttrs: rec { version = "1.4.46"; @@ -125,91 +182,128 @@ let }; }; - boost = boost175.override { + boost = boost179.override { enablePython = true; inherit python; }; - ceph-python-env = python.withPackages (ps: [ - # Check .requires files below https://github.com/ceph/ceph/tree/main/debian for dependencies - ps.sphinx - ps.flask - ps.routes - ps.cython - ps.setuptools - ps.virtualenv - # Libraries needed by the python tools - ps.mako + # TODO: split this off in build and runtime environment + ceph-python-env = python.withPackages (ps: with ps; [ ceph-common - ps.cherrypy - ps.cmd2 - ps.colorama - ps.python-dateutil - ps.jsonpatch - ps.pecan - ps.prettytable - ps.pyopenssl - ps.pyjwt - ps.webob - ps.bcrypt - ps.scipy - ps.six - ps.pyyaml + + # build time + cython + + # debian/control + bcrypt + cherrypy + influxdb + jinja2 + kubernetes + natsort + numpy + pecan + prettytable + pyjwt + pyopenssl + python-dateutil + pyyaml + requests + routes + scikit-learn + scipy + setuptools + sphinx + virtualenv + werkzeug + + # src/pybind/mgr/requirements-required.txt + cryptography + jsonpatch + + # src/tools/cephfs/shell/setup.py + cmd2 + colorama ]); sitePackages = ceph-python-env.python.sitePackages; - version = "16.2.10"; + version = "17.2.5"; src = fetchurl { url = "http://download.ceph.com/tarballs/ceph-${version}.tar.gz"; - sha256 = "sha256-342+nUV3mCX7QJfZSnKEfnQFCJwJmVQeYnefJwW/AtU="; + hash = "sha256-NiJpwUeROvh0siSaRoRrDm+C0s61CvRiIrbd7JmRspo="; }; in rec { ceph = stdenv.mkDerivation { pname = "ceph"; inherit src version; - patches = [ - ./0000-fix-SPDK-build-env.patch - # pacific: include/buffer: include - # fixes build with gcc 12 - # https://github.com/ceph/ceph/pull/47295 - (fetchpatch { - url = "https://github.com/ceph/ceph/pull/47295/commits/df88789a38c053513d3b2a9b7d12a952fc0c9042.patch"; - hash = "sha256-je65kBfa5hR0ZKo6ZI10XmD5ZUbKj5rxlGxxI9ZJVfo="; - }) - (fetchpatch { - url = "https://github.com/ceph/ceph/pull/47295/commits/2abcbe4e47705e6e0fcc7d9d9b75625f563199af.patch"; - hash = "sha256-8sWQKoZNHuGuhzX/F+3fY4+kjsrwsfoMdVpfVSj2x5w="; - }) - (fetchpatch { - url = "https://github.com/ceph/ceph/pull/47295/commits/13dc077cf6c65a3b8c4f13d896847b9964b3fcbb.patch"; - hash = "sha256-byfiZh9OJrux/y5m3QCPg0LET6q33ZDXmp/CN+yOSQQ="; - }) - ]; - nativeBuildInputs = [ cmake - pkg-config which git python.pkgs.wrapPython makeWrapper - python.pkgs.python # for the toPythonPath function - (ensureNewerSourcesHook { year = "1980"; }) - python fmt + git + makeWrapper + pkg-config + python + python.pkgs.python # for the toPythonPath function + python.pkgs.wrapPython + which + (ensureNewerSourcesHook { year = "1980"; }) # for building docs/man-pages presumably doxygen graphviz ]; + enableParallelBuilding = true; + buildInputs = cryptoLibsMap.${cryptoStr} ++ [ - boost xz ceph-python-env libxml2 optYasm optLibatomic_ops optLibs3 - malloc zlib openldap lttng-ust babeltrace gperf gtest cunit - snappy lz4 oath-toolkit leveldb libnl libcap_ng rdkafka - cryptsetup sqlite lua icu bzip2 + arrow-cpp + babeltrace + boost + bzip2 + ceph-python-env + cimg + cryptsetup + cunit + gperf + gtest + jsoncpp + icu + libcap_ng + libnl + libxml2 + lttng-ust + lua + lz4 + malloc + oath-toolkit + openldap + optLibatomic_ops + optLibs3 + optYasm + rdkafka + rocksdb' + snappy + sqlite + utf8proc + zlib + zstd ] ++ lib.optionals stdenv.isLinux [ - linuxHeaders util-linux libuuid udev keyutils liburing optLibaio optLibxfs optZfs - # ceph 14 - rdma-core rabbitmq-c + keyutils + liburing + libuuid + linuxHeaders + optLibaio + optLibxfs + optZfs + rabbitmq-c + rdma-core + udev + util-linux ] ++ lib.optionals hasRadosgw [ - optFcgi optExpat optCurl optFuse optLibedit + optCurl + optExpat + optFuse + optLibedit ]; pythonPath = [ ceph-python-env "${placeholder "out"}/${ceph-python-env.sitePackages}" ]; @@ -226,17 +320,27 @@ in rec { ''; cmakeFlags = [ - "-DWITH_SYSTEM_ROCKSDB=OFF" # breaks Bluestore "-DCMAKE_INSTALL_DATADIR=${placeholder "lib"}/lib" - "-DWITH_SYSTEM_BOOST=ON" - "-DWITH_SYSTEM_GTEST=ON" "-DMGR_PYTHON_VERSION=${ceph-python-env.python.pythonVersion}" - "-DWITH_SYSTEMD=OFF" - "-DWITH_TESTS=OFF" - "-DWITH_CEPHFS_SHELL=ON" + "-DWITH_CEPHFS_SHELL:BOOL=ON" + "-DWITH_SYSTEMD:BOOL=OFF" + "-DWITH_TESTS:BOOL=OFF" + + # Use our own libraries, where possible + "-DWITH_SYSTEM_ARROW:BOOL=ON" + "-DWITH_SYSTEM_BOOST:BOOL=ON" + "-DWITH_SYSTEM_CIMG:BOOL=ON" + "-DWITH_SYSTEM_JSONCPP:BOOL=ON" + "-DWITH_SYSTEM_GTEST:BOOL=ON" + "-DWITH_SYSTEM_ROCKSDB:BOOL=ON" + "-DWITH_SYSTEM_UTF8PROC:BOOL=ON" + "-DWITH_SYSTEM_ZSTD:BOOL=ON" + # TODO breaks with sandbox, tries to download stuff with npm - "-DWITH_MGR_DASHBOARD_FRONTEND=OFF" + "-DWITH_MGR_DASHBOARD_FRONTEND:BOOL=OFF" + # no matching function for call to 'parquet::PageReader::Open(std::shared_ptr&, int64_t, arrow::Compression::type, parquet::MemoryPool*, parquet::CryptoContext*)' + "-DWITH_RADOSGW_SELECT_PARQUET:BOOL=OFF" # WITH_XFS has been set default ON from Ceph 16, keeping it optional in nixpkgs for now ''-DWITH_XFS=${if optLibxfs != null then "ON" else "OFF"}'' ] ++ lib.optional stdenv.isLinux "-DWITH_SYSTEM_LIBURING=ON"; @@ -259,8 +363,15 @@ in rec { meta = getMeta "Distributed storage system"; - passthru.version = version; - passthru.tests = { inherit (nixosTests) ceph-single-node ceph-multi-node ceph-single-node-bluestore; }; + passthru = { + inherit version; + tests = { + inherit (nixosTests) + ceph-multi-node + ceph-single-node + ceph-single-node-bluestore; + }; + }; }; ceph-client = runCommand "ceph-client-${version}" { From adb0c3560b93e5390825a03147a44383dd54050e Mon Sep 17 00:00:00 2001 From: Martin Weinelt Date: Fri, 24 Mar 2023 14:29:07 +0100 Subject: [PATCH 2/2] nixos/tests/ceph*: Enable on aarch64-linux --- nixos/tests/all-tests.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 625eddf97bdc..8d6e64d32630 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -121,9 +121,9 @@ in { cassandra_3_0 = handleTest ./cassandra.nix { testPackage = pkgs.cassandra_3_0; }; cassandra_3_11 = handleTest ./cassandra.nix { testPackage = pkgs.cassandra_3_11; }; cassandra_4 = handleTest ./cassandra.nix { testPackage = pkgs.cassandra_4; }; - ceph-multi-node = handleTestOn ["x86_64-linux"] ./ceph-multi-node.nix {}; - ceph-single-node = handleTestOn ["x86_64-linux"] ./ceph-single-node.nix {}; - ceph-single-node-bluestore = handleTestOn ["x86_64-linux"] ./ceph-single-node-bluestore.nix {}; + ceph-multi-node = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./ceph-multi-node.nix {}; + ceph-single-node = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./ceph-single-node.nix {}; + ceph-single-node-bluestore = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./ceph-single-node-bluestore.nix {}; certmgr = handleTest ./certmgr.nix {}; cfssl = handleTestOn ["aarch64-linux" "x86_64-linux"] ./cfssl.nix {}; cgit = handleTest ./cgit.nix {};