osquery: 5.5.1 -> 5.12.2

- Update to 5.12.2
- Switch to the upstream supported osquery-toolchain
- Upstream binaries for the toolchain were used. Efforts to reproduce
  the upstream toolchain from source were unsuccessful. Future work
  could involve building this from source.
- Efforts to use a newer toolchain provided by nixpkgs were
  unsuccessful. osquery assumes that osquery-toolchain and its included
  LLVM 9 compiler and libraries are available. LLVM 9 is no longer present
  in nixpkgs.
- Remove patches that are no longer necessary
- Use the exact version of openssl provided by upstream. This was
  necessary due to build errors when using the nixpkgs openssl package.
- Add code that will fail the build if the openssl hashes from the
  fetcher do not match what upstream expects

pkgs/tools/system/osquery/Remove-circular-definition-of-AUDIT_FILTER_EXCLUDE.patch

@@ -1,25 +0,0 @@
-From: Jack Baldry <jack.baldry@grafana.com>
-Date: Tue, 15 Nov 2022 15:40:31 -0400
-Subject: [PATCH] Remove circular definition of AUDIT_FILTER_EXCLUDE
-
-https://github.com/osquery/osquery/issues/6551
-
-Signed-off-by: Jack Baldry <jack.baldry@grafana.com>
----
- libraries/cmake/source/libaudit/src/lib/libaudit.h | 1 -
- 1 file changed, 1 deletion(-)
-
-diff --git a/libraries/cmake/source/libaudit/src/lib/libaudit.h b/libraries/cmake/source/libaudit/src/libaudit.h
---- a/libraries/cmake/source/libaudit/src/lib/libaudit.h
-+++ b/libraries/cmake/source/libaudit/src/lib/libaudit.h
-@@ -260,7 +260,6 @@ extern "C" {
- #define AUDIT_KEY_SEPARATOR 0x01
- 
- /* These are used in filter control */
--#define AUDIT_FILTER_EXCLUDE	AUDIT_FILTER_TYPE
- #define AUDIT_FILTER_MASK	0x07	/* Mask to get actual filter */
- #define AUDIT_FILTER_UNSET	0x80	/* This value means filter is unset */
- 
--- 
-2.38.1
-

pkgs/tools/system/osquery/Remove-system-controls-table.patch

@@ -1,157 +0,0 @@
-From: Jack Baldry <jack.baldry@grafana.com>
-Date: Wed, 16 Nov 2022 22:00:06 -0400
-Subject: [PATCH] Remove system controls table
-
-Relies on <sys/sysctl.h> which is not present in glibc since 2.32.
-
-Signed-off-by: Jack Baldry <jack.baldry@grafana.com>
----
- osquery/tables/system/CMakeLists.txt         |  4 --
- specs/CMakeLists.txt                         |  1 -
- specs/posix/system_controls.table            | 21 -------
- tests/integration/tables/system_controls.cpp | 61 --------------------
- 4 files changed, 87 deletions(-)
- delete mode 100644 specs/posix/system_controls.table
- delete mode 100644 tests/integration/tables/system_controls.cpp
-
-diff --git a/osquery/tables/system/CMakeLists.txt b/osquery/tables/system/CMakeLists.txt
---- a/osquery/tables/system/CMakeLists.txt
-+++ b/osquery/tables/system/CMakeLists.txt
-@@ -43,7 +43,6 @@ function(generateOsqueryTablesSystemSystemtable)
-       posix/smbios_utils.cpp
-       posix/sudoers.cpp
-       posix/suid_bin.cpp
--      posix/system_controls.cpp
-       posix/ulimit_info.cpp
-     )
-   endif()
-@@ -82,7 +81,6 @@ function(generateOsqueryTablesSystemSystemtable)
-       linux/shared_memory.cpp
-       linux/smbios_tables.cpp
-       linux/startup_items.cpp
--      linux/sysctl_utils.cpp
-       linux/system_info.cpp
-       linux/usb_devices.cpp
-       linux/user_groups.cpp
-@@ -156,7 +154,6 @@ function(generateOsqueryTablesSystemSystemtable)
-       darwin/smbios_tables.cpp
-       darwin/smc_keys.cpp
-       darwin/startup_items.cpp
--      darwin/sysctl_utils.cpp
-       darwin/system_extensions.mm
-       darwin/system_info.cpp
-       darwin/time_machine.cpp
-@@ -326,7 +323,6 @@ function(generateOsqueryTablesSystemSystemtable)
-       posix/shell_history.h
-       posix/ssh_keys.h
-       posix/sudoers.h
--      posix/sysctl_utils.h
-       posix/last.h
-       posix/openssl_utils.h
-       posix/authorized_keys.h
-diff --git a/specs/CMakeLists.txt b/specs/CMakeLists.txt
---- a/specs/CMakeLists.txt
-+++ b/specs/CMakeLists.txt
-@@ -246,7 +246,6 @@ function(generateNativeTables)
-     "posix/socket_events.table:linux,macos"
-     "posix/sudoers.table:linux,macos,freebsd"
-     "posix/suid_bin.table:linux,macos,freebsd"
--    "posix/system_controls.table:linux,macos,freebsd"
-     "posix/ulimit_info.table:linux,macos,freebsd"
-     "posix/usb_devices.table:linux,macos"
-     "posix/user_events.table:linux,macos,freebsd"
-diff --git a/specs/posix/system_controls.table b/specs/posix/system_controls.table
-deleted file mode 100644
---- a/specs/posix/system_controls.table
-+++ /dev/null
-@@ -1,21 +0,0 @@
--table_name("system_controls")
--description("sysctl names, values, and settings information.")
--schema([
--    Column("name", TEXT, "Full sysctl MIB name", index=True),
--    Column("oid", TEXT, "Control MIB", additional=True),
--    Column("subsystem", TEXT, "Subsystem ID, control type", additional=True),
--    Column("current_value", TEXT, "Value of setting"),
--    Column("config_value", TEXT, "The MIB value set in /etc/sysctl.conf"),
--    Column("type", TEXT, "Data type"),
--])
--extended_schema(DARWIN, [
--    Column("field_name", TEXT, "Specific attribute of opaque type"),
--])
--
--implementation("system_controls@genSystemControls")
--fuzz_paths([
--    "/run/sysctl.d/",
--    "/usr/lib/sysctl.d/",
--    "/lib/sysctl.d/",
--    "/sys"
--])
-diff --git a/tests/integration/tables/system_controls.cpp b/tests/integration/tables/system_controls.cpp
-deleted file mode 100644
---- a/tests/integration/tables/system_controls.cpp
-+++ /dev/null
-@@ -1,61 +0,0 @@
--/**
-- * Copyright (c) 2014-present, The osquery authors
-- *
-- * This source code is licensed as defined by the LICENSE file found in the
-- * root directory of this source tree.
-- *
-- * SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only)
-- */
--
--// Sanity check integration test for system_controls
--// Spec file: specs/posix/system_controls.table
--
--#include <osquery/tests/integration/tables/helper.h>
--
--namespace osquery {
--namespace table_tests {
--namespace {
--
--class SystemControlsTest : public testing::Test {
-- protected:
--  void SetUp() override {
--    setUpEnvironment();
--  }
--};
--
--TEST_F(SystemControlsTest, test_sanity) {
--  auto const rows = execute_query("select * from system_controls");
--  auto const row_map = ValidationMap{
--      {"name", NonEmptyString},
--      {"oid", NormalType},
--      {"subsystem",
--       SpecificValuesCheck{"",
--                           "abi",
--                           "debug",
--                           "dev",
--                           "fs",
--                           "fscache",
--                           "hw",
--                           "kern",
--                           "kernel",
--                           "machdep",
--                           "net",
--                           "sunrpc",
--                           "user",
--                           "vfs",
--                           "vm"}},
--      {"current_value", NormalType},
--      {"config_value", NormalType},
--      {"type",
--       SpecificValuesCheck{
--           "", "node", "int", "string", "quad", "opaque", "struct"}},
--#ifdef __APPLE__
--      {"field_name", NormalType},
--#endif
--  };
--  validate_rows(rows, row_map);
--}
--
--} // namespace
--} // namespace table_tests
--} // namespace osquery
--- 
-2.38.1
-

pkgs/tools/system/osquery/Use-locale.h-instead-of-removed-xlocale.h-header.patch

@@ -1,32 +0,0 @@
-diff --git a/libraries/cmake/source/augeas/gnulib/generated/linux/aarch64/lib/locale.h b/libraries/cmake/source/augeas/gnulib/generated/linux/aarch64/lib/locale.h
-index 4f9baece2..afe947956 100644
---- a/libraries/cmake/source/augeas/gnulib/generated/linux/aarch64/lib/locale.h
-+++ b/libraries/cmake/source/augeas/gnulib/generated/linux/aarch64/lib/locale.h
-@@ -48,9 +48,9 @@
- /* NetBSD 5.0 mis-defines NULL.  */
- #include <stddef.h>
- 
--/* Mac OS X 10.5 defines the locale_t type in <xlocale.h>.  */
-+/* Mac OS X 10.5 defines the locale_t type in <locale.h>.  */
- #if 1
--# include <xlocale.h>
-+# include <locale.h>
- #endif
- 
- /* The definitions of _GL_FUNCDECL_RPL etc. are copied here.  */
-diff --git a/libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h b/libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h
-index 4f9baece2..afe947956 100644
---- a/libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h
-+++ b/libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h
-@@ -48,9 +48,9 @@
- /* NetBSD 5.0 mis-defines NULL.  */
- #include <stddef.h>
- 
--/* Mac OS X 10.5 defines the locale_t type in <xlocale.h>.  */
-+/* Mac OS X 10.5 defines the locale_t type in <locale.h>.  */
- #if 1
--# include <xlocale.h>
-+# include <locale.h>
- #endif
- 
- /* The definitions of _GL_FUNCDECL_RPL etc. are copied here.  */

pkgs/tools/system/osquery/default.nix

@@ -1,97 +1,131 @@
 { lib
 , cmake
 , fetchFromGitHub
-, fetchpatch
+, fetchzip
+, fetchurl
 , git
-, llvmPackages
-, nixosTests
-, overrideCC
 , perl
 , python3
-, stdenv
+, stdenvNoCC
-, openssl_1_1
+, ninja
+, autoPatchelfHook
+, writeShellApplication
+, jq
+, removeReferencesTo
 }:
 
 let
-  buildStdenv = overrideCC stdenv llvmPackages.clangUseLLVM;
+
-in
+  version = "5.12.2";
-buildStdenv.mkDerivation rec {
+
-  pname = "osquery";
+  opensslVersion = "3.2.1";
-  version = "5.5.1";
+
+  opensslSha256 = "83c7329fe52c850677d75e5d0b0ca245309b97e8ecbcfdc1dfdc4ab9fac35b39";
 
   src = fetchFromGitHub {
     owner = "osquery";
     repo = "osquery";
     rev = version;
     fetchSubmodules = true;
-    sha256 = "sha256-Q6PQVnBjAjAlR725fyny+RhQFUNwxWGjLDuS5p9JKlU=";
+    hash = "sha256-PJrGAqDxo5l6jtQdpTqraR195G6kaLQ2ik08WtlWEmk=";
   };
 
+  extractOpensslInfo = writeShellApplication {
+    name = "extractOpensslInfo";
+    text = ''
+      if [ $# -ne 1 ]; then
+        echo "Usage: $0 <osquery-source-directory>"
+        exit 1
+      fi
+      opensslCmake="$1"/libraries/cmake/formula/openssl/CMakeLists.txt
+      version=$(gawk 'match($0, /OPENSSL_VERSION "(.*)"/, a) {print a[1]}' < "$opensslCmake")
+      sha256=$(gawk 'match($0, /OPENSSL_ARCHIVE_SHA256 "(.*)"/, a) {print a[1]}' < "$opensslCmake")
+      echo "{\"version\": \"$version\", \"sha256\": \"$sha256\"}"
+    '';
+  };
+
+  opensslSrc = fetchurl {
+    url = "https://www.openssl.org/source/openssl-${opensslVersion}.tar.gz";
+    sha256 = opensslSha256;
+  };
+
+  toolchain = import ./toolchain-bin.nix { inherit autoPatchelfHook stdenvNoCC lib fetchzip; };
+
+in
+
+stdenvNoCC.mkDerivation rec {
+
+  pname = "osquery";
+
+  inherit src version;
+
   patches = [
     ./Remove-git-reset.patch
-    ./Use-locale.h-instead-of-removed-xlocale.h-header.patch
-    ./Remove-circular-definition-of-AUDIT_FILTER_EXCLUDE.patch
-    # For current state of compilation against glibc in the clangWithLLVM toolchain, refer to the upstream issue in https://github.com/osquery/osquery/issues/7823.
-    ./Remove-system-controls-table.patch
-
-    # osquery uses a vendored boost library that still relies on old standard types (e.g. `std::unary_function`)
-    # which have been removed as of C++17. The patch is already checked in upstream, but there have been no
-    # releases yet. Can likely be removed with versions > 5.10.2.
-    (fetchpatch {
-      name = "fix-build-on-clang-16.patch";
-      url  = "https://github.com/osquery/osquery/commit/222991a15b4ae0a0fb919e4965603616536e1b0a.patch";
-      hash = "sha256-PdzEoeR1LXVri1Cd+7KMhKmDC8yZhAx3f1+9tjLJKyo=";
-    })
   ];
 
-
-  buildInputs = [
-    llvmPackages.libunwind
-  ];
   nativeBuildInputs = [
     cmake
     git
     perl
     python3
+    ninja
+    autoPatchelfHook
+    extractOpensslInfo
+    jq
+    removeReferencesTo
   ];
 
   postPatch = ''
     substituteInPlace cmake/install_directives.cmake --replace "/control" "control"
-    # This is required to build libarchive with our glibc version
-    # which provides the ARC4RANDOM_BUF function
-    substituteInPlace libraries/cmake/source/libarchive/CMakeLists.txt --replace "  target_compile_definitions(thirdparty_libarchive PRIVATE" "  target_compile_definitions(thirdparty_libarchive PRIVATE HAVE_ARC4RANDOM_BUF"
-    # We need to override this hash because we use our own openssl 1.1 version
-    substituteInPlace libraries/cmake/formula/openssl/CMakeLists.txt --replace \
-      "d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca" \
-      "$(sha256sum ${openssl_1_1.src} | cut -f1 '-d ')"
-    cat libraries/cmake/formula/openssl/CMakeLists.txt
   '';
 
-  # For explanation of these deletions, refer to the ./Use-locale.h-instead-of-removed-xlocale.h-header.patch file.
+  configurePhase = ''
-  preConfigure = ''
+    expectedOpensslVersion=$(extractOpensslInfo . | jq -r .version)
-    find libraries/cmake/source -name 'config.h' -exec sed -i '/#define HAVE_XLOCALE_H 1/d' {} \;
+    expectedOpensslSha256=$(extractOpensslInfo . | jq -r .sha256)
+
+    if [ "$expectedOpensslVersion" != "${opensslVersion}" ]; then
+      echo "openssl version mismatch: expected=$expectedOpensslVersion actual=${opensslVersion}"
+      opensslMismatch=1
+    fi
+
+    if [ "$expectedOpensslSha256" != "${opensslSha256}" ]; then
+      echo "openssl sha256 mismatch: expected=$expectedOpensslSha256 actual=${opensslSha256}"
+      opensslMismatch=1
+    fi
+
+    if [ -n "$opensslMismatch" ]; then
+      exit 1
+    fi
+
+    mkdir build
+    cd build
+    cmake .. \
+      -DCMAKE_INSTALL_PREFIX=$out \
+      -DOSQUERY_TOOLCHAIN_SYSROOT=${toolchain} \
+      -DOSQUERY_VERSION=${version} \
+      -DCMAKE_PREFIX_PATH=${toolchain}/usr/lib/cmake \
+      -DCMAKE_LIBRARY_PATH=${toolchain}/usr/lib \
+      -DOSQUERY_OPENSSL_ARCHIVE_PATH=${opensslSrc} \
+      -GNinja
   '';
 
-  cmakeFlags = [
+  disallowedReferences = [ toolchain ];
-    "-DOSQUERY_VERSION=${version}"
-    "-DOSQUERY_OPENSSL_ARCHIVE_PATH=${openssl_1_1.src}"
-  ];
 
-  postFixup = ''
+  postInstall = ''
-    patchelf --set-rpath "${llvmPackages.libunwind}/lib:$(patchelf --print-rpath $out/bin/osqueryd)" "$out/bin/osqueryd"
+    rm -rf $out/control
+    remove-references-to -t ${toolchain} $out/bin/osqueryd
   '';
 
-  passthru.tests.osquery = nixosTests.osquery;
+  passthru = {
+    inherit extractOpensslInfo opensslSrc toolchain;
+  };
 
   meta = with lib; {
     description = "SQL powered operating system instrumentation, monitoring, and analytics";
-    longDescription = ''
-      The system controls table is not included as it does not presently compile with glibc >= 2.32.
-      For more information, refer to https://github.com/osquery/osquery/issues/7823
-    '';
     homepage = "https://osquery.io";
-    license = licenses.bsd3;
+    license = with licenses; [ gpl2Only asl20 ];
     platforms = platforms.linux;
-    maintainers = with maintainers; [ znewman01 lewo ];
+    sourceProvenance = with sourceTypes; [ fromSource ];
+    maintainers = with maintainers; [ znewman01 lewo squalus ];
   };
 }

pkgs/tools/system/osquery/toolchain-bin.nix

@@ -0,0 +1,42 @@
+{ stdenvNoCC, lib, autoPatchelfHook, fetchzip }:
+let
+
+  version = "1.1.0";
+
+  dist = {
+    "x86_64-linux" = {
+      url = "https://github.com/osquery/osquery-toolchain/releases/download/${version}/osquery-toolchain-${version}-x86_64.tar.xz";
+      hash = "sha256-irekR8a0d+T64+ZObgblsLoc4kVBmb6Gv0Qf8dLDCMk=";
+    };
+    "aarch64-linux" = {
+      url = "https://github.com/osquery/osquery-toolchain/releases/download/${version}/osquery-toolchain-${version}-aarch64.tar.xz";
+      hash = "sha256-cQlx9AtO6ggIQqHowa+42wQ4YCMCN4Gb+0qqVl2JElw=";
+    };
+  };
+
+in
+
+stdenvNoCC.mkDerivation {
+
+  name = "osquery-toolchain-bin";
+
+  inherit version;
+
+  src = fetchzip dist.${stdenvNoCC.hostPlatform.system};
+
+  nativeBuildInputs = [ autoPatchelfHook ];
+
+  installPhase = ''
+    mkdir $out
+    cp -r * $out
+  '';
+
+  meta = with lib; {
+    description = "A LLVM-based toolchain for Linux designed to build a portable osquery";
+    homepage = "https://github.com/osquery/osquery-toolchain";
+    platforms = [ "x86_64-linux" "aarch64-linux" ];
+    sourceProvenance = with sourceTypes; [ binaryNativeCode ];
+    license = with licenses; [ gpl2Only asl20 ];
+    maintainers = with maintainers; [ squalus ];
+  };
+}