From 53426f6cb93f3fbaa2ad974659da271d08ea0594 Mon Sep 17 00:00:00 2001 From: Charles Strahan Date: Mon, 24 Jul 2017 21:47:32 -0400 Subject: [PATCH] osquery: init at 2.5.2 --- nixos/modules/module-list.nix | 1 + nixos/modules/services/monitoring/osquery.nix | 91 +++++++++++++ pkgs/tools/system/osquery/default.nix | 76 +++++++++++ pkgs/tools/system/osquery/misc.patch | 126 ++++++++++++++++++ .../tools/system/osquery/platform-nixos.patch | 22 +++ pkgs/top-level/all-packages.nix | 2 + 6 files changed, 318 insertions(+) create mode 100644 nixos/modules/services/monitoring/osquery.nix create mode 100644 pkgs/tools/system/osquery/default.nix create mode 100644 pkgs/tools/system/osquery/misc.patch create mode 100644 pkgs/tools/system/osquery/platform-nixos.patch diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 59419a5e8c56..de3de20e7716 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -350,6 +350,7 @@ ./services/monitoring/munin.nix ./services/monitoring/nagios.nix ./services/monitoring/netdata.nix + ./services/monitoring/osquery.nix ./services/monitoring/prometheus/default.nix ./services/monitoring/prometheus/alertmanager.nix ./services/monitoring/prometheus/blackbox-exporter.nix diff --git a/nixos/modules/services/monitoring/osquery.nix b/nixos/modules/services/monitoring/osquery.nix new file mode 100644 index 000000000000..ba0dc4c21768 --- /dev/null +++ b/nixos/modules/services/monitoring/osquery.nix @@ -0,0 +1,91 @@ +{ config, lib, pkgs, ... }: + +with builtins; +with lib; + +let + cfg = config.services.osquery; + +in + +{ + + options = { + + services.osquery = { + + enable = mkEnableOption "osquery"; + + loggerPath = mkOption { + type = types.path; + description = "Base directory used for logging."; + default = "/var/log/osquery"; + }; + + pidfile = mkOption { + type = types.path; + description = "Path used for pid file."; + default = "/var/osquery/osqueryd.pidfile"; + }; + + utc = mkOption { + type = types.bool; + description = "Attempt to convert all UNIX calendar times to UTC."; + default = true; + }; + + databasePath = mkOption { + type = types.path; + description = "Path used for database file."; + default = "/var/osquery/osquery.db"; + }; + + extraConfig = mkOption { + type = types.attrs // { + merge = loc: foldl' (res: def: recursiveUpdate res def.value) {}; + }; + description = "Extra config to be recursively merged into the JSON config file."; + default = { }; + }; + }; + + }; + + config = mkIf cfg.enable { + + environment.systemPackages = [ pkgs.osquery ]; + + environment.etc."osquery/osquery.conf".text = toJSON ( + recursiveUpdate { + options = { + config_plugin = "filesystem"; + logger_plugin = "filesystem"; + logger_path = cfg.loggerPath; + database_path = cfg.databasePath; + utc = cfg.utc; + }; + } cfg.extraConfig + ); + + systemd.services.osqueryd = { + description = "The osquery Daemon"; + after = [ "network.target" "syslog.service" ]; + wantedBy = [ "multi-user.target" ]; + path = [ pkgs.osquery ]; + preStart = '' + mkdir -p ${escapeShellArg cfg.loggerPath} + mkdir -p "$(dirname ${escapeShellArg cfg.pidfile})" + mkdir -p "$(dirname ${escapeShellArg cfg.databasePath})" + ''; + serviceConfig = { + TimeoutStartSec = 0; + ExecStart = "${pkgs.osquery}/bin/osqueryd --logger_path ${escapeShellArg cfg.loggerPath} --pidfile ${escapeShellArg cfg.pidfile} --database_path ${escapeShellArg cfg.databasePath}"; + KillMode = "process"; + KillSignal = "SIGTERM"; + Restart = "on-failure"; + }; + }; + + }; + +} diff --git a/pkgs/tools/system/osquery/default.nix b/pkgs/tools/system/osquery/default.nix new file mode 100644 index 000000000000..72d43354ff73 --- /dev/null +++ b/pkgs/tools/system/osquery/default.nix @@ -0,0 +1,76 @@ +{ stdenv, lib, fetchFromGitHub, pkgconfig, cmake, pythonPackages +, udev, audit, aws-sdk-cpp, cryptsetup, lvm2, libgcrypt, libarchive +, libgpgerror, libuuid, iptables, apt, dpkg, lzma, lz4, bzip2, rpm +, beecrypt, augeas, libxml2, sleuthkit, yara, lldpd, google-gflags +, thrift, boost, rocksdb_lite, cpp-netlib, glog, gbenchmark, snappy +, openssl, linenoise-ng, file, doxygen, devicemapper +}: + +let + thirdparty = fetchFromGitHub { + owner = "osquery"; + repo = "third-party"; + rev = "6919841175b2c9cb2dee8986e0cfe49191ecb868"; + sha256 = "1kjxrky586jd1b2z1vs9cm7x1dxw51cizpys9kddiarapc2ih65j"; + }; + +in + +stdenv.mkDerivation rec { + name = "osquery-${version}"; + version = "2.5.2"; + + # this is what `osquery --help` will show as the version. + OSQUERY_BUILD_VERSION = version; + + src = fetchFromGitHub { + owner = "facebook"; + repo = "osquery"; + rev = version; + sha256 = "16isplk66qpvhrf041l0lxb4z6k7wwd1sg7kpsw2q6kivkxpnk3z"; + }; + + patches = [ ./misc.patch ] ++ lib.optional stdenv.isLinux ./platform-nixos.patch; + + nativeBuildInputs = [ + pkgconfig cmake pythonPackages.python pythonPackages.jinja2 + ]; + + buildInputs = [ + udev audit + + (aws-sdk-cpp.override { + apis = [ "firehose" "kinesis" "sts" ]; + customMemoryManagement = false; + }) + + lvm2 libgcrypt libarchive libgpgerror libuuid iptables.dev apt dpkg + lzma lz4 bzip2 rpm beecrypt augeas libxml2 sleuthkit + yara lldpd google-gflags thrift boost + cpp-netlib glog gbenchmark snappy openssl linenoise-ng + file doxygen devicemapper cryptsetup + + # need to be consistent about the malloc implementation + (rocksdb_lite.override { jemalloc = null; gperftools = null; }) + ]; + + preConfigure = '' + export NIX_CFLAGS_COMPILE="-I${libxml2.dev}/include/libxml2 $NIX_CFLAGS_COMPILE" + + cmakeFlagsArray+=( + -DCMAKE_LIBRARY_PATH=${cryptsetup}/lib + -DCMAKE_VERBOSE_MAKEFILE=ON + ) + + cp -r ${thirdparty}/* third-party + chmod +w -R third-party + ''; + + meta = with lib; { + description = "SQL powered operating system instrumentation, monitoring, and analytics"; + homepage = "https://osquery.io/"; + license = licenses.bsd3; + platforms = platforms.linux; + maintainers = with maintainers; [ cstrahan ]; + }; +} diff --git a/pkgs/tools/system/osquery/misc.patch b/pkgs/tools/system/osquery/misc.patch new file mode 100644 index 000000000000..bcd393e5e239 --- /dev/null +++ b/pkgs/tools/system/osquery/misc.patch @@ -0,0 +1,126 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index a976a46d..73a95575 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -125,14 +125,13 @@ else() + set(CXX_COMPILE_FLAGS "${CXX_COMPILE_FLAGS} -std=c++14 -stdlib=libc++") + else() + set(LINUX TRUE) +- set(CXX_COMPILE_FLAGS "${CXX_COMPILE_FLAGS} -std=c++14 -stdlib=libstdc++") ++ set(CXX_COMPILE_FLAGS "${CXX_COMPILE_FLAGS} -std=c++14") + endif() + set(POSIX TRUE) + endif() + + if(POSIX) + add_compile_options( +- -Qunused-arguments + -Wstrict-aliasing + -Wno-missing-field-initializers + -Wno-unused-local-typedef +@@ -154,7 +153,6 @@ if(POSIX) + ) + if(NOT FREEBSD) + add_compile_options( +- -Werror=shadow + -fvisibility=hidden + -fvisibility-inlines-hidden + ) +@@ -439,6 +437,8 @@ endif() + + if(APPLE) + LOG_PLATFORM("OS X") ++elseif(OSQUERY_BUILD_PLATFORM STREQUAL "nixos") ++ LOG_PLATFORM("NixOS") + elseif(OSQUERY_BUILD_PLATFORM STREQUAL "debian") + LOG_PLATFORM("Debian") + elseif(OSQUERY_BUILD_PLATFORM STREQUAL "ubuntu") +diff --git a/include/osquery/core.h b/include/osquery/core.h +index b597edee..b0628037 100644 +--- a/include/osquery/core.h ++++ b/include/osquery/core.h +@@ -15,8 +15,9 @@ + #include + #include + +-#if defined(__APPLE__) || defined(__FreeBSD__) ++#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__linux__) + #include ++#include + #else + #include + #endif +@@ -188,7 +189,7 @@ inline bool isPlatform(PlatformType a, const PlatformType& t = kPlatformType) { + return (static_cast(t) & static_cast(a)) != 0; + } + +-#if defined(__APPLE__) || defined(__FreeBSD__) ++#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__linux__) + #define MUTEX_IMPL boost + #else + #define MUTEX_IMPL std +@@ -204,10 +205,10 @@ using WriteLock = MUTEX_IMPL::unique_lock; + using ReadLock = MUTEX_IMPL::shared_lock; + + /// Helper alias for defining recursive mutexes. +-using RecursiveMutex = std::recursive_mutex; ++using RecursiveMutex = MUTEX_IMPL::recursive_mutex; + + /// Helper alias for write locking a recursive mutex. +-using RecursiveLock = std::lock_guard; ++using RecursiveLock = MUTEX_IMPL::lock_guard; + } + + /** +diff --git a/osquery/CMakeLists.txt b/osquery/CMakeLists.txt +index 77913d31..c833c289 100644 +--- a/osquery/CMakeLists.txt ++++ b/osquery/CMakeLists.txt +@@ -157,6 +157,7 @@ ADD_OSQUERY_LINK_ADDITIONAL("cppnetlib-client-connections${WO_KEY}") + ADD_OSQUERY_LINK_CORE("glog${WO_KEY}") + + if(POSIX) ++ ADD_OSQUERY_LINK_ADDITIONAL("benchmark") + ADD_OSQUERY_LINK_ADDITIONAL("snappy") + ADD_OSQUERY_LINK_ADDITIONAL("ssl") + ADD_OSQUERY_LINK_ADDITIONAL("crypto") +@@ -336,13 +337,6 @@ if(NOT OSQUERY_BUILD_SDK_ONLY) + + install(DIRECTORY "${CMAKE_SOURCE_DIR}/packs/" + DESTINATION "${CMAKE_INSTALL_PREFIX}/share/osquery/packs" COMPONENT main) +- if(APPLE) +- install(FILES "${CMAKE_SOURCE_DIR}/tools/deployment/com.facebook.osqueryd.plist" +- DESTINATION "${CMAKE_INSTALL_PREFIX}/share/osquery/" COMPONENT main) +- else() +- install(PROGRAMS "${CMAKE_SOURCE_DIR}/tools/deployment/osqueryd.initd" +- DESTINATION "/etc/init.d/" RENAME "osqueryd" COMPONENT main) +- endif() + endif() + + if(NOT SKIP_TESTS) +diff --git a/osquery/tables/system/linux/tests/md_tables_tests.cpp b/osquery/tables/system/linux/tests/md_tables_tests.cpp +index 126be362..119d361d 100644 +--- a/osquery/tables/system/linux/tests/md_tables_tests.cpp ++++ b/osquery/tables/system/linux/tests/md_tables_tests.cpp +@@ -72,7 +72,7 @@ void GetDrivesForArrayTestHarness(std::string arrayName, + EXPECT_CALL(md, getArrayInfo(arrayDevPath, _)) + .WillOnce(DoAll(SetArgReferee<1>(arrayInfo), Return(true))); + +- Sequence::Sequence s1; ++ Sequence s1; + for (int i = 0; i < MD_SB_DISKS; i++) { + mdu_disk_info_t diskInfo; + diskInfo.number = i; +diff --git a/specs/windows/services.table b/specs/windows/services.table +index 4ac24ee9..657d8b99 100644 +--- a/specs/windows/services.table ++++ b/specs/windows/services.table +@@ -12,7 +12,7 @@ schema([ + Column("path", TEXT, "Path to Service Executable"), + Column("module_path", TEXT, "Path to ServiceDll"), + Column("description", TEXT, "Service Description"), +- Column("user_account", TEXT, "The name of the account that the service process will be logged on as when it runs. This name can be of the form Domain\UserName. If the account belongs to the built-in domain, the name can be of the form .\UserName."), ++ Column("user_account", TEXT, "The name of the account that the service process will be logged on as when it runs. This name can be of the form Domain\\UserName. If the account belongs to the built-in domain, the name can be of the form .\\UserName."), + ]) + implementation("system/windows/services@genServices") + examples([ diff --git a/pkgs/tools/system/osquery/platform-nixos.patch b/pkgs/tools/system/osquery/platform-nixos.patch new file mode 100644 index 000000000000..7e1afcb640b3 --- /dev/null +++ b/pkgs/tools/system/osquery/platform-nixos.patch @@ -0,0 +1,22 @@ +diff --git a/tools/get_platform.py b/tools/get_platform.py +index 3dd34516..f53ca83a 100644 +--- a/tools/get_platform.py ++++ b/tools/get_platform.py +@@ -26,6 +26,8 @@ DEBIAN_VERSION = "/etc/debian_version" + GENTOO_RELEASE = "/etc/gentoo-release" + + def _platform(): ++ return ("nixos", "nixos") ++ + osType, _, _, _, _, _ = platform.uname() + + if osType == "Windows": +@@ -75,6 +77,8 @@ def _platform(): + return (None, osType.lower()) + + def _distro(osType): ++ return "unknown_version" ++ + def getRedhatDistroVersion(pattern): + with open(SYSTEM_RELEASE, "r") as fd: + contents = fd.read() diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index a015b2faa887..927b17419e7c 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -15340,6 +15340,8 @@ with pkgs; osmo = callPackage ../applications/office/osmo { }; + osquery = callPackage ../tools/system/osquery { }; + palemoon = callPackage ../applications/networking/browsers/palemoon { # https://forum.palemoon.org/viewtopic.php?f=57&t=15296#p111146 stdenv = overrideCC stdenv gcc49;