arrow-cpp: 0.10.0 -> 0.11.0 (#48066)

* arrow-cpp: 0.10.0 -> 0.11.0 * arrow-cpp: enable testing
2025-01-30 08:44:31 +00:00 · 2018-10-13 03:12:26 -05:00 · 2018-10-13 03:12:26 -05:00 · 4ed993367f
commit 4ed993367f
parent 045a4a4b72
6 changed files with 90 additions and 62 deletions
--- a/pkgs/development/libraries/arrow-cpp/default.nix
+++ b/pkgs/development/libraries/arrow-cpp/default.nix
@ -1,42 +1,75 @@
-{ stdenv, symlinkJoin, fetchurl, boost, brotli, cmake, flatbuffers, gtest, gflags, lz4, python, rapidjson, snappy, zlib, zstd }:
+{ stdenv, symlinkJoin, fetchurl, fetchFromGitHub, boost, brotli, cmake, double-conversion, flatbuffers, gflags, glog, gtest, lz4, perl, python, rapidjson, snappy, thrift, which, zlib, zstd }:
+
+let
+  parquet-testing = fetchFromGitHub {
+    owner = "apache";
+    repo = "parquet-testing";
+    rev = "46ae2605c2de306f5740587107dcf333a527f2d1";
+    sha256 = "07ps745gas2zcfmg56m3vwl63yyzmalnxwb5dc40vd004cx5hdik";
+  };
+in

 stdenv.mkDerivation rec {
  name = "arrow-cpp-${version}";
-  version = "0.10.0";
+  version = "0.11.0";

  src = fetchurl {
    url = "mirror://apache/arrow/arrow-${version}/apache-arrow-${version}.tar.gz";
-    sha256 = "0bc4krapz1kzdm16npzmgdz7zvg9lip6rnqbwph8vfn7zji0fcll";
+    sha256 = "0pc5pqr0dbnx8s1ji102dhw9bbrsq3ml4ac3mmi2022yfyizlf0q";
  };

  sourceRoot = "apache-arrow-${version}/cpp";

-  # patch to fix python-test
-  patches = [ ./darwin.patch ];
+  patches = [
+    # fix ARROW-3467
+    ./double-conversion_cmake.patch
+
+    # patch to fix python-test
+    ./darwin.patch
+    ];

  nativeBuildInputs = [ cmake ];
-  buildInputs = [ boost python.pkgs.python python.pkgs.numpy ];
+  buildInputs = [ boost double-conversion glog python.pkgs.python python.pkgs.numpy ];

  preConfigure = ''
+    substituteInPlace cmake_modules/FindThrift.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
    substituteInPlace cmake_modules/FindBrotli.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
+    substituteInPlace cmake_modules/FindGLOG.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
    substituteInPlace cmake_modules/FindLz4.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
    substituteInPlace cmake_modules/FindSnappy.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
+
+    patchShebangs build-support/
  '';

  BROTLI_HOME = symlinkJoin { name="brotli-wrap"; paths = [ brotli.lib brotli.dev ]; };
+  DOUBLE_CONVERSION_HOME = double-conversion;
  FLATBUFFERS_HOME = flatbuffers;
-  GTEST_HOME = gtest;
  GFLAGS_HOME = gflags;
+  GLOG_HOME = glog;
+  GTEST_HOME = gtest;
  LZ4_HOME = symlinkJoin { name="lz4-wrap"; paths = [ lz4 lz4.dev ]; };
  RAPIDJSON_HOME = rapidjson;
  SNAPPY_HOME = symlinkJoin { name="snappy-wrap"; paths = [ snappy snappy.dev ]; };
-  ZLIB_HOME = symlinkJoin { name="zlib-wrap"; paths = [ zlib.dev zlib.static ]; };
+  THRIFT_HOME = thrift;
+  ZLIB_HOME = symlinkJoin { name="zlib-wrap"; paths = [ zlib zlib.dev ]; };
  ZSTD_HOME = zstd;

  cmakeFlags = [
    "-DARROW_PYTHON=ON"
+    "-DARROW_PARQUET=ON"
  ];

+  doInstallCheck = true;
+  PARQUET_TEST_DATA = if doInstallCheck then "${parquet-testing}/data" else null;
+  installCheckInputs = [ perl which ];
+  installCheckPhase = (stdenv.lib.optionalString stdenv.isDarwin ''
+    for f in release/*-test; do
+      install_name_tool -add_rpath "$out"/lib  "$f"
+    done
+  '') + ''
+    ctest -L unittest -V
+  '';
+
  meta = {
    description = "A  cross-language development platform for in-memory data";
    homepage = https://arrow.apache.org/;
--- a/pkgs/development/libraries/arrow-cpp/double-conversion_cmake.patch
+++ b/pkgs/development/libraries/arrow-cpp/double-conversion_cmake.patch
@ -0,0 +1,43 @@
+diff --git a/CMakeLists.txt b/cpp/CMakeLists.txt
+--- a/CMakeLists.txt
+++ b/CMakeLists.txt
+@@ -615,7 +615,7 @@ endif(UNIX)
+ set(ARROW_LINK_LIBS)
+ 
+ # Libraries to link statically with libarrow.so
+-set(ARROW_STATIC_LINK_LIBS double-conversion)
+set(ARROW_STATIC_LINK_LIBS ${DOUBLE_CONVERSION_TARGET})
+ 
+ if (ARROW_WITH_BROTLI)
+   SET(ARROW_STATIC_LINK_LIBS
+@@ -694,7 +694,7 @@ else ()
+   set(ARROW_MIN_TEST_LIBS
+     arrow_shared
+     ${ARROW_LINK_LIBS}
+-    double-conversion
+    ${DOUBLE_CONVERSION_TARGET}
+     ${BOOST_SYSTEM_LIBRARY}
+     ${BOOST_FILESYSTEM_LIBRARY}
+     ${BOOST_REGEX_LIBRARY}
+diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
+--- a/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cmake_modules/ThirdpartyToolchain.cmake
+@@ -469,14 +469,16 @@ if("${DOUBLE_CONVERSION_HOME}" STREQUAL "")
+   set(DOUBLE_CONVERSION_VENDORED 1)
+ else()
+   find_package(double-conversion REQUIRED)
+  set(DOUBLE_CONVERSION_TARGET double-conversion::double-conversion)
+   set(DOUBLE_CONVERSION_VENDORED 0)
+ endif()
+ 
+ include_directories(SYSTEM ${DOUBLE_CONVERSION_INCLUDE_DIR})
+-ADD_THIRDPARTY_LIB(double-conversion
+-  STATIC_LIB ${DOUBLE_CONVERSION_STATIC_LIB})
+ 
+ if (DOUBLE_CONVERSION_VENDORED)
+  ADD_THIRDPARTY_LIB(double-conversion
+    STATIC_LIB ${DOUBLE_CONVERSION_STATIC_LIB})
+  set(DOUBLE_CONVERSION_TARGET double-conversion)
+   add_dependencies(arrow_dependencies double-conversion_ep)
+ endif()
+ 
--- a/pkgs/development/libraries/parquet-cpp/default.nix
+++ b/pkgs/development/libraries/parquet-cpp/default.nix
@ -1,39 +0,0 @@
-{ stdenv, symlinkJoin, fetchurl, arrow-cpp, boost, cmake, gtest, snappy, thrift, zlib }:
-
-stdenv.mkDerivation rec {
-  name = "parquet-cpp-${version}";
-  version = "1.5.0";
-
-  src = fetchurl {
-    url = "https://github.com/apache/parquet-cpp/archive/apache-${name}.tar.gz";
-    sha256 = "19nwqahc0igr0jfprbf2m86rmzz6zicw4z7b8z832wbsyc904wli";
-  };
-
-  patches = [ ./api.patch ];
-
-  nativeBuildInputs = [ cmake ];
-  buildInputs = [ boost ];
-
-  preConfigure = ''
-    substituteInPlace cmake_modules/FindThrift.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
-    substituteInPlace cmake_modules/FindSnappy.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
-  '';
-
-  ARROW_HOME = arrow-cpp;
-  THRIFT_HOME = thrift;
-  GTEST_HOME = gtest;
-  SNAPPY_HOME = symlinkJoin { name="snappy-wrap"; paths = [ snappy snappy.dev ]; };
-  ZLIB_HOME = symlinkJoin { name="zlib-wrap"; paths = [ zlib.dev zlib.static ]; };
-
-  cmakeFlags = [
-    "-DPARQUET_BUILD_BENCHMARKS=OFF"
-  ];
-
-  meta = {
-    description = "A C++ library to read and write the Apache Parquet columnar data format";
-    homepage = http://parquet.apache.org;
-    license = stdenv.lib.licenses.asl20;
-    platforms = stdenv.lib.platforms.unix;
-    maintainers = with stdenv.lib.maintainers; [ veprbl ];
-  };
-}
--- a/pkgs/development/python-modules/pyarrow/default.nix
+++ b/pkgs/development/python-modules/pyarrow/default.nix
@ -1,8 +1,7 @@
-{ lib, buildPythonPackage, python, isPy3k, fetchurl, arrow-cpp, cmake, cython, futures, JPype1, numpy, pandas, pytest, pytestrunner, parquet-cpp, pkgconfig, setuptools_scm, six }:
+{ lib, buildPythonPackage, python, isPy3k, fetchurl, arrow-cpp, cmake, cython, futures, JPype1, numpy, pandas, pytest, pytestrunner, pkgconfig, setuptools_scm, six }:

 let
-  _arrow-cpp = arrow-cpp.override { inherit python;};
-  _parquet-cpp = parquet-cpp.override { arrow-cpp = _arrow-cpp; };
+  _arrow-cpp = arrow-cpp.override { inherit python; };
 in

 buildPythonPackage rec {
@ -17,18 +16,11 @@ buildPythonPackage rec {
  checkInputs = [ pandas pytest pytestrunner JPype1 ];

  PYARROW_BUILD_TYPE = "release";
-  PYARROW_CMAKE_OPTIONS = "-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib;${PARQUET_HOME}/lib";
-
-  preBuild = ''
-    substituteInPlace CMakeLists.txt --replace "\''${ARROW_ABI_VERSION}" '"0.0.0"'
-    substituteInPlace CMakeLists.txt --replace "\''${ARROW_SO_VERSION}" '"0"'
-
-    # fix the hardcoded value
-    substituteInPlace cmake_modules/FindParquet.cmake --replace 'set(PARQUET_ABI_VERSION "1.0.0")' 'set(PARQUET_ABI_VERSION "${_parquet-cpp.version}")'
-  '';
+  PYARROW_CMAKE_OPTIONS = "-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib";

  preCheck = ''
    rm pyarrow/tests/test_hdfs.py
+    rm pyarrow/tests/test_cuda.py

    # fails: "ArrowNotImplementedError: Unsupported numpy type 22"
    substituteInPlace pyarrow/tests/test_feather.py --replace "test_timedelta_with_nulls" "_disabled"
@ -47,7 +39,7 @@ buildPythonPackage rec {
  '';

  ARROW_HOME = _arrow-cpp;
-  PARQUET_HOME = _parquet-cpp;
+  PARQUET_HOME = _arrow-cpp;

  setupPyBuildFlags = ["--with-parquet" ];

--- a/pkgs/top-level/aliases.nix
+++ b/pkgs/top-level/aliases.nix
@ -207,6 +207,7 @@ mapAliases ({
  openssh_with_kerberos = openssh; # added 2018-01-28
  owncloudclient = owncloud-client;  # added 2016-08
  p11_kit = p11-kit; # added 2018-02-25
+  parquet-cpp = arrow-cpp; # added 2018-09-08
  pass-otp = pass.withExtensions (ext: [ext.pass-otp]); # added 2018-05-04
  pgp-tools = signing-party; # added 2017-03-26
  pidgin-with-plugins = pidgin; # added 2016-06
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@ -11600,8 +11600,6 @@ with pkgs;

  paperkey = callPackage ../tools/security/paperkey { };

-  parquet-cpp = callPackage ../development/libraries/parquet-cpp {};
-
  pangoxsl = callPackage ../development/libraries/pangoxsl { };

  pcaudiolib = callPackage ../development/libraries/pcaudiolib {