From 015f46beba0d753572f617951a295fa134d7a6f5 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 20 Apr 2020 13:53:29 -0500 Subject: [PATCH 1/9] numpy: set lapack,blas library names correctly --- pkgs/development/python-modules/numpy/default.nix | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkgs/development/python-modules/numpy/default.nix b/pkgs/development/python-modules/numpy/default.nix index 949d5e5898b3..7009d1d1cd03 100644 --- a/pkgs/development/python-modules/numpy/default.nix +++ b/pkgs/development/python-modules/numpy/default.nix @@ -21,9 +21,7 @@ let ${blas.implementation} = { include_dirs = "${blas}/include:${lapack}/include"; library_dirs = "${blas}/lib:${lapack}/lib"; - } // lib.optionalAttrs (blas.implementation == "mkl") { - mkl_libs = "mkl_rt"; - lapack_libs = ""; + libraries = "lapack,lapacke,blas,cblas"; }; }); }; From 3c9c894f836fffa07a8d269f6394bdf2cbfdc940 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 20 Apr 2020 15:46:45 -0500 Subject: [PATCH 2/9] =?UTF-8?q?blas,lapack:=20don=E2=80=99t=20patchelf?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have some unused RPATHs we don’t want shrunk. These are used in MKL to dlopen based on architecture. --- pkgs/build-support/alternatives/blas/default.nix | 2 ++ pkgs/build-support/alternatives/lapack/default.nix | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pkgs/build-support/alternatives/blas/default.nix b/pkgs/build-support/alternatives/blas/default.nix index 36708ce8841a..bce0c7af2bc1 100644 --- a/pkgs/build-support/alternatives/blas/default.nix +++ b/pkgs/build-support/alternatives/blas/default.nix @@ -58,6 +58,8 @@ stdenv.mkDerivation { dontConfigure = true; unpackPhase = "src=$PWD"; + dontPatchELF = true; + installPhase = ('' mkdir -p $out/lib $dev/include $dev/lib/pkgconfig diff --git a/pkgs/build-support/alternatives/lapack/default.nix b/pkgs/build-support/alternatives/lapack/default.nix index 24c339042a2f..75dc2ae3b343 100644 --- a/pkgs/build-support/alternatives/lapack/default.nix +++ b/pkgs/build-support/alternatives/lapack/default.nix @@ -36,6 +36,8 @@ stdenv.mkDerivation { dontConfigure = true; unpackPhase = "src=$PWD"; + dontPatchELF = true; + installPhase = ('' mkdir -p $out/lib $dev/include $dev/lib/pkgconfig From f86d582ea7cdf9c969f2294d9dc953f36f2a63e9 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 20 Apr 2020 15:49:56 -0500 Subject: [PATCH 3/9] doc/overlays.xml: add information on BLAS/LAPACK switching --- doc/using/overlays.xml | 76 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/doc/using/overlays.xml b/doc/using/overlays.xml index 26a888368abf..7732e0ac2179 100644 --- a/doc/using/overlays.xml +++ b/doc/using/overlays.xml @@ -137,4 +137,80 @@ self: super: Overlays are similar to other methods for customizing Nixpkgs, in particular the packageOverrides attribute described in . Indeed, packageOverrides acts as an overlay with only the super argument. It is therefore appropriate for basic use, but overlays are more powerful and easier to distribute. +
+ Using overlays to configure alternatives + + Certain software has different implementations of the same + interface. Other distributions have functionality to switch + between these. For example, Debian provides DebianAlternatives. + Nixpkgs has what we call alternatives, which + are configured through overlays. + +
+ BLAS/LAPACK + + In Nixpkgs, we have multiple implementations of the BLAS/LAPACK + numerical linear algebra interfaces. They are: + + + + + OpenBLAS + + + The Nixpkgs attribute is openblas for + ILP64 and openblasCompat for LP64. This + is the default. + + + + + LAPACK + reference (also provides BLAS) + + + The Nixpkgs attribute is lapack-reference. + + + + + Intel + MKL (only works on x86 architecture) + + + The Nixpkgs attribute is mkl. + + + + + Introduced in PR + #83888, we are able to override the ‘blas’ and ‘lapack’ + packages to use different implementations, through the + ‘blasProvider’ and ‘lapackProvider’ argument. This can be used + to select a different provider. For example, an overlay can be + created that looks like: + + +self: super: + +{ + blas = super.blas.override { + blasProvider = self.mkl; + } + lapack = super.lapack.override { + lapackProvider = self.mkl; + } +} + + + This overlay uses Intel’s MKL library for both BLAS and LAPACK + interfaces. Note that the same can be accomplished at runtime + using LD_PRELOAD of libblas.so.3 and + liblapack.so.3. + +
+
From ff2f2644f8ea1b364dde5dfee2bc76027afccaf9 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 20 Apr 2020 15:50:55 -0500 Subject: [PATCH 4/9] blas,lapack: use isILP64 instead of is64bit This is a better name since we have multiple 64-bit things that could be referred to. LP64 : integer=32, long=64, pointer=64 ILP64 : integer=64, long=64, pointer=64 --- .../science/chemistry/octopus/default.nix | 2 +- .../science/machine-learning/shogun/default.nix | 2 +- pkgs/applications/science/math/R/default.nix | 2 +- pkgs/applications/science/math/giac/default.nix | 2 +- pkgs/applications/science/math/gmsh/default.nix | 2 +- pkgs/applications/science/math/sage/sage-env.nix | 2 +- .../applications/science/math/sage/sage-with-env.nix | 2 +- pkgs/applications/science/math/sage/sagelib.nix | 2 +- pkgs/build-support/alternatives/blas/default.nix | 12 ++++++------ pkgs/build-support/alternatives/lapack/default.nix | 10 +++++----- pkgs/development/compilers/julia/1.3.nix | 4 ++-- pkgs/development/compilers/julia/shared.nix | 4 ++-- pkgs/development/interpreters/octave/default.nix | 6 +++--- pkgs/development/libraries/fflas-ffpack/default.nix | 2 +- pkgs/development/libraries/linbox/default.nix | 2 +- pkgs/development/libraries/qrupdate/default.nix | 2 +- .../libraries/science/math/arpack/default.nix | 2 +- .../libraries/science/math/ipopt/default.nix | 2 +- .../libraries/science/math/scalapack/default.nix | 2 +- .../libraries/science/math/scs/default.nix | 2 +- .../libraries/science/math/suitesparse/4.2.nix | 2 +- .../libraries/science/math/suitesparse/4.4.nix | 2 +- .../libraries/science/math/suitesparse/default.nix | 2 +- .../libraries/science/math/superlu/default.nix | 2 +- pkgs/development/libraries/sundials/2.x.nix | 2 +- pkgs/development/libraries/sundials/default.nix | 2 +- pkgs/development/ocaml-modules/lacaml/default.nix | 2 +- pkgs/development/ocaml-modules/owl/default.nix | 2 +- pkgs/development/python-modules/cvxopt/default.nix | 2 +- pkgs/development/python-modules/numpy/default.nix | 2 +- pkgs/tools/misc/hpl/default.nix | 2 +- pkgs/top-level/release-alternatives.nix | 6 +++--- 32 files changed, 47 insertions(+), 47 deletions(-) diff --git a/pkgs/applications/science/chemistry/octopus/default.nix b/pkgs/applications/science/chemistry/octopus/default.nix index f30954367cf7..adb31d326509 100644 --- a/pkgs/applications/science/chemistry/octopus/default.nix +++ b/pkgs/applications/science/chemistry/octopus/default.nix @@ -2,7 +2,7 @@ , libyaml, libxc, fftw, blas, lapack, gsl, netcdf, arpack, autoreconfHook }: -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); stdenv.mkDerivation rec { pname = "octopus"; diff --git a/pkgs/applications/science/machine-learning/shogun/default.nix b/pkgs/applications/science/machine-learning/shogun/default.nix index 33871df87f33..031ca55a0480 100644 --- a/pkgs/applications/science/machine-learning/shogun/default.nix +++ b/pkgs/applications/science/machine-learning/shogun/default.nix @@ -13,7 +13,7 @@ assert pythonSupport -> pythonPackages != null; assert opencvSupport -> opencv != null; -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); let pname = "shogun"; diff --git a/pkgs/applications/science/math/R/default.nix b/pkgs/applications/science/math/R/default.nix index 836ce004fff2..617a18265371 100644 --- a/pkgs/applications/science/math/R/default.nix +++ b/pkgs/applications/science/math/R/default.nix @@ -9,7 +9,7 @@ , static ? false }: -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); stdenv.mkDerivation rec { name = "R-3.6.3"; diff --git a/pkgs/applications/science/math/giac/default.nix b/pkgs/applications/science/math/giac/default.nix index be3c1e6dcc88..ea9c921f19e1 100644 --- a/pkgs/applications/science/math/giac/default.nix +++ b/pkgs/applications/science/math/giac/default.nix @@ -5,7 +5,7 @@ }: assert enableGUI -> libGLU != null && libGL != null && xorg != null && fltk != null; -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); stdenv.mkDerivation rec { pname = "giac${lib.optionalString enableGUI "-with-xcas"}"; diff --git a/pkgs/applications/science/math/gmsh/default.nix b/pkgs/applications/science/math/gmsh/default.nix index 8ece2e7819cf..946beff8003f 100644 --- a/pkgs/applications/science/math/gmsh/default.nix +++ b/pkgs/applications/science/math/gmsh/default.nix @@ -1,7 +1,7 @@ { stdenv, fetchurl, cmake, blas, lapack, gfortran, gmm, fltk, libjpeg , zlib, libGL, libGLU, xorg, opencascade-occt }: -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); stdenv.mkDerivation rec { pname = "gmsh"; diff --git a/pkgs/applications/science/math/sage/sage-env.nix b/pkgs/applications/science/math/sage/sage-env.nix index c0dfeef1119b..626e1bf1ea92 100644 --- a/pkgs/applications/science/math/sage/sage-env.nix +++ b/pkgs/applications/science/math/sage/sage-env.nix @@ -54,7 +54,7 @@ , less }: -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); # This generates a `sage-env` shell file that will be sourced by sage on startup. # It sets up various environment variables, telling sage where to find its diff --git a/pkgs/applications/science/math/sage/sage-with-env.nix b/pkgs/applications/science/math/sage/sage-with-env.nix index 77bc65c2be12..87c1e07f5307 100644 --- a/pkgs/applications/science/math/sage/sage-with-env.nix +++ b/pkgs/applications/science/math/sage/sage-with-env.nix @@ -23,7 +23,7 @@ }: # lots of segfaults with (64 bit) blas -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); # Wrapper that combined `sagelib` with `sage-env` to produce an actually # executable sage. No tests are run yet and no documentation is built. diff --git a/pkgs/applications/science/math/sage/sagelib.nix b/pkgs/applications/science/math/sage/sagelib.nix index 92b4e8efa2ec..830d806a8bcd 100644 --- a/pkgs/applications/science/math/sage/sagelib.nix +++ b/pkgs/applications/science/math/sage/sagelib.nix @@ -53,7 +53,7 @@ , pplpy }: -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); # This is the core sage python package. Everything else is just wrappers gluing # stuff together. It is not very useful on its own though, since it will not diff --git a/pkgs/build-support/alternatives/blas/default.nix b/pkgs/build-support/alternatives/blas/default.nix index bce0c7af2bc1..45fb69eddb7d 100644 --- a/pkgs/build-support/alternatives/blas/default.nix +++ b/pkgs/build-support/alternatives/blas/default.nix @@ -1,7 +1,7 @@ { lib, stdenv , lapack-reference, openblasCompat, openblas -, is64bit ? false -, blasProvider ? if is64bit then openblas else openblasCompat }: +, isILP64 ? false +, blasProvider ? if isILP64 then openblas else openblasCompat }: let blasFortranSymbols = [ @@ -31,12 +31,12 @@ let else stdenv.hostPlatform.extensions.sharedLibrary; - is64bit = blasProvider.blas64 or false; + isILP64 = blasProvider.blas64 or false; blasImplementation = lib.getName blasProvider; in -assert is64bit -> (blasImplementation == "openblas" && blasProvider.blas64) || blasImplementation == "mkl"; +assert isILP64 -> (blasImplementation == "openblas" && blasProvider.blas64) || blasImplementation == "mkl"; stdenv.mkDerivation { pname = "blas"; @@ -49,7 +49,7 @@ stdenv.mkDerivation { }; passthru = { - inherit is64bit; + inherit isILP64; provider = blasProvider; implementation = blasImplementation; }; @@ -134,6 +134,6 @@ Libs: -L$out/lib -lcblas EOF '' + stdenv.lib.optionalString (blasImplementation == "mkl") '' mkdir -p $out/nix-support - echo 'export MKL_INTERFACE_LAYER=${lib.optionalString is64bit "I"}LP64,GNU' > $out/nix-support/setup-hook + echo 'export MKL_INTERFACE_LAYER=${lib.optionalString isILP64 "I"}LP64,GNU' > $out/nix-support/setup-hook ''); } diff --git a/pkgs/build-support/alternatives/lapack/default.nix b/pkgs/build-support/alternatives/lapack/default.nix index 75dc2ae3b343..c90eb43494d8 100644 --- a/pkgs/build-support/alternatives/lapack/default.nix +++ b/pkgs/build-support/alternatives/lapack/default.nix @@ -1,7 +1,7 @@ { lib, stdenv , lapack-reference, openblasCompat, openblas -, is64bit ? false -, lapackProvider ? if is64bit then openblas else openblasCompat }: +, isILP64 ? false +, lapackProvider ? if isILP64 then openblas else openblasCompat }: let @@ -14,7 +14,7 @@ let in -assert is64bit -> (lapackImplementation == "openblas" && lapackProvider.blas64) || lapackImplementation == "mkl"; +assert isILP64 -> (lapackImplementation == "openblas" && lapackProvider.blas64) || lapackImplementation == "mkl"; stdenv.mkDerivation { pname = "lapack"; @@ -27,7 +27,7 @@ stdenv.mkDerivation { }; passthru = { - inherit is64bit; + inherit isILP64; provider = lapackProvider; implementation = lapackImplementation; }; @@ -108,6 +108,6 @@ Libs: -L$out/lib -llapacke EOF '' + stdenv.lib.optionalString (lapackImplementation == "mkl") '' mkdir -p $out/nix-support - echo 'export MKL_INTERFACE_LAYER=${lib.optionalString is64bit "I"}LP64,GNU' > $out/nix-support/setup-hook + echo 'export MKL_INTERFACE_LAYER=${lib.optionalString isILP64 "I"}LP64,GNU' > $out/nix-support/setup-hook ''); } diff --git a/pkgs/development/compilers/julia/1.3.nix b/pkgs/development/compilers/julia/1.3.nix index 49dc17a27f45..b67a78b4a5ab 100644 --- a/pkgs/development/compilers/julia/1.3.nix +++ b/pkgs/development/compilers/julia/1.3.nix @@ -12,7 +12,7 @@ , CoreServices, ApplicationServices }: -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); with stdenv.lib; @@ -88,7 +88,7 @@ stdenv.mkDerivation rec { "SHELL=${stdenv.shell}" "USE_SYSTEM_BLAS=1" - "USE_BLAS64=${if blas.is64bit then "1" else "0"}" + "USE_BLAS64=${if blas.isILP64 then "1" else "0"}" "USE_SYSTEM_LAPACK=1" diff --git a/pkgs/development/compilers/julia/shared.nix b/pkgs/development/compilers/julia/shared.nix index 92e3d4a5c14b..29ac41150506 100644 --- a/pkgs/development/compilers/julia/shared.nix +++ b/pkgs/development/compilers/julia/shared.nix @@ -22,7 +22,7 @@ with stdenv.lib; -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); let dsfmtVersion = "2.2.3"; @@ -137,7 +137,7 @@ stdenv.mkDerivation rec { "SHELL=${stdenv.shell}" "USE_SYSTEM_BLAS=1" - "USE_BLAS64=${if blas.is64bit then "1" else "0"}" + "USE_BLAS64=${if blas.isILP64 then "1" else "0"}" "USE_SYSTEM_LAPACK=1" diff --git a/pkgs/development/interpreters/octave/default.nix b/pkgs/development/interpreters/octave/default.nix index 4d2bf40b3292..06d69ed8117f 100644 --- a/pkgs/development/interpreters/octave/default.nix +++ b/pkgs/development/interpreters/octave/default.nix @@ -53,7 +53,7 @@ , darwin }: -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); stdenv.mkDerivation rec { version = "5.2.0"; @@ -125,12 +125,12 @@ stdenv.mkDerivation rec { enableParallelBuilding = true; # See https://savannah.gnu.org/bugs/?50339 - F77_INTEGER_8_FLAG = if blas.is64bit then "-fdefault-integer-8" else ""; + F77_INTEGER_8_FLAG = if blas.isILP64 then "-fdefault-integer-8" else ""; configureFlags = [ "--with-blas=blas" "--with-lapack=lapack" - (if blas.is64bit then "--enable-64" else "--disable-64") + (if blas.isILP64 then "--enable-64" else "--disable-64") ] ++ (if stdenv.isDarwin then [ "--enable-link-all-dependencies" ] else [ ]) ++ stdenv.lib.optionals enableReadline [ "--enable-readline" ] diff --git a/pkgs/development/libraries/fflas-ffpack/default.nix b/pkgs/development/libraries/fflas-ffpack/default.nix index 16a9e3e79d45..bc84039a9aa2 100644 --- a/pkgs/development/libraries/fflas-ffpack/default.nix +++ b/pkgs/development/libraries/fflas-ffpack/default.nix @@ -2,7 +2,7 @@ , gmpxx }: -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); stdenv.mkDerivation rec { pname = "fflas-ffpack"; diff --git a/pkgs/development/libraries/linbox/default.nix b/pkgs/development/libraries/linbox/default.nix index 6d25cb459870..2217996acdbc 100644 --- a/pkgs/development/libraries/linbox/default.nix +++ b/pkgs/development/libraries/linbox/default.nix @@ -10,7 +10,7 @@ , withSage ? false # sage support }: -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); stdenv.mkDerivation rec { pname = "linbox"; diff --git a/pkgs/development/libraries/qrupdate/default.nix b/pkgs/development/libraries/qrupdate/default.nix index f6c16583ad75..424502dce0d4 100644 --- a/pkgs/development/libraries/qrupdate/default.nix +++ b/pkgs/development/libraries/qrupdate/default.nix @@ -18,7 +18,7 @@ stdenv.mkDerivation { -e 's,^LAPACK=.*,LAPACK=-L${lapack}/lib -llapack,' \ Makeconf '' - + stdenv.lib.optionalString blas.is64bit + + stdenv.lib.optionalString blas.isILP64 '' sed -i Makeconf -e '/^FFLAGS=.*/ s/$/-fdefault-integer-8/' ''; diff --git a/pkgs/development/libraries/science/math/arpack/default.nix b/pkgs/development/libraries/science/math/arpack/default.nix index a5fbb679df5b..0829557d0f3d 100644 --- a/pkgs/development/libraries/science/math/arpack/default.nix +++ b/pkgs/development/libraries/science/math/arpack/default.nix @@ -27,7 +27,7 @@ stdenv.mkDerivation { cmakeFlags = [ "-DBUILD_SHARED_LIBS=ON" - "-DINTERFACE64=${optionalString blas.is64bit "1"}" + "-DINTERFACE64=${optionalString blas.isILP64 "1"}" ]; preCheck = if stdenv.isDarwin then '' diff --git a/pkgs/development/libraries/science/math/ipopt/default.nix b/pkgs/development/libraries/science/math/ipopt/default.nix index 647fd82ec0e9..97374fb0d477 100644 --- a/pkgs/development/libraries/science/math/ipopt/default.nix +++ b/pkgs/development/libraries/science/math/ipopt/default.nix @@ -1,6 +1,6 @@ { stdenv, fetchurl, unzip, blas, lapack, gfortran }: -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); stdenv.mkDerivation rec { pname = "ipopt"; diff --git a/pkgs/development/libraries/science/math/scalapack/default.nix b/pkgs/development/libraries/science/math/scalapack/default.nix index d20219860296..1cf9c2ed8d53 100644 --- a/pkgs/development/libraries/science/math/scalapack/default.nix +++ b/pkgs/development/libraries/science/math/scalapack/default.nix @@ -2,7 +2,7 @@ , gfortran, mpi, blas, lapack } : -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); stdenv.mkDerivation rec { pname = "scalapack"; diff --git a/pkgs/development/libraries/science/math/scs/default.nix b/pkgs/development/libraries/science/math/scs/default.nix index 2dcb47f11baa..3820f2b95275 100644 --- a/pkgs/development/libraries/science/math/scs/default.nix +++ b/pkgs/development/libraries/science/math/scs/default.nix @@ -1,6 +1,6 @@ { stdenv, fetchFromGitHub, blas, lapack, gfortran, fixDarwinDylibNames }: -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); stdenv.mkDerivation rec { pname = "scs"; diff --git a/pkgs/development/libraries/science/math/suitesparse/4.2.nix b/pkgs/development/libraries/science/math/suitesparse/4.2.nix index 34a1fb7ad9e6..b1c1202c5780 100644 --- a/pkgs/development/libraries/science/math/suitesparse/4.2.nix +++ b/pkgs/development/libraries/science/math/suitesparse/4.2.nix @@ -1,7 +1,7 @@ { stdenv, fetchurl, gfortran, blas, lapack }: let - int_t = if blas.is64bit then "int64_t" else "int32_t"; + int_t = if blas.isILP64 then "int64_t" else "int32_t"; in stdenv.mkDerivation rec { version = "4.2.1"; diff --git a/pkgs/development/libraries/science/math/suitesparse/4.4.nix b/pkgs/development/libraries/science/math/suitesparse/4.4.nix index b2b7e666b7ef..81a80c920b53 100644 --- a/pkgs/development/libraries/science/math/suitesparse/4.4.nix +++ b/pkgs/development/libraries/science/math/suitesparse/4.4.nix @@ -6,7 +6,7 @@ let version = "4.4.4"; name = "suitesparse-${version}"; - int_t = if blas.is64bit then "int64_t" else "int32_t"; + int_t = if blas.isILP64 then "int64_t" else "int32_t"; SHLIB_EXT = stdenv.hostPlatform.extensions.sharedLibrary; in stdenv.mkDerivation { diff --git a/pkgs/development/libraries/science/math/suitesparse/default.nix b/pkgs/development/libraries/science/math/suitesparse/default.nix index 48177987a7a2..f0480e1e2381 100644 --- a/pkgs/development/libraries/science/math/suitesparse/default.nix +++ b/pkgs/development/libraries/science/math/suitesparse/default.nix @@ -44,7 +44,7 @@ stdenv.mkDerivation rec { "BLAS=-lblas" "LAPACK=-llapack" "MY_METIS_LIB=-lmetis" - ] ++ stdenv.lib.optionals blas.is64bit [ + ] ++ stdenv.lib.optionals blas.isILP64 [ "CFLAGS=-DBLAS64" ] ++ stdenv.lib.optionals enableCuda [ "CUDA_PATH=${cudatoolkit}" diff --git a/pkgs/development/libraries/science/math/superlu/default.nix b/pkgs/development/libraries/science/math/superlu/default.nix index 05ceb21387d3..ff59297bd839 100644 --- a/pkgs/development/libraries/science/math/superlu/default.nix +++ b/pkgs/development/libraries/science/math/superlu/default.nix @@ -1,7 +1,7 @@ { stdenv, fetchurl, cmake, gfortran, blas, lapack}: -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); stdenv.mkDerivation rec { version = "5.2.1"; diff --git a/pkgs/development/libraries/sundials/2.x.nix b/pkgs/development/libraries/sundials/2.x.nix index 08cbd52ba394..b54e537fb823 100644 --- a/pkgs/development/libraries/sundials/2.x.nix +++ b/pkgs/development/libraries/sundials/2.x.nix @@ -8,7 +8,7 @@ , gfortran , lapackSupport ? true }: -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); stdenv.mkDerivation rec { pname = "sundials"; diff --git a/pkgs/development/libraries/sundials/default.nix b/pkgs/development/libraries/sundials/default.nix index 8ed8d6cfcb55..46cf437d72ca 100644 --- a/pkgs/development/libraries/sundials/default.nix +++ b/pkgs/development/libraries/sundials/default.nix @@ -7,7 +7,7 @@ , gfortran , lapackSupport ? true }: -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); stdenv.mkDerivation rec { pname = "sundials"; diff --git a/pkgs/development/ocaml-modules/lacaml/default.nix b/pkgs/development/ocaml-modules/lacaml/default.nix index e4da2216a83d..23b38e469b4b 100644 --- a/pkgs/development/ocaml-modules/lacaml/default.nix +++ b/pkgs/development/ocaml-modules/lacaml/default.nix @@ -1,7 +1,7 @@ { stdenv, fetchFromGitHub, darwin, ocaml, findlib, dune, base, stdio, lapack, blas }: assert stdenv.lib.versionAtLeast (stdenv.lib.getVersion ocaml) "4.05.0"; -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); stdenv.mkDerivation rec { pname = "ocaml${ocaml.version}-lacaml"; diff --git a/pkgs/development/ocaml-modules/owl/default.nix b/pkgs/development/ocaml-modules/owl/default.nix index c84743a8da52..b306ee9ce6d2 100644 --- a/pkgs/development/ocaml-modules/owl/default.nix +++ b/pkgs/development/ocaml-modules/owl/default.nix @@ -11,7 +11,7 @@ , npy }: -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); assert blas.implementation == "openblas" && lapack.implementation == "openblas"; buildDunePackage rec { diff --git a/pkgs/development/python-modules/cvxopt/default.nix b/pkgs/development/python-modules/cvxopt/default.nix index 9266f406b151..2b6d1b4d2484 100644 --- a/pkgs/development/python-modules/cvxopt/default.nix +++ b/pkgs/development/python-modules/cvxopt/default.nix @@ -14,7 +14,7 @@ , withFftw ? true }: -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); buildPythonPackage rec { pname = "cvxopt"; diff --git a/pkgs/development/python-modules/numpy/default.nix b/pkgs/development/python-modules/numpy/default.nix index 7009d1d1cd03..f3d8dcba31d3 100644 --- a/pkgs/development/python-modules/numpy/default.nix +++ b/pkgs/development/python-modules/numpy/default.nix @@ -12,7 +12,7 @@ , setuptoolsBuildHook }: -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); let cfg = writeTextFile { diff --git a/pkgs/tools/misc/hpl/default.nix b/pkgs/tools/misc/hpl/default.nix index c131de021806..d688f3adf30e 100644 --- a/pkgs/tools/misc/hpl/default.nix +++ b/pkgs/tools/misc/hpl/default.nix @@ -1,6 +1,6 @@ { stdenv, fetchurl, blas, lapack, mpi } : -assert (!blas.is64bit) && (!lapack.is64bit); +assert (!blas.isILP64) && (!lapack.isILP64); stdenv.mkDerivation rec { pname = "hpl"; diff --git a/pkgs/top-level/release-alternatives.nix b/pkgs/top-level/release-alternatives.nix index eef239d4e0a8..7479377bd302 100644 --- a/pkgs/top-level/release-alternatives.nix +++ b/pkgs/top-level/release-alternatives.nix @@ -59,7 +59,7 @@ in { blas = mapListToAttrs supportedSystems (system': let system = lib.systems.elaborate { system = system'; }; in mapListToAttrs (blasProviders system) (provider: let - is64bit = builtins.elem provider (["mkl64"] ++ lib.optional system.is64bit "openblas"); + isILP64 = builtins.elem provider (["mkl64"] ++ lib.optional system.is64bit "openblas"); pkgs = pkgsFun { config = { inherit allowUnfree; }; system = system'; @@ -68,13 +68,13 @@ in lapackProvider = if provider == "mkl64" then super.mkl else builtins.getAttr provider super; - inherit is64bit; + inherit isILP64; }; blas = super.blas.override { blasProvider = if provider == "mkl64" then super.mkl else builtins.getAttr provider super; - inherit is64bit; + inherit isILP64; }; })]; }; From a6a502fca05a308a727450978417240e2f6ee0d7 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 20 Apr 2020 15:51:34 -0500 Subject: [PATCH 5/9] magma: remove mklSupport flag This now relies on the "blas" and "lapack" packages. --- .../libraries/science/math/magma/default.nix | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/pkgs/development/libraries/science/math/magma/default.nix b/pkgs/development/libraries/science/math/magma/default.nix index b34139e7f484..dbe162c60b34 100644 --- a/pkgs/development/libraries/science/math/magma/default.nix +++ b/pkgs/development/libraries/science/math/magma/default.nix @@ -1,8 +1,4 @@ -{ stdenv, fetchurl, cmake, gfortran, cudatoolkit, libpthreadstubs, lapack, blas -, mklSupport ? false, mkl ? null -}: - -assert !mklSupport || mkl != null; +{ stdenv, fetchurl, cmake, gfortran, cudatoolkit, libpthreadstubs, lapack, blas }: with stdenv.lib; @@ -17,13 +13,10 @@ in stdenv.mkDerivation { name = "magma-${version}.tar.gz"; }; - buildInputs = [ gfortran cudatoolkit libpthreadstubs cmake ] - ++ (if mklSupport then [ mkl ] else [ lapack blas ]); + buildInputs = [ gfortran cudatoolkit libpthreadstubs cmake lapack blas ]; doCheck = false; - MKLROOT = optionalString mklSupport mkl; - preConfigure = '' export CC=${cudatoolkit.cc}/bin/gcc CXX=${cudatoolkit.cc}/bin/g++ ''; From 607522c78db958555de68ef6342857398f2002f4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 20 Apr 2020 16:31:59 -0400 Subject: [PATCH 6/9] numpy: Use blas and lapack dev outputs for include dirs --- pkgs/development/python-modules/numpy/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/numpy/default.nix b/pkgs/development/python-modules/numpy/default.nix index f3d8dcba31d3..4a062ca7b97d 100644 --- a/pkgs/development/python-modules/numpy/default.nix +++ b/pkgs/development/python-modules/numpy/default.nix @@ -19,7 +19,7 @@ let name = "site.cfg"; text = (lib.generators.toINI {} { ${blas.implementation} = { - include_dirs = "${blas}/include:${lapack}/include"; + include_dirs = "${lib.getDev blas}/include:${lib.getDev lapack}/include"; library_dirs = "${blas}/lib:${lapack}/lib"; libraries = "lapack,lapacke,blas,cblas"; }; From 0caa375b4ceac2031b3b6d270019d476c9e61bd6 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 20 Apr 2020 23:32:03 -0500 Subject: [PATCH 7/9] blas,lapack: provide symlink for libmkl_rt.so This is needed for numpy to detect mkl correctly. --- pkgs/build-support/alternatives/blas/default.nix | 1 + pkgs/build-support/alternatives/lapack/default.nix | 1 + 2 files changed, 2 insertions(+) diff --git a/pkgs/build-support/alternatives/blas/default.nix b/pkgs/build-support/alternatives/blas/default.nix index 45fb69eddb7d..90cafbf53ce0 100644 --- a/pkgs/build-support/alternatives/blas/default.nix +++ b/pkgs/build-support/alternatives/blas/default.nix @@ -135,5 +135,6 @@ EOF '' + stdenv.lib.optionalString (blasImplementation == "mkl") '' mkdir -p $out/nix-support echo 'export MKL_INTERFACE_LAYER=${lib.optionalString isILP64 "I"}LP64,GNU' > $out/nix-support/setup-hook + ln -s $out/lib/libblas${canonicalExtension} $out/lib/libmkl_rt${stdenv.hostPlatform.extensions.sharedLibrary} ''); } diff --git a/pkgs/build-support/alternatives/lapack/default.nix b/pkgs/build-support/alternatives/lapack/default.nix index c90eb43494d8..935150bdb7df 100644 --- a/pkgs/build-support/alternatives/lapack/default.nix +++ b/pkgs/build-support/alternatives/lapack/default.nix @@ -109,5 +109,6 @@ EOF '' + stdenv.lib.optionalString (lapackImplementation == "mkl") '' mkdir -p $out/nix-support echo 'export MKL_INTERFACE_LAYER=${lib.optionalString isILP64 "I"}LP64,GNU' > $out/nix-support/setup-hook + ln -s $out/lib/liblapack${canonicalExtension} $out/lib/libmkl_rt${stdenv.hostPlatform.extensions.sharedLibrary} ''); } From 8f58fa044dca8becdc865c2feb51e5048bf14fef Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 21 Apr 2020 00:29:52 -0500 Subject: [PATCH 8/9] blas,lapack: symlink headers when using mkl Some of these are necessary for things like python.pkgs.numexpr --- pkgs/build-support/alternatives/blas/default.nix | 1 + pkgs/build-support/alternatives/lapack/default.nix | 1 + 2 files changed, 2 insertions(+) diff --git a/pkgs/build-support/alternatives/blas/default.nix b/pkgs/build-support/alternatives/blas/default.nix index 90cafbf53ce0..8bba49b45504 100644 --- a/pkgs/build-support/alternatives/blas/default.nix +++ b/pkgs/build-support/alternatives/blas/default.nix @@ -136,5 +136,6 @@ EOF mkdir -p $out/nix-support echo 'export MKL_INTERFACE_LAYER=${lib.optionalString isILP64 "I"}LP64,GNU' > $out/nix-support/setup-hook ln -s $out/lib/libblas${canonicalExtension} $out/lib/libmkl_rt${stdenv.hostPlatform.extensions.sharedLibrary} + ln -sf ${blasProvider}/include/* $dev/include ''); } diff --git a/pkgs/build-support/alternatives/lapack/default.nix b/pkgs/build-support/alternatives/lapack/default.nix index 935150bdb7df..e260ad1bd931 100644 --- a/pkgs/build-support/alternatives/lapack/default.nix +++ b/pkgs/build-support/alternatives/lapack/default.nix @@ -110,5 +110,6 @@ EOF mkdir -p $out/nix-support echo 'export MKL_INTERFACE_LAYER=${lib.optionalString isILP64 "I"}LP64,GNU' > $out/nix-support/setup-hook ln -s $out/lib/liblapack${canonicalExtension} $out/lib/libmkl_rt${stdenv.hostPlatform.extensions.sharedLibrary} + ln -sf ${lapackProvider}/include/* $dev/include ''); } From 3c41d61514029d5aa4ea748c1e46098767ae448b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 21 Apr 2020 11:03:10 -0500 Subject: [PATCH 9/9] doc/overlays.xml: update documentation for BLAS/LAPACK This expands the documentation and explains how to assert LP64. --- doc/languages-frameworks/python.section.md | 29 ++-------------- doc/using/overlays.xml | 40 +++++++++++++++++++++- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md index f8884785e90d..e6f0b64fa9cc 100644 --- a/doc/languages-frameworks/python.section.md +++ b/doc/languages-frameworks/python.section.md @@ -1290,32 +1290,9 @@ self: super: { ### How to use Intel's MKL with numpy and scipy? -A `site.cfg` is created that configures BLAS based on the `blas` parameter of -the `numpy` derivation. By passing in `mkl`, `numpy` and packages depending on -`numpy` will be built with `mkl`. - -The following is an overlay that configures `numpy` to use `mkl`: - -```nix -self: super: { - python37 = super.python37.override { - packageOverrides = python-self: python-super: { - numpy = python-super.numpy.override { - blas = super.pkgs.mkl; - }; - }; - }; -} -``` - -`mkl` requires an `openmp` implementation when running with multiple processors. -By default, `mkl` will use Intel's `iomp` implementation if no other is -specified, but this is a runtime-only dependency and binary compatible with the -LLVM implementation. To use that one instead, Intel recommends users set it with -`LD_PRELOAD`. - -Note that `mkl` is only available on `x86_64-{linux,darwin}` platforms; -moreover, Hydra is not building and distributing pre-compiled binaries using it. +MKL can be configured using an overlay. See the section “[Using +overlays to configure +alternatives](#sec-overlays-alternatives-blas-lapack)”. ### What inputs do `setup_requires`, `install_requires` and `tests_require` map to? diff --git a/doc/using/overlays.xml b/doc/using/overlays.xml index 7732e0ac2179..5f808839dd0d 100644 --- a/doc/using/overlays.xml +++ b/doc/using/overlays.xml @@ -177,7 +177,7 @@ self: super: Intel - MKL (only works on x86 architecture) + MKL (only works on x86 architecture, unfree) The Nixpkgs attribute is mkl. @@ -211,6 +211,44 @@ self: super: using LD_PRELOAD of libblas.so.3 and liblapack.so.3. + + Intel MKL requires an openmp implementation + when running with multiple processors. By default, + mkl will use Intel’s iomp + implementation if no other is specified, but this is a + runtime-only dependency and binary compatible with the LLVM + implementation. To use that one instead, Intel recommends users + set it with LD_PRELOAD. Note that + mkl is only available on + x86_64-linux and + x86_64-darwin. Moreover, Hydra is not build + and distributing pre-compiled binaries using it. + + + For BLAS/LAPACK switching to work correctly, all packages must + depend on blas or lapack. + This ensures that only one BLAS/LAPACK library is used at one + time. There are two versions versions of BLAS/LAPACK currently + in the wild, LP64 (integer size = 32 bits) + and ILP64 (integer size = 64 bits). Some + software needs special flags or patches to work with + ILP64. You can check if + ILP64 is used in Nixpkgs with + blas.isILP64 and + lapack.isILP64. Some software does NOT work + with ILP64, and derivations need to specify + an assertion to prevent this. You can prevent + ILP64 from being used with the following: + + +{ stdenv, blas, lapack, ... }: + +assert (!blas.isILP64) && (!lapack.isILP64); + +stdenv.mkDerivation { + ... +} +