Merge pull request #240333 from mschwaig/tflite-bazel

maintainers/maintainer-list.nix

@@ -11318,6 +11318,12 @@
     name = "Maxim Schuwalow";
     email = "maxim.schuwalow@gmail.com";
   };
+  mschwaig = {
+    name = "Martin Schwaighofer";
+    github = "mschwaig";
+    githubId = 3856390;
+    email = "mschwaig+nixpkgs@eml.cc";
+  };
   msfjarvis = {
     github = "msfjarvis";
     githubId = 13348378;

pkgs/development/libraries/science/math/tensorflow-lite/default.nix

@@ -1,203 +1,104 @@
 { stdenv
 , bash
-, abseil-cpp
+, buildPackages
+, buildBazelPackage
 , fetchFromGitHub
-, fetchFromGitLab
-, fetchpatch
-, fetchurl
-, flatbuffers
 , lib
-, zlib
 }:
 let
-  tflite-eigen = fetchFromGitLab {
+  buildPlatform = stdenv.buildPlatform;
-    owner = "libeigen";
+  hostPlatform = stdenv.hostPlatform;
-    repo = "eigen";
+  pythonEnv = buildPackages.python3.withPackages (ps: [ ps.numpy ]);
-    rev = "3d9051ea84a5089b277c88dac456b3b1576bfa7f";
+  bazelDepsSha256ByBuildAndHost = {
-    sha256 = "1y3f2jvimb5i904f4n37h23cv2pkdlbz8656s0kga1y7c0p50wif";
+    x86_64-linux = {
-  };
+      x86_64-linux = "sha256-61qmnAB80syYhURWYJOiOnoGOtNa1pPkxfznrFScPAo=";
-
+      aarch64-linux = "sha256-sOIYpp98wJRz3RGvPasyNEJ05W29913Lsm+oi/aq/Ag=";
-  gemmlowp-src = fetchFromGitHub {
+    };
-    owner = "google";
+    aarch64-linux = {
-    repo = "gemmlowp";
+      aarch64-linux = "sha256-MJU4y9Dt9xJWKgw7iKW+9Ur856rMIHeFD5u05s+Q7rQ=";
-    rev = "fda83bdc38b118cc6b56753bd540caa49e570745";
+    };
-    sha256 = "1sbp8kmr2azwlvfbzryy1frxi99jhsh1nc93bdbxdf8zdgpv0kxl";
-  };
-
-  neon-2-sse-src = fetchFromGitHub {
-    owner = "intel";
-    repo = "ARM_NEON_2_x86_SSE";
-    rev = "1200fe90bb174a6224a525ee60148671a786a71f";
-    sha256 = "0fhxch711ck809dpq1myxz63jiiwfcnxvj45ww0kg8s0pqpn5kv6";
-  };
-
-  farmhash-src = fetchFromGitHub {
-    owner = "google";
-    repo = "farmhash";
-    rev = "816a4ae622e964763ca0862d9dbd19324a1eaf45";
-    sha256 = "1mqxsljq476n1hb8ilkrpb39yz3ip2hnc7rhzszz4sri8ma7qzp6";
-  };
-
-  fft2d-src = fetchurl {
-    url = "http://www.kurims.kyoto-u.ac.jp/~ooura/fft2d.tgz";
-    sha256 = "ada7e99087c4ed477bfdf11413f2ba8db8a840ba9bbf8ac94f4f3972e2a7cec9";
-  };
-
-  fp16-src = fetchFromGitHub {
-    owner = "Maratyszcza";
-    repo = "FP16";
-    rev = "4dfe081cf6bcd15db339cf2680b9281b8451eeb3";
-    sha256 = "06a8dfl3a29r93nxpp6hpywsajz5d555n3sqd3i6krybb6swnvh7";
-  };
-
-  ruy-src = fetchFromGitHub {
-    owner = "google";
-    repo = "ruy";
-    rev = "23633b37099b614a2f836ef012cafc8087fdb98c";
-    sha256 = "14k9hz6ss8qy8nsajk6lrq25f6qxrldxky31ijw0dpqnfnnswrx4";
-  };
-
-  cpuinfo-src = fetchFromGitHub {
-    owner = "pytorch";
-    repo = "cpuinfo";
-    rev = "5916273f79a21551890fd3d56fc5375a78d1598d";
-    sha256 = "0q6760xdxsg18acdv8vq3yrq7ksr7wsm8zbyan01zf2khnb6fw4x";
   };
+  bazelHostConfigName.aarch64-linux = "elinux_aarch64";
+  bazelDepsSha256ByHost =
+    bazelDepsSha256ByBuildAndHost.${buildPlatform.system} or
+      (throw "unsupported build system ${buildPlatform.system}");
+  bazelDepsSha256 = bazelDepsSha256ByHost.${hostPlatform.system} or
+      (throw "unsupported host system ${hostPlatform.system} with build system ${buildPlatform.system}");
 in
-stdenv.mkDerivation rec {
+buildBazelPackage rec {
-  pname = "tensorflow-lite";
+  name = "tensorflow-lite";
-  version = "2.5.0";
+  version = "2.13.0";
 
   src = fetchFromGitHub {
     owner = "tensorflow";
     repo = "tensorflow";
     rev = "v${version}";
-    sha256 = "1jdw2i1rq06zqd6aabh7bbm0avsg4pygnfmd7gviv0blhih9054l";
+    hash = "sha256-Rq5pAVmxlWBVnph20fkAwbfy+iuBNlfFy14poDPd5h0=";
   };
 
-  patches = [
+  bazel = buildPackages.bazel_5;
-    # TODO: remove on the next version bump
+
-    (fetchpatch {
+  nativeBuildInputs = [ pythonEnv buildPackages.perl ];
-      name = "include-schema-conversion-utils-source.patch";
+
-      url = "https://github.com/tensorflow/tensorflow/commit/f3c4f4733692150fd6174f2cd16438cfaba2e5ab.patch";
+  bazelTargets = [
-      sha256 = "0zx4hbz679kn79f30159rl1mq74dg45cvaawii0cyv48z472yy4k";
+    "//tensorflow/lite:libtensorflowlite.so"
-    })
+    "//tensorflow/lite/c:tensorflowlite_c"
-    # TODO: remove on the next version bump
+    "//tensorflow/lite/tools/benchmark:benchmark_model"
-    (fetchpatch {
+    "//tensorflow/lite/tools/benchmark:benchmark_model_performance_options"
-      name = "cxxstandard-var.patch";
-      url = "https://github.com/tensorflow/tensorflow/commit/9b128ae4200e10b4752f903492d1e7d11957ed5c.patch";
-      sha256 = "1q0izdwdji5fbyqll6k4dmkzfykyvvz5cvc6hysdj285nkn2wy6h";
-    })
   ];
 
-  buildInputs = [ zlib flatbuffers ];
+  bazelFlags = [
+    "--config=opt"
+  ] ++ lib.optionals (hostPlatform.system != buildPlatform.system) [
+    "--config=${bazelHostConfigName.${hostPlatform.system}}"
+  ];
 
-  dontConfigure = true;
+  bazelBuildFlags = [ "--cxxopt=--std=c++17" ];
+
+  buildAttrs = {
+    installPhase = ''
+      mkdir -p $out/{bin,lib}
+
+      # copy the libs and binaries into the output dir
+      cp ./bazel-bin/tensorflow/lite/c/libtensorflowlite_c.so $out/lib
+      cp ./bazel-bin/tensorflow/lite/libtensorflowlite.so $out/lib
+      cp ./bazel-bin/tensorflow/lite/tools/benchmark/benchmark_model $out/bin
+      cp ./bazel-bin/tensorflow/lite/tools/benchmark/benchmark_model_performance_options $out/bin
+
+      find . -type f -name '*.h' | while read f; do
+        path="$out/include/''${f/.\//}"
+        install -D "$f" "$path"
+
+        # remove executable bit from headers
+        chmod -x "$path"
+      done
+    '';
+  };
+
+  fetchAttrs.sha256 = bazelDepsSha256;
+
+  PYTHON_BIN_PATH = pythonEnv.interpreter;
+
+  dontAddBazelOpts = true;
+  removeRulesCC = false;
 
   postPatch = ''
-    substituteInPlace ./tensorflow/lite/tools/make/Makefile \
+    rm .bazelversion
-      --replace /bin/bash ${bash}/bin/bash \
-      --replace /bin/sh ${bash}/bin/sh
   '';
 
-  makefile = "tensorflow/lite/tools/make/Makefile";
+  preConfigure = ''
-
+    patchShebangs configure
-  preBuild =
-    let
-      includes =
-        lib.concatMapStringsSep
-          " "
-          (subdir: "-I $PWD/tensorflow/lite/tools/make/downloads/${subdir}")
-          [
-            "neon_2_sse"
-            "gemmlowp"
-            "absl"
-            "fp16/include"
-            "farmhash/src"
-            "ruy"
-            "cpuinfo"
-            "cpuinfo/src"
-            "cpuinfo/include"
-            "cpuinfo/deps/clog/include"
-            "eigen"
-          ];
-    in
-    ''
-      # enter the vendoring lair of doom
-
-      prefix="$PWD/tensorflow/lite/tools/make/downloads"
-
-      mkdir -p "$prefix"
-
-      tar xzf ${fft2d-src} -C "$prefix"
-
-      ln -s ${ruy-src} "$prefix/ruy"
-      ln -s ${gemmlowp-src} "$prefix/gemmlowp"
-      ln -s ${neon-2-sse-src} "$prefix/neon_2_sse"
-      ln -s ${farmhash-src} "$prefix/farmhash"
-      ln -s ${cpuinfo-src} "$prefix/cpuinfo"
-      ln -s ${fp16-src} "$prefix/fp16"
-      ln -s ${tflite-eigen} "$prefix/eigen"
-
-      # tensorflow lite is using the *source* of flatbuffers
-      ln -s ${flatbuffers.src} "$prefix/flatbuffers"
-
-      # tensorflow lite expects to compile abseil into `libtensorflow-lite.a`
-      ln -s ${abseil-cpp.src} "$prefix/absl"
-
-      # set CXXSTANDARD=c++17 here because abseil-cpp in nixpkgs is set as
-      # such and would be used in dependents like libedgetpu
-      buildFlagsArray+=(
-        INCLUDES="-I $PWD ${includes}"
-        CXXSTANDARD="-std=c++17"
-        TARGET_TOOLCHAIN_PREFIX=""
-        -j$NIX_BUILD_CORES
-        all)
-    '';
-
-  installPhase = ''
-    mkdir "$out"
-
-    # copy the static lib and binaries into the output dir
-    cp -r ./tensorflow/lite/tools/make/gen/linux_${stdenv.hostPlatform.uname.processor}/{bin,lib} "$out"
-
-    find ./tensorflow/lite -type f -name '*.h' | while read f; do
-      path="$out/include/''${f/.\//}"
-      install -D "$f" "$path"
-
-      # remove executable bit from headers
-      chmod -x "$path"
-    done
   '';
 
+  # configure script freaks out when parameters are passed
+  dontAddPrefix = true;
+  configurePlatforms = [];
+
   meta = with lib; {
     description = "An open source deep learning framework for on-device inference.";
     homepage = "https://www.tensorflow.org/lite";
     license = licenses.asl20;
-    maintainers = with maintainers; [ cpcloud ];
+    maintainers = with maintainers; [ mschwaig cpcloud ];
     platforms = [ "x86_64-linux" "aarch64-linux" ];
-    knownVulnerabilities = [
-      # at least some of
-      "CVE-2023-27579"
-      "CVE-2023-25801"
-      "CVE-2023-25676"
-      "CVE-2023-25675"
-      "CVE-2023-25674"
-      "CVE-2023-25673"
-      "CVE-2023-25671"
-      "CVE-2023-25670"
-      "CVE-2023-25669"
-      "CVE-2023-25668"
-      "CVE-2023-25667"
-      "CVE-2023-25665"
-      "CVE-2023-25666"
-      "CVE-2023-25664"
-      "CVE-2023-25663"
-      "CVE-2023-25662"
-      "CVE-2023-25660"
-      "CVE-2023-25659"
-      "CVE-2023-25658"
-      # and many many more
-    ];
   };
 }