Merge pull request #305406 from GaetanLepage/transformers

python311Packages.transformers: 4.39.3 -> 4.40.0

pkgs/development/python-modules/tokenizers/default.nix

@@ -63,16 +63,16 @@ let
 in
 buildPythonPackage rec {
   pname = "tokenizers";
-  version = "0.15.0";
-  format = "pyproject";
+  version = "0.19.1";
+  pyproject = true;
 
   disabled = pythonOlder "3.7";
 
   src = fetchFromGitHub {
     owner = "huggingface";
-    repo = pname;
-    rev = "v${version}";
-    hash = "sha256-+yfX12eKtgZV1OQvPOlMVTONbpFuigHcl4SjoCIZkSk=";
+    repo = "tokenizers";
+    rev = "refs/tags/v${version}";
+    hash = "sha256-sKEAt46cdme821tzz9WSKnQb3hPmFJ4zvHgBNRxjEuk=";
   };
 
   cargoDeps = rustPlatform.importCargoLock {
@@ -97,7 +97,13 @@ buildPythonPackage rec {
     Security
   ];
 
-  propagatedBuildInputs = [
+  # Cargo.lock is outdated
+  # TODO: remove at next release
+  preConfigure = ''
+    cargo update --offline
+  '';
+
+  dependencies = [
     numpy
   ];
 
@@ -123,6 +129,8 @@ buildPythonPackage rec {
 
   disabledTests = [
     # Downloads data using the datasets module
+    "test_encode_special_tokens"
+    "test_splitting"
     "TestTrainFromIterators"
     # Those tests require more data
     "test_from_pretrained"

pkgs/development/python-modules/transformers/default.nix

@@ -2,6 +2,7 @@
 , buildPythonPackage
 , fetchFromGitHub
 , pythonOlder
+, setuptools
 # propagated build inputs
 , filelock
 , huggingface-hub
@@ -53,8 +54,8 @@
 
 buildPythonPackage rec {
   pname = "transformers";
-  version = "4.39.3";
-  format = "setuptools";
+  version = "4.40.0";
+  pyproject = true;
 
   disabled = pythonOlder "3.8";
 
@@ -62,9 +63,13 @@ buildPythonPackage rec {
     owner = "huggingface";
     repo = "transformers";
     rev = "refs/tags/v${version}";
-    hash = "sha256-MJZvPbj9ypr6YwFGkzwCp9fVuM3vEGpeXK+gEIFzwRA=";
+    hash = "sha256-k0AloOG7fRQhTW8IF6uQSfav8p1KC5633SaLNYZrZ2M=";
   };
 
+  build-system = [
+    setuptools
+  ];
+
   propagatedBuildInputs = [
     filelock
     huggingface-hub