Merge pull request #213038 from mattmelling/dvc-2.43.1

dvc: 2.17.0 -> 2.43.1

pkgs/applications/version-management/dvc/default.nix

@@ -10,56 +10,56 @@
 
 python3.pkgs.buildPythonApplication rec {
   pname = "dvc";
-  version = "2.17.0";
-  format = "setuptools";
+  version = "2.56.0";
+  format = "pyproject";
 
   src = fetchFromGitHub {
     owner = "iterative";
     repo = pname;
-    rev = version;
-    hash = "sha256-P0J+3TNHGqMw3krfs1uLnf8nEiIBK6UrrB37mY+fBA0=";
+    rev = "refs/tags/${version}";
+    hash = "sha256-IpdlNwOuUNWgfphRH2UTQ/IvBHo39PafCqyioju8miI=";
   };
 
+  pythonRelaxDeps = [
+    "dvc-data"
+    "platformdirs"
+  ];
+
   postPatch = ''
-    substituteInPlace setup.cfg \
-      --replace "grandalf==0.6" "grandalf" \
-      --replace "scmrepo==0.0.25" "scmrepo" \
-      --replace "pathspec>=0.9.0,<0.10.0" "pathspec"
     substituteInPlace dvc/daemon.py \
       --subst-var-by dvc "$out/bin/dcv"
   '';
 
   nativeBuildInputs = with python3.pkgs; [
+    pythonRelaxDepsHook
     setuptools-scm
   ];
 
   propagatedBuildInputs = with python3.pkgs; [
-    aiohttp-retry
     appdirs
     colorama
     configobj
-    dictdiffer
-    diskcache
     distro
     dpath
-    dvclive
     dvc-data
+    dvc-http
     dvc-render
+    dvc-studio-client
     dvc-task
     flatten-dict
     flufl_lock
     funcy
     grandalf
-    nanotime
+    hydra-core
+    iterative-telemetry
     networkx
     packaging
     pathspec
-    ply
+    platformdirs
     psutil
     pydot
     pygtrie
     pyparsing
-    python-benedict
     requests
     rich
     ruamel-yaml
@@ -73,17 +73,13 @@ python3.pkgs.buildPythonApplication rec {
     voluptuous
     zc_lockfile
   ] ++ lib.optionals enableGoogle [
-    gcsfs
-    google-cloud-storage
+    dvc-gs
   ] ++ lib.optionals enableAWS [
-    aiobotocore
-    boto3
-    s3fs
+    dvc-s3
   ] ++ lib.optionals enableAzure [
-    azure-identity
-    knack
+    dvc-azure
   ] ++ lib.optionals enableSSH [
-    bcrypt
+    dvc-ssh
   ] ++ lib.optionals (pythonOlder "3.8") [
     importlib-metadata
   ] ++ lib.optionals (pythonOlder "3.9") [
@@ -96,8 +92,8 @@ python3.pkgs.buildPythonApplication rec {
   meta = with lib; {
     description = "Version Control System for Machine Learning Projects";
     homepage = "https://dvc.org";
+    changelog = "https://github.com/iterative/dvc/releases/tag/${version}";
     license = licenses.asl20;
     maintainers = with maintainers; [ cmcdragonkai fab ];
-    broken = true; # requires new python package: dvc-studio-client
   };
 }

pkgs/development/python-modules/dvc-azure/default.nix

@@ -0,0 +1,41 @@
+{ lib
+, adlfs
+, azure-identity
+, buildPythonPackage
+, fetchPypi
+, knack
+, pythonRelaxDepsHook
+, setuptools-scm }:
+
+buildPythonPackage rec {
+  pname = "dvc-azure";
+  version = "2.21.0";
+  format = "setuptools";
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-VN3QSGb4cLhxX8JV1Pg4/449SJOWv9Tu3kcDGbDwAYw=";
+  };
+
+  # Prevent circular dependency
+  pythonRemoveDeps = [ "dvc" ];
+
+  nativeBuildInputs = [ setuptools-scm pythonRelaxDepsHook ];
+
+  propagatedBuildInputs = [
+    adlfs azure-identity knack
+  ];
+
+  # Network access is needed for tests
+  doCheck = false;
+
+  pythonCheckImports = [ "dvc_azure" ];
+
+  meta = with lib; {
+    description = "azure plugin for dvc";
+    homepage = "https://pypi.org/project/dvc-azure/${version}";
+    changelog = "https://github.com/iterative/dvc-azure/releases/tag/${version}";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ melling ];
+  };
+}

pkgs/development/python-modules/dvc-data/default.nix

@@ -15,7 +15,7 @@
 
 buildPythonPackage rec {
   pname = "dvc-data";
-  version = "0.41.3";
+  version = "0.47.2";
   format = "pyproject";
 
   disabled = pythonOlder "3.8";
@@ -24,7 +24,7 @@ buildPythonPackage rec {
     owner = "iterative";
     repo = pname;
     rev = "refs/tags/${version}";
-    hash = "sha256-+rK5OGbIBP7g69swhsgnaJovllxp7XdO8iT3hfYwhV4=";
+    hash = "sha256-5DbnvIScSq+bu8ki0eUwUZd9Gf3KlhD+I0/PpWfGOu0=";
   };
 
   SETUPTOOLS_SCM_PRETEND_VERSION = version;

pkgs/development/python-modules/dvc-gs/default.nix

@@ -0,0 +1,38 @@
+{ lib
+, buildPythonPackage
+, dvc-objects
+, fetchPypi
+, gcsfs
+, pythonRelaxDepsHook
+, setuptools-scm }:
+
+buildPythonPackage rec {
+  pname = "dvc-gs";
+  version = "2.21.0";
+  format = "setuptools";
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-MGNDEhJJGSQIPDXGv/y4u1UHnh4HnNbKtQbGHys0dSA=";
+  };
+
+  # Prevent circular dependency
+  pythonRemoveDeps = [ "dvc" ];
+
+  nativeBuildInputs = [ setuptools-scm pythonRelaxDepsHook ];
+
+  propagatedBuildInputs = [ gcsfs dvc-objects ];
+
+  # Network access is needed for tests
+  doCheck = false;
+
+  pythonCheckImports = [ "dvc_gs" ];
+
+  meta = with lib; {
+    description = "gs plugin for dvc";
+    homepage = "https://pypi.org/project/dvc-gs/version";
+    changelog = "https://github.com/iterative/dvc-gs/releases/tag/${version}";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ melling ];
+  };
+}

pkgs/development/python-modules/dvc-objects/default.nix

@@ -16,7 +16,7 @@
 
 buildPythonPackage rec {
   pname = "dvc-objects";
-  version = "0.21.1";
+  version = "0.21.2";
   format = "pyproject";
 
   disabled = pythonOlder "3.8";
@@ -25,7 +25,7 @@ buildPythonPackage rec {
     owner = "iterative";
     repo = pname;
     rev = "refs/tags/${version}";
-    hash = "sha256-JUpK4sAn5ZivjlpHO3XSBXZVDSWkjch/HRqHNrdC7b4=";
+    hash = "sha256-XKfhjzN69U0qiArXJHeLcdv/sMy0gF33FVrk9DuCGLk=";
   };
 
   SETUPTOOLS_SCM_PRETEND_VERSION = version;

pkgs/development/python-modules/dvc-render/default.nix

@@ -14,7 +14,7 @@
 
 buildPythonPackage rec {
   pname = "dvc-render";
-  version = "0.2.0";
+  version = "0.4.0";
   format = "pyproject";
 
   disabled = pythonOlder "3.7";
@@ -23,7 +23,7 @@ buildPythonPackage rec {
     owner = "iterative";
     repo = pname;
     rev = "refs/tags/${version}";
-    hash = "sha256-1nXNi++vNNRxoA/ptTDN9PtePP67oWdkAtqAbZpTfDg=";
+    hash = "sha256-bt2jqjhgtGmJsyfYNSzETQZUHFKni/poyeqpQzIJX08=";
   };
 
   SETUPTOOLS_SCM_PRETEND_VERSION = version;

pkgs/development/python-modules/dvc-s3/default.nix

@@ -0,0 +1,47 @@
+{ lib
+, aiobotocore
+, boto3
+, buildPythonPackage
+, fetchPypi
+, flatten-dict
+, pythonRelaxDepsHook
+, s3fs
+, setuptools-scm }:
+
+buildPythonPackage rec {
+  pname = "dvc-s3";
+  version = "2.21.0";
+  format = "setuptools";
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-AEB5Nyp6j7mX0AOA0rhegd4q8xP/POx9J6yn1Ppu0nk=";
+  };
+
+  # Prevent circular dependency
+  pythonRemoveDeps = [ "dvc" ];
+
+  # dvc-s3 uses boto3 directly, we add in propagatedBuildInputs
+  postPatch = ''
+    substituteInPlace setup.cfg --replace 'aiobotocore[boto3]' 'aiobotocore'
+  '';
+
+  nativeBuildInputs = [ setuptools-scm pythonRelaxDepsHook ];
+
+  propagatedBuildInputs = [
+    aiobotocore boto3 flatten-dict s3fs
+  ];
+
+  # Network access is needed for tests
+  doCheck = false;
+
+  pythonCheckImports = [ "dvc_s3" ];
+
+  meta = with lib; {
+    description = "s3 plugin for dvc";
+    homepage = "https://pypi.org/project/dvc-s3/${version}";
+    changelog = "https://github.com/iterative/dvc-s3/releases/tag/${version}";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ melling ];
+  };
+}

pkgs/development/python-modules/dvc-ssh/default.nix

@@ -0,0 +1,44 @@
+{ lib
+, bcrypt
+, buildPythonPackage
+, fetchPypi
+, pythonRelaxDepsHook
+, setuptools-scm
+, sshfs
+}:
+
+buildPythonPackage rec {
+  pname = "dvc-ssh";
+  version = "2.21.0";
+  format = "setuptools";
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-iew/+/Ww6Uvz6Ctvswd8l5YFa3zihYxo1jvTe0ZTW9M=";
+  };
+
+  # Prevent circular dependency
+  pythonRemoveDeps = [ "dvc" ];
+
+  nativeBuildInputs = [ setuptools-scm pythonRelaxDepsHook ];
+
+  propagatedBuildInputs = [ bcrypt sshfs ];
+
+  # bcrypt is enabled for sshfs in nixpkgs
+  postPatch = ''
+    substituteInPlace setup.cfg --replace "sshfs[bcrypt]" "sshfs"
+  '';
+
+  # Network access is needed for tests
+  doCheck = false;
+
+  pythonCheckImports = [ "dvc_ssh" ];
+
+  meta = with lib; {
+    description = "ssh plugin for dvc";
+    homepage = "https://pypi.org/project/dvc-ssh/${version}";
+    changelog = "https://github.com/iterative/dvc-ssh/releases/tag/${version}";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ melling ];
+  };
+}

pkgs/development/python-modules/dvc-studio-client/default.nix

@@ -0,0 +1,53 @@
+{ lib
+, buildPythonPackage
+, dulwich
+, fetchFromGitHub
+, gitpython
+, pythonOlder
+, requests
+, setuptools-scm
+, voluptuous
+}:
+
+buildPythonPackage rec {
+  pname = "dvc-studio-client";
+  version = "0.8.0";
+  format = "pyproject";
+
+  disabled = pythonOlder "3.8";
+
+  src = fetchFromGitHub {
+    owner = "iterative";
+    repo = pname;
+    rev = "refs/tags/${version}";
+    hash = "sha256-Uk9P/QUlvu3XgGMGZF7d2dPJyYKe3/Ex68HVP8eZqU0=";
+  };
+
+  SETUPTOOLS_SCM_PRETEND_VERSION = version;
+
+  nativeBuildInputs = [
+    setuptools-scm
+  ];
+
+  propagatedBuildInputs = [
+    dulwich
+    gitpython
+    requests
+    voluptuous
+  ];
+
+  pythonImportsCheck = [
+    "dvc_studio_client"
+  ];
+
+  # Tests try to access network
+  doCheck = false;
+
+  meta = with lib; {
+    description = "Library to post data from DVC/DVCLive to Iterative Studio";
+    homepage = "https://github.com/iterative/dvc-studio-client";
+    changelog = "https://github.com/iterative/dvc-studio-client/releases/tag/${version}";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ melling ];
+  };
+}

pkgs/development/python-modules/dvc-task/default.nix

@@ -15,7 +15,7 @@
 
 buildPythonPackage rec {
   pname = "dvc-task";
-  version = "0.1.11";
+  version = "0.2.1";
   format = "pyproject";
 
   disabled = pythonOlder "3.7";
@@ -24,7 +24,7 @@ buildPythonPackage rec {
     owner = "iterative";
     repo = pname;
     rev = "refs/tags/${version}";
-    hash = "sha256-0KfH0/rghq+03sFLaR8lsIi4TJuwwca/YhQgILCdHq8=";
+    hash = "sha256-DIjS56QDdjH8lp8yZE8UIccOM+OoafJK3DWZ1d/q29k=";
   };
 
   SETUPTOOLS_SCM_PRETEND_VERSION = version;

pkgs/development/python-modules/dvclive/default.nix

@@ -1,17 +1,20 @@
 { lib
 , buildPythonPackage
-, dvc-render
+, dvc
+, dvc-studio-client
 , fetchFromGitHub
+, funcy
 , pytestCheckHook
 , pythonOlder
 , ruamel-yaml
-, setuptools
+, scmrepo
+, setuptools-scm
 , tabulate
 }:
 
 buildPythonPackage rec {
   pname = "dvclive";
-  version = "2.1.0";
+  version = "2.8.1";
   format = "pyproject";
 
   disabled = pythonOlder "3.8";
@@ -20,17 +23,22 @@ buildPythonPackage rec {
     owner = "iterative";
     repo = pname;
     rev = "refs/tags/${version}";
-    hash = "sha256-n3JjZrh2ImpjW1MUTwag716qaVjSChrYqNJuNp6K1s8=";
+    hash = "sha256-Wb0347y+oDFOctKjJUYPtqIFdsRm2wt16ZlQWDgjr7g=";
   };
 
+  SETUPTOOLS_SCM_PRETEND_VERSION = version;
+
   nativeBuildInputs = [
-    setuptools
+    setuptools-scm
   ];
 
   propagatedBuildInputs = [
-    dvc-render
+    dvc
+    dvc-studio-client
+    funcy
     ruamel-yaml
-  ] ++ dvc-render.optional-dependencies.table;
+    scmrepo
+  ];
 
   # Circular dependency with dvc
   doCheck = false;

pkgs/development/python-modules/iterative-telemetry/default.nix

@@ -0,0 +1,57 @@
+{ lib
+, appdirs
+, buildPythonPackage
+, distro
+, fetchFromGitHub
+, filelock
+, pytestCheckHook
+, pytest-mock
+, pythonOlder
+, requests
+, setuptools-scm
+}:
+
+buildPythonPackage rec {
+  pname = "iterative-telemtry";
+  version = "0.0.7";
+  format = "pyproject";
+
+  disabled = pythonOlder "3.8";
+
+  src = fetchFromGitHub {
+    owner = "iterative";
+    repo = "telemetry-python";
+    rev = "refs/tags/${version}";
+    hash = "sha256-n67nc9a/Qrz2v1EYbHZb+pGhuMDqofUMpgfD/0BwqLM=";
+  };
+
+  SETUPTOOLS_SCM_PRETEND_VERSION = version;
+
+  nativeBuildInputs = [
+    setuptools-scm
+  ];
+
+  propagatedBuildInputs = [
+    requests
+    appdirs
+    filelock
+    distro
+  ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+    pytest-mock
+  ];
+
+  pythonImportsCheck = [
+    "iterative_telemetry"
+  ];
+
+  meta = with lib; {
+    description = "Common library to send usage telemetry";
+    homepage = "https://github.com/iterative/iterative-telemetry";
+    changelog = "https://github.com/iterative/iterative-telemetry/releases/tag/${version}";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ melling ];
+  };
+}

pkgs/development/python-modules/omegaconf/default.nix

@@ -60,6 +60,11 @@ buildPythonPackage rec {
     "omegaconf"
   ];
 
+  pytestFlagsArray = [
+    "-W"
+    "ignore::DeprecationWarning"
+  ];
+
   meta = with lib; {
     description = "Framework for configuring complex applications";
     homepage = "https://github.com/omry/omegaconf";

pkgs/development/python-modules/pydevd/default.nix

@@ -1,22 +1,26 @@
 { lib
-, fetchFromGitHub
 , buildPythonPackage
-, pytestCheckHook
-, untangle
-, psutil
-, trio
+, fetchFromGitHub
 , numpy
+, psutil
+, pytestCheckHook
+, pythonOlder
+, trio
+, untangle
 }:
 
 buildPythonPackage rec {
   pname = "pydevd";
-  version = "2.8.0";
+  version = "2.9.6";
+  format = "setuptools";
+
+  disabled = pythonOlder "3.8";
 
   src = fetchFromGitHub {
     owner = "fabioz";
     repo = "PyDev.Debugger";
     rev = "pydev_debugger_${lib.replaceStrings ["."] ["_"] version}";
-    hash = "sha256-+yRngN10654trB09ZZa8QQsTPdM7VxVj7r6jh7OcgAA=";
+    hash = "sha256-TDU/V7kY7zVxiP4OVjGqpsRVYplpkgCly2qAOqhZONo=";
   };
 
   nativeCheckInputs = [
@@ -46,13 +50,14 @@ buildPythonPackage rec {
     "test_tracing_basic"
   ];
 
-  pythonImportsCheck = [ "pydevd" ];
+  pythonImportsCheck = [
+    "pydevd"
+  ];
 
   meta = with lib; {
     description = "PyDev.Debugger (used in PyDev, PyCharm and VSCode Python)";
     homepage = "https://github.com/fabioz/PyDev.Debugger";
     license = licenses.epl10;
     maintainers = with maintainers; [ onny ];
-    broken = true;
   };
 }

pkgs/development/python-modules/scmrepo/default.nix

@@ -11,6 +11,7 @@
 , pygtrie
 , pythonOlder
 , setuptools
+, setuptools-scm
 , shortuuid
 }:
 
@@ -28,14 +29,11 @@ buildPythonPackage rec {
     hash = "sha256-MREY8i6FIeRyjcCKvS8gthsVql81x4Ab7gA7yFgwNoQ=";
   };
 
-  postPatch = ''
-    substituteInPlace setup.cfg \
-      --replace "asyncssh>=2.7.1,<2.9" "asyncssh>=2.7.1" \
-      --replace "pathspec>=0.9.0,<0.10.0" "pathspec"
-  '';
+  SETUPTOOLS_SCM_PRETEND_VERSION = version;
 
   nativeBuildInputs = [
     setuptools
+    setuptools-scm
   ];
 
   propagatedBuildInputs = [

pkgs/top-level/python-packages.nix

@@ -2997,14 +2997,24 @@ self: super: with self; {
 
   durus = callPackage ../development/python-modules/durus {  };
 
+  dvc-azure = callPackage ../development/python-modules/dvc-azure {  };
+
   dvc-data = callPackage ../development/python-modules/dvc-data {  };
 
+  dvc-gs = callPackage ../development/python-modules/dvc-gs { };
+
   dvc-http = callPackage ../development/python-modules/dvc-http {  };
 
   dvc-objects = callPackage ../development/python-modules/dvc-objects {  };
 
   dvc-render = callPackage ../development/python-modules/dvc-render {  };
 
+  dvc-s3 = callPackage ../development/python-modules/dvc-s3 { };
+
+  dvc-ssh = callPackage ../development/python-modules/dvc-ssh { };
+
+  dvc-studio-client = callPackage ../development/python-modules/dvc-studio-client {  };
+
   dvc-task = callPackage ../development/python-modules/dvc-task {  };
 
   dvclive = callPackage ../development/python-modules/dvclive {  };
@@ -4951,6 +4961,8 @@ self: super: with self; {
 
   iteration-utilities = callPackage ../development/python-modules/iteration-utilities { };
 
+  iterative-telemetry = callPackage ../development/python-modules/iterative-telemetry { };
+
   iterm2 = callPackage ../development/python-modules/iterm2 { };
 
   itsdangerous = callPackage ../development/python-modules/itsdangerous { };