Merge pull request #326690 from natsukium/graphrag/init

python312Packages.graphrag: init at 0.1.1 and introduce some dependencies

pkgs/development/python-modules/azure-search-documents/default.nix

@@ -0,0 +1,46 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  setuptools,
+  azure-common,
+  azure-core,
+  isodate,
+  pytestCheckHook,
+}:
+
+buildPythonPackage rec {
+  pname = "azure-search-documents";
+  version = "11.4.0";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "Azure";
+    repo = "azure-sdk-for-python";
+    rev = "azure-search-documents_${version}";
+    hash = "sha256-0J9AXDH7TOkcKDwFbICiMatLAwiFq3Jtoji8fJSOg8k=";
+  };
+
+  sourceRoot = "${src.name}/sdk/search/azure-search-documents";
+
+  build-system = [ setuptools ];
+
+  dependencies = [
+    azure-common
+    azure-core
+    isodate
+  ];
+
+  pythonImportsCheck = [ "azure.search.documents" ];
+
+  # require devtools_testutils which is a internal package for azure-sdk
+  doCheck = false;
+
+  meta = {
+    description = "Microsoft Azure Cognitive Search Client Library for Python";
+    homepage = "https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/search/azure-search-documents";
+    changelog = "https://github.com/Azure/azure-sdk-for-python/blob/${src.rev}/sdk/search/azure-search-documents/CHANGELOG.md";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ natsukium ];
+  };
+}

pkgs/development/python-modules/datashaper/default.nix

@@ -0,0 +1,52 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchPypi,
+  pythonOlder,
+  pythonRelaxDepsHook,
+  poetry-core,
+  dacite,
+  diskcache,
+  jsonschema,
+  pandas,
+  pyarrow,
+}:
+
+buildPythonPackage rec {
+  pname = "datashaper";
+  version = "0.0.49";
+  pyproject = true;
+
+  disabled = pythonOlder "3.10";
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-Bb+6WWRHSmK91SWew/oBc9AeNlIItqSv9OoOYwlqdTM=";
+  };
+
+  build-system = [ poetry-core ];
+
+  nativeBuildInputs = [ pythonRelaxDepsHook ];
+
+  pythonRelaxDeps = [ "pyarrow" ];
+
+  dependencies = [
+    dacite
+    diskcache
+    jsonschema
+    pandas
+    pyarrow
+  ];
+
+  pythonImportsCheck = [ "datashaper" ];
+
+  # pypi tarball has no tests
+  doCheck = false;
+
+  meta = {
+    description = "Collection of utilities for doing lightweight data wrangling";
+    homepage = "https://github.com/microsoft/datashaper/tree/main/python/datashaper";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ natsukium ];
+  };
+}

pkgs/development/python-modules/graphrag/default.nix

@@ -0,0 +1,122 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  poetry-core,
+  poetry-dynamic-versioning,
+  aiofiles,
+  aiolimiter,
+  azure-identity,
+  azure-search-documents,
+  azure-storage-blob,
+  datashaper,
+  devtools,
+  environs,
+  fastparquet,
+  graspologic,
+  lancedb,
+  networkx,
+  nltk,
+  numba,
+  numpy,
+  openai,
+  pyaml-env,
+  pydantic,
+  python-dotenv,
+  pyyaml,
+  rich,
+  scipy,
+  swifter,
+  tenacity,
+  textual,
+  tiktoken,
+  typing-extensions,
+  uvloop,
+  nbformat,
+  pytest-asyncio,
+  pytestCheckHook,
+}:
+
+buildPythonPackage rec {
+  pname = "graphrag";
+  version = "0.1.1";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "microsoft";
+    repo = "graphrag";
+    rev = "refs/tags/v${version}";
+    hash = "sha256-hIAQOIqm9S9AtssE6UxcXfaIbSt3+506ueMrlathNaQ=";
+  };
+
+  build-system = [
+    poetry-core
+    poetry-dynamic-versioning
+  ];
+
+  pythonRelaxDeps = [
+    "aiofiles"
+    "azure-identity"
+    "scipy"
+    "tiktoken"
+  ];
+
+  dependencies = [
+    aiofiles
+    aiolimiter
+    azure-identity
+    azure-search-documents
+    azure-storage-blob
+    datashaper
+    devtools
+    environs
+    fastparquet
+    graspologic
+    lancedb
+    networkx
+    nltk
+    numba
+    numpy
+    openai
+    pyaml-env
+    pydantic
+    python-dotenv
+    pyyaml
+    rich
+    scipy
+    swifter
+    tenacity
+    textual
+    tiktoken
+    typing-extensions
+    uvloop
+  ];
+
+  env.NUMBA_CACHE_DIR = "$TMPDIR";
+
+  pythonImportsCheck = [ "graphrag" ];
+
+  nativeCheckInputs = [
+    nbformat
+    pytest-asyncio
+    pytestCheckHook
+  ];
+
+  pytestFlagsArray = [ "tests/unit" ];
+
+  disabledTests = [
+    # touch the network
+    "test_child"
+    "test_dotprefix"
+    "test_find"
+    "test_run_extract_entities_multiple_documents"
+    "test_run_extract_entities_single_document"
+  ];
+
+  meta = {
+    description = "Modular graph-based Retrieval-Augmented Generation (RAG) system";
+    homepage = "https://github.com/microsoft/graphrag";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ natsukium ];
+  };
+}

pkgs/development/python-modules/pyaml-env/default.nix

@@ -0,0 +1,37 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  setuptools,
+  pyyaml,
+  pytestCheckHook,
+}:
+
+buildPythonPackage rec {
+  pname = "pyaml-env";
+  version = "1.2.1";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "mkaranasou";
+    repo = "pyaml_env";
+    rev = "refs/tags/v${version}";
+    hash = "sha256-xSu+dksSVugShJwOqedXBrXIKaH0G5JAsynauOuP3OA=";
+  };
+
+  build-system = [ setuptools ];
+
+  dependencies = [ pyyaml ];
+
+  pythonImportsCheck = [ "pyaml_env" ];
+
+  nativeCheckInputs = [ pytestCheckHook ];
+
+  meta = {
+    description = "Parse YAML configuration with environment variables in Python";
+    homepage = "https://github.com/mkaranasou/pyaml_env";
+    changelog = "https://github.com/mkaranasou/pyaml_env/releases/tag/v${version}";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ natsukium ];
+  };
+}

pkgs/development/python-modules/swifter/default.nix

@@ -0,0 +1,52 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  setuptools,
+  dask,
+  pandas,
+  psutil,
+  tqdm,
+  ipywidgets,
+  ray,
+}:
+
+buildPythonPackage rec {
+  pname = "swifter";
+  version = "1.4.0";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "jmcarpenter2";
+    repo = "swifter";
+    rev = "refs/tags/${version}";
+    hash = "sha256-lgdf8E9GGjeLY4ERzxqtjQuYVtdtIZt2HFLSiNBbtX4=";
+  };
+
+  build-system = [ setuptools ];
+
+  dependencies = [
+    pandas
+    psutil
+    dask
+    tqdm
+  ] ++ dask.optional-dependencies.dataframe;
+
+  optional-dependencies = {
+    groupby = [ ray ];
+    notebook = [ ipywidgets ];
+  };
+
+  pythonImportsCheck = [ "swifter" ];
+
+  # tests may hang due to ignoring cpu core limit
+  # https://github.com/jmcarpenter2/swifter/issues/221
+  doCheck = false;
+
+  meta = {
+    description = "Package which efficiently applies any function to a pandas dataframe or series in the fastest available manner";
+    homepage = "https://github.com/jmcarpenter2/swifter";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ natsukium ];
+  };
+}

pkgs/top-level/python-packages.nix

@@ -1316,6 +1316,8 @@ self: super: with self; {
 
   azure-nspkg = callPackage ../development/python-modules/azure-nspkg { };
 
+  azure-search-documents = callPackage ../development/python-modules/azure-search-documents { };
+
   azure-servicebus = callPackage ../development/python-modules/azure-servicebus { };
 
   azure-servicefabric = callPackage ../development/python-modules/azure-servicefabric { };
@@ -2845,6 +2847,8 @@ self: super: with self; {
 
   datashape = callPackage ../development/python-modules/datashape { };
 
+  datashaper = callPackage ../development/python-modules/datashaper { };
+
   datatable = callPackage ../development/python-modules/datatable { };
 
   datauri = callPackage ../development/python-modules/datauri { };
@@ -5238,6 +5242,8 @@ self: super: with self; {
 
   graphql-subscription-manager = callPackage ../development/python-modules/graphql-subscription-manager { };
 
+  graphrag = callPackage ../development/python-modules/graphrag { };
+
   graph-tool = callPackage ../development/python-modules/graph-tool { };
 
   graphtage = callPackage ../development/python-modules/graphtage { };
@@ -10892,6 +10898,8 @@ self: super: with self; {
 
   pyaml = callPackage ../development/python-modules/pyaml { };
 
+  pyaml-env = callPackage ../development/python-modules/pyaml-env { };
+
   pyannotate = callPackage ../development/python-modules/pyannotate { };
 
   pyannote-audio = callPackage ../development/python-modules/pyannote-audio { };
@@ -15038,6 +15046,8 @@ self: super: with self; {
 
   swift = callPackage ../development/python-modules/swift { };
 
+  swifter = callPackage ../development/python-modules/swifter { };
+
   swisshydrodata = callPackage ../development/python-modules/swisshydrodata { };
 
   swspotify = callPackage ../development/python-modules/swspotify { };