From f2f5d1266881c0e415fa5a88016555efb08c9c91 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 13 May 2022 21:45:05 +1200 Subject: [PATCH] poetry2nix: 1.28.0 -> 1.29.0 --- .../tools/poetry2nix/poetry2nix/default.nix | 2 +- .../poetry2nix/poetry2nix/hooks/default.nix | 66 +++---- .../hooks/pyproject-without-special-deps.py | 20 +- .../hooks/remove-special-dependencies.sh | 17 +- .../poetry2nix/overrides/build-systems.json | 63 ++++++ .../poetry2nix/overrides/default.nix | 181 +++++++----------- .../poetry2nix/overrides/shapely-rewrite.py | 46 +++++ .../tools/poetry2nix/poetry2nix/pep425.nix | 4 +- 8 files changed, 236 insertions(+), 163 deletions(-) create mode 100644 pkgs/development/tools/poetry2nix/poetry2nix/overrides/shapely-rewrite.py diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/default.nix b/pkgs/development/tools/poetry2nix/poetry2nix/default.nix index f42ee6b79cc0..45a7e6619d1b 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/default.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/default.nix @@ -5,7 +5,7 @@ }: let # Poetry2nix version - version = "1.28.0"; + version = "1.29.0"; inherit (poetryLib) isCompatible readTOML moduleName; diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix index 5d562017b8fb..f6d1ea5418fa 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/default.nix @@ -3,48 +3,46 @@ , makeSetupHook , wheel , pip +, pkgs }: let callPackage = python.pythonForBuild.pkgs.callPackage; pythonInterpreter = python.pythonForBuild.interpreter; pythonSitePackages = python.sitePackages; + + nonOverlayedPython = pkgs.python3.pythonForBuild.withPackages (ps: [ ps.tomlkit ]); + makeRemoveSpecialDependenciesHook = { fields, kind }: + nonOverlayedPython.pkgs.callPackage + ( + {}: + makeSetupHook + { + name = "remove-path-dependencies.sh"; + deps = [ ]; + substitutions = { + # NOTE: We have to use a non-overlayed Python here because otherwise we run into an infinite recursion + # because building of tomlkit and its dependencies also use these hooks. + pythonPath = nonOverlayedPython.pkgs.makePythonPath [ nonOverlayedPython ]; + pythonInterpreter = nonOverlayedPython.interpreter; + pyprojectPatchScript = "${./pyproject-without-special-deps.py}"; + fields = fields; + kind = kind; + }; + } ./remove-special-dependencies.sh + ) + { }; in { + removePathDependenciesHook = makeRemoveSpecialDependenciesHook { + fields = [ "path" ]; + kind = "path"; + }; - removePathDependenciesHook = callPackage - ( - {}: - makeSetupHook - { - name = "remove-path-dependencies.sh"; - deps = [ ]; - substitutions = { - inherit pythonInterpreter; - yj = "${buildPackages.yj}/bin/yj"; - pyprojectPatchScript = "${./pyproject-without-special-deps.py}"; - fields = [ "path" ]; - kind = "path"; - }; - } ./remove-special-dependencies.sh - ) - { }; + removeGitDependenciesHook = makeRemoveSpecialDependenciesHook { + fields = [ "git" "branch" "rev" "tag" ]; + kind = "git"; + }; - removeGitDependenciesHook = callPackage - ({}: - makeSetupHook - { - name = "remove-git-dependencies.sh"; - deps = [ ]; - substitutions = { - inherit pythonInterpreter; - yj = "${buildPackages.yj}/bin/yj"; - pyprojectPatchScript = "${./pyproject-without-special-deps.py}"; - fields = [ "git" "branch" "rev" "tag" ]; - kind = "git"; - }; - } ./remove-special-dependencies.sh - ) - { }; pipBuildHook = callPackage ( @@ -89,6 +87,4 @@ in } ./wheel-unpack-hook.sh ) { }; - - } diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/pyproject-without-special-deps.py b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/pyproject-without-special-deps.py index 9f79f9afab56..b74bfebf6400 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/pyproject-without-special-deps.py +++ b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/pyproject-without-special-deps.py @@ -1,13 +1,14 @@ #!/usr/bin/env python -# Patch out special dependencies (git and path) from a pyproject.json file +# Patch out special dependencies (git and path) from a pyproject.toml file import argparse -import json import sys +import tomlkit + def main(input, output, fields_to_remove): - data = json.load(input) + data = tomlkit.loads(input.read()) try: deps = data["tool"]["poetry"]["dependencies"] @@ -22,12 +23,7 @@ def main(input, output, fields_to_remove): if any_removed: dep["version"] = "*" - # Set ensure_ascii to False because TOML is valid UTF-8 so text that can't - # be represented in ASCII is perfectly legitimate - # HACK: Setting ensure_asscii to False breaks Python2 for some dependencies (like cachy==0.3.0) - json.dump( - data, output, separators=(",", ":"), ensure_ascii=sys.version_info.major < 3 - ) + output.write(tomlkit.dumps(data)) if __name__ == "__main__": @@ -37,20 +33,20 @@ if __name__ == "__main__": "--input", type=argparse.FileType("r"), default=sys.stdin, - help="Location from which to read input JSON", + help="Location from which to read input TOML", ) p.add_argument( "-o", "--output", type=argparse.FileType("w"), default=sys.stdout, - help="Location to write output JSON", + help="Location to write output TOML", ) p.add_argument( "-f", "--fields-to-remove", nargs="+", - help="The fields to remove from the dependency's JSON", + help="The fields to remove from the dependency's TOML", ) args = p.parse_args() diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/remove-special-dependencies.sh b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/remove-special-dependencies.sh index 2d37667d6fdf..904efa615b1d 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/remove-special-dependencies.sh +++ b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/remove-special-dependencies.sh @@ -1,18 +1,21 @@ remove-@kind@-dependencies-hook() { + # Tell poetry not to resolve special dependencies. Any version is fine! + if ! test -f pyproject.toml; then return fi echo "Removing @kind@ dependencies" - # Tell poetry not to resolve special dependencies. Any version is fine! - @yj@ -tj < pyproject.toml | \ - @pythonInterpreter@ \ - @pyprojectPatchScript@ \ - --fields-to-remove @fields@ > pyproject.json - @yj@ -jt < pyproject.json > pyproject.toml + # NOTE: We have to reset PYTHONPATH to avoid having propagatedBuildInputs + # from the currently building derivation leaking into our unrelated Python + # environment. + PYTHONPATH=@pythonPath@ \ + @pythonInterpreter@ \ + @pyprojectPatchScript@ \ + --fields-to-remove @fields@ < pyproject.toml > pyproject.formatted.toml - rm pyproject.json + mv pyproject.formatted.toml pyproject.toml echo "Finished removing @kind@ dependencies" } diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/overrides/build-systems.json b/pkgs/development/tools/poetry2nix/poetry2nix/overrides/build-systems.json index e4d64b05b997..78e1e87f6bf7 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/overrides/build-systems.json +++ b/pkgs/development/tools/poetry2nix/poetry2nix/overrides/build-systems.json @@ -50,6 +50,9 @@ "aiokafka": [ "cython" ], + "aiolimiter": [ + "poetry-core" + ], "aiomultiprocess": [ "flit-core" ], @@ -229,6 +232,9 @@ "flit-core", "flitBuildHook" ], + "copier": [ + "poetry-core" + ], "cpyparsing": [ "cython" ], @@ -239,6 +245,9 @@ "cython", "poetry-core" ], + "cruft": [ + "poetry-core" + ], "cssselect2": [ "flit", "flit-core" @@ -350,6 +359,9 @@ "enumatch": [ "poetry" ], + "envs": [ + "poetry-core" + ], "exceptiongroup": [ "flit-core", "flitBuildHook" @@ -366,6 +378,9 @@ "fastavro": [ "cython" ], + "fastbencode": [ + "cython" + ], "fastdtw": [ "cython" ], @@ -429,6 +444,9 @@ "gensim": [ "cython" ], + "gidgethub": [ + "flitBuildHook" + ], "glances-api": [ "poetry-core" ], @@ -480,6 +498,9 @@ "hashids": [ "flit-core" ], + "hatch-vcs": [ + "hatchling" + ], "hdate": [ "poetry-core" ], @@ -711,6 +732,9 @@ "monosat": [ "cython" ], + "more-itertools": [ + "flit-core" + ], "motioneye-client": [ "poetry-core" ], @@ -720,6 +744,9 @@ "msgpack-numpy": [ "cython" ], + "msgpack-types": [ + "poetry" + ], "msoffcrypto-tool": [ "poetry-core" ], @@ -735,6 +762,9 @@ "mypy-boto3-builder": [ "poetry-core" ], + "myst-parser": [ + "flit-core" + ], "nats-python": [ "poetry-core" ], @@ -807,6 +837,9 @@ "openstacksdk": [ "pbr" ], + "openstep-plist": [ + "cython" + ], "openvino": [ "cython" ], @@ -889,6 +922,9 @@ "pixelmatch": [ "poetry-core" ], + "pkgconfig": [ + "poetry-core" + ], "poetry": [ "poetry-core" ], @@ -925,6 +961,12 @@ "prometheus-fastapi-instrumentator": [ "poetry" ], + "prospector": [ + "poetry-core" + ], + "protoletariat": [ + "poetry-core" + ], "ptyprocess": [ "flit-core" ], @@ -1234,6 +1276,9 @@ "python-swiftclient": [ "pbr" ], + "python-trovo": [ + "poetry" + ], "python_openzwave": [ "cython" ], @@ -1276,6 +1321,9 @@ "qiskit-terra": [ "cython" ], + "qstylizer": [ + "pbr" + ], "questionary": [ "poetry-core" ], @@ -1330,6 +1378,9 @@ "rich": [ "poetry-core" ], + "rio-tiler": [ + "flit-core" + ], "ripser": [ "cython" ], @@ -1367,6 +1418,9 @@ "sarif-om": [ "pbr" ], + "sat-search": [ + "pytest-runner" + ], "scikit-bio": [ "cython" ], @@ -1430,6 +1484,9 @@ "single-version": [ "poetry-core" ], + "skia-pathops": [ + "cython" + ], "slowapi": [ "poetry-core" ], @@ -1602,6 +1659,9 @@ "toggl-cli": [ "pbr" ], + "toml-cli": [ + "poetry" + ], "toml-sort": [ "poetry" ], @@ -1620,6 +1680,9 @@ "traceback2": [ "pbr" ], + "traitlets": [ + "flit-core" + ], "transmission-rpc": [ "poetry-core" ], diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/overrides/default.nix b/pkgs/development/tools/poetry2nix/poetry2nix/overrides/default.nix index a04e2d65385c..83e5bb339dd5 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/overrides/default.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/overrides/default.nix @@ -74,6 +74,12 @@ lib.composeManyExtensions [ inherit (pkgs) buildPackages; pyBuildPackages = self.python.pythonForBuild.pkgs; + selectQt5 = version: + let + selector = builtins.concatStringsSep "" (lib.take 2 (builtins.splitVersion version)); + in + pkgs."qt${selector}" or pkgs.qt5; + in { @@ -287,6 +293,7 @@ lib.composeManyExtensions [ "36.0.0" = "sha256-Y6TuW7AryVgSvZ6G8WNoDIvi+0tvx8ZlEYF5qB0jfNk="; "36.0.1" = "sha256-kozYXkqt1Wpqyo9GYCwN08J+zV92ZWFJY/f+rulxmeQ="; "36.0.2" = "1a0ni1a3dbv2dvh6gx2i54z8v5j9m6asqg97kkv7gqb1ivihsbp8"; + "37.0.2" = "sha256-qvrxvneoBXjP96AnUPyrtfmCnZo+IriHR5HbtWQ5Gk8="; }.${version} or null; sha256 = getCargoHash super.cryptography.version; scrypto = @@ -517,6 +524,14 @@ lib.composeManyExtensions [ } ); + file-magic = super.file-magic.overridePythonAttrs ( + old: { + postPatch = '' + substituteInPlace magic.py --replace "find_library('magic')" "'${pkgs.file}/lib/libmagic${pkgs.stdenv.hostPlatform.extensions.sharedLibrary}'" + ''; + } + ); + fiona = super.fiona.overridePythonAttrs ( old: { buildInputs = (old.buildInputs or [ ]) ++ [ pkgs.gdal_2 ]; @@ -735,6 +750,14 @@ lib.composeManyExtensions [ } ); + jinja2-ansible-filters = super.jinja2-ansible-filters.overridePythonAttrs ( + old: { + preBuild = (old.preBuild or "") + '' + echo "${old.version}" > VERSION + ''; + } + ); + jira = super.jira.overridePythonAttrs ( old: { inherit (pkgs.python3Packages.jira) patches; @@ -1056,6 +1079,18 @@ lib.composeManyExtensions [ mypy = super.mypy.overridePythonAttrs ( old: { + buildInputs = (old.buildInputs or [ ]) ++ [ + self.types-typed-ast + ]; + # Compile mypy with mypyc, which makes mypy about 4 times faster. The compiled + # version is also the default in the wheels on Pypi that include binaries. + # is64bit: unfortunately the build would exhaust all possible memory on i686-linux. + MYPY_USE_MYPYC = stdenv.buildPlatform.is64bit; + + # when testing reduce optimisation level to drastically reduce build time + # (default is 3) + # MYPYC_OPT_LEVEL = 1; + } // lib.optionalAttrs (old.format != "wheel") { # FIXME: Remove patch after upstream has decided the proper solution. # https://github.com/python/mypy/pull/11143 patches = (old.patches or [ ]) ++ lib.optionals ((lib.strings.versionAtLeast old.version "0.900") && lib.strings.versionOlder old.version "0.940") [ @@ -1069,17 +1104,6 @@ lib.composeManyExtensions [ sha256 = "sha256-waIZ+m3tfvYE4HJ8kL6rN/C4fMjvLEe9UoPbt9mHWIM="; }) ]; - buildInputs = (old.buildInputs or [ ]) ++ [ - self.types-typed-ast - ]; - # Compile mypy with mypyc, which makes mypy about 4 times faster. The compiled - # version is also the default in the wheels on Pypi that include binaries. - # is64bit: unfortunately the build would exhaust all possible memory on i686-linux. - MYPY_USE_MYPYC = stdenv.buildPlatform.is64bit; - - # when testing reduce optimisation level to drastically reduce build time - # (default is 3) - # MYPYC_OPT_LEVEL = 1; } ); @@ -1566,106 +1590,35 @@ lib.composeManyExtensions [ pyqt5 = let - drv = super.pyqt5; - withConnectivity = drv.passthru.args.withConnectivity or false; - withMultimedia = drv.passthru.args.withMultimedia or false; - withWebKit = drv.passthru.args.withWebKit or false; - withWebSockets = drv.passthru.args.withWebSockets or false; + qt5 = selectQt5 super.pyqt5.version; in super.pyqt5.overridePythonAttrs ( old: { - format = "other"; - + dontConfigure = true; dontWrapQtApps = true; - - nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ - pkg-config - pkgs.qt5.qmake - pkgs.xorg.lndir - pkgs.qt5.qtbase - pkgs.qt5.qtsvg - pkgs.qt5.qtdeclarative - pkgs.qt5.qtwebchannel - pkgs.qt5.qt3d - # self.pyqt5-sip + nativeBuildInputs = old.nativeBuildInputs or [ ] ++ [ + self.pyqt-builder self.sip - ] - ++ lib.optional withConnectivity pkgs.qt5.qtconnectivity - ++ lib.optional withMultimedia pkgs.qt5.qtmultimedia - ++ lib.optional withWebKit pkgs.qt5.qtwebkit - ++ lib.optional withWebSockets pkgs.qt5.qtwebsockets - ; + qt5.full + ]; + } + ); - buildInputs = (old.buildInputs or [ ]) ++ [ - pkgs.dbus - pkgs.qt5.qtbase - pkgs.qt5.qtsvg - pkgs.qt5.qtdeclarative - self.sip - ] - ++ lib.optional withConnectivity pkgs.qt5.qtconnectivity - ++ lib.optional withWebKit pkgs.qt5.qtwebkit - ++ lib.optional withWebSockets pkgs.qt5.qtwebsockets - ; - - # Fix dbus mainloop - patches = pkgs.python3.pkgs.pyqt5.patches or [ ]; - - configurePhase = '' - runHook preConfigure - - export PYTHONPATH=$PYTHONPATH:$out/${self.python.sitePackages} - - mkdir -p $out/${self.python.sitePackages}/dbus/mainloop - ${self.python.executable} configure.py -w \ - --confirm-license \ - --no-qml-plugin \ - --bindir=$out/bin \ - --destdir=$out/${self.python.sitePackages} \ - --stubsdir=$out/${self.python.sitePackages}/PyQt5 \ - --sipdir=$out/share/sip/PyQt5 \ - --designer-plugindir=$out/plugins/designer - - runHook postConfigure - ''; - - postInstall = '' - ln -s ${self.pyqt5-sip}/${self.python.sitePackages}/PyQt5/sip.* $out/${self.python.sitePackages}/PyQt5/ - for i in $out/bin/*; do - wrapProgram $i --prefix PYTHONPATH : "$PYTHONPATH" - done - - # Let's make it a namespace package - cat << EOF > $out/${self.python.sitePackages}/PyQt5/__init__.py - from pkgutil import extend_path - __path__ = extend_path(__path__, __name__) - EOF - ''; - - installCheckPhase = - let - modules = [ - "PyQt5" - "PyQt5.QtCore" - "PyQt5.QtQml" - "PyQt5.QtWidgets" - "PyQt5.QtGui" - ] - ++ lib.optional withWebSockets "PyQt5.QtWebSockets" - ++ lib.optional withWebKit "PyQt5.QtWebKit" - ++ lib.optional withMultimedia "PyQt5.QtMultimedia" - ++ lib.optional withConnectivity "PyQt5.QtConnectivity" - ; - imports = lib.concatMapStrings (module: "import ${module};") modules; - in - '' - echo "Checking whether modules can be imported..." - ${self.python.interpreter} -c "${imports}" - ''; - - doCheck = true; - - enableParallelBuilding = true; + pyqt5-qt5 = + let + qt5 = selectQt5 super.pyqt5-qt5.version; + in + super.pyqt5-qt5.overridePythonAttrs ( + old: { + dontWrapQtApps = true; + propagatedBuildInputs = old.propagatedBuildInputs or [ ] ++ [ + qt5.full + qt5.qtgamepad # As of 2022-05-13 not a port of qt5.full + pkgs.gtk3 + pkgs.speechd + pkgs.postgresql + pkgs.unixODBC + ]; } ); @@ -1681,7 +1634,10 @@ lib.composeManyExtensions [ old: { # Fixes https://github.com/pytest-dev/pytest/issues/7891 postPatch = old.postPatch or "" + '' - sed -i '/\[metadata\]/aversion = ${old.version}' setup.cfg + # sometimes setup.cfg doesn't exist + if [ -f setup.cfg ]; then + sed -i '/\[metadata\]/aversion = ${old.version}' setup.cfg + fi ''; } ); @@ -1758,6 +1714,10 @@ lib.composeManyExtensions [ } ); + python-twitter = super.python-twitter.overridePythonAttrs (old: { + buildInputs = (old.buildInputs or [ ]) ++ [ self.pytest-runner ]; + }); + pythran = super.pythran.overridePythonAttrs (old: { buildInputs = (old.buildInputs or [ ]) ++ [ self.pytest-runner ]; }); @@ -1925,7 +1885,14 @@ lib.composeManyExtensions [ shapely = super.shapely.overridePythonAttrs ( old: { buildInputs = (old.buildInputs or [ ]) ++ [ pkgs.geos ]; - inherit (pkgs.python3.pkgs.shapely) patches GEOS_LIBRARY_PATH; + inherit (pkgs.python3.pkgs.shapely) GEOS_LIBRARY_PATH; + + GEOS_LIBC = lib.optionalString (!stdenv.isDarwin) "${stdenv.cc.libc}/lib/libc${stdenv.hostPlatform.extensions.sharedLibrary}.6"; + + # Fix library paths + postPatch = old.postPatch or "" + '' + ${self.python.pythonForBuild.interpreter} ${./shapely-rewrite.py} shapely/geos.py + ''; } ); diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/overrides/shapely-rewrite.py b/pkgs/development/tools/poetry2nix/poetry2nix/overrides/shapely-rewrite.py new file mode 100644 index 000000000000..d3b365459bd3 --- /dev/null +++ b/pkgs/development/tools/poetry2nix/poetry2nix/overrides/shapely-rewrite.py @@ -0,0 +1,46 @@ +""" +Rewrite libc/library path references to Nix store paths +Nixpkgs uses a normal patch for this but we need to be less +sensitive to changes between versions. +""" +from textwrap import dedent +import sys +import ast +import os + + +with open(sys.argv[1]) as f: + mod = ast.parse(f.read(), "geos.py") + + +class LibTransformer(ast.NodeTransformer): + _lgeos_replaced = False + + def visit_If(self, node): + if ast.unparse(node).startswith("if sys.platform.startswith('linux')"): + return ast.parse( + dedent( + """ + free = CDLL(%s).free + free.argtypes = [c_void_p] + free.restype = None + """ + ) + % (lambda x: "'" + x + "'" if x else None)(os.environ.get("GEOS_LIBC")) + ) + return node + + def visit_Assign(self, node): + _target = node.targets[0] + if ( + not self._lgeos_replaced + and isinstance(_target, ast.Name) + and _target.id == "_lgeos" + ): + self._lgeos_replaced = True + return ast.parse("_lgeos = CDLL('%s')" % os.environ["GEOS_LIBRARY_PATH"]) + return node + + +with open(sys.argv[1], "w") as f: + f.write(ast.unparse(LibTransformer().visit(mod))) diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix b/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix index b16bd5f3cebc..00c33956edd1 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix @@ -23,7 +23,9 @@ let let entries' = splitString "-" str; # Hack: Remove version "suffixes" like 2.11.4-1 - entries = builtins.filter (x: builtins.match "[0-9]" x == null) entries'; + # Some wheels have build tag with more than one digit + # like openvino-2022.1.0-7019-cp36-cp36m-manylinux_2_27_x86_64.whl + entries = builtins.filter (x: builtins.match "[0-9]*" x == null) entries'; p = removeSuffix ".whl" (builtins.elemAt entries 4); in {