Merge pull request #188306 from gbpdt/fix/airflow_2.3.4

maintainers/maintainer-list.nix

@@ -4788,6 +4788,12 @@
     githubId = 16470252;
     name = "Gemini Lasswell";
   };
+  gbpdt = {
+    email = "nix@pdtpartners.com";
+    github = "gbpdt";
+    githubId = 25106405;
+    name = "Graham Bennett";
+  };
   gbtb = {
     email = "goodbetterthebeast3@gmail.com";
     github = "gbtb";

pkgs/development/python-modules/apache-airflow/default.nix

@@ -11,15 +11,19 @@
 , cattrs
 , clickclick
 , colorlog
+, connexion
+, cron-descriptor
 , croniter
 , cryptography
 , dataclasses
+, deprecated
 , dill
 , flask
 , flask_login
-, flask-wtf
 , flask-appbuilder
 , flask-caching
+, flask-session
+, flask-wtf
 , GitPython
 , graphviz
 , gunicorn
@@ -32,13 +36,16 @@
 , jinja2
 , jsonschema
 , lazy-object-proxy
+, linkify-it-py
 , lockfile
 , markdown
 , markupsafe
 , marshmallow-oneofschema
+, mdit-py-plugins
 , numpy
 , openapi-spec-validator
 , pandas
+, pathspec
 , pendulum
 , psutil
 , pygments
@@ -58,20 +65,27 @@
 , tabulate
 , tenacity
 , termcolor
+, typing-extensions
 , unicodecsv
 , werkzeug
 , pytestCheckHook
 , freezegun
 , mkYarnPackage
+, writeScript
+
+# Extra airflow providers to enable
+, enabledProviders ? []
 }:
 let
-  version = "2.3.3";
+  version = "2.3.4";
 
   airflow-src = fetchFromGitHub rec {
     owner = "apache";
     repo = "airflow";
     rev = "refs/tags/${version}";
-    sha256 = "sha256-N+6ljfSo6+UvSAnvDav6G0S49JZ1VJwxmaiKPV3/DjA=";
+    # Required because the GitHub archive tarballs don't appear to include tests
+    leaveDotGit = true;
+    sha256 = "sha256-rxvLyz/hvZ6U8QKy9MiVofU0qeeo7OHctAj2PkxLh2c=";
   };
 
   # airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree.
@@ -87,6 +101,12 @@ let
 
     distPhase = "true";
 
+    # The webpack license plugin tries to create /licenses when given the
+    # original relative path
+    postPatch = ''
+      sed -i 's!../../../../licenses/LICENSES-ui.txt!licenses/LICENSES-ui.txt!' webpack.config.js
+    '';
+
     configurePhase = ''
       cp -r $node_modules node_modules
     '';
@@ -102,6 +122,13 @@ let
     '';
   };
 
+  # Import generated file with metadata for provider dependencies and imports.
+  # Enable additional providers using enabledProviders above.
+  providers = import ./providers.nix;
+  getProviderDeps = provider: map (dep: python.pkgs.${dep}) providers.${provider}.deps;
+  getProviderImports = provider: providers.${provider}.imports;
+  providerDependencies = lib.concatMap getProviderDeps enabledProviders;
+  providerImports = lib.concatMap getProviderImports enabledProviders;
 in
 buildPythonPackage rec {
   pname = "apache-airflow";
@@ -119,14 +146,18 @@ buildPythonPackage rec {
     cattrs
     clickclick
     colorlog
+    connexion
+    cron-descriptor
     croniter
     cryptography
+    deprecated
     dill
     flask
     flask-appbuilder
     flask-caching
-    flask_login
+    flask-session
     flask-wtf
+    flask_login
     GitPython
     graphviz
     gunicorn
@@ -138,13 +169,16 @@ buildPythonPackage rec {
     jinja2
     jsonschema
     lazy-object-proxy
+    linkify-it-py
     lockfile
     markdown
     markupsafe
     marshmallow-oneofschema
+    mdit-py-plugins
     numpy
     openapi-spec-validator
     pandas
+    pathspec
     pendulum
     psutil
     pygments
@@ -163,13 +197,14 @@ buildPythonPackage rec {
     tabulate
     tenacity
     termcolor
+    typing-extensions
     unicodecsv
     werkzeug
   ] ++ lib.optionals (pythonOlder "3.7") [
     dataclasses
   ] ++ lib.optionals (pythonOlder "3.9") [
     importlib-metadata
-  ];
+  ] ++ providerDependencies;
 
   buildInputs = [
     airflow-frontend
@@ -180,29 +215,15 @@ buildPythonPackage rec {
     pytestCheckHook
   ];
 
+  # By default, source code of providers is included but unusable due to missing
+  # transitive dependencies. To enable a provider, add it to extraProviders
+  # above
   INSTALL_PROVIDERS_FROM_SOURCES = "true";
 
   postPatch = ''
     substituteInPlace setup.cfg \
-      --replace "attrs>=20.0, <21.0" "attrs" \
+      --replace "colorlog>=4.0.2, <5.0" "colorlog" \
-      --replace "cattrs~=1.1, <1.7.0" "cattrs" \
+      --replace "flask-login>=0.6.2" "flask-login"
-      --replace "colorlog>=4.0.2, <6.0" "colorlog" \
-      --replace "croniter>=0.3.17, <1.1" "croniter" \
-      --replace "docutils<0.17" "docutils" \
-      --replace "flask-login>=0.3, <0.5" "flask-login" \
-      --replace "flask-wtf>=0.14.3, <0.15" "flask-wtf" \
-      --replace "flask>=1.1.0, <2.0" "flask" \
-      --replace "importlib_resources~=1.4" "importlib_resources" \
-      --replace "itsdangerous>=1.1.0, <2.0" "itsdangerous" \
-      --replace "markupsafe>=1.1.1, <2.0" "markupsafe" \
-      --replace "pyjwt<2" "pyjwt" \
-      --replace "python-slugify>=3.0.0,<5.0" "python-slugify" \
-      --replace "sqlalchemy_jsonfield~=1.0" "sqlalchemy-jsonfield" \
-      --replace "tenacity~=6.2.0" "tenacity" \
-      --replace "werkzeug~=1.0, >=1.0.1" "werkzeug"
-
-    substituteInPlace tests/core/test_core.py \
-      --replace "/bin/bash" "${stdenv.shell}"
   '' + lib.optionalString stdenv.isDarwin ''
     # Fix failing test on Hydra
     substituteInPlace airflow/utils/db.py \
@@ -214,7 +235,11 @@ buildPythonPackage rec {
     "--prefix PYTHONPATH : $PYTHONPATH"
   ];
 
-  preCheck = ''
+  pythonImportsCheck = [
+    "airflow"
+  ] ++ providerImports;
+
+  checkPhase = ''
     export HOME=$(mktemp -d)
     export AIRFLOW_HOME=$HOME
     export AIRFLOW__CORE__UNIT_TEST_MODE=True
@@ -238,12 +263,37 @@ buildPythonPackage rec {
     cp -rv ${airflow-frontend}/static/dist $out/lib/${python.libPrefix}/site-packages/airflow/www/static
   '';
 
+  # Updates yarn.lock and package.json
+  passthru.updateScript = writeScript "update.sh" ''
+    #!/usr/bin/env nix-shell
+    #!nix-shell -i bash -p common-updater-scripts curl pcre "python3.withPackages (ps: with ps; [ pyyaml ])" yarn2nix
+
+    set -euo pipefail
+
+    # Get new version
+    new_version="$(curl -s https://airflow.apache.org/docs/apache-airflow/stable/release_notes.html |
+      pcregrep -o1 'Airflow ([0-9.]+).' | head -1)"
+    update-source-version ${pname} "$new_version"
+
+    # Update frontend
+    cd ./pkgs/development/python-modules/apache-airflow
+    curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/yarn.lock
+    curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/package.json
+    # Note: for 2.3.4 a manual change was needed to get a fully resolved URL for
+    # caniuse-lite@1.0.30001312 (with the sha after the #). The error from yarn
+    # was 'Can't make a request in offline mode' from yarn. Corrected install by
+    # manually running yarn add caniuse-lite@1.0.30001312 and copying the
+    # requisite section from the generated yarn.lock.
+    yarn2nix > yarn.nix
+
+    # update provider dependencies
+    ./update-providers.py
+  '';
+
   meta = with lib; {
     description = "Programmatically author, schedule and monitor data pipelines";
     homepage = "https://airflow.apache.org/";
     license = licenses.asl20;
-    maintainers = with maintainers; [ bhipple costrouc ingenieroariel ];
+    maintainers = with maintainers; [ bhipple gbpdt ingenieroariel ];
-    # requires extremely outdated versions of multiple dependencies
-    broken = true;
   };
 }

pkgs/development/python-modules/apache-airflow/package.json

@@ -1,14 +1,14 @@
 {
-  "name": "airflow-frontend",
+  "name": "airflow-www",
-  "version": "2.1.1rc1",
+  "version": "1.0.0",
   "description": "Apache Airflow is a platform to programmatically author, schedule and monitor workflows.",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test": "jest",
-    "dev": "NODE_ENV=dev webpack --watch --colors --progress --debug --output-pathinfo --devtool eval-cheap-source-map --mode development",
+    "dev": "NODE_ENV=development webpack --watch --progress --devtool eval-cheap-source-map --mode development",
-    "prod": "NODE_ENV=production node --max_old_space_size=4096 ./node_modules/webpack/bin/webpack.js -p --colors --progress",
+    "prod": "NODE_ENV=production node --max_old_space_size=4096 ./node_modules/webpack/bin/webpack.js --mode production --progress",
-    "build": "NODE_ENV=production webpack --colors --progress",
+    "build": "NODE_ENV=production webpack --progress --mode production",
-    "lint": "eslint --ignore-path=.eslintignore --ext .js,.html .",
+    "lint": "eslint --ignore-path=.eslintignore --ext .js,.jsx,.ts,.tsx . && tsc --noEmit",
-    "lint:fix": "eslint --fix --ignore-path=.eslintignore --ext .js,.html ."
+    "lint:fix": "eslint --fix --ignore-path=.eslintignore --ext .js,.jsx,.ts,.tsx . && tsc --noEmit"
   },
   "author": "Apache",
   "license": "Apache-2.0",
@@ -28,53 +28,82 @@
     "flask"
   ],
   "devDependencies": {
-    "babel": "^6.23.0",
+    "@babel/core": "^7.18.5",
-    "babel-core": "^6.26.3",
+    "@babel/eslint-parser": "^7.18.2",
-    "babel-eslint": "^10.1.0",
+    "@babel/plugin-transform-runtime": "^7.16.0",
+    "@babel/preset-env": "^7.16.0",
+    "@babel/preset-react": "^7.16.0",
+    "@babel/preset-typescript": "^7.17.12",
+    "@testing-library/jest-dom": "^5.16.0",
+    "@testing-library/react": "^13.0.0",
+    "@types/react": "^18.0.12",
+    "@types/react-dom": "^18.0.5",
+    "@typescript-eslint/eslint-plugin": "^5.13.0",
+    "@typescript-eslint/parser": "^5.0.0",
+    "babel-jest": "^27.3.1",
     "babel-loader": "^8.1.0",
-    "babel-plugin-css-modules-transform": "^1.6.1",
-    "babel-polyfill": "^6.26.0",
     "clean-webpack-plugin": "^3.0.0",
     "copy-webpack-plugin": "^6.0.3",
-    "css-loader": "^3.4.2",
+    "css-loader": "5.2.7",
-    "eslint": "^7.5.0",
+    "css-minimizer-webpack-plugin": "^4.0.0",
-    "eslint-config-airbnb-base": "^14.2.0",
+    "eslint": "^8.6.0",
+    "eslint-config-airbnb": "^19.0.4",
+    "eslint-config-airbnb-typescript": "^17.0.0",
     "eslint-plugin-html": "^6.0.2",
-    "eslint-plugin-import": "^2.22.0",
+    "eslint-plugin-import": "^2.25.3",
+    "eslint-plugin-jsx-a11y": "^6.5.0",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-promise": "^4.2.1",
+    "eslint-plugin-react": "^7.30.0",
+    "eslint-plugin-react-hooks": "^4.5.0",
     "eslint-plugin-standard": "^4.0.1",
     "file-loader": "^6.0.0",
     "imports-loader": "^1.1.0",
-    "mini-css-extract-plugin": "1.6.0",
+    "jest": "^27.3.1",
+    "mini-css-extract-plugin": "^1.6.2",
+    "moment": "^2.29.3",
     "moment-locales-webpack-plugin": "^1.2.0",
-    "optimize-css-assets-webpack-plugin": "6.0.0",
+    "nock": "^13.2.4",
     "style-loader": "^1.2.1",
     "stylelint": "^13.6.1",
     "stylelint-config-standard": "^20.0.0",
+    "terser-webpack-plugin": "<5.0.0",
+    "typescript": "^4.6.3",
     "url-loader": "4.1.0",
-    "webpack": "^4.16.3",
+    "webpack": "^5.73.0",
-    "webpack-cli": "^3.1.0",
+    "webpack-cli": "^4.0.0",
-    "webpack-manifest-plugin": "^2.2.0"
+    "webpack-license-plugin": "^4.2.1",
+    "webpack-manifest-plugin": "^4.0.0"
   },
   "dependencies": {
+    "@chakra-ui/react": "^2.2.0",
+    "@emotion/cache": "^11.9.3",
+    "@emotion/react": "^11.9.3",
+    "@emotion/styled": "^11",
+    "axios": "^0.26.0",
     "bootstrap-3-typeahead": "^4.0.2",
+    "camelcase-keys": "^7.0.0",
     "codemirror": "^5.59.1",
     "d3": "^3.4.4",
     "d3-shape": "^2.1.0",
     "d3-tip": "^0.9.1",
     "dagre-d3": "^0.6.4",
-    "datatables.net": "^1.10.23",
+    "datatables.net": "^1.11.4",
-    "datatables.net-bs": "^1.10.23",
+    "datatables.net-bs": "^1.11.4",
     "eonasdan-bootstrap-datetimepicker": "^4.17.47",
-    "jquery": ">=3.4.0",
+    "framer-motion": "^6.0.0",
-    "jshint": "^2.12.0",
+    "jquery": ">=3.5.0",
-    "moment-timezone": "^0.5.28",
+    "jshint": "^2.13.4",
+    "lodash": "^4.17.21",
+    "moment-timezone": "^0.5.34",
     "nvd3": "^1.8.6",
-    "redoc": "^2.0.0-rc.48",
+    "react": "^18.0.0",
+    "react-dom": "^18.0.0",
+    "react-icons": "^4.3.1",
+    "react-query": "^3.39.1",
+    "react-router-dom": "^6.3.0",
+    "react-table": "^7.8.0",
+    "redoc": "^2.0.0-rc.72",
     "url-search-params-polyfill": "^8.1.0"
-  },
-  "resolutions": {
-    "lodash": "^4.17.21"
   }
 }

pkgs/development/python-modules/apache-airflow/update-providers.py

@@ -0,0 +1,226 @@
+#! /usr/bin/env python3
+
+from itertools import chain
+import json
+import logging
+from pathlib import Path
+import os
+import re
+import subprocess
+import sys
+from typing import Dict, List, Optional, Set, TextIO
+from urllib.request import urlopen
+from urllib.error import HTTPError
+import yaml
+
+PKG_SET = "pkgs.python3Packages"
+
+# If some requirements are matched by multiple or no Python packages, the
+# following can be used to choose the correct one
+PKG_PREFERENCES = {
+    "dnspython": "dnspython",
+    "google-api-python-client": "google-api-python-client",
+    "psycopg2-binary": "psycopg2",
+    "requests_toolbelt": "requests-toolbelt",
+}
+
+# Requirements missing from the airflow provider metadata
+EXTRA_REQS = {
+    "sftp": ["pysftp"],
+}
+
+
+def get_version():
+    with open(os.path.dirname(sys.argv[0]) + "/default.nix") as fh:
+        # A version consists of digits, dots, and possibly a "b" (for beta)
+        m = re.search('version = "([\\d\\.b]+)";', fh.read())
+        return m.group(1)
+
+
+def get_file_from_github(version: str, path: str):
+    with urlopen(
+        f"https://raw.githubusercontent.com/apache/airflow/{version}/{path}"
+    ) as response:
+        return yaml.safe_load(response)
+
+
+def repository_root() -> Path:
+    return Path(os.path.dirname(sys.argv[0])) / "../../../.."
+
+
+def dump_packages() -> Dict[str, Dict[str, str]]:
+    # Store a JSON dump of Nixpkgs' python3Packages
+    output = subprocess.check_output(
+        [
+            "nix-env",
+            "-f",
+            repository_root(),
+            "-qa",
+            "-A",
+            PKG_SET,
+            "--arg",
+            "config",
+            "{ allowAliases = false; }",
+            "--json",
+        ]
+    )
+    return json.loads(output)
+
+
+def remove_version_constraint(req: str) -> str:
+    return re.sub(r"[=><~].*$", "", req)
+
+
+def name_to_attr_path(req: str, packages: Dict[str, Dict[str, str]]) -> Optional[str]:
+    if req in PKG_PREFERENCES:
+        return f"{PKG_SET}.{PKG_PREFERENCES[req]}"
+    attr_paths = []
+    names = [req]
+    # E.g. python-mpd2 is actually called python3.6-mpd2
+    # instead of python-3.6-python-mpd2 inside Nixpkgs
+    if req.startswith("python-") or req.startswith("python_"):
+        names.append(req[len("python-") :])
+    for name in names:
+        # treat "-" and "_" equally
+        name = re.sub("[-_]", "[-_]", name)
+        # python(minor).(major)-(pname)-(version or unstable-date)
+        # we need the version qualifier, or we'll have multiple matches
+        # (e.g. pyserial and pyserial-asyncio when looking for pyserial)
+        pattern = re.compile(
+            f"^python\\d+\\.\\d+-{name}-(?:\\d|unstable-.*)", re.I
+        )
+        for attr_path, package in packages.items():
+            # logging.debug("Checking match for %s with %s", name, package["name"])
+            if pattern.match(package["name"]):
+                attr_paths.append(attr_path)
+    # Let's hope there's only one derivation with a matching name
+    assert len(attr_paths) <= 1, f"{req} matches more than one derivation: {attr_paths}"
+    if attr_paths:
+        return attr_paths[0]
+    return None
+
+
+def provider_reqs_to_attr_paths(reqs: List, packages: Dict) -> List:
+    no_version_reqs = map(remove_version_constraint, reqs)
+    filtered_reqs = [
+        req for req in no_version_reqs if not re.match(r"^apache-airflow", req)
+    ]
+    attr_paths = []
+    for req in filtered_reqs:
+        attr_path = name_to_attr_path(req, packages)
+        if attr_path is not None:
+            # Add attribute path without "python3Packages." prefix
+            pname = attr_path[len(PKG_SET + ".") :]
+            attr_paths.append(pname)
+        else:
+            # If we can't find it, we just skip and warn the user
+            logging.warning("Could not find package attr for %s", req)
+    return attr_paths
+
+
+def get_cross_provider_reqs(
+    provider: str, provider_reqs: Dict, cross_provider_deps: Dict, seen: List = None
+) -> Set:
+    # Unfortunately there are circular cross-provider dependencies, so keep a
+    # list of ones we've seen already
+    seen = seen or []
+    reqs = set(provider_reqs[provider])
+    if len(cross_provider_deps[provider]) > 0:
+        reqs.update(
+            chain.from_iterable(
+                get_cross_provider_reqs(
+                    d, provider_reqs, cross_provider_deps, seen + [provider]
+                )
+                if d not in seen
+                else []
+                for d in cross_provider_deps[provider]
+            )
+        )
+    return reqs
+
+
+def get_provider_reqs(version: str, packages: Dict) -> Dict:
+    provider_dependencies = get_file_from_github(
+        version, "generated/provider_dependencies.json"
+    )
+    provider_reqs = {}
+    cross_provider_deps = {}
+    for provider, provider_data in provider_dependencies.items():
+        provider_reqs[provider] = list(
+            provider_reqs_to_attr_paths(provider_data["deps"], packages)
+        ) + EXTRA_REQS.get(provider, [])
+        cross_provider_deps[provider] = [
+            d for d in provider_data["cross-providers-deps"] if d != "common.sql"
+        ]
+    transitive_provider_reqs = {}
+    # Add transitive cross-provider reqs
+    for provider in provider_reqs:
+        transitive_provider_reqs[provider] = get_cross_provider_reqs(
+            provider, provider_reqs, cross_provider_deps
+        )
+    return transitive_provider_reqs
+
+
+def get_provider_yaml(version: str, provider: str) -> Dict:
+    provider_dir = provider.replace(".", "/")
+    path = f"airflow/providers/{provider_dir}/provider.yaml"
+    try:
+        return get_file_from_github(version, path)
+    except HTTPError:
+        logging.warning("Couldn't get provider yaml for %s", provider)
+        return {}
+
+
+def get_provider_imports(version: str, providers) -> Dict:
+    provider_imports = {}
+    for provider in providers:
+        provider_yaml = get_provider_yaml(version, provider)
+        imports: List[str] = []
+        if "hooks" in provider_yaml:
+            imports.extend(
+                chain.from_iterable(
+                    hook["python-modules"] for hook in provider_yaml["hooks"]
+                )
+            )
+        if "operators" in provider_yaml:
+            imports.extend(
+                chain.from_iterable(
+                    operator["python-modules"]
+                    for operator in provider_yaml["operators"]
+                )
+            )
+        provider_imports[provider] = imports
+    return provider_imports
+
+
+def to_nix_expr(provider_reqs: Dict, provider_imports: Dict, fh: TextIO) -> None:
+    fh.write("# Warning: generated by update-providers.py, do not update manually\n")
+    fh.write("{\n")
+    for provider, reqs in provider_reqs.items():
+        provider_name = provider.replace(".", "_")
+        fh.write(f"  {provider_name} = {{\n")
+        fh.write(
+            "    deps = [ " + " ".join(sorted(f'"{req}"' for req in reqs)) + " ];\n"
+        )
+        fh.write(
+            "    imports = [ "
+            + " ".join(sorted(f'"{imp}"' for imp in provider_imports[provider]))
+            + " ];\n"
+        )
+        fh.write("  };\n")
+    fh.write("}\n")
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO)
+    version = get_version()
+    packages = dump_packages()
+    logging.info("Generating providers.nix for version %s", version)
+    provider_reqs = get_provider_reqs(version, packages)
+    provider_imports = get_provider_imports(version, provider_reqs.keys())
+    with open("providers.nix", "w") as fh:
+        to_nix_expr(provider_reqs, provider_imports, fh)
+
+
+if __name__ == "__main__":
+    main()