nordic-dev.net/nixpkgs
nordic-dev.net/nixpkgs
2
0
Fork 0
mirror of https://github.com/NixOS/nixpkgs.git synced 2024-11-23 23:43:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #248737 from figsoda/pluginupdate

Browse Source 
pluginupdate: don't rely on NIX_PATH
This commit is contained in:
Nick Cao 2023-08-13 00:36:11 -06:00 committed by GitHub
commit 8f9ad8b8ec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 106 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

183
maintainers/scripts/pluginupdate.py
View File

@ -4,20 +4,20 @@
# - maintainers/scripts/update-luarocks-packages
# format:
# $ nix run nixpkgs.python3Packages.black -c black update.py
# $ nix run nixpkgs#black maintainers/scripts/pluginupdate.py
# type-check:
# $ nix run nixpkgs.python3Packages.mypy -c mypy update.py
# $ nix run nixpkgs#python3.pkgs.mypy maintainers/scripts/pluginupdate.py
# linted:
# $ nix run nixpkgs.python3Packages.flake8 -c flake8 --ignore E501,E265 update.py
# $ nix run nixpkgs#python3.pkgs.flake8 -- --ignore E501,E265 maintainers/scripts/pluginupdate.py
import argparse
import csv
import functools
import http
import json
import logging
import os
import subprocess
import logging
import sys
import time
import traceback
@ -25,14 +25,14 @@ import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from dataclasses import asdict, dataclass
from datetime import datetime
from functools import wraps
from multiprocessing.dummy import Pool
from pathlib import Path
from typing import Dict, List, Optional, Tuple, Union, Any, Callable
from urllib.parse import urljoin, urlparse
from tempfile import NamedTemporaryFile
from dataclasses import dataclass, asdict
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
from urllib.parse import urljoin, urlparse
import git
@ -41,12 +41,13 @@ ATOM_LINK = "{http://www.w3.org/2005/Atom}link" # "
ATOM_UPDATED = "{http://www.w3.org/2005/Atom}updated" # "
LOG_LEVELS = {
logging.getLevelName(level): level for level in [
logging.DEBUG, logging.INFO, logging.WARN, logging.ERROR ]
logging.getLevelName(level): level
for level in [logging.DEBUG, logging.INFO, logging.WARN, logging.ERROR]
}
log = logging.getLogger()
def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: float = 2):
"""Retry calling the decorated function using an exponential backoff.
http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
@ -77,6 +78,7 @@ def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: floa
return deco_retry
@dataclass
class FetchConfig:
proc: int
@ -91,22 +93,21 @@ def make_request(url: str, token=None) -> urllib.request.Request:
# a dictionary of plugins and their new repositories
Redirects = Dict['PluginDesc', 'Repo']
Redirects = Dict["PluginDesc", "Repo"]
class Repo:
def __init__(
self, uri: str, branch: str
) -> None:
def __init__(self, uri: str, branch: str) -> None:
self.uri = uri
'''Url to the repo'''
"""Url to the repo"""
self._branch = branch
# Redirect is the new Repo to use
self.redirect: Optional['Repo'] = None
self.redirect: Optional["Repo"] = None
self.token = "dummy_token"
@property
def name(self):
return self.uri.split('/')[-1]
return self.uri.split("/")[-1]
@property
def branch(self):
@ -114,6 +115,7 @@ class Repo:
def __str__(self) -> str:
return f"{self.uri}"
def __repr__(self) -> str:
return f"Repo({self.name}, {self.uri})"
@ -125,9 +127,9 @@ class Repo:
def latest_commit(self) -> Tuple[str, datetime]:
log.debug("Latest commit")
loaded = self._prefetch(None)
updated = datetime.strptime(loaded['date'], "%Y-%m-%dT%H:%M:%S%z")
updated = datetime.strptime(loaded["date"], "%Y-%m-%dT%H:%M:%S%z")
return loaded['rev'], updated
return loaded["rev"], updated
def _prefetch(self, ref: Optional[str]):
cmd = ["nix-prefetch-git", "--quiet", "--fetch-submodules", self.uri]
@ -144,23 +146,23 @@ class Repo:
return loaded["sha256"]
def as_nix(self, plugin: "Plugin") -> str:
return f'''fetchgit {{
return f"""fetchgit {{
url = "{self.uri}";
rev = "{plugin.commit}";
sha256 = "{plugin.sha256}";
}}'''
}}"""
class RepoGitHub(Repo):
def __init__(
self, owner: str, repo: str, branch: str
) -> None:
def __init__(self, owner: str, repo: str, branch: str) -> None:
self.owner = owner
self.repo = repo
self.token = None
'''Url to the repo'''
"""Url to the repo"""
super().__init__(self.url(""), branch)
log.debug("Instantiating github repo owner=%s and repo=%s", self.owner, self.repo)
log.debug(
"Instantiating github repo owner=%s and repo=%s", self.owner, self.repo
)
@property
def name(self):
@ -213,7 +215,6 @@ class RepoGitHub(Repo):
new_repo = RepoGitHub(owner=new_owner, repo=new_name, branch=self.branch)
self.redirect = new_repo
def prefetch(self, commit: str) -> str:
if self.has_submodules():
sha256 = super().prefetch(commit)
@ -233,12 +234,12 @@ class RepoGitHub(Repo):
else:
submodule_attr = ""
return f'''fetchFromGitHub {{
return f"""fetchFromGitHub {{
owner = "{self.owner}";
repo = "{self.repo}";
rev = "{plugin.commit}";
sha256 = "{plugin.sha256}";{submodule_attr}
}}'''
}}"""
@dataclass(frozen=True)
@ -258,15 +259,14 @@ class PluginDesc:
return self.repo.name < other.repo.name
@staticmethod
def load_from_csv(config: FetchConfig, row: Dict[str, str]) -> 'PluginDesc':
def load_from_csv(config: FetchConfig, row: Dict[str, str]) -> "PluginDesc":
branch = row["branch"]
repo = make_repo(row['repo'], branch.strip())
repo = make_repo(row["repo"], branch.strip())
repo.token = config.github_token
return PluginDesc(repo, branch.strip(), row["alias"])
@staticmethod
def load_from_string(config: FetchConfig, line: str) -> 'PluginDesc':
def load_from_string(config: FetchConfig, line: str) -> "PluginDesc":
branch = "HEAD"
alias = None
uri = line
@ -279,6 +279,7 @@ class PluginDesc:
repo.token = config.github_token
return PluginDesc(repo, branch.strip(), alias)
@dataclass
class Plugin:
name: str
@ -302,22 +303,38 @@ class Plugin:
return copy
def load_plugins_from_csv(config: FetchConfig, input_file: Path,) -> List[PluginDesc]:
def load_plugins_from_csv(
config: FetchConfig,
input_file: Path,
) -> List[PluginDesc]:
log.debug("Load plugins from csv %s", input_file)
plugins = []
with open(input_file, newline='') as csvfile:
with open(input_file, newline="") as csvfile:
log.debug("Writing into %s", input_file)
reader = csv.DictReader(csvfile,)
reader = csv.DictReader(
csvfile,
)
for line in reader:
plugin = PluginDesc.load_from_csv(config, line)
plugins.append(plugin)
return plugins
def run_nix_expr(expr):
with CleanEnvironment():
cmd = ["nix", "eval", "--extra-experimental-features",
"nix-command", "--impure", "--json", "--expr", expr]
with CleanEnvironment() as nix_path:
cmd = [
"nix",
"eval",
"--extra-experimental-features",
"nix-command",
"--impure",
"--json",
"--expr",
expr,
"--nix-path",
nix_path,
]
log.debug("Running command %s", " ".join(cmd))
out = subprocess.check_output(cmd)
data = json.loads(out)
@ -348,7 +365,7 @@ class Editor:
self.nixpkgs_repo = None
def add(self, args):
'''CSV spec'''
"""CSV spec"""
log.debug("called the 'add' command")
fetch_config = FetchConfig(args.proc, args.github_token)
editor = self
@ -356,23 +373,27 @@ class Editor:
log.debug("using plugin_line", plugin_line)
pdesc = PluginDesc.load_from_string(fetch_config, plugin_line)
log.debug("loaded as pdesc", pdesc)
append = [ pdesc ]
editor.rewrite_input(fetch_config, args.input_file, editor.deprecated, append=append)
plugin, _ = prefetch_plugin(pdesc, )
append = [pdesc]
editor.rewrite_input(
fetch_config, args.input_file, editor.deprecated, append=append
)
plugin, _ = prefetch_plugin(
pdesc,
)
autocommit = not args.no_commit
if autocommit:
commit(
editor.nixpkgs_repo,
"{drv_name}: init at {version}".format(
drv_name=editor.get_drv_name(plugin.normalized_name),
version=plugin.version
version=plugin.version,
),
[args.outfile, args.input_file],
)
# Expects arguments generated by 'update' subparser
def update(self, args ):
'''CSV spec'''
def update(self, args):
"""CSV spec"""
print("the update member function should be overriden in subclasses")
def get_current_plugins(self) -> List[Plugin]:
@ -385,11 +406,11 @@ class Editor:
return plugins
def load_plugin_spec(self, config: FetchConfig, plugin_file) -> List[PluginDesc]:
'''CSV spec'''
"""CSV spec"""
return load_plugins_from_csv(config, plugin_file)
def generate_nix(self, _plugins, _outfile: str):
'''Returns nothing for now, writes directly to outfile'''
"""Returns nothing for now, writes directly to outfile"""
raise NotImplementedError()
def get_update(self, input_file: str, outfile: str, config: FetchConfig):
@ -413,7 +434,6 @@ class Editor:
return update
@property
def attr_path(self):
return self.name + "Plugins"
@ -427,10 +447,11 @@ class Editor:
def create_parser(self):
common = argparse.ArgumentParser(
add_help=False,
description=(f"""
description=(
f"""
Updates nix derivations for {self.name} plugins.\n
By default from {self.default_in} to {self.default_out}"""
)
),
)
common.add_argument(
"--input-names",
@ -463,26 +484,33 @@ class Editor:
Uses GITHUB_API_TOKEN environment variables as the default value.""",
)
common.add_argument(
"--no-commit", "-n", action="store_true", default=False,
help="Whether to autocommit changes"
"--no-commit",
"-n",
action="store_true",
default=False,
help="Whether to autocommit changes",
)
common.add_argument(
"--debug", "-d", choices=LOG_LEVELS.keys(),
"--debug",
"-d",
choices=LOG_LEVELS.keys(),
default=logging.getLevelName(logging.WARN),
help="Adjust log level"
help="Adjust log level",
)
main = argparse.ArgumentParser(
parents=[common],
description=(f"""
description=(
f"""
Updates nix derivations for {self.name} plugins.\n
By default from {self.default_in} to {self.default_out}"""
)
),
)
subparsers = main.add_subparsers(dest="command", required=False)
padd = subparsers.add_parser(
"add", parents=[],
"add",
parents=[],
description="Add new plugin",
add_help=False,
)
@ -502,10 +530,12 @@ class Editor:
pupdate.set_defaults(func=self.update)
return main
def run(self,):
'''
def run(
self,
):
"""
Convenience function
'''
"""
parser = self.create_parser()
args = parser.parse_args()
command = args.command or "update"
@ -518,17 +548,15 @@ class Editor:
getattr(self, command)(args)
class CleanEnvironment(object):
def __enter__(self) -> None:
def __enter__(self) -> str:
self.old_environ = os.environ.copy()
local_pkgs = str(Path(__file__).parent.parent.parent)
os.environ["NIX_PATH"] = f"localpkgs={local_pkgs}"
self.empty_config = NamedTemporaryFile()
self.empty_config.write(b"{}")
self.empty_config.flush()
os.environ["NIXPKGS_CONFIG"] = self.empty_config.name
return f"localpkgs={local_pkgs}"
def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
os.environ.update(self.old_environ)
@ -570,14 +598,15 @@ def print_download_error(plugin: PluginDesc, ex: Exception):
]
print("\n".join(tb_lines))
def check_results(
results: List[Tuple[PluginDesc, Union[Exception, Plugin], Optional[Repo]]]
) -> Tuple[List[Tuple[PluginDesc, Plugin]], Redirects]:
''' '''
""" """
failures: List[Tuple[PluginDesc, Exception]] = []
plugins = []
redirects: Redirects = {}
for (pdesc, result, redirect) in results:
for pdesc, result, redirect in results:
if isinstance(result, Exception):
failures.append((pdesc, result))
else:
@ -594,17 +623,18 @@ def check_results(
else:
print(f", {len(failures)} plugin(s) could not be downloaded:\n")
for (plugin, exception) in failures:
for plugin, exception in failures:
print_download_error(plugin, exception)
sys.exit(1)
def make_repo(uri: str, branch) -> Repo:
'''Instantiate a Repo with the correct specialization depending on server (gitub spec)'''
"""Instantiate a Repo with the correct specialization depending on server (gitub spec)"""
# dumb check to see if it's of the form owner/repo (=> github) or https://...
res = urlparse(uri)
if res.netloc in [ "github.com", ""]:
res = res.path.strip('/').split('/')
if res.netloc in ["github.com", ""]:
res = res.path.strip("/").split("/")
repo = RepoGitHub(res[0], res[1], branch)
else:
repo = Repo(uri.strip(), branch)
@ -675,7 +705,6 @@ def prefetch(
return (pluginDesc, e, None)
def rewrite_input(
config: FetchConfig,
input_file: Path,
@ -684,12 +713,14 @@ def rewrite_input(
redirects: Redirects = {},
append: List[PluginDesc] = [],
):
plugins = load_plugins_from_csv(config, input_file,)
plugins = load_plugins_from_csv(
config,
input_file,
)
plugins.extend(append)
if redirects:
cur_date_iso = datetime.now().strftime("%Y-%m-%d")
with open(deprecated, "r") as f:
deprecations = json.load(f)
@ -709,8 +740,8 @@ def rewrite_input(
with open(input_file, "w") as f:
log.debug("Writing into %s", input_file)
# fields = dataclasses.fields(PluginDesc)
fieldnames = ['repo', 'branch', 'alias']
writer = csv.DictWriter(f, fieldnames, dialect='unix', quoting=csv.QUOTE_NONE)
fieldnames = ["repo", "branch", "alias"]
writer = csv.DictWriter(f, fieldnames, dialect="unix", quoting=csv.QUOTE_NONE)
writer.writeheader()
for plugin in sorted(plugins):
writer.writerow(asdict(plugin))
@ -726,7 +757,6 @@ def commit(repo: git.Repo, message: str, files: List[Path]) -> None:
print("no changes in working tree to commit")
def update_plugins(editor: Editor, args):
"""The main entry function of this module. All input arguments are grouped in the `Editor`."""
@ -751,4 +781,3 @@ def update_plugins(editor: Editor, args):
f"{editor.attr_path}: resolve github repository redirects",
[args.outfile, args.input_file, editor.deprecated],
)