gitlab: add update.py script

pkgs/applications/version-management/gitlab/update.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env nix-shell
+#! nix-shell -i python3 -p bundix common-updater-scripts nix nix-prefetch-git python3 python3Packages.requests python3Packages.lxml python3Packages.click python3Packages.click-log
+
+import click
+import click_log
+import os
+import re
+import logging
+import subprocess
+import json
+import pathlib
+from typing import Iterable
+
+import requests
+from xml.etree import ElementTree
+
+logger = logging.getLogger(__name__)
+
+
+class GitLabRepo:
+    def __init__(self, owner: str, repo: str):
+        self.owner = owner
+        self.repo = repo
+
+    @property
+    def url(self):
+        return f"https://gitlab.com/{self.owner}/{self.repo}"
+
+    @property
+    def tags(self) -> Iterable[str]:
+        r = requests.get(self.url + "/tags?format=atom", stream=True)
+
+        tree = ElementTree.fromstring(r.content)
+        return sorted((e.text for e in tree.findall(
+            '{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}title')), reverse=True)
+
+    def get_git_hash(self, rev: str):
+        out = subprocess.check_output(['nix-prefetch-git', self.url, rev])
+        j = json.loads(out)
+        return j['sha256']
+
+    def get_deb_url(self, flavour: str, version: str, arch: str = 'amd64') -> str:
+        """
+        gitlab builds debian packages, which we currently need as we don't build the frontend on our own
+        this returns the url of a given flavour, version and arch
+        :param flavour: 'ce' or 'ee'
+        :param version: a version, without 'v' prefix and '-ee' suffix
+        :param arch: amd64
+        :return: url of the debian package
+        """
+        if self.owner != "gitlab-org" or self.repo not in ['gitlab-ce', 'gitlab-ee']:
+            raise Exception(f"don't know how to get deb_url for {self.url}")
+        return f"https://packages.gitlab.com/gitlab/gitlab-{flavour}/packages" + \
+               f"/debian/stretch/gitlab-{flavour}_{version}-{flavour}.0_{arch}.deb/download.deb"
+
+    def get_deb_hash(self, flavour: str, version: str) -> str:
+        out = subprocess.check_output(['nix-prefetch-url', self.get_deb_url(flavour, version)])
+        return out.decode('utf-8').strip()
+
+    @staticmethod
+    def rev2version(tag: str) -> str:
+        """
+        normalize a tag to a version number.
+        This obviously isn't very smart if we don't pass something that looks like a tag
+        :param tag: the tag to normalize
+        :return: a normalized version number
+        """
+        # strip v prefix
+        version = re.sub(r"^v", '', tag)
+        # strip -ee suffix
+        return re.sub(r"-ee$", '', version)
+
+    def get_file(self, filepath, rev):
+        """
+        returns file contents at a given rev
+        :param filepath: the path to the file, relative to the repo root
+        :param rev: the rev to fetch at
+        :return:
+        """
+        return requests.get(self.url + f"/raw/{rev}/{filepath}").text
+
+    def get_data(self, rev, flavour):
+        version = self.rev2version(rev)
+
+        passthru = {v: self.get_file(v, rev).strip() for v in ['GITALY_SERVER_VERSION', 'GITLAB_PAGES_VERSION',
+                                                               'GITLAB_SHELL_VERSION', 'GITLAB_WORKHORSE_VERSION']}
+        return dict(version=self.rev2version(rev),
+                    repo_hash=self.get_git_hash(rev),
+                    deb_hash=self.get_deb_hash(flavour, version),
+                    deb_url=self.get_deb_url(flavour, version),
+                    owner=self.owner,
+                    repo=self.repo,
+                    rev=rev,
+                    passthru=passthru)
+
+
+def _flavour2gitlabrepo(flavour: str):
+    if flavour not in ['ce', 'ee']:
+        raise Exception(f"unknown gitlab flavour: {flavour}, needs to be ce or ee")
+
+    owner = 'gitlab-org'
+    repo = 'gitlab-' + flavour
+
+    return GitLabRepo(owner, repo)
+
+
+def _update_data_json(filename: str, repo: GitLabRepo, rev: str, flavour: str):
+    flavour_data = repo.get_data(rev, flavour)
+
+    if not os.path.exists(filename):
+        with open(filename, 'w') as f:
+            json.dump({flavour: flavour_data}, f, indent=2)
+    else:
+        with open(filename, 'r+') as f:
+            data = json.load(f)
+            data[flavour] = flavour_data
+            f.seek(0)
+            json.dump(data, f, indent=2)
+
+
+def _get_data_json():
+    data_file_path = pathlib.Path(__file__).parent / 'data.json'
+    with open(data_file_path, 'r') as f:
+        return json.load(f)
+
+
+def _call_update_source_version(pkg, version):
+    """calls update-source-version from nixpkgs root dir"""
+    nixpkgs_path = pathlib.Path(__file__).parent / '../../../../'
+    return subprocess.check_output(['update-source-version', pkg, version], cwd=nixpkgs_path)
+
+
+@click_log.simple_verbosity_option(logger)
+@click.group()
+def cli():
+    pass
+
+
+@cli.command('update-data')
+@click.option('--rev', default='latest', help='The rev to use, \'latest\' points to the latest (stable) tag')
+@click.argument('flavour')
+def update_data(rev: str, flavour: str):
+    """Update data.nix for a selected flavour"""
+    r = _flavour2gitlabrepo(flavour)
+
+    if rev == 'latest':
+        # filter out pre and re releases
+        rev = next(filter(lambda x: not ('rc' in x or x.endswith('pre')), r.tags))
+    logger.debug(f"Using rev {rev}")
+
+    version = r.rev2version(rev)
+    logger.debug(f"Using version {version}")
+
+    data_file_path = pathlib.Path(__file__).parent / 'data.json'
+
+    _update_data_json(filename=data_file_path.as_posix(),
+                      repo=r,
+                      rev=rev,
+                      flavour=flavour)
+
+
+@cli.command('update-rubyenv')
+@click.argument('flavour')
+def update_rubyenv(flavour):
+    """Update rubyEnv-${flavour}"""
+    if flavour not in ['ce', 'ee']:
+        raise Exception(f"unknown gitlab flavour: {flavour}, needs to be ce or ee")
+
+    r = _flavour2gitlabrepo(flavour)
+    rubyenv_dir = pathlib.Path(__file__).parent / f"rubyEnv-{flavour}"
+
+    # load rev from data.json
+    data = _get_data_json()
+    rev = data[flavour]['rev']
+
+    for fn in ['Gemfile.lock', 'Gemfile']:
+        with open(rubyenv_dir / fn, 'w') as f:
+            f.write(r.get_file(fn, rev))
+
+    subprocess.check_output(['bundix'], cwd=rubyenv_dir)
+
+
+@cli.command('update-gitaly')
+def update_gitaly():
+    """Update gitaly"""
+    data = _get_data_json()
+    gitaly_server_version = data['ce']['passthru']['GITALY_SERVER_VERSION']
+    r = GitLabRepo('gitlab-org', 'gitaly')
+    rubyenv_dir = pathlib.Path(__file__).parent / 'gitaly'
+
+    for fn in ['Gemfile.lock', 'Gemfile']:
+        with open(rubyenv_dir / fn, 'w') as f:
+            f.write(r.get_file(f"ruby/{fn}", f"v{gitaly_server_version}"))
+
+    subprocess.check_output(['bundix'], cwd=rubyenv_dir)
+    _call_update_source_version('gitaly', gitaly_server_version)
+
+
+@cli.command('update-gitlab-shell')
+def update_gitlab_shell():
+    """Update gitlab-shell"""
+    data = _get_data_json()
+    gitlab_shell_version = data['ce']['passthru']['GITLAB_SHELL_VERSION']
+    _call_update_source_version('gitlab-shell', gitlab_shell_version)
+
+
+@cli.command('update-gitlab-workhorse')
+def update_gitlab_workhorse():
+    """Update gitlab-shell"""
+    data = _get_data_json()
+    gitlab_workhorse_version = data['ce']['passthru']['GITLAB_WORKHORSE_VERSION']
+    _call_update_source_version('gitlab-workhorse', gitlab_workhorse_version)
+
+
+@cli.command('update-all')
+@click.pass_context
+def update_all(ctx):
+    """Update gitlab ce and ee data.nix and rubyenvs to the latest stable release"""
+    for flavour in ['ce', 'ee']:
+        ctx.invoke(update_data, rev='latest', flavour=flavour)
+        ctx.invoke(update_rubyenv, flavour=flavour)
+    ctx.invoke(update_gitaly)
+    ctx.invoke(update_gitlab_shell)
+    ctx.invoke(update_gitlab_workhorse)
+
+
+if __name__ == '__main__':
+    cli()