From 6c5d199d57ee937dc8f4f036c38aa594751250ce Mon Sep 17 00:00:00 2001
From: flip1995 <philipp.krones@embecosm.com>
Date: Mon, 31 May 2021 17:46:20 +0200
Subject: [PATCH 1/3] Update deploy CI
This updates all the deploy scripts and the deploy workflow.
The deploy workflow now runs the metadata collector to collect the lint
documentation. It also changes the files that are checked out in the
deploy workflow from master and adds an explanation why we have to do
this.
---
.github/deploy.sh | 9 ++++-----
.github/workflows/deploy.yml | 17 +++++++++++++++--
.../utils/internal_lints/metadata_collector.rs | 2 +-
tests/dogfood.rs | 2 +-
4 files changed, 21 insertions(+), 9 deletions(-)
diff --git a/.github/deploy.sh b/.github/deploy.sh
index e85e8874ba6..a3c57232f55 100644
--- a/.github/deploy.sh
+++ b/.github/deploy.sh
@@ -8,13 +8,12 @@ rm -rf out/master/ || exit 0
echo "Making the docs for master"
mkdir out/master/
cp util/gh-pages/index.html out/master
-python3 ./util/export.py out/master/lints.json
+cp util/gh-pages/lints.json out/master
if [[ -n $TAG_NAME ]]; then
echo "Save the doc for the current tag ($TAG_NAME) and point stable/ to it"
- cp -r out/master "out/$TAG_NAME"
- rm -f out/stable
- ln -s "$TAG_NAME" out/stable
+ cp -Tr out/master "out/$TAG_NAME"
+ ln -sf "$TAG_NAME" out/stable
fi
if [[ $BETA = "true" ]]; then
@@ -28,8 +27,8 @@ cp util/gh-pages/versions.html out/index.html
echo "Making the versions.json file"
python3 ./util/versions.py out
-cd out
# Now let's go have some fun with the cloned repo
+cd out
git config user.name "GHA CI"
git config user.email "gha@ci.invalid"
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 15aeaf907dc..b8be730be32 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -39,10 +39,23 @@ jobs:
if: github.ref == 'refs/heads/beta'
run: echo "BETA=true" >> $GITHUB_ENV
- - name: Use scripts and templates from master branch
+ # We need to check out all files that (transitively) depend on the
+ # structure of the gh-pages branch, so that we're able to change that
+ # structure without breaking the deployment.
+ - name: Use deploy files from master branch
run: |
git fetch --no-tags --prune --depth=1 origin master
- git checkout origin/master -- .github/deploy.sh util/gh-pages/ util/*.py
+ git checkout origin/master -- .github/deploy.sh util/versions.py util/gh-pages/versions.html
+
+ # Generate lockfile for caching to avoid build problems with cached deps
+ - name: cargo generate-lockfile
+ run: cargo generate-lockfile
+
+ - name: Cache
+ uses: Swatinem/rust-cache@v1.3.0
+
+ - name: cargo collect-metadata
+ run: cargo collect-metadata
- name: Deploy
run: |
diff --git a/clippy_lints/src/utils/internal_lints/metadata_collector.rs b/clippy_lints/src/utils/internal_lints/metadata_collector.rs
index 3eccc89cdeb..598450a725f 100644
--- a/clippy_lints/src/utils/internal_lints/metadata_collector.rs
+++ b/clippy_lints/src/utils/internal_lints/metadata_collector.rs
@@ -32,7 +32,7 @@ use clippy_utils::{
};
/// This is the output file of the lint collector.
-const OUTPUT_FILE: &str = "../util/gh-pages/metadata_collection.json";
+const OUTPUT_FILE: &str = "../util/gh-pages/lints.json";
/// These lints are excluded from the export.
const BLACK_LISTED_LINTS: [&str; 3] = ["lint_author", "deep_code_inspection", "internal_metadata_collector"];
/// These groups will be ignored by the lint group matcher. This is useful for collections like
diff --git a/tests/dogfood.rs b/tests/dogfood.rs
index a996f9df144..4ede20c5258 100644
--- a/tests/dogfood.rs
+++ b/tests/dogfood.rs
@@ -183,7 +183,7 @@ fn run_metadata_collection_lint() {
use std::time::SystemTime;
// Setup for validation
- let metadata_output_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("util/gh-pages/metadata_collection.json");
+ let metadata_output_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("util/gh-pages/lints.json");
let start_time = SystemTime::now();
// Run collection as is
From fe25282aaa656dda273d5913430a20a450cf11a3 Mon Sep 17 00:00:00 2001
From: flip1995 <philipp.krones@embecosm.com>
Date: Wed, 28 Jul 2021 14:15:34 +0200
Subject: [PATCH 2/3] Remove old python lint doc generation scripts
---
util/export.py | 84 ----------------------------------
util/lintlib.py | 115 -----------------------------------------------
util/versions.py | 6 +--
3 files changed, 3 insertions(+), 202 deletions(-)
delete mode 100755 util/export.py
delete mode 100644 util/lintlib.py
diff --git a/util/export.py b/util/export.py
deleted file mode 100755
index 1248e6b6a26..00000000000
--- a/util/export.py
+++ /dev/null
@@ -1,84 +0,0 @@
-#!/usr/bin/env python
-
-# Build the gh-pages
-
-from collections import OrderedDict
-import re
-import sys
-import json
-
-from lintlib import parse_all, log
-
-lint_subheadline = re.compile(r'''^\*\*([\w\s]+?)[:?.!]?\*\*(.*)''')
-rust_code_block = re.compile(r'''```rust.+?```''', flags=re.DOTALL)
-
-CONF_TEMPLATE = """\
-This lint has the following configuration variables:
-
-* `%s: %s`: %s (defaults to `%s`)."""
-
-
-def parse_code_block(match):
- lines = []
-
- for line in match.group(0).split('\n'):
- # fix syntax highlighting for headers like ```rust,ignore
- if line.startswith('```rust'):
- lines.append('```rust')
- elif not line.startswith('# '):
- lines.append(line)
-
- return '\n'.join(lines)
-
-
-def parse_lint_def(lint):
- lint_dict = {}
- lint_dict['id'] = lint.name
- lint_dict['group'] = lint.group
- lint_dict['level'] = lint.level
- lint_dict['docs'] = OrderedDict()
-
- last_section = None
-
- for line in lint.doc:
- match = re.match(lint_subheadline, line)
- if match:
- last_section = match.groups()[0]
- text = match.groups()[1]
- else:
- text = line
-
- if not last_section:
- log.warning("Skipping comment line as it was not preceded by a heading")
- log.debug("in lint `%s`, line `%s`", lint.name, line)
-
- if last_section not in lint_dict['docs']:
- lint_dict['docs'][last_section] = ""
-
- lint_dict['docs'][last_section] += text + "\n"
-
- for section in lint_dict['docs']:
- lint_dict['docs'][section] = re.sub(rust_code_block, parse_code_block, lint_dict['docs'][section].strip())
-
- return lint_dict
-
-
-def main():
- lintlist, configs = parse_all()
- lints = {}
- for lint in lintlist:
- lints[lint.name] = parse_lint_def(lint)
- if lint.name in configs:
- lints[lint.name]['docs']['Configuration'] = \
- CONF_TEMPLATE % configs[lint.name]
-
- outfile = sys.argv[1] if len(sys.argv) > 1 else "util/gh-pages/lints.json"
- with open(outfile, "w") as fp:
- lints = list(lints.values())
- lints.sort(key=lambda x: x['id'])
- json.dump(lints, fp, indent=2)
- log.info("wrote JSON for great justice")
-
-
-if __name__ == "__main__":
- main()
diff --git a/util/lintlib.py b/util/lintlib.py
deleted file mode 100644
index 9cefb2dbb19..00000000000
--- a/util/lintlib.py
+++ /dev/null
@@ -1,115 +0,0 @@
-# Common utils for the several housekeeping scripts.
-
-import os
-import re
-import collections
-
-import logging as log
-log.basicConfig(level=log.INFO, format='%(levelname)s: %(message)s')
-
-Lint = collections.namedtuple('Lint', 'name level doc sourcefile group')
-Config = collections.namedtuple('Config', 'name ty doc default')
-
-lintname_re = re.compile(r'''pub\s+([A-Z_][A-Z_0-9]*)''')
-group_re = re.compile(r'''\s*([a-z_][a-z_0-9]+)''')
-conf_re = re.compile(r'''define_Conf! {\n((?!\n})[\s\S])*\n}''', re.MULTILINE)
-confvar_re = re.compile(
- r'''/// Lint: ([\w,\s]+)\. (.*)\n\s*\(([^:]+):\s*([^\s=]+)\s*=\s*([^\.\)]+).*\),''', re.MULTILINE)
-comment_re = re.compile(r'''\s*/// ?(.*)''')
-
-lint_levels = {
- "correctness": 'Deny',
- "suspicious": 'Warn',
- "style": 'Warn',
- "complexity": 'Warn',
- "perf": 'Warn',
- "restriction": 'Allow',
- "pedantic": 'Allow',
- "nursery": 'Allow',
- "cargo": 'Allow',
-}
-
-
-def parse_lints(lints, filepath):
- comment = []
- clippy = False
- deprecated = False
- name = ""
-
- with open(filepath) as fp:
- for line in fp:
- if clippy or deprecated:
- m = lintname_re.search(line)
- if m:
- name = m.group(1).lower()
- line = next(fp)
-
- if deprecated:
- level = "Deprecated"
- group = "deprecated"
- else:
- while True:
- g = group_re.search(line)
- if g:
- group = g.group(1).lower()
- level = lint_levels.get(group, None)
- break
- line = next(fp)
-
- if level is None:
- continue
-
- log.info("found %s with level %s in %s",
- name, level, filepath)
- lints.append(Lint(name, level, comment, filepath, group))
- comment = []
-
- clippy = False
- deprecated = False
- name = ""
- else:
- m = comment_re.search(line)
- if m:
- comment.append(m.group(1))
- elif line.startswith("declare_clippy_lint!"):
- clippy = True
- deprecated = False
- elif line.startswith("declare_deprecated_lint!"):
- clippy = False
- deprecated = True
- elif line.startswith("declare_lint!"):
- import sys
- print(
- "don't use `declare_lint!` in Clippy, "
- "use `declare_clippy_lint!` instead"
- )
- sys.exit(42)
-
-
-def parse_configs(path):
- configs = {}
- with open(os.path.join(path, 'utils/conf.rs')) as fp:
- contents = fp.read()
-
- match = re.search(conf_re, contents)
- confvars = re.findall(confvar_re, match.group(0))
-
- for (lints, doc, name, ty, default) in confvars:
- for lint in lints.split(','):
- configs[lint.strip().lower()] = Config(name.replace("_", "-"), ty, doc, default)
- return configs
-
-
-def parse_all(path="clippy_lints/src"):
- lints = []
- for root, dirs, files in os.walk(path):
- for fn in files:
- if fn.endswith('.rs'):
- parse_lints(lints, os.path.join(root, fn))
-
- log.info("got %s lints", len(lints))
-
- configs = parse_configs(path)
- log.info("got %d configs", len(configs))
-
- return lints, configs
diff --git a/util/versions.py b/util/versions.py
index 5cdc7313f54..0cfa007d1b2 100755
--- a/util/versions.py
+++ b/util/versions.py
@@ -3,8 +3,8 @@
import json
import os
import sys
-
-from lintlib import log
+import logging as log
+log.basicConfig(level=log.INFO, format='%(levelname)s: %(message)s')
def key(v):
@@ -26,7 +26,7 @@ def key(v):
def main():
if len(sys.argv) < 2:
- print("Error: specify output directory")
+ log.error("specify output directory")
return
outdir = sys.argv[1]
From c951a3c68daefb078d7e52e87e3cebbfa9a13f10 Mon Sep 17 00:00:00 2001
From: flip1995 <philipp.krones@embecosm.com>
Date: Wed, 28 Jul 2021 15:05:11 +0200
Subject: [PATCH 3/3] Run cargo collect-metadata in cargo dev serve
---
clippy_dev/src/serve.rs | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/clippy_dev/src/serve.rs b/clippy_dev/src/serve.rs
index d13c27a1957..b36e2a28ee4 100644
--- a/clippy_dev/src/serve.rs
+++ b/clippy_dev/src/serve.rs
@@ -15,8 +15,8 @@ pub fn run(port: u16, lint: Option<&str>) -> ! {
loop {
if mtime("util/gh-pages/lints.json") < mtime("clippy_lints/src") {
- Command::new("python3")
- .arg("util/export.py")
+ Command::new("cargo")
+ .arg("collect-metadata")
.spawn()
.unwrap()
.wait()