condense llvm licensing into a single item

.reuse/dep5

@@ -93,3 +93,11 @@ License: MIT OR Apache-2.0
 Files: src/doc/rustc-dev-guide/mermaid.min.js
 Copyright: Knut Sveidqvist
 License: MIT
+
+# Note that the LLVM submodule here only specifies the NCSA as the license for
+# LLVM, even though LLVM is in the process of a relicensing to Apache 2.0 with
+# the LLVM Exception. That's because relicensed files have a SPDX header with
+# the correct license, so it will automatically be included here.
+Files: src/llvm-project/*
+Copyright: The LLVM Authors
+License: NCSA

LICENSES/NCSA.txt

@@ -0,0 +1,29 @@
+Copyright (c) <YEAR> <OWNER ORGANIZATION NAME>.  All rights reserved.
+
+Developed by: <NAME OF DEVELOPMENT GROUP>
+              <NAME OF INSTITUTION>
+              <URL FOR DEVELOPMENT GROUP/INSTITUTION>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal with
+the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to
+do so, subject to the following conditions:
+* Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimers.
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimers in the documentation
+  and/or other materials provided with the distribution.
+* Neither the names of <NAME OF DEVELOPMENT GROUP>, <NAME OF INSTITUTION>,
+  nor the names of its contributors may be used to endorse or promote products
+  derived from this Software without specific prior written permission.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
+SOFTWARE.
+

src/tools/collect-license-metadata/src/path_tree.rs

@@ -4,14 +4,20 @@
 //! passes over the tree to remove redundant information.
 
 use crate::licenses::{License, LicenseId, LicensesInterner};
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 use std::path::{Path, PathBuf};
 
+// Some directories have too many slight license differences that'd result in a huge report, and
+// could be considered a standalone project anyway. Those directories are "condensed" into a single
+// licensing block for ease of reading, merging the licensing information.
+const CONDENSED_DIRECTORIED: &[&str] = &["./src/llvm-project/"];
+
 #[derive(serde::Serialize)]
 #[serde(rename_all = "kebab-case", tag = "type")]
 pub(crate) enum Node<L> {
     Root { children: Vec<Node<L>> },
     Directory { name: PathBuf, children: Vec<Node<L>>, license: Option<L> },
+    CondensedDirectory { name: PathBuf, licenses: Vec<L> },
     File { name: PathBuf, license: L },
     Group { files: Vec<PathBuf>, directories: Vec<PathBuf>, license: L },
     Empty,
@@ -57,9 +63,9 @@ impl Node<LicenseId> {
                         Node::Directory { name, mut children, license: None } => {
                             directories.entry(name).or_insert_with(Vec::new).append(&mut children);
                         }
-                        file @ Node::File { .. } => {
+                        file @ Node::File { .. } => files.push(file),
-                            files.push(file);
+                        // Propagate condensed directories as-is.
-                        }
+                        condensed @ Node::CondensedDirectory { .. } => files.push(condensed),
                         Node::Empty => {}
                         Node::Root { .. } => {
                             panic!("can't have a root inside another element");
@@ -86,6 +92,7 @@ impl Node<LicenseId> {
             }
             Node::Empty => {}
             Node::File { .. } => {}
+            Node::CondensedDirectory { .. } => {}
             Node::Group { .. } => {
                 panic!("Group should not be present at this stage");
             }
@@ -132,6 +139,7 @@ impl Node<LicenseId> {
                 }
             }
             Node::File { .. } => {}
+            Node::CondensedDirectory { .. } => {}
             Node::Group { .. } => panic!("group should not be present at this stage"),
             Node::Empty => {}
         }
@@ -174,6 +182,9 @@ impl Node<LicenseId> {
                             Node::Directory { name: child_child_name, .. } => {
                                 *child_child_name = child_name.join(&child_child_name);
                             }
+                            Node::CondensedDirectory { name: child_child_name, .. } => {
+                                *child_child_name = child_name.join(&child_child_name);
+                            }
                             Node::File { name: child_child_name, .. } => {
                                 *child_child_name = child_name.join(&child_child_name);
                             }
@@ -188,6 +199,7 @@ impl Node<LicenseId> {
             }
             Node::Empty => {}
             Node::File { .. } => {}
+            Node::CondensedDirectory { .. } => {}
             Node::Group { .. } => panic!("Group should not be present at this stage"),
         }
     }
@@ -255,6 +267,7 @@ impl Node<LicenseId> {
                 }
             }
             Node::File { .. } => {}
+            Node::CondensedDirectory { .. } => {}
             Node::Group { .. } => panic!("FileGroup should not be present at this stage"),
             Node::Empty => {}
         }
@@ -270,6 +283,7 @@ impl Node<LicenseId> {
                 }
                 children.retain(|child| !matches!(child, Node::Empty));
             }
+            Node::CondensedDirectory { .. } => {}
             Node::Group { .. } => {}
             Node::File { .. } => {}
             Node::Empty => {}
@@ -293,7 +307,19 @@ pub(crate) fn build(mut input: Vec<(PathBuf, LicenseId)>) -> Node<LicenseId> {
     // Ensure reproducibility of all future steps.
     input.sort();
 
-    for (path, license) in input {
+    let mut condensed_directories = BTreeMap::new();
+    'outer: for (path, license) in input {
+        // Files in condensed directories are handled separately.
+        for directory in CONDENSED_DIRECTORIED {
+            if path.starts_with(directory) {
+                condensed_directories
+                    .entry(*directory)
+                    .or_insert_with(BTreeSet::new)
+                    .insert(license);
+                continue 'outer;
+            }
+        }
+
         let mut node = Node::File { name: path.file_name().unwrap().into(), license };
         for component in path.parent().unwrap_or_else(|| Path::new(".")).components().rev() {
             node = Node::Directory {
@@ -306,6 +332,22 @@ pub(crate) fn build(mut input: Vec<(PathBuf, LicenseId)>) -> Node<LicenseId> {
         children.push(node);
     }
 
+    for (path, licenses) in condensed_directories {
+        let path = Path::new(path);
+        let mut node = Node::CondensedDirectory {
+            name: path.file_name().unwrap().into(),
+            licenses: licenses.iter().copied().collect(),
+        };
+        for name in path.parent().unwrap_or_else(|| Path::new(".")).components().rev() {
+            node = Node::Directory {
+                name: name.as_os_str().into(),
+                children: vec![node],
+                license: None,
+            };
+        }
+        children.push(node);
+    }
+
     Node::Root { children }
 }
 
@@ -334,6 +376,10 @@ pub(crate) fn expand_interned_licenses(
         Node::Group { files, directories, license } => {
             Node::Group { files, directories, license: interner.resolve(license) }
         }
+        Node::CondensedDirectory { name, licenses } => Node::CondensedDirectory {
+            name,
+            licenses: licenses.into_iter().map(|license| interner.resolve(license)).collect(),
+        },
         Node::Empty => Node::Empty,
     }
 }

src/tools/generate-copyright/src/main.rs

@@ -1,4 +1,5 @@
 use anyhow::Error;
+use std::collections::BTreeSet;
 use std::io::Write;
 use std::path::PathBuf;
 
@@ -26,7 +27,7 @@ fn render_recursive(node: &Node, buffer: &mut Vec<u8>, depth: usize) -> Result<(
             }
         }
         Node::Directory { name, children, license } => {
-            render_license(&prefix, std::iter::once(name), license, buffer)?;
+            render_license(&prefix, std::iter::once(name), std::iter::once(license), buffer)?;
             if !children.is_empty() {
                 writeln!(buffer, "{prefix}")?;
                 writeln!(buffer, "{prefix}*Exceptions:*")?;
@@ -36,11 +37,19 @@ fn render_recursive(node: &Node, buffer: &mut Vec<u8>, depth: usize) -> Result<(
                 }
             }
         }
+        Node::CondensedDirectory { name, licenses } => {
+            render_license(&prefix, std::iter::once(name), licenses.iter(), buffer)?;
+        }
         Node::Group { files, directories, license } => {
-            render_license(&prefix, directories.iter().chain(files.iter()), license, buffer)?;
+            render_license(
+                &prefix,
+                directories.iter().chain(files.iter()),
+                std::iter::once(license),
+                buffer,
+            )?;
         }
         Node::File { name, license } => {
-            render_license(&prefix, std::iter::once(name), license, buffer)?;
+            render_license(&prefix, std::iter::once(name), std::iter::once(license), buffer)?;
         }
     }
 
@@ -50,15 +59,26 @@ fn render_recursive(node: &Node, buffer: &mut Vec<u8>, depth: usize) -> Result<(
 fn render_license<'a>(
     prefix: &str,
     names: impl Iterator<Item = &'a String>,
-    license: &License,
+    licenses: impl Iterator<Item = &'a License>,
     buffer: &mut Vec<u8>,
 ) -> Result<(), Error> {
+    let mut spdxs = BTreeSet::new();
+    let mut copyrights = BTreeSet::new();
+    for license in licenses {
+        spdxs.insert(&license.spdx);
+        for copyright in &license.copyright {
+            copyrights.insert(copyright);
+        }
+    }
+
     for name in names {
         writeln!(buffer, "{prefix}**`{name}`**  ")?;
     }
-    writeln!(buffer, "{prefix}License: `{}`  ", license.spdx)?;
+    for spdx in spdxs.iter() {
-    for (i, copyright) in license.copyright.iter().enumerate() {
+        writeln!(buffer, "{prefix}License: `{spdx}`  ")?;
-        let suffix = if i == license.copyright.len() - 1 { "" } else { "  " };
+    }
+    for (i, copyright) in copyrights.iter().enumerate() {
+        let suffix = if i == copyrights.len() - 1 { "" } else { "  " };
         writeln!(buffer, "{prefix}Copyright: {copyright}{suffix}")?;
     }
 
@@ -75,6 +95,7 @@ struct Metadata {
 pub(crate) enum Node {
     Root { children: Vec<Node> },
     Directory { name: String, children: Vec<Node>, license: License },
+    CondensedDirectory { name: String, licenses: Vec<License> },
     File { name: String, license: License },
     Group { files: Vec<String>, directories: Vec<String>, license: License },
 }