From 552eb3295a06a1a6841686f064647c4d847552cb Mon Sep 17 00:00:00 2001 From: David Wood Date: Mon, 23 May 2022 18:24:55 +0100 Subject: [PATCH] macros: introduce `fluent_messages` macro Adds a new `fluent_messages` macro which performs compile-time validation of the compiler's Fluent resources (i.e. that the resources parse and don't multiply define the same messages) and generates constants that make using those messages in diagnostics more ergonomic. For example, given the following invocation of the macro.. ```ignore (rust) fluent_messages! { typeck => "./typeck.ftl", } ``` ..where `typeck.ftl` has the following contents.. ```fluent typeck-field-multiply-specified-in-initializer = field `{$ident}` specified more than once .label = used more than once .label-previous-use = first use of `{$ident}` ``` ...then the macro parse the Fluent resource, emitting a diagnostic if it fails to do so, and will generate the following code: ```ignore (rust) pub static DEFAULT_LOCALE_RESOURCES: &'static [&'static str] = &[ include_str!("./typeck.ftl"), ]; mod fluent_generated { mod typeck { pub const field_multiply_specified_in_initializer: DiagnosticMessage = DiagnosticMessage::fluent("typeck-field-multiply-specified-in-initializer"); pub const field_multiply_specified_in_initializer_label_previous_use: DiagnosticMessage = DiagnosticMessage::fluent_attr( "typeck-field-multiply-specified-in-initializer", "previous-use-label" ); } } ``` When emitting a diagnostic, the generated constants can be used as follows: ```ignore (rust) let mut err = sess.struct_span_err( span, fluent::typeck::field_multiply_specified_in_initializer ); err.span_default_label(span); err.span_label( previous_use_span, fluent::typeck::field_multiply_specified_in_initializer_label_previous_use ); err.emit(); ``` Signed-off-by: David Wood --- Cargo.lock | 4 + compiler/rustc_error_messages/src/lib.rs | 11 +- compiler/rustc_errors/src/lib.rs | 4 +- compiler/rustc_macros/Cargo.toml | 4 + .../rustc_macros/src/diagnostics/fluent.rs | 254 ++++++++++++++++++ compiler/rustc_macros/src/diagnostics/mod.rs | 2 + compiler/rustc_macros/src/lib.rs | 59 ++++ .../fluent-messages/duplicate-a.ftl | 1 + .../fluent-messages/duplicate-b.ftl | 1 + .../fluent-messages/missing-message.ftl | 1 + src/test/ui-fulldeps/fluent-messages/test.rs | 60 +++++ .../ui-fulldeps/fluent-messages/test.stderr | 45 ++++ .../ui-fulldeps/fluent-messages/valid.ftl | 1 + 13 files changed, 442 insertions(+), 5 deletions(-) create mode 100644 compiler/rustc_macros/src/diagnostics/fluent.rs create mode 100644 src/test/ui-fulldeps/fluent-messages/duplicate-a.ftl create mode 100644 src/test/ui-fulldeps/fluent-messages/duplicate-b.ftl create mode 100644 src/test/ui-fulldeps/fluent-messages/missing-message.ftl create mode 100644 src/test/ui-fulldeps/fluent-messages/test.rs create mode 100644 src/test/ui-fulldeps/fluent-messages/test.stderr create mode 100644 src/test/ui-fulldeps/fluent-messages/valid.ftl diff --git a/Cargo.lock b/Cargo.lock index 9fa6e1d51d2..25b43227d6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4008,10 +4008,14 @@ dependencies = [ name = "rustc_macros" version = "0.1.0" dependencies = [ + "annotate-snippets", + "fluent-bundle", + "fluent-syntax", "proc-macro2", "quote", "syn", "synstructure", + "unic-langid", ] [[package]] diff --git a/compiler/rustc_error_messages/src/lib.rs b/compiler/rustc_error_messages/src/lib.rs index e1e0ed7222d..7faf14a2472 100644 --- a/compiler/rustc_error_messages/src/lib.rs +++ b/compiler/rustc_error_messages/src/lib.rs @@ -6,7 +6,7 @@ use fluent_bundle::FluentResource; use fluent_syntax::parser::ParserError; use rustc_data_structures::sync::Lrc; -use rustc_macros::{Decodable, Encodable}; +use rustc_macros::{fluent_messages, Decodable, Encodable}; use rustc_span::Span; use std::borrow::Cow; use std::error::Error; @@ -29,8 +29,13 @@ use intl_memoizer::IntlLangMemoizer; pub use fluent_bundle::{FluentArgs, FluentError, FluentValue}; pub use unic_langid::{langid, LanguageIdentifier}; -pub static DEFAULT_LOCALE_RESOURCES: &'static [&'static str] = - &[include_str!("../locales/en-US/typeck.ftl"), include_str!("../locales/en-US/parser.ftl")]; +// Generates `DEFAULT_LOCALE_RESOURCES` static and `fluent_generated` module. +fluent_messages! { + parser => "../locales/en-US/parser.ftl", + typeck => "../locales/en-US/typeck.ftl", +} + +pub use fluent_generated::{self as fluent, DEFAULT_LOCALE_RESOURCES}; pub type FluentBundle = fluent_bundle::bundle::FluentBundle; diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs index d2f50d5df54..5b9b65da343 100644 --- a/compiler/rustc_errors/src/lib.rs +++ b/compiler/rustc_errors/src/lib.rs @@ -31,8 +31,8 @@ use rustc_data_structures::stable_hasher::StableHasher; use rustc_data_structures::sync::{self, Lock, Lrc}; use rustc_data_structures::AtomicRef; pub use rustc_error_messages::{ - fallback_fluent_bundle, fluent_bundle, DiagnosticMessage, FluentBundle, LanguageIdentifier, - LazyFallbackBundle, MultiSpan, SpanLabel, DEFAULT_LOCALE_RESOURCES, + fallback_fluent_bundle, fluent, fluent_bundle, DiagnosticMessage, FluentBundle, + LanguageIdentifier, LazyFallbackBundle, MultiSpan, SpanLabel, DEFAULT_LOCALE_RESOURCES, }; pub use rustc_lint_defs::{pluralize, Applicability}; use rustc_span::source_map::SourceMap; diff --git a/compiler/rustc_macros/Cargo.toml b/compiler/rustc_macros/Cargo.toml index a9192be4d6e..25b3aadc1c5 100644 --- a/compiler/rustc_macros/Cargo.toml +++ b/compiler/rustc_macros/Cargo.toml @@ -7,7 +7,11 @@ edition = "2021" proc-macro = true [dependencies] +annotate-snippets = "0.8.0" +fluent-bundle = "0.15.2" +fluent-syntax = "0.11" synstructure = "0.12.1" syn = { version = "1", features = ["full"] } proc-macro2 = "1" quote = "1" +unic-langid = { version = "0.9.0", features = ["macros"] } diff --git a/compiler/rustc_macros/src/diagnostics/fluent.rs b/compiler/rustc_macros/src/diagnostics/fluent.rs new file mode 100644 index 00000000000..8523d7fa9f9 --- /dev/null +++ b/compiler/rustc_macros/src/diagnostics/fluent.rs @@ -0,0 +1,254 @@ +use annotate_snippets::{ + display_list::DisplayList, + snippet::{Annotation, AnnotationType, Slice, Snippet, SourceAnnotation}, +}; +use fluent_bundle::{FluentBundle, FluentError, FluentResource}; +use fluent_syntax::{ + ast::{Attribute, Entry, Identifier, Message}, + parser::ParserError, +}; +use proc_macro::{Diagnostic, Level, Span}; +use proc_macro2::TokenStream; +use quote::quote; +use std::{ + collections::HashMap, + fs::File, + io::Read, + path::{Path, PathBuf}, +}; +use syn::{ + parse::{Parse, ParseStream}, + parse_macro_input, + punctuated::Punctuated, + token, Ident, LitStr, Result, +}; +use unic_langid::langid; + +struct Resource { + ident: Ident, + #[allow(dead_code)] + fat_arrow_token: token::FatArrow, + resource: LitStr, +} + +impl Parse for Resource { + fn parse(input: ParseStream<'_>) -> Result { + Ok(Resource { + ident: input.parse()?, + fat_arrow_token: input.parse()?, + resource: input.parse()?, + }) + } +} + +struct Resources(Punctuated); + +impl Parse for Resources { + fn parse(input: ParseStream<'_>) -> Result { + let mut resources = Punctuated::new(); + loop { + if input.is_empty() || input.peek(token::Brace) { + break; + } + let value = input.parse()?; + resources.push_value(value); + if !input.peek(token::Comma) { + break; + } + let punct = input.parse()?; + resources.push_punct(punct); + } + Ok(Resources(resources)) + } +} + +/// Helper function for returning an absolute path for macro-invocation relative file paths. +/// +/// If the input is already absolute, then the input is returned. If the input is not absolute, +/// then it is appended to the directory containing the source file with this macro invocation. +fn invocation_relative_path_to_absolute(span: Span, path: &str) -> PathBuf { + let path = Path::new(path); + if path.is_absolute() { + path.to_path_buf() + } else { + // `/a/b/c/foo/bar.rs` contains the current macro invocation + let mut source_file_path = span.source_file().path(); + // `/a/b/c/foo/` + source_file_path.pop(); + // `/a/b/c/foo/../locales/en-US/example.ftl` + source_file_path.push(path); + source_file_path + } +} + +/// See [rustc_macros::fluent_messages]. +pub(crate) fn fluent_messages(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let resources = parse_macro_input!(input as Resources); + + // Cannot iterate over individual messages in a bundle, so do that using the + // `FluentResource` instead. Construct a bundle anyway to find out if there are conflicting + // messages in the resources. + let mut bundle = FluentBundle::new(vec![langid!("en-US")]); + + // Map of Fluent identifiers to the `Span` of the resource that defined them, used for better + // diagnostics. + let mut previous_defns = HashMap::new(); + + let mut includes = TokenStream::new(); + let mut generated = TokenStream::new(); + for res in resources.0 { + let ident_span = res.ident.span().unwrap(); + let path_span = res.resource.span().unwrap(); + + let relative_ftl_path = res.resource.value(); + let absolute_ftl_path = + invocation_relative_path_to_absolute(ident_span, &relative_ftl_path); + // As this macro also outputs an `include_str!` for this file, the macro will always be + // re-executed when the file changes. + let mut resource_file = match File::open(absolute_ftl_path) { + Ok(resource_file) => resource_file, + Err(e) => { + Diagnostic::spanned(path_span, Level::Error, "could not open Fluent resource") + .note(e.to_string()) + .emit(); + continue; + } + }; + let mut resource_contents = String::new(); + if let Err(e) = resource_file.read_to_string(&mut resource_contents) { + Diagnostic::spanned(path_span, Level::Error, "could not read Fluent resource") + .note(e.to_string()) + .emit(); + continue; + } + let resource = match FluentResource::try_new(resource_contents) { + Ok(resource) => resource, + Err((this, errs)) => { + Diagnostic::spanned(path_span, Level::Error, "could not parse Fluent resource") + .help("see additional errors emitted") + .emit(); + for ParserError { pos, slice: _, kind } in errs { + let mut err = kind.to_string(); + // Entirely unnecessary string modification so that the error message starts + // with a lowercase as rustc errors do. + err.replace_range( + 0..1, + &err.chars().next().unwrap().to_lowercase().to_string(), + ); + + let line_starts: Vec = std::iter::once(0) + .chain( + this.source() + .char_indices() + .filter_map(|(i, c)| Some(i + 1).filter(|_| c == '\n')), + ) + .collect(); + let line_start = line_starts + .iter() + .enumerate() + .map(|(line, idx)| (line + 1, idx)) + .filter(|(_, idx)| **idx <= pos.start) + .last() + .unwrap() + .0; + + let snippet = Snippet { + title: Some(Annotation { + label: Some(&err), + id: None, + annotation_type: AnnotationType::Error, + }), + footer: vec![], + slices: vec![Slice { + source: this.source(), + line_start, + origin: Some(&relative_ftl_path), + fold: true, + annotations: vec![SourceAnnotation { + label: "", + annotation_type: AnnotationType::Error, + range: (pos.start, pos.end - 1), + }], + }], + opt: Default::default(), + }; + let dl = DisplayList::from(snippet); + eprintln!("{}\n", dl); + } + continue; + } + }; + + let mut constants = TokenStream::new(); + for entry in resource.entries() { + let span = res.ident.span(); + if let Entry::Message(Message { id: Identifier { name }, attributes, .. }) = entry { + let _ = previous_defns.entry(name.to_string()).or_insert(ident_span); + + // `typeck-foo-bar` => `foo_bar` + let snake_name = Ident::new( + &name.replace(&format!("{}-", res.ident), "").replace("-", "_"), + span, + ); + constants.extend(quote! { + pub const #snake_name: crate::DiagnosticMessage = + crate::DiagnosticMessage::FluentIdentifier( + std::borrow::Cow::Borrowed(#name), + None + ); + }); + + for Attribute { id: Identifier { name: attr_name }, .. } in attributes { + let attr_snake_name = attr_name.replace("-", "_"); + let snake_name = Ident::new(&format!("{snake_name}_{attr_snake_name}"), span); + constants.extend(quote! { + pub const #snake_name: crate::DiagnosticMessage = + crate::DiagnosticMessage::FluentIdentifier( + std::borrow::Cow::Borrowed(#name), + Some(std::borrow::Cow::Borrowed(#attr_name)) + ); + }); + } + } + } + + if let Err(errs) = bundle.add_resource(resource) { + for e in errs { + match e { + FluentError::Overriding { kind, id } => { + Diagnostic::spanned( + ident_span, + Level::Error, + format!("overrides existing {}: `{}`", kind, id), + ) + .span_help(previous_defns[&id], "previously defined in this resource") + .emit(); + } + FluentError::ResolverError(_) | FluentError::ParserError(_) => unreachable!(), + } + } + } + + includes.extend(quote! { include_str!(#relative_ftl_path), }); + + let ident = res.ident; + generated.extend(quote! { + pub mod #ident { + #constants + } + }); + } + + quote! { + #[allow(non_upper_case_globals)] + #[doc(hidden)] + pub mod fluent_generated { + pub static DEFAULT_LOCALE_RESOURCES: &'static [&'static str] = &[ + #includes + ]; + + #generated + } + } + .into() +} diff --git a/compiler/rustc_macros/src/diagnostics/mod.rs b/compiler/rustc_macros/src/diagnostics/mod.rs index ccd3880057b..69573d904d4 100644 --- a/compiler/rustc_macros/src/diagnostics/mod.rs +++ b/compiler/rustc_macros/src/diagnostics/mod.rs @@ -1,9 +1,11 @@ mod diagnostic; mod error; +mod fluent; mod subdiagnostic; mod utils; use diagnostic::SessionDiagnosticDerive; +pub(crate) use fluent::fluent_messages; use proc_macro2::TokenStream; use quote::format_ident; use subdiagnostic::SessionSubdiagnosticDerive; diff --git a/compiler/rustc_macros/src/lib.rs b/compiler/rustc_macros/src/lib.rs index 0baebdb7130..7c8e3c6d140 100644 --- a/compiler/rustc_macros/src/lib.rs +++ b/compiler/rustc_macros/src/lib.rs @@ -2,6 +2,7 @@ #![feature(let_else)] #![feature(never_type)] #![feature(proc_macro_diagnostic)] +#![feature(proc_macro_span)] #![allow(rustc::default_hash_types)] #![recursion_limit = "128"] @@ -49,6 +50,64 @@ pub fn newtype_index(input: TokenStream) -> TokenStream { newtype::newtype(input) } +/// Implements the `fluent_messages` macro, which performs compile-time validation of the +/// compiler's Fluent resources (i.e. that the resources parse and don't multiply define the same +/// messages) and generates constants that make using those messages in diagnostics more ergonomic. +/// +/// For example, given the following invocation of the macro.. +/// +/// ```ignore (rust) +/// fluent_messages! { +/// typeck => "./typeck.ftl", +/// } +/// ``` +/// ..where `typeck.ftl` has the following contents.. +/// +/// ```fluent +/// typeck-field-multiply-specified-in-initializer = +/// field `{$ident}` specified more than once +/// .label = used more than once +/// .label-previous-use = first use of `{$ident}` +/// ``` +/// ...then the macro parse the Fluent resource, emitting a diagnostic if it fails to do so, and +/// will generate the following code: +/// +/// ```ignore (rust) +/// pub static DEFAULT_LOCALE_RESOURCES: &'static [&'static str] = &[ +/// include_str!("./typeck.ftl"), +/// ]; +/// +/// mod fluent_generated { +/// mod typeck { +/// pub const field_multiply_specified_in_initializer: DiagnosticMessage = +/// DiagnosticMessage::fluent("typeck-field-multiply-specified-in-initializer"); +/// pub const field_multiply_specified_in_initializer_label_previous_use: DiagnosticMessage = +/// DiagnosticMessage::fluent_attr( +/// "typeck-field-multiply-specified-in-initializer", +/// "previous-use-label" +/// ); +/// } +/// } +/// ``` +/// When emitting a diagnostic, the generated constants can be used as follows: +/// +/// ```ignore (rust) +/// let mut err = sess.struct_span_err( +/// span, +/// fluent::typeck::field_multiply_specified_in_initializer +/// ); +/// err.span_default_label(span); +/// err.span_label( +/// previous_use_span, +/// fluent::typeck::field_multiply_specified_in_initializer_label_previous_use +/// ); +/// err.emit(); +/// ``` +#[proc_macro] +pub fn fluent_messages(input: TokenStream) -> TokenStream { + diagnostics::fluent_messages(input) +} + decl_derive!([HashStable, attributes(stable_hasher)] => hash_stable::hash_stable_derive); decl_derive!( [HashStable_Generic, attributes(stable_hasher)] => diff --git a/src/test/ui-fulldeps/fluent-messages/duplicate-a.ftl b/src/test/ui-fulldeps/fluent-messages/duplicate-a.ftl new file mode 100644 index 00000000000..fd9976b5a41 --- /dev/null +++ b/src/test/ui-fulldeps/fluent-messages/duplicate-a.ftl @@ -0,0 +1 @@ +key = Value diff --git a/src/test/ui-fulldeps/fluent-messages/duplicate-b.ftl b/src/test/ui-fulldeps/fluent-messages/duplicate-b.ftl new file mode 100644 index 00000000000..fd9976b5a41 --- /dev/null +++ b/src/test/ui-fulldeps/fluent-messages/duplicate-b.ftl @@ -0,0 +1 @@ +key = Value diff --git a/src/test/ui-fulldeps/fluent-messages/missing-message.ftl b/src/test/ui-fulldeps/fluent-messages/missing-message.ftl new file mode 100644 index 00000000000..372b1a2e453 --- /dev/null +++ b/src/test/ui-fulldeps/fluent-messages/missing-message.ftl @@ -0,0 +1 @@ +missing-message = diff --git a/src/test/ui-fulldeps/fluent-messages/test.rs b/src/test/ui-fulldeps/fluent-messages/test.rs new file mode 100644 index 00000000000..b05d3d08ccb --- /dev/null +++ b/src/test/ui-fulldeps/fluent-messages/test.rs @@ -0,0 +1,60 @@ +// normalize-stderr-test "note.*" -> "note: os-specific message" + +#![feature(rustc_private)] +#![crate_type = "lib"] + +extern crate rustc_macros; +use rustc_macros::fluent_messages; + +/// Copy of the relevant `DiagnosticMessage` variant constructed by `fluent_messages` as it +/// expects `crate::DiagnosticMessage` to exist. +pub enum DiagnosticMessage { + FluentIdentifier(std::borrow::Cow<'static, str>, Option>), +} + +mod missing_absolute { + use super::fluent_messages; + + fluent_messages! { + missing_absolute => "/definitely_does_not_exist.ftl", +//~^ ERROR could not open Fluent resource + } +} + +mod missing_relative { + use super::fluent_messages; + + fluent_messages! { + missing_relative => "../definitely_does_not_exist.ftl", +//~^ ERROR could not open Fluent resource + } +} + +mod missing_message { + use super::fluent_messages; + + fluent_messages! { + missing_message => "./missing-message.ftl", +//~^ ERROR could not parse Fluent resource + } +} + +mod duplicate { + use super::fluent_messages; + + fluent_messages! { + a => "./duplicate-a.ftl", + b => "./duplicate-b.ftl", +//~^ ERROR overrides existing message: `key` + } +} + +mod valid { + use super::fluent_messages; + + fluent_messages! { + valid => "./valid.ftl", + } + + use self::fluent_generated::{DEFAULT_LOCALE_RESOURCES, valid::valid}; +} diff --git a/src/test/ui-fulldeps/fluent-messages/test.stderr b/src/test/ui-fulldeps/fluent-messages/test.stderr new file mode 100644 index 00000000000..f88d09bee6e --- /dev/null +++ b/src/test/ui-fulldeps/fluent-messages/test.stderr @@ -0,0 +1,45 @@ +error: could not open Fluent resource + --> $DIR/test.rs:19:29 + | +LL | missing_absolute => "/definitely_does_not_exist.ftl", + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: os-specific message + +error: could not open Fluent resource + --> $DIR/test.rs:28:29 + | +LL | missing_relative => "../definitely_does_not_exist.ftl", + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: os-specific message + +error: could not parse Fluent resource + --> $DIR/test.rs:37:28 + | +LL | missing_message => "./missing-message.ftl", + | ^^^^^^^^^^^^^^^^^^^^^^^ + | + = help: see additional errors emitted + +error: expected a message field for "missing-message" + --> ./missing-message.ftl:1:1 + | +1 | missing-message = + | ^^^^^^^^^^^^^^^^^^ + | + +error: overrides existing message: `key` + --> $DIR/test.rs:47:9 + | +LL | b => "./duplicate-b.ftl", + | ^ + | +help: previously defined in this resource + --> $DIR/test.rs:46:9 + | +LL | a => "./duplicate-a.ftl", + | ^ + +error: aborting due to 4 previous errors + diff --git a/src/test/ui-fulldeps/fluent-messages/valid.ftl b/src/test/ui-fulldeps/fluent-messages/valid.ftl new file mode 100644 index 00000000000..0eee4a02b96 --- /dev/null +++ b/src/test/ui-fulldeps/fluent-messages/valid.ftl @@ -0,0 +1 @@ +valid = Valid!