Rollup merge of #39820 - jonasbb:export-attributes, r=nrc

Export attributes in save-analysis data

Since this is my first pull-request to rust, I would like to get some feedback about obvious errors in this implementation.

I would like to change the save-analysis data to include arbitrary attribute data.
A use-case I have in mind for this is identifying functions with `#[test]` annotations such that tools like rls can offer a test-runner feature. I described my idea here [rls#173](https://github.com/rust-lang-nursery/rls/issues/173).

My changes contain:

1. track a vector of attributes in the various `*Data` types in `data.rs` and `external_data.rs`
2. implement lowering for `Attribute` and `MetaItem`
3. adjust `JsonDumper` to print the attributes

In the lowering of `Attribute` I remove the distinction between `MetaItem` and `NestedMetaItem`. I did this because this distinction is somewhat confusing. For example, `NestedMetaItemKind::Literal` has two identical spans, because both `NestedMetaItem` and `Lit` are defined as `Spanned<_>`.
My model is strictly more general, as it allows an `LitKind` instead of a `Symbol` for `MetaItem` and `Symbol`s are converted into a cooked string. As a consumer of the save-analysis data this shouldn't affect you much.

Example json output of `#[test]` annotation:
```
"attributes": [
  {
    "value": {
      "name": {
        "variant": "Str",
        "fields": [
          "test",
          "Cooked"
        ]
      },
      "kind": "Literal",
      "span": {
        "file_name": "test.rs",
        "byte_start": 2,
        "byte_end": 6,
        "line_start": 1,
        "line_end": 1,
        "column_start": 3,
        "column_end": 7
      }
    },
    "span": {
      "file_name": "test.rs",
      "byte_start": 0,
      "byte_end": 7,
      "line_start": 1,
      "line_end": 1,
      "column_start": 1,
      "column_end": 8
    }
  }
]
```

src/librustc_save_analysis/data.rs

@@ -15,7 +15,7 @@
 
 use rustc::hir;
 use rustc::hir::def_id::{CrateNum, DefId};
-use syntax::ast::{self, NodeId};
+use syntax::ast::{self, Attribute, NodeId};
 use syntax_pos::Span;
 
 pub struct CrateData {
@@ -136,6 +136,7 @@ pub struct EnumData {
     pub visibility: Visibility,
     pub docs: String,
     pub sig: Signature,
+    pub attributes: Vec<Attribute>,
 }
 
 /// Data for extern crates.
@@ -171,6 +172,7 @@ pub struct FunctionData {
     pub parent: Option<DefId>,
     pub docs: String,
     pub sig: Signature,
+    pub attributes: Vec<Attribute>,
 }
 
 /// Data about a function call.
@@ -256,6 +258,7 @@ pub struct MethodData {
     pub visibility: Visibility,
     pub docs: String,
     pub sig: Signature,
+    pub attributes: Vec<Attribute>,
 }
 
 /// Data for modules.
@@ -271,6 +274,7 @@ pub struct ModData {
     pub visibility: Visibility,
     pub docs: String,
     pub sig: Signature,
+    pub attributes: Vec<Attribute>,
 }
 
 /// Data for a reference to a module.
@@ -295,6 +299,7 @@ pub struct StructData {
     pub visibility: Visibility,
     pub docs: String,
     pub sig: Signature,
+    pub attributes: Vec<Attribute>,
 }
 
 #[derive(Debug, RustcEncodable)]
@@ -309,6 +314,7 @@ pub struct StructVariantData {
     pub parent: Option<DefId>,
     pub docs: String,
     pub sig: Signature,
+    pub attributes: Vec<Attribute>,
 }
 
 #[derive(Debug, RustcEncodable)]
@@ -323,6 +329,7 @@ pub struct TraitData {
     pub visibility: Visibility,
     pub docs: String,
     pub sig: Signature,
+    pub attributes: Vec<Attribute>,
 }
 
 #[derive(Debug, RustcEncodable)]
@@ -337,6 +344,7 @@ pub struct TupleVariantData {
     pub parent: Option<DefId>,
     pub docs: String,
     pub sig: Signature,
+    pub attributes: Vec<Attribute>,
 }
 
 /// Data for a typedef.
@@ -351,6 +359,7 @@ pub struct TypeDefData {
     pub parent: Option<DefId>,
     pub docs: String,
     pub sig: Option<Signature>,
+    pub attributes: Vec<Attribute>,
 }
 
 /// Data for a reference to a type or trait.
@@ -396,6 +405,7 @@ pub struct VariableData {
     pub visibility: Visibility,
     pub docs: String,
     pub sig: Option<Signature>,
+    pub attributes: Vec<Attribute>,
 }
 
 #[derive(Debug, RustcEncodable)]

src/librustc_save_analysis/dump_visitor.rs

@@ -373,6 +373,7 @@ impl<'l, 'tcx: 'l, 'll, D: Dump + 'll> DumpVisitor<'l, 'tcx, 'll, D> {
                         visibility: Visibility::Inherited,
                         docs: String::new(),
                         sig: None,
+                        attributes: vec![],
                     }.lower(self.tcx));
                 }
             }
@@ -448,6 +449,7 @@ impl<'l, 'tcx: 'l, 'll, D: Dump + 'll> DumpVisitor<'l, 'tcx, 'll, D> {
                     visibility: vis,
                     docs: docs_for_attrs(attrs),
                     sig: method_data.sig,
+                    attributes: attrs.to_vec(),
                 }.lower(self.tcx));
             }
 
@@ -519,6 +521,7 @@ impl<'l, 'tcx: 'l, 'll, D: Dump + 'll> DumpVisitor<'l, 'tcx, 'll, D> {
                     parent: None,
                     docs: String::new(),
                     sig: None,
+                    attributes: vec![],
                 }.lower(self.tcx));
             }
         }
@@ -592,6 +595,7 @@ impl<'l, 'tcx: 'l, 'll, D: Dump + 'll> DumpVisitor<'l, 'tcx, 'll, D> {
                 visibility: vis,
                 docs: docs_for_attrs(attrs),
                 sig: None,
+                attributes: attrs.to_vec(),
             }.lower(self.tcx));
         }
 
@@ -636,6 +640,7 @@ impl<'l, 'tcx: 'l, 'll, D: Dump + 'll> DumpVisitor<'l, 'tcx, 'll, D> {
                 visibility: From::from(&item.vis),
                 docs: docs_for_attrs(&item.attrs),
                 sig: self.save_ctxt.sig_base(item),
+                attributes: item.attrs.clone(),
             }.lower(self.tcx));
         }
 
@@ -701,6 +706,7 @@ impl<'l, 'tcx: 'l, 'll, D: Dump + 'll> DumpVisitor<'l, 'tcx, 'll, D> {
                             parent: Some(make_def_id(item.id, &self.tcx.hir)),
                             docs: docs_for_attrs(&variant.node.attrs),
                             sig: sig,
+                            attributes: variant.node.attrs.clone(),
                         }.lower(self.tcx));
                     }
                 }
@@ -727,6 +733,7 @@ impl<'l, 'tcx: 'l, 'll, D: Dump + 'll> DumpVisitor<'l, 'tcx, 'll, D> {
                             parent: Some(make_def_id(item.id, &self.tcx.hir)),
                             docs: docs_for_attrs(&variant.node.attrs),
                             sig: sig,
+                            attributes: variant.node.attrs.clone(),
                         }.lower(self.tcx));
                     }
                 }
@@ -798,6 +805,7 @@ impl<'l, 'tcx: 'l, 'll, D: Dump + 'll> DumpVisitor<'l, 'tcx, 'll, D> {
                 visibility: From::from(&item.vis),
                 docs: docs_for_attrs(&item.attrs),
                 sig: self.save_ctxt.sig_base(item),
+                attributes: item.attrs.clone(),
             }.lower(self.tcx));
         }
 
@@ -1064,6 +1072,7 @@ impl<'l, 'tcx: 'l, 'll, D: Dump + 'll> DumpVisitor<'l, 'tcx, 'll, D> {
                     visibility: Visibility::Inherited,
                     docs: String::new(),
                     sig: None,
+                    attributes: vec![],
                 }.lower(self.tcx));
             }
         }
@@ -1305,6 +1314,7 @@ impl<'l, 'tcx: 'l, 'll, D: Dump +'ll> Visitor<'l> for DumpVisitor<'l, 'tcx, 'll,
                         parent: None,
                         docs: docs_for_attrs(&item.attrs),
                         sig: Some(self.save_ctxt.sig_base(item)),
+                        attributes: item.attrs.clone(),
                     }.lower(self.tcx));
                 }
 
@@ -1527,6 +1537,7 @@ impl<'l, 'tcx: 'l, 'll, D: Dump +'ll> Visitor<'l> for DumpVisitor<'l, 'tcx, 'll,
                             visibility: Visibility::Inherited,
                             docs: String::new(),
                             sig: None,
+                            attributes: vec![],
                         }.lower(self.tcx));
                     }
                 }

src/librustc_save_analysis/external_data.rs

@@ -11,8 +11,10 @@
 use rustc::hir::def_id::{CrateNum, DefId, DefIndex};
 use rustc::hir::map::Map;
 use rustc::ty::TyCtxt;
-use syntax::ast::NodeId;
+use syntax::ast::{self, NodeId};
 use syntax::codemap::CodeMap;
+use syntax::print::pprust;
+use syntax::symbol::Symbol;
 use syntax_pos::Span;
 
 use data::{self, Visibility, SigElement};
@@ -64,6 +66,39 @@ impl SpanData {
     }
 }
 
+/// Represent an arbitrary attribute on a code element
+#[derive(Clone, Debug, RustcEncodable)]
+pub struct Attribute {
+    value: String,
+    span: SpanData,
+}
+
+impl Lower for Vec<ast::Attribute> {
+    type Target = Vec<Attribute>;
+
+    fn lower(self, tcx: TyCtxt) -> Vec<Attribute> {
+        let doc = Symbol::intern("doc");
+        self.into_iter()
+        // Only retain real attributes. Doc comments are lowered separately.
+        .filter(|attr| attr.name() != doc)
+        .map(|mut attr| {
+            // Remove the surrounding '#[..]' or '#![..]' of the pretty printed
+            // attribute. First normalize all inner attribute (#![..]) to outer
+            // ones (#[..]), then remove the two leading and the one trailing character.
+            attr.style = ast::AttrStyle::Outer;
+            let value = pprust::attribute_to_string(&attr);
+            // This str slicing works correctly, because the leading and trailing characters
+            // are in the ASCII range and thus exactly one byte each.
+            let value = value[2..value.len()-1].to_string();
+
+            Attribute {
+                value: value,
+                span: SpanData::from_span(attr.span, tcx.sess.codemap()),
+            }
+        }).collect()
+    }
+}
+
 #[derive(Debug, RustcEncodable)]
 pub struct CratePreludeData {
     pub crate_name: String,
@@ -98,6 +133,7 @@ pub struct EnumData {
     pub visibility: Visibility,
     pub docs: String,
     pub sig: Signature,
+    pub attributes: Vec<Attribute>,
 }
 
 impl Lower for data::EnumData {
@@ -115,6 +151,7 @@ impl Lower for data::EnumData {
             visibility: self.visibility,
             docs: self.docs,
             sig: self.sig.lower(tcx),
+            attributes: self.attributes.lower(tcx),
         }
     }
 }
@@ -179,6 +216,7 @@ pub struct FunctionData {
     pub parent: Option<DefId>,
     pub docs: String,
     pub sig: Signature,
+    pub attributes: Vec<Attribute>,
 }
 
 impl Lower for data::FunctionData {
@@ -197,6 +235,7 @@ impl Lower for data::FunctionData {
             parent: self.parent,
             docs: self.docs,
             sig: self.sig.lower(tcx),
+            attributes: self.attributes.lower(tcx),
         }
     }
 }
@@ -346,6 +385,7 @@ pub struct MethodData {
     pub parent: Option<DefId>,
     pub docs: String,
     pub sig: Signature,
+    pub attributes: Vec<Attribute>,
 }
 
 impl Lower for data::MethodData {
@@ -364,6 +404,7 @@ impl Lower for data::MethodData {
             parent: self.parent,
             docs: self.docs,
             sig: self.sig.lower(tcx),
+            attributes: self.attributes.lower(tcx),
         }
     }
 }
@@ -381,6 +422,7 @@ pub struct ModData {
     pub visibility: Visibility,
     pub docs: String,
     pub sig: Signature,
+    pub attributes: Vec<Attribute>,
 }
 
 impl Lower for data::ModData {
@@ -398,6 +440,7 @@ impl Lower for data::ModData {
             visibility: self.visibility,
             docs: self.docs,
             sig: self.sig.lower(tcx),
+            attributes: self.attributes.lower(tcx),
         }
     }
 }
@@ -437,6 +480,7 @@ pub struct StructData {
     pub visibility: Visibility,
     pub docs: String,
     pub sig: Signature,
+    pub attributes: Vec<Attribute>,
 }
 
 impl Lower for data::StructData {
@@ -455,6 +499,7 @@ impl Lower for data::StructData {
             visibility: self.visibility,
             docs: self.docs,
             sig: self.sig.lower(tcx),
+            attributes: self.attributes.lower(tcx),
         }
     }
 }
@@ -471,6 +516,7 @@ pub struct StructVariantData {
     pub parent: Option<DefId>,
     pub docs: String,
     pub sig: Signature,
+    pub attributes: Vec<Attribute>,
 }
 
 impl Lower for data::StructVariantData {
@@ -488,6 +534,7 @@ impl Lower for data::StructVariantData {
             parent: self.parent,
             docs: self.docs,
             sig: self.sig.lower(tcx),
+            attributes: self.attributes.lower(tcx),
         }
     }
 }
@@ -504,6 +551,7 @@ pub struct TraitData {
     pub visibility: Visibility,
     pub docs: String,
     pub sig: Signature,
+    pub attributes: Vec<Attribute>,
 }
 
 impl Lower for data::TraitData {
@@ -521,6 +569,7 @@ impl Lower for data::TraitData {
             visibility: self.visibility,
             docs: self.docs,
             sig: self.sig.lower(tcx),
+            attributes: self.attributes.lower(tcx),
         }
     }
 }
@@ -537,6 +586,7 @@ pub struct TupleVariantData {
     pub parent: Option<DefId>,
     pub docs: String,
     pub sig: Signature,
+    pub attributes: Vec<Attribute>,
 }
 
 impl Lower for data::TupleVariantData {
@@ -554,6 +604,7 @@ impl Lower for data::TupleVariantData {
             parent: self.parent,
             docs: self.docs,
             sig: self.sig.lower(tcx),
+            attributes: self.attributes.lower(tcx),
         }
     }
 }
@@ -570,6 +621,7 @@ pub struct TypeDefData {
     pub parent: Option<DefId>,
     pub docs: String,
     pub sig: Option<Signature>,
+    pub attributes: Vec<Attribute>,
 }
 
 impl Lower for data::TypeDefData {
@@ -586,6 +638,7 @@ impl Lower for data::TypeDefData {
             parent: self.parent,
             docs: self.docs,
             sig: self.sig.map(|s| s.lower(tcx)),
+            attributes: self.attributes.lower(tcx),
         }
     }
 }
@@ -675,6 +728,7 @@ pub struct VariableData {
     pub visibility: Visibility,
     pub docs: String,
     pub sig: Option<Signature>,
+    pub attributes: Vec<Attribute>,
 }
 
 impl Lower for data::VariableData {
@@ -694,6 +748,7 @@ impl Lower for data::VariableData {
             visibility: self.visibility,
             docs: self.docs,
             sig: self.sig.map(|s| s.lower(tcx)),
+            attributes: self.attributes.lower(tcx),
         }
     }
 }

src/librustc_save_analysis/json_dumper.rs

@@ -87,6 +87,7 @@ impl<'b, W: Write + 'b> Dump for JsonDumper<'b, W> {
             decl_id: None,
             docs: data.docs,
             sig: Some(From::from(data.sig)),
+            attributes: data.attributes,
         };
         if def.span.file_name != def.value {
             // If the module is an out-of-line defintion, then we'll make the
@@ -232,6 +233,7 @@ struct Def {
     decl_id: Option<Id>,
     docs: String,
     sig: Option<JsonSignature>,
+    attributes: Vec<Attribute>,
 }
 
 #[derive(Debug, RustcEncodable)]
@@ -274,6 +276,7 @@ impl From<EnumData> for Def {
             decl_id: None,
             docs: data.docs,
             sig: Some(From::from(data.sig)),
+            attributes: data.attributes,
         }
     }
 }
@@ -291,6 +294,7 @@ impl From<TupleVariantData> for Def {
             decl_id: None,
             docs: data.docs,
             sig: Some(From::from(data.sig)),
+            attributes: data.attributes,
         }
     }
 }
@@ -307,6 +311,7 @@ impl From<StructVariantData> for Def {
             decl_id: None,
             docs: data.docs,
             sig: Some(From::from(data.sig)),
+            attributes: data.attributes,
         }
     }
 }
@@ -323,6 +328,7 @@ impl From<StructData> for Def {
             decl_id: None,
             docs: data.docs,
             sig: Some(From::from(data.sig)),
+            attributes: data.attributes,
         }
     }
 }
@@ -339,6 +345,7 @@ impl From<TraitData> for Def {
             decl_id: None,
             docs: data.docs,
             sig: Some(From::from(data.sig)),
+            attributes: data.attributes,
         }
     }
 }
@@ -355,6 +362,7 @@ impl From<FunctionData> for Def {
             decl_id: None,
             docs: data.docs,
             sig: Some(From::from(data.sig)),
+            attributes: data.attributes,
         }
     }
 }
@@ -371,6 +379,7 @@ impl From<MethodData> for Def {
             decl_id: data.decl_id.map(|id| From::from(id)),
             docs: data.docs,
             sig: Some(From::from(data.sig)),
+            attributes: data.attributes,
         }
     }
 }
@@ -387,6 +396,7 @@ impl From<MacroData> for Def {
             decl_id: None,
             docs: data.docs,
             sig: None,
+            attributes: vec![],
         }
     }
 }
@@ -403,6 +413,7 @@ impl From<TypeDefData> for Def {
             decl_id: None,
             docs: String::new(),
             sig: data.sig.map(|s| From::from(s)),
+            attributes: data.attributes,
         }
     }
 }
@@ -424,6 +435,7 @@ impl From<VariableData> for Def {
             decl_id: None,
             docs: data.docs,
             sig: None,
+            attributes: data.attributes,
         }
     }
 }

src/librustc_save_analysis/lib.rs

@@ -136,6 +136,7 @@ impl<'l, 'tcx: 'l> SaveContext<'l, 'tcx> {
                     parent: None,
                     docs: docs_for_attrs(&item.attrs),
                     sig: self.sig_base(item),
+                    attributes: item.attrs.clone(),
                 }))
             }
             ast::ItemKind::Static(ref typ, mt, ref expr) => {
@@ -164,6 +165,7 @@ impl<'l, 'tcx: 'l> SaveContext<'l, 'tcx> {
                     visibility: From::from(&item.vis),
                     docs: docs_for_attrs(&item.attrs),
                     sig: Some(self.sig_base(item)),
+                    attributes: item.attrs.clone(),
                 }))
             }
             ast::ItemKind::Const(ref typ, ref expr) => {
@@ -183,6 +185,7 @@ impl<'l, 'tcx: 'l> SaveContext<'l, 'tcx> {
                     visibility: From::from(&item.vis),
                     docs: docs_for_attrs(&item.attrs),
                     sig: Some(self.sig_base(item)),
+                    attributes: item.attrs.clone(),
                 }))
             }
             ast::ItemKind::Mod(ref m) => {
@@ -205,6 +208,7 @@ impl<'l, 'tcx: 'l> SaveContext<'l, 'tcx> {
                     visibility: From::from(&item.vis),
                     docs: docs_for_attrs(&item.attrs),
                     sig: self.sig_base(item),
+                    attributes: item.attrs.clone(),
                 }))
             }
             ast::ItemKind::Enum(ref def, _) => {
@@ -228,6 +232,7 @@ impl<'l, 'tcx: 'l> SaveContext<'l, 'tcx> {
                     visibility: From::from(&item.vis),
                     docs: docs_for_attrs(&item.attrs),
                     sig: self.sig_base(item),
+                    attributes: item.attrs.clone(),
                 }))
             }
             ast::ItemKind::Impl(.., ref trait_ref, ref typ, _) => {
@@ -315,6 +320,7 @@ impl<'l, 'tcx: 'l> SaveContext<'l, 'tcx> {
                 visibility: From::from(&field.vis),
                 docs: docs_for_attrs(&field.attrs),
                 sig: Some(sig),
+                attributes: field.attrs.clone(),
             })
         } else {
             None
@@ -327,7 +333,7 @@ impl<'l, 'tcx: 'l> SaveContext<'l, 'tcx> {
                            name: ast::Name, span: Span) -> Option<FunctionData> {
         // The qualname for a method is the trait name or name of the struct in an impl in
         // which the method is declared in, followed by the method's name.
-        let (qualname, parent_scope, decl_id, vis, docs) =
+        let (qualname, parent_scope, decl_id, vis, docs, attributes) =
           match self.tcx.impl_of_method(self.tcx.hir.local_def_id(id)) {
             Some(impl_id) => match self.tcx.hir.get_if_local(impl_id) {
                 Some(Node::NodeItem(item)) => {
@@ -349,7 +355,8 @@ impl<'l, 'tcx: 'l> SaveContext<'l, 'tcx> {
 
                             (result, trait_id, decl_id,
                              From::from(&item.vis),
-                             docs_for_attrs(&item.attrs))
+                             docs_for_attrs(&item.attrs),
+                             item.attrs.to_vec())
                         }
                         _ => {
                             span_bug!(span,
@@ -374,7 +381,8 @@ impl<'l, 'tcx: 'l> SaveContext<'l, 'tcx> {
                             (format!("::{}", self.tcx.item_path_str(def_id)),
                              Some(def_id), None,
                              From::from(&item.vis),
-                             docs_for_attrs(&item.attrs))
+                             docs_for_attrs(&item.attrs),
+                             item.attrs.to_vec())
                         }
                         r => {
                             span_bug!(span,
@@ -423,6 +431,7 @@ impl<'l, 'tcx: 'l> SaveContext<'l, 'tcx> {
             parent: parent_scope,
             docs: docs,
             sig: sig,
+            attributes: attributes,
         })
     }