Auto merge of #12976 - tjdevries:scip, r=Veykril

feat: emit SCIP from rust-analyzer

hi rust-analyzer team

I'm one of the engineers at Sourcegraph (and have done a few small changes related to the LSIF work done in rust-analyzer). Recently, we've moved to a new protocol as the primary way to interact with Sourcegraph (LSIF is still possible to upload, so existing jobs will not stop working any time soon). This new protocol is SCIP (I linked a blog post below with more information).

I've implemented SCIP support (based largely on the existing LSIF support). In addition to supporting the existing features that `rust-analyzer`'s LSIF support does, this PR adds the ability to move between crates on sourcegraph.com. So if both your project and a dependency are indexed, you would be able to hop to the particular version and view the source code. I'd be happy to record a demo of that on my local instance if you're interested.

There are a few TODO's left in the code (some that you might have insights on) which I'm happy to fix in this PR, but I just wanted to open this up for discussion first.

Thanks for your time :)

TJ

- [announcing scip](https://about.sourcegraph.com/blog/announcing-scip)
This commit is contained in:
bors 2022-08-23 07:53:36 +00:00
commit b2bf37cdde
8 changed files with 630 additions and 14 deletions

50
Cargo.lock generated
View File

@ -1229,6 +1229,26 @@ dependencies = [
"tracing",
]
[[package]]
name = "protobuf"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ee4a7d8b91800c8f167a6268d1a1026607368e1adc84e98fe044aeb905302f7"
dependencies = [
"once_cell",
"protobuf-support",
"thiserror",
]
[[package]]
name = "protobuf-support"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ca157fe12fc7ee2e315f2f735e27df41b3d97cdd70ea112824dac1ffb08ee1c"
dependencies = [
"thiserror",
]
[[package]]
name = "pulldown-cmark"
version = "0.9.2"
@ -1360,6 +1380,7 @@ dependencies = [
"project-model",
"rayon",
"rustc-hash",
"scip",
"serde",
"serde_json",
"sourcegen",
@ -1446,6 +1467,15 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "scip"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2bfbb10286f69fad7c78db71004b7839bf957788359fe0c479f029f9849136b"
dependencies = [
"protobuf",
]
[[package]]
name = "scoped-tls"
version = "1.0.0"
@ -1631,6 +1661,26 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "288cb548dbe72b652243ea797201f3d481a0609a967980fcc5b2315ea811560a"
[[package]]
name = "thiserror"
version = "1.0.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "thread_local"
version = "1.1.4"

View File

@ -87,7 +87,7 @@ pub use crate::{
},
join_lines::JoinLinesConfig,
markup::Markup,
moniker::{MonikerKind, MonikerResult, PackageInformation},
moniker::{MonikerDescriptorKind, MonikerKind, MonikerResult, PackageInformation},
move_item::Direction,
navigation_target::NavigationTarget,
prime_caches::ParallelPrimeCachesProgress,

View File

@ -13,17 +13,39 @@ use syntax::{AstNode, SyntaxKind::*, T};
use crate::{doc_links::token_as_doc_comment, RangeInfo};
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum MonikerDescriptorKind {
Namespace,
Type,
Term,
Method,
TypeParameter,
Parameter,
Macro,
Meta,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct MonikerDescriptor {
pub name: Name,
pub desc: MonikerDescriptorKind,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct MonikerIdentifier {
crate_name: String,
path: Vec<Name>,
pub crate_name: String,
pub description: Vec<MonikerDescriptor>,
}
impl ToString for MonikerIdentifier {
fn to_string(&self) -> String {
match self {
MonikerIdentifier { path, crate_name } => {
format!("{}::{}", crate_name, path.iter().map(|x| x.to_string()).join("::"))
MonikerIdentifier { description, crate_name } => {
format!(
"{}::{}",
crate_name,
description.iter().map(|x| x.name.to_string()).join("::")
)
}
}
}
@ -42,6 +64,12 @@ pub struct MonikerResult {
pub package_information: PackageInformation,
}
impl MonikerResult {
pub fn from_def(db: &RootDatabase, def: Definition, from_crate: Crate) -> Option<Self> {
def_to_moniker(db, def, from_crate)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct PackageInformation {
pub name: String,
@ -105,13 +133,23 @@ pub(crate) fn def_to_moniker(
def: Definition,
from_crate: Crate,
) -> Option<MonikerResult> {
if matches!(def, Definition::GenericParam(_) | Definition::SelfType(_) | Definition::Local(_)) {
if matches!(
def,
Definition::GenericParam(_)
| Definition::Label(_)
| Definition::DeriveHelper(_)
| Definition::BuiltinAttr(_)
| Definition::ToolModule(_)
) {
return None;
}
let module = def.module(db)?;
let krate = module.krate();
let mut path = vec![];
path.extend(module.path_to_root(db).into_iter().filter_map(|x| x.name(db)));
let mut description = vec![];
description.extend(module.path_to_root(db).into_iter().filter_map(|x| {
Some(MonikerDescriptor { name: x.name(db)?, desc: MonikerDescriptorKind::Namespace })
}));
// Handle associated items within a trait
if let Some(assoc) = def.as_assoc_item(db) {
@ -120,31 +158,98 @@ pub(crate) fn def_to_moniker(
AssocItemContainer::Trait(trait_) => {
// Because different traits can have functions with the same name,
// we have to include the trait name as part of the moniker for uniqueness.
path.push(trait_.name(db));
description.push(MonikerDescriptor {
name: trait_.name(db),
desc: MonikerDescriptorKind::Type,
});
}
AssocItemContainer::Impl(impl_) => {
// Because a struct can implement multiple traits, for implementations
// we add both the struct name and the trait name to the path
if let Some(adt) = impl_.self_ty(db).as_adt() {
path.push(adt.name(db));
description.push(MonikerDescriptor {
name: adt.name(db),
desc: MonikerDescriptorKind::Type,
});
}
if let Some(trait_) = impl_.trait_(db) {
path.push(trait_.name(db));
description.push(MonikerDescriptor {
name: trait_.name(db),
desc: MonikerDescriptorKind::Type,
});
}
}
}
}
if let Definition::Field(it) = def {
path.push(it.parent_def(db).name(db));
description.push(MonikerDescriptor {
name: it.parent_def(db).name(db),
desc: MonikerDescriptorKind::Type,
});
}
path.push(def.name(db)?);
let name_desc = match def {
// These are handled by top-level guard (for performance).
Definition::GenericParam(_)
| Definition::Label(_)
| Definition::DeriveHelper(_)
| Definition::BuiltinAttr(_)
| Definition::ToolModule(_) => return None,
Definition::Local(local) => {
if !local.is_param(db) {
return None;
}
MonikerDescriptor { name: local.name(db), desc: MonikerDescriptorKind::Parameter }
}
Definition::Macro(m) => {
MonikerDescriptor { name: m.name(db), desc: MonikerDescriptorKind::Macro }
}
Definition::Function(f) => {
MonikerDescriptor { name: f.name(db), desc: MonikerDescriptorKind::Method }
}
Definition::Variant(v) => {
MonikerDescriptor { name: v.name(db), desc: MonikerDescriptorKind::Type }
}
Definition::Const(c) => {
MonikerDescriptor { name: c.name(db)?, desc: MonikerDescriptorKind::Term }
}
Definition::Trait(trait_) => {
MonikerDescriptor { name: trait_.name(db), desc: MonikerDescriptorKind::Type }
}
Definition::TypeAlias(ta) => {
MonikerDescriptor { name: ta.name(db), desc: MonikerDescriptorKind::TypeParameter }
}
Definition::Module(m) => {
MonikerDescriptor { name: m.name(db)?, desc: MonikerDescriptorKind::Namespace }
}
Definition::BuiltinType(b) => {
MonikerDescriptor { name: b.name(), desc: MonikerDescriptorKind::Type }
}
Definition::SelfType(imp) => MonikerDescriptor {
name: imp.self_ty(db).as_adt()?.name(db),
desc: MonikerDescriptorKind::Type,
},
Definition::Field(it) => {
MonikerDescriptor { name: it.name(db), desc: MonikerDescriptorKind::Term }
}
Definition::Adt(adt) => {
MonikerDescriptor { name: adt.name(db), desc: MonikerDescriptorKind::Type }
}
Definition::Static(s) => {
MonikerDescriptor { name: s.name(db), desc: MonikerDescriptorKind::Meta }
}
};
description.push(name_desc);
Some(MonikerResult {
identifier: MonikerIdentifier {
crate_name: krate.display_name(db)?.crate_name().to_string(),
path,
description,
},
kind: if krate == from_crate { MonikerKind::Export } else { MonikerKind::Import },
package_information: {

View File

@ -23,6 +23,7 @@ crossbeam-channel = "0.5.5"
dissimilar = "1.0.4"
itertools = "0.10.3"
lsp-types = { version = "0.93.0", features = ["proposed"] }
scip = "0.1.1"
parking_lot = "0.12.1"
xflags = "0.2.4"
oorandom = "11.1.3"

View File

@ -93,6 +93,7 @@ fn try_main() -> Result<()> {
flags::RustAnalyzerCmd::Ssr(cmd) => cmd.run()?,
flags::RustAnalyzerCmd::Search(cmd) => cmd.run()?,
flags::RustAnalyzerCmd::Lsif(cmd) => cmd.run()?,
flags::RustAnalyzerCmd::Scip(cmd) => cmd.run()?,
}
Ok(())
}

View File

@ -9,6 +9,7 @@ mod analysis_stats;
mod diagnostics;
mod ssr;
mod lsif;
mod scip;
mod progress_report;

View File

@ -112,6 +112,10 @@ xflags::xflags! {
cmd lsif
required path: PathBuf
{}
cmd scip
required path: PathBuf
{}
}
}
@ -140,6 +144,7 @@ pub enum RustAnalyzerCmd {
Search(Search),
ProcMacro(ProcMacro),
Lsif(Lsif),
Scip(Scip),
}
#[derive(Debug)]
@ -207,6 +212,11 @@ pub struct Lsif {
pub path: PathBuf,
}
#[derive(Debug)]
pub struct Scip {
pub path: PathBuf,
}
impl RustAnalyzer {
pub const HELP: &'static str = Self::HELP_;

View File

@ -0,0 +1,448 @@
//! SCIP generator
use std::{
collections::{HashMap, HashSet},
time::Instant,
};
use crate::line_index::{LineEndings, LineIndex, OffsetEncoding};
use hir::Name;
use ide::{
LineCol, MonikerDescriptorKind, MonikerResult, StaticIndex, StaticIndexedFile, TextRange,
TokenId,
};
use ide_db::LineIndexDatabase;
use project_model::{CargoConfig, ProjectManifest, ProjectWorkspace};
use scip::types as scip_types;
use std::env;
use crate::cli::{
flags,
load_cargo::{load_workspace, LoadCargoConfig},
Result,
};
impl flags::Scip {
pub fn run(self) -> Result<()> {
eprintln!("Generating SCIP start...");
let now = Instant::now();
let cargo_config = CargoConfig::default();
let no_progress = &|s| (eprintln!("rust-analyzer: Loading {}", s));
let load_cargo_config = LoadCargoConfig {
load_out_dirs_from_check: true,
with_proc_macro: true,
prefill_caches: true,
};
let path = vfs::AbsPathBuf::assert(env::current_dir()?.join(&self.path));
let rootpath = path.normalize();
let manifest = ProjectManifest::discover_single(&path)?;
let workspace = ProjectWorkspace::load(manifest, &cargo_config, no_progress)?;
let (host, vfs, _) = load_workspace(workspace, &load_cargo_config)?;
let db = host.raw_database();
let analysis = host.analysis();
let si = StaticIndex::compute(&analysis);
let mut index = scip_types::Index {
metadata: Some(scip_types::Metadata {
version: scip_types::ProtocolVersion::UnspecifiedProtocolVersion.into(),
tool_info: Some(scip_types::ToolInfo {
name: "rust-analyzer".to_owned(),
version: "0.1".to_owned(),
arguments: vec![],
..Default::default()
})
.into(),
project_root: format!(
"file://{}",
path.normalize()
.as_os_str()
.to_str()
.ok_or(anyhow::anyhow!("Unable to normalize project_root path"))?
.to_string()
),
text_document_encoding: scip_types::TextEncoding::UTF8.into(),
..Default::default()
})
.into(),
..Default::default()
};
let mut symbols_emitted: HashSet<TokenId> = HashSet::default();
let mut tokens_to_symbol: HashMap<TokenId, String> = HashMap::new();
for file in si.files {
let mut local_count = 0;
let mut new_local_symbol = || {
let new_symbol = scip::types::Symbol::new_local(local_count);
local_count += 1;
new_symbol
};
let StaticIndexedFile { file_id, tokens, .. } = file;
let relative_path = match get_relative_filepath(&vfs, &rootpath, file_id) {
Some(relative_path) => relative_path,
None => continue,
};
let line_index = LineIndex {
index: db.line_index(file_id),
encoding: OffsetEncoding::Utf8,
endings: LineEndings::Unix,
};
let mut doc = scip_types::Document {
relative_path,
language: "rust".to_string(),
..Default::default()
};
tokens.into_iter().for_each(|(range, id)| {
let token = si.tokens.get(id).unwrap();
let mut occurrence = scip_types::Occurrence::default();
occurrence.range = text_range_to_scip_range(&line_index, range);
occurrence.symbol = match tokens_to_symbol.get(&id) {
Some(symbol) => symbol.clone(),
None => {
let symbol = match &token.moniker {
Some(moniker) => moniker_to_symbol(&moniker),
None => new_local_symbol(),
};
let symbol = scip::symbol::format_symbol(symbol);
tokens_to_symbol.insert(id, symbol.clone());
symbol
}
};
if let Some(def) = token.definition {
if def.range == range {
occurrence.symbol_roles |= scip_types::SymbolRole::Definition as i32;
}
if !symbols_emitted.contains(&id) {
symbols_emitted.insert(id);
let mut symbol_info = scip_types::SymbolInformation::default();
symbol_info.symbol = occurrence.symbol.clone();
if let Some(hover) = &token.hover {
if !hover.markup.as_str().is_empty() {
symbol_info.documentation = vec![hover.markup.as_str().to_string()];
}
}
doc.symbols.push(symbol_info)
}
}
doc.occurrences.push(occurrence);
});
if doc.occurrences.is_empty() {
continue;
}
index.documents.push(doc);
}
scip::write_message_to_file("index.scip", index)
.map_err(|err| anyhow::anyhow!("Failed to write scip to file: {}", err))?;
eprintln!("Generating SCIP finished {:?}", now.elapsed());
Ok(())
}
}
fn get_relative_filepath(
vfs: &vfs::Vfs,
rootpath: &vfs::AbsPathBuf,
file_id: ide::FileId,
) -> Option<String> {
Some(vfs.file_path(file_id).as_path()?.strip_prefix(&rootpath)?.as_ref().to_str()?.to_string())
}
// SCIP Ranges have a (very large) optimization that ranges if they are on the same line
// only encode as a vector of [start_line, start_col, end_col].
//
// This transforms a line index into the optimized SCIP Range.
fn text_range_to_scip_range(line_index: &LineIndex, range: TextRange) -> Vec<i32> {
let LineCol { line: start_line, col: start_col } = line_index.index.line_col(range.start());
let LineCol { line: end_line, col: end_col } = line_index.index.line_col(range.end());
if start_line == end_line {
vec![start_line as i32, start_col as i32, end_col as i32]
} else {
vec![start_line as i32, start_col as i32, end_line as i32, end_col as i32]
}
}
fn new_descriptor_str(
name: &str,
suffix: scip_types::descriptor::Suffix,
) -> scip_types::Descriptor {
scip_types::Descriptor {
name: name.to_string(),
disambiguator: "".to_string(),
suffix: suffix.into(),
..Default::default()
}
}
fn new_descriptor(name: Name, suffix: scip_types::descriptor::Suffix) -> scip_types::Descriptor {
let mut name = name.to_string();
if name.contains("'") {
name = format!("`{}`", name);
}
new_descriptor_str(name.as_str(), suffix)
}
/// Loosely based on `def_to_moniker`
///
/// Only returns a Symbol when it's a non-local symbol.
/// So if the visibility isn't outside of a document, then it will return None
fn moniker_to_symbol(moniker: &MonikerResult) -> scip_types::Symbol {
use scip_types::descriptor::Suffix::*;
let package_name = moniker.package_information.name.clone();
let version = moniker.package_information.version.clone();
let descriptors = moniker
.identifier
.description
.iter()
.map(|desc| {
new_descriptor(
desc.name.clone(),
match desc.desc {
MonikerDescriptorKind::Namespace => Namespace,
MonikerDescriptorKind::Type => Type,
MonikerDescriptorKind::Term => Term,
MonikerDescriptorKind::Method => Method,
MonikerDescriptorKind::TypeParameter => TypeParameter,
MonikerDescriptorKind::Parameter => Parameter,
MonikerDescriptorKind::Macro => Macro,
MonikerDescriptorKind::Meta => Meta,
},
)
})
.collect();
scip_types::Symbol {
scheme: "rust-analyzer".into(),
package: Some(scip_types::Package {
manager: "cargo".to_string(),
name: package_name,
version,
..Default::default()
})
.into(),
descriptors,
..Default::default()
}
}
#[cfg(test)]
mod test {
use super::*;
use hir::Semantics;
use ide::{AnalysisHost, FilePosition};
use ide_db::defs::IdentClass;
use ide_db::{base_db::fixture::ChangeFixture, helpers::pick_best_token};
use scip::symbol::format_symbol;
use syntax::SyntaxKind::*;
use syntax::{AstNode, T};
fn position(ra_fixture: &str) -> (AnalysisHost, FilePosition) {
let mut host = AnalysisHost::default();
let change_fixture = ChangeFixture::parse(ra_fixture);
host.raw_database_mut().apply_change(change_fixture.change);
let (file_id, range_or_offset) =
change_fixture.file_position.expect("expected a marker ($0)");
let offset = range_or_offset.expect_offset();
(host, FilePosition { file_id, offset })
}
/// If expected == "", then assert that there are no symbols (this is basically local symbol)
#[track_caller]
fn check_symbol(ra_fixture: &str, expected: &str) {
let (host, position) = position(ra_fixture);
let FilePosition { file_id, offset } = position;
let db = host.raw_database();
let sema = &Semantics::new(db);
let file = sema.parse(file_id).syntax().clone();
let original_token = pick_best_token(file.token_at_offset(offset), |kind| match kind {
IDENT
| INT_NUMBER
| LIFETIME_IDENT
| T![self]
| T![super]
| T![crate]
| T![Self]
| COMMENT => 2,
kind if kind.is_trivia() => 0,
_ => 1,
})
.expect("OK OK");
let navs = sema
.descend_into_macros(original_token.clone())
.into_iter()
.filter_map(|token| {
IdentClass::classify_token(sema, &token).map(IdentClass::definitions).map(|it| {
it.into_iter().flat_map(|def| {
let module = def.module(db).unwrap();
let current_crate = module.krate();
match MonikerResult::from_def(sema.db, def, current_crate) {
Some(moniker_result) => Some(moniker_to_symbol(&moniker_result)),
None => None,
}
})
})
})
.flatten()
.collect::<Vec<_>>();
if expected == "" {
assert_eq!(0, navs.len(), "must have no symbols {:?}", navs);
return;
}
assert_eq!(1, navs.len(), "must have one symbol {:?}", navs);
let res = navs.get(0).unwrap();
let formatted = format_symbol(res.clone());
assert_eq!(formatted, expected);
}
#[test]
fn basic() {
check_symbol(
r#"
//- /lib.rs crate:main deps:foo
use foo::example_mod::func;
fn main() {
func$0();
}
//- /foo/lib.rs crate:foo@CratesIo:0.1.0,https://a.b/foo.git
pub mod example_mod {
pub fn func() {}
}
"#,
"rust-analyzer cargo foo 0.1.0 example_mod/func().",
);
}
#[test]
fn symbol_for_trait() {
check_symbol(
r#"
//- /foo/lib.rs crate:foo@CratesIo:0.1.0,https://a.b/foo.git
pub mod module {
pub trait MyTrait {
pub fn func$0() {}
}
}
"#,
"rust-analyzer cargo foo 0.1.0 module/MyTrait#func().",
);
}
#[test]
fn symbol_for_trait_constant() {
check_symbol(
r#"
//- /foo/lib.rs crate:foo@CratesIo:0.1.0,https://a.b/foo.git
pub mod module {
pub trait MyTrait {
const MY_CONST$0: u8;
}
}
"#,
"rust-analyzer cargo foo 0.1.0 module/MyTrait#MY_CONST.",
);
}
#[test]
fn symbol_for_trait_type() {
check_symbol(
r#"
//- /foo/lib.rs crate:foo@CratesIo:0.1.0,https://a.b/foo.git
pub mod module {
pub trait MyTrait {
type MyType$0;
}
}
"#,
// "foo::module::MyTrait::MyType",
"rust-analyzer cargo foo 0.1.0 module/MyTrait#[MyType]",
);
}
#[test]
fn symbol_for_trait_impl_function() {
check_symbol(
r#"
//- /foo/lib.rs crate:foo@CratesIo:0.1.0,https://a.b/foo.git
pub mod module {
pub trait MyTrait {
pub fn func() {}
}
struct MyStruct {}
impl MyTrait for MyStruct {
pub fn func$0() {}
}
}
"#,
// "foo::module::MyStruct::MyTrait::func",
"rust-analyzer cargo foo 0.1.0 module/MyStruct#MyTrait#func().",
);
}
#[test]
fn symbol_for_field() {
check_symbol(
r#"
//- /lib.rs crate:main deps:foo
use foo::St;
fn main() {
let x = St { a$0: 2 };
}
//- /foo/lib.rs crate:foo@CratesIo:0.1.0,https://a.b/foo.git
pub struct St {
pub a: i32,
}
"#,
"rust-analyzer cargo foo 0.1.0 St#a.",
);
}
#[test]
fn local_symbol_for_local() {
check_symbol(
r#"
//- /lib.rs crate:main deps:foo
use foo::module::func;
fn main() {
func();
}
//- /foo/lib.rs crate:foo@CratesIo:0.1.0,https://a.b/foo.git
pub mod module {
pub fn func() {
let x$0 = 2;
}
}
"#,
"",
);
}
}