Auto merge of #15616 - HKalbasi:rustc-deps, r=HKalbasi

Switch to in-tree rustc dependencies with a cfg flag

We can use this flag to detect and prevent breakages in rustc CI. (see #14846 and #15569)

~The `IN_RUSTC_REPOSITORY` is just a placeholder. Is there any existing cfg flag that rustc CI sets?~

Cargo.lock

@@ -531,8 +531,6 @@ dependencies = [
  "fst",
  "hashbrown 0.12.3",
  "hir-expand",
- "hkalbasi-rustc-ap-rustc_abi",
- "hkalbasi-rustc-ap-rustc_index",
  "indexmap 2.0.0",
  "intern",
  "itertools",
@@ -541,7 +539,7 @@ dependencies = [
  "mbe",
  "once_cell",
  "profile",
- "ra-ap-rustc_parse_format",
+ "rustc-dependencies",
  "rustc-hash",
  "smallvec",
  "stdx",
@@ -594,7 +592,6 @@ dependencies = [
  "expect-test",
  "hir-def",
  "hir-expand",
- "hkalbasi-rustc-ap-rustc_index",
  "intern",
  "itertools",
  "la-arena 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -604,6 +601,7 @@ dependencies = [
  "oorandom",
  "profile",
  "project-model",
+ "rustc-dependencies",
  "rustc-hash",
  "scoped-tls",
  "smallvec",
@@ -1277,7 +1275,7 @@ dependencies = [
  "drop_bomb",
  "expect-test",
  "limit",
- "ra-ap-rustc_lexer",
+ "rustc-dependencies",
  "sourcegen",
  "stdx",
 ]
@@ -1594,10 +1592,12 @@ dependencies = [
  "oorandom",
  "parking_lot 0.12.1",
  "parking_lot_core 0.9.6",
+ "parser",
  "proc-macro-api",
  "profile",
  "project-model",
  "rayon",
+ "rustc-dependencies",
  "rustc-hash",
  "scip",
  "serde",
@@ -1626,6 +1626,16 @@ version = "0.1.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
 
+[[package]]
+name = "rustc-dependencies"
+version = "0.0.0"
+dependencies = [
+ "hkalbasi-rustc-ap-rustc_abi",
+ "hkalbasi-rustc-ap-rustc_index",
+ "ra-ap-rustc_lexer",
+ "ra-ap-rustc_parse_format",
+]
+
 [[package]]
 name = "rustc-hash"
 version = "1.1.0"
@@ -1853,9 +1863,9 @@ dependencies = [
  "proc-macro2",
  "profile",
  "quote",
- "ra-ap-rustc_lexer",
  "rayon",
  "rowan",
+ "rustc-dependencies",
  "rustc-hash",
  "smol_str",
  "sourcegen",

Cargo.toml

@@ -79,6 +79,7 @@ toolchain = { path = "./crates/toolchain", version = "0.0.0" }
 tt = { path = "./crates/tt", version = "0.0.0" }
 vfs-notify = { path = "./crates/vfs-notify", version = "0.0.0" }
 vfs = { path = "./crates/vfs", version = "0.0.0" }
+rustc-dependencies = { path = "./crates/rustc-dependencies", version = "0.0.0" }
 
 # local crates that aren't published to crates.io. These should not have versions.
 proc-macro-test = { path = "./crates/proc-macro-test" }
@@ -90,9 +91,9 @@ lsp-server = { version = "0.7.4" }
 
 # non-local crates
 smallvec = { version = "1.10.0", features = [
-  "const_new",
+    "const_new",
-  "union",
+    "union",
-  "const_generics",
+    "const_generics",
 ] }
 smol_str = "0.2.0"
 nohash-hasher = "0.2.0"
@@ -101,11 +102,6 @@ serde = { version = "1.0.156", features = ["derive"] }
 serde_json = "1.0.96"
 triomphe = { version = "0.1.8", default-features = false, features = ["std"] }
 # can't upgrade due to dashmap depending on 0.12.3 currently
-hashbrown = { version = "0.12.3", features = ["inline-more"], default-features = false }
+hashbrown = { version = "0.12.3", features = [
-
+    "inline-more",
-rustc_lexer = { version = "0.10.0", package = "ra-ap-rustc_lexer" }
+], default-features = false }
-rustc_parse_format = { version = "0.10.0", package = "ra-ap-rustc_parse_format", default-features = false }
-
-# Upstream broke this for us so we can't update it
-rustc_abi = { version = "0.0.20221221", package = "hkalbasi-rustc-ap-rustc_abi", default-features = false }
-rustc_index = { version = "0.0.20221221", package = "hkalbasi-rustc-ap-rustc_index", default-features = false }

crates/hir-def/Cargo.toml

@@ -31,10 +31,7 @@ smallvec.workspace = true
 hashbrown.workspace = true
 triomphe.workspace = true
 
-rustc_abi.workspace = true
+rustc-dependencies.workspace = true
-rustc_index.workspace = true
-rustc_parse_format.workspace = true
-
 
 # local deps
 stdx.workspace = true
@@ -53,3 +50,6 @@ expect-test = "1.4.0"
 
 # local deps
 test-utils.workspace = true
+
+[features]
+in-rust-tree = ["rustc-dependencies/in-rust-tree"]

crates/hir-def/src/data/adt.rs

@@ -11,7 +11,7 @@ use hir_expand::{
 };
 use intern::Interned;
 use la_arena::{Arena, ArenaMap};
-use rustc_abi::{Align, Integer, IntegerType, ReprFlags, ReprOptions};
+use rustc_dependencies::abi::{Align, Integer, IntegerType, ReprFlags, ReprOptions};
 use syntax::ast::{self, HasName, HasVisibility};
 use triomphe::Arc;
 

crates/hir-def/src/hir/format_args.rs

@@ -2,7 +2,7 @@
 use std::mem;
 
 use hir_expand::name::Name;
-use rustc_parse_format as parse;
+use rustc_dependencies::parse_format as parse;
 use syntax::{
     ast::{self, IsString},
     AstToken, SmolStr, TextRange,

crates/hir-def/src/lib.rs

@@ -8,6 +8,7 @@
 //! actually true.
 
 #![warn(rust_2018_idioms, unused_lifetimes, semicolon_in_expressions_from_macros)]
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
 
 #[allow(unused)]
 macro_rules! eprintln {
@@ -48,7 +49,7 @@ pub mod visibility;
 pub mod find_path;
 pub mod import_map;
 
-pub use rustc_abi as layout;
+pub use rustc_dependencies::abi as layout;
 use triomphe::Arc;
 
 #[cfg(test)]

crates/hir-ty/Cargo.toml

@@ -33,7 +33,7 @@ triomphe.workspace = true
 nohash-hasher.workspace = true
 typed-arena = "2.0.1"
 
-rustc_index.workspace = true
+rustc-dependencies.workspace = true
 
 # local deps
 stdx.workspace = true
@@ -56,3 +56,6 @@ project-model = { path = "../project-model" }
 
 # local deps
 test-utils.workspace = true
+
+[features]
+in-rust-tree = ["rustc-dependencies/in-rust-tree"]

crates/hir-ty/src/layout.rs

@@ -34,7 +34,7 @@ mod target;
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub struct RustcEnumVariantIdx(pub LocalEnumVariantId);
 
-impl rustc_index::vec::Idx for RustcEnumVariantIdx {
+impl rustc_dependencies::index::vec::Idx for RustcEnumVariantIdx {
     fn new(idx: usize) -> Self {
         RustcEnumVariantIdx(Idx::from_raw(RawIdx::from(idx as u32)))
     }

crates/parser/Cargo.toml

@@ -13,7 +13,7 @@ doctest = false
 
 [dependencies]
 drop_bomb = "0.1.5"
-rustc_lexer.workspace = true
+rustc-dependencies.workspace = true
 
 limit.workspace = true
 
@@ -22,3 +22,6 @@ expect-test = "1.4.0"
 
 stdx.workspace = true
 sourcegen.workspace = true
+
+[features]
+in-rust-tree = ["rustc-dependencies/in-rust-tree"]

crates/parser/src/lexed_str.rs

@@ -8,6 +8,7 @@
 //! Note that these tokens, unlike the tokens we feed into the parser, do
 //! include info about comments and whitespace.
 
+use rustc_dependencies::lexer as rustc_lexer;
 use std::ops;
 
 use crate::{

crates/parser/src/lib.rs

@@ -19,6 +19,7 @@
 
 #![warn(rust_2018_idioms, unused_lifetimes, semicolon_in_expressions_from_macros)]
 #![allow(rustdoc::private_intra_doc_links)]
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
 
 mod lexed_str;
 mod token_set;

crates/rust-analyzer/Cargo.toml

@@ -57,6 +57,7 @@ flycheck.workspace = true
 hir-def.workspace = true
 hir-ty.workspace = true
 hir.workspace = true
+rustc-dependencies.workspace = true
 ide-db.workspace = true
 # This should only be used in CLI
 ide-ssr.workspace = true
@@ -67,6 +68,7 @@ profile.workspace = true
 project-model.workspace = true
 stdx.workspace = true
 syntax.workspace = true
+parser.workspace = true
 toolchain.workspace = true
 vfs-notify.workspace = true
 vfs.workspace = true
@@ -89,4 +91,12 @@ mbe.workspace = true
 jemalloc = ["jemallocator", "profile/jemalloc"]
 force-always-assert = ["always-assert/force"]
 sysroot-abi = []
-in-rust-tree = ["sysroot-abi", "ide/in-rust-tree", "syntax/in-rust-tree"]
+in-rust-tree = [
+    "sysroot-abi",
+    "ide/in-rust-tree",
+    "syntax/in-rust-tree",
+    "parser/in-rust-tree",
+    "rustc-dependencies/in-rust-tree",
+    "hir-def/in-rust-tree",
+    "hir-ty/in-rust-tree",
+]

crates/rustc-dependencies/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "rustc-dependencies"
+version = "0.0.0"
+rust-version.workspace = true
+edition.workspace = true
+license.workspace = true
+authors.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+ra-ap-rustc_lexer = { version = "0.10.0" }
+ra-ap-rustc_parse_format = { version = "0.10.0", default-features = false }
+
+# Upstream broke this for us so we can't update it
+hkalbasi-rustc-ap-rustc_abi = { version = "0.0.20221221", default-features = false }
+hkalbasi-rustc-ap-rustc_index = { version = "0.0.20221221", default-features = false }
+
+[features]
+in-rust-tree = []

crates/rustc-dependencies/src/lib.rs

@@ -0,0 +1,39 @@
+//! A wrapper around rustc internal crates, which enables switching between compiler provided
+//! ones and stable ones published in crates.io
+
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+
+#[cfg(feature = "in-rust-tree")]
+extern crate rustc_lexer;
+
+#[cfg(feature = "in-rust-tree")]
+pub mod lexer {
+    pub use ::rustc_lexer::*;
+}
+
+#[cfg(not(feature = "in-rust-tree"))]
+pub mod lexer {
+    pub use ::ra_ap_rustc_lexer::*;
+}
+
+#[cfg(feature = "in-rust-tree")]
+extern crate rustc_parse_format;
+
+#[cfg(feature = "in-rust-tree")]
+pub mod parse_format {
+    pub use ::rustc_parse_format::*;
+}
+
+#[cfg(not(feature = "in-rust-tree"))]
+pub mod parse_format {
+    pub use ::ra_ap_rustc_parse_format::*;
+}
+
+// Upstream broke this for us so we can't update it
+pub mod abi {
+    pub use ::hkalbasi_rustc_ap_rustc_abi::*;
+}
+
+pub mod index {
+    pub use ::hkalbasi_rustc_ap_rustc_index::*;
+}

crates/syntax/Cargo.toml

@@ -23,7 +23,7 @@ indexmap = "2.0.0"
 smol_str.workspace = true
 triomphe.workspace = true
 
-rustc_lexer.workspace = true
+rustc-dependencies.workspace = true
 
 parser.workspace = true
 profile.workspace = true
@@ -41,4 +41,4 @@ test-utils.workspace = true
 sourcegen.workspace = true
 
 [features]
-in-rust-tree = []
+in-rust-tree = ["rustc-dependencies/in-rust-tree"]

crates/syntax/src/ast/token_ext.rs

@@ -2,6 +2,8 @@
 
 use std::borrow::Cow;
 
+use rustc_dependencies::lexer as rustc_lexer;
+
 use rustc_lexer::unescape::{
     unescape_byte, unescape_c_string, unescape_char, unescape_literal, CStrUnit, Mode,
 };

crates/syntax/src/lib.rs

@@ -19,6 +19,7 @@
 //! [RFC]: <https://github.com/rust-lang/rfcs/pull/2256>
 //! [Swift]: <https://github.com/apple/swift/blob/13d593df6f359d0cb2fc81cfaac273297c539455/lib/Syntax/README.md>
 
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
 #![warn(rust_2018_idioms, unused_lifetimes, semicolon_in_expressions_from_macros)]
 
 #[allow(unused)]

crates/syntax/src/validation.rs

@@ -5,7 +5,7 @@
 mod block;
 
 use rowan::Direction;
-use rustc_lexer::unescape::{self, unescape_literal, Mode};
+use rustc_dependencies::lexer::unescape::{self, unescape_literal, Mode};
 
 use crate::{
     algo,