Cleanup feature generation

crates/ide/src/completion.rs

@@ -3,6 +3,7 @@ mod completion_item;
 mod completion_context;
 mod presentation;
 mod patterns;
+mod generated_features;
 #[cfg(test)]
 mod test_utils;
 
@@ -18,7 +19,6 @@ mod complete_unqualified_path;
 mod complete_postfix;
 mod complete_macro_in_item_position;
 mod complete_trait_impl;
-mod unstable_feature_descriptor;
 
 use ide_db::RootDatabase;
 
@@ -30,11 +30,6 @@ use crate::{
     FilePosition,
 };
 
-//FIXME: cyclic imports caused by xtask generation, this should be better
-use crate::completion::{
-    complete_attribute::LintCompletion, unstable_feature_descriptor::UNSTABLE_FEATURE_DESCRIPTOR,
-};
-
 pub use crate::completion::{
     completion_config::CompletionConfig,
     completion_item::{CompletionItem, CompletionItemKind, CompletionScore, InsertTextFormat},

crates/ide/src/completion/complete_attribute.rs

@@ -9,7 +9,7 @@ use syntax::{ast, AstNode, SyntaxKind};
 use crate::completion::{
     completion_context::CompletionContext,
     completion_item::{CompletionItem, CompletionItemKind, CompletionKind, Completions},
-    UNSTABLE_FEATURE_DESCRIPTOR,
+    generated_features::FEATURES,
 };
 
 pub(super) fn complete_attribute(acc: &mut Completions, ctx: &CompletionContext) -> Option<()> {
@@ -19,7 +19,7 @@ pub(super) fn complete_attribute(acc: &mut Completions, ctx: &CompletionContext)
             complete_derive(acc, ctx, token_tree)
         }
         (Some(path), Some(token_tree)) if path.to_string() == "feature" => {
-            complete_lint(acc, ctx, token_tree, UNSTABLE_FEATURE_DESCRIPTOR)
+            complete_lint(acc, ctx, token_tree, FEATURES)
         }
         (Some(path), Some(token_tree))
             if ["allow", "warn", "deny", "forbid"]
@@ -237,7 +237,7 @@ fn get_derive_names_in_scope(ctx: &CompletionContext) -> FxHashSet<String> {
     result
 }
 
-pub(crate) struct DeriveCompletion {
+struct DeriveCompletion {
     label: &'static str,
     dependencies: &'static [&'static str],
 }
@@ -257,9 +257,9 @@ const DEFAULT_DERIVE_COMPLETIONS: &[DeriveCompletion] = &[
     DeriveCompletion { label: "Ord", dependencies: &["PartialOrd", "Eq", "PartialEq"] },
 ];
 
-pub(crate) struct LintCompletion {
-    pub(crate) label: &'static str,
-    pub(crate) description: &'static str,
+pub(super) struct LintCompletion {
+    pub(super) label: &'static str,
+    pub(super) description: &'static str,
 }
 
 #[rustfmt::skip]

xtask/src/codegen.rs

@@ -9,7 +9,7 @@ mod gen_syntax;
 mod gen_parser_tests;
 mod gen_assists_docs;
 mod gen_feature_docs;
-mod gen_unstable_future_descriptor;
+mod gen_features;
 
 use std::{
     fmt, mem,
@@ -25,35 +25,35 @@ use crate::{
 pub use self::{
     gen_assists_docs::{generate_assists_docs, generate_assists_tests},
     gen_feature_docs::generate_feature_docs,
+    gen_features::generate_features,
     gen_parser_tests::generate_parser_tests,
     gen_syntax::generate_syntax,
-    gen_unstable_future_descriptor::generate_unstable_future_descriptor,
 };
 
-// Directory used by xtask
-const STORAGE: &str = ".xtask";
-
-const GRAMMAR_DIR: &str = "crates/parser/src/grammar";
-const OK_INLINE_TESTS_DIR: &str = "crates/syntax/test_data/parser/inline/ok";
-const ERR_INLINE_TESTS_DIR: &str = "crates/syntax/test_data/parser/inline/err";
-
-const SYNTAX_KINDS: &str = "crates/parser/src/syntax_kind/generated.rs";
-const AST_NODES: &str = "crates/syntax/src/ast/generated/nodes.rs";
-const AST_TOKENS: &str = "crates/syntax/src/ast/generated/tokens.rs";
-
-const ASSISTS_DIR: &str = "crates/assists/src/handlers";
-const ASSISTS_TESTS: &str = "crates/assists/src/tests/generated.rs";
-
-const REPOSITORY_URL: &str = "https://github.com/rust-lang/rust";
-const UNSTABLE_FEATURE: &str = "crates/ide/src/completion/unstable_feature_descriptor.rs";
-const REPO_PATH: &str = "src/doc/unstable-book/src";
-
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 pub enum Mode {
     Overwrite,
     Verify,
 }
 
+pub struct CodegenCmd {
+    pub features: bool,
+}
+
+impl CodegenCmd {
+    pub fn run(self) -> Result<()> {
+        if self.features {
+            generate_features(Mode::Overwrite)?;
+        }
+        generate_syntax(Mode::Overwrite)?;
+        generate_parser_tests(Mode::Overwrite)?;
+        generate_assists_tests(Mode::Overwrite)?;
+        generate_assists_docs(Mode::Overwrite)?;
+        generate_feature_docs(Mode::Overwrite)?;
+        Ok(())
+    }
+}
+
 /// A helper to update file on disk if it has changed.
 /// With verify = false,
 fn update(path: &Path, contents: &str, mode: Mode) -> Result<()> {

xtask/src/codegen/gen_assists_docs.rs

@@ -32,7 +32,7 @@ struct Assist {
 impl Assist {
     fn collect() -> Result<Vec<Assist>> {
         let mut res = Vec::new();
-        for path in rust_files(&project_root().join(codegen::ASSISTS_DIR)) {
+        for path in rust_files(&project_root().join("crates/assists/src/handlers")) {
             collect_file(&mut res, path.as_path())?;
         }
         res.sort_by(|lhs, rhs| lhs.id.cmp(&rhs.id));
@@ -135,7 +135,7 @@ r#####"
         buf.push_str(&test)
     }
     let buf = reformat(buf)?;
-    codegen::update(&project_root().join(codegen::ASSISTS_TESTS), &buf, mode)
+    codegen::update(&project_root().join("crates/assists/src/tests/generated.rs"), &buf, mode)
 }
 
 fn hide_hash_comments(text: &str) -> String {

xtask/src/codegen/gen_features.rs

@@ -0,0 +1,50 @@
+//! Generates descriptors structure for unstable feature from Unstable Book
+use std::path::{Path, PathBuf};
+
+use quote::quote;
+use walkdir::WalkDir;
+
+use crate::{
+    codegen::{project_root, reformat, update, Mode, Result},
+    not_bash::{fs2, run},
+};
+
+pub fn generate_features(mode: Mode) -> Result<()> {
+    if !Path::new("./target/rust").exists() {
+        run!("git clone https://github.com/rust-lang/rust ./target/rust")?;
+    }
+
+    let contents = generate_descriptor("./target/rust/src/doc/unstable-book/src".into())?;
+
+    let destination = project_root().join("crates/ide/src/completion/generated_features.rs");
+    update(destination.as_path(), &contents, mode)?;
+
+    Ok(())
+}
+
+fn generate_descriptor(src_dir: PathBuf) -> Result<String> {
+    let definitions = ["language-features", "library-features"]
+        .iter()
+        .flat_map(|it| WalkDir::new(src_dir.join(it)))
+        .filter_map(|e| e.ok())
+        .filter(|entry| {
+            // Get all `.md ` files
+            entry.file_type().is_file() && entry.path().extension().unwrap_or_default() == "md"
+        })
+        .map(|entry| {
+            let path = entry.path();
+            let feature_ident = path.file_stem().unwrap().to_str().unwrap().replace("-", "_");
+            let doc = fs2::read_to_string(path).unwrap();
+
+            quote! { LintCompletion { label: #feature_ident, description: #doc } }
+        });
+
+    let ts = quote! {
+        use crate::completion::complete_attribute::LintCompletion;
+
+        pub(super) const FEATURES:  &[LintCompletion] = &[
+            #(#definitions),*
+        ];
+    };
+    reformat(ts)
+}

xtask/src/codegen/gen_parser_tests.rs

@@ -8,12 +8,12 @@ use std::{
 };
 
 use crate::{
-    codegen::{self, extract_comment_blocks, update, Mode},
+    codegen::{extract_comment_blocks, update, Mode},
     project_root, Result,
 };
 
 pub fn generate_parser_tests(mode: Mode) -> Result<()> {
-    let tests = tests_from_dir(&project_root().join(Path::new(codegen::GRAMMAR_DIR)))?;
+    let tests = tests_from_dir(&project_root().join(Path::new("crates/parser/src/grammar")))?;
     fn install_tests(tests: &HashMap<String, Test>, into: &str, mode: Mode) -> Result<()> {
         let tests_dir = project_root().join(into);
         if !tests_dir.is_dir() {
@@ -39,8 +39,8 @@ pub fn generate_parser_tests(mode: Mode) -> Result<()> {
         }
         Ok(())
     }
-    install_tests(&tests.ok, codegen::OK_INLINE_TESTS_DIR, mode)?;
-    install_tests(&tests.err, codegen::ERR_INLINE_TESTS_DIR, mode)
+    install_tests(&tests.ok, "crates/syntax/test_data/parser/inline/ok", mode)?;
+    install_tests(&tests.err, "crates/syntax/test_data/parser/inline/err", mode)
 }
 
 #[derive(Debug)]

xtask/src/codegen/gen_syntax.rs

@@ -14,7 +14,7 @@ use ungrammar::{rust_grammar, Grammar, Rule};
 
 use crate::{
     ast_src::{AstEnumSrc, AstNodeSrc, AstSrc, Cardinality, Field, KindsSrc, KINDS_SRC},
-    codegen::{self, reformat, update, Mode},
+    codegen::{reformat, update, Mode},
     project_root, Result,
 };
 
@@ -22,15 +22,15 @@ pub fn generate_syntax(mode: Mode) -> Result<()> {
     let grammar = rust_grammar();
     let ast = lower(&grammar);
 
-    let syntax_kinds_file = project_root().join(codegen::SYNTAX_KINDS);
+    let syntax_kinds_file = project_root().join("crates/parser/src/syntax_kind/generated.rs");
     let syntax_kinds = generate_syntax_kinds(KINDS_SRC)?;
     update(syntax_kinds_file.as_path(), &syntax_kinds, mode)?;
 
-    let ast_tokens_file = project_root().join(codegen::AST_TOKENS);
+    let ast_tokens_file = project_root().join("crates/syntax/src/ast/generated/tokens.rs");
     let contents = generate_tokens(&ast)?;
     update(ast_tokens_file.as_path(), &contents, mode)?;
 
-    let ast_nodes_file = project_root().join(codegen::AST_NODES);
+    let ast_nodes_file = project_root().join("crates/syntax/src/ast/generated/nodes.rs");
     let contents = generate_nodes(KINDS_SRC, &ast)?;
     update(ast_nodes_file.as_path(), &contents, mode)?;
 

xtask/src/codegen/gen_unstable_future_descriptor.rs

@@ -1,61 +0,0 @@
-//! Generates descriptors structure for unstable feature from Unstable Book
-
-use crate::codegen::{self, project_root, Mode, Result};
-use crate::codegen::{reformat, update};
-use crate::not_bash::{fs2, pushd, run};
-use proc_macro2::TokenStream;
-use quote::quote;
-use std::path::PathBuf;
-use walkdir::WalkDir;
-
-fn generate_descriptor(src_dir: PathBuf) -> Result<TokenStream> {
-    let files = WalkDir::new(src_dir.join("language-features"))
-        .into_iter()
-        .chain(WalkDir::new(src_dir.join("library-features")))
-        .filter_map(|e| e.ok())
-        .filter(|entry| {
-            // Get all `.md ` files
-            entry.file_type().is_file() && entry.path().extension().unwrap_or_default() == "md"
-        })
-        .collect::<Vec<_>>();
-
-    let definitions = files
-        .iter()
-        .map(|entry| {
-            let path = entry.path();
-            let feature_ident =
-                format!("{}", path.file_stem().unwrap().to_str().unwrap().replace("-", "_"));
-            let doc = format!("{}", std::fs::read_to_string(path).unwrap());
-
-            quote! { LintCompletion { label: #feature_ident, description: #doc } }
-        })
-        .collect::<Vec<_>>();
-
-    let ts = quote! {
-        use crate::completion::LintCompletion;
-
-        pub(crate) const UNSTABLE_FEATURE_DESCRIPTOR:  &[LintCompletion] = &[
-            #(#definitions),*
-        ];
-    };
-    Ok(ts)
-}
-
-pub fn generate_unstable_future_descriptor(mode: Mode) -> Result<()> {
-    let path = project_root().join(codegen::STORAGE);
-    fs2::create_dir_all(path.clone())?;
-
-    let _d = pushd(path.clone());
-    run!("git init")?;
-    run!("git remote add -f origin {}", codegen::REPOSITORY_URL)?;
-    run!("git pull origin master")?;
-
-    let src_dir = path.join(codegen::REPO_PATH);
-    let content = generate_descriptor(src_dir)?.to_string();
-
-    let contents = reformat(content)?;
-    let destination = project_root().join(codegen::UNSTABLE_FEATURE);
-    update(destination.as_path(), &contents, mode)?;
-
-    Ok(())
-}

xtask/src/main.rs

@@ -10,6 +10,7 @@
 
 use std::env;
 
+use codegen::CodegenCmd;
 use pico_args::Arguments;
 use xtask::{
     codegen::{self, Mode},
@@ -75,14 +76,9 @@ FLAGS:
             .run()
         }
         "codegen" => {
+            let features = args.contains("--features");
             args.finish()?;
-            codegen::generate_syntax(Mode::Overwrite)?;
-            codegen::generate_unstable_future_descriptor(Mode::Overwrite)?;
-            codegen::generate_parser_tests(Mode::Overwrite)?;
-            codegen::generate_assists_tests(Mode::Overwrite)?;
-            codegen::generate_assists_docs(Mode::Overwrite)?;
-            codegen::generate_feature_docs(Mode::Overwrite)?;
-            Ok(())
+            CodegenCmd { features }.run()
         }
         "format" => {
             args.finish()?;

xtask/tests/tidy.rs

@@ -113,7 +113,7 @@ fn check_todo(path: &Path, text: &str) {
         // `ast::make`.
         "ast/make.rs",
         // The documentation in string literals may contain anything for its own purposes
-        "completion/unstable_feature_descriptor.rs",
+        "completion/generated_features.rs",
     ];
     if need_todo.iter().any(|p| path.ends_with(p)) {
         return;