Extract tags from test fixtures during parsing

Summary:
The goal of this change is to ensure we can extract "tags" (used to specify ranges for folding, highlighting or other) from multi-file test fixtures.

The original implementation of `extract_tags` was ported directly from Rust Analyzer, but it only supported extracting tags from a single-file fixture.

Adding multi-file support is non trivial, though. In the current `check` function, we extract the tags from a snippet with no *metadata* lines, then generate a DB with the version of the snippet with the tags removed. The moment we want to support multiple files, we have a problem: we'd need to run the `extract_tags` function on each snippet, without the metadata lines, or the ranges would be off.

We therefore move tags extraction to the parsing stage. In a test fixture, the user can specify an optional `tag` attribute (`fold` or tag` in the tests below), that can be used to extract tags from the snippet. The tags are stored in the `ChangeFixture`, so they can be later retrieved by `FileId`. In the test fixture used for the actual test, the tags are actually removed, so that the snippet still contains valid syntax. This is necessary for the ranges calculated from the snippet to coincide with the actual ranges (which would otherwise would be different due to syntax errors in the file).

This ultimately allows us to write the test for `test_doc_attributes_with_included_file`, which highlights a current bug: ranges for definitions belonging to included files are reported for the incorrect file. This will be fixed as a follow-up diff.

Reviewed By: alanz

Differential Revision: D67160534

fbshipit-source-id: 4d61d5766e6fd973035285138eae342fe6d11108

crates/base_db/src/fixture.rs

@@ -103,6 +103,7 @@ pub struct ChangeFixture {
     pub files: Vec<FileId>,
     pub files_by_path: FxHashMap<VfsPath, FileId>,
     pub diagnostics_enabled: DiagnosticsEnabled,
+    pub tags: FxHashMap<FileId, Vec<(TextRange, Option<String>)>>,
 }
 
 struct Builder {
@@ -188,6 +189,7 @@ impl ChangeFixture {
         let mut otp: Option<Otp> = None;
         let mut app_files = SourceRootMap::default();
         let mut files_by_path: FxHashMap<VfsPath, FileId> = FxHashMap::default();
+        let mut tags: FxHashMap<FileId, Vec<(TextRange, Option<String>)>> = FxHashMap::default();
 
         for entry in fixture.clone() {
             let (text, file_pos) = Self::get_text_and_pos(&entry.text, file_id);
@@ -225,6 +227,7 @@ impl ChangeFixture {
             app_files.insert(app_name, file_id, path.clone());
             files_by_path.insert(path, file_id);
             files.push(file_id);
+            tags.insert(file_id, entry.tags);
 
             inc_file_id(&mut file_id);
         }
@@ -343,6 +346,7 @@ impl ChangeFixture {
                 files,
                 files_by_path,
                 diagnostics_enabled,
+                tags,
             },
             change,
             project,
@@ -517,73 +521,6 @@ fn try_extract_range(text: &str) -> Option<(TextRange, String)> {
     Some((TextRange::new(start, end), text))
 }
 
-/// Extracts ranges, marked with `<tag> </tag>` pairs from the `text`
-pub fn extract_tags(mut text: &str, tag: &str) -> (Vec<(TextRange, Option<String>)>, String) {
-    let open = format!("<{tag}");
-    let close = format!("</{tag}>");
-    let mut ranges = Vec::new();
-    let mut res = String::new();
-    let mut stack = Vec::new();
-    loop {
-        match text.find('<') {
-            None => {
-                res.push_str(text);
-                break;
-            }
-            Some(i) => {
-                res.push_str(&text[..i]);
-                text = &text[i..];
-                if text.starts_with(&open) {
-                    let close_open = text.find('>').unwrap();
-                    let attr = text[open.len()..close_open].trim();
-                    let attr = if attr.is_empty() {
-                        None
-                    } else {
-                        Some(attr.to_string())
-                    };
-                    text = &text[close_open + '>'.len_utf8()..];
-                    let from = TextSize::of(&res);
-                    stack.push((from, attr));
-                } else if text.starts_with(&close) {
-                    text = &text[close.len()..];
-                    let (from, attr) = stack.pop().unwrap_or_else(|| panic!("unmatched </{tag}>"));
-                    let to = TextSize::of(&res);
-                    ranges.push((TextRange::new(from, to), attr));
-                } else {
-                    res.push('<');
-                    text = &text['<'.len_utf8()..];
-                }
-            }
-        }
-    }
-    assert!(stack.is_empty(), "unmatched <{}>", tag);
-    ranges.sort_by_key(|r| (r.0.start(), r.0.end()));
-    (ranges, res)
-}
-
-#[test]
-fn test_extract_tags_1() {
-    let (tags, text) = extract_tags(r#"<tag region>foo() -> ok.</tag>"#, "tag");
-    let actual = tags
-        .into_iter()
-        .map(|(range, attr)| (&text[range], attr))
-        .collect::<Vec<_>>();
-    assert_eq!(actual, vec![("foo() -> ok.", Some("region".into()))]);
-}
-
-#[test]
-fn test_extract_tags_2() {
-    let (tags, text) = extract_tags(
-        r#"bar() -> ok.\n<tag region>foo() -> ok.</tag>\nbaz() -> ok."#,
-        "tag",
-    );
-    let actual = tags
-        .into_iter()
-        .map(|(range, attr)| (&text[range], attr))
-        .collect::<Vec<_>>();
-    assert_eq!(actual, vec![("foo() -> ok.", Some("region".into()))]);
-}
-
 #[derive(Clone, Copy, Debug)]
 pub enum RangeOrOffset {
     Range(TextRange),

crates/ide/src/folding_ranges.rs

@@ -118,15 +118,18 @@ pub(crate) fn folding_ranges(db: &RootDatabase, file_id: FileId) -> Vec<FoldingR
 
 #[cfg(test)]
 mod tests {
-    use elp_ide_db::elp_base_db::fixture::extract_tags;
+    use elp_ide_db::elp_base_db::fixture::WithFixture;
+    use stdx::trim_indent;
 
     use super::*;
-    use crate::fixture;
 
     fn check(fixture: &str) {
-        let (ranges, fixture) = extract_tags(fixture.trim_start(), "fold");
-        let (analysis, file_id) = fixture::single_file(&fixture);
-        let mut folding_ranges = analysis.folding_ranges(file_id).unwrap_or_default();
+        let fixture = trim_indent(fixture);
+        let (db, fixture) = RootDatabase::with_fixture(&fixture);
+        let ranges = fixture.tags.get(&fixture.file_id()).unwrap().clone();
+        let file_id = fixture.file_id();
+
+        let mut folding_ranges = folding_ranges(&db, file_id);
         folding_ranges
             .sort_by_key(|folding_range| (folding_range.range.start(), folding_range.range.end()));
 
@@ -162,6 +165,7 @@ mod tests {
     fn test_function() {
         check(
             r#"
+//- /src/my_module tag:fold
 -module(my_module).
 <fold region>one() ->
   ok.</fold>
@@ -173,6 +177,7 @@ mod tests {
     fn test_record() {
         check(
             r#"
+//- /src/my_module tag:fold
 -module(my_module).
 <fold region>-record(my_record, {a :: integer(), b :: binary()}).</fold>
 "#,
@@ -183,6 +188,7 @@ mod tests {
     fn test_records_and_functions() {
         check(
             r#"
+//- /src/my_module tag:fold
 -module(my_module).
 
 <fold region>-record(my_record, {a :: integer(),
@@ -202,6 +208,7 @@ mod tests {
     fn test_module_doc_attributes() {
         check(
             r#"
+//- /src/my_module tag:fold
 -module(my_module).
 <fold region>-moduledoc """
 This is a module doc
@@ -218,6 +225,7 @@ This is a module doc
     fn test_doc_attributes() {
         check(
             r#"
+//- /src/my_module tag:fold
 -module(my_module).
 
 -export([one/0]).
@@ -226,6 +234,25 @@ This is a module doc
 This is one function
 ".</fold>
 <fold region>one() -> 1.</fold>
+"#,
+        );
+    }
+
+    #[test]
+    fn test_doc_attributes_with_included_file() {
+        check(
+            r#"
+//- /src/my_header.hrl
+-record(my_record, {a :: integer()}).
+//- /src/my_module.erl tag:fold
+<fold region>                                     </fold>
+-module(my_module).
+-export([one/0]).
+-include("my_header.hrl").
+<fold region>-doc "
+This is one function
+".</fold>
+<fold region>one() -> 1.</fold>~
 "#,
         );
     }

crates/ide/src/syntax_highlighting.rs

@@ -299,7 +299,6 @@ fn is_dynamic(t: &Type) -> bool {
 mod tests {
     use elp_base_db::fixture::WithFixture;
     use elp_ide_db::elp_base_db;
-    use elp_ide_db::elp_base_db::fixture::extract_tags;
     use elp_ide_db::EqwalizerDatabase;
     use elp_ide_db::RootDatabase;
     use elp_project_model::otp::otp_supported_by_eqwalizer;
@@ -324,14 +323,13 @@ mod tests {
     #[track_caller]
     fn do_check_highlights(fixture: &str, provide_types: bool) {
         let fixture = trim_indent(fixture);
-        let (ranges, fixture) = extract_tags(fixture.trim_start(), "tag");
+        let (db, fixture) = RootDatabase::with_fixture(&fixture);
+        let ranges = fixture.tags.get(&fixture.file_id()).unwrap();
         let range = if !ranges.is_empty() {
             Some(ranges[0].0)
         } else {
             None
         };
-
-        let (db, fixture) = RootDatabase::with_fixture(&fixture);
         let annotations = fixture.annotations(&db);
         let expected: Vec<_> = annotations
             .into_iter()
@@ -399,7 +397,7 @@ mod tests {
     fn deprecated_highlight() {
         check_highlights(
             r#"
-              //- /src/deprecated_highlight.erl
+           //- /src/deprecated_highlight.erl
               -module(deprecated_highlight).
               -deprecated([{f, 1}]).
               f(1) -> 1.
@@ -413,6 +411,8 @@ mod tests {
     fn highlights_in_range() {
         check_highlights(
             r#"
+           //- /src/highlights_in_range.erl tag:tag
+              -module(highlights_in_range).
               -export([f/1]).
               foo(X) -> ok.
               f(Var1) ->

crates/project_model/Cargo.toml

@@ -27,6 +27,7 @@ serde.workspace = true
 serde_json.workspace = true
 stdx.workspace = true
 tempfile.workspace = true
+text-size.workspace = true
 thiserror.workspace = true
 toml.workspace = true
 

crates/project_model/src/test_fixture.rs

@@ -91,6 +91,8 @@ use paths::Utf8Path;
 use paths::Utf8PathBuf;
 pub use stdx::trim_indent;
 use tempfile::tempdir;
+use text_size::TextRange;
+use text_size::TextSize;
 
 use crate::otp::Otp;
 use crate::temp_dir::TempDir;
@@ -105,6 +107,8 @@ pub struct Fixture {
     pub app_data: ProjectAppData,
     pub otp: Option<Otp>,
     pub scratch_buffer: Option<PathBuf>,
+    pub tag: Option<String>,
+    pub tags: Vec<(TextRange, Option<String>)>,
 }
 
 #[derive(Clone, Debug, Default)]
@@ -276,6 +280,14 @@ impl FixtureWithProjectMeta {
             }
         }
 
+        for fixture in &mut res {
+            if let Some(tag) = &fixture.tag {
+                let (tags, text) = extract_tags(&fixture.text, tag);
+                fixture.tags = tags;
+                fixture.text = text;
+            }
+        }
+
         FixtureWithProjectMeta {
             fixture: res,
             diagnostics_enabled,
@@ -321,6 +333,7 @@ impl FixtureWithProjectMeta {
         let mut extra_dirs = Vec::new();
         let mut otp = None;
         let mut scratch_buffer = None;
+        let mut tag = None;
 
         for component in components[1..].iter() {
             let (key, value) = component
@@ -353,6 +366,9 @@ impl FixtureWithProjectMeta {
                     path = tmp_path.to_str().unwrap().to_string();
                     scratch_buffer = Some(tmp_path);
                 }
+                "tag" => {
+                    tag = Some(value.to_string());
+                }
                 _ => panic!("bad component: {:?}", component),
             }
         }
@@ -386,10 +402,56 @@ impl FixtureWithProjectMeta {
             app_data,
             otp,
             scratch_buffer,
+            tag,
+            tags: Vec::new(),
         }
     }
 }
 
+/// Extracts ranges, marked with `<tag> </tag>` pairs from the `text`
+pub fn extract_tags(mut text: &str, tag: &str) -> (Vec<(TextRange, Option<String>)>, String) {
+    let open = format!("<{tag}");
+    let close = format!("</{tag}>");
+    let mut ranges = Vec::new();
+    let mut res = String::new();
+    let mut stack = Vec::new();
+    loop {
+        match text.find('<') {
+            None => {
+                res.push_str(text);
+                break;
+            }
+            Some(i) => {
+                res.push_str(&text[..i]);
+                text = &text[i..];
+                if text.starts_with(&open) {
+                    let close_open = text.find('>').unwrap();
+                    let attr = text[open.len()..close_open].trim();
+                    let attr = if attr.is_empty() {
+                        None
+                    } else {
+                        Some(attr.to_string())
+                    };
+                    text = &text[close_open + '>'.len_utf8()..];
+                    let from = TextSize::of(&res);
+                    stack.push((from, attr));
+                } else if text.starts_with(&close) {
+                    text = &text[close.len()..];
+                    let (from, attr) = stack.pop().unwrap_or_else(|| panic!("unmatched </{tag}>"));
+                    let to = TextSize::of(&res);
+                    ranges.push((TextRange::new(from, to), attr));
+                } else {
+                    res.push('<');
+                    text = &text['<'.len_utf8()..];
+                }
+            }
+        }
+    }
+    assert!(stack.is_empty(), "unmatched <{}>", tag);
+    ranges.sort_by_key(|r| (r.0.start(), r.0.end()));
+    (ranges, res)
+}
+
 // ---------------------------------------------------------------------
 
 #[cfg(test)]
@@ -550,3 +612,26 @@ bar() -> ok.
         .assert_eq(format!("{:#?}", meta0.app_data).as_str());
     }
 }
+
+#[test]
+fn test_extract_tags_1() {
+    let (tags, text) = extract_tags(r#"<tag region>foo() -> ok.</tag>"#, "tag");
+    let actual = tags
+        .into_iter()
+        .map(|(range, attr)| (&text[range], attr))
+        .collect::<Vec<_>>();
+    assert_eq!(actual, vec![("foo() -> ok.", Some("region".into()))]);
+}
+
+#[test]
+fn test_extract_tags_2() {
+    let (tags, text) = extract_tags(
+        r#"bar() -> ok.\n<tag region>foo() -> ok.</tag>\nbaz() -> ok."#,
+        "tag",
+    );
+    let actual = tags
+        .into_iter()
+        .map(|(range, attr)| (&text[range], attr))
+        .collect::<Vec<_>>();
+    assert_eq!(actual, vec![("foo() -> ok.", Some("region".into()))]);
+}