Merge pull request #529 from posit-dev/feature/bump-tree-sitter-5

Bump tree-sitter-r 5 - Leading newline skipping, tree-sitter 0.23.0 update

crates/ark/Cargo.toml

@@ -47,8 +47,8 @@ serde_json = { version = "1.0.94", features = ["preserve_order"]}
 stdext = { path = "../stdext" }
 tokio = { version = "1.26.0", features = ["full"] }
 tower-lsp = "0.19.0"
-tree-sitter = "0.22.6"
-tree-sitter-r = { git = "https://github.com/r-lib/tree-sitter-r", rev = "99bf614d9d7e6ac9c7445fa7dc54a590fcdf3ce0" }
+tree-sitter = "0.23.0"
+tree-sitter-r = { git = "https://github.com/r-lib/tree-sitter-r", rev = "2097fa502efa21349d26af0ffee55d773015e481" }
 uuid = "1.3.0"
 url = "2.4.1"
 walkdir = "2"

crates/ark/src/lsp/diagnostics_syntactic.rs

@@ -478,12 +478,17 @@ identity(1)
 
     #[test]
     fn test_error_precision() {
+        // The actual error is up to tree-sitter-r's error recovery,
+        // but it should always be decent
         let diagnostics = text_diagnostics("sum(1 * 2 + )");
         assert_eq!(diagnostics.len(), 1);
         let diagnostic = diagnostics.get(0).unwrap();
-        assert!(diagnostic.message.starts_with("Syntax error"));
-        assert_eq!(diagnostic.range.start, Position::new(0, 10));
-        assert_eq!(diagnostic.range.end, Position::new(0, 11));
+        assert_eq!(
+            diagnostic.message,
+            "Unmatched closing token. Missing an opening '('."
+        );
+        assert_eq!(diagnostic.range.start, Position::new(0, 12));
+        assert_eq!(diagnostic.range.end, Position::new(0, 13));
     }
 
     #[test]
@@ -522,6 +527,8 @@ identity(1)
 
     #[test]
     fn test_unmatched_binary_operator() {
+        // The actual error is up to tree-sitter-r's error recovery,
+        // but it should always be decent
         let text = "
 {
  1 +
@@ -533,24 +540,32 @@ identity(1)
         let diagnostic = diagnostics.get(0).unwrap();
         assert_eq!(
             diagnostic.message,
-            String::from("Invalid binary operator '+'. Missing a right hand side.")
+            String::from("Unmatched closing token. Missing an opening '{'.")
         );
-        assert_eq!(diagnostic.range.start, Position::new(2, 3));
-        assert_eq!(diagnostic.range.end, Position::new(2, 4));
+        assert_eq!(diagnostic.range.start, Position::new(3, 0));
+        assert_eq!(diagnostic.range.end, Position::new(3, 1));
     }
 
     #[test]
     fn test_unmatched_function_parameters_parentheses() {
+        // Exact set of diagnostics are up to tree-sitter-r's error recovery,
+        // but they should be decent at pointing you to the right place
         let text = "
 function(x {
+  1 + 1
 }";
 
         let diagnostics = text_diagnostics(text);
-        assert_eq!(diagnostics.len(), 1);
+        assert_eq!(diagnostics.len(), 2);
 
         let diagnostic = diagnostics.get(0).unwrap();
-        assert!(diagnostic.message.starts_with("Unmatched opening token"));
-        assert_eq!(diagnostic.range.start, Position::new(1, 8));
-        assert_eq!(diagnostic.range.end, Position::new(1, 9));
+        assert!(diagnostic.message.starts_with("Syntax error"));
+        assert_eq!(diagnostic.range.start, Position::new(1, 11));
+        assert_eq!(diagnostic.range.end, Position::new(1, 12));
+
+        let diagnostic = diagnostics.get(1).unwrap();
+        assert!(diagnostic.message.starts_with("Unmatched closing token"));
+        assert_eq!(diagnostic.range.start, Position::new(3, 0));
+        assert_eq!(diagnostic.range.end, Position::new(3, 1));
     }
 }

crates/ark/src/lsp/documents.rs

@@ -64,9 +64,10 @@ impl Document {
     pub fn new(contents: &str, version: Option<i32>) -> Self {
         // A one-shot parser, assumes the `Document` won't be incrementally reparsed.
         // Useful for testing, `with_document()`, and `index_file()`.
-        let language = tree_sitter_r::language();
         let mut parser = Parser::new();
-        parser.set_language(&language).unwrap();
+        parser
+            .set_language(&tree_sitter_r::LANGUAGE.into())
+            .unwrap();
 
         Self::new_with_parser(contents, &mut parser, version)
     }
@@ -224,4 +225,11 @@ mod tests {
         let point = compute_point(Point::new(0, 0), "abcdefghi\n");
         assert_eq!(point, Point::new(1, 0));
     }
+
+    #[test]
+    fn test_document_starts_at_0_0_with_leading_whitespace() {
+        let document = Document::new("\n\n# hi there", None);
+        let root = document.ast.root_node();
+        assert_eq!(root.start_position(), Point::new(0, 0));
+    }
 }

crates/ark/src/lsp/help_topic.rs

@@ -105,11 +105,9 @@ mod tests {
 
     #[test]
     fn test_locate_help_node() {
-        let language = tree_sitter_r::language();
-
         let mut parser = Parser::new();
         parser
-            .set_language(&language)
+            .set_language(&tree_sitter_r::LANGUAGE.into())
             .expect("failed to create parser");
 
         // On the RHS

crates/ark/src/lsp/selection_range.rs

@@ -139,11 +139,9 @@ mod tests {
 
         let (text, point) = point_from_cursor(text);
 
-        let language = tree_sitter_r::language();
-
         let mut parser = Parser::new();
         parser
-            .set_language(&language)
+            .set_language(&tree_sitter_r::LANGUAGE.into())
             .expect("failed to create parser");
 
         let tree = parser.parse(text, None).unwrap();
@@ -158,7 +156,7 @@ mod tests {
         assert_eq!(selection.range.end_point, Point::new(3, 1));
 
         let selection = selection.parent.as_ref().unwrap();
-        assert_eq!(selection.range.start_point, Point::new(1, 0));
+        assert_eq!(selection.range.start_point, Point::new(0, 0));
         assert_eq!(selection.range.end_point, Point::new(6, 0));
         assert!(selection.parent.is_none());
     }
@@ -176,11 +174,9 @@ mod tests {
 
         let (text, point) = point_from_cursor(text);
 
-        let language = tree_sitter_r::language();
-
         let mut parser = Parser::new();
         parser
-            .set_language(&language)
+            .set_language(&tree_sitter_r::LANGUAGE.into())
             .expect("failed to create parser");
 
         let tree = parser.parse(text, None).unwrap();
@@ -191,7 +187,7 @@ mod tests {
 
         // Just 1 selection, the whole document
         let selection = selections.get(0).unwrap();
-        assert_eq!(selection.range.start_point, Point::new(1, 0));
+        assert_eq!(selection.range.start_point, Point::new(0, 0));
         assert_eq!(selection.range.end_point, Point::new(6, 0));
         assert!(selection.parent.is_none());
     }
@@ -209,12 +205,10 @@ fn <- function(x, arg) {
 
         let (text, point) = point_from_cursor(text);
 
-        let language = tree_sitter_r::language();
-
         // create a parser for this document
         let mut parser = Parser::new();
         parser
-            .set_language(&language)
+            .set_language(&tree_sitter_r::LANGUAGE.into())
             .expect("failed to create parser");
 
         let tree = parser.parse(text, None).unwrap();
@@ -250,19 +244,17 @@ fn <- function(x, arg) {
 
         // Whole document
         let selection = selection.parent.as_ref().unwrap();
-        assert_eq!(selection.range.start_point, Point::new(1, 0));
+        assert_eq!(selection.range.start_point, Point::new(0, 0));
         assert_eq!(selection.range.end_point, Point::new(6, 0));
         assert!(selection.parent.is_none());
     }
 
     #[test]
     #[rustfmt::skip]
     fn test_selection_range_assignment() {
-        let language = tree_sitter_r::language();
-
         let mut parser = Parser::new();
         parser
-            .set_language(&language)
+            .set_language(&tree_sitter_r::LANGUAGE.into())
             .expect("failed to create parser");
 
         let text = "
@@ -295,11 +287,9 @@ fn <- function() {
     #[test]
     #[rustfmt::skip]
     fn test_selection_range_call_arguments() {
-        let language = tree_sitter_r::language();
-
         let mut parser = Parser::new();
         parser
-            .set_language(&language)
+            .set_language(&tree_sitter_r::LANGUAGE.into())
             .unwrap();
 
         let text = "
@@ -337,11 +327,9 @@ fn(@a, b, c)
     #[test]
     #[rustfmt::skip]
     fn test_selection_range_subset_arguments() {
-        let language = tree_sitter_r::language();
-
         let mut parser = Parser::new();
         parser
-            .set_language(&language)
+            .set_language(&tree_sitter_r::LANGUAGE.into())
             .unwrap();
 
         let text = "
@@ -384,11 +372,9 @@ x[a, @fn(), c]
     #[test]
     #[rustfmt::skip]
     fn test_selection_range_subset2_arguments() {
-        let language = tree_sitter_r::language();
-
         let mut parser = Parser::new();
         parser
-            .set_language(&language)
+            .set_language(&tree_sitter_r::LANGUAGE.into())
             .unwrap();
 
         let text = "
@@ -431,11 +417,9 @@ x[[a, @fn(), c]]
     #[test]
     #[rustfmt::skip]
     fn test_selection_range_namespaced_calls() {
-        let language = tree_sitter_r::language();
-
         let mut parser = Parser::new();
         parser
-            .set_language(&language)
+            .set_language(&tree_sitter_r::LANGUAGE.into())
             .unwrap();
 
         let text = "

crates/ark/src/lsp/snapshots/indent.R

@@ -1098,11 +1098,11 @@ fun_call(object1 + object2 ~ object3 +
            argument)
 
 ## 32
-fun_call(object ~
+fun_call(~ object
   )
 
 ## 33
-fun_call(object +
+fun_call(object + object2
   )
 
 ## 34

crates/ark/src/lsp/state_handlers.rs

@@ -165,9 +165,10 @@ pub(crate) fn did_open(
     let uri = params.text_document.uri;
     let version = params.text_document.version;
 
-    let language = tree_sitter_r::language();
     let mut parser = Parser::new();
-    parser.set_language(&language).unwrap();
+    parser
+        .set_language(&tree_sitter_r::LANGUAGE.into())
+        .unwrap();
 
     let document = Document::new_with_parser(contents, &mut parser, Some(version));
 

crates/ark/src/lsp/statement_range.rs

@@ -509,6 +509,8 @@ fn test_statement_range() {
     // by tree-sitter. It is generally best to left align the string against the
     // far left margin to avoid unexpected whitespace and mimic real life.
     fn statement_range_test(x: &str) {
+        let original = x;
+
         let lines = x.split("\n").collect::<Vec<&str>>();
 
         let mut cursor: Option<Point> = None;
@@ -649,11 +651,9 @@ fn test_statement_range() {
         let x = x.replace("@", "");
         let x = x.replace(">>", "");
 
-        let language = tree_sitter_r::language();
-
         let mut parser = Parser::new();
         parser
-            .set_language(&language)
+            .set_language(&tree_sitter_r::LANGUAGE.into())
             .expect("Failed to create parser");
 
         let ast = parser.parse(x, None).unwrap();
@@ -662,8 +662,8 @@ fn test_statement_range() {
 
         let node = find_statement_range_node(&root, cursor.unwrap().row).unwrap();
 
-        assert_eq!(node.start_position(), sel_start.unwrap());
-        assert_eq!(node.end_position(), sel_end.unwrap());
+        assert_eq!(node.start_position(), sel_start.unwrap(), "Failed on test {original}");
+        assert_eq!(node.end_position(), sel_end.unwrap(), "Failed on test {original}");
     }
 
     // Simple test
@@ -1286,10 +1286,9 @@ test_that('stuff', {
 
 
 ";
-    let language = tree_sitter_r::language();
     let mut parser = Parser::new();
     parser
-        .set_language(&language)
+        .set_language(&tree_sitter_r::LANGUAGE.into())
         .expect("Failed to create parser");
     let ast = parser.parse(contents, None).unwrap();
     let root = ast.root_node();
@@ -1304,10 +1303,9 @@ test_that('stuff', {
 
 }
 ";
-    let language = tree_sitter_r::language();
     let mut parser = Parser::new();
     parser
-        .set_language(&language)
+        .set_language(&tree_sitter_r::LANGUAGE.into())
         .expect("Failed to create parser");
     let ast = parser.parse(contents, None).unwrap();
     let root = ast.root_node();