feat: multiline Text literals (#3995)

Some discussion here: #3939

This PR extends the current `Text` literal to allow multiline. It does not enable interpolation as `Text` is implemented using ropes under the hood, which already has fast appends.

Example:
```
"here is a 

multiline 
text value"
```

Some other solution for nicely expressing `json`, `html`, `xml` and other similar structures are being considered (aside from using `Text` with interpolation), but that is outside the scope of this PR.

Changelog.md

@@ -2,6 +2,20 @@
 
 * motoko (`moc`)
 
+  * Allow multiline text literals (#3995).
+    For example,
+
+    ```
+    "A horse walks into a bar.
+    The barman says: `Why the long face?`"
+    ```
+
+    parses as:
+
+    ```
+    "A horse walks into a bar.\nThe barman says: `Why the long face?`"
+    ```
+
   * Added pipe operator `<exp1> |> <exp2>` and placeholder expression `_`  (#3987).
     For example:
 
@@ -48,7 +62,6 @@
     and can call both `query` and other `composite query` functions.
 
     See the documentation for full details.
-
   * Allow canister imports of Candid service constructors, ignoring the service arguments to
     import the instantiated service instead (with a warning) (#4041).
 

doc/md/language-manual.md

@@ -172,6 +172,7 @@ character ::=
   | '\\'escape
   | '\\'hexdigit hexdigit
   | "\\u{" hexnum '}'
+  | '\n'        // literal newline
 
 char := '\'' character '\''
 ```
@@ -184,6 +185,8 @@ A text literal is `"`-delimited sequence of characters:
 text ::= '"' character* '"'
 ```
 
+Note that a text literal may span multiple lines.
+
 ### Literals
 
 ``` bnf

src/mo_frontend/source_lexer.mll

@@ -49,6 +49,7 @@ let text lexbuf s =
   let b = Buffer.create (String.length s) in
   let i = ref 1 in
   while !i < String.length s - 1 do
+    if s.[!i] = '\n' then Lexing.new_line lexbuf;
     let bs = codepoint lexbuf s i in
     Buffer.add_substring b bs 0 (String.length bs)
   done;
@@ -98,6 +99,7 @@ let character =
   | byte
   | '\\'escape
   | "\\u{" hexnum '}'
+  | '\n'
 
 let nat = num | "0x" hexnum
 let frac = num
@@ -182,7 +184,7 @@ rule token mode = parse
   | float as s { FLOAT s }
   | char as s { CHAR (char lexbuf s) }
   | text as s { TEXT (text lexbuf s) }
-  | '"'character*('\n'|eof)
+  | '"'character*eof
     { error lexbuf "unclosed text literal" }
   | '"'character*['\x00'-'\x09''\x0b'-'\x1f''\x7f']
     { error lexbuf "illegal control character in text literal" }

test/fail/multiline-text-line-number.mo

@@ -0,0 +1,8 @@
+"check that the
+
+line number of the error is correct.
+
+The asterisks should
+produce an error
+
+" *;

test/fail/ok/multiline-text-line-number.tc.ok

@@ -0,0 +1,2 @@
+multiline-text-line-number.mo:8.3-8.4: syntax error [M0001], unexpected token ';', expected one of token or <phrase> sequence:
+  <exp_bin(ob)>

test/fail/ok/multiline-text-line-number.tc.ret.ok

@@ -0,0 +1 @@
+Return code 1

test/repl/ok/stateful.stderr.ok

@@ -1,4 +1,5 @@
-stdin:2.9-2.17: syntax error [M0002], unclosed text literal
+stdin:2.18-2.19: syntax error [M0001], unexpected token ';', expected one of token or <phrase> sequence:
+  <exp_bin(ob)>
 stdin:1.9-1.10: type error [M0050], literal of type
   Nat
 does not have expected type

test/repl/stateful.sh

@@ -3,7 +3,7 @@
 # syntax and type errors
 moc -i <<__END__
 let x = 42;
-let x = "foobar
+let x = "foobar"*;
 let x = 1 + true;
 assert (x == 42);
 __END__

test/run/ok/text-multiline.run-ir.ok

@@ -0,0 +1,5 @@
+ Text 
+  with
+
+    newline 
+

test/run/ok/text-multiline.run-low.ok

@@ -0,0 +1,5 @@
+ Text 
+  with
+
+    newline 
+

test/run/ok/text-multiline.run.ok

@@ -0,0 +1,5 @@
+ Text 
+  with
+
+    newline 
+

test/run/ok/text-multiline.wasm-run.ok

@@ -0,0 +1,5 @@
+ Text 
+  with
+
+    newline 
+

test/run/text-multiline.mo

@@ -0,0 +1,7 @@
+import Prim "mo:⛔";
+
+Prim.debugPrint " Text 
+  with
+
+    newline 
+";

test/run/text-pats.mo

@@ -1,13 +1,20 @@
 switch "foo" {
   case "bar" assert false;
-  case ""    assert false;
+  case "" assert false;
   case "foo" assert true;
-  case _     assert false;
+  case _ assert false;
 };
 
 switch (?"foo") {
   case (?"bar") assert false;
-  case (?"")    assert false;
+  case (?"") assert false;
   case (?"foo") assert true;
-  case _        assert false;
-}
+  case _ assert false;
+};
+
+switch "foo
+" {
+  case "foo
+" assert true;
+  case _ assert false;
+};