feat: pipe operator and placeholder expression (#3987)

Alternative to #3968 that places no restrictions on rhs of pipe, including repeated uses of placeholder '_'.

We just define 

```
exp_nullary := ...
  | `_`                        // placeholder reference
exp_bin := ...
  | <exp_bin> `|>` <exp_bin>   // pipe (bind the placeholder)
```
( `|>` has almost lowest precedence, just above `:`, and associates to the left).

and define sugar:

```
'_' ::= _underscore_

<e1> |> <e2>  ::= do { let _underscore_ = <e1>; <e2> }
```


(here _underscore_ is some inexpressible identifier - literally '_' in the implementation)


pros:
- more flexible: can pipe in to anything, not just arguments of function calls
- simpler grammar
- more natural in a language without linear typing

cons: 
- no rejection of dubious, non-linear: `text.chars() |> Iter.zip(_, _)`;
- our recursive `let`s means `_` is also available in `e1`, which is odd, but consistent, and rarely typechecks.


Additional examples, not possible in linear #3986 
```
> import I = "mo:base/Iter";
let I : module {...}
> import L = "mo:base/List";
let L : module {...}
> "hello".chars() |> I.toList _ |> L.zip (_, _) |> { size = L.size _ };
{size = 5} : {size : Nat}
```
Note non-linear piping into (safe) `List.zip`; terminal piping into non-application (a record).


TODO:

[ ] bespoke error message for use of unbound  (_underscore_) (reference not enclosed by a pipe).

Changelog.md

@@ -2,6 +2,27 @@
 
 * motoko (`moc`)
 
+  * Added pipe operator `<exp1> |> <exp2>` and placeholder expression `_`  (#3987).
+    For example:
+
+    ``` motoko
+    Iter.range(0, 10) |>
+      Iter.toList _ |>
+        List.filter<Nat>(_, func n { n % 3 == 0 }) |>
+          { multiples = _ };
+    ```
+
+    may, according to taste, be a more readable rendition of:
+
+    ``` motoko
+    { multiples =
+       List.filter<Nat>(
+         Iter.toList(Iter.range(0, 10)),
+           func n { n % 3 == 0 }) };
+    ```
+
+    However, beware the change of evaluation order for code with side-effects.
+
   * BREAKING CHANGE (Minor):
 
     New keyword `composite` allows one to declare Internet Computer *composite queries* (#4003).

doc/md/examples/Piped.mo

@@ -0,0 +1,7 @@
+import Iter "mo:base/Iter";
+import List "mo:base/List";
+
+Iter.range(0, 10) |>
+  Iter.toList _ |>
+    List.filter<Nat>(_, func n { n % 3 == 0 }) |>
+      { multiples = _ };

doc/md/examples/Unpiped.mo

@@ -0,0 +1,7 @@
+import Iter "mo:base/Iter";
+import List "mo:base/List";
+
+{ multiples =
+   List.filter<Nat>(
+     Iter.toList(Iter.range(0, 10)),
+     func n { n % 3 == 0 }) };

doc/md/examples/grammar.txt

@@ -157,6 +157,7 @@
     <exp_obj>
     <exp_plain>
     <id>
+    '_'
 
 <exp_post> ::= 
     <exp_nullary>
@@ -188,6 +189,7 @@
     <exp_bin> 'and' <exp_bin>
     <exp_bin> 'or' <exp_bin>
     <exp_bin> ':' <typ_nobin>
+    <exp_bin> '|>' <exp_bin>
 
 <exp_nondec> ::= 
     <exp_bin>

doc/md/language-manual.md

@@ -53,21 +53,21 @@ In the definition of some lexemes, the quick reference uses the symbol `␣` to
 
 Single line comments are all characters following `//` until the end of the same line.
 
-``` motoko
+``` motoko no-repl
 // single line comment
 x = 1
 ```
 
 Single or multi-line comments are any sequence of characters delimited by `/*` and `*/`:
 
-``` motoko
+``` motoko no-repl
 /* multi-line comments
    look like this, as in C and friends */
 ```
 
 Comments delimited by `/*` and `*/` may be nested, provided the nesting is well-bracketed.
 
-``` motoko
+``` motoko no-repl
 /// I'm a documentation comment
 /// for a function
 ```
@@ -217,7 +217,6 @@ Some types have several categories. For example, type `Int` is both arithmetic (
 | `-`      | A        | numeric negation |
 | `+`      | A        | numeric identity |
 | `^`      | B        | bitwise negation |
-| `!`      |          | null break       |
 
 ### Relational operators
 
@@ -304,6 +303,7 @@ The following table defines the relative precedence and associativity of operato
 | (higher)   | none          | `else`, `while`                                                                                                               |
 | (higher)   | right         | `:=`, `+=`, `-=`, `*=`, `/=`, `%=`, `**=`, `#=`, `&=`, `\|=`, `^=`, `<<=`, `>>=`, `<<>=`, `<>>=`, `+%=`, `-%=`, `*%=`, `**%=` |
 | (higher)   | left          | `:`                                                                                                                           |
+| (higher)   | left          | `|>`                                                                                                                           |
 | (higher)   | left          | `or`                                                                                                                          |
 | (higher)   | left          | `and`                                                                                                                         |
 | (higher)   | none          | `==`, `!=`, `<`, `>`, `<=`, `>`, `>=`                                                                                         |
@@ -469,6 +469,8 @@ The syntax of an *expression* is as follows:
   <unop> <exp>                                   unary operator
   <exp> <binop> <exp>                            binary operator
   <exp> <relop> <exp>                            binary relational operator
+  _                                              placeholder expression
+  <exp> |> <exp>                                 pipe operator
   ( <exp>,* )                                    tuple
   <exp> . <nat>                                  tuple projection
   ? <exp>                                        option injection
@@ -1769,6 +1771,63 @@ Otherwise, `r1` and `r2` are values `v1` and `v2` and the expression returns the
 
 For equality and inequality, the meaning of `v1 <relop> v2` depends on the compile-time, static choice of `T` (not the run-time types of `v1` and `v2`, which, due to subtyping, may be more precise).
 
+### Pipe operators and placeholder expressions
+
+The pipe expression `<exp1> |> <exp2>` binds the value of `<exp1>` to the special placeholder expression `_`, that can be referenced in `<exp2>` (and recursively in `<exp1>`).
+Referencing the placeholder expression outside of a pipe operation is a compile-time error.
+
+The pipe expression `<exp1> |> <exp2>` is just syntactic sugar for a `let` binding to a
+placeholder identifier, `p`:
+
+``` bnf
+do { let p = <exp1>; <exp2> }
+```
+
+The placeholder expression `_` is just syntactic sugar for the expression referencing the placeholder identifier:
+
+``` bnf
+p
+```
+
+The placeholder identifier, `p`, is a fixed, reserved identifier that cannot be bound by any other expression or pattern other than a pipe operation,
+and can only be referenced using the placeholder expression `_`.
+
+`|>` has lowest precedence amongst all operators except `:` and associates to the left.
+
+Judicious use of the pipe operator allows one to express a more complicated nested expression by piping arguments of that expression into their nested positions within that expression.
+
+For example:
+
+``` motoko no-repl
+Iter.range(0, 10) |>
+  Iter.toList _ |>
+    List.filter<Nat>(_, func n { n % 3 == 0 }) |>
+      { multiples = _ };
+```
+
+may, according to taste, be a more readable rendition of:
+
+``` motoko no-repl
+{ multiples =
+   List.filter<Nat>(
+     Iter.toList(Iter.range(0, 10)),
+     func n { n % 3 == 0 }) };
+```
+
+Above, each occurence of `_` refers to the value of the left-hand-size of the nearest enclosing
+pipe operation, after associating nested pipes to the left.
+
+Note that the evaluation order of the two examples is different, but consistently left-to-right.
+
+:::note
+
+Although syntactically identical, the placeholder expression is
+semantically distinct from, and should not be confused with, the wildcard pattern `_`.
+Occurrences of the forms can be distinguished by their syntactic role as pattern or
+expression.
+
+:::
+
 ### Tuples
 
 Tuple expression `(<exp1>, …​, <expn>)` has tuple type `(T1, …​, Tn)`, provided `<exp1>`, …​, `<expn>` have types `T1`, …​, `Tn`.

doc/md/pipes.md

@@ -0,0 +1,46 @@
+# Piping values into expressions
+
+It can sometimes be hard to read deeply nested expressions involving several function applications.
+
+``` motoko file=./examples/Unpiped.mo#L1-L8
+```
+
+This expression take the range of numbers `0`..`10`, converts it to a list, filters the list for multiples of three and returns a record containing the result.
+
+To make such expressions more readable, you can use Motoko's pipe operator `<exp1> |> <exp2>`.
+The operator evaluates the first argument `<exp1>`, and lets you refer to its value in `<exp2>` using the special placeholder expression `_`.
+
+Using this, you can write the former expression as:
+
+``` motoko file=./examples/Piped.mo#L1-L8
+```
+
+Now, the textual order of operations corresponds to our English explanation above: "this expression takes the range of numbers `0`..`10`, converts it to a list, filters the list for multiples of three and returns a record containing the result".
+
+
+The pipe expression `<exp1> |> <exp2>` is just syntactic sugar for the following block binding `<exp1>` to a reserved
+placeholder identifier, `p`, before returning `<exp2>`:
+
+``` bnf
+do { let p = <exp1>; <exp2> }
+```
+
+The otherwise inaccessible placeholder identifier `p` can only referenced by the placeholder expression `_`. 
+Multiple references to `_` are allowed and refer to the same value within the same pipe operation.
+
+
+Note that using `_` as an expression outside of a pipe operation, where it is undefined, is an error.
+
+For example:
+
+``` motoko
+let x = _;
+```
+
+produces the compile-time error "type error [M0057], unbound variable _".
+
+(Internally, the compiler uses the reserved identifier `_` as the name for the placeholder called `p` above, so this `let` is just referencing an undefined variable).
+
+
+See [here](language-manual.md#pipe-operators-and-placeholder-expressions) for more details.
+

src/gen-grammar/grammar.sed

@@ -73,6 +73,7 @@ s/RBRACKET/\']\'/g
 s/QUEST/\'?\'/g
 s/BANG/\'!\'/g
 s/QUERY/\'query\'/g
+s/PIPE/\'|>\'/g
 s/PUBLIC/\'public\'/g
 s/PRIVATE/\'private\'/g
 s/POWOP/\'**\'/g

src/mo_frontend/error_reporting.ml

@@ -132,3 +132,4 @@ let terminal2token (type a) (symbol : a terminal) : token =
       | T_WRAPSUBASSIGN -> WRAPSUBASSIGN
       | T_WRAPMULASSIGN -> WRAPMULASSIGN
       | T_WRAPPOWASSIGN -> WRAPPOWASSIGN
+      | T_PIPE -> PIPE

src/mo_frontend/parser.mly

@@ -233,6 +233,7 @@ and objblock s dec_fields =
 %token<bool> BOOL
 %token<string> ID
 %token<string> TEXT
+%token PIPE
 %token PRIM
 %token UNDERSCORE
 %token COMPOSITE
@@ -243,6 +244,7 @@ and objblock s dec_fields =
 %nonassoc ELSE WHILE
 
 %left COLON
+%left PIPE
 %left OR
 %left AND
 %nonassoc EQOP NEQOP LEOP LTOP GTOP GEOP
@@ -579,6 +581,8 @@ exp_nullary(B) :
     { VarE(x) @? at $sloc }
   | PRIM s=TEXT
     { PrimE(s) @? at $sloc }
+  | UNDERSCORE
+    { VarE ("_" @@ at $sloc) @? at $sloc }
 
 exp_post(B) :
   | e=exp_nullary(B)
@@ -642,6 +646,13 @@ exp_un(B) :
     { OrE(e1, e2) @? at $sloc }
   | e=exp_bin(B) COLON t=typ_nobin
     { AnnotE(e, t) @? at $sloc }
+  | e1=exp_bin(B) PIPE e2=exp_bin(ob)
+    { let x = "_" @@ e1.at in
+      BlockE [
+        LetD (VarP x @! x.at, e1, None) @? e1.at;
+        ExpD e2 @? e2.at
+      ] @? at $sloc }
+
 
 %public exp_nondec(B) :
   | e=exp_bin(B)

src/mo_frontend/printers.ml

@@ -139,6 +139,7 @@ let string_of_symbol = function
   | X (T T_ADDOP) -> unop "+"
   | X (T T_ACTOR) -> "actor"
   | X (T T_INVARIANT) -> "invariant"
+  | X (T T_PIPE) -> "|>"
   (* non-terminals *)
   | X (N N_bl) -> "<bl>"
   | X (N N_case) -> "<case>"

src/mo_frontend/source_lexer.mll

@@ -139,6 +139,7 @@ rule token mode = parse
   | "**" { POWOP }
   | "&" { ANDOP }
   | "|" { OROP }
+  | "|>" { PIPE }
   | "^" { XOROP }
   | "<<" { SHLOP }
   | "<<>" { ROTLOP }

src/mo_frontend/source_token.ml

@@ -126,6 +126,7 @@ type token =
   | SPACE of int
   | TAB of int (* shudders *)
   | COMMENT of string
+  | PIPE
 
 let to_parser_token :
     token -> (Parser.token, line_feed trivia) result = function
@@ -248,6 +249,7 @@ let to_parser_token :
   | UNDERSCORE -> Ok Parser.UNDERSCORE
   | COMPOSITE -> Ok Parser.COMPOSITE
   | INVARIANT -> Ok Parser.INVARIANT
+  | PIPE -> Ok Parser.PIPE
   (*Trivia *)
   | SINGLESPACE -> Error (Space 1)
   | SPACE n -> Error (Space n)
@@ -379,6 +381,7 @@ let string_of_parser_token = function
   | Parser.INVARIANT -> "INVARIANT"
   | Parser.IMPLIES -> "IMPLIES"
   | Parser.OLD -> "OLD"
+  | Parser.PIPE -> "PIPE"
 
 let is_lineless_trivia : token -> void trivia option = function
   | SINGLESPACE -> Some (Space 1)

test/fail/ok/pipe-ill-typed.tc.ok

@@ -0,0 +1,12 @@
+pipe-ill-typed.mo:7.33-7.34: type error [M0096], expression of type
+  {#B}
+cannot produce expected type
+  {#A}
+pipe-ill-typed.mo:11.37-11.38: type error [M0096], expression of type
+  {#A}
+cannot produce expected type
+  {#B}
+pipe-ill-typed.mo:16.33-16.34: type error [M0096], expression of type
+  {#A}
+cannot produce expected type
+  (A__9, B)

test/fail/ok/pipe-ill-typed.tc.ret.ok

@@ -0,0 +1 @@
+Return code 1

test/fail/ok/syntax2.tc.ok

@@ -5,6 +5,7 @@ syntax2.mo:2.1-2.4: syntax error [M0001], unexpected token 'let', expected one o
   <exp_nullary(ob)>
   <binop> <exp(ob)>
   ; seplist(<dec>,<semicolon>)
+  |> <exp_bin(ob)>
   or <exp_bin(ob)>
   <unassign> <exp(ob)>
   implies <exp_bin(ob)>

test/fail/ok/syntax3.tc.ok

@@ -4,6 +4,7 @@ syntax3.mo:1.3-1.4: syntax error [M0001], unexpected token ';', expected one of
   !
   <exp_nullary(ob)>
   <binop> <exp(ob)>
+  |> <exp_bin(ob)>
   or <exp_bin(ob)>
   <unassign> <exp(ob)>
   implies <exp_bin(ob)>

test/fail/ok/syntax5.tc.ok

@@ -4,6 +4,7 @@ syntax5.mo:3.1: syntax error [M0001], unexpected end of input, expected one of t
   !
   <exp_nullary(ob)>
   <binop> <exp(ob)>
+  |> <exp_bin(ob)>
   or <exp_bin(ob)>
   <unassign> <exp(ob)>
   implies <exp_bin(ob)>

test/fail/pipe-ill-typed.mo

@@ -0,0 +1,17 @@
+type A = {#A};
+type B = {#B};
+
+func f2<T1, T2>(x1 : T1, x2 : T2) : (T1, T2) { (x1, x2) };
+
+do {
+  let (#A, #B) = #B |> f2<A, B>(_, #B); // type error
+};
+
+do {
+  let (#A, #B) = #A |> f2<A, B>(#A, _); // type error
+};
+
+
+do {
+  let (#A, #B) = #A |> f2<A, B> _; // type error
+};

test/run/ok/pipes.run-ir.ok

@@ -0,0 +1,23 @@
+1:
+#A
+f
+#B
+2:
+#B
+f
+#A
+3:
+#A
+f
+#B
+#C
+4:
+#B
+f
+#A
+#C
+5:
+#C
+f
+#A
+#B