the comma (#23)

It turns out that `;` cannot consistently be used for parameter lists
with defaults, so we revert back to using `,` to separate identdefs
except when doing so would result in AST inequality - this works for all
known cases and returns us to comma-land generally.

* fix missing `:` in command post-statements
* fix parsing of top-level statements with `;`
(nim-lang/Nim#23088)
* fix doc-comment support after `=` for vars, consts etc

src/astcmp.nim

@@ -2,8 +2,9 @@
 #        (c) Copyright 2023 Jacek Sieka
 ## Compare two AST's for semantic equivalence - aka undo whitespace bugs in the
 ## Nim parser / grammar
-
-import "$nim"/compiler/[ast, parser, idents, options], std/sequtils
+import
+  "$nim"/compiler/[ast, llstream, parser, idents, options, pathutils],
+  std/sequtils
 
 from std/math import isNaN
 
@@ -20,6 +21,38 @@ type
     of Different:
       a*, b*: PNode
 
+# TODO https://github.com/nim-lang/Nim/pull/23088
+proc parseAll(p: var Parser): PNode =
+  ## Parses the rest of the input stream held by the parser into a PNode.
+  result = newNodeP(nkStmtList, p)
+  while true:
+    let nextStmt = p.parseTopLevelStmt()
+    if nextStmt.kind == nkEmpty:
+      break
+    result &= nextStmt
+  # setEndInfo()
+
+proc parseString2(
+    s: string,
+    cache: IdentCache,
+    config: ConfigRef,
+    filename: string = "",
+    line: int = 0,
+    printTokens = false,
+): PNode =
+  ## Parses a string into an AST, returning the top node.
+  ## `filename` and `line`, although optional, provide info so that the
+  ## compiler can generate correct error messages referring to the original
+  ## source.
+  var stream = llStreamOpen(s)
+  stream.lineOffset = line
+
+  var p: Parser
+
+  openParser(p, AbsoluteFile filename, stream, cache, config)
+  result = p.parseAll
+  closeParser(p)
+
 proc similarKinds(ak, bk: TNodeKind): bool =
   ak == bk or (ak in {nkElseExpr, nkElse} and bk in {nkElseExpr, nkElse}) or (
     ak in {nkElifExpr, nkElifBranch} and bk in {nkElifExpr, nkElifBranch}
@@ -108,8 +141,8 @@ proc makeConfigRef(): ConfigRef =
 proc equivalent*(a, afile, b, bfile: string): Outcome =
   let
     conf = makeConfigRef()
-    aa = parseString(a, newIdentCache(), conf, afile)
-    bb = parseString(b, newIdentCache(), conf, bfile)
+    aa = parseString2(a, newIdentCache(), conf, afile)
+    bb = parseString2(b, newIdentCache(), conf, bfile)
   if conf.errorCounter > 0:
     Outcome(kind: ParseError)
   else:

src/astyaml.nim

@@ -12,7 +12,7 @@
 import "$nim"/compiler/[ast, lineinfos, msgs, options, rodutils]
 import std/[intsets, strutils]
 
-proc addYamlString*(res: var string; s: string) =
+proc addYamlString*(res: var string, s: string) =
   # We have to split long strings into many ropes. Otherwise
   # this could trigger InternalError(111). See the ropes module for
   # further information.
@@ -42,42 +42,42 @@ proc flagsToStr[T](flags: set[T]): string =
       result.addYamlString($x)
     result.add "]"
 
-proc lineInfoToStr(conf: ConfigRef; info: TLineInfo): string =
+proc lineInfoToStr(conf: ConfigRef, info: TLineInfo): string =
   result.add "["
   result.addYamlString(toFilename(conf, info))
   result.addf ", $1, $2]", [toLinenumber(info), toColumn(info)]
 
 proc treeToYamlAux(
-  res: var string;
-  conf: ConfigRef;
-  n: PNode;
-  marker: var IntSet;
-  indent, maxRecDepth: int;
+  res: var string,
+  conf: ConfigRef,
+  n: PNode,
+  marker: var IntSet,
+  indent, maxRecDepth: int,
 )
 
 proc symToYamlAux(
-  res: var string;
-  conf: ConfigRef;
-  n: PSym;
-  marker: var IntSet;
-  indent, maxRecDepth: int;
+  res: var string,
+  conf: ConfigRef,
+  n: PSym,
+  marker: var IntSet,
+  indent, maxRecDepth: int,
 )
 
 proc typeToYamlAux(
-  res: var string;
-  conf: ConfigRef;
-  n: PType;
-  marker: var IntSet;
-  indent, maxRecDepth: int;
+  res: var string,
+  conf: ConfigRef,
+  n: PType,
+  marker: var IntSet,
+  indent, maxRecDepth: int,
 )
 
 proc symToYamlAux(
-    res: var string;
-    conf: ConfigRef;
-    n: PSym;
-    marker: var IntSet;
-    indent: int;
-    maxRecDepth: int;
+    res: var string,
+    conf: ConfigRef,
+    n: PSym,
+    marker: var IntSet,
+    indent: int,
+    maxRecDepth: int,
 ) =
   if n == nil:
     res.add("null")
@@ -109,12 +109,12 @@ proc symToYamlAux(
     res.treeToYamlAux(conf, n.loc.lode, marker, indent + 1, maxRecDepth - 1)
 
 proc typeToYamlAux(
-    res: var string;
-    conf: ConfigRef;
-    n: PType;
-    marker: var IntSet;
-    indent: int;
-    maxRecDepth: int;
+    res: var string,
+    conf: ConfigRef,
+    n: PType,
+    marker: var IntSet,
+    indent: int,
+    maxRecDepth: int,
 ) =
   if n == nil:
     res.add("null")
@@ -139,12 +139,12 @@ proc typeToYamlAux(
         res.typeToYamlAux(conf, s, marker, indent + 1, maxRecDepth - 1)
 
 proc treeToYamlAux(
-    res: var string;
-    conf: ConfigRef;
-    n: PNode;
-    marker: var IntSet;
-    indent: int;
-    maxRecDepth: int;
+    res: var string,
+    conf: ConfigRef,
+    n: PNode,
+    marker: var IntSet,
+    indent: int,
+    maxRecDepth: int,
 ) =
   if n == nil:
     res.add("null")
@@ -181,19 +181,19 @@ proc treeToYamlAux(
         res.typeToYamlAux(conf, n.typ, marker, indent + 1, maxRecDepth)
 
 proc treeToYaml*(
-    conf: ConfigRef; n: PNode; indent: int = 0; maxRecDepth: int = -1
+    conf: ConfigRef, n: PNode, indent: int = 0, maxRecDepth: int = -1
 ): string =
   var marker = initIntSet()
   result.treeToYamlAux(conf, n, marker, indent, maxRecDepth)
 
 proc typeToYaml*(
-    conf: ConfigRef; n: PType; indent: int = 0; maxRecDepth: int = -1
+    conf: ConfigRef, n: PType, indent: int = 0, maxRecDepth: int = -1
 ): string =
   var marker = initIntSet()
   result.typeToYamlAux(conf, n, marker, indent, maxRecDepth)
 
 proc symToYaml*(
-    conf: ConfigRef; n: PSym; indent: int = 0; maxRecDepth: int = -1
+    conf: ConfigRef, n: PSym, indent: int = 0, maxRecDepth: int = -1
 ): string =
   var marker = initIntSet()
   result.symToYamlAux(conf, n, marker, indent, maxRecDepth)

src/nph.nim

@@ -31,8 +31,9 @@ Options:
   --help                show this help
 """
   ErrCheckFailed = 1
-  ErrParseFailed = 2
-  ErrEqFailed = 3
+  ErrParseInputFailed = 2
+  ErrParseOutputFailed = 3
+  ErrEqFailed = 4
 
 proc writeHelp() =
   stdout.write(Usage)
@@ -44,7 +45,7 @@ proc writeVersion() =
   stdout.flushFile()
   quit(0)
 
-proc parse(input, filename: string; printTokens: bool; conf: ConfigRef): PNode =
+proc parse(input, filename: string, printTokens: bool, conf: ConfigRef): PNode =
   let fn = if filename == "-": "stdin" else: filename
 
   parseString(input, newIdentCache(), conf, fn, printTokens = printTokens)
@@ -59,7 +60,7 @@ proc makeConfigRef(): ConfigRef =
   conf.errorMax = int.high
   conf
 
-proc prettyPrint(infile, outfile: string; debug, check, printTokens: bool): int =
+proc prettyPrint(infile, outfile: string, debug, check, printTokens: bool): int =
   let
     conf = makeConfigRef()
     input =
@@ -70,14 +71,18 @@ proc prettyPrint(infile, outfile: string; debug, check, printTokens: bool): int
     node = parse(input, infile, printTokens, conf)
 
   if conf.errorCounter > 0:
-    return ErrParseFailed
+    localError(
+      conf, TLineInfo(fileIndex: FileIndex(0)), "Skipped file, input cannot be parsed"
+    )
+
+    return ErrParseInputFailed
 
   var output = renderTree(node, conf)
   if not output.endsWith("\n"):
     output.add "\n"
 
   if conf.errorCounter > 0:
-    return ErrParseFailed
+    return ErrParseOutputFailed
 
   if infile != "-":
     if debug: