the stable comment

.github/workflows/ci.yml

@@ -37,6 +37,10 @@ jobs:
     name: CI '${{ matrix.target.name }}'
     runs-on: ${{ matrix.builder }}
     steps:
+      - name: Set git to use LF
+        run: |
+          git config --global core.autocrlf false
+          git config --global core.eol lf
       - name: Checkout
         uses: actions/checkout@v3
 
@@ -64,6 +68,8 @@ jobs:
 
       - name: Check formatting
         run: |
-          find src tests/after -name "*.nim" -print0 | xargs -0 -n1 ./nph
-          git diff
-          # find -name "*.nim"  ! -path "./tests/before/*" ! -path "./nimbledeps/*" -print0 | xargs -0 -n1 ./nph --check
+          ! ./nph --check tests/before
+          ./nph --check tests/after
+
+          ./nph src
+          git diff --exit-code

.gitignore

@@ -1,2 +1,5 @@
 nimbledeps
 /nph
+nimble.develop
+nimble.paths
+*.exe

src/astcmp.nim

@@ -10,11 +10,12 @@ from std/math import isNaN
 type
   Equivalence* = enum
     Same
+    ParseError
     Different
 
   Outcome* = object
     case kind*: Equivalence
-    of Same:
+    of Same, ParseError:
       discard
     of Different:
       a*, b*: PNode
@@ -71,10 +72,16 @@ proc equivalent*(a, b: PNode): Outcome =
     of nkIdent:
       a.ident.s == b.ident.s
     else:
+      let skipped =
+        if a.kind == nkStmtListExpr:
+          # When inserting a `;`, we might get some extra empty statements (?)
+          {nkEmpty, nkCommentStmt}
+        else:
+          {nkCommentStmt}
       # TODO don't break comments!
       let
-        af = a.sons.filterIt(it.kind != nkCommentStmt)
-        bf = b.sons.filterIt(it.kind != nkCommentStmt)
+        af = a.sons.filterIt(it.kind notin skipped)
+        bf = b.sons.filterIt(it.kind notin skipped)
 
       if af.len() != bf.len():
         false
@@ -91,8 +98,17 @@ proc equivalent*(a, b: PNode): Outcome =
   else:
     Outcome(kind: Same)
 
+proc makeConfigRef(): ConfigRef =
+  let conf = newConfigRef()
+  conf.errorMax = int.high
+  conf
+
 proc equivalent*(a, afile, b, bfile: string): Outcome =
-  equivalent(
-    parseString(a, newIdentCache(), newConfigRef(), afile),
-    parseString(b, newIdentCache(), newConfigRef(), bfile),
-  )
+  let
+    conf = makeConfigRef()
+    aa = parseString(a, newIdentCache(), conf, afile)
+    bb = parseString(b, newIdentCache(), conf, bfile)
+  if conf.errorCounter > 0:
+    Outcome(kind: ParseError)
+  else:
+    equivalent(aa, bb)

src/nph.nim

@@ -17,7 +17,7 @@ static:
     "nph needs to be compiled with nim 2.0.0 exactly for now"
 
 const
-  Version = "0.1"
+  Version = gorge("git describe --long --dirty --always")
   Usage =
     "nph - Nim formatter " & Version &
       """
@@ -30,6 +30,9 @@ Options:
   --version             show the version
   --help                show this help
 """
+  ErrCheckFailed = 1
+  ErrParseFailed = 2
+  ErrEqFailed = 3
 
 proc writeHelp() =
   stdout.write(Usage)
@@ -55,16 +58,28 @@ func isNimFile(file: string): bool =
   let (_, _, ext) = file.splitFile()
   ext in [".nim", ".nims", ".nimble"]
 
-proc prettyPrint(infile, outfile: string; debug, check, printTokens: bool): bool =
+proc makeConfigRef(): ConfigRef =
+  let conf = newConfigRef()
+  conf.errorMax = int.high
+  conf
+
+proc prettyPrint(infile, outfile: string; debug, check, printTokens: bool): int =
   let
-    conf = newConfigRef()
+    conf = makeConfigRef()
     input =
       if infile == "-":
         readAll(stdin)
       else:
         readFile(infile)
     node = parse(input, infile, printTokens, conf)
-    output = renderTree(node, conf) & "\n"
+
+  if conf.errorCounter > 0:
+    return ErrParseFailed
+
+  let output = renderTree(node, conf) & "\n"
+
+  if conf.errorCounter > 0:
+    return ErrParseFailed
 
   if infile != "-":
     if debug:
@@ -78,7 +93,7 @@ proc prettyPrint(infile, outfile: string; debug, check, printTokens: bool): bool
         )
     elif fileExists(outFile) and output == readFile(outFile):
       # No formatting difference - don't touch file modificuation date
-      return true
+      return QuitSuccess
 
   let eq =
     equivalent(
@@ -92,7 +107,41 @@ proc prettyPrint(infile, outfile: string; debug, check, printTokens: bool): bool
       ,
     )
 
-  if eq.kind == Different:
+  template writeUnformatted() =
+    if not debug and (infile != outfile or infile == "-"):
+      # Write unformatted content
+      if not check:
+        if infile == "-":
+          write(stdout, input)
+        else:
+          writeFile(outfile, input)
+
+  case eq.kind
+  of Same:
+    if check:
+      ErrCheckFailed # We failed the equivalence check above
+    else:
+      # Formatting changed the file
+      if not debug or infile == "-":
+        if infile == "-":
+          write(stdout, output)
+        else:
+          writeFile(outfile, output)
+
+      QuitSuccess
+  of ParseError:
+    writeUnformatted()
+
+    localError(
+      conf,
+      TLineInfo(fileIndex: FileIndex(0)),
+      "Skipped file, formatted output cannot be parsed (bug! " & Version & ")",
+    )
+
+    ErrEqFailed
+  of Different:
+    writeUnformatted()
+
     stderr.writeLine "--- Input ---"
     stderr.writeLine input
     stderr.writeLine "--- Formatted ---"
@@ -102,32 +151,13 @@ proc prettyPrint(infile, outfile: string; debug, check, printTokens: bool): bool
     stderr.writeLine "--- POST ---"
     stderr.writeLine treeToYaml(nil, eq.b)
 
-    internalError(
+    localError(
       conf,
       TLineInfo(fileIndex: FileIndex(0)),
-      "Formatted output does not match input, report bug!",
+      "Skipped file, formatted output does not match input (bug! " & Version & ")",
     )
-    if infile != outfile or infile == "-":
-      # Write unformatted content
-      if not check:
-        if infile == "-":
-          write(stdout, input)
-        else:
-          writeFile(outfile, input)
-
-    quit 2
-
-  if check:
-    false # We failed the equivalence check above
-  else:
-    # Formatting changed the file
-    if not debug or infile == "-":
-      if infile == "-":
-        write(stdout, output)
-      else:
-        writeFile(outfile, output)
 
-    true
+    ErrEqFailed
 
 proc main() =
   var
@@ -194,14 +224,20 @@ proc main() =
   elif outdir.len != 0:
     outfiles = infiles.mapIt($(joinPath(outdir, it)))
 
+  var res = QuitSuccess
   for (infile, outfile) in zip(infiles, outfiles):
     let (dir, _, _) = splitFile(outfile)
 
     createDir(dir)
-    if not prettyPrint(infile, outfile, debug, check, printTokens):
-      quit 1
+    let err = prettyPrint(infile, outfile, debug, check, printTokens)
+    case err
+    of ErrCheckFailed:
+      quit ErrCheckFailed
+    else:
+      # Keep going on source code errors but fail the program eventually
+      res = max(res, err)
 
-  quit 0
+  quit res
 
 when isMainModule:
   main()

src/phastyaml.nim

@@ -11,9 +11,10 @@
 
 # * yaml formatting for the nph-specific fields
 
-import "."/[phast, phlexer, phlineinfos, phoptions, phmsgs]
-import "$nim"/compiler/[rodutils]
-import std/[intsets, strutils]
+import
+  "."/[phast, phlexer, phlineinfos, phoptions, phmsgs],
+  "$nim"/compiler/[rodutils],
+  std/[intsets, strutils]
 
 proc addYamlString*(res: var string; s: string) =
   # We have to split long strings into many ropes. Otherwise

src/phparser.nim

@@ -138,14 +138,17 @@ proc parMessage(p: Parser; arg: string) =
   ## Produce and emit the parser message `arg` to output.
   lexMessageTok(p.lex, errGenerated, p.tok, arg)
 
-template withInd(p, body: untyped) =
+template withInd(p, indent, body: untyped) =
   let oldInd = p.currInd
-  p.currInd = p.tok.indent
+  p.currInd = indent
 
   body
 
   p.currInd = oldInd
 
+template withInd(p, body: untyped) =
+  withInd(p, p.tok.indent, body)
+
 template realInd(p): bool =
   p.tok.indent > p.currInd
 
@@ -229,7 +232,7 @@ proc drainSkipped(p: var Parser; indent: int): seq[Token] =
     for c in p.skipped:
       if c.indent == -1 or c.indent > indent:
         result.add(c)
-        p.lineNumberPrevious = int c.prevLine
+        p.lineNumberPrevious = int c.lineB
 
     p.skipped.delete(0..result.high)
 
@@ -358,8 +361,16 @@ proc isOperator(tok: Token): bool =
   }
 
 proc parseComStmt(p: var Parser; n: PNode; commentLoc = clPostfix): PNode =
-  splitLookahead(p, n, commentLoc)
-  parseStmt(p)
+  splitLookahead(p, n, int.high, commentLoc)
+
+  # Let parseStmt deal with statement lists that contain only doc comments
+  if not (
+    p.tok.indent >= 0 and p.tok.indent < p.currInd and p.skipped.len > 0 and
+    p.skipped[0].indent > p.currInd
+  ):
+    splitLookahead(p, n, commentLoc)
+
+  parseStmt(p, allowEmpty = true)
 
 proc parseColComStmt(p: var Parser; n: PNode; commentLoc = clPostfix): PNode =
   # `:` followed by a list of statements, with comments interspresed
@@ -1109,6 +1120,9 @@ proc parseOperators(
     a.add(opNode)
     a.add(result)
     a.add(b)
+    # Reset the "beginning" of the infix to capture empty lines correctly
+    a.info = result.info
+    a.endInfo = b.endInfo
 
     result = a
     opPrec = getPrecedence(p.tok)
@@ -1232,7 +1246,7 @@ proc parseIdentColonEquals(p: var Parser; flags: DeclaredIdentFlags): PNode =
     if p.tok.tokType != tkComma:
       break
     getTok(p)
-
+    splitLookahead(p, a, clPostfix)
     optInd(p, a)
 
   # We let comments sit as prefixes to whatever comes next - this works well
@@ -2501,7 +2515,6 @@ proc parseSection(
   #| section(RULE) = COMMENT? RULE / (IND{>} (RULE / COMMENT)^+IND{=} DED)
   if result.kind != nkTypeSection:
     getTok(p)
-
   splitLookahead(p, result, clMid)
   if realInd(p):
     withInd(p):
@@ -2838,7 +2851,6 @@ proc parseTypeDef(p: var Parser): PNode =
   #| typeDef = identVisDot genericParamList? pragma '=' optInd typeDefValue
   #|             indAndComment?
   result = newNodeP(nkTypeDef, p)
-
   var identifier = identVis(p, allowDot = true)
   var identPragma = identifier
   var pragma: PNode
@@ -3119,11 +3131,17 @@ proc complexOrSimpleStmt(p: var Parser): PNode =
 proc parseStmt(p: var Parser; allowEmpty: bool): PNode =
   #| stmt = (IND{>} complexOrSimpleStmt^+(IND{=} / ';') DED)
   #|      / simpleStmt ^+ ';'
-  if p.tok.indent > p.currInd:
+
+  let nextInd =
+    if p.tok.indent < p.currInd and p.skipped.len > 0 and p.skipped[0].indent > p.currInd:
+      p.skipped[0].indent
+    else:
+      p.tok.indent
+  if nextInd > p.currInd:
     result = newNodeP(nkStmtList, p, withPrefix = false)
-    addSkipped(p, result)
 
-    withInd(p):
+    withInd(p, nextInd):
+      addSkipped(p, result)
       splitLookahead(p, result, clPrefix)
       while true:
         if p.tok.indent == p.currInd:
@@ -3172,6 +3190,7 @@ proc parseStmt(p: var Parser; allowEmpty: bool): PNode =
         tkMacro, tkType, tkConst, tkWhen, tkVar:
       if allowEmpty:
         result = newNodeP(nkStmtList, p, withPrefix = false)
+        addSkipped(p, result)
       else:
         parMessage(p, "nestable statement requires indentation")
         result = p.emptyNode