fix do in call in dotexpr, comment in tuple, long files and lines

src/phlexer.nim

@@ -1425,8 +1425,8 @@ proc rawGetTok*(L: var Lexer, tok: var Token) =
   template atTokenEnd() {.dirty.} =
     L.previousTokenEnd.line = L.tokenEnd.line
     L.previousTokenEnd.col = L.tokenEnd.col
-    L.tokenEnd.line = tok.line.uint16
-    L.tokenEnd.col = getColNumber(L, L.bufpos).int16
+    L.tokenEnd.line = tok.line
+    L.tokenEnd.col = getColNumber(L, L.bufpos)
 
   let lineB = tok.lineB
   reset(tok)

src/phlineinfos.nim

@@ -352,14 +352,8 @@ type
 
   FileIndex* = distinct int32
   TLineInfo* = object
-    # This is designed to be as small as possible,
-    # because it is used
-    # in syntax nodes. We save space here by using
-    # two int16 and an int32.
-    # On 64 bit and on 32 bit systems this is
-    # only 8 bytes.
-    line*: uint16
-    col*: int16
+    line*: int
+    col*: int
     fileIndex*: FileIndex
     offsetA*, offsetB*: int
 

src/phmsgs.nim

@@ -163,15 +163,8 @@ proc fileInfoIdx*(conf: ConfigRef, filename: RelativeFile): FileIndex =
 
 proc newLineInfo*(fileInfoIdx: FileIndex, line, col: int): TLineInfo =
   result.fileIndex = fileInfoIdx
-  if line < int high(uint16):
-    result.line = uint16(line)
-  else:
-    result.line = high(uint16)
-
-  if col < int high(int16):
-    result.col = int16(col)
-  else:
-    result.col = -1
+  result.line = line
+  result.col = col
 
 proc newLineInfo*(
     conf: ConfigRef, filename: AbsoluteFile, line, col: int

src/phparser.nim

@@ -51,7 +51,7 @@ type
     inSemiStmtList*: int
     emptyNode: PNode
     skipped*: seq[Token]
-    endLine: uint16
+    endLine: int
 
   SymbolMode = enum
     smNormal
@@ -219,7 +219,7 @@ template setEndInfo(node: PNode) =
   #      for statement lists, which is the only place where we retain whitespace
   node.endInfo = TLineInfo(
     fileIndex: p.lex.fileIdx,
-    line: uint16 p.lineNumberPrevious,
+    line: p.lineNumberPrevious,
     col: p.lex.previousTokenEnd.col,
   )
 
@@ -488,7 +488,7 @@ proc exprList(p: var Parser, endTok: TokType, result: PNode) =
     if p.tok.tokType != tkComma:
       break
     getTok(p)
-    splitLookahead(p, a, clPostfix)
+    splitLookahead(p, result[^1], clPostfix)
     optInd(p, a)
     var a = parseExpr(p)
     result.add(a)
@@ -600,6 +600,8 @@ proc setOrTableConstr(p: var Parser): PNode =
         result.transitionSonsKind(nkTableConstr)
       result.add(a)
       if p.tok.tokType != tkComma:
+        # All comments before closing token
+        a.postfix.add move(p.skipped)
         break
       getTok(p)
       splitLookahead(p, a, clPostfix)
@@ -753,6 +755,8 @@ proc parsePar(p: var Parser): PNode =
       result.add(a)
       if p.tok.tokType == tkComma:
         getTok(p)
+        splitLookahead(p, a, clPostfix)
+
         # (1,) produces a tuple expression:
         result.transitionSonsKind(nkTupleConstr)
         # progress guaranteed
@@ -762,6 +766,9 @@ proc parsePar(p: var Parser): PNode =
           if p.tok.tokType != tkComma:
             break
           getTok(p)
+          splitLookahead(p, a, clPostfix)
+      # All comments before closing token
+      result[^1].postfix.add move(p.skipped)
   optPar(p)
   eat(p, tkParRi)
   setEndInfo()
@@ -1163,6 +1170,8 @@ proc parseTuple(p: var Parser, indentAllowed = false): PNode =
       var a = parseIdentColonEquals(p, {})
       result.add(a)
       if p.tok.tokType notin {tkComma, tkSemiColon}:
+        # All comments before closing token
+        a.postfix.add move(p.skipped)
         break
       getTok(p)
       splitLookahead(p, a, clPostfix)
@@ -1178,6 +1187,7 @@ proc parseTuple(p: var Parser, indentAllowed = false): PNode =
           case p.tok.tokType
           of tkSymbol, tkAccent:
             var a = parseIdentColonEquals(p, {})
+            splitLookahead(p, a, clPostfix)
             result.add(a)
           of tkEof:
             break
@@ -1223,10 +1233,12 @@ proc parseParamList(p: var Parser, retColon = true): PNode =
         parMessage(p, "expected closing ')'")
         break
       result.add(a)
-      splitLookahead(p, a, clPostfix)
       if p.tok.tokType notin {tkComma, tkSemiColon}:
+        # All comments before closing token
+        a.postfix.add move(p.skipped)
         break
       getTok(p)
+      splitLookahead(p, a, clPostfix)
     # Not ideal, but we'll attach the last comment in the par to the last
     # parameter
     if result.len > 0:
@@ -2111,6 +2123,8 @@ proc parseGenericParamList(p: var Parser): PNode =
     var a = parseGenericParam(p)
     result.add(a)
     if p.tok.tokType notin {tkComma, tkSemiColon}:
+      # All comments before closing token
+      a.postfix.add move(p.skipped)
       break
     getTok(p)
     splitLookahead(p, a, clPostfix)
@@ -2497,6 +2511,8 @@ proc parseVarTuple(p: var Parser): PNode =
       a = identWithPragma(p, allowDot = true)
     result.add(a)
     if p.tok.tokType != tkComma:
+      # All comments before closing token
+      a.postfix.add move(p.skipped)
       break
     getTok(p)
     splitLookahead(p, a, clPostfix)

src/phrenderer.nim

@@ -36,7 +36,7 @@ const
 type
   TRenderTok* = object
     kind*: TokType
-    length*: int16
+    length*: int
     sym*: PSym
 
   TRenderTokSeq* = seq[TRenderTok]
@@ -101,6 +101,7 @@ type
     sfOneLine ## Use single-line formatting (if possible)
     sfStackDot ## Stack multiple dot-calls
     sfStackDotInCall ## Stacked dotting opportunity
+    sfParDo ## Add parens to `do` to avoid dot-expr ambiguity
 
   SubFlags = set[SubFlag]
   TOutput = TSrcGen | TSrcLen
@@ -211,11 +212,11 @@ proc containsNL(s: string): bool =
 
 proc addTok(g: var TSrcLen, kind: TokType, s: string) =
   g.nl = g.nl or containsNL(s)
-  g.tokens.add TRenderTok(kind: kind, length: int16(s.len))
+  g.tokens.add TRenderTok(kind: kind, length: s.len)
 
 proc addTok(g: var TSrcGen, kind: TokType, s: string) =
   # debugEcho "addTok ", kind, " " , s.len, " ", s
-  g.tokens.add TRenderTok(kind: kind, length: int16(s.len))
+  g.tokens.add TRenderTok(kind: kind, length: s.len)
 
   g.buf.add(s)
   g.line += count(s, "\n")
@@ -808,24 +809,28 @@ proc gcomma(
   # If a full, comma-separate list fits on one line, go for it. If not, we put
   # each element on its own line unless it's a list of trivial things (so as to
   # avoid wasting significant vertical space on lists of numbers and the like)
-  let onePerLine =
-    if not overflows(
-      g,
-      lcomma(
+  let
+    onePerLine =
+      if not overflows(
         g,
-        n,
-        start,
-        theEnd,
-        separator,
-        indentNL,
-        flags - {lfFirstSticky, lfFirstAlone},
-        subFlags,
-      ),
-    ):
-      false
-    else:
-      count > 1 and
-        anyIt(n.sons[sstart .. n.len + theEnd], not isSimple(it, n.kind == nkIdentDefs))
+        lcomma(
+          g,
+          n,
+          start,
+          theEnd,
+          separator,
+          indentNL,
+          flags - {lfFirstSticky, lfFirstAlone},
+          subFlags,
+        ),
+      ):
+        false
+      else:
+        count > 1 and
+          anyIt(
+            n.sons[sstart .. n.len + theEnd], not isSimple(it, n.kind == nkIdentDefs)
+          )
+    sepAtEnd = lfSepAtEnd in flags or onePerLine and lfLongSepAtEnd in flags
 
   for i in start .. n.len + theEnd:
     let c = i < n.len + theEnd
@@ -836,20 +841,21 @@ proc gcomma(
 
     gsub(g, n[i], flags = subFlags + {sfSkipPostfix})
 
-    if c:
+    # In sticky comment mode we put a separator before the comment, else a
+    # comment for a single parameter in an argument list fails to parse
+    if c or sepAtEnd or (lfFirstCommentSticky in flags and n[i].postfix.len > 0):
       if g.tokens.len > oldLen:
         if lfSkipPushComma notin flags or not eqIdent(n[i], "push"):
           let sep = separator(n[i])
-          putWithSpace(g, sep, $sep)
+          put(g, sep, $sep)
+
+          if c:
+            optSpace(g)
         else:
           optSpace(g)
 
     gpostfixes(g, n[i], lfFirstCommentSticky in flags)
 
-  if lfSepAtEnd in flags or onePerLine and lfLongSepAtEnd in flags:
-    let sep = separator(n[n.len + theEnd])
-    put(g, sep, $sep)
-
 proc gsons(
     g: var TOutput, n: PNode, start: int = 0, theEnd: int = -1, flags: SubFlags = {}
 ) =
@@ -1495,6 +1501,18 @@ proc gsub(g: var TOutput, n: PNode, flags: SubFlags, extra: int) =
     put(g, tkNil, atom(g, n)) # complex expressions
   of nkCall, nkConv, nkPattern, nkObjConstr:
     if n.len > 1 and n.lastSon.kind in postExprBlocks:
+      let
+        doPars =
+          if n.lastSon.kind == nkDo and sfParDo in flags:
+            # A dot-expr that ends with a call with a `do` needs an extra set of
+            # parens to highlight where the `do` ends.
+            put(g, tkParLe, $tkParLe)
+            optNL(g)
+            true
+          else:
+            false
+        ind = condIndent(g, doPars)
+
       accentedName(g, n[0], flags = (flags * {sfStackDot}) + {sfStackDotInCall})
 
       var i = 1
@@ -1507,6 +1525,10 @@ proc gsub(g: var TOutput, n: PNode, flags: SubFlags, extra: int) =
         )
 
       postStatements(g, n, i, sfSkipDo in flags)
+      dedent(g, ind)
+
+      if n.lastSon.kind == nkDo and sfParDo in flags:
+        put(g, tkParRi, $tkParRi)
     elif n.len >= 1:
       accentedName(g, n[0], flags = (flags * {sfStackDot}) + {sfStackDotInCall})
       glist(g, n, tkParLe, start = 1, flags = {lfLongSepAtEnd})
@@ -1595,10 +1617,12 @@ proc gsub(g: var TOutput, n: PNode, flags: SubFlags, extra: int) =
     glist(g, n, tkParLe, subflags = {sfNoIndent})
   of nkTupleConstr:
     let flags =
-      if n.len == 1 and n[0].kind != nkExprColonExpr:
-        {lfSepAtEnd}
-      else:
-        {lfLongSepAtEnd}
+      {lfFirstCommentSticky} + (
+        if n.len == 1 and n[0].kind != nkExprColonExpr:
+          {lfSepAtEnd}
+        else:
+          {lfLongSepAtEnd}
+      )
 
     glist(g, n, tkParLe, flags = flags)
   of nkCurly:
@@ -1629,10 +1653,12 @@ proc gsub(g: var TOutput, n: PNode, flags: SubFlags, extra: int) =
           )
         stackNL = stackDot and sfStackDotInCall in flags
         subFlags =
-          if stackDot:
-            {sfStackDot}
-          else:
-            {}
+          {sfParDo} + (
+            if stackDot:
+              {sfStackDot}
+            else:
+              {}
+          )
 
       gsub(g, n[0], flags = subFlags)
 
@@ -2177,15 +2203,23 @@ proc gsub(g: var TOutput, n: PNode, flags: SubFlags, extra: int) =
         extra = retExtra,
         start = 1,
         indentNL = flagIndent(flags),
-        flags = {lfLongSepAtEnd},
+        flags = {lfLongSepAtEnd, lfFirstCommentSticky},
       )
 
     if n.len > 0 and n[0].kind != nkEmpty:
       putWithSpace(g, tkColon, ":")
       gsub(g, n[0], flags)
   of nkTupleTy:
     put(g, tkTuple, "tuple")
-    glist(g, n, tkBracketLe)
+    if anyIt(n, hasComments(it)) or n.mid.len > 0:
+      withIndent(g):
+        optNL(g)
+        gmids(g, n)
+        for child in n:
+          optNL(g)
+          gsub(g, child)
+    else:
+      glist(g, n, tkBracketLe)
   of nkTupleClassTy:
     put(g, tkTuple, "tuple")
   of nkTypeClassTy:

tests/after/comments.nim

@@ -125,6 +125,17 @@ type
     ## comment nofield1 eol
     ## comment nl
 
+  CommentedTuple* =
+    tuple
+      ## Comment here
+      field: int ## comment tuple field
+      field2: int ## comment tuple field2
+
+  CommentedTuple2* =
+    tuple
+      ## comment tuple only mid
+      field: int
+
 when defined(somecond): # when colon line
   # when first line
   discard
@@ -375,6 +386,11 @@ proc a(): int =
   ## even two lines again
   42
 
+proc a(
+    param: int, ## doc comment here
+) =
+  discard
+
 command "a", "b", "c" # command eol comment
 
 command "first arg", # first arg comment