diff --git a/src/phlexer.nim b/src/phlexer.nim index 59c9ecb..f2bf033 100644 --- a/src/phlexer.nim +++ b/src/phlexer.nim @@ -1425,8 +1425,8 @@ proc rawGetTok*(L: var Lexer, tok: var Token) = template atTokenEnd() {.dirty.} = L.previousTokenEnd.line = L.tokenEnd.line L.previousTokenEnd.col = L.tokenEnd.col - L.tokenEnd.line = tok.line.uint16 - L.tokenEnd.col = getColNumber(L, L.bufpos).int16 + L.tokenEnd.line = tok.line + L.tokenEnd.col = getColNumber(L, L.bufpos) let lineB = tok.lineB reset(tok) diff --git a/src/phlineinfos.nim b/src/phlineinfos.nim index 7af58b2..94b8ec0 100644 --- a/src/phlineinfos.nim +++ b/src/phlineinfos.nim @@ -352,14 +352,8 @@ type FileIndex* = distinct int32 TLineInfo* = object - # This is designed to be as small as possible, - # because it is used - # in syntax nodes. We save space here by using - # two int16 and an int32. - # On 64 bit and on 32 bit systems this is - # only 8 bytes. - line*: uint16 - col*: int16 + line*: int + col*: int fileIndex*: FileIndex offsetA*, offsetB*: int diff --git a/src/phmsgs.nim b/src/phmsgs.nim index 144f42f..c9d464e 100644 --- a/src/phmsgs.nim +++ b/src/phmsgs.nim @@ -163,15 +163,8 @@ proc fileInfoIdx*(conf: ConfigRef, filename: RelativeFile): FileIndex = proc newLineInfo*(fileInfoIdx: FileIndex, line, col: int): TLineInfo = result.fileIndex = fileInfoIdx - if line < int high(uint16): - result.line = uint16(line) - else: - result.line = high(uint16) - - if col < int high(int16): - result.col = int16(col) - else: - result.col = -1 + result.line = line + result.col = col proc newLineInfo*( conf: ConfigRef, filename: AbsoluteFile, line, col: int diff --git a/src/phparser.nim b/src/phparser.nim index 2e725af..ef8d43c 100644 --- a/src/phparser.nim +++ b/src/phparser.nim @@ -51,7 +51,7 @@ type inSemiStmtList*: int emptyNode: PNode skipped*: seq[Token] - endLine: uint16 + endLine: int SymbolMode = enum smNormal @@ -219,7 +219,7 @@ template setEndInfo(node: PNode) = # for statement lists, which is the only place where we retain whitespace node.endInfo = TLineInfo( fileIndex: p.lex.fileIdx, - line: uint16 p.lineNumberPrevious, + line: p.lineNumberPrevious, col: p.lex.previousTokenEnd.col, ) @@ -488,7 +488,7 @@ proc exprList(p: var Parser, endTok: TokType, result: PNode) = if p.tok.tokType != tkComma: break getTok(p) - splitLookahead(p, a, clPostfix) + splitLookahead(p, result[^1], clPostfix) optInd(p, a) var a = parseExpr(p) result.add(a) @@ -600,6 +600,8 @@ proc setOrTableConstr(p: var Parser): PNode = result.transitionSonsKind(nkTableConstr) result.add(a) if p.tok.tokType != tkComma: + # All comments before closing token + a.postfix.add move(p.skipped) break getTok(p) splitLookahead(p, a, clPostfix) @@ -753,6 +755,8 @@ proc parsePar(p: var Parser): PNode = result.add(a) if p.tok.tokType == tkComma: getTok(p) + splitLookahead(p, a, clPostfix) + # (1,) produces a tuple expression: result.transitionSonsKind(nkTupleConstr) # progress guaranteed @@ -762,6 +766,9 @@ proc parsePar(p: var Parser): PNode = if p.tok.tokType != tkComma: break getTok(p) + splitLookahead(p, a, clPostfix) + # All comments before closing token + result[^1].postfix.add move(p.skipped) optPar(p) eat(p, tkParRi) setEndInfo() @@ -1163,6 +1170,8 @@ proc parseTuple(p: var Parser, indentAllowed = false): PNode = var a = parseIdentColonEquals(p, {}) result.add(a) if p.tok.tokType notin {tkComma, tkSemiColon}: + # All comments before closing token + a.postfix.add move(p.skipped) break getTok(p) splitLookahead(p, a, clPostfix) @@ -1178,6 +1187,7 @@ proc parseTuple(p: var Parser, indentAllowed = false): PNode = case p.tok.tokType of tkSymbol, tkAccent: var a = parseIdentColonEquals(p, {}) + splitLookahead(p, a, clPostfix) result.add(a) of tkEof: break @@ -1223,10 +1233,12 @@ proc parseParamList(p: var Parser, retColon = true): PNode = parMessage(p, "expected closing ')'") break result.add(a) - splitLookahead(p, a, clPostfix) if p.tok.tokType notin {tkComma, tkSemiColon}: + # All comments before closing token + a.postfix.add move(p.skipped) break getTok(p) + splitLookahead(p, a, clPostfix) # Not ideal, but we'll attach the last comment in the par to the last # parameter if result.len > 0: @@ -2111,6 +2123,8 @@ proc parseGenericParamList(p: var Parser): PNode = var a = parseGenericParam(p) result.add(a) if p.tok.tokType notin {tkComma, tkSemiColon}: + # All comments before closing token + a.postfix.add move(p.skipped) break getTok(p) splitLookahead(p, a, clPostfix) @@ -2497,6 +2511,8 @@ proc parseVarTuple(p: var Parser): PNode = a = identWithPragma(p, allowDot = true) result.add(a) if p.tok.tokType != tkComma: + # All comments before closing token + a.postfix.add move(p.skipped) break getTok(p) splitLookahead(p, a, clPostfix) diff --git a/src/phrenderer.nim b/src/phrenderer.nim index b360baf..c0ca456 100644 --- a/src/phrenderer.nim +++ b/src/phrenderer.nim @@ -36,7 +36,7 @@ const type TRenderTok* = object kind*: TokType - length*: int16 + length*: int sym*: PSym TRenderTokSeq* = seq[TRenderTok] @@ -101,6 +101,7 @@ type sfOneLine ## Use single-line formatting (if possible) sfStackDot ## Stack multiple dot-calls sfStackDotInCall ## Stacked dotting opportunity + sfParDo ## Add parens to `do` to avoid dot-expr ambiguity SubFlags = set[SubFlag] TOutput = TSrcGen | TSrcLen @@ -211,11 +212,11 @@ proc containsNL(s: string): bool = proc addTok(g: var TSrcLen, kind: TokType, s: string) = g.nl = g.nl or containsNL(s) - g.tokens.add TRenderTok(kind: kind, length: int16(s.len)) + g.tokens.add TRenderTok(kind: kind, length: s.len) proc addTok(g: var TSrcGen, kind: TokType, s: string) = # debugEcho "addTok ", kind, " " , s.len, " ", s - g.tokens.add TRenderTok(kind: kind, length: int16(s.len)) + g.tokens.add TRenderTok(kind: kind, length: s.len) g.buf.add(s) g.line += count(s, "\n") @@ -808,24 +809,28 @@ proc gcomma( # If a full, comma-separate list fits on one line, go for it. If not, we put # each element on its own line unless it's a list of trivial things (so as to # avoid wasting significant vertical space on lists of numbers and the like) - let onePerLine = - if not overflows( - g, - lcomma( + let + onePerLine = + if not overflows( g, - n, - start, - theEnd, - separator, - indentNL, - flags - {lfFirstSticky, lfFirstAlone}, - subFlags, - ), - ): - false - else: - count > 1 and - anyIt(n.sons[sstart .. n.len + theEnd], not isSimple(it, n.kind == nkIdentDefs)) + lcomma( + g, + n, + start, + theEnd, + separator, + indentNL, + flags - {lfFirstSticky, lfFirstAlone}, + subFlags, + ), + ): + false + else: + count > 1 and + anyIt( + n.sons[sstart .. n.len + theEnd], not isSimple(it, n.kind == nkIdentDefs) + ) + sepAtEnd = lfSepAtEnd in flags or onePerLine and lfLongSepAtEnd in flags for i in start .. n.len + theEnd: let c = i < n.len + theEnd @@ -836,20 +841,21 @@ proc gcomma( gsub(g, n[i], flags = subFlags + {sfSkipPostfix}) - if c: + # In sticky comment mode we put a separator before the comment, else a + # comment for a single parameter in an argument list fails to parse + if c or sepAtEnd or (lfFirstCommentSticky in flags and n[i].postfix.len > 0): if g.tokens.len > oldLen: if lfSkipPushComma notin flags or not eqIdent(n[i], "push"): let sep = separator(n[i]) - putWithSpace(g, sep, $sep) + put(g, sep, $sep) + + if c: + optSpace(g) else: optSpace(g) gpostfixes(g, n[i], lfFirstCommentSticky in flags) - if lfSepAtEnd in flags or onePerLine and lfLongSepAtEnd in flags: - let sep = separator(n[n.len + theEnd]) - put(g, sep, $sep) - proc gsons( g: var TOutput, n: PNode, start: int = 0, theEnd: int = -1, flags: SubFlags = {} ) = @@ -1495,6 +1501,18 @@ proc gsub(g: var TOutput, n: PNode, flags: SubFlags, extra: int) = put(g, tkNil, atom(g, n)) # complex expressions of nkCall, nkConv, nkPattern, nkObjConstr: if n.len > 1 and n.lastSon.kind in postExprBlocks: + let + doPars = + if n.lastSon.kind == nkDo and sfParDo in flags: + # A dot-expr that ends with a call with a `do` needs an extra set of + # parens to highlight where the `do` ends. + put(g, tkParLe, $tkParLe) + optNL(g) + true + else: + false + ind = condIndent(g, doPars) + accentedName(g, n[0], flags = (flags * {sfStackDot}) + {sfStackDotInCall}) var i = 1 @@ -1507,6 +1525,10 @@ proc gsub(g: var TOutput, n: PNode, flags: SubFlags, extra: int) = ) postStatements(g, n, i, sfSkipDo in flags) + dedent(g, ind) + + if n.lastSon.kind == nkDo and sfParDo in flags: + put(g, tkParRi, $tkParRi) elif n.len >= 1: accentedName(g, n[0], flags = (flags * {sfStackDot}) + {sfStackDotInCall}) glist(g, n, tkParLe, start = 1, flags = {lfLongSepAtEnd}) @@ -1595,10 +1617,12 @@ proc gsub(g: var TOutput, n: PNode, flags: SubFlags, extra: int) = glist(g, n, tkParLe, subflags = {sfNoIndent}) of nkTupleConstr: let flags = - if n.len == 1 and n[0].kind != nkExprColonExpr: - {lfSepAtEnd} - else: - {lfLongSepAtEnd} + {lfFirstCommentSticky} + ( + if n.len == 1 and n[0].kind != nkExprColonExpr: + {lfSepAtEnd} + else: + {lfLongSepAtEnd} + ) glist(g, n, tkParLe, flags = flags) of nkCurly: @@ -1629,10 +1653,12 @@ proc gsub(g: var TOutput, n: PNode, flags: SubFlags, extra: int) = ) stackNL = stackDot and sfStackDotInCall in flags subFlags = - if stackDot: - {sfStackDot} - else: - {} + {sfParDo} + ( + if stackDot: + {sfStackDot} + else: + {} + ) gsub(g, n[0], flags = subFlags) @@ -2177,7 +2203,7 @@ proc gsub(g: var TOutput, n: PNode, flags: SubFlags, extra: int) = extra = retExtra, start = 1, indentNL = flagIndent(flags), - flags = {lfLongSepAtEnd}, + flags = {lfLongSepAtEnd, lfFirstCommentSticky}, ) if n.len > 0 and n[0].kind != nkEmpty: @@ -2185,7 +2211,15 @@ proc gsub(g: var TOutput, n: PNode, flags: SubFlags, extra: int) = gsub(g, n[0], flags) of nkTupleTy: put(g, tkTuple, "tuple") - glist(g, n, tkBracketLe) + if anyIt(n, hasComments(it)) or n.mid.len > 0: + withIndent(g): + optNL(g) + gmids(g, n) + for child in n: + optNL(g) + gsub(g, child) + else: + glist(g, n, tkBracketLe) of nkTupleClassTy: put(g, tkTuple, "tuple") of nkTypeClassTy: diff --git a/tests/after/comments.nim b/tests/after/comments.nim index cac82a9..1025ba3 100644 --- a/tests/after/comments.nim +++ b/tests/after/comments.nim @@ -125,6 +125,17 @@ type ## comment nofield1 eol ## comment nl + CommentedTuple* = + tuple + ## Comment here + field: int ## comment tuple field + field2: int ## comment tuple field2 + + CommentedTuple2* = + tuple + ## comment tuple only mid + field: int + when defined(somecond): # when colon line # when first line discard @@ -375,6 +386,11 @@ proc a(): int = ## even two lines again 42 +proc a( + param: int, ## doc comment here +) = + discard + command "a", "b", "c" # command eol comment command "first arg", # first arg comment diff --git a/tests/after/comments.nim.nph.yaml b/tests/after/comments.nim.nph.yaml index 2f71046..73c3956 100644 --- a/tests/after/comments.nim.nph.yaml +++ b/tests/after/comments.nim.nph.yaml @@ -492,6 +492,57 @@ sons: - kind: "nkIdent" ident: "RootObj" - kind: "nkEmpty" + - kind: "nkTypeDef" + sons: + - kind: "nkPostfix" + sons: + - kind: "nkIdent" + ident: "*" + - kind: "nkIdent" + ident: "CommentedTuple" + - kind: "nkEmpty" + - kind: "nkTupleTy" + sons: + - kind: "nkIdentDefs" + prefix: + - "## Comment here" + sons: + - kind: "nkIdent" + ident: "field" + - kind: "nkIdent" + ident: "int" + - kind: "nkEmpty" + postfix: + - "## comment tuple field" + - kind: "nkIdentDefs" + sons: + - kind: "nkIdent" + ident: "field2" + - kind: "nkIdent" + ident: "int" + - kind: "nkEmpty" + postfix: + - "## comment tuple field2" + - kind: "nkTypeDef" + sons: + - kind: "nkPostfix" + sons: + - kind: "nkIdent" + ident: "*" + - kind: "nkIdent" + ident: "CommentedTuple2" + - kind: "nkEmpty" + - kind: "nkTupleTy" + sons: + - kind: "nkIdentDefs" + prefix: + - "## comment tuple only mid" + sons: + - kind: "nkIdent" + ident: "field" + - kind: "nkIdent" + ident: "int" + - kind: "nkEmpty" - kind: "nkWhenStmt" sons: - kind: "nkElifBranch" @@ -1424,6 +1475,31 @@ sons: "comment": "## even two lines again" - kind: "nkIntLit" intVal: 42 + - kind: "nkProcDef" + sons: + - kind: "nkIdent" + ident: "a" + - kind: "nkEmpty" + - kind: "nkEmpty" + - kind: "nkFormalParams" + sons: + - kind: "nkEmpty" + - kind: "nkIdentDefs" + sons: + - kind: "nkIdent" + ident: "param" + - kind: "nkIdent" + ident: "int" + - kind: "nkEmpty" + postfix: + - "## doc comment here" + - kind: "nkEmpty" + - kind: "nkEmpty" + - kind: "nkStmtList" + sons: + - kind: "nkDiscardStmt" + sons: + - kind: "nkEmpty" - kind: "nkCommand" sons: - kind: "nkIdent" diff --git a/tests/after/postexprs.nim b/tests/after/postexprs.nim index 0e400cf..da6f108 100644 --- a/tests/after/postexprs.nim +++ b/tests/after/postexprs.nim @@ -91,3 +91,11 @@ of a: discard else: discard + +discard + ( + aaa.bbb + .exec do(res: int64): + size = res + ) + .ccc() diff --git a/tests/after/postexprs.nim.nph.yaml b/tests/after/postexprs.nim.nph.yaml index 3afc24c..005e7d1 100644 --- a/tests/after/postexprs.nim.nph.yaml +++ b/tests/after/postexprs.nim.nph.yaml @@ -531,3 +531,48 @@ sons: - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" + - kind: "nkDiscardStmt" + sons: + - kind: "nkCall" + sons: + - kind: "nkDotExpr" + sons: + - kind: "nkCall" + sons: + - kind: "nkDotExpr" + sons: + - kind: "nkDotExpr" + sons: + - kind: "nkIdent" + ident: "aaa" + - kind: "nkIdent" + ident: "bbb" + - kind: "nkIdent" + ident: "exec" + - kind: "nkDo" + sons: + - kind: "nkEmpty" + - kind: "nkEmpty" + - kind: "nkEmpty" + - kind: "nkFormalParams" + sons: + - kind: "nkEmpty" + - kind: "nkIdentDefs" + sons: + - kind: "nkIdent" + ident: "res" + - kind: "nkIdent" + ident: "int64" + - kind: "nkEmpty" + - kind: "nkEmpty" + - kind: "nkEmpty" + - kind: "nkStmtList" + sons: + - kind: "nkAsgn" + sons: + - kind: "nkIdent" + ident: "size" + - kind: "nkIdent" + ident: "res" + - kind: "nkIdent" + ident: "ccc" diff --git a/tests/before/comments.nim b/tests/before/comments.nim index bf5446d..b030e87 100644 --- a/tests/before/comments.nim +++ b/tests/before/comments.nim @@ -118,6 +118,15 @@ type NoField1* = object of RootObj ## comment nofield1 eol ## comment nl + CommentedTuple* = tuple + ## Comment here + field: int ## comment tuple field + field2: int ## comment tuple field2 + + CommentedTuple2* = + tuple + ## comment tuple only mid + field: int when defined(somecond): # when colon line # when first line @@ -353,6 +362,10 @@ proc a(): int = 42 ## Doc comment that needs body reordering ## even two lines again +proc a( + param: int, ## doc comment here +) = discard + command "a", "b", "c" # command eol comment command "first arg", # first arg comment diff --git a/tests/before/comments.nim.nph.yaml b/tests/before/comments.nim.nph.yaml index 0b17916..baa019f 100644 --- a/tests/before/comments.nim.nph.yaml +++ b/tests/before/comments.nim.nph.yaml @@ -491,6 +491,57 @@ sons: - kind: "nkIdent" ident: "RootObj" - kind: "nkEmpty" + - kind: "nkTypeDef" + sons: + - kind: "nkPostfix" + sons: + - kind: "nkIdent" + ident: "*" + - kind: "nkIdent" + ident: "CommentedTuple" + - kind: "nkEmpty" + - kind: "nkTupleTy" + sons: + - kind: "nkIdentDefs" + prefix: + - "## Comment here" + sons: + - kind: "nkIdent" + ident: "field" + - kind: "nkIdent" + ident: "int" + - kind: "nkEmpty" + postfix: + - "## comment tuple field" + - kind: "nkIdentDefs" + sons: + - kind: "nkIdent" + ident: "field2" + - kind: "nkIdent" + ident: "int" + - kind: "nkEmpty" + postfix: + - "## comment tuple field2" + - kind: "nkTypeDef" + sons: + - kind: "nkPostfix" + sons: + - kind: "nkIdent" + ident: "*" + - kind: "nkIdent" + ident: "CommentedTuple2" + - kind: "nkEmpty" + - kind: "nkTupleTy" + sons: + - kind: "nkIdentDefs" + prefix: + - "## comment tuple only mid" + sons: + - kind: "nkIdent" + ident: "field" + - kind: "nkIdent" + ident: "int" + - kind: "nkEmpty" - kind: "nkWhenStmt" sons: - kind: "nkElifBranch" @@ -1420,6 +1471,31 @@ sons: postfix: - "## Doc comment that needs body reordering" - "## even two lines again" + - kind: "nkProcDef" + sons: + - kind: "nkIdent" + ident: "a" + - kind: "nkEmpty" + - kind: "nkEmpty" + - kind: "nkFormalParams" + sons: + - kind: "nkEmpty" + - kind: "nkIdentDefs" + sons: + - kind: "nkIdent" + ident: "param" + - kind: "nkIdent" + ident: "int" + - kind: "nkEmpty" + postfix: + - "## doc comment here" + - kind: "nkEmpty" + - kind: "nkEmpty" + - kind: "nkStmtList" + sons: + - kind: "nkDiscardStmt" + sons: + - kind: "nkEmpty" - kind: "nkCommand" sons: - kind: "nkIdent" diff --git a/tests/before/postexprs.nim b/tests/before/postexprs.nim index c87f908..9bd72df 100644 --- a/tests/before/postexprs.nim +++ b/tests/before/postexprs.nim @@ -93,4 +93,7 @@ command "llllllllllllllllooooooooooooooooooonnnnnnnnnnnnnnnnnnnnnngggggggggggggg command param: of a: discard -else: discard \ No newline at end of file +else: discard + +discard (aaa.bbb.exec do(res: int64): + size = res).ccc() diff --git a/tests/before/postexprs.nim.nph.yaml b/tests/before/postexprs.nim.nph.yaml index 3afc24c..005e7d1 100644 --- a/tests/before/postexprs.nim.nph.yaml +++ b/tests/before/postexprs.nim.nph.yaml @@ -531,3 +531,48 @@ sons: - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" + - kind: "nkDiscardStmt" + sons: + - kind: "nkCall" + sons: + - kind: "nkDotExpr" + sons: + - kind: "nkCall" + sons: + - kind: "nkDotExpr" + sons: + - kind: "nkDotExpr" + sons: + - kind: "nkIdent" + ident: "aaa" + - kind: "nkIdent" + ident: "bbb" + - kind: "nkIdent" + ident: "exec" + - kind: "nkDo" + sons: + - kind: "nkEmpty" + - kind: "nkEmpty" + - kind: "nkEmpty" + - kind: "nkFormalParams" + sons: + - kind: "nkEmpty" + - kind: "nkIdentDefs" + sons: + - kind: "nkIdent" + ident: "res" + - kind: "nkIdent" + ident: "int64" + - kind: "nkEmpty" + - kind: "nkEmpty" + - kind: "nkEmpty" + - kind: "nkStmtList" + sons: + - kind: "nkAsgn" + sons: + - kind: "nkIdent" + ident: "size" + - kind: "nkIdent" + ident: "res" + - kind: "nkIdent" + ident: "ccc"