diff --git a/.fantomasignore b/.fantomasignore index 20249273f54..d731fb6d9a5 100644 --- a/.fantomasignore +++ b/.fantomasignore @@ -116,6 +116,7 @@ src/Compiler/Utilities/HashMultiMap.fs src/Compiler/Facilities/AsyncMemoize.fsi src/Compiler/Facilities/AsyncMemoize.fs src/Compiler/AbstractIL/il.fs +src/Compiler/SyntaxTree/LexerStore.fs src/Compiler/Driver/GraphChecking/Graph.fsi src/Compiler/Driver/GraphChecking/Graph.fs diff --git a/src/Compiler/Driver/ParseAndCheckInputs.fs b/src/Compiler/Driver/ParseAndCheckInputs.fs index 4fff95653d4..22ea3c7f033 100644 --- a/src/Compiler/Driver/ParseAndCheckInputs.fs +++ b/src/Compiler/Driver/ParseAndCheckInputs.fs @@ -27,6 +27,7 @@ open FSharp.Compiler.Diagnostics open FSharp.Compiler.DiagnosticsLogger open FSharp.Compiler.Features open FSharp.Compiler.IO +open FSharp.Compiler.LexerStore open FSharp.Compiler.Lexhelp open FSharp.Compiler.NameResolution open FSharp.Compiler.ParseHelpers @@ -239,7 +240,7 @@ let GetScopedPragmasForHashDirective hd (langVersion: LanguageVersion) = let private collectCodeComments (lexbuf: UnicodeLexing.Lexbuf) (tripleSlashComments: range list) = [ - yield! LexbufCommentStore.GetComments(lexbuf) + yield! CommentStore.GetComments(lexbuf) yield! (List.map CommentTrivia.LineComment tripleSlashComments) ] |> List.sortBy (function @@ -285,7 +286,7 @@ let PostParseModuleImpls yield! GetScopedPragmasForHashDirective hd lexbuf.LanguageVersion ] - let conditionalDirectives = LexbufIfdefStore.GetTrivia(lexbuf) + let conditionalDirectives = IfdefStore.GetTrivia(lexbuf) let codeComments = collectCodeComments lexbuf tripleSlashComments let trivia: ParsedImplFileInputTrivia = @@ -336,7 +337,7 @@ let PostParseModuleSpecs yield! GetScopedPragmasForHashDirective hd lexbuf.LanguageVersion ] - let conditionalDirectives = LexbufIfdefStore.GetTrivia(lexbuf) + let conditionalDirectives = IfdefStore.GetTrivia(lexbuf) let codeComments = collectCodeComments lexbuf tripleSlashComments let trivia: ParsedSigFileInputTrivia = @@ -488,15 +489,13 @@ let ParseInput if FSharpImplFileSuffixes |> List.exists (FileSystemUtils.checkSuffix fileName) then let impl = Parser.implementationFile lexer lexbuf - let tripleSlashComments = - LexbufLocalXmlDocStore.ReportInvalidXmlDocPositions(lexbuf) + let tripleSlashComments = XmlDocStore.ReportInvalidXmlDocPositions(lexbuf) PostParseModuleImpls(defaultNamespace, fileName, isLastCompiland, impl, lexbuf, tripleSlashComments, Set identStore) elif FSharpSigFileSuffixes |> List.exists (FileSystemUtils.checkSuffix fileName) then let intfs = Parser.signatureFile lexer lexbuf - let tripleSlashComments = - LexbufLocalXmlDocStore.ReportInvalidXmlDocPositions(lexbuf) + let tripleSlashComments = XmlDocStore.ReportInvalidXmlDocPositions(lexbuf) PostParseModuleSpecs(defaultNamespace, fileName, isLastCompiland, intfs, lexbuf, tripleSlashComments, Set identStore) else if lexbuf.SupportsFeature LanguageFeature.MLCompatRevisions then diff --git a/src/Compiler/FSharp.Compiler.Service.fsproj b/src/Compiler/FSharp.Compiler.Service.fsproj index 659eb21edb9..419746af6c0 100644 --- a/src/Compiler/FSharp.Compiler.Service.fsproj +++ b/src/Compiler/FSharp.Compiler.Service.fsproj @@ -247,11 +247,11 @@ SyntaxTree\pplex.fsl - --module FSharp.Compiler.PPParser --open FSharp.Compiler.ParseHelpers --internal --lexlib Internal.Utilities.Text.Lexing --parslib Internal.Utilities.Text.Parsing --buffer-type-argument char + --module FSharp.Compiler.PPParser --open FSharp.Compiler.ParseHelpers --open FSharp.Compiler.LexerStore --internal --lexlib Internal.Utilities.Text.Lexing --parslib Internal.Utilities.Text.Parsing --buffer-type-argument char SyntaxTree\pppars.fsy - --module FSharp.Compiler.Lexer --open FSharp.Compiler.Lexhelp --open Internal.Utilities.Text.Lexing --open FSharp.Compiler.Parser --open FSharp.Compiler.Text --open FSharp.Compiler.ParseHelpers --internal --unicode --lexlib Internal.Utilities.Text.Lexing + --module FSharp.Compiler.Lexer --open FSharp.Compiler.Lexhelp --open Internal.Utilities.Text.Lexing --open FSharp.Compiler.Parser --open FSharp.Compiler.Text --open FSharp.Compiler.ParseHelpers --open FSharp.Compiler.LexerStore --internal --unicode --lexlib Internal.Utilities.Text.Lexing SyntaxTree\lex.fsl @@ -280,6 +280,8 @@ + + diff --git a/src/Compiler/SyntaxTree/LexFilter.fs b/src/Compiler/SyntaxTree/LexFilter.fs index 19f7be5d31b..20af46524b0 100644 --- a/src/Compiler/SyntaxTree/LexFilter.fs +++ b/src/Compiler/SyntaxTree/LexFilter.fs @@ -7,11 +7,11 @@ module internal FSharp.Compiler.LexFilter open System open System.Collections.Generic open Internal.Utilities.Text.Lexing -open FSharp.Compiler open Internal.Utilities.Library open FSharp.Compiler.AbstractIL.Diagnostics open FSharp.Compiler.DiagnosticsLogger open FSharp.Compiler.Features +open FSharp.Compiler.LexerStore open FSharp.Compiler.Lexhelp open FSharp.Compiler.ParseHelpers open FSharp.Compiler.Parser @@ -665,7 +665,7 @@ type LexFilterImpl ( let lastTokenEnd = state.EndPos let token = lexer lexbuf - LexbufLocalXmlDocStore.AddGrabPoint(lexbuf) + XmlDocStore.AddGrabPoint(lexbuf) // Now we've got the token, remember the lexbuf state, associating it with the token // and remembering it as the last observed lexbuf state for the wrapped lexer function. diff --git a/src/Compiler/SyntaxTree/LexHelpers.fs b/src/Compiler/SyntaxTree/LexHelpers.fs index 96b4a9a570b..f7f090a6b00 100644 --- a/src/Compiler/SyntaxTree/LexHelpers.fs +++ b/src/Compiler/SyntaxTree/LexHelpers.fs @@ -12,6 +12,7 @@ open Internal.Utilities.Text.Lexing open FSharp.Compiler.IO open FSharp.Compiler.DiagnosticsLogger open FSharp.Compiler.Features +open FSharp.Compiler.LexerStore open FSharp.Compiler.ParseHelpers open FSharp.Compiler.UnicodeLexing open FSharp.Compiler.Parser @@ -98,10 +99,8 @@ let mkLexargs } /// Register the lexbuf and call the given function -let reusingLexbufForParsing lexbuf f = +let reusingLexbufForParsing (lexbuf: Lexbuf) f = use _ = UseBuildPhase BuildPhase.Parse - LexbufLocalXmlDocStore.ClearXmlDoc lexbuf - LexbufCommentStore.ClearComments lexbuf try f () diff --git a/src/Compiler/SyntaxTree/LexerStore.fs b/src/Compiler/SyntaxTree/LexerStore.fs new file mode 100644 index 00000000000..711e9530cd9 --- /dev/null +++ b/src/Compiler/SyntaxTree/LexerStore.fs @@ -0,0 +1,169 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +module internal FSharp.Compiler.LexerStore + +open FSharp.Compiler.SyntaxTreeOps +open FSharp.Compiler.SyntaxTrivia +open FSharp.Compiler.UnicodeLexing +open FSharp.Compiler.Text +open FSharp.Compiler.Text.Position +open FSharp.Compiler.Text.Range +open FSharp.Compiler.Xml + +//------------------------------------------------------------------------ +// Lexbuf.BufferLocalStore is used during lexing/parsing of a file for different purposes. +// All access happens through the functions and modules below. +//------------------------------------------------------------------------ + +let private getStoreData<'T when 'T: not null> (lexbuf: Lexbuf) key (getInitialData: unit -> 'T) = + let store = lexbuf.BufferLocalStore + + match store.TryGetValue key with + | true, data -> data :?> 'T + | _ -> + let data = getInitialData () + store[key] <- data + data + +let private tryGetStoreData<'T when 'T: not null> (lexbuf: Lexbuf) key = + let store = lexbuf.BufferLocalStore + + match store.TryGetValue key with + | true, data -> Some(data :?> 'T) + | _ -> None + +let private setStoreData (lexbuf: Lexbuf) key data = lexbuf.BufferLocalStore[key] <- data + +//------------------------------------------------------------------------ +// A SynArgNameGenerator for the current file, used by the parser +//------------------------------------------------------------------------ + +let getSynArgNameGenerator (lexbuf: Lexbuf) = + getStoreData lexbuf "SynArgNameGenerator" SynArgNameGenerator + +//------------------------------------------------------------------------ +// A XmlDocCollector, used to hold the current accumulated Xml doc lines, and related access functions +//------------------------------------------------------------------------ + +[] +module XmlDocStore = + let private xmlDocKey = "XmlDoc" + + let private getCollector (lexbuf: Lexbuf) = + getStoreData lexbuf xmlDocKey XmlDocCollector + + /// Called from the lexer to save a single line of XML doc comment. + let SaveXmlDocLine (lexbuf: Lexbuf, lineText, range: range) = + let collector = getCollector lexbuf + collector.AddXmlDocLine(lineText, range) + + let AddGrabPoint (lexbuf: Lexbuf) = + let collector = getCollector lexbuf + let startPos = lexbuf.StartPos + collector.AddGrabPoint(mkPos startPos.Line startPos.Column) + + /// Allowed cases when there are comments after XmlDoc + /// + /// /// X xmlDoc + /// // comment + /// //// comment + /// (* multiline comment *) + /// let x = ... // X xmlDoc + /// + /// Remember the first position when a comment (//, (* *), ////) is encountered after the XmlDoc block + /// then add a grab point if a new XmlDoc block follows the comments + let AddGrabPointDelayed (lexbuf: Lexbuf) = + let collector = getCollector lexbuf + let startPos = lexbuf.StartPos + collector.AddGrabPointDelayed(mkPos startPos.Line startPos.Column) + + /// Called from the parser each time we parse a construct that marks the end of an XML doc comment range, + /// e.g. a 'type' declaration. The markerRange is the range of the keyword that delimits the construct. + let GrabXmlDocBeforeMarker (lexbuf: Lexbuf, markerRange: range) = + match tryGetStoreData lexbuf xmlDocKey with + | Some collector -> PreXmlDoc.CreateFromGrabPoint(collector, markerRange.Start) + | _ -> PreXmlDoc.Empty + + let ReportInvalidXmlDocPositions (lexbuf: Lexbuf) = + let collector = getCollector lexbuf + collector.CheckInvalidXmlDocPositions() + +//------------------------------------------------------------------------ +// Storage to hold the current accumulated ConditionalDirectiveTrivia, and related types and access functions +//------------------------------------------------------------------------ + +type LexerIfdefExpression = + | IfdefAnd of LexerIfdefExpression * LexerIfdefExpression + | IfdefOr of LexerIfdefExpression * LexerIfdefExpression + | IfdefNot of LexerIfdefExpression + | IfdefId of string + +let rec LexerIfdefEval (lookup: string -> bool) = + function + | IfdefAnd(l, r) -> (LexerIfdefEval lookup l) && (LexerIfdefEval lookup r) + | IfdefOr(l, r) -> (LexerIfdefEval lookup l) || (LexerIfdefEval lookup r) + | IfdefNot e -> not (LexerIfdefEval lookup e) + | IfdefId id -> lookup id + +[] +module IfdefStore = + let private getStore (lexbuf: Lexbuf) = + getStoreData lexbuf "Ifdef" ResizeArray + + let private mkRangeWithoutLeadingWhitespace (lexed: string) (m: range) : range = + let startColumn = lexed.Length - lexed.TrimStart().Length + mkFileIndexRange m.FileIndex (mkPos m.StartLine startColumn) m.End + + let SaveIfHash (lexbuf: Lexbuf, lexed: string, expr: LexerIfdefExpression, range: range) = + let store = getStore lexbuf + + let expr = + let rec visit (expr: LexerIfdefExpression) : IfDirectiveExpression = + match expr with + | LexerIfdefExpression.IfdefAnd(l, r) -> IfDirectiveExpression.And(visit l, visit r) + | LexerIfdefExpression.IfdefOr(l, r) -> IfDirectiveExpression.Or(visit l, visit r) + | LexerIfdefExpression.IfdefNot e -> IfDirectiveExpression.Not(visit e) + | LexerIfdefExpression.IfdefId id -> IfDirectiveExpression.Ident id + + visit expr + + let m = mkRangeWithoutLeadingWhitespace lexed range + + store.Add(ConditionalDirectiveTrivia.If(expr, m)) + + let SaveElseHash (lexbuf: Lexbuf, lexed: string, range: range) = + let store = getStore lexbuf + let m = mkRangeWithoutLeadingWhitespace lexed range + store.Add(ConditionalDirectiveTrivia.Else(m)) + + let SaveEndIfHash (lexbuf: Lexbuf, lexed: string, range: range) = + let store = getStore lexbuf + let m = mkRangeWithoutLeadingWhitespace lexed range + store.Add(ConditionalDirectiveTrivia.EndIf(m)) + + let GetTrivia (lexbuf: Lexbuf) : ConditionalDirectiveTrivia list = + let store = getStore lexbuf + Seq.toList store + +//------------------------------------------------------------------------ +// Storage to hold the current accumulated CommentTrivia, and related access functions +//------------------------------------------------------------------------ + +[] +module CommentStore = + let private getStore (lexbuf: Lexbuf) = + getStoreData lexbuf "Comments" ResizeArray + + let SaveSingleLineComment (lexbuf: Lexbuf, startRange: range, endRange: range) = + let store = getStore lexbuf + let m = unionRanges startRange endRange + store.Add(CommentTrivia.LineComment(m)) + + let SaveBlockComment (lexbuf: Lexbuf, startRange: range, endRange: range) = + let store = getStore lexbuf + let m = unionRanges startRange endRange + store.Add(CommentTrivia.BlockComment(m)) + + let GetComments (lexbuf: Lexbuf) : CommentTrivia list = + let store = getStore lexbuf + Seq.toList store diff --git a/src/Compiler/SyntaxTree/LexerStore.fsi b/src/Compiler/SyntaxTree/LexerStore.fsi new file mode 100644 index 00000000000..952a0cc6f81 --- /dev/null +++ b/src/Compiler/SyntaxTree/LexerStore.fsi @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +module internal FSharp.Compiler.LexerStore + +open FSharp.Compiler.SyntaxTreeOps +open FSharp.Compiler.SyntaxTrivia +open FSharp.Compiler.UnicodeLexing +open FSharp.Compiler.Text +open FSharp.Compiler.Xml + +val getSynArgNameGenerator: Lexbuf -> SynArgNameGenerator + +[] +module XmlDocStore = + + val SaveXmlDocLine: lexbuf: Lexbuf * lineText: string * range: range -> unit + + val GrabXmlDocBeforeMarker: lexbuf: Lexbuf * markerRange: range -> PreXmlDoc + + val AddGrabPoint: lexbuf: Lexbuf -> unit + + val AddGrabPointDelayed: lexbuf: Lexbuf -> unit + + val ReportInvalidXmlDocPositions: lexbuf: Lexbuf -> range list + +type LexerIfdefExpression = + | IfdefAnd of LexerIfdefExpression * LexerIfdefExpression + | IfdefOr of LexerIfdefExpression * LexerIfdefExpression + | IfdefNot of LexerIfdefExpression + | IfdefId of string + +val LexerIfdefEval: lookup: (string -> bool) -> _arg1: LexerIfdefExpression -> bool + +[] +module IfdefStore = + + val SaveIfHash: lexbuf: Lexbuf * lexed: string * expr: LexerIfdefExpression * range: range -> unit + + val SaveElseHash: lexbuf: Lexbuf * lexed: string * range: range -> unit + + val SaveEndIfHash: lexbuf: Lexbuf * lexed: string * range: range -> unit + + val GetTrivia: lexbuf: Lexbuf -> ConditionalDirectiveTrivia list + +[] +module CommentStore = + + val SaveSingleLineComment: lexbuf: Lexbuf * startRange: range * endRange: range -> unit + + val SaveBlockComment: lexbuf: Lexbuf * startRange: range * endRange: range -> unit + + val GetComments: lexbuf: Lexbuf -> CommentTrivia list diff --git a/src/Compiler/SyntaxTree/ParseHelpers.fs b/src/Compiler/SyntaxTree/ParseHelpers.fs index 59fb645bd0a..1c0428eb835 100644 --- a/src/Compiler/SyntaxTree/ParseHelpers.fs +++ b/src/Compiler/SyntaxTree/ParseHelpers.fs @@ -16,6 +16,7 @@ open FSharp.Compiler.Xml open Internal.Utilities.Library open Internal.Utilities.Text.Lexing open Internal.Utilities.Text.Parsing +open FSharp.Compiler.LexerStore //------------------------------------------------------------------------ // Parsing: Error recovery exception for fsyacc @@ -68,86 +69,6 @@ let rhs2 (parseState: IParseState) i j = /// Get the range corresponding to one of the r.h.s. symbols of a grammar rule while it is being reduced let rhs parseState i = rhs2 parseState i i -type IParseState with - - /// Get the generator used for compiler-generated argument names. - member x.SynArgNameGenerator = - let key = "SynArgNameGenerator" - let bls = x.LexBuffer.BufferLocalStore - - let gen = - match bls.TryGetValue key with - | true, gen -> gen - | _ -> - let gen = !!(box (SynArgNameGenerator())) - bls[key] <- gen - gen - - gen :?> SynArgNameGenerator - - /// Reset the generator used for compiler-generated argument names. - member x.ResetSynArgNameGenerator() = x.SynArgNameGenerator.Reset() - -//------------------------------------------------------------------------ -// Parsing: grabbing XmlDoc -//------------------------------------------------------------------------ - -/// XmlDoc F# lexer/parser state, held in the BufferLocalStore for the lexer. -module LexbufLocalXmlDocStore = - // The key into the BufferLocalStore used to hold the current accumulated XmlDoc lines - let private xmlDocKey = "XmlDoc" - - let private getCollector (lexbuf: Lexbuf) = - match lexbuf.BufferLocalStore.TryGetValue xmlDocKey with - | true, collector -> collector - | _ -> - let collector = !!(box (XmlDocCollector())) - lexbuf.BufferLocalStore[xmlDocKey] <- collector - collector - - |> unbox - - let ClearXmlDoc (lexbuf: Lexbuf) = - lexbuf.BufferLocalStore[xmlDocKey] <- box (XmlDocCollector()) |> Unchecked.nonNull - - /// Called from the lexer to save a single line of XML doc comment. - let SaveXmlDocLine (lexbuf: Lexbuf, lineText, range: range) = - let collector = getCollector lexbuf - collector.AddXmlDocLine(lineText, range) - - let AddGrabPoint (lexbuf: Lexbuf) = - let collector = getCollector lexbuf - let startPos = lexbuf.StartPos - collector.AddGrabPoint(mkPos startPos.Line startPos.Column) - - /// Allowed cases when there are comments after XmlDoc - /// - /// /// X xmlDoc - /// // comment - /// //// comment - /// (* multiline comment *) - /// let x = ... // X xmlDoc - /// - /// Remember the first position when a comment (//, (* *), ////) is encountered after the XmlDoc block - /// then add a grab point if a new XmlDoc block follows the comments - let AddGrabPointDelayed (lexbuf: Lexbuf) = - let collector = getCollector lexbuf - let startPos = lexbuf.StartPos - collector.AddGrabPointDelayed(mkPos startPos.Line startPos.Column) - - /// Called from the parser each time we parse a construct that marks the end of an XML doc comment range, - /// e.g. a 'type' declaration. The markerRange is the range of the keyword that delimits the construct. - let GrabXmlDocBeforeMarker (lexbuf: Lexbuf, markerRange: range) = - match lexbuf.BufferLocalStore.TryGetValue xmlDocKey with - | true, collector -> - let collector = unbox (collector) - PreXmlDoc.CreateFromGrabPoint(collector, markerRange.Start) - | _ -> PreXmlDoc.Empty - - let ReportInvalidXmlDocPositions (lexbuf: Lexbuf) = - let collector = getCollector lexbuf - collector.CheckInvalidXmlDocPositions() - //------------------------------------------------------------------------ // Parsing/lexing: status of #if/#endif processing in lexing, used for continuations // for whitespace tokens in parser specification. @@ -170,100 +91,6 @@ type LexerEndlineContinuation = | Token | IfdefSkip of int * range: range -type LexerIfdefExpression = - | IfdefAnd of LexerIfdefExpression * LexerIfdefExpression - | IfdefOr of LexerIfdefExpression * LexerIfdefExpression - | IfdefNot of LexerIfdefExpression - | IfdefId of string - -let rec LexerIfdefEval (lookup: string -> bool) = - function - | IfdefAnd(l, r) -> (LexerIfdefEval lookup l) && (LexerIfdefEval lookup r) - | IfdefOr(l, r) -> (LexerIfdefEval lookup l) || (LexerIfdefEval lookup r) - | IfdefNot e -> not (LexerIfdefEval lookup e) - | IfdefId id -> lookup id - -/// Ifdef F# lexer/parser state, held in the BufferLocalStore for the lexer. -/// Used to capture #if, #else and #endif as syntax trivia. -module LexbufIfdefStore = - // The key into the BufferLocalStore used to hold the compiler directives - let private ifDefKey = "Ifdef" - - let private getStore (lexbuf: Lexbuf) : ResizeArray = - match lexbuf.BufferLocalStore.TryGetValue ifDefKey with - | true, store -> store - | _ -> - let store = !!(box (ResizeArray())) - lexbuf.BufferLocalStore[ifDefKey] <- store - store - |> unbox> - - let private mkRangeWithoutLeadingWhitespace (lexed: string) (m: range) : range = - let startColumn = lexed.Length - lexed.TrimStart().Length - mkFileIndexRange m.FileIndex (mkPos m.StartLine startColumn) m.End - - let SaveIfHash (lexbuf: Lexbuf, lexed: string, expr: LexerIfdefExpression, range: range) = - let store = getStore lexbuf - - let expr = - let rec visit (expr: LexerIfdefExpression) : IfDirectiveExpression = - match expr with - | LexerIfdefExpression.IfdefAnd(l, r) -> IfDirectiveExpression.And(visit l, visit r) - | LexerIfdefExpression.IfdefOr(l, r) -> IfDirectiveExpression.Or(visit l, visit r) - | LexerIfdefExpression.IfdefNot e -> IfDirectiveExpression.Not(visit e) - | LexerIfdefExpression.IfdefId id -> IfDirectiveExpression.Ident id - - visit expr - - let m = mkRangeWithoutLeadingWhitespace lexed range - - store.Add(ConditionalDirectiveTrivia.If(expr, m)) - - let SaveElseHash (lexbuf: Lexbuf, lexed: string, range: range) = - let store = getStore lexbuf - let m = mkRangeWithoutLeadingWhitespace lexed range - store.Add(ConditionalDirectiveTrivia.Else(m)) - - let SaveEndIfHash (lexbuf: Lexbuf, lexed: string, range: range) = - let store = getStore lexbuf - let m = mkRangeWithoutLeadingWhitespace lexed range - store.Add(ConditionalDirectiveTrivia.EndIf(m)) - - let GetTrivia (lexbuf: Lexbuf) : ConditionalDirectiveTrivia list = - let store = getStore lexbuf - Seq.toList store - -/// Used to capture the ranges of code comments as syntax trivia -module LexbufCommentStore = - // The key into the BufferLocalStore used to hold the compiler directives - let private commentKey = "Comments" - - let private getStore (lexbuf: Lexbuf) : ResizeArray = - match lexbuf.BufferLocalStore.TryGetValue commentKey with - | true, store -> store - | _ -> - let store = !!(box (ResizeArray())) - lexbuf.BufferLocalStore[commentKey] <- store - store - |> unbox> - - let SaveSingleLineComment (lexbuf: Lexbuf, startRange: range, endRange: range) = - let store = getStore lexbuf - let m = unionRanges startRange endRange - store.Add(CommentTrivia.LineComment(m)) - - let SaveBlockComment (lexbuf: Lexbuf, startRange: range, endRange: range) = - let store = getStore lexbuf - let m = unionRanges startRange endRange - store.Add(CommentTrivia.BlockComment(m)) - - let GetComments (lexbuf: Lexbuf) : CommentTrivia list = - let store = getStore lexbuf - Seq.toList store - - let ClearComments (lexbuf: Lexbuf) : unit = - lexbuf.BufferLocalStore.Remove(commentKey) |> ignore - //------------------------------------------------------------------------ // Parsing: continuations for whitespace tokens //------------------------------------------------------------------------ @@ -407,7 +234,7 @@ let grabXmlDocAtRangeStart (parseState: IParseState, optAttributes: SynAttribute | [] -> range | h :: _ -> h.Range - LexbufLocalXmlDocStore.GrabXmlDocBeforeMarker(parseState.LexBuffer, grabPoint) + XmlDocStore.GrabXmlDocBeforeMarker(parseState.LexBuffer, grabPoint) let grabXmlDoc (parseState: IParseState, optAttributes: SynAttributeList list, elemIdx) = grabXmlDocAtRangeStart (parseState, optAttributes, rhs parseState elemIdx) diff --git a/src/Compiler/SyntaxTree/ParseHelpers.fsi b/src/Compiler/SyntaxTree/ParseHelpers.fsi index f769b14a864..84be5f0e4f7 100644 --- a/src/Compiler/SyntaxTree/ParseHelpers.fsi +++ b/src/Compiler/SyntaxTree/ParseHelpers.fsi @@ -38,25 +38,6 @@ val rhs2: parseState: IParseState -> i: int -> j: int -> range val rhs: parseState: IParseState -> i: int -> range -type IParseState with - - member SynArgNameGenerator: SyntaxTreeOps.SynArgNameGenerator - member ResetSynArgNameGenerator: unit -> unit - -module LexbufLocalXmlDocStore = - - val ClearXmlDoc: lexbuf: UnicodeLexing.Lexbuf -> unit - - val SaveXmlDocLine: lexbuf: UnicodeLexing.Lexbuf * lineText: string * range: range -> unit - - val GrabXmlDocBeforeMarker: lexbuf: UnicodeLexing.Lexbuf * markerRange: range -> PreXmlDoc - - val AddGrabPoint: lexbuf: UnicodeLexing.Lexbuf -> unit - - val AddGrabPointDelayed: lexbuf: UnicodeLexing.Lexbuf -> unit - - val ReportInvalidXmlDocPositions: lexbuf: UnicodeLexing.Lexbuf -> range list - type LexerIfdefStackEntry = | IfDefIf | IfDefElse @@ -69,34 +50,6 @@ type LexerEndlineContinuation = | Token | IfdefSkip of int * range: range -type LexerIfdefExpression = - | IfdefAnd of LexerIfdefExpression * LexerIfdefExpression - | IfdefOr of LexerIfdefExpression * LexerIfdefExpression - | IfdefNot of LexerIfdefExpression - | IfdefId of string - -val LexerIfdefEval: lookup: (string -> bool) -> _arg1: LexerIfdefExpression -> bool - -module LexbufIfdefStore = - - val SaveIfHash: lexbuf: UnicodeLexing.Lexbuf * lexed: string * expr: LexerIfdefExpression * range: range -> unit - - val SaveElseHash: lexbuf: UnicodeLexing.Lexbuf * lexed: string * range: range -> unit - - val SaveEndIfHash: lexbuf: UnicodeLexing.Lexbuf * lexed: string * range: range -> unit - - val GetTrivia: lexbuf: UnicodeLexing.Lexbuf -> ConditionalDirectiveTrivia list - -module LexbufCommentStore = - - val SaveSingleLineComment: lexbuf: UnicodeLexing.Lexbuf * startRange: range * endRange: range -> unit - - val SaveBlockComment: lexbuf: UnicodeLexing.Lexbuf * startRange: range * endRange: range -> unit - - val GetComments: lexbuf: UnicodeLexing.Lexbuf -> CommentTrivia list - - val ClearComments: lexbuf: UnicodeLexing.Lexbuf -> unit - [] type LexerStringStyle = | Verbatim diff --git a/src/Compiler/lex.fsl b/src/Compiler/lex.fsl index 9de91ebab74..34bfe4bfe17 100644 --- a/src/Compiler/lex.fsl +++ b/src/Compiler/lex.fsl @@ -185,7 +185,7 @@ let trySaveXmlDoc (lexbuf: LexBuffer) (buff: (range * StringBuilder) optio | None -> () | Some (start, sb) -> let xmlCommentLineRange = mkFileIndexRange start.FileIndex start.Start (posOfLexPosition lexbuf.StartPos) - LexbufLocalXmlDocStore.SaveXmlDocLine (lexbuf, sb.ToString(), xmlCommentLineRange) + XmlDocStore.SaveXmlDocLine (lexbuf, sb.ToString(), xmlCommentLineRange) let tryAppendXmlDoc (buff: (range * StringBuilder) option) (s:string) = match buff with @@ -733,7 +733,7 @@ rule token (args: LexArgs) (skip: bool) = parse | "////" op_char* { // 4+ slash are 1-line comments, online 3 slash are XmlDoc let m = lexbuf.LexemeRange - LexbufLocalXmlDocStore.AddGrabPointDelayed(lexbuf) + XmlDocStore.AddGrabPointDelayed(lexbuf) if not skip then LINE_COMMENT (LexCont.SingleLineComment(args.ifdefStack, args.stringNest, 1, m)) else singleLineComment (None,1,m,m,args) skip lexbuf } @@ -748,7 +748,7 @@ rule token (args: LexArgs) (skip: bool) = parse | "//" op_char* { // Need to read all operator symbols too, otherwise it might be parsed by a rule below let m = lexbuf.LexemeRange - LexbufLocalXmlDocStore.AddGrabPointDelayed(lexbuf) + XmlDocStore.AddGrabPointDelayed(lexbuf) if not skip then LINE_COMMENT (LexCont.SingleLineComment(args.ifdefStack, args.stringNest, 1, m)) else singleLineComment (None,1,m,m,args) skip lexbuf } @@ -1036,7 +1036,7 @@ rule token (args: LexArgs) (skip: bool) = parse let lexed = lexeme lexbuf let isTrue, expr = evalIfDefExpression lexbuf.StartPos lexbuf.ReportLibraryOnlyFeatures lexbuf.LanguageVersion lexbuf.StrictIndentation args lookup lexed args.ifdefStack <- (IfDefIf,m) :: args.ifdefStack - LexbufIfdefStore.SaveIfHash(lexbuf, lexed, expr, m) + IfdefStore.SaveIfHash(lexbuf, lexed, expr, m) let contCase = if isTrue then LexerEndlineContinuation.Token else LexerEndlineContinuation.IfdefSkip(0, m) let tok = HASH_IF(m, lexed, LexCont.EndLine(args.ifdefStack, args.stringNest, contCase)) if skip then endline contCase args skip lexbuf else tok } @@ -1050,7 +1050,7 @@ rule token (args: LexArgs) (skip: bool) = parse let m = lexbuf.LexemeRange shouldStartLine args lexbuf m (FSComp.SR.lexHashElseMustBeFirst()) args.ifdefStack <- (IfDefElse,m) :: rest - LexbufIfdefStore.SaveElseHash(lexbuf, lexed, m) + IfdefStore.SaveElseHash(lexbuf, lexed, m) let tok = HASH_ELSE(m, lexed, LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.IfdefSkip(0, m))) if skip then endline (LexerEndlineContinuation.IfdefSkip(0, m)) args skip lexbuf else tok } @@ -1062,7 +1062,7 @@ rule token (args: LexArgs) (skip: bool) = parse | _ :: rest -> shouldStartLine args lexbuf m (FSComp.SR.lexHashEndifMustBeFirst()) args.ifdefStack <- rest - LexbufIfdefStore.SaveEndIfHash(lexbuf, lexed, m) + IfdefStore.SaveEndIfHash(lexbuf, lexed, m) let tok = HASH_ENDIF(m,lexed,LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.Token)) if not skip then tok else endline LexerEndlineContinuation.Token args skip lexbuf } @@ -1102,7 +1102,7 @@ and ifdefSkip (n: int) (m: range) (args: LexArgs) (skip: bool) = parse let lexed = lexeme lexbuf let lookup id = List.contains id args.conditionalDefines let _, expr = evalIfDefExpression lexbuf.StartPos lexbuf.ReportLibraryOnlyFeatures lexbuf.LanguageVersion lexbuf.StrictIndentation args lookup lexed - LexbufIfdefStore.SaveIfHash(lexbuf, lexed, expr, m) + IfdefStore.SaveIfHash(lexbuf, lexed, expr, m) let tok = INACTIVECODE(LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.IfdefSkip(n+1, m))) if skip then endline (LexerEndlineContinuation.IfdefSkip(n+1, m)) args skip lexbuf else tok } @@ -1120,12 +1120,12 @@ and ifdefSkip (n: int) (m: range) (args: LexArgs) (skip: bool) = parse | (IfDefElse,_) :: _rest -> LEX_FAILURE (FSComp.SR.lexHashEndifRequiredForElse()) | (IfDefIf,_) :: rest -> let m = lexbuf.LexemeRange - LexbufIfdefStore.SaveElseHash(lexbuf, lexed, m) + IfdefStore.SaveElseHash(lexbuf, lexed, m) args.ifdefStack <- (IfDefElse,m) :: rest if not skip then HASH_ELSE(m,lexed,LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.Token)) else endline LexerEndlineContinuation.Token args skip lexbuf else - LexbufIfdefStore.SaveElseHash(lexbuf, lexed, m) + IfdefStore.SaveElseHash(lexbuf, lexed, m) if not skip then INACTIVECODE(LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.IfdefSkip(n, m))) else endline (LexerEndlineContinuation.IfdefSkip(n, m)) args skip lexbuf } @@ -1141,13 +1141,13 @@ and ifdefSkip (n: int) (m: range) (args: LexArgs) (skip: bool) = parse match args.ifdefStack with | [] -> LEX_FAILURE (FSComp.SR.lexHashEndingNoMatchingIf()) | _ :: rest -> - LexbufIfdefStore.SaveEndIfHash(lexbuf, lexed, m) + IfdefStore.SaveEndIfHash(lexbuf, lexed, m) args.ifdefStack <- rest if not skip then HASH_ENDIF(m,lexed,LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.Token)) else endline LexerEndlineContinuation.Token args skip lexbuf else shouldStartLine args lexbuf m (FSComp.SR.lexWrongNestedHashEndif()) - LexbufIfdefStore.SaveEndIfHash(lexbuf, lexed, m) + IfdefStore.SaveEndIfHash(lexbuf, lexed, m) let tok = INACTIVECODE(LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.IfdefSkip(n-1, m))) if not skip then tok else endline (LexerEndlineContinuation.IfdefSkip(n-1, m)) args skip lexbuf } @@ -1707,13 +1707,13 @@ and singleLineComment (cargs: SingleLineCommentArgs) (skip: bool) = parse // Saves the documentation (if we're collecting any) into a buffer-local variable. if not skip then LINE_COMMENT (LexCont.Token(args.ifdefStack, args.stringNest)) else - if Option.isNone buff then LexbufCommentStore.SaveSingleLineComment(lexbuf, mStart, mEnd) + if Option.isNone buff then CommentStore.SaveSingleLineComment(lexbuf, mStart, mEnd) token args skip lexbuf } | eof { let buff, _n, mStart, mEnd, args = cargs trySaveXmlDoc lexbuf buff - LexbufCommentStore.SaveSingleLineComment(lexbuf, mStart, mEnd) + CommentStore.SaveSingleLineComment(lexbuf, mStart, mEnd) // NOTE: it is legal to end a file with this comment, so we'll return EOF as a token EOF (LexCont.Token(args.ifdefStack, args.stringNest)) } @@ -1754,7 +1754,7 @@ and comment (cargs: BlockCommentArgs) (skip: bool) = parse | "(*)" { let n, m, args = cargs - LexbufLocalXmlDocStore.AddGrabPoint(lexbuf) + XmlDocStore.AddGrabPoint(lexbuf) if not skip then COMMENT (LexCont.Comment(args.ifdefStack, args.stringNest, n, m)) else comment cargs skip lexbuf } @@ -1775,10 +1775,10 @@ and comment (cargs: BlockCommentArgs) (skip: bool) = parse if not skip then COMMENT (LexCont.Comment(args.ifdefStack, args.stringNest, n-1, m)) else comment (n-1,m,args) skip lexbuf else - LexbufLocalXmlDocStore.AddGrabPointDelayed(lexbuf) + XmlDocStore.AddGrabPointDelayed(lexbuf) if not skip then COMMENT (LexCont.Token(args.ifdefStack, args.stringNest)) else - LexbufCommentStore.SaveBlockComment(lexbuf, m, lexbuf.LexemeRange) + CommentStore.SaveBlockComment(lexbuf, m, lexbuf.LexemeRange) token args skip lexbuf } | anywhite+ diff --git a/src/Compiler/pars.fsy b/src/Compiler/pars.fsy index b21a51d345c..7cac0ad0dc0 100644 --- a/src/Compiler/pars.fsy +++ b/src/Compiler/pars.fsy @@ -17,6 +17,7 @@ open FSharp.Compiler.AbstractIL open FSharp.Compiler.AbstractIL open FSharp.Compiler.DiagnosticsLogger open FSharp.Compiler.Features +open FSharp.Compiler.LexerStore open FSharp.Compiler.ParseHelpers open FSharp.Compiler.Syntax open FSharp.Compiler.SyntaxTrivia @@ -1302,7 +1303,7 @@ moduleDefn: /* 'let' definitions in non-#light*/ | opt_attributes opt_access defnBindings %prec decl_let { if Option.isSome $2 then errorR(Error(FSComp.SR.parsVisibilityDeclarationsShouldComePriorToIdentifier(), rhs parseState 2)) - parseState.ResetSynArgNameGenerator() + (getSynArgNameGenerator parseState.LexBuffer).Reset() let (BindingSetPreAttrs(_, _, _, _, mWhole)) = $3 mkDefnBindings (mWhole, $3, $1, $2, mWhole) } @@ -1310,7 +1311,7 @@ moduleDefn: | opt_attributes opt_access hardwhiteLetBindings %prec decl_let { let hwlb, m, _ = $3 if Option.isSome $2 then errorR(Error(FSComp.SR.parsVisibilityDeclarationsShouldComePriorToIdentifier(), rhs parseState 2)) - parseState.ResetSynArgNameGenerator() + (getSynArgNameGenerator parseState.LexBuffer).Reset() mkDefnBindings (m, hwlb, $1, $2, m) } /* 'do' definitions in non-#light*/ @@ -5829,44 +5830,44 @@ anonLambdaExpr: | FUN atomicPatterns RARROW typedSequentialExprBlock { let mAll = unionRanges (rhs parseState 1) $4.Range let mArrow = Some(rhs parseState 3) - mkSynFunMatchLambdas parseState.SynArgNameGenerator false mAll $2 mArrow $4 } + mkSynFunMatchLambdas (getSynArgNameGenerator parseState.LexBuffer) false mAll $2 mArrow $4 } | FUN atomicPatterns RARROW error { let mAll = rhs2 parseState 1 3 let mArrow = Some(rhs parseState 3) - mkSynFunMatchLambdas parseState.SynArgNameGenerator false mAll $2 mArrow (arbExpr ("anonLambdaExpr1", (rhs parseState 4))) } + mkSynFunMatchLambdas (getSynArgNameGenerator parseState.LexBuffer) false mAll $2 mArrow (arbExpr ("anonLambdaExpr1", (rhs parseState 4))) } | OFUN atomicPatterns RARROW typedSequentialExprBlockR OEND { let mArrow = rhs parseState 3 let expr = $4 mArrow let mAll = unionRanges (rhs parseState 1) expr.Range - mkSynFunMatchLambdas parseState.SynArgNameGenerator false mAll $2 (Some mArrow) expr } + mkSynFunMatchLambdas (getSynArgNameGenerator parseState.LexBuffer) false mAll $2 (Some mArrow) expr } | OFUN atomicPatterns RARROW typedSequentialExprBlockR recover { if not $5 then reportParseErrorAt (rhs parseState 1) (FSComp.SR.parsUnexpectedEndOfFileFunBody ()) let mArrow = rhs parseState 3 let expr = $4 mArrow let mAll = unionRanges (rhs parseState 1) expr.Range - exprFromParseError (mkSynFunMatchLambdas parseState.SynArgNameGenerator false mAll $2 (Some mArrow) expr) } + exprFromParseError (mkSynFunMatchLambdas (getSynArgNameGenerator parseState.LexBuffer) false mAll $2 (Some mArrow) expr) } | OFUN atomicPatterns RARROW ORIGHT_BLOCK_END OEND { let mLambda = rhs2 parseState 1 3 reportParseErrorAt mLambda (FSComp.SR.parsMissingFunctionBody()) let mArrow = Some(rhs parseState 3) - mkSynFunMatchLambdas parseState.SynArgNameGenerator false mLambda $2 mArrow (arbExpr ("anonLambdaExpr2", mLambda.EndRange)) } + mkSynFunMatchLambdas (getSynArgNameGenerator parseState.LexBuffer) false mLambda $2 mArrow (arbExpr ("anonLambdaExpr2", mLambda.EndRange)) } | OFUN atomicPatterns RARROW recover { if not $4 then reportParseErrorAt (rhs parseState 1) (FSComp.SR.parsUnexpectedEndOfFileFunBody()) let mLambda = rhs2 parseState 1 3 let mArrow = Some(rhs parseState 3) - exprFromParseError (mkSynFunMatchLambdas parseState.SynArgNameGenerator false mLambda $2 mArrow (arbExpr ("anonLambdaExpr3", mLambda.EndRange))) } + exprFromParseError (mkSynFunMatchLambdas (getSynArgNameGenerator parseState.LexBuffer) false mLambda $2 mArrow (arbExpr ("anonLambdaExpr3", mLambda.EndRange))) } | OFUN atomicPatterns error OEND { let mLambda = rhs2 parseState 1 2 - exprFromParseError (mkSynFunMatchLambdas parseState.SynArgNameGenerator false mLambda $2 None (arbExpr ("anonLambdaExpr4", mLambda.EndRange))) } + exprFromParseError (mkSynFunMatchLambdas (getSynArgNameGenerator parseState.LexBuffer) false mLambda $2 None (arbExpr ("anonLambdaExpr4", mLambda.EndRange))) } | OFUN error OEND - { exprFromParseError (mkSynFunMatchLambdas parseState.SynArgNameGenerator false (rhs parseState 1) [] None (arbExpr ("anonLambdaExpr5", (rhs parseState 2)))) } + { exprFromParseError (mkSynFunMatchLambdas (getSynArgNameGenerator parseState.LexBuffer) false (rhs parseState 1) [] None (arbExpr ("anonLambdaExpr5", (rhs parseState 2)))) } anonMatchingExpr: | FUNCTION withPatternClauses %prec expr_function diff --git a/tests/FSharp.Compiler.Service.Tests/HashIfExpression.fs b/tests/FSharp.Compiler.Service.Tests/HashIfExpression.fs index f871ee96582..d348f692aaa 100644 --- a/tests/FSharp.Compiler.Service.Tests/HashIfExpression.fs +++ b/tests/FSharp.Compiler.Service.Tests/HashIfExpression.fs @@ -16,7 +16,7 @@ open FSharp.Compiler.Diagnostics open FSharp.Compiler.Lexhelp open FSharp.Compiler.DiagnosticsLogger open FSharp.Compiler.Features -open FSharp.Compiler.ParseHelpers +open FSharp.Compiler.LexerStore type public HashIfExpression() = let preludes = [|"#if "; "#elif "|]