Add ISourceText to language service and use Roslyn's SourceText for F…

…Sharp.Editor (#6001)

* Initial ISourceText implementation (does not work yet)

* Lexbuffer works

* Removing Source. Now using only ISourceText. Added SourceText.ofString.

* Fixing tests

* We need to use addNewLine for tests to pass

* Added test for SourceText.ofString

* Trying to fix tests

* Simplified ISourceText API. Added RoslynSourceTextTests

* Trying to get the build working again

* Re-organize prim-lexing.fsi

* Handling format strings

* Trying to get tests to pass

* Trying to fix tests

* Ignoring test

* unignoring test

* Fixed weak table

* Removing addNewLine in sourcetext

* Fixing interactive checker tests

* Fixing more tests

* Removed addNewLine

* Removed addNewLine

* Removed addNewLine

* Removed addNewLine

* Removed addNewLine

* Removed addNewLine

* Removed addNewLine

* Removed addNewLine

* Removed addNewLine

* Removing last addNewLine. It's done

* Better tests and small optimizations

* Adjusting comment

* Updating CompilerServiceBenchmarks

* Updated nits

benchmarks/CompilerServiceBenchmarks/CompilerServiceBenchmarks.fsproj

@@ -12,12 +12,17 @@
     <DefaultValueTuplePackageVersion>$(SystemValueTuplePackageVersion)</DefaultValueTuplePackageVersion>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+  </PropertyGroup>
+
   <ItemGroup>
     <Compile Include="Program.fs" />
   </ItemGroup>
 
   <ItemGroup>
     <PackageReference Include="BenchmarkDotNet" Version="0.11.3" />
+    <PackageReference Include="Microsoft.CodeAnalysis.EditorFeatures.Text" Version="2.9.0" />
   </ItemGroup>
 
   <ItemGroup>

benchmarks/CompilerServiceBenchmarks/Program.fs

@@ -4,9 +4,80 @@ open BenchmarkDotNet.Attributes
 open BenchmarkDotNet.Running
 open Microsoft.FSharp.Compiler.ErrorLogger
 open Microsoft.FSharp.Compiler.SourceCodeServices
+open Microsoft.FSharp.Compiler.Text
 open System.Text
+open Microsoft.CodeAnalysis.Text
 
-[<ClrJob(baseline = true); MemoryDiagnoser>]
+module private SourceText =
+
+    open System.Runtime.CompilerServices
+
+    let weakTable = ConditionalWeakTable<SourceText, ISourceText>()
+
+    let create (sourceText: SourceText) =
+
+        let sourceText =
+            { new ISourceText with
+
+                member __.Item with get index = sourceText.[index]
+
+                member __.GetLineString(lineIndex) =
+                    sourceText.Lines.[lineIndex].ToString()
+
+                member __.GetLineCount() =
+                    sourceText.Lines.Count
+
+                member __.GetLastCharacterPosition() =
+                    if sourceText.Lines.Count > 0 then
+                        (sourceText.Lines.Count, sourceText.Lines.[sourceText.Lines.Count - 1].Span.Length)
+                    else
+                        (0, 0)
+
+                member __.GetSubTextString(start, length) =
+                    sourceText.GetSubText(TextSpan(start, length)).ToString()
+
+                member __.SubTextEquals(target, startIndex) =
+                    if startIndex < 0 || startIndex >= sourceText.Length then
+                        raise (ArgumentOutOfRangeException("startIndex"))
+
+                    if String.IsNullOrEmpty(target) then
+                        raise (ArgumentException("Target is null or empty.", "target"))
+
+                    let lastIndex = startIndex + target.Length
+                    if lastIndex <= startIndex || lastIndex >= sourceText.Length then
+                        raise (ArgumentException("Target is too big.", "target"))
+
+                    let mutable finished = false
+                    let mutable didEqual = true
+                    let mutable i = 0
+                    while not finished && i < target.Length do
+                        if target.[i] <> sourceText.[startIndex + i] then
+                            didEqual <- false
+                            finished <- true // bail out early                        
+                        else
+                            i <- i + 1
+
+                    didEqual
+
+                member __.ContentEquals(sourceText) =
+                    match sourceText with
+                    | :? SourceText as sourceText -> sourceText.ContentEquals(sourceText)
+                    | _ -> false
+
+                member __.Length = sourceText.Length
+
+                member __.CopyTo(sourceIndex, destination, destinationIndex, count) =
+                    sourceText.CopyTo(sourceIndex, destination, destinationIndex, count)
+            }
+
+        sourceText
+
+type SourceText with
+
+    member this.ToFSharpSourceText() =
+        SourceText.weakTable.GetValue(this, Runtime.CompilerServices.ConditionalWeakTable<_,_>.CreateValueCallback(SourceText.create))
+
+[<ClrJob; MemoryDiagnoser>]
 type CompilerServiceParsing() =
 
     let mutable checkerOpt = None
@@ -32,8 +103,7 @@ type CompilerServiceParsing() =
 
         match sourceOpt with
         | None ->
-            let source = File.ReadAllText("""..\..\..\..\..\src\fsharp\TypeChecker.fs""")
-            sourceOpt <- Some(source)
+            sourceOpt <- Some <| SourceText.From(File.OpenRead("""..\..\..\..\..\src\fsharp\TypeChecker.fs"""), Encoding.Default, SourceHashAlgorithm.Sha1, true)
         | _ -> ()
 
     [<IterationSetup>]
@@ -43,18 +113,18 @@ type CompilerServiceParsing() =
         | Some(checker) ->
             checker.InvalidateAll()
             checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients()
-            checker.ParseFile("dummy.fs", "dummy", parsingOptions) |> Async.RunSynchronously |> ignore
+            checker.ParseFile("dummy.fs", SourceText.ofString "dummy", parsingOptions) |> Async.RunSynchronously |> ignore
 
     [<Benchmark>]
     member __.Parsing() =
         match checkerOpt, sourceOpt with
         | None, _ -> failwith "no checker"
         | _, None -> failwith "no source"
         | Some(checker), Some(source) ->
-            let results = checker.ParseFile("TypeChecker.fs", source, parsingOptions) |> Async.RunSynchronously
+            let results = checker.ParseFile("TypeChecker.fs", source.ToFSharpSourceText(), parsingOptions) |> Async.RunSynchronously
             if results.ParseHadErrors then failwithf "parse had errors: %A" results.Errors
 
 [<EntryPoint>]
 let main argv =
     let _ = BenchmarkRunner.Run<CompilerServiceParsing>()
-    0
+    0

fcs/samples/EditorService/Program.fs

@@ -8,6 +8,7 @@ open Microsoft.FSharp.Compiler.QuickParse
 let checker = FSharpChecker.Create()
 
 let parseWithTypeInfo (file, input) = 
+    let input = Microsoft.FSharp.Compiler.Text.SourceText.ofString input
     let checkOptions, _errors = checker.GetProjectOptionsFromScript(file, input) |> Async.RunSynchronously
     let parsingOptions, _errors = checker.GetParsingOptionsFromProjectOptions(checkOptions)
     let untypedRes = checker.ParseFile(file, input, parsingOptions) |> Async.RunSynchronously

fcs/samples/UntypedTree/Program.fs

@@ -11,7 +11,7 @@ let checker = FSharpChecker.Create()
 // Get untyped tree for a specified input
 let getUntypedTree (file, input) = 
   let parsingOptions = { FSharpParsingOptions.Default with SourceFiles = [| file |] }
-  let untypedRes = checker.ParseFile(file, input, parsingOptions) |> Async.RunSynchronously
+  let untypedRes = checker.ParseFile(file, Microsoft.FSharp.Compiler.Text.SourceText.ofString input, parsingOptions) |> Async.RunSynchronously
   match untypedRes.ParseTree with
   | Some tree -> tree
   | None -> failwith "Something went wrong during parsing!"

src/fsharp/CheckFormatStrings.fs

@@ -54,15 +54,17 @@ let parseFormatStringInternal (m:range) (g: TcGlobals) (context: FormatStringChe
     let (offset, fmt) = 
         match context with
         | Some context ->
-            let length = context.Source.Length
-            if m.EndLine < context.LineStartPositions.Length then
-                let startIndex = context.LineStartPositions.[m.StartLine-1] + m.StartColumn
-                let endIndex = context.LineStartPositions.[m.EndLine-1] + m.EndColumn - 1
-                if startIndex < length-3 && context.Source.[startIndex..startIndex+2] = "\"\"\"" then
-                    (3, context.Source.[startIndex+3..endIndex-3])
-                elif startIndex < length-2 && context.Source.[startIndex..startIndex+1] = "@\"" then
-                    (2, context.Source.[startIndex+2..endIndex-1])
-                else (1, context.Source.[startIndex+1..endIndex-1])
+            let sourceText = context.SourceText
+            let lineStartPositions = context.LineStartPositions
+            let length = sourceText.Length
+            if m.EndLine < lineStartPositions.Length then
+                let startIndex = lineStartPositions.[m.StartLine-1] + m.StartColumn
+                let endIndex = lineStartPositions.[m.EndLine-1] + m.EndColumn - 1
+                if startIndex < length-3 && sourceText.SubTextEquals("\"\"\"", startIndex) then
+                    (3, sourceText.GetSubTextString(startIndex + 3, endIndex - startIndex))
+                elif startIndex < length-2 && sourceText.SubTextEquals("@\"", startIndex) then
+                    (2, sourceText.GetSubTextString(startIndex + 2, endIndex + 1 - startIndex))
+                else (1, sourceText.GetSubTextString(startIndex + 1, endIndex - startIndex))
             else (1, fmt)
         | None -> (1, fmt)
 

src/fsharp/CompileOps.fs

@@ -22,7 +22,8 @@ open Microsoft.FSharp.Compiler.AbstractIL.Internal.Library
 open Microsoft.FSharp.Compiler.AbstractIL.Extensions.ILX
 open Microsoft.FSharp.Compiler.AbstractIL.Diagnostics 
 
-open Microsoft.FSharp.Compiler 
+open Microsoft.FSharp.Compiler
+open Microsoft.FSharp.Compiler.Text
 open Microsoft.FSharp.Compiler.Ast
 open Microsoft.FSharp.Compiler.AttributeChecking
 open Microsoft.FSharp.Compiler.ConstraintSolver
@@ -5037,7 +5038,7 @@ module private ScriptPreprocessClosure =
     open Internal.Utilities.Text.Lexing
 
     /// Represents an input to the closure finding process
-    type ClosureSource = ClosureSource of filename: string * referenceRange: range * sourceText: string * parseRequired: bool 
+    type ClosureSource = ClosureSource of filename: string * referenceRange: range * sourceText: ISourceText * parseRequired: bool 
 
     /// Represents an output of the closure finding process
     type ClosureFile = ClosureFile  of string * range * ParsedInput option * (PhasedDiagnostic * bool) list * (PhasedDiagnostic * bool) list * (string * range) list // filename, range, errors, warnings, nowarns
@@ -5052,7 +5053,7 @@ module private ScriptPreprocessClosure =
             seen.ContainsKey(check)
 
     /// Parse a script from source.
-    let ParseScriptText(filename:string, source:string, tcConfig:TcConfig, codeContext, lexResourceManager:Lexhelp.LexResourceManager, errorLogger:ErrorLogger) =    
+    let ParseScriptText(filename:string, sourceText:ISourceText, tcConfig:TcConfig, codeContext, lexResourceManager:Lexhelp.LexResourceManager, errorLogger:ErrorLogger) =    
 
         // fsc.exe -- COMPILED\!INTERACTIVE
         // fsi.exe -- !COMPILED\INTERACTIVE
@@ -5064,7 +5065,7 @@ module private ScriptPreprocessClosure =
             | CodeContext.CompilationAndEvaluation -> ["INTERACTIVE"]
             | CodeContext.Compilation -> ["COMPILED"]
             | CodeContext.Editing -> "EDITING" :: (if IsScript filename then ["INTERACTIVE"] else ["COMPILED"])
-        let lexbuf = UnicodeLexing.StringAsLexbuf source 
+        let lexbuf = UnicodeLexing.SourceTextAsLexbuf(sourceText) 
 
         let isLastCompiland = (IsScript filename), tcConfig.target.IsExe        // The root compiland is last in the list of compilands.
         ParseOneInputLexbuf (tcConfig, lexResourceManager, defines, lexbuf, filename, isLastCompiland, errorLogger) 
@@ -5101,7 +5102,7 @@ module private ScriptPreprocessClosure =
                 | None -> new  StreamReader(stream, true)
                 | Some (n: int) -> new  StreamReader(stream, Encoding.GetEncoding(n)) 
             let source = reader.ReadToEnd()
-            [ClosureSource(filename, m, source, parseRequired)]
+            [ClosureSource(filename, m, SourceText.ofString source, parseRequired)]
         with e -> 
             errorRecovery e m 
             []
@@ -5129,15 +5130,15 @@ module private ScriptPreprocessClosure =
         let tcConfig = ref tcConfig
 
         let observedSources = Observed()
-        let rec loop (ClosureSource(filename, m, source, parseRequired)) = 
+        let rec loop (ClosureSource(filename, m, sourceText, parseRequired)) = 
           [     if not (observedSources.HaveSeen(filename)) then
                     observedSources.SetSeen(filename)
                     //printfn "visiting %s" filename
                     if IsScript(filename) || parseRequired then 
                         let parseResult, parseDiagnostics =
                             let errorLogger = CapturingErrorLogger("FindClosureParse")                    
                             use _unwindEL = PushErrorLoggerPhaseUntilUnwind (fun _ -> errorLogger)
-                            let result = ParseScriptText (filename, source, !tcConfig, codeContext, lexResourceManager, errorLogger) 
+                            let result = ParseScriptText (filename, sourceText, !tcConfig, codeContext, lexResourceManager, errorLogger) 
                             result, errorLogger.Diagnostics
 
                         match parseResult with 
@@ -5237,7 +5238,7 @@ module private ScriptPreprocessClosure =
         result
 
     /// Given source text, find the full load closure. Used from service.fs, when editing a script file
-    let GetFullClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename, source, codeContext, useSimpleResolution, useFsiAuxLib, lexResourceManager:Lexhelp.LexResourceManager, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage) = 
+    let GetFullClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename, sourceText, codeContext, useSimpleResolution, useFsiAuxLib, lexResourceManager:Lexhelp.LexResourceManager, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage) = 
         // Resolve the basic references such as FSharp.Core.dll first, before processing any #I directives in the script
         //
         // This is tries to mimic the action of running the script in F# Interactive - the initial context for scripting is created
@@ -5250,7 +5251,7 @@ module private ScriptPreprocessClosure =
 
         let tcConfig = CreateScriptTextTcConfig(legacyReferenceResolver, defaultFSharpBinariesDir, filename, codeContext, useSimpleResolution, useFsiAuxLib, Some references0, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage)
 
-        let closureSources = [ClosureSource(filename, range0, source, true)]
+        let closureSources = [ClosureSource(filename, range0, sourceText, true)]
         let closureFiles, tcConfig = FindClosureFiles(closureSources, tcConfig, codeContext, lexResourceManager)
         GetLoadClosure(ctok, filename, closureFiles, tcConfig, codeContext)
 
@@ -5268,9 +5269,9 @@ type LoadClosure with
     //
     /// A temporary TcConfig is created along the way, is why this routine takes so many arguments. We want to be sure to use exactly the
     /// same arguments as the rest of the application.
-    static member ComputeClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename:string, source:string, codeContext, useSimpleResolution:bool, useFsiAuxLib, lexResourceManager:Lexhelp.LexResourceManager, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage) : LoadClosure = 
+    static member ComputeClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename:string, sourceText:ISourceText, codeContext, useSimpleResolution:bool, useFsiAuxLib, lexResourceManager:Lexhelp.LexResourceManager, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage) : LoadClosure = 
         use unwindBuildPhase = PushThreadBuildPhaseUntilUnwind BuildPhase.Parse
-        ScriptPreprocessClosure.GetFullClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename, source, codeContext, useSimpleResolution, useFsiAuxLib, lexResourceManager, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage)
+        ScriptPreprocessClosure.GetFullClosureOfScriptText(ctok, legacyReferenceResolver, defaultFSharpBinariesDir, filename, sourceText, codeContext, useSimpleResolution, useFsiAuxLib, lexResourceManager, applyCommmandLineArgs, assumeDotNetFramework, tryGetMetadataSnapshot, reduceMemoryUsage)
 
     /// Analyze a set of script files and find the closure of their references. The resulting references are then added to the given TcConfig.
     /// Used from fsi.fs and fsc.fs, for #load and command line. 

src/fsharp/CompileOps.fsi

@@ -11,6 +11,7 @@ open Microsoft.FSharp.Compiler.AbstractIL.IL
 open Microsoft.FSharp.Compiler.AbstractIL.ILBinaryReader
 open Microsoft.FSharp.Compiler.AbstractIL.Internal.Library 
 open Microsoft.FSharp.Compiler 
+open Microsoft.FSharp.Compiler.Text
 open Microsoft.FSharp.Compiler.TypeChecker
 open Microsoft.FSharp.Compiler.Range
 open Microsoft.FSharp.Compiler.Ast
@@ -796,7 +797,7 @@ type LoadClosure =
     //
     /// A temporary TcConfig is created along the way, is why this routine takes so many arguments. We want to be sure to use exactly the
     /// same arguments as the rest of the application.
-    static member ComputeClosureOfScriptText: CompilationThreadToken * legacyReferenceResolver: ReferenceResolver.Resolver * defaultFSharpBinariesDir: string * filename: string * source: string * implicitDefines:CodeContext * useSimpleResolution: bool * useFsiAuxLib: bool * lexResourceManager: Lexhelp.LexResourceManager * applyCompilerOptions: (TcConfigBuilder -> unit) * assumeDotNetFramework: bool * tryGetMetadataSnapshot: ILReaderTryGetMetadataSnapshot * reduceMemoryUsage: ReduceMemoryFlag -> LoadClosure
+    static member ComputeClosureOfScriptText: CompilationThreadToken * legacyReferenceResolver: ReferenceResolver.Resolver * defaultFSharpBinariesDir: string * filename: string * sourceText: ISourceText * implicitDefines:CodeContext * useSimpleResolution: bool * useFsiAuxLib: bool * lexResourceManager: Lexhelp.LexResourceManager * applyCompilerOptions: (TcConfigBuilder -> unit) * assumeDotNetFramework: bool * tryGetMetadataSnapshot: ILReaderTryGetMetadataSnapshot * reduceMemoryUsage: ReduceMemoryFlag -> LoadClosure
 
     /// Analyze a set of script files and find the closure of their references. The resulting references are then added to the given TcConfig.
     /// Used from fsi.fs and fsc.fs, for #load and command line.