Add range to MarkdownParagraph and MarkdownSpan

src/Common/StringParsing.fs

@@ -14,44 +14,57 @@ open FSharp.Collections
 
 module String =
   /// Matches when a string is a whitespace or null
-  let (|WhiteSpace|_|) s = 
+  let (|WhiteSpaceS|_|) (s) = 
+    if String.IsNullOrWhiteSpace(s) then Some() else None
+
+  /// Matches when a string is a whitespace or null
+  let (|WhiteSpace|_|) (s, n: int) = 
     if String.IsNullOrWhiteSpace(s) then Some() else None
 
   /// Matches when a string does starts with non-whitespace
-  let (|Unindented|_|) (s:string) = 
+  let (|Unindented|_|) (s:string, n:int) = 
     if not (String.IsNullOrWhiteSpace(s)) && s.TrimStart() = s then Some() else None
 
   /// Returns a string trimmed from both start and end
-  let (|TrimBoth|) (text:string) = text.Trim()
+  let (|TrimBothS|) (text:string) = text.Trim()
+  /// Returns a string trimmed from both start and end
+  let (|TrimBoth|) (text:string, n:int) = (text.Trim(), n)
   /// Returns a string trimmed from the end
-  let (|TrimEnd|) (text:string) = text.TrimEnd()
+  let (|TrimEnd|) (text:string, n:int) = (text.TrimEnd(), n)
   /// Returns a string trimmed from the start
-  let (|TrimStart|) (text:string) = text.TrimStart()
+  let (|TrimStart|) (text:string, n:int) = (text.TrimStart(), n)
 
   /// Retrusn a string trimmed from the end using characters given as a parameter
-  let (|TrimEndUsing|) chars (text:string) = text.TrimEnd(Array.ofSeq chars)
+  let (|TrimEndUsing|) chars (text:string, n:int) = text.TrimEnd(Array.ofSeq chars)
 
   /// Returns a string trimmed from the start together with 
   /// the number of skipped whitespace characters
-  let (|TrimStartAndCount|) (text:string) = 
+  let (|TrimStartAndCount|) (text:string, n:int) = 
     let trimmed = text.TrimStart([|' '; '\t'|])
     let len = text.Length - trimmed.Length
-    len, text.Substring(0, len).Replace("\t", "    ").Length, trimmed
+    len, text.Substring(0, len).Replace("\t", "    ").Length, (trimmed, n)
 
   /// Matches when a string starts with any of the specified sub-strings
-  let (|StartsWithAny|_|) (starts:seq<string>) (text:string) = 
+  let (|StartsWithAny|_|) (starts:seq<string>) (text:string, n:int) = 
     if starts |> Seq.exists (text.StartsWith) then Some() else None
   /// Matches when a string starts with the specified sub-string
-  let (|StartsWith|_|) (start:string) (text:string) = 
+  let (|StartsWithS|_|) (start:string) (text:string) = 
     if text.StartsWith(start) then Some(text.Substring(start.Length)) else None
   /// Matches when a string starts with the specified sub-string
+  let (|StartsWith|_|) (start:string) (text:string, n:int) = 
+    if text.StartsWith(start) then Some(text.Substring(start.Length), n) else None
+  /// Matches when a string starts with the specified sub-string
   /// The matched string is trimmed from all whitespace.
-  let (|StartsWithTrim|_|) (start:string) (text:string) = 
+  let (|StartsWithTrimS|_|) (start:string) (text:string) = 
     if text.StartsWith(start) then Some(text.Substring(start.Length).Trim()) else None
+  /// Matches when a string starts with the specified sub-string
+  /// The matched string is trimmed from all whitespace.
+  let (|StartsWithTrim|_|) (start:string) (text:string, n:int) = 
+    if text.StartsWith(start) then Some(text.Substring(start.Length).Trim(), n) else None
 
   /// Matches when a string starts with the specified sub-string (ignoring whitespace at the start)
   /// The matched string is trimmed from all whitespace.
-  let (|StartsWithNTimesTrimIgnoreStartWhitespace|_|) (start:string) (text:string) =
+  let (|StartsWithNTimesTrimIgnoreStartWhitespace|_|) (start:string) (text:string, n:int) =
     if text.Contains(start) then
       let beforeStart = text.Substring(0, text.IndexOf(start))
       if String.IsNullOrWhiteSpace (beforeStart) then
@@ -67,12 +80,26 @@ module String =
 
   /// Matches when a string starts with the given value and ends 
   /// with a given value (and returns the rest of it)
-  let (|StartsAndEndsWith|_|) (starts, ends) (s:string) =
+  let (|StartsAndEndsWithS|_|) (starts, ends) (s:string) =
     if s.StartsWith(starts) && s.EndsWith(ends) && 
        s.Length >= starts.Length + ends.Length then 
       Some(s.Substring(starts.Length, s.Length - starts.Length - ends.Length))
     else None
 
+  /// Matches when a string starts with the given value and ends 
+  /// with a given value (and returns the rest of it)
+  let (|StartsAndEndsWith|_|) (starts, ends) (s:string, n:int) =
+    if s.StartsWith(starts) && s.EndsWith(ends) && 
+       s.Length >= starts.Length + ends.Length then 
+      Some(s.Substring(starts.Length, s.Length - starts.Length - ends.Length), n)
+    else None
+
+  /// Matches when a string starts with the given value and ends 
+  /// with a given value (and returns trimmed body)
+  let (|StartsAndEndsWithTrimS|_|) args = function
+    | StartsAndEndsWithS args (TrimBothS res) -> Some res
+    | _ -> None
+
   /// Matches when a string starts with the given value and ends 
   /// with a given value (and returns trimmed body)
   let (|StartsAndEndsWithTrim|_|) args = function
@@ -85,15 +112,15 @@ module String =
   ///
   ///    let (StartsWithRepeated "/\" (2, " abc")) = "/\/\ abc"
   ///
-  let (|StartsWithRepeated|_|) (repeated:string) (text:string) = 
+  let (|StartsWithRepeated|_|) (repeated:string) (text:string, ln:int) = 
     let rec loop i = 
       if i = text.Length then i
       elif text.[i] <> repeated.[i % repeated.Length] then i
       else loop (i + 1)
 
     let n = loop 0 
     if n = 0 || n % repeated.Length <> 0 then None
-    else Some(n/repeated.Length, text.Substring(n, text.Length - n)) 
+    else Some(n/repeated.Length, (text.Substring(n, text.Length - n), ln)) 
 
   /// Ignores everything until a end-line character is detected, returns the remaining string.
   let (|SkipSingleLine|) (text:string) =
@@ -114,12 +141,11 @@ module String =
       FSharp.Formatting.Common.Log.warnf "could not skip a line of %s, because no line-ending character was found!" text
     result
 
-
   /// Matches when a string starts with a sub-string wrapped using the 
   /// opening and closing sub-string specified in the parameter.
   /// For example "[aa]bc" is wrapped in [ and ] pair. Returns the wrapped
   /// text together with the rest.
-  let (|StartsWithWrapped|_|) (starts:string, ends:string) (text:string) = 
+  let (|StartsWithWrappedS|_|) (starts:string, ends:string) (text:string) = 
     if text.StartsWith(starts) then 
       let id = text.IndexOf(ends, starts.Length)
       if id >= 0 then 
@@ -129,10 +155,24 @@ module String =
       else None
     else None
 
+  /// Matches when a string starts with a sub-string wrapped using the 
+  /// opening and closing sub-string specified in the parameter.
+  /// For example "[aa]bc" is wrapped in [ and ] pair. Returns the wrapped
+  /// text together with the rest.
+  let (|StartsWithWrapped|_|) (starts:string, ends:string) (text:string, n:int) = 
+    if text.StartsWith(starts) then 
+      let id = text.IndexOf(ends, starts.Length)
+      if id >= 0 then 
+        let wrapped = text.Substring(starts.Length, id - starts.Length)
+        let rest = text.Substring(id + ends.Length, text.Length - id - ends.Length)
+        Some(wrapped, (rest, n))
+      else None
+    else None
+
   /// Matches when a string consists of some number of 
   /// complete repetitions of a specified sub-string.
-  let (|EqualsRepeated|_|) repeated = function
-    | StartsWithRepeated repeated (n, "") -> Some()
+  let (|EqualsRepeated|_|) (repeated, n:int) = function
+    | StartsWithRepeated repeated (n, ("", _)) -> Some()
     | _ -> None 
 
   /// Given a list of lines indented with certan number of whitespace 
@@ -197,8 +237,8 @@ module Lines =
   /// Removes blank lines from the start and the end of a list
   let (|TrimBlank|) lines = 
     lines
-    |> List.skipWhile String.IsNullOrWhiteSpace |> List.rev
-    |> List.skipWhile String.IsNullOrWhiteSpace |> List.rev
+    |> List.skipWhile (fun (s, n) -> String.IsNullOrWhiteSpace s) |> List.rev
+    |> List.skipWhile (fun (s, n) -> String.IsNullOrWhiteSpace s) |> List.rev
 
   /// Matches when there are some lines at the beginning that are 
   /// either empty (or whitespace) or start with the specified string.
@@ -213,7 +253,7 @@ module Lines =
   /// either empty (or whitespace) or start with at least 4 spaces (a tab counts as 4 spaces here).
   /// Returns all such lines from the beginning until a different line and
   /// the number of spaces the first line started with.
-  let (|TakeCodeBlock|_|) (input:string list) =
+  let (|TakeCodeBlock|_|) (input:(string * int) list) =
     let spaceNum = 4
       //match input with
       //| h :: _ ->
@@ -225,20 +265,20 @@ module Lines =
       let normalized = s.Replace("\t", "    ")
       normalized.Length >= spaceNum &&
         normalized.Substring(0, spaceNum) = System.String(' ', spaceNum)
-    match List.partitionWhile (fun s ->
+    match List.partitionWhile (fun (s, n) ->
             String.IsNullOrWhiteSpace s || startsWithSpaces s) input with
     | matching, rest when matching <> [] && spaceNum >= 4 ->
       Some(spaceNum, matching, rest)
     | _ -> None
 
   /// Removes whitespace lines from the beginning of the list
-  let (|TrimBlankStart|) = List.skipWhile (String.IsNullOrWhiteSpace)
+  let (|TrimBlankStart|) = List.skipWhile (fun (s:string, n:int) -> String.IsNullOrWhiteSpace s)
 
   /// Trims all lines of the current paragraph
   let (|TrimParagraphLines|) lines =
     lines
     // first remove all whitespace on the beginning of the line
-    |> List.map (fun (s:string) -> s.TrimStart())
+    |> List.map (fun (s:string, n:int) -> s.TrimStart())
     // Now remove all additional spaces at the end, but keep two spaces if existent
     |> List.map (fun s ->
       let endsWithTwoSpaces = s.EndsWith("  ")
@@ -258,7 +298,21 @@ open System.Collections.Generic
 /// recognize `key1=value, key2=value` and also `key1:value, key2:value`
 /// The key of the command should be identifier with just 
 /// characters in it - otherwise, the parsing fails.
-let (|ParseCommands|_|) (str:string) = 
+let (|ParseCommandsS|_|) (str:string) = 
+  let kvs = 
+    [ for cmd in str.Split(',') do
+        let kv = cmd.Split([| '='; ':' |])
+        if kv.Length = 2 then yield kv.[0].Trim(), kv.[1].Trim()
+        elif kv.Length = 1 then yield kv.[0].Trim(), "" ] 
+  let allKeysValid = 
+    kvs |> Seq.forall (fst >> Seq.forall (fun c -> Char.IsLetter c || c = '_' || c = '-'))
+  if allKeysValid && kvs <> [] then Some(dict kvs) else None
+
+/// Utility for parsing commands. Commands can be used in different places. We 
+/// recognize `key1=value, key2=value` and also `key1:value, key2:value`
+/// The key of the command should be identifier with just 
+/// characters in it - otherwise, the parsing fails.
+let (|ParseCommands|_|) (str:string, n:int) = 
   let kvs = 
     [ for cmd in str.Split(',') do
         let kv = cmd.Split([| '='; ':' |])

src/FSharp.CodeFormat/CommentFilter.fs

@@ -49,12 +49,12 @@ let rec getSnippets (state:NamedSnippet option) (snippets:NamedSnippet list)
   match source with 
   | [] -> snippets
   | (line, tokens)::rest ->
-    let text = lines.[line].Trim()
+    let text = lines.[line].Trim(), line
     match state, text with
 
     // We're not inside a snippet and we found a beginning of one
     | None, String.StartsWithTrim "//" (String.StartsWithTrim "[snippet:" title) ->
-        let title = title.Substring(0, title.IndexOf(']'))
+        let title = (fst title).Substring(0, (fst title).IndexOf(']'))
         getSnippets (Some(title, [])) snippets rest lines
     // Not inside a snippet and there is a usual line
     | None, _ -> 
@@ -92,7 +92,7 @@ let rec shrinkOmittedCode (text:StringBuilder) line content (source:Snippet) =
   // Take the next line, merge comments and continue looking for end
   | [], (line, content)::source -> 
       shrinkOmittedCode (text.Append("\n")) line (mergeComments content None []) source
-  | (String.StartsAndEndsWithTrim ("(*", "*)") "[/omit]", tok)::rest, source 
+  | (String.StartsAndEndsWithTrimS ("(*", "*)") "[/omit]", tok)::rest, source 
         when tok.TokenName = "COMMENT" ->
       line, rest, source, text
   | (str, tok)::rest, _ -> 
@@ -105,13 +105,13 @@ let rec shrinkOmittedCode (text:StringBuilder) line content (source:Snippet) =
 let rec shrinkLine line (content:SnippetLine) (source:Snippet) = 
   match content with 
   | [] -> [], source
-  | (String.StartsAndEndsWithTrim ("(*", "*)") (String.StartsAndEndsWithTrim ("[omit:", "]") body), (tok:FSharpTokenInfo))::rest
+  | (String.StartsAndEndsWithTrimS ("(*", "*)") (String.StartsAndEndsWithTrimS ("[omit:", "]") body), (tok:FSharpTokenInfo))::rest
         when tok.TokenName = "COMMENT" -> 
       let line, remcontent, source, text = 
         shrinkOmittedCode (StringBuilder()) line rest source
       let line, source = shrinkLine line remcontent source
       (body, { tok with TokenName = "OMIT" + (text.ToString()) })::line, source
-  | (String.StartsWithTrim "//" (String.StartsAndEndsWith ("[fsi:", "]") fsi), (tok:FSharpTokenInfo))::rest ->
+  | (String.StartsWithTrimS "//" (String.StartsAndEndsWithS ("[fsi:", "]") fsi), (tok:FSharpTokenInfo))::rest ->
       let line, source = shrinkLine line rest source
       (fsi, { tok with TokenName = "FSI"})::line, source
   | (str, tok)::rest -> 

src/FSharp.Literate/Document.fs

@@ -112,4 +112,4 @@ type LiterateDocument(paragraphs, formattedTips, links, source, sourceFile, erro
 /// Markdown documents.
 module Matching =
   let (|LiterateParagraph|_|) = function
-    | EmbedParagraphs(:? LiterateParagraph as lp) -> Some lp | _ -> None
+    | EmbedParagraphs(:? LiterateParagraph as lp, _) -> Some lp | _ -> None

src/FSharp.Literate/Evaluator.fs

@@ -111,7 +111,7 @@ type FsiEvaluator(?options:string[], ?fsiObj) =
   /// Registered transformations for pretty printing values
   /// (the default formats value as a string and emits single CodeBlock)
   let mutable valueTransformations = 
-    [ (fun (o:obj, t:Type) ->Some([CodeBlock (sprintf "%A" o, "", "")]) ) ]
+    [ (fun (o:obj, t:Type) ->Some([CodeBlock (sprintf "%A" o, "", "", None)]) ) ]
 
   /// Register a function that formats (some) values that are produced by the evaluator.
   /// The specified function should return 'Some' when it knows how to format a value
@@ -130,12 +130,12 @@ type FsiEvaluator(?options:string[], ?fsiObj) =
       match result :?> FsiEvaluationResult, kind with
       | result, FsiEmbedKind.Output -> 
           let s = defaultArg result.Output "No output has been produced."
-          [ CodeBlock(s.Trim(), "", "") ]
+          [ CodeBlock(s.Trim(), "", "", None) ]
       | { ItValue = Some v }, FsiEmbedKind.ItValue
       | { Result = Some v }, FsiEmbedKind.Value ->
           valueTransformations |> Seq.pick (fun f -> lock lockObj (fun () -> f v))
-      | _, FsiEmbedKind.ItValue -> [ CodeBlock ("No value has been returned", "", "") ]
-      | _, FsiEmbedKind.Value -> [ CodeBlock ("No value has been returned", "", "") ]
+      | _, FsiEmbedKind.ItValue -> [ CodeBlock ("No value has been returned", "", "", None) ]
+      | _, FsiEmbedKind.Value -> [ CodeBlock ("No value has been returned", "", "", None) ]
 
     /// Evaluates the given text in an fsi session and returns
     /// an FsiEvaluationResult.

src/FSharp.Literate/Formatting.fs

@@ -27,8 +27,8 @@ module Formatting =
   /// Try find first-level heading in the paragraph collection
   let findHeadings paragraphs generateAnchors (outputKind:OutputKind) =              
     paragraphs |> Seq.tryPick (function 
-      | (Heading(1, text)) ->           
-          let doc = MarkdownDocument([Span(text)], dict [])
+      | Heading(1, text, r) ->           
+          let doc = MarkdownDocument([Span(text, r)], dict [])
           Some(format doc generateAnchors outputKind)
       | _ -> None)
 
@@ -37,13 +37,13 @@ module Formatting =
   let getSourceDocument (doc:LiterateDocument) =
     match doc.Source with
     | LiterateSource.Markdown text ->
-        doc.With(paragraphs = [CodeBlock (text, "", "")])
+        doc.With(paragraphs = [CodeBlock (text, "", "", None)])
     | LiterateSource.Script snippets ->
         let paragraphs = 
           [ for Snippet(name, lines) in snippets do
               if snippets.Length > 1 then
-                yield Heading(3, [Literal name])
-              yield EmbedParagraphs(FormattedCode(lines)) ]
+                yield Heading(3, [Literal(name, None)], None)
+              yield EmbedParagraphs(FormattedCode(lines), None) ]
         doc.With(paragraphs = paragraphs)
 
 // --------------------------------------------------------------------------------------

src/FSharp.Literate/Main.fs

@@ -104,7 +104,7 @@ type Literate private () =
   static member WriteHtml(doc:LiterateDocument, ?prefix, ?lineNumbers, ?generateAnchors) =
     let ctx = formattingContext None (Some OutputKind.Html) prefix lineNumbers None generateAnchors None None
     let doc = Transformations.replaceLiterateParagraphs ctx doc
-    let doc = MarkdownDocument(doc.Paragraphs @ [InlineBlock doc.FormattedTips], doc.DefinedLinks)
+    let doc = MarkdownDocument(doc.Paragraphs @ [InlineBlock(doc.FormattedTips, None)], doc.DefinedLinks)
     let sb = new System.Text.StringBuilder()
     use wr = new StringWriter(sb)
     Html.formatMarkdown wr ctx.GenerateHeaderAnchors Environment.NewLine true doc.DefinedLinks doc.Paragraphs
@@ -113,7 +113,7 @@ type Literate private () =
   static member WriteHtml(doc:LiterateDocument, writer:TextWriter, ?prefix, ?lineNumbers, ?generateAnchors) =
     let ctx = formattingContext None (Some OutputKind.Html) prefix lineNumbers None generateAnchors None None
     let doc = Transformations.replaceLiterateParagraphs ctx doc
-    let doc = MarkdownDocument(doc.Paragraphs @ [InlineBlock doc.FormattedTips], doc.DefinedLinks)
+    let doc = MarkdownDocument(doc.Paragraphs @ [InlineBlock(doc.FormattedTips, None)], doc.DefinedLinks)
     Html.formatMarkdown writer ctx.GenerateHeaderAnchors Environment.NewLine true doc.DefinedLinks doc.Paragraphs
 
   static member WriteLatex(doc:LiterateDocument, ?prefix, ?lineNumbers, ?generateAnchors) =