diff --git a/docs/content/codeformat.fsx b/docs/content/codeformat.fsx index 570c84e12..12ae4d9bb 100644 --- a/docs/content/codeformat.fsx +++ b/docs/content/codeformat.fsx @@ -24,7 +24,7 @@ can be called to format snippets repeatedly: *) let fsharpCompiler = Assembly.Load("FSharp.Compiler") -let formattingAgent = CodeFormat.CreateAgent(fsharpCompiler) +let formattingAgent = CodeFormat.CreateAgent() (** If you want to process multiple snippets, it is a good idea to keep the diff --git a/docs/content/markdown.fsx b/docs/content/markdown.fsx index 87b1593d7..3818c6bcd 100644 --- a/docs/content/markdown.fsx +++ b/docs/content/markdown.fsx @@ -10,7 +10,10 @@ First, we need to load the assembly and open necessary namespaces: *) #r "../../bin/FSharp.Markdown.dll" +#r "../../bin/FSharp.Formatting.Common.dll" open FSharp.Markdown +open FSharp.Formatting.Common + (** Parsing documents @@ -64,7 +67,7 @@ The following snippet prints the heading of the document: // Iterate over all the paragraph elements for par in parsed.Paragraphs do match par with - | Heading(1, [Literal text]) -> + | Heading(size=1; body=[Literal(text=text)]) -> // Recognize heading that has a simple content // containing just a literal (no other formatting) printfn "%s" text @@ -92,8 +95,8 @@ to recognize any paragraph or span that can contain child elements: /// Returns all links in a specified span node let rec collectSpanLinks span = seq { match span with - | DirectLink(_, (url, _)) -> yield url - | IndirectLink(_, _, key) -> yield fst (parsed.DefinedLinks.[key]) + | DirectLink(_, (url, _), _) -> yield url + | IndirectLink(_, _, key, _) -> yield fst (parsed.DefinedLinks.[key]) | Matching.SpanLeaf _ -> () | Matching.SpanNode(_, spans) -> for s in spans do yield! collectSpanLinks s } diff --git a/src/Common/StringParsing.fs b/src/Common/StringParsing.fs index 0f376562a..e33c38c92 100644 --- a/src/Common/StringParsing.fs +++ b/src/Common/StringParsing.fs @@ -7,6 +7,7 @@ module internal FSharp.Patterns open System open FSharp.Collections +open FSharp.Formatting.Common // -------------------------------------------------------------------------------------- // Active patterns that simplify parsing of strings and lists of strings (lines) @@ -14,44 +15,133 @@ open FSharp.Collections module String = /// Matches when a string is a whitespace or null - let (|WhiteSpace|_|) s = + let (|WhiteSpace|_|) (s) = + if String.IsNullOrWhiteSpace(s) then Some() else None + + /// Returns a string trimmed from both start and end + let (|TrimBoth|) (text:string) = text.Trim() + + /// Matches when a string starts with the specified sub-string + let (|StartsWith|_|) (start:string) (text:string) = + if text.StartsWith(start) then Some(text.Substring(start.Length)) else None + + /// Matches when a string starts with the specified sub-string + /// The matched string is trimmed from all whitespace. + let (|StartsWithTrim|_|) (start:string) (text:string) = + if text.StartsWith(start) then Some(text.Substring(start.Length).Trim()) else None + + /// Matches when a string starts with the given value and ends + /// with a given value (and returns the rest of it) + let (|StartsAndEndsWith|_|) (starts, ends) (s:string) = + if s.StartsWith(starts) && s.EndsWith(ends) && + s.Length >= starts.Length + ends.Length then + Some(s.Substring(starts.Length, s.Length - starts.Length - ends.Length)) + else None + + /// Matches when a string starts with the given value and ends + /// with a given value (and returns trimmed body) + let (|StartsAndEndsWithTrim|_|) args = function + | StartsAndEndsWith args (TrimBoth res) -> Some res + | _ -> None + + /// Matches when a string starts with a sub-string wrapped using the + /// opening and closing sub-string specified in the parameter. + /// For example "[aa]bc" is wrapped in [ and ] pair. Returns the wrapped + /// text together with the rest. + let (|StartsWithWrapped|_|) (starts:string, ends:string) (text:string) = + if text.StartsWith(starts) then + let id = text.IndexOf(ends, starts.Length) + if id >= 0 then + let wrapped = text.Substring(starts.Length, id - starts.Length) + let rest = text.Substring(id + ends.Length, text.Length - id - ends.Length) + Some(wrapped, rest) + else None + else None + + /// Ignores everything until a end-line character is detected, returns the remaining string. + let (|SkipSingleLine|) (text:string) = + let rec tryEol eolList = + match eolList with + | h : string :: t -> + match text.IndexOf(h) with + | i when i < 0 -> tryEol t + | i -> + text.Substring (i + h.Length) + | _ -> + text + let result = tryEol [ "\r\n"; "\n" ] + let skipped = text.Substring(0, text.Length - result.Length) + if not <| String.IsNullOrWhiteSpace(skipped) then + FSharp.Formatting.Common.Log.warnf "skipped '%s' which contains non-whitespace character!" skipped + if result = text then + FSharp.Formatting.Common.Log.warnf "could not skip a line of %s, because no line-ending character was found!" text + result + + /// Given a list of lines indented with certan number of whitespace + /// characters (spaces), remove the spaces from the beginning of each line + /// and return the string as a list of lines + let removeSpaces lines = + let spaces = + lines + |> Seq.filter (String.IsNullOrWhiteSpace >> not) + |> Seq.map (fun line -> line |> Seq.takeWhile Char.IsWhiteSpace |> Seq.length) + |> fun xs -> if Seq.isEmpty xs then 0 else Seq.min xs + lines + |> Seq.map (fun line -> + if String.IsNullOrWhiteSpace(line) then "" + else line.Substring(spaces)) + +module StringPosition = + /// Matches when a string is a whitespace or null + let (|WhiteSpace|_|) (s, n: MarkdownRange) = if String.IsNullOrWhiteSpace(s) then Some() else None /// Matches when a string does starts with non-whitespace - let (|Unindented|_|) (s:string) = + let (|Unindented|_|) (s:string, n:MarkdownRange) = if not (String.IsNullOrWhiteSpace(s)) && s.TrimStart() = s then Some() else None - + /// Returns a string trimmed from both start and end - let (|TrimBoth|) (text:string) = text.Trim() + let (|TrimBoth|) (text:string, n:MarkdownRange) = + let trimmedStart = text.TrimStart() + let trimmed = trimmedStart.TrimEnd() + (trimmed, { n with StartColumn = n.StartColumn + text.Length - trimmedStart.Length; EndColumn = n.EndColumn - trimmedStart.Length + trimmed.Length }) /// Returns a string trimmed from the end - let (|TrimEnd|) (text:string) = text.TrimEnd() + let (|TrimEnd|) (text:string, n:MarkdownRange) = + let trimmed = text.TrimEnd() + (trimmed, { n with EndColumn = n.EndColumn - text.Length + trimmed.Length }) /// Returns a string trimmed from the start - let (|TrimStart|) (text:string) = text.TrimStart() + let (|TrimStart|) (text:string, n:MarkdownRange) = + let trimmed = text.TrimStart() + (trimmed, { n with StartColumn = n.StartColumn + text.Length - trimmed.Length }) - /// Retrusn a string trimmed from the end using characters given as a parameter - let (|TrimEndUsing|) chars (text:string) = text.TrimEnd(Array.ofSeq chars) + /// Returns a string trimmed from the end using characters given as a parameter + let (|TrimEndUsing|) chars (text:string, n:MarkdownRange) = + let trimmed = text.TrimEnd(Array.ofSeq chars) + (trimmed, { n with EndColumn = n.EndColumn - text.Length + trimmed.Length }) /// Returns a string trimmed from the start together with /// the number of skipped whitespace characters - let (|TrimStartAndCount|) (text:string) = + let (|TrimStartAndCount|) (text:string, n:MarkdownRange) = let trimmed = text.TrimStart([|' '; '\t'|]) let len = text.Length - trimmed.Length - len, text.Substring(0, len).Replace("\t", " ").Length, trimmed + len, text.Substring(0, len).Replace("\t", " ").Length, (trimmed, { n with StartColumn = n.StartColumn + text.Length - trimmed.Length }) /// Matches when a string starts with any of the specified sub-strings - let (|StartsWithAny|_|) (starts:seq) (text:string) = + let (|StartsWithAny|_|) (starts:seq) (text:string, n:MarkdownRange) = if starts |> Seq.exists (text.StartsWith) then Some() else None + /// Matches when a string starts with the specified sub-string - let (|StartsWith|_|) (start:string) (text:string) = - if text.StartsWith(start) then Some(text.Substring(start.Length)) else None + let (|StartsWith|_|) (start:string) (text:string, n:MarkdownRange) = + if text.StartsWith(start) then Some(text.Substring(start.Length), { n with StartColumn = n.StartColumn + text.Length - start.Length }) else None + /// Matches when a string starts with the specified sub-string /// The matched string is trimmed from all whitespace. - let (|StartsWithTrim|_|) (start:string) (text:string) = - if text.StartsWith(start) then Some(text.Substring(start.Length).Trim()) else None + let (|StartsWithTrim|_|) (start:string) (text:string, n:MarkdownRange) = + if text.StartsWith(start) then Some(text.Substring(start.Length).Trim(), { n with StartColumn = n.StartColumn + text.Length - start.Length }) else None /// Matches when a string starts with the specified sub-string (ignoring whitespace at the start) /// The matched string is trimmed from all whitespace. - let (|StartsWithNTimesTrimIgnoreStartWhitespace|_|) (start:string) (text:string) = + let (|StartsWithNTimesTrimIgnoreStartWhitespace|_|) (start:string) (text:string, n:MarkdownRange) = if text.Contains(start) then let beforeStart = text.Substring(0, text.IndexOf(start)) if String.IsNullOrWhiteSpace (beforeStart) then @@ -67,10 +157,10 @@ module String = /// Matches when a string starts with the given value and ends /// with a given value (and returns the rest of it) - let (|StartsAndEndsWith|_|) (starts, ends) (s:string) = + let (|StartsAndEndsWith|_|) (starts, ends) (s:string, n:MarkdownRange) = if s.StartsWith(starts) && s.EndsWith(ends) && s.Length >= starts.Length + ends.Length then - Some(s.Substring(starts.Length, s.Length - starts.Length - ends.Length)) + Some(s.Substring(starts.Length, s.Length - starts.Length - ends.Length), { n with StartColumn = n.StartColumn + s.Length - starts.Length; EndColumn = n.EndColumn - s.Length + ends.Length }) else None /// Matches when a string starts with the given value and ends @@ -85,7 +175,7 @@ module String = /// /// let (StartsWithRepeated "/\" (2, " abc")) = "/\/\ abc" /// - let (|StartsWithRepeated|_|) (repeated:string) (text:string) = + let (|StartsWithRepeated|_|) (repeated:string) (text:string, ln:MarkdownRange) = let rec loop i = if i = text.Length then i elif text.[i] <> repeated.[i % repeated.Length] then i @@ -93,70 +183,35 @@ module String = let n = loop 0 if n = 0 || n % repeated.Length <> 0 then None - else Some(n/repeated.Length, text.Substring(n, text.Length - n)) - - /// Ignores everything until a end-line character is detected, returns the remaining string. - let (|SkipSingleLine|) (text:string) = - let rec tryEol eolList = - match eolList with - | h : string :: t -> - match text.IndexOf(h) with - | i when i < 0 -> tryEol t - | i -> - text.Substring (i + h.Length) - | _ -> - text - let result = tryEol [ "\r\n"; "\n" ] - let skipped = text.Substring(0, text.Length - result.Length) - if not <| String.IsNullOrWhiteSpace(skipped) then - FSharp.Formatting.Common.Log.warnf "skipped '%s' which contains non-whitespace character!" skipped - if result = text then - FSharp.Formatting.Common.Log.warnf "could not skip a line of %s, because no line-ending character was found!" text - result - + else Some(n/repeated.Length, (text.Substring(n, text.Length - n), { ln with StartColumn = n })) /// Matches when a string starts with a sub-string wrapped using the /// opening and closing sub-string specified in the parameter. /// For example "[aa]bc" is wrapped in [ and ] pair. Returns the wrapped /// text together with the rest. - let (|StartsWithWrapped|_|) (starts:string, ends:string) (text:string) = + let (|StartsWithWrapped|_|) (starts:string, ends:string) (text:string, n:MarkdownRange) = if text.StartsWith(starts) then let id = text.IndexOf(ends, starts.Length) if id >= 0 then let wrapped = text.Substring(starts.Length, id - starts.Length) let rest = text.Substring(id + ends.Length, text.Length - id - ends.Length) - Some(wrapped, rest) + Some(wrapped, (rest, { n with StartColumn = id + ends.Length })) else None else None /// Matches when a string consists of some number of /// complete repetitions of a specified sub-string. - let (|EqualsRepeated|_|) repeated = function - | StartsWithRepeated repeated (n, "") -> Some() + let (|EqualsRepeated|_|) (repeated, n:MarkdownRange) = function + | StartsWithRepeated repeated (n, ("", _)) -> Some() | _ -> None - /// Given a list of lines indented with certan number of whitespace - /// characters (spaces), remove the spaces from the beginning of each line - /// and return the string as a list of lines - let removeSpaces lines = - let spaces = - lines - |> Seq.filter (String.IsNullOrWhiteSpace >> not) - |> Seq.map (fun line -> line |> Seq.takeWhile Char.IsWhiteSpace |> Seq.length) - |> fun xs -> if Seq.isEmpty xs then 0 else Seq.min xs - lines - |> Seq.map (fun line -> - if String.IsNullOrWhiteSpace(line) then "" - else line.Substring(spaces)) - - module List = /// Matches a list if it starts with a sub-list that is delimited /// using the specified delimiters. Returns a wrapped list and the rest. let inline (|DelimitedWith|_|) startl endl input = if List.startsWith startl input then match List.partitionUntilEquals endl (List.skip startl.Length input) with - | Some(pre, post) -> Some(pre, List.skip endl.Length post) + | Some(pre, post) -> Some(pre, List.skip endl.Length post, startl.Length, endl.Length) | None -> None else None @@ -197,8 +252,8 @@ module Lines = /// Removes blank lines from the start and the end of a list let (|TrimBlank|) lines = lines - |> List.skipWhile String.IsNullOrWhiteSpace |> List.rev - |> List.skipWhile String.IsNullOrWhiteSpace |> List.rev + |> List.skipWhile (fun (s, n) -> String.IsNullOrWhiteSpace s) |> List.rev + |> List.skipWhile (fun (s, n) -> String.IsNullOrWhiteSpace s) |> List.rev /// Matches when there are some lines at the beginning that are /// either empty (or whitespace) or start with the specified string. @@ -213,7 +268,7 @@ module Lines = /// either empty (or whitespace) or start with at least 4 spaces (a tab counts as 4 spaces here). /// Returns all such lines from the beginning until a different line and /// the number of spaces the first line started with. - let (|TakeCodeBlock|_|) (input:string list) = + let (|TakeCodeBlock|_|) (input:(string * MarkdownRange) list) = let spaceNum = 4 //match input with //| h :: _ -> @@ -225,24 +280,25 @@ module Lines = let normalized = s.Replace("\t", " ") normalized.Length >= spaceNum && normalized.Substring(0, spaceNum) = System.String(' ', spaceNum) - match List.partitionWhile (fun s -> + match List.partitionWhile (fun (s, n) -> String.IsNullOrWhiteSpace s || startsWithSpaces s) input with | matching, rest when matching <> [] && spaceNum >= 4 -> Some(spaceNum, matching, rest) | _ -> None /// Removes whitespace lines from the beginning of the list - let (|TrimBlankStart|) = List.skipWhile (String.IsNullOrWhiteSpace) + let (|TrimBlankStart|) = List.skipWhile (fun (s:string, n:MarkdownRange) -> String.IsNullOrWhiteSpace s) /// Trims all lines of the current paragraph let (|TrimParagraphLines|) lines = lines // first remove all whitespace on the beginning of the line - |> List.map (fun (s:string) -> s.TrimStart()) + |> List.map StringPosition.(|TrimStart|) // Now remove all additional spaces at the end, but keep two spaces if existent - |> List.map (fun s -> + |> List.map (fun (s, n) -> let endsWithTwoSpaces = s.EndsWith(" ") - s.TrimEnd([|' '|]) + if endsWithTwoSpaces then " " else "") + let trimmed = s.TrimEnd([|' '|]) + if endsWithTwoSpaces then " " else "" + (trimmed, { n with EndColumn = n.EndColumn - s.Length + trimmed.Length })) /// Parameterized pattern that assigns the specified value to the /// first component of a tuple. Usage: diff --git a/src/FSharp.CodeFormat/CodeFormatAgent.fs b/src/FSharp.CodeFormat/CodeFormatAgent.fs index 1c7dfb1e0..29c3e291e 100644 --- a/src/FSharp.CodeFormat/CodeFormatAgent.fs +++ b/src/FSharp.CodeFormat/CodeFormatAgent.fs @@ -73,7 +73,7 @@ module private Helpers = state := nstate yield! parseLine() | None, nstate -> state := nstate } - yield n, parseLine() |> List.ofSeq ] + yield { StartLine = n; StartColumn = 0; EndLine = n; EndColumn = 0 }, parseLine() |> List.ofSeq ] // Count the minimal number of spaces at the beginning of lines // (so that we can remove spaces for indented text) @@ -238,10 +238,10 @@ type CodeFormatAgent() = processSnippetLine checkResults (categorizedSpans - |> Map.tryFind ((fst snippetLine) + 1) + |> Map.tryFind ((fst snippetLine).StartLine + 1) |> function None -> [] | Some spans -> List.ofSeq spans) lines - snippetLine) + ((fst snippetLine).StartLine, snd snippetLine)) // -------------------------------------------------------------------------------------- diff --git a/src/FSharp.CodeFormat/CommentFilter.fs b/src/FSharp.CodeFormat/CommentFilter.fs index 279f5e176..52043ba92 100644 --- a/src/FSharp.CodeFormat/CommentFilter.fs +++ b/src/FSharp.CodeFormat/CommentFilter.fs @@ -11,6 +11,7 @@ open System.Web open FSharp.Patterns open FSharp.Collections +open FSharp.Formatting.Common open Microsoft.FSharp.Compiler open Microsoft.FSharp.Compiler.SourceCodeServices @@ -34,7 +35,7 @@ open Microsoft.FSharp.Compiler.SourceCodeServices type Token = string * FSharpTokenInfo type SnippetLine = Token list -type IndexedSnippetLine = int * SnippetLine +type IndexedSnippetLine = MarkdownRange * SnippetLine type Snippet = IndexedSnippetLine list type NamedSnippet = string * Snippet @@ -49,19 +50,19 @@ let rec getSnippets (state:NamedSnippet option) (snippets:NamedSnippet list) match source with | [] -> snippets | (line, tokens)::rest -> - let text = lines.[line].Trim() + let text = lines.[line.StartLine].Trim(), line match state, text with // We're not inside a snippet and we found a beginning of one - | None, String.StartsWithTrim "//" (String.StartsWithTrim "[snippet:" title) -> - let title = title.Substring(0, title.IndexOf(']')) + | None, StringPosition.StartsWithTrim "//" (StringPosition.StartsWithTrim "[snippet:" title) -> + let title = (fst title).Substring(0, (fst title).IndexOf(']')) getSnippets (Some(title, [])) snippets rest lines // Not inside a snippet and there is a usual line | None, _ -> getSnippets state snippets rest lines // We're inside a snippet and it ends - | Some(title, acc), String.StartsWithTrim "//" (String.StartsWithTrim "[/snippet]" _) -> + | Some(title, acc), StringPosition.StartsWithTrim "//" (StringPosition.StartsWithTrim "[/snippet]" _) -> getSnippets None ((title, acc |> List.rev)::snippets) rest lines // We're inside snippet - add current line to it | Some(title, acc), _ -> diff --git a/src/FSharp.Formatting.Common/FSharp.Formatting.Common.fsproj b/src/FSharp.Formatting.Common/FSharp.Formatting.Common.fsproj index 07fc84a1c..33e7125c9 100644 --- a/src/FSharp.Formatting.Common/FSharp.Formatting.Common.fsproj +++ b/src/FSharp.Formatting.Common/FSharp.Formatting.Common.fsproj @@ -69,6 +69,7 @@ Common\AssemblyInfo.fs + diff --git a/src/FSharp.Formatting.Common/Range.fs b/src/FSharp.Formatting.Common/Range.fs new file mode 100644 index 000000000..f0946edad --- /dev/null +++ b/src/FSharp.Formatting.Common/Range.fs @@ -0,0 +1,12 @@ +namespace FSharp.Formatting.Common + +type MarkdownRange = { StartLine : int; StartColumn : int; EndLine : int; EndColumn : int } + +[] +module MarkdownRange = + let Zero = { StartLine = 0; StartColumn = 0; EndLine = 0; EndColumn = 0 } + + let MergeRanges (ranges:MarkdownRange list) = + let startRange = ranges |> List.minBy (fun r -> r.StartLine * 10 + r.StartColumn) + let endRange = ranges |> List.maxBy (fun r -> r.EndLine * 10 + r.EndColumn) + { StartLine = startRange.StartLine; StartColumn = startRange.StartColumn; EndLine = endRange.EndLine; EndColumn = endRange.EndColumn } \ No newline at end of file diff --git a/src/FSharp.Literate/Document.fs b/src/FSharp.Literate/Document.fs index 0441373d4..e7bc9323b 100644 --- a/src/FSharp.Literate/Document.fs +++ b/src/FSharp.Literate/Document.fs @@ -112,4 +112,4 @@ type LiterateDocument(paragraphs, formattedTips, links, source, sourceFile, erro /// Markdown documents. module Matching = let (|LiterateParagraph|_|) = function - | EmbedParagraphs(:? LiterateParagraph as lp) -> Some lp | _ -> None \ No newline at end of file + | EmbedParagraphs(:? LiterateParagraph as lp, _) -> Some lp | _ -> None \ No newline at end of file diff --git a/src/FSharp.Literate/Evaluator.fs b/src/FSharp.Literate/Evaluator.fs index 27a87486f..2241114d5 100644 --- a/src/FSharp.Literate/Evaluator.fs +++ b/src/FSharp.Literate/Evaluator.fs @@ -111,7 +111,7 @@ type FsiEvaluator(?options:string[], ?fsiObj) = /// Registered transformations for pretty printing values /// (the default formats value as a string and emits single CodeBlock) let mutable valueTransformations = - [ (fun (o:obj, t:Type) ->Some([CodeBlock (sprintf "%A" o, "", "")]) ) ] + [ (fun (o:obj, t:Type) ->Some([CodeBlock (sprintf "%A" o, "", "", None)]) ) ] /// Register a function that formats (some) values that are produced by the evaluator. /// The specified function should return 'Some' when it knows how to format a value @@ -130,12 +130,12 @@ type FsiEvaluator(?options:string[], ?fsiObj) = match result :?> FsiEvaluationResult, kind with | result, FsiEmbedKind.Output -> let s = defaultArg result.Output "No output has been produced." - [ CodeBlock(s.Trim(), "", "") ] + [ CodeBlock(s.Trim(), "", "", None) ] | { ItValue = Some v }, FsiEmbedKind.ItValue | { Result = Some v }, FsiEmbedKind.Value -> valueTransformations |> Seq.pick (fun f -> lock lockObj (fun () -> f v)) - | _, FsiEmbedKind.ItValue -> [ CodeBlock ("No value has been returned", "", "") ] - | _, FsiEmbedKind.Value -> [ CodeBlock ("No value has been returned", "", "") ] + | _, FsiEmbedKind.ItValue -> [ CodeBlock ("No value has been returned", "", "", None) ] + | _, FsiEmbedKind.Value -> [ CodeBlock ("No value has been returned", "", "", None) ] /// Evaluates the given text in an fsi session and returns /// an FsiEvaluationResult. diff --git a/src/FSharp.Literate/Formatting.fs b/src/FSharp.Literate/Formatting.fs index 9d02c7c23..93bbad5a3 100644 --- a/src/FSharp.Literate/Formatting.fs +++ b/src/FSharp.Literate/Formatting.fs @@ -27,8 +27,8 @@ module Formatting = /// Try find first-level heading in the paragraph collection let findHeadings paragraphs generateAnchors (outputKind:OutputKind) = paragraphs |> Seq.tryPick (function - | (Heading(1, text)) -> - let doc = MarkdownDocument([Span(text)], dict []) + | Heading(1, text, r) -> + let doc = MarkdownDocument([Span(text, r)], dict []) Some(format doc generateAnchors outputKind) | _ -> None) @@ -37,13 +37,13 @@ module Formatting = let getSourceDocument (doc:LiterateDocument) = match doc.Source with | LiterateSource.Markdown text -> - doc.With(paragraphs = [CodeBlock (text, "", "")]) + doc.With(paragraphs = [CodeBlock (text, "", "", None)]) | LiterateSource.Script snippets -> let paragraphs = [ for Snippet(name, lines) in snippets do if snippets.Length > 1 then - yield Heading(3, [Literal name]) - yield EmbedParagraphs(FormattedCode(lines)) ] + yield Heading(3, [Literal(name, None)], None) + yield EmbedParagraphs(FormattedCode(lines), None) ] doc.With(paragraphs = paragraphs) // -------------------------------------------------------------------------------------- diff --git a/src/FSharp.Literate/Main.fs b/src/FSharp.Literate/Main.fs index bc7b0cef0..37a7314ac 100644 --- a/src/FSharp.Literate/Main.fs +++ b/src/FSharp.Literate/Main.fs @@ -104,7 +104,7 @@ type Literate private () = static member WriteHtml(doc:LiterateDocument, ?prefix, ?lineNumbers, ?generateAnchors) = let ctx = formattingContext None (Some OutputKind.Html) prefix lineNumbers None generateAnchors None None let doc = Transformations.replaceLiterateParagraphs ctx doc - let doc = MarkdownDocument(doc.Paragraphs @ [InlineBlock doc.FormattedTips], doc.DefinedLinks) + let doc = MarkdownDocument(doc.Paragraphs @ [InlineBlock(doc.FormattedTips, None)], doc.DefinedLinks) let sb = new System.Text.StringBuilder() use wr = new StringWriter(sb) Html.formatMarkdown wr ctx.GenerateHeaderAnchors Environment.NewLine true doc.DefinedLinks doc.Paragraphs @@ -113,7 +113,7 @@ type Literate private () = static member WriteHtml(doc:LiterateDocument, writer:TextWriter, ?prefix, ?lineNumbers, ?generateAnchors) = let ctx = formattingContext None (Some OutputKind.Html) prefix lineNumbers None generateAnchors None None let doc = Transformations.replaceLiterateParagraphs ctx doc - let doc = MarkdownDocument(doc.Paragraphs @ [InlineBlock doc.FormattedTips], doc.DefinedLinks) + let doc = MarkdownDocument(doc.Paragraphs @ [InlineBlock(doc.FormattedTips, None)], doc.DefinedLinks) Html.formatMarkdown writer ctx.GenerateHeaderAnchors Environment.NewLine true doc.DefinedLinks doc.Paragraphs static member WriteLatex(doc:LiterateDocument, ?prefix, ?lineNumbers, ?generateAnchors) = diff --git a/src/FSharp.Literate/ParseScript.fs b/src/FSharp.Literate/ParseScript.fs index 70edf8308..34e2c4076 100644 --- a/src/FSharp.Literate/ParseScript.fs +++ b/src/FSharp.Literate/ParseScript.fs @@ -132,19 +132,19 @@ module internal ParseScript = // Reference to code snippet defined later | BlockCommand(Command "include" ref)::blocks -> - let p = EmbedParagraphs(CodeReference(ref)) + let p = EmbedParagraphs(CodeReference(ref), None) transformBlocks noEval (p::acc) defs blocks | BlockCommand(Command "include-output" ref)::blocks -> - let p = EmbedParagraphs(OutputReference(ref)) + let p = EmbedParagraphs(OutputReference(ref), None) transformBlocks noEval (p::acc) defs blocks | BlockCommand(Command "include-it" ref)::blocks -> - let p = EmbedParagraphs(ItValueReference(ref)) + let p = EmbedParagraphs(ItValueReference(ref), None) transformBlocks noEval (p::acc) defs blocks | BlockCommand(Command "include-value" ref)::blocks -> - let p = EmbedParagraphs(ValueReference(ref)) + let p = EmbedParagraphs(ValueReference(ref), None) transformBlocks noEval (p::acc) defs blocks | BlockCommand(Command "raw" _) ::BlockSnippet(snip):: blocks -> - let p = EmbedParagraphs(RawBlock(snip)) + let p = EmbedParagraphs(RawBlock(snip), None) transformBlocks noEval (p::acc) defs blocks // Parse commands in [foo=bar,zoo], followed by a source code snippet @@ -166,7 +166,7 @@ module internal ParseScript = { Evaluate = not (noEval || cmds.ContainsKey("do-not-eval")) OutputName = outputName Visibility = visibility } - let code = EmbedParagraphs(LiterateCode(snip, opts)) + let code = EmbedParagraphs(LiterateCode(snip, opts), None) transformBlocks noEval (code::acc) defs blocks // Unknown command @@ -178,7 +178,7 @@ module internal ParseScript = transformBlocks noEval acc defs blocks // Ordinary F# code snippet | BlockSnippet(snip)::blocks -> - let p = EmbedParagraphs(FormattedCode(snip)) + let p = EmbedParagraphs(FormattedCode(snip), None) transformBlocks noEval (p::acc) defs blocks // Markdown documentation block | BlockComment(text)::blocks -> diff --git a/src/FSharp.Literate/Transformations.fs b/src/FSharp.Literate/Transformations.fs index 141a42ca0..d5253e096 100644 --- a/src/FSharp.Literate/Transformations.fs +++ b/src/FSharp.Literate/Transformations.fs @@ -21,10 +21,10 @@ module Transformations = /// to colorize. We skip snippets that specify non-fsharp langauge e.g. [lang=csharp]. let rec collectCodeSnippets par = seq { match par with - | CodeBlock((String.StartsWithWrapped ("[", "]") (ParseCommands cmds, String.SkipSingleLine code)), language, _) + | CodeBlock((String.StartsWithWrapped ("[", "]") (ParseCommands cmds, String.SkipSingleLine code)), language, _, _) when (not (String.IsNullOrWhiteSpace(language)) && language <> "fsharp") || (cmds.ContainsKey("lang") && cmds.["lang"] <> "fsharp") -> () - | CodeBlock((String.StartsWithWrapped ("[", "]") (ParseCommands cmds, String.SkipSingleLine code)), _, _) - | CodeBlock(Let (dict []) (cmds, code), _, _) -> + | CodeBlock((String.StartsWithWrapped ("[", "]") (ParseCommands cmds, String.SkipSingleLine code)), _, _, _) + | CodeBlock(Let (dict []) (cmds, code), _, _, _) -> let modul = match cmds.TryGetValue("module") with | true, v -> Some v | _ -> None @@ -39,8 +39,8 @@ module Transformations = /// Replace CodeBlock elements with formatted HTML that was processed by the F# snippets tool /// (The dictionary argument is a map from original code snippets to formatted HTML snippets.) let rec replaceCodeSnippets path (codeLookup:IDictionary<_, _>) = function - | CodeBlock ((String.StartsWithWrapped ("[", "]") (ParseCommands cmds, String.SkipSingleLine code)), language, _) - | CodeBlock(Let (dict []) (cmds, code), language, _) -> + | CodeBlock ((String.StartsWithWrapped ("[", "]") (ParseCommands cmds, String.SkipSingleLine code)), language, _, r) + | CodeBlock(Let (dict []) (cmds, code), language, _, r) -> if cmds.ContainsKey("hide") then None else let code = if cmds.ContainsKey("file") && cmds.ContainsKey("key") then @@ -59,9 +59,9 @@ module Transformations = | String.WhiteSpace when cmds.ContainsKey("lang") -> cmds.["lang"] | language -> language if not (String.IsNullOrWhiteSpace(lang)) && lang <> "fsharp" then - Some (EmbedParagraphs(LanguageTaggedCode(lang, code))) + Some (EmbedParagraphs(LanguageTaggedCode(lang, code), r)) else - Some (EmbedParagraphs(FormattedCode(codeLookup.[code]))) + Some (EmbedParagraphs(FormattedCode(codeLookup.[code]), r)) // Recursively process nested paragraphs, other nodes return without change | Matching.ParagraphNested(pn, nested) -> @@ -116,7 +116,7 @@ module Transformations = // Collect IndirectLinks in a span let rec collectSpanReferences span = seq { match span with - | IndirectLink(_, _, key) -> yield key + | IndirectLink(_, _, key, _) -> yield key | Matching.SpanLeaf _ -> () | Matching.SpanNode(_, spans) -> for s in spans do yield! collectSpanReferences s } @@ -137,13 +137,13 @@ module Transformations = let replaceReferences (refIndex:IDictionary) = // Replace IndirectLinks with a nice link given a single span element let rec replaceSpans = function - | IndirectLink(body, original, key) -> - [ yield IndirectLink(body, original, key) + | IndirectLink(body, original, key, r) -> + [ yield IndirectLink(body, original, key, r) match refIndex.TryGetValue(key) with | true, i -> - yield Literal " [" - yield DirectLink([Literal (string i)], ("#rf" + DateTime.Now.ToString("yyMMddhh"), None)) - yield Literal "]" + yield Literal(" [", r) + yield DirectLink([Literal (string i, r)], ("#rf" + DateTime.Now.ToString("yyMMddhh"), None), r) + yield Literal("]", r) | _ -> () ] | Matching.SpanLeaf(sl) -> [Matching.SpanLeaf(sl)] | Matching.SpanNode(nd, spans) -> @@ -179,18 +179,18 @@ module Transformations = if colon > 0 then let auth = title.Substring(0, colon) let name = title.Substring(colon + 1, title.Length - 1 - colon) - yield [Span [ Literal (sprintf "[%d] " i) - DirectLink([Literal (name.Trim())], (link, Some title)) - Literal (" - " + auth)] ] + yield [Span([ Literal (sprintf "[%d] " i, None) + DirectLink([Literal (name.Trim(), None)], (link, Some title), None) + Literal (" - " + auth, None)], None) ] else - yield [Span [ Literal (sprintf "[%d] " i) - DirectLink([Literal title], (link, Some title))]] ] + yield [Span([ Literal (sprintf "[%d] " i, None) + DirectLink([Literal(title, None)], (link, Some title), None)], None)] ] // Return the document together with dictionary for looking up indices let id = DateTime.Now.ToString("yyMMddhh") - [ Paragraph [AnchorLink id]; - Heading(3, [Literal "References"]) - ListBlock(MarkdownListKind.Unordered, refList) ], refLookup + [ Paragraph([AnchorLink(id, None)], None) + Heading(3, [Literal("References", None)], None) + ListBlock(MarkdownListKind.Unordered, refList, None) ], refLookup /// Turn all indirect links into a references /// and add paragraph to the document @@ -270,8 +270,8 @@ module Transformations = | _ -> None match special with | EvalFormat(Some result, _, kind) -> ctx.Evaluator.Value.Format(result, kind) - | EvalFormat(None, ref, _) -> [ CodeBlock("Could not find reference '" + ref + "'", "", "") ] - | other -> [ EmbedParagraphs(other) ] + | EvalFormat(None, ref, _) -> [ CodeBlock("Could not find reference '" + ref + "'", "", "", None) ] + | other -> [ EmbedParagraphs(other, None) ] // Traverse all other structrues recursively | Matching.ParagraphNested(pn, nested) -> @@ -311,7 +311,7 @@ module Transformations = let rec replaceSpecialCodes ctx (formatted:IDictionary<_, _>) = function | Matching.LiterateParagraph(special) -> match special with - | RawBlock lines -> Some (InlineBlock (unparse lines)) + | RawBlock lines -> Some (InlineBlock(unparse lines, None)) | LiterateCode(_, { Visibility = (HiddenCode | NamedCode _) }) -> None | FormattedCode lines | LiterateCode(lines, _) -> Some (formatted.[Choice1Of2 lines]) @@ -352,7 +352,7 @@ module Transformations = | OutputKind.Latex -> sprintf "\\begin{lstlisting}\n%s\n\\end{lstlisting}" code - Some(InlineBlock(inlined)) + Some(InlineBlock(inlined, None)) // Traverse all other structures recursively | Matching.ParagraphNested(pn, nested) -> let nested = List.map (List.choose (replaceSpecialCodes ctx formatted)) nested @@ -379,7 +379,7 @@ module Transformations = | OutputKind.Latex -> CodeFormat.FormatLatex(snippets, ctx.GenerateLineNumbers) let lookup = [ for (key, _), fmtd in Seq.zip replacements formatted.Snippets -> - key, InlineBlock(fmtd.Content) ] |> dict + key, InlineBlock(fmtd.Content, None) ] |> dict // Replace original snippets with formatted HTML/Latex and return document let newParagraphs = List.choose (replaceSpecialCodes ctx lookup) doc.Paragraphs diff --git a/src/FSharp.Markdown/HtmlFormatting.fs b/src/FSharp.Markdown/HtmlFormatting.fs index a256cd19c..45badc162 100644 --- a/src/FSharp.Markdown/HtmlFormatting.fs +++ b/src/FSharp.Markdown/HtmlFormatting.fs @@ -67,19 +67,19 @@ let noBreak (ctx:FormattingContext) () = () /// Write MarkdownSpan value to a TextWriter let rec formatSpan (ctx:FormattingContext) = function - | LatexDisplayMath(body) -> + | LatexDisplayMath(body, _) -> // use mathjax grammar, for detail, check: http://www.mathjax.org/ ctx.Writer.Write("\\[" + (htmlEncode body) + "\\]") - | LatexInlineMath(body) -> + | LatexInlineMath(body, _) -> // use mathjax grammar, for detail, check: http://www.mathjax.org/ ctx.Writer.Write("\\(" + (htmlEncode body) + "\\)") - | AnchorLink(id) -> ctx.Writer.Write(" ") - | EmbedSpans(cmd) -> formatSpans ctx (cmd.Render()) - | Literal(str) -> ctx.Writer.Write(str) - | HardLineBreak -> ctx.Writer.Write("
" + ctx.Newline) - | IndirectLink(body, _, LookupKey ctx.Links (link, title)) - | DirectLink(body, (link, title)) -> + | AnchorLink(id, _) -> ctx.Writer.Write(" ") + | EmbedSpans(cmd, _) -> formatSpans ctx (cmd.Render()) + | Literal(str, _) -> ctx.Writer.Write(str) + | HardLineBreak(_) -> ctx.Writer.Write("
" + ctx.Newline) + | IndirectLink(body, _, LookupKey ctx.Links (link, title), _) + | DirectLink(body, (link, title), _) -> ctx.Writer.Write("") - | IndirectLink(body, original, _) -> + | IndirectLink(body, original, _, _) -> ctx.Writer.Write("[") formatSpans ctx body ctx.Writer.Write("]") ctx.Writer.Write(original) - | IndirectImage(body, _, LookupKey ctx.Links (link, title)) - | DirectImage(body, (link, title)) -> + | IndirectImage(body, _, LookupKey ctx.Links (link, title), _) + | DirectImage(body, link, title, _) -> ctx.Writer.Write("\"") () ctx.Writer.Write("\" />") - | IndirectImage(body, original, _) -> + | IndirectImage(body, original, _, _) -> ctx.Writer.Write("[") ctx.Writer.Write(body) ctx.Writer.Write("]") ctx.Writer.Write(original) - | Strong(body) -> + | Strong(body, _) -> ctx.Writer.Write("") formatSpans ctx body ctx.Writer.Write("") - | InlineCode(body) -> + | InlineCode(body, _) -> ctx.Writer.Write("") ctx.Writer.Write(htmlEncode body) ctx.Writer.Write("") - | Emphasis(body) -> + | Emphasis(body, _) -> ctx.Writer.Write("") formatSpans ctx body ctx.Writer.Write("") @@ -141,10 +141,10 @@ let formatAnchor (ctx:FormattingContext) (spans:MarkdownSpans) = let rec gather (span:MarkdownSpan) : seq = seq { match span with - | Literal str -> yield! extractWords str - | Strong body -> yield! gathers body - | Emphasis body -> yield! gathers body - | DirectLink (body,_) -> yield! gathers body + | Literal(str, _) -> yield! extractWords str + | Strong(body, _) -> yield! gathers body + | Emphasis(body, _) -> yield! gathers body + | DirectLink(body, _, _) -> yield! gathers body | _ -> () } @@ -163,13 +163,13 @@ let withInner ctx f = /// Write a MarkdownParagraph value to a TextWriter let rec formatParagraph (ctx:FormattingContext) paragraph = match paragraph with - | LatexBlock(lines) -> + | LatexBlock(lines, _) -> // use mathjax grammar, for detail, check: http://www.mathjax.org/ let body = String.concat ctx.Newline lines ctx.Writer.Write("

\\[" + (htmlEncode body) + "\\]

") - | EmbedParagraphs(cmd) -> formatParagraphs ctx (cmd.Render()) - | Heading(n, spans) -> + | EmbedParagraphs(cmd, _) -> formatParagraphs ctx (cmd.Render()) + | Heading(n, spans, _) -> ctx.Writer.Write("") if ctx.GenerateHeaderAnchors then let anchorName = formatAnchor ctx spans @@ -179,28 +179,28 @@ let rec formatParagraph (ctx:FormattingContext) paragraph = else formatSpans ctx spans ctx.Writer.Write("") - | Paragraph(spans) -> + | Paragraph(spans, _) -> ctx.ParagraphIndent() ctx.Writer.Write("

") for span in spans do formatSpan ctx span ctx.Writer.Write("

") - | HorizontalRule(_) -> + | HorizontalRule(_, _) -> ctx.Writer.Write("
") - | CodeBlock(code, String.WhiteSpace, _) -> + | CodeBlock(code, String.WhiteSpace, _, _) -> if ctx.WrapCodeSnippets then ctx.Writer.Write("
") ctx.Writer.Write("
")
       ctx.Writer.Write(htmlEncode code)
       ctx.Writer.Write("
") if ctx.WrapCodeSnippets then ctx.Writer.Write("
") - | CodeBlock(code, codeLanguage, _) -> + | CodeBlock(code, codeLanguage, _, _) -> if ctx.WrapCodeSnippets then ctx.Writer.Write("
") let langCode = sprintf "language-%s" codeLanguage ctx.Writer.Write(sprintf "
" langCode)
       ctx.Writer.Write(htmlEncode code)
       ctx.Writer.Write("
") if ctx.WrapCodeSnippets then ctx.Writer.Write("
") - | TableBlock(headers, alignments, rows) -> + | TableBlock(headers, alignments, rows, _) -> let aligns = alignments |> List.map (function | AlignLeft -> " align=\"left\"" | AlignRight -> " align=\"right\"" @@ -229,14 +229,14 @@ let rec formatParagraph (ctx:FormattingContext) paragraph = ctx.Writer.Write("") ctx.Writer.Write(ctx.Newline) - | ListBlock(kind, items) -> + | ListBlock(kind, items, _) -> let tag = if kind = Ordered then "ol" else "ul" ctx.Writer.Write("<" + tag + ">" + ctx.Newline) for body in items do ctx.Writer.Write("
  • ") match body with // Simple Paragraph - | [ Paragraph [MarkdownSpan.Literal s] ] when not (s.Contains(ctx.Newline)) -> + | [ Paragraph([MarkdownSpan.Literal(s, _)], _) ] when not (s.Contains(ctx.Newline)) -> ctx.Writer.Write s | _ -> let inner = @@ -249,15 +249,15 @@ let rec formatParagraph (ctx:FormattingContext) paragraph = ctx.Writer.Write(wrappedInner) ctx.Writer.Write("
    • " + ctx.Newline) ctx.Writer.Write("") - | QuotedBlock(body) -> + | QuotedBlock(body, _) -> ctx.ParagraphIndent() ctx.Writer.Write("
    " + ctx.Newline) formatParagraphs { ctx with ParagraphIndent = fun () -> ctx.ParagraphIndent() (*; ctx.Writer.Write(" ")*) } body ctx.ParagraphIndent() ctx.Writer.Write("
    ") - | Span spans -> + | Span(spans, _) -> formatSpans ctx spans - | InlineBlock(code) -> + | InlineBlock(code, _) -> ctx.Writer.Write(code) ctx.LineBreak() diff --git a/src/FSharp.Markdown/LatexFormatting.fs b/src/FSharp.Markdown/LatexFormatting.fs index cb3bb8bc0..5b58e79e3 100644 --- a/src/FSharp.Markdown/LatexFormatting.fs +++ b/src/FSharp.Markdown/LatexFormatting.fs @@ -55,25 +55,25 @@ let noBreak (ctx:FormattingContext) () = () /// Write MarkdownSpan value to a TextWriter let rec formatSpan (ctx:FormattingContext) = function - | LatexInlineMath(body) -> ctx.Writer.Write(sprintf "$%s$" body) - | LatexDisplayMath(body) -> ctx.Writer.Write(sprintf "$$%s$$" body) - | EmbedSpans(cmd) -> formatSpans ctx (cmd.Render()) - | Literal(str) -> ctx.Writer.Write(latexEncode str) - | HardLineBreak -> ctx.LineBreak(); ctx.LineBreak() + | LatexInlineMath(body, _) -> ctx.Writer.Write(sprintf "$%s$" body) + | LatexDisplayMath(body, _) -> ctx.Writer.Write(sprintf "$$%s$$" body) + | EmbedSpans(cmd, _) -> formatSpans ctx (cmd.Render()) + | Literal(str, _) -> ctx.Writer.Write(latexEncode str) + | HardLineBreak(_) -> ctx.LineBreak(); ctx.LineBreak() | AnchorLink _ -> () - | IndirectLink(body, _, LookupKey ctx.Links (link, _)) - | DirectLink(body, (link, _)) - | IndirectLink(body, link, _) -> + | IndirectLink(body, _, LookupKey ctx.Links (link, _), _) + | DirectLink(body, (link, _), _) + | IndirectLink(body, link, _, _) -> ctx.Writer.Write(@"\href{") ctx.Writer.Write(latexEncode link) ctx.Writer.Write("}{") formatSpans ctx body ctx.Writer.Write("}") - | IndirectImage(body, _, LookupKey ctx.Links (link, _)) - | DirectImage(body, (link, _)) - | IndirectImage(body, link, _) -> + | IndirectImage(body, _, LookupKey ctx.Links (link, _), _) + | DirectImage(body, link, _, _) + | IndirectImage(body, link, _, _) -> // Use the technique introduced at // http://stackoverflow.com/q/14014827 if not (System.String.IsNullOrWhiteSpace(body)) then @@ -91,15 +91,15 @@ let rec formatSpan (ctx:FormattingContext) = function ctx.Writer.Write(@"\end{figure}") ctx.LineBreak() - | Strong(body) -> + | Strong(body, _) -> ctx.Writer.Write(@"\textbf{") formatSpans ctx body ctx.Writer.Write("}") - | InlineCode(body) -> + | InlineCode(body, _) -> ctx.Writer.Write(@"\texttt{") ctx.Writer.Write(latexEncode body) ctx.Writer.Write("}") - | Emphasis(body) -> + | Emphasis(body, _) -> ctx.Writer.Write(@"\emph{") formatSpans ctx body ctx.Writer.Write("}") @@ -110,7 +110,7 @@ and formatSpans ctx = List.iter (formatSpan ctx) /// Write a MarkdownParagraph value to a TextWriter let rec formatParagraph (ctx:FormattingContext) paragraph = match paragraph with - | LatexBlock(lines) -> + | LatexBlock(lines, _) -> ctx.LineBreak(); ctx.LineBreak() ctx.Writer.Write("\["); ctx.LineBreak() for line in lines do @@ -119,8 +119,8 @@ let rec formatParagraph (ctx:FormattingContext) paragraph = ctx.Writer.Write("\]") ctx.LineBreak(); ctx.LineBreak() - | EmbedParagraphs(cmd) -> formatParagraphs ctx (cmd.Render()) - | Heading(n, spans) -> + | EmbedParagraphs(cmd, _) -> formatParagraphs ctx (cmd.Render()) + | Heading(n, spans, _) -> let level = match n with | 1 -> @"\section*" @@ -133,7 +133,7 @@ let rec formatParagraph (ctx:FormattingContext) paragraph = formatSpans ctx spans ctx.Writer.Write("}") ctx.LineBreak() - | Paragraph(spans) -> + | Paragraph(spans, _) -> ctx.LineBreak(); ctx.LineBreak() for span in spans do formatSpan ctx span @@ -143,7 +143,7 @@ let rec formatParagraph (ctx:FormattingContext) paragraph = ctx.Writer.Write(@"\noindent\makebox[\linewidth]{\rule{\linewidth}{0.4pt}}\medskip") ctx.LineBreak() - | CodeBlock(code, _, _) -> + | CodeBlock(code, _, _, _) -> ctx.Writer.Write(@"\begin{lstlisting}") ctx.LineBreak() ctx.Writer.Write(code) @@ -151,7 +151,7 @@ let rec formatParagraph (ctx:FormattingContext) paragraph = ctx.Writer.Write(@"\end{lstlisting}") ctx.LineBreak() - | TableBlock(headers, alignments, rows) -> + | TableBlock(headers, alignments, rows, _) -> let aligns = alignments |> List.map (function | AlignRight -> "|r" | AlignCenter -> "|c" @@ -178,7 +178,7 @@ let rec formatParagraph (ctx:FormattingContext) paragraph = ctx.Writer.Write(@"\end{tabular}") ctx.LineBreak() - | ListBlock(kind, items) -> + | ListBlock(kind, items, _) -> let tag = if kind = Ordered then "enumerate" else "itemize" ctx.Writer.Write(@"\begin{" + tag + "}") ctx.LineBreak() @@ -189,16 +189,16 @@ let rec formatParagraph (ctx:FormattingContext) paragraph = ctx.Writer.Write(@"\end{" + tag + "}") ctx.LineBreak() - | QuotedBlock(body) -> + | QuotedBlock(body, _) -> ctx.Writer.Write(@"\begin{quote}") ctx.LineBreak() formatParagraphs ctx body ctx.Writer.Write(@"\end{quote}") ctx.LineBreak() - | Span spans -> + | Span(spans, _) -> formatSpans ctx spans - | InlineBlock(code) -> + | InlineBlock(code, _) -> ctx.Writer.Write(code) ctx.LineBreak() diff --git a/src/FSharp.Markdown/Main.fs b/src/FSharp.Markdown/Main.fs index 2e174bd7a..155a729ce 100644 --- a/src/FSharp.Markdown/Main.fs +++ b/src/FSharp.Markdown/Main.fs @@ -12,6 +12,7 @@ open System.Collections.Generic open FSharp.Patterns open FSharp.Markdown.Parser open FSharp.Markdown.Html +open FSharp.Formatting.Common module private Utils = /// Replace tabs with four spaces - tab will end at the @@ -48,17 +49,19 @@ type Markdown = use reader = new StringReader(text) let lines = [ let line = ref "" + let mutable lineNo = 1 while (line := reader.ReadLine(); line.Value <> null) do - yield line.Value + yield (line.Value, { StartLine = lineNo; StartColumn = 0; EndLine = lineNo; EndColumn = line.Value.Length }) + lineNo <- lineNo + 1 if text.EndsWith(newline) then - yield "" ] + yield ("", { StartLine = lineNo; StartColumn = 0; EndLine = lineNo; EndColumn = 0 }) ] //|> Utils.replaceTabs 4 let links = Dictionary<_, _>() //let (Lines.TrimBlank lines) = lines - let ctx : ParsingContext = { Newline = newline; Links = links } + let ctx : ParsingContext = { Newline = newline; Links = links; CurrentRange = Some(MarkdownRange.Zero) } let paragraphs = lines - |> FSharp.Collections.List.skipWhile String.IsNullOrWhiteSpace + |> FSharp.Collections.List.skipWhile (fun (s, n) -> String.IsNullOrWhiteSpace s) |> parseParagraphs ctx |> List.ofSeq MarkdownDocument(paragraphs, links) diff --git a/src/FSharp.Markdown/Markdown.fs b/src/FSharp.Markdown/Markdown.fs index a70fc0e00..11e06f149 100644 --- a/src/FSharp.Markdown/Markdown.fs +++ b/src/FSharp.Markdown/Markdown.fs @@ -8,6 +8,7 @@ namespace FSharp.Markdown open System open System.IO open System.Collections.Generic +open FSharp.Formatting.Common // -------------------------------------------------------------------------------------- // Definition of the Markdown format @@ -28,19 +29,19 @@ type MarkdownColumnAlignment = /// Represents inline formatting inside a paragraph. This can be literal (with text), various /// formattings (string, emphasis, etc.), hyperlinks, images, inline maths etc. type MarkdownSpan = - | Literal of string - | InlineCode of string - | Strong of MarkdownSpans - | Emphasis of MarkdownSpans - | AnchorLink of string - | DirectLink of MarkdownSpans * (string * option) - | IndirectLink of MarkdownSpans * string * string - | DirectImage of string * (string * option) - | IndirectImage of string * string * string - | HardLineBreak - | LatexInlineMath of string - | LatexDisplayMath of string - | EmbedSpans of MarkdownEmbedSpans + | Literal of text:string * range:MarkdownRange option + | InlineCode of code:string * range:MarkdownRange option + | Strong of body:MarkdownSpans * range:MarkdownRange option + | Emphasis of body:MarkdownSpans * range:MarkdownRange option + | AnchorLink of link:string * range:MarkdownRange option + | DirectLink of body:MarkdownSpans * linkAndTitle:(string * option) * range:MarkdownRange option + | IndirectLink of body:MarkdownSpans * link:string * key:string * range:MarkdownRange option + | DirectImage of body:string * link:string * title:option * range:MarkdownRange option + | IndirectImage of body:string * link:string * key:string * range:MarkdownRange option + | HardLineBreak of range:MarkdownRange option + | LatexInlineMath of code:string * range:MarkdownRange option + | LatexDisplayMath of code:string * range:MarkdownRange option + | EmbedSpans of customSpans:MarkdownEmbedSpans * range:MarkdownRange option /// A type alias for a list of `MarkdownSpan` values and MarkdownSpans = list @@ -54,17 +55,17 @@ and MarkdownEmbedSpans = /// Paragraphs are headings, inline paragraphs, code blocks, lists, quotations, tables and /// also embedded LaTeX blocks. type MarkdownParagraph = - | Heading of int * MarkdownSpans - | Paragraph of MarkdownSpans - | CodeBlock of string * string * string - | InlineBlock of string - | ListBlock of MarkdownListKind * list - | QuotedBlock of MarkdownParagraphs - | Span of MarkdownSpans - | LatexBlock of list - | HorizontalRule of char - | TableBlock of option * list * list - | EmbedParagraphs of MarkdownEmbedParagraphs + | Heading of size:int * body:MarkdownSpans * range:MarkdownRange option + | Paragraph of body:MarkdownSpans * range:MarkdownRange option + | CodeBlock of code:string * language:string * ignoredLine:string * range:MarkdownRange option + | InlineBlock of code:string * range:MarkdownRange option + | ListBlock of kind:MarkdownListKind * items:list * range:MarkdownRange option + | QuotedBlock of paragraphs:MarkdownParagraphs * range:MarkdownRange option + | Span of body:MarkdownSpans * range:MarkdownRange option + | LatexBlock of body:list * range:MarkdownRange option + | HorizontalRule of character:char * range:MarkdownRange option + | TableBlock of headers:option * alignments:list * rows:list * range:MarkdownRange option + | EmbedParagraphs of customParagraphs:MarkdownEmbedParagraphs * range:MarkdownRange option /// A type alias for a list of paragraphs and MarkdownParagraphs = list @@ -97,21 +98,21 @@ module Matching = | LatexInlineMath _ | LatexDisplayMath _ | EmbedSpans _ - | HardLineBreak -> + | HardLineBreak _ -> SpanLeaf(SL span) - | Strong spans - | Emphasis spans - | DirectLink(spans, _) - | IndirectLink(spans, _, _) -> + | Strong(spans, _) + | Emphasis(spans , _) + | DirectLink(spans, _, _) + | IndirectLink(spans, _, _, _) -> SpanNode(SN span, spans) let SpanLeaf (SL(span)) = span let SpanNode (SN(span), spans) = match span with - | Strong _ -> Strong spans - | Emphasis _ -> Emphasis spans - | DirectLink(_, a) -> DirectLink(spans, a) - | IndirectLink(_, a, b) -> IndirectLink(spans, a, b) + | Strong(_, r) -> Strong(spans , r) + | Emphasis(_, r) -> Emphasis(spans, r) + | DirectLink(_, a, r) -> DirectLink(spans, a, r) + | IndirectLink(_, a, b, r) -> IndirectLink(spans, a, b, r) | _ -> invalidArg "" "Incorrect SpanNodeInfo" type ParagraphSpansInfo = private PS of MarkdownParagraph @@ -120,9 +121,9 @@ module Matching = let (|ParagraphLeaf|ParagraphNested|ParagraphSpans|) par = match par with - | Heading(_, spans) - | Paragraph(spans) - | Span(spans) -> + | Heading(_, spans, _) + | Paragraph(spans, _) + | Span(spans, _) -> ParagraphSpans(PS par, spans) | CodeBlock _ | InlineBlock _ @@ -130,20 +131,20 @@ module Matching = | LatexBlock _ | HorizontalRule _ -> ParagraphLeaf(PL par) - | ListBlock(_, pars) -> + | ListBlock(_, pars, _) -> ParagraphNested(PN par, pars) - | QuotedBlock(nested) -> + | QuotedBlock(nested, _) -> ParagraphNested(PN par, [nested]) - | TableBlock(headers, alignments, rows) -> + | TableBlock(headers, alignments, rows, _) -> match headers with | None -> ParagraphNested(PN par, rows |> List.concat) | Some columns -> ParagraphNested(PN par, columns::rows |> List.concat) let ParagraphSpans (PS(par), spans) = match par with - | Heading(a, _) -> Heading(a, spans) - | Paragraph(_) -> Paragraph(spans) - | Span(_) -> Span(spans) + | Heading(a, _, r) -> Heading(a, spans, r) + | Paragraph(_, r) -> Paragraph(spans, r) + | Span(_, r) -> Span(spans, r) | _ -> invalidArg "" "Incorrect ParagraphSpansInfo." let ParagraphLeaf (PL(par)) = par @@ -157,10 +158,10 @@ module Matching = loop n n [] [] list match par with - | ListBlock(a, _) -> ListBlock(a, pars) - | QuotedBlock(_) -> QuotedBlock(List.concat pars) - | TableBlock(headers, alignments, _) -> + | ListBlock(a, _, r) -> ListBlock(a, pars, r) + | QuotedBlock(_, r) -> QuotedBlock(List.concat pars, r) + | TableBlock(headers, alignments, _, r) -> let rows = splitEach (alignments.Length) pars - if List.isEmpty rows || headers.IsNone then TableBlock(None, alignments, rows) - else TableBlock(Some(List.head rows), alignments, List.tail rows) + if List.isEmpty rows || headers.IsNone then TableBlock(None, alignments, rows, r) + else TableBlock(Some(List.head rows), alignments, List.tail rows, r) | _ -> invalidArg "" "Incorrect ParagraphNestedInfo." diff --git a/src/FSharp.Markdown/MarkdownParser.fs b/src/FSharp.Markdown/MarkdownParser.fs index fba6eddb1..97a808d25 100644 --- a/src/FSharp.Markdown/MarkdownParser.fs +++ b/src/FSharp.Markdown/MarkdownParser.fs @@ -11,6 +11,7 @@ open System.Text.RegularExpressions open FSharp.Patterns open FSharp.Collections +open FSharp.Formatting.Common // -------------------------------------------------------------------------------------- // Parsing of Markdown - first part handles inline formatting @@ -18,7 +19,7 @@ open FSharp.Collections /// Splits a link formatted as `http://link "title"` into a link part /// and an optional title part (may be wrapped using quote or double-quotes) -let getLinkAndTitle (String.TrimBoth input) = +let getLinkAndTitle (StringPosition.TrimBoth(input, n)) = let url, title = if input.Length = 0 then "", None else let c = input.[input.Length - 1] @@ -118,7 +119,7 @@ let (|AutoLink|_|) input = let linkFor (scheme:string) = let prefix = scheme.ToCharArray() |> Array.toList match input with - | List.DelimitedWith prefix [' '] (List.AsString link, rest) -> + | List.DelimitedWith prefix [' '] (List.AsString link, rest, s, e) -> Some(scheme + link, ' '::rest) | List.StartsWith prefix (List.AsString link) -> Some(link, []) @@ -135,7 +136,7 @@ let (|Emphasised|_|) = function match input with | DelimitedMarkdown ['_'; '_'; '_'] (body, rest) | DelimitedMarkdown ['*'; '*'; '*'] (body, rest) -> - Some(body, Emphasis >> List.singleton >> Strong, rest) + Some(body, Emphasis >> List.singleton >> (fun s -> Strong(s, None)), rest) | DelimitedMarkdown ['_'; '_'] (body, rest) | DelimitedMarkdown ['*'; '*'] (body, rest) -> Some(body, Strong, rest) @@ -145,107 +146,137 @@ let (|Emphasised|_|) = function | _ -> None | _ -> None +/// Defines a context for the main `parseParagraphs` function +type ParsingContext = + { Links : Dictionary> + Newline : string + CurrentRange : MarkdownRange option } + /// Parses a body of a paragraph and recognizes all inline tags. -let rec parseChars acc input = seq { +let rec parseChars acc input (ctx:ParsingContext) = seq { - // Zero or one literals, depending whether there is some accumulated input + // Zero or one literals, depending whether there is some accumulated input and update the ctx let accLiterals = Lazy.Create(fun () -> - if List.isEmpty acc then [] - else [Literal(String(List.rev acc |> Array.ofList))] ) + if List.isEmpty acc then ([], ctx) + else + let range = match ctx.CurrentRange with + | Some(n) -> Some({ n with EndColumn = n.StartColumn + acc.Length }) + | None -> None + let ctx = { ctx with CurrentRange = match ctx.CurrentRange with + | Some(n) -> Some({ n with StartColumn = n.StartColumn + acc.Length }) + | None -> None } + let text = String(List.rev acc |> Array.ofList) + ([Literal(text, range)], ctx) ) match input with // Recognizes explicit line-break at the end of line | ' '::' '::'\r'::'\n'::rest | ' '::' '::('\n' | '\r')::rest -> - yield! accLiterals.Value - yield HardLineBreak - yield! parseChars [] rest + let (value, ctx) = accLiterals.Value + yield! value + yield HardLineBreak(ctx.CurrentRange) + yield! parseChars [] rest ctx // Encode & as an HTML entity | '&'::'a'::'m'::'p'::';'::rest | '&'::rest -> - yield! parseChars (';'::'p'::'m'::'a'::'&'::acc) rest + yield! parseChars (';'::'p'::'m'::'a'::'&'::acc) rest ctx // Ignore escaped characters that might mean something else | EscapedChar(c, rest) -> - yield! parseChars (c::acc) rest + yield! parseChars (c::acc) rest ctx // Inline code delimited either using double `` or single ` // (if there are spaces around, then body can contain more backticks) - | List.DelimitedWith ['`'; ' '] [' '; '`'] (body, rest) - | List.DelimitedNTimes '`' (body, rest) -> - yield! accLiterals.Value - yield InlineCode(String(Array.ofList body).Trim()) - yield! parseChars [] rest + | List.DelimitedWith ['`'; ' '] [' '; '`'] (body, rest, s, e) + | List.DelimitedNTimes '`' (body, rest, s, e) -> + let (value, ctx) = accLiterals.Value + yield! value + yield InlineCode(String(Array.ofList body).Trim(), match ctx.CurrentRange with Some(n) -> Some({ n with StartColumn = n.StartColumn + s; EndColumn = n.EndColumn - e }) | None -> None) + yield! parseChars [] rest ctx // Display Latex inline math mode | DelimitedLatexDisplayMath ['$';'$'] (body, rest) -> - yield! accLiterals.Value - yield LatexDisplayMath(String(Array.ofList body).Trim()) - yield! parseChars [] rest + let (value, ctx) = accLiterals.Value + yield! value + yield LatexDisplayMath(String(Array.ofList body).Trim(), ctx.CurrentRange) + yield! parseChars [] rest ctx // Inline Latex inline math mode | DelimitedLatexInlineMath ['$'] (body, rest) -> - yield! accLiterals.Value - yield LatexInlineMath(String(Array.ofList body).Trim()) - yield! parseChars [] rest + let (value, ctx) = accLiterals.Value + let ctx = { ctx with CurrentRange = match ctx.CurrentRange with | Some(n) -> Some({ n with StartColumn = n.StartColumn + 1 }) | None -> None } + yield! value + let code = String(Array.ofList body).Trim() + yield LatexInlineMath(code, match ctx.CurrentRange with | Some(n) -> Some({ n with EndColumn = n.StartColumn + code.Length }) | None -> None) + yield! parseChars [] rest ctx // Inline link wrapped as - | List.DelimitedWith ['<'] ['>'] (List.AsString link, rest) + | List.DelimitedWith ['<'] ['>'] (List.AsString link, rest, s, e) when Seq.forall (Char.IsWhiteSpace >> not) link && (link.Contains("@") || link.Contains("://")) -> - yield! accLiterals.Value - yield DirectLink([Literal link], (link, None)) - yield! parseChars [] rest + let (value, ctx) = accLiterals.Value + yield! value + yield DirectLink([Literal(link, ctx.CurrentRange)], (link, None), ctx.CurrentRange) + yield! parseChars [] rest ctx // Not an inline link - leave as an inline HTML tag - | List.DelimitedWith ['<'] ['>'] (tag, rest) -> - yield! parseChars ('>'::(List.rev tag) @ '<' :: acc) rest + | List.DelimitedWith ['<'] ['>'] (tag, rest, s, e) -> + yield! parseChars ('>'::(List.rev tag) @ '<' :: acc) rest ctx // Recognize direct link [foo](http://bar) or indirect link [foo][bar] or auto link http://bar | DirectLink (body, link, rest) -> - yield! accLiterals.Value - let info = getLinkAndTitle (String(Array.ofList link)) - yield DirectLink(parseChars [] body |> List.ofSeq, info) - yield! parseChars [] rest + let (value, ctx) = accLiterals.Value + yield! value + let info = getLinkAndTitle (String(Array.ofList link), MarkdownRange.Zero) + yield DirectLink(parseChars [] body ctx |> List.ofSeq, info, ctx.CurrentRange) + yield! parseChars [] rest ctx | IndirectLink(body, link, original, rest) -> - yield! accLiterals.Value + let (value, ctx) = accLiterals.Value + yield! value let key = if String.IsNullOrEmpty(link) then String(body |> Array.ofSeq) else link - yield IndirectLink(parseChars [] body |> List.ofSeq, original, key) - yield! parseChars [] rest + yield IndirectLink(parseChars [] body ctx |> List.ofSeq, original, key, ctx.CurrentRange) + yield! parseChars [] rest ctx | AutoLink (link, rest) -> - yield! accLiterals.Value - yield DirectLink([Literal link], (link, None)) - yield! parseChars [] rest + let (value, ctx) = accLiterals.Value + yield! value + yield DirectLink([Literal(link, ctx.CurrentRange)], (link, None), ctx.CurrentRange) + yield! parseChars [] rest ctx // Recognize image - this is a link prefixed with the '!' symbol | '!'::DirectLink (body, link, rest) -> - yield! accLiterals.Value - yield DirectImage(String(Array.ofList body), getLinkAndTitle (String(Array.ofList link))) - yield! parseChars [] rest + let (value, ctx) = accLiterals.Value + yield! value + let link, title = getLinkAndTitle (String(Array.ofList link), MarkdownRange.Zero) + yield DirectImage(String(Array.ofList body), link, title, ctx.CurrentRange) + yield! parseChars [] rest ctx | '!'::IndirectLink(body, link, original, rest) -> - yield! accLiterals.Value + let (value, ctx) = accLiterals.Value + yield! value let key = if String.IsNullOrEmpty(link) then String(body |> Array.ofSeq) else link - yield IndirectImage(String(Array.ofList body), original, key) - yield! parseChars [] rest + yield IndirectImage(String(Array.ofList body), original, key, ctx.CurrentRange) + yield! parseChars [] rest ctx // Handle emphasised text | Emphasised (body, f, rest) -> - yield! accLiterals.Value - let body = parseChars [] body |> List.ofSeq - yield f(body) - yield! parseChars [] rest + let (value, ctx) = accLiterals.Value + yield! value + let body = parseChars [] body ctx |> List.ofSeq + yield f(body, ctx.CurrentRange) + yield! parseChars [] rest ctx // Encode '<' char if it is not link or inline HTML | '<'::rest -> - yield! parseChars (';'::'t'::'l'::'&'::acc) rest + yield! parseChars (';'::'t'::'l'::'&'::acc) rest ctx | '>'::rest -> - yield! parseChars (';'::'t'::'g'::'&'::acc) rest + yield! parseChars (';'::'t'::'g'::'&'::acc) rest ctx | x::xs -> - yield! parseChars (x::acc) xs + yield! parseChars (x::acc) xs ctx | [] -> - yield! accLiterals.Value } + let (value, ctx) = accLiterals.Value + yield! value } /// Parse body of a paragraph into a list of Markdown inline spans -let parseSpans (String.TrimBoth s) = - parseChars [] (s.ToCharArray() |> List.ofArray) |> List.ofSeq +let parseSpans (StringPosition.TrimBoth(s, n)) ctx = + let ctx = { ctx with CurrentRange = Some(n) } + parseChars [] (s.ToCharArray() |> List.ofArray) ctx |> List.ofSeq let rec trimSpaces numSpaces (s:string) = if numSpaces <= 0 then s @@ -259,11 +290,11 @@ let rec trimSpaces numSpaces (s:string) = /// Recognizes heading, either prefixed with #s or followed by === or --- line let (|Heading|_|) = function - | (String.TrimBoth header) :: (String.TrimEnd (String.EqualsRepeated "=")) :: rest -> + | (StringPosition.TrimBoth header) :: (StringPosition.TrimEnd (StringPosition.EqualsRepeated("=", MarkdownRange.Zero))) :: rest -> Some(1, header, rest) - | (String.TrimBoth header) :: (String.TrimEnd (String.EqualsRepeated "-")) :: rest -> + | (StringPosition.TrimBoth header) :: (StringPosition.TrimEnd (StringPosition.EqualsRepeated("-", MarkdownRange.Zero))) :: rest -> Some(2, header, rest) - | String.StartsWithRepeated "#" (n, header) :: rest -> + | StringPosition.StartsWithRepeated "#" (n, StringPosition.TrimBoth(header, ln)) :: rest -> let header = // Drop "##" at the end, but only when it is preceded by some whitespace // (For example "## Hello F#" should be "Hello F#") @@ -272,12 +303,12 @@ let (|Heading|_|) = function if noHash.Length > 0 && Char.IsWhiteSpace(noHash.Chars(noHash.Length - 1)) then noHash else header else header - Some(n, header.Trim(), rest) + Some(n, (header, ln), rest) | rest -> None /// Recognizes a horizontal rule written using *, _ or - -let (|HorizontalRule|_|) (line:string) = +let (|HorizontalRule|_|) (line:string, n:MarkdownRange) = let rec loop ((h, a, u) as arg) i = if (h >= 3 || a >= 3 || u >= 3) && i = line.Length then Some(line.[0]) elif i = line.Length then None @@ -292,7 +323,7 @@ let (|HorizontalRule|_|) (line:string) = let (|NestedCodeBlock|_|) = function | Lines.TakeCodeBlock (numspaces, Lines.TrimBlank lines, rest) when lines <> [] -> let code = - [ for l in lines -> + [ for (l, n) in lines -> if String.IsNullOrEmpty l then "" else trimSpaces 4 l ] Some(code @ [""], rest, "", "") @@ -300,9 +331,9 @@ let (|NestedCodeBlock|_|) = function /// Recognizes a fenced code block - starting and ending with at least ``` or ~~~ let (|FencedCodeBlock|_|) = function - | String.StartsWithNTimesTrimIgnoreStartWhitespace "~" (Let "~" (start,num), indent, header) :: lines + | StringPosition.StartsWithNTimesTrimIgnoreStartWhitespace "~" (Let "~" (start,num), indent, header) :: lines // when num > 2 - | String.StartsWithNTimesTrimIgnoreStartWhitespace "`" (Let "`" (start,num), indent, header) :: lines + | StringPosition.StartsWithNTimesTrimIgnoreStartWhitespace "`" (Let "`" (start,num), indent, header) :: lines when num > 2 -> let mutable endStr = String.replicate num start if header.Contains (start) then None // info string cannot contain backspaces @@ -311,7 +342,7 @@ let (|FencedCodeBlock|_|) = function match [line] with // end cannot contain info string afterwards (see http://spec.commonmark.org/0.23/#example-104) // end must be indended with less then 4 spaces: http://spec.commonmark.org/0.23/#example-95 - | String.StartsWithNTimesTrimIgnoreStartWhitespace start (n, i, h) :: _ when n >= num && i < 4 && String.IsNullOrWhiteSpace h -> + | StringPosition.StartsWithNTimesTrimIgnoreStartWhitespace start (n, i, h) :: _ when n >= num && i < 4 && String.IsNullOrWhiteSpace h -> endStr <- String.replicate n start true | _ -> false) @@ -320,7 +351,7 @@ let (|FencedCodeBlock|_|) = function elif l.Length > indent && String.IsNullOrWhiteSpace (l.Substring(0, indent)) then l.Substring(indent, l.Length - indent) else l.TrimStart() let code = - [ for l in code -> handleIndent l ] + [ for (l, n) in code -> handleIndent l ] // langString is the part after ``` and ignoredString is the rest until the line ends. let langString, ignoredString = @@ -334,12 +365,12 @@ let (|FencedCodeBlock|_|) = function // Handle the ending line let code, rest = match rest with - | hd :: tl -> + | (hd, n) :: tl -> let idx = hd.IndexOf(endStr) if idx > -1 && idx + endStr.Length <= hd.Length then let pre = hd.Substring(0, idx) let after = hd.Substring(idx + endStr.Length) - code @ [""], (if String.IsNullOrWhiteSpace after then tl else after :: tl) + code @ [""], (if String.IsNullOrWhiteSpace after then tl else (after, n) :: tl) else code @ [""], tl | _ -> @@ -349,7 +380,7 @@ let (|FencedCodeBlock|_|) = function /// Matches when the input starts with a number. Returns the /// rest of the input, following the last number. -let (|SkipSomeNumbers|_|) (input:string) = +let (|SkipSomeNumbers|_|) (input:string, n:MarkdownRange) = match List.ofSeq input with | x::xs when Char.IsDigit x -> let _, rest = List.partitionUntil (Char.IsDigit >> not) xs @@ -359,40 +390,41 @@ let (|SkipSomeNumbers|_|) (input:string) = /// Recognizes a staring of a list (either 1. or +, *, -). /// Returns the rest of the line, together with the indent. let (|ListStart|_|) = function - | String.TrimStartAndCount + | StringPosition.TrimStartAndCount (startIndent, spaces, // NOTE: a tab character after +, * or - isn't supported by the reference implementation // (it will be parsed as paragraph for 0.22) - (String.StartsWithAny ["+ "; "* "; "- " (*; "+\t"; "*\t"; "-\t"*)] as item)) -> - let li = item.Substring(2) - let (String.TrimStartAndCount (startIndent2, spaces2, _)) = li + (StringPosition.StartsWithAny ["+ "; "* "; "- " (*; "+\t"; "*\t"; "-\t"*)] as item)) -> + let range = snd item + let li = ((fst item).Substring(2), { range with StartColumn = range.StartColumn + 2 }) + let (StringPosition.TrimStartAndCount (startIndent2, spaces2, _)) = li let endIndent = startIndent + 2 + // Handle case of code block if startIndent2 >= 5 then 1 else startIndent2 Some(Unordered, startIndent, endIndent, li) - | String.TrimStartAndCount // Remove leading spaces + | StringPosition.TrimStartAndCount // Remove leading spaces (startIndent, spaces, (SkipSomeNumbers // read a number (skipNumCount, '.' :: ' ' :: List.AsString item))) -> - let (String.TrimStartAndCount (startIndent2, spaces2, _)) = item + let (StringPosition.TrimStartAndCount (startIndent2, spaces2, _)) = (item, MarkdownRange.Zero) let endIndent = startIndent + 2 + skipNumCount + // Handle case of code block if startIndent2 >= 5 then 1 else startIndent2 - Some(Ordered, startIndent, endIndent, item) + Some(Ordered, startIndent, endIndent, (item, MarkdownRange.Zero)) | _ -> None /// Splits input into lines until whitespace or starting of a list and the rest. let (|LinesUntilListOrWhite|) = List.partitionUntil (function - | ListStart _ | String.WhiteSpace -> true | _ -> false) + | ListStart _ | StringPosition.WhiteSpace -> true | _ -> false) /// Splits input into lines until not-indented line or starting of a list and the rest. let (|LinesUntilListOrUnindented|) = List.partitionUntilLookahead (function - | (ListStart _ | String.Unindented)::_ - | String.WhiteSpace::String.WhiteSpace::_ -> true | _ -> false) + | (ListStart _ | StringPosition.Unindented)::_ + | StringPosition.WhiteSpace::StringPosition.WhiteSpace::_ -> true | _ -> false) /// Recognizes a list item until the next list item (possibly nested) or end of a list. /// The parameter specifies whether the previous line was simple (single-line not @@ -408,26 +440,26 @@ let (|ListItem|_|) prevSimple = function (LinesUntilListOrUnindented (more, rest) as next)) -> let simple = match item with - | String.TrimStartAndCount (_, spaces, _) when spaces >= 4-> + | StringPosition.TrimStartAndCount (_, spaces, _) when spaces >= 4-> // Code Block false | _ -> match next, rest with - | String.WhiteSpace::_, (ListStart _)::_ -> false + | StringPosition.WhiteSpace::_, (ListStart _)::_ -> false | (ListStart _)::_, _ -> true | [], _ -> true - | [ String.WhiteSpace ], _ -> true - | String.WhiteSpace::String.WhiteSpace::_, _ -> true - | _, String.Unindented::_ -> prevSimple + | [ StringPosition.WhiteSpace ], _ -> true + | StringPosition.WhiteSpace::StringPosition.WhiteSpace::_, _ -> true + | _, StringPosition.Unindented::_ -> prevSimple | _, _ -> false let lines = [ yield item - for line in continued do - yield line.Trim() - for line in more do + for (line, n) in continued do + yield (line.Trim(), n) + for (line, n) in more do let trimmed = trimSpaces endIndent line - yield trimmed ] + yield (trimmed, { n with StartColumn = n.StartColumn + line.Length - trimmed.Length }) ] //let trimmed = line.TrimStart() //if trimmed.Length >= line.Length - endIndent then yield trimmed //else yield line.Substring(endIndent) ] @@ -460,8 +492,8 @@ let rec pipeTableFindSplits (delim : char array) (line : char list) = match line with | DelimitedLatexDisplayMath [ '$'; '$' ] (body, rest) -> ptfs delim rest | DelimitedLatexInlineMath [ '$' ] (body, rest) -> ptfs delim rest - | List.DelimitedWith [ '`'; ' ' ] [ ' '; '`' ] (body, rest) -> ptfs delim rest - | List.DelimitedNTimes '`' (body, rest) -> ptfs delim rest + | List.DelimitedWith [ '`'; ' ' ] [ ' '; '`' ] (body, rest, s, e) -> ptfs delim rest + | List.DelimitedNTimes '`' (body, rest, s, e) -> ptfs delim rest | x :: rest when Array.exists ((=) x) delim -> Some rest | '\\' :: _ :: rest | _ :: rest -> ptfs delim rest | [] -> None @@ -480,20 +512,20 @@ let rec pipeTableFindSplits (delim : char array) (line : char list) = /// Recognizes alignment specified in the passed separator line. let (|TableCellSeparator|_|) = function - | String.StartsAndEndsWith (":", ":") (String.EqualsRepeated "-") -> Some(AlignCenter) - | String.StartsWith ":" (String.EqualsRepeated "-") -> Some(AlignLeft) - | String.StartsAndEndsWith ("", ":") (String.EqualsRepeated "-") -> Some(AlignRight) - | String.EqualsRepeated "-" -> Some(AlignDefault) + | StringPosition.StartsAndEndsWith (":", ":") (StringPosition.EqualsRepeated("-", MarkdownRange.Zero)) -> Some(AlignCenter) + | StringPosition.StartsWith ":" (StringPosition.EqualsRepeated("-", MarkdownRange.Zero)) -> Some(AlignLeft) + | StringPosition.StartsAndEndsWith ("", ":") (StringPosition.EqualsRepeated("-", MarkdownRange.Zero)) -> Some(AlignRight) + | StringPosition.EqualsRepeated("-", MarkdownRange.Zero) -> Some(AlignDefault) | _ -> None /// Recognizes row of pipe table. /// The function takes number of expected columns and array of delimiters. /// Returns list of strings between delimiters. -let (|PipeTableRow|_|) (size : option) delimiters (line : string) = +let (|PipeTableRow|_|) (size : option) delimiters (line : string, n:MarkdownRange) = let parts = pipeTableFindSplits delimiters (line.ToCharArray() |> Array.toList) |> List.toArray - |> Array.map (fun s -> s.Trim()) + |> Array.map (fun s -> (s.Trim(), n)) let n = parts.Length @@ -502,11 +534,11 @@ let (|PipeTableRow|_|) (size : option) delimiters (line : string) = else size.Value let x = - if String.IsNullOrEmpty parts.[0] && n > m then 1 + if String.IsNullOrEmpty (fst parts.[0]) && n > m then 1 else 0 let y = - if String.IsNullOrEmpty parts.[n - 1] && n - x > m then n - 2 + if String.IsNullOrEmpty (fst parts.[n - 1]) && n - x > m then n - 2 else n - 1 if n = 1 || (size.IsSome && y - x + 1 <> m) then None @@ -546,20 +578,20 @@ let (|PipeTableBlock|_|) input = /// The function takes positions of grid columns (if known) and expected grid separator. /// Passed function is used to check whether all parts within grid are valid. /// Retuns tuple (position of grid columns, text between grid columns). -let (|EmacsTableLine|_|) (grid:option) (c:char) (check:string -> bool) (line:string) = +let (|EmacsTableLine|_|) (grid:option) (c:char) (check:string * MarkdownRange -> bool) (line:string, n:MarkdownRange) = let p = if grid.IsSome then grid.Value else Array.FindAll([|0..line.Length - 1|], fun i -> line.[i] = c) let n = p.Length - 1 if n < 2 || line.Length <= p.[n] || Array.exists (fun i -> line.[i] <> c) p then None else - let parts = [1..n] |> List.map (fun i -> line.Substring(p.[i - 1] + 1, p.[i] - p.[i - 1] - 1)) + let parts = [1..n] |> List.map (fun i -> line.Substring(p.[i - 1] + 1, p.[i] - p.[i - 1] - 1), { StartLine = n; StartColumn = 0; EndLine = n; EndColumn = p.[i] - p.[i - 1] - 1 }) if List.forall check parts then Some(p, parts) else None /// Recognizes emacs table -let (|EmacsTableBlock|_|) input = - let isCellSep = String.(|EqualsRepeated|_|) "-" >> Option.isSome +let (|EmacsTableBlock|_|) (input) = + let isCellSep = StringPosition.(|EqualsRepeated|_|)("-", MarkdownRange.Zero) >> Option.isSome let isAlignedCellSep = ( |TableCellSeparator|_| ) >> Option.isSome - let isHeadCellSep = String.(|EqualsRepeated|_|) "=" >> Option.isSome - let isText (s:string) = true + let isHeadCellSep = StringPosition.(|EqualsRepeated|_|)("=", MarkdownRange.Zero) >> Option.isSome + let isText (s:string, n:MarkdownRange) = true match input with | (EmacsTableLine None '+' isAlignedCellSep (grid, parts)) :: rest -> let alignments = List.choose ( |TableCellSeparator|_| ) parts @@ -568,10 +600,10 @@ let (|EmacsTableBlock|_|) input = // prevRow - content of the processed rows // cur - list of paragraphs in the current row (list of empty lists after each separator line) // flag indicates whether current row is empty (similar to List.forall (List.isEmpty) cur) - let emptyCur = List.replicate (grid.Length - 1) [] - let rec loop flag headers prevRows cur = function + let emptyCur = List.replicate<(string * MarkdownRange) list> (grid.Length - 1) [] + let rec loop flag headers (prevRows:(string * MarkdownRange) list list list) (cur:(string * MarkdownRange) list list) = function | (EmacsTableLine (Some grid) '|' isText (_, parts)) :: others -> - loop false headers prevRows (List.zip parts cur |> List.map (fun (h, t) -> h.TrimEnd() :: t)) others + loop false headers prevRows (List.zip parts cur |> List.map (fun ((h, n), t) -> (h.TrimEnd(), n) :: t)) others | (EmacsTableLine (Some grid) '+' isCellSep _) :: others -> loop true headers (List.map (List.rev) cur :: prevRows) emptyCur others | (EmacsTableLine (Some grid) '+' isHeadCellSep _) :: others when Option.isNone headers -> @@ -582,14 +614,16 @@ let (|EmacsTableBlock|_|) input = | _ -> None /// Recognizes a start of a blockquote -let (|BlockquoteStart|_|) (line:string) = +let (|BlockquoteStart|_|) (line:string, n:MarkdownRange) = let regex = "^ {0,3}" // Up to three leading spaces + ">" // Blockquote character + "\s?" // Maybe one whitespace character + "(.*)" // Capture everything else let match' = Regex.Match(line, regex) - if match'.Success then Some (match'.Groups.Item(1)).Value + if match'.Success then + let group = match'.Groups.Item(1) + Some (group.Value, { n with StartColumn = n.StartColumn + group.Index; EndColumn = n.StartColumn + group.Index + group.Length }) else None /// Takes lines that belong to a continuing paragraph until @@ -599,7 +633,7 @@ let (|TakeParagraphLines|_|) input = | Heading _ -> false | FencedCodeBlock _ -> false | BlockquoteStart _::_ -> false - | String.WhiteSpace::_ -> false + | StringPosition.WhiteSpace::_ -> false | _ -> true) input with | matching, rest when matching <> [] -> Some(matching, rest) | _ -> None @@ -607,7 +641,7 @@ let (|TakeParagraphLines|_|) input = /// Recognize nested HTML block /// TODO: This is too simple - takes paragraph that starts with < let (|HtmlBlock|_|) = function - | first::_ & TakeParagraphLines(html, rest) when first.StartsWith("<") -> + | (first, n)::_ & TakeParagraphLines(html, rest) when first.StartsWith("<") -> Some(html, rest) | _ -> None @@ -617,7 +651,7 @@ let (|LinesUntilBlockquoteEnds|) input = match next with | BlockquoteStart _ :: _ | Heading _ - | String.WhiteSpace :: _ -> true + | StringPosition.WhiteSpace :: _ -> true | _ -> false) input @@ -625,7 +659,7 @@ let (|LinesUntilBlockquoteEnds|) input = /// starting with '>' until there is something else let rec (|Blockquote|_|) = function | EmptyBlockquote(Lines.TrimBlankStart rest) -> - Some ([""], rest) + Some ([("", MarkdownRange.Zero)], rest) | BlockquoteStart(line)::LinesUntilBlockquoteEnds(continued, Lines.TrimBlankStart rest) -> let moreLines, rest = match rest with @@ -637,33 +671,34 @@ let rec (|Blockquote|_|) = function /// Recognizes a special case: an empty blockquote line should terminate /// the blockquote if the next line is not a blockquote and (|EmptyBlockquote|_|) = function - | BlockquoteStart(String.WhiteSpace) :: Blockquote(_) -> None - | BlockquoteStart(String.WhiteSpace) :: rest -> Some rest + | BlockquoteStart(StringPosition.WhiteSpace) :: Blockquote(_) -> None + | BlockquoteStart(StringPosition.WhiteSpace) :: rest -> Some rest | _ -> None /// Recognizes Latex block - start with "$$$" -let (|LatexBlock|_|) (lines:string list) = lines |> function - | first::rest when (first.TrimEnd()) = "$$$" -> rest |> function +let (|LatexBlock|_|) (lines:(string * MarkdownRange) list) = lines |> function + | (first, n)::rest when (first.TrimEnd()) = "$$$" -> rest |> function | TakeParagraphLines(body, rest) -> Some(body, rest) | _ -> None | _ -> None /// Recognize a definition of a link as in `[key]: http://url ...` let (|LinkDefinition|_|) = function - | ( String.StartsWithWrapped ("[", "]:") (wrapped, String.TrimBoth link) - | String.StartsWithWrapped (" [", "]:") (wrapped, String.TrimBoth link) - | String.StartsWithWrapped (" [", "]:") (wrapped, String.TrimBoth link) - | String.StartsWithWrapped (" [", "]:") (wrapped, String.TrimBoth link) ) :: rest -> + | ( StringPosition.StartsWithWrapped ("[", "]:") (wrapped, StringPosition.TrimBoth link) + | StringPosition.StartsWithWrapped (" [", "]:") (wrapped, StringPosition.TrimBoth link) + | StringPosition.StartsWithWrapped (" [", "]:") (wrapped, StringPosition.TrimBoth link) + | StringPosition.StartsWithWrapped (" [", "]:") (wrapped, StringPosition.TrimBoth link) ) :: rest -> Some((wrapped, link), rest) | _ -> None -/// Defines a context for the main `parseParagraphs` function -type ParsingContext = - { Links : Dictionary> - Newline : string } +let updateCurrentRange lines = + match lines with + | [] -> None + | (_, l)::_ -> Some(l) /// Parse a list of lines into a sequence of markdown paragraphs -let rec parseParagraphs (ctx:ParsingContext) lines = seq { +let rec parseParagraphs (ctx:ParsingContext) (lines:(string * MarkdownRange) list) = seq { + let ctx = { ctx with CurrentRange = updateCurrentRange lines } match lines with // Recognize various kinds of standard paragraphs | LinkDefinition ((key, link), Lines.TrimBlankStart lines) -> @@ -671,10 +706,10 @@ let rec parseParagraphs (ctx:ParsingContext) lines = seq { yield! parseParagraphs ctx lines | NestedCodeBlock(code, Lines.TrimBlankStart lines, langString, ignoredLine) | FencedCodeBlock(code, Lines.TrimBlankStart lines, langString, ignoredLine) -> - yield CodeBlock(code |> String.concat ctx.Newline, langString, ignoredLine) + yield CodeBlock(code |> String.concat ctx.Newline, langString, ignoredLine, ctx.CurrentRange) yield! parseParagraphs ctx lines | Blockquote(body, Lines.TrimBlankStart rest) -> - yield QuotedBlock(parseParagraphs ctx (body @ [""]) |> List.ofSeq) + yield QuotedBlock(parseParagraphs ctx (body @ [("", MarkdownRange.Zero)]) |> List.ofSeq, ctx.CurrentRange) yield! parseParagraphs ctx rest | EmacsTableBlock((headers, alignments, rows), Lines.TrimBlankStart rest) | PipeTableBlock((headers, alignments, rows), Lines.TrimBlankStart rest) -> @@ -682,13 +717,13 @@ let rec parseParagraphs (ctx:ParsingContext) lines = seq { if headers.IsNone then None else Some(headers.Value |> List.map (fun i -> parseParagraphs ctx i |> List.ofSeq)) yield TableBlock(headParagraphs, alignments, - rows |> List.map (List.map (fun i -> parseParagraphs ctx i |> List.ofSeq))) + rows |> List.map (List.map (fun i -> parseParagraphs ctx i |> List.ofSeq)), ctx.CurrentRange) yield! parseParagraphs ctx rest | HorizontalRule(c) :: (Lines.TrimBlankStart lines) -> - yield HorizontalRule(c) + yield HorizontalRule(c, ctx.CurrentRange) yield! parseParagraphs ctx lines | LatexBlock(body, Lines.TrimBlankStart rest) -> - yield LatexBlock(body) + yield LatexBlock(body |> List.map fst, ctx.CurrentRange) yield! parseParagraphs ctx rest @@ -708,18 +743,18 @@ let rec parseParagraphs (ctx:ParsingContext) lines = seq { | [] -> [] // Turn tree into nested list definitions - let rec formatTree (nodes:Tree list) = + let rec formatTree (nodes:Tree list) = let kind = match nodes with Node((_, kind, _), _)::_ -> kind | _ -> Unordered let items = [ for (Node((simple, _, body), nested)) in nodes -> [ if not simple then yield! parseParagraphs ctx body - else yield Span(parseSpans(String.concat ctx.Newline body)) + else yield Span(parseSpans(String.concat ctx.Newline (body |> List.map fst), body |> List.map snd |> MarkdownRange.MergeRanges) ctx, ctx.CurrentRange) if nested <> [] then yield formatTree nested ] ] - ListBlock(kind, items) + ListBlock(kind, items, ctx.CurrentRange) // Make sure all items of the list have are either simple or not. - let rec unifySimpleProperty (nodes:Tree list) = + let rec unifySimpleProperty (nodes:Tree list) = let containsNonSimple = tree |> Seq.exists (function | Node ((false, _, _), _) -> true @@ -734,16 +769,16 @@ let rec parseParagraphs (ctx:ParsingContext) lines = seq { // Recognize remaining types of paragraphs | Heading(n, body, Lines.TrimBlankStart lines) -> - yield Heading(n, parseSpans body) + yield Heading(n, parseSpans body ctx, ctx.CurrentRange) yield! parseParagraphs ctx lines | HtmlBlock(code, Lines.TrimBlankStart lines) when - ( let all = String.concat ctx.Newline code + ( let all = String.concat ctx.Newline (code |> List.map fst) not (all.StartsWith(" - let all = String.concat ctx.Newline code - yield InlineBlock(all) + let all = String.concat ctx.Newline (code |> List.map fst) + yield InlineBlock(all, ctx.CurrentRange) yield! parseParagraphs ctx lines | TakeParagraphLines(Lines.TrimParagraphLines lines, Lines.TrimBlankStart rest) -> - yield Paragraph (parseSpans (String.concat ctx.Newline lines)) + yield Paragraph (parseSpans (lines |> List.map fst |> String.concat ctx.Newline, lines |> List.map snd |> MarkdownRange.MergeRanges) ctx, ctx.CurrentRange) yield! parseParagraphs ctx rest | Lines.TrimBlankStart [] -> () diff --git a/src/FSharp.MetadataFormat/Main.fs b/src/FSharp.MetadataFormat/Main.fs index 55873871f..ce1d74f99 100755 --- a/src/FSharp.MetadataFormat/Main.fs +++ b/src/FSharp.MetadataFormat/Main.fs @@ -519,7 +519,7 @@ module Reader = groups.Add(current, []) for par in doc.Paragraphs do match par with - | Heading(2, [Literal text]) -> + | Heading(2, [Literal(text, _)], _) -> current <- text.Trim() groups.Add(current, [par]) | par -> @@ -535,7 +535,7 @@ module Reader = Comment.Create(blurb, full, sections) let findCommand = (function - | String.StartsWithWrapped ("[", "]") (ParseCommand(k, v), rest) -> + | StringPosition.StartsWithWrapped ("[", "]") (ParseCommand(k, v), rest) -> Some (k, v) | _ -> None) @@ -546,7 +546,7 @@ module Reader = Seq.iter (fun (x : XNode) -> if x.NodeType = XmlNodeType.Text then let text = (x :?> XText).Value - match findCommand text with + match findCommand (text, MarkdownRange.Zero) with | Some (k,v) -> cmds.Add(k,v) | None -> full.Append(text) |> ignore elif x.NodeType = XmlNodeType.Element then @@ -635,7 +635,7 @@ module Reader = /// Returns all indirect links in a specified span node let rec collectSpanIndirectLinks span = seq { match span with - | IndirectLink (_, _, key) -> yield key + | IndirectLink (_, _, key, _) -> yield key | Matching.SpanLeaf _ -> () | Matching.SpanNode(_, spans) -> for s in spans do yield! collectSpanIndirectLinks s } @@ -673,9 +673,9 @@ module Reader = /// Wraps the span inside an `IndirectLink` if it is an inline code that can be converted to a link let wrapInlineCodeLinksInSpans (ctx:ReadingContext) span = match span with - | InlineCode(code) -> + | InlineCode(code, r) -> match getTypeLink ctx code with - | Some _ -> IndirectLink([span], code, code) + | Some _ -> IndirectLink([span], code, code, r) | None -> span | _ -> span @@ -713,7 +713,7 @@ module Reader = | null -> dict[], (Comment.Create ("", el.Value, [])) | sum -> - let lines = removeSpaces sum.Value + let lines = removeSpaces sum.Value |> Seq.map (fun s -> (s, MarkdownRange.Zero)) let cmds = new System.Collections.Generic.Dictionary<_, _>() if ctx.MarkdownComments then @@ -722,7 +722,7 @@ module Reader = | Some (k, v) -> cmds.Add(k, v) false - | _ -> true)) |> String.concat "\n" + | _ -> true)) |> Seq.map fst |> String.concat "\n" let doc = Literate.ParseMarkdownString ( text, path=Path.Combine(ctx.AssemblyPath, "docs.fsx"), diff --git a/tests/FSharp.Literate.Tests/EvalTests.fs b/tests/FSharp.Literate.Tests/EvalTests.fs index 9756d4a92..c294d2193 100644 --- a/tests/FSharp.Literate.Tests/EvalTests.fs +++ b/tests/FSharp.Literate.Tests/EvalTests.fs @@ -53,15 +53,15 @@ printf ">>%d<<" 12343 doc.Paragraphs |> shouldMatchPar (function | Matching.LiterateParagraph(FormattedCode(_)) -> true | _ -> false) doc.Paragraphs |> shouldMatchPar (function - | Paragraph [Strong [Literal "hello"]] -> true | _ -> false) + | Paragraph([Strong([Literal("hello", _)], _)], _) -> true | _ -> false) // Contains transformed output doc.Paragraphs |> shouldMatchPar (function - | CodeBlock ("42", _, _) -> true | _ -> false) + | CodeBlock ("42", _, _, _) -> true | _ -> false) doc.Paragraphs |> shouldMatchPar (function - | CodeBlock ("85", _, _) -> true | _ -> false) + | CodeBlock ("85", _, _, _) -> true | _ -> false) doc.Paragraphs |> shouldMatchPar (function - | CodeBlock (">>12343<<", _, _) -> true | _ -> false) + | CodeBlock (">>12343<<", _, _, _) -> true | _ -> false) [] let ``Can evaluate hidden code snippets`` () = @@ -86,15 +86,15 @@ let test = [1;2;3] fsiEvaluator.RegisterTransformation(fun (o, ty) -> if ty.IsGenericType && ty.GetGenericTypeDefinition() = typedefof> then let items = - [ for it in Seq.cast (unbox o) -> [ Paragraph[Literal (it.ToString())] ] ] - Some [ ListBlock(MarkdownListKind.Ordered, items) ] + [ for it in Seq.cast (unbox o) -> [ Paragraph([Literal (it.ToString(), None)], None) ] ] + Some [ ListBlock(MarkdownListKind.Ordered, items, None) ] else None) let doc = Literate.ParseScriptString(content, "." @@ "A.fsx", getFormatAgent(), fsiEvaluator = fsiEvaluator) doc.Paragraphs |> shouldMatchPar (function - | ListBlock(Ordered, items) -> - items = [ [Paragraph [Literal "1"]]; [Paragraph [Literal "2"]]; [Paragraph [Literal "3"]] ] + | ListBlock(Ordered, items, None) -> + items = [ [Paragraph([Literal("1", None)], None)]; [Paragraph([Literal("2", None)], None)]; [Paragraph([Literal("3", None)], None)] ] | _ -> false) [] diff --git a/tests/FSharp.Literate.Tests/Tests.fs b/tests/FSharp.Literate.Tests/Tests.fs index 11676b471..75432718a 100644 --- a/tests/FSharp.Literate.Tests/Tests.fs +++ b/tests/FSharp.Literate.Tests/Tests.fs @@ -61,10 +61,10 @@ a b""", __SOURCE_DIRECTORY__ @@ "Test.fsx") //[/test] - doc.Paragraphs |> shouldMatchPar (function Paragraph [Literal "a"] -> true | _ -> false) - doc.Paragraphs |> shouldMatchPar (function Paragraph [Literal "b"] -> true | _ -> false) + doc.Paragraphs |> shouldMatchPar (function Paragraph([Literal("a", Some({ StartLine = 2 }))], Some({ StartLine = 2 })) -> true | _ -> false) + doc.Paragraphs |> shouldMatchPar (function Paragraph([Literal("b", Some({ StartLine = 6 }))], Some({ StartLine = 6 })) -> true | _ -> false) doc.Paragraphs |> shouldMatchPar (function - | EmbedParagraphs(:? LiterateParagraph as cd) -> + | EmbedParagraphs(:? LiterateParagraph as cd, Some({ StartLine = 4 })) -> match cd with LanguageTaggedCode("csharp", text) -> text.Contains "magic" | _ -> false | _ -> false) @@ -82,7 +82,7 @@ let test = 42""" doc.Paragraphs |> shouldMatchPar (function | Matching.LiterateParagraph(FormattedCode(_)) -> true | _ -> false) doc.Paragraphs |> shouldMatchPar (function - | Paragraph [Strong [Literal "hello"]] -> true | _ -> false) + | Paragraph([Strong([Literal("hello", Some({ StartLine = 1 }))], Some({ StartLine = 1 }))], Some({ StartLine = 1 })) -> true | _ -> false) [] let ``Can parse heading on the same line as opnening comment (#147)`` () = @@ -92,7 +92,7 @@ content *) let test = 42""" let doc = Literate.ParseScriptString(content, "C" @@ "A.fsx", getFormatAgent()) doc.Paragraphs |> shouldMatchPar (function - | Heading(2, [Literal "Heading"]) -> true | _ -> false) + | Heading(2, [Literal("Heading", Some({ StartLine = 1 }))], Some({ StartLine = 1 })) -> true | _ -> false) [] let ``Can parse and format markdown with F# snippet`` () = @@ -105,7 +105,7 @@ let ``Can parse and format markdown with F# snippet`` () = doc.Paragraphs |> shouldMatchPar (function | Matching.LiterateParagraph(FormattedCode(_)) -> true | _ -> false) doc.Paragraphs |> shouldMatchPar (function - | Paragraph [Strong [Literal "hello"]] -> true | _ -> false) + | Paragraph([Strong([Literal("hello", Some({ StartLine = 2 }))], Some({ StartLine = 2 }))], Some({ StartLine = 2 })) -> true | _ -> false) [] let ``Can parse and format markdown with Github-flavoured F# snippet`` () = @@ -120,7 +120,7 @@ let test = 42 doc.Paragraphs |> shouldMatchPar (function | Matching.LiterateParagraph(FormattedCode(_)) -> true | _ -> false) doc.Paragraphs |> shouldMatchPar (function - | Paragraph [Strong [Literal "hello"]] -> true | _ -> false) + | Paragraph([Strong([Literal("hello", Some({ StartLine = 2 }))], Some({ StartLine = 2 }))], Some({ StartLine = 2 })) -> true | _ -> false) [] let ``Can parse and format markdown with Github-flavoured F# snippet starting and ending with empty lines`` () = @@ -144,9 +144,9 @@ some [link][ref] to [ref]: http://there "Author: Article" *)""" let doc = Literate.ParseScriptString(content, "C" @@ "A.fsx", getFormatAgent(), references=true) - doc.Paragraphs |> shouldMatchPar (function ListBlock(_, _) -> true | _ -> false) - doc.Paragraphs |> shouldMatchSpan (function Literal("Article") -> true | _ -> false) - doc.Paragraphs |> shouldMatchSpan (function Literal(" - Author") -> true | _ -> false) + doc.Paragraphs |> shouldMatchPar (function ListBlock(_, _, _) -> true | _ -> false) + doc.Paragraphs |> shouldMatchSpan (function Literal("Article", None) -> true | _ -> false) + doc.Paragraphs |> shouldMatchSpan (function Literal(" - Author", None) -> true | _ -> false) [] let ``Can report errors in F# code snippets (in F# script file)`` () = @@ -581,7 +581,7 @@ let ``Can format single snippet with label using literate parser`` () = let add a b = a + b // [/snippet]""" let doc = Literate.ParseScriptString(source, "/somewhere/test.fsx", getFormatAgent()) - doc.Paragraphs |> shouldMatchPar (function Heading(_, [Literal "demo"]) -> true | _ -> false) + doc.Paragraphs |> shouldMatchPar (function Heading(_, [Literal("demo", Some({ StartLine = 1 }))], Some({ StartLine = 1 })) -> true | _ -> false) [] @@ -594,8 +594,8 @@ let add a b = a + b let mul a b = a * b // [/snippet]""" let doc = Literate.ParseScriptString(source, "/somewhere/test.fsx", getFormatAgent()) - doc.Paragraphs |> shouldMatchPar (function Heading(_, [Literal "demo1"]) -> true | _ -> false) - doc.Paragraphs |> shouldMatchPar (function Heading(_, [Literal "demo2"]) -> true | _ -> false) + doc.Paragraphs |> shouldMatchPar (function Heading(_, [Literal("demo1", Some({ StartLine = 1 }))], Some({ StartLine = 1 })) -> true | _ -> false) + doc.Paragraphs |> shouldMatchPar (function Heading(_, [Literal("demo2", Some({ StartLine = 1 }))], Some({ StartLine = 1 })) -> true | _ -> false) [] let ``Formatter does not crash on source that contains invalid string`` () = diff --git a/tests/FSharp.Markdown.Tests/FSharp.Markdown.Tests.fsproj b/tests/FSharp.Markdown.Tests/FSharp.Markdown.Tests.fsproj index cdebb63c3..01ed64dc6 100644 --- a/tests/FSharp.Markdown.Tests/FSharp.Markdown.Tests.fsproj +++ b/tests/FSharp.Markdown.Tests/FSharp.Markdown.Tests.fsproj @@ -95,6 +95,11 @@ --> + + FSharp.Formatting.Common + {91bad90e-bf3b-4646-a1a7-1568f8f25075} + True + FSharp.Markdown {c44c1c05-599a-40dd-9590-465eab8960c5} diff --git a/tests/FSharp.Markdown.Tests/Markdown.fs b/tests/FSharp.Markdown.Tests/Markdown.fs index 8b7aaf952..8d90faf69 100644 --- a/tests/FSharp.Markdown.Tests/Markdown.fs +++ b/tests/FSharp.Markdown.Tests/Markdown.fs @@ -9,6 +9,7 @@ module FSharp.Markdown.Tests.Parsing open FsUnit open NUnit.Framework open FSharp.Markdown +open FSharp.Formatting.Common let properNewLines (text: string) = text.Replace("\r\n", System.Environment.NewLine) @@ -19,7 +20,7 @@ let shouldEqualNoWhiteSpace (x:string) (y:string) = let ``Inline HTML tag containing 'at' is not turned into hyperlink`` () = let doc = """    hi""" |> Markdown.Parse doc.Paragraphs - |> shouldEqual [ Paragraph [Literal """hi""" ]] + |> shouldEqual [ Paragraph([Literal("""hi""", Some({ StartLine = 1; StartColumn = 0; EndLine = 1; EndColumn = 29 })) ], Some({ StartLine = 1; StartColumn = 0; EndLine = 1; EndColumn = 29 }))] [] let ``Encode '<' and '>' characters as HTML entities`` () = @@ -36,8 +37,8 @@ Some more""" |> Markdown.Parse doc.Paragraphs |> shouldEqual [ - Heading(2, [Literal "Hello F#"]); - Paragraph [Literal "Some more"]] + Heading(2, [Literal("Hello F#", Some({ StartLine = 2; StartColumn = 3; EndLine = 2; EndColumn = 11 }))], Some({ StartLine = 2; StartColumn = 0; EndLine = 2; EndColumn = 11 })) + Paragraph([Literal("Some more", Some({ StartLine = 3; StartColumn = 0; EndLine = 3; EndColumn = 9 }))], Some({ StartLine = 3; StartColumn = 0; EndLine = 3; EndColumn = 9 }))] [] let ``Headings ending with spaces followed by # are parsed correctly`` () = @@ -47,8 +48,8 @@ Some more""" |> Markdown.Parse doc.Paragraphs |> shouldEqual [ - Heading(2, [Literal "Hello"]); - Paragraph [Literal "Some more"]] + Heading(2, [Literal("Hello", Some({ StartLine = 2; StartColumn = 3; EndLine = 2; EndColumn = 8 }))], Some({ StartLine = 2; StartColumn = 0; EndLine = 2; EndColumn = 13 })) + Paragraph([Literal("Some more", Some({ StartLine = 3; StartColumn = 0; EndLine = 3; EndColumn = 9 }))], Some({ StartLine = 3; StartColumn = 0; EndLine = 3; EndColumn = 9 }))] [] let ``Should be able to create nested list item with two paragraphs`` () = @@ -59,38 +60,38 @@ let ``Should be able to create nested list item with two paragraphs`` () = c""" |> Markdown.Parse let expectedBody = - [ Paragraph [Literal "b"] - Paragraph [Literal "c"] ] + [ Paragraph([Literal("b", Some({ StartLine = 3; StartColumn = 4; EndLine = 3; EndColumn = 5 }))] , Some({ StartLine = 3; StartColumn = 4; EndLine = 3; EndColumn = 5 })) + Paragraph([Literal("c", Some({ StartLine = 5; StartColumn = 4; EndLine = 5; EndColumn = 5 }))], Some({ StartLine = 5; StartColumn = 4; EndLine = 5; EndColumn = 5 })) ] match doc.Paragraphs.Head with - | ListBlock(Unordered, [ [Span [Literal "a"]; ListBlock (Unordered, [ body ])] ]) -> + | ListBlock(Unordered, [ [Span([Literal("a", Some({ StartLine = 2; StartColumn = 2; EndLine = 2; EndColumn = 3 }))], _); ListBlock(Unordered, [ body ], _)] ], _) -> body |> shouldEqual expectedBody | _ -> Assert.Fail "Expected list block with a nested list block" [] let ``Can escape special characters such as "*" in emphasis`` () = let doc = """*foo\*\*bar*""" |> Markdown.Parse - let expected = Paragraph [Emphasis [Literal "foo**bar"]] + let expected = Paragraph([Emphasis([Literal("foo**bar", Some({ StartLine = 1; StartColumn = 0; EndLine = 1; EndColumn = 8 }))], Some({ StartLine = 1; StartColumn = 0; EndLine = 1; EndColumn = 12 }))], Some({ StartLine = 1; StartColumn = 0; EndLine = 1; EndColumn = 12 })) doc.Paragraphs.Head |> shouldEqual expected [] let ``Can escape special characters in LaTex inline math`` () = let doc = """test \$ is: $foo\$\$bar<>\$\&\%\$\#\_\{\}$""" |> Markdown.Parse - let expected = Paragraph [Literal "test $ is: "; LatexInlineMath "foo\$\$bar<>\$\&\%\$\#\_\{\}"] + let expected = Paragraph([Literal("test $ is: ", Some({ StartLine = 1; StartColumn = 0; EndLine = 1; EndColumn = 11 })); LatexInlineMath("foo\$\$bar<>\$\&\%\$\#\_\{\}", Some({ StartLine = 1; StartColumn = 12; EndLine = 1; EndColumn = 40 }))], Some({ StartLine = 1; StartColumn = 0; EndLine = 1; EndColumn = 42 })) doc.Paragraphs.Head |> shouldEqual expected [] let ``Test special character _ in LaTex inline math`` () = let doc = """$\bigcap_{x \in A} p_{x}A$""" |> Markdown.Parse - let expected = Paragraph [ LatexInlineMath "\\bigcap_{x \\in A} p_{x}A" ] + let expected = Paragraph([ LatexInlineMath("\\bigcap_{x \\in A} p_{x}A", Some({ StartLine = 1; StartColumn = 1; EndLine = 1; EndColumn = 25 })) ], Some({ StartLine = 1; StartColumn = 0; EndLine = 1; EndColumn = 26 })) doc.Paragraphs.Head |> shouldEqual expected [] let ``Inline code can contain backticks when wrapped with spaces`` () = let doc = """` ``h`` `""" |> Markdown.Parse - let expected = Paragraph [InlineCode "``h``"] + let expected = Paragraph([InlineCode("``h``", Some({ StartLine = 1; StartColumn = 2; EndLine = 1; EndColumn = 7 }))], Some({ StartLine = 1; StartColumn = 0; EndLine = 1; EndColumn = 9 })) doc.Paragraphs.Head |> shouldEqual expected @@ -284,7 +285,7 @@ let ``Transform horizontal rules correctly``() = let doc = "* * *\r\n\r\n***\r\n\r\n*****\r\n\r\n- - -\r\n\r\n---------------------------------------\r\n\r\n"; let expected = "
    \r\n
    \r\n
    \r\n
    \r\n
    \r\n" |> properNewLines Markdown.Parse(doc).Paragraphs - |> shouldEqual [ HorizontalRule '*'; HorizontalRule '*'; HorizontalRule '*'; HorizontalRule '-'; HorizontalRule '-' ] + |> shouldEqual [ HorizontalRule('*', Some({ StartLine = 1; StartColumn = 0; EndLine = 1; EndColumn = 5 })); HorizontalRule('*', Some({ StartLine = 3; StartColumn = 0; EndLine = 3; EndColumn = 3 })); HorizontalRule('*', Some({ StartLine = 5; StartColumn = 0; EndLine = 5; EndColumn = 5 })); HorizontalRule('-', Some({ StartLine = 7; StartColumn = 0; EndLine = 7; EndColumn = 5 })); HorizontalRule('-', Some({ StartLine = 9; StartColumn = 0; EndLine = 9; EndColumn = 39 })) ] Markdown.TransformHtml doc |> shouldEqual expected @@ -326,8 +327,8 @@ let ``Transform tables with delimiters in code or math correctly``() = let ``Parse empty blockquote followed by content``() = let doc = "> a" - let expected = [ QuotedBlock [] - Paragraph [ Literal "a" ] ] + let expected = [ QuotedBlock([], Some({ StartLine = 1; StartColumn = 0; EndLine = 1; EndColumn = 1 })) + Paragraph([ Literal("a", Some({ StartLine = 2; StartColumn = 0; EndLine = 2; EndColumn = 1 })) ], Some({ StartLine = 2; StartColumn = 0; EndLine = 2; EndColumn = 1 })) ] (Markdown.Parse doc).Paragraphs |> shouldEqual expected @@ -337,8 +338,8 @@ let ``Parse blockquote teriminated by empty blockquote line and followed by cont let doc = ">a > a" - let expected = [ QuotedBlock [ Paragraph [ Literal "a" ] ] - Paragraph [ Literal "a" ] ] + let expected = [ QuotedBlock([ Paragraph([ Literal("a", Some({ StartLine = 1; StartColumn = 1; EndLine = 1; EndColumn = 2 })) ], Some({ StartLine = 1; StartColumn = 1; EndLine = 1; EndColumn = 2 })) ], Some({ StartLine = 1; StartColumn = 0; EndLine = 1; EndColumn = 2 })) + Paragraph([ Literal("a", Some({ StartLine = 3; StartColumn = 0; EndLine = 3; EndColumn = 1 })) ], Some({ StartLine = 3; StartColumn = 0; EndLine = 3; EndColumn = 1 })) ] (Markdown.Parse doc).Paragraphs |> shouldEqual expected @@ -346,7 +347,7 @@ a" [] let ``Parse blockquote with three leading spaces``() = let doc = " >a" - let expected = [ QuotedBlock [ Paragraph [ Literal "a" ] ] ] + let expected = [ QuotedBlock([ Paragraph([ Literal("a", Some({ StartLine = 1; StartColumn = 4; EndLine = 1; EndColumn = 5 })) ], Some({ StartLine = 1; StartColumn = 4; EndLine = 1; EndColumn = 5 })) ], Some({ StartLine = 1; StartColumn = 0; EndLine = 1; EndColumn = 5 })) ] (Markdown.Parse doc).Paragraphs |> shouldEqual expected \ No newline at end of file