Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add range to MarkdownParagraph and MarkdownSpan #411

Closed
wants to merge 6 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/content/codeformat.fsx
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ can be called to format snippets repeatedly:
*)

let fsharpCompiler = Assembly.Load("FSharp.Compiler")
let formattingAgent = CodeFormat.CreateAgent(fsharpCompiler)
let formattingAgent = CodeFormat.CreateAgent()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Out of curiosity, is this a needed change for some reason? (Happy to delete the parameter if it's not needed, but then we should probably delete the whole line 26.)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I didnt removed the parameter from the method declaration when I looked at all scrpts to see if I need to change something I saw error on this line so I change it


(**
If you want to process multiple snippets, it is a good idea to keep the
Expand Down
9 changes: 6 additions & 3 deletions docs/content/markdown.fsx
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,10 @@ First, we need to load the assembly and open necessary namespaces:
*)

#r "../../bin/FSharp.Markdown.dll"
#r "../../bin/FSharp.Formatting.Common.dll"
open FSharp.Markdown
open FSharp.Formatting.Common


(**
Parsing documents
Expand Down Expand Up @@ -64,7 +67,7 @@ The following snippet prints the heading of the document:
// Iterate over all the paragraph elements
for par in parsed.Paragraphs do
match par with
| Heading(1, [Literal text]) ->
| Heading(size=1; body=[Literal(text=text)]) ->
// Recognize heading that has a simple content
// containing just a literal (no other formatting)
printfn "%s" text
Expand Down Expand Up @@ -92,8 +95,8 @@ to recognize any paragraph or span that can contain child elements:
/// Returns all links in a specified span node
let rec collectSpanLinks span = seq {
match span with
| DirectLink(_, (url, _)) -> yield url
| IndirectLink(_, _, key) -> yield fst (parsed.DefinedLinks.[key])
| DirectLink(_, (url, _), _) -> yield url
| IndirectLink(_, _, key, _) -> yield fst (parsed.DefinedLinks.[key])
| Matching.SpanLeaf _ -> ()
| Matching.SpanNode(_, spans) ->
for s in spans do yield! collectSpanLinks s }
Expand Down
192 changes: 124 additions & 68 deletions src/Common/StringParsing.fs
Original file line number Diff line number Diff line change
Expand Up @@ -7,51 +7,141 @@ module internal FSharp.Patterns

open System
open FSharp.Collections
open FSharp.Formatting.Common

// --------------------------------------------------------------------------------------
// Active patterns that simplify parsing of strings and lists of strings (lines)
// --------------------------------------------------------------------------------------

module String =
/// Matches when a string is a whitespace or null
let (|WhiteSpace|_|) s =
let (|WhiteSpace|_|) (s) =
if String.IsNullOrWhiteSpace(s) then Some() else None

/// Returns a string trimmed from both start and end
let (|TrimBoth|) (text:string) = text.Trim()

/// Matches when a string starts with the specified sub-string
let (|StartsWith|_|) (start:string) (text:string) =
if text.StartsWith(start) then Some(text.Substring(start.Length)) else None

/// Matches when a string starts with the specified sub-string
/// The matched string is trimmed from all whitespace.
let (|StartsWithTrim|_|) (start:string) (text:string) =
if text.StartsWith(start) then Some(text.Substring(start.Length).Trim()) else None

/// Matches when a string starts with the given value and ends
/// with a given value (and returns the rest of it)
let (|StartsAndEndsWith|_|) (starts, ends) (s:string) =
if s.StartsWith(starts) && s.EndsWith(ends) &&
s.Length >= starts.Length + ends.Length then
Some(s.Substring(starts.Length, s.Length - starts.Length - ends.Length))
else None

/// Matches when a string starts with the given value and ends
/// with a given value (and returns trimmed body)
let (|StartsAndEndsWithTrim|_|) args = function
| StartsAndEndsWith args (TrimBoth res) -> Some res
| _ -> None

/// Matches when a string starts with a sub-string wrapped using the
/// opening and closing sub-string specified in the parameter.
/// For example "[aa]bc" is wrapped in [ and ] pair. Returns the wrapped
/// text together with the rest.
let (|StartsWithWrapped|_|) (starts:string, ends:string) (text:string) =
if text.StartsWith(starts) then
let id = text.IndexOf(ends, starts.Length)
if id >= 0 then
let wrapped = text.Substring(starts.Length, id - starts.Length)
let rest = text.Substring(id + ends.Length, text.Length - id - ends.Length)
Some(wrapped, rest)
else None
else None

/// Ignores everything until a end-line character is detected, returns the remaining string.
let (|SkipSingleLine|) (text:string) =
let rec tryEol eolList =
match eolList with
| h : string :: t ->
match text.IndexOf(h) with
| i when i < 0 -> tryEol t
| i ->
text.Substring (i + h.Length)
| _ ->
text
let result = tryEol [ "\r\n"; "\n" ]
let skipped = text.Substring(0, text.Length - result.Length)
if not <| String.IsNullOrWhiteSpace(skipped) then
FSharp.Formatting.Common.Log.warnf "skipped '%s' which contains non-whitespace character!" skipped
if result = text then
FSharp.Formatting.Common.Log.warnf "could not skip a line of %s, because no line-ending character was found!" text
result

/// Given a list of lines indented with certan number of whitespace
/// characters (spaces), remove the spaces from the beginning of each line
/// and return the string as a list of lines
let removeSpaces lines =
let spaces =
lines
|> Seq.filter (String.IsNullOrWhiteSpace >> not)
|> Seq.map (fun line -> line |> Seq.takeWhile Char.IsWhiteSpace |> Seq.length)
|> fun xs -> if Seq.isEmpty xs then 0 else Seq.min xs
lines
|> Seq.map (fun line ->
if String.IsNullOrWhiteSpace(line) then ""
else line.Substring(spaces))

module StringPosition =
/// Matches when a string is a whitespace or null
let (|WhiteSpace|_|) (s, n: MarkdownRange) =
if String.IsNullOrWhiteSpace(s) then Some() else None

/// Matches when a string does starts with non-whitespace
let (|Unindented|_|) (s:string) =
let (|Unindented|_|) (s:string, n:MarkdownRange) =
if not (String.IsNullOrWhiteSpace(s)) && s.TrimStart() = s then Some() else None

/// Returns a string trimmed from both start and end
let (|TrimBoth|) (text:string) = text.Trim()
let (|TrimBoth|) (text:string, n:MarkdownRange) =
let trimmedStart = text.TrimStart()
let trimmed = trimmedStart.TrimEnd()
(trimmed, { n with StartColumn = n.StartColumn + text.Length - trimmedStart.Length; EndColumn = n.EndColumn - trimmedStart.Length + trimmed.Length })
/// Returns a string trimmed from the end
let (|TrimEnd|) (text:string) = text.TrimEnd()
let (|TrimEnd|) (text:string, n:MarkdownRange) =
let trimmed = text.TrimEnd()
(trimmed, { n with EndColumn = n.EndColumn - text.Length + trimmed.Length })
/// Returns a string trimmed from the start
let (|TrimStart|) (text:string) = text.TrimStart()
let (|TrimStart|) (text:string, n:MarkdownRange) =
let trimmed = text.TrimStart()
(trimmed, { n with StartColumn = n.StartColumn + text.Length - trimmed.Length })

/// Retrusn a string trimmed from the end using characters given as a parameter
let (|TrimEndUsing|) chars (text:string) = text.TrimEnd(Array.ofSeq chars)
/// Returns a string trimmed from the end using characters given as a parameter
let (|TrimEndUsing|) chars (text:string, n:MarkdownRange) =
let trimmed = text.TrimEnd(Array.ofSeq chars)
(trimmed, { n with EndColumn = n.EndColumn - text.Length + trimmed.Length })

/// Returns a string trimmed from the start together with
/// the number of skipped whitespace characters
let (|TrimStartAndCount|) (text:string) =
let (|TrimStartAndCount|) (text:string, n:MarkdownRange) =
let trimmed = text.TrimStart([|' '; '\t'|])
let len = text.Length - trimmed.Length
len, text.Substring(0, len).Replace("\t", " ").Length, trimmed
len, text.Substring(0, len).Replace("\t", " ").Length, (trimmed, { n with StartColumn = n.StartColumn + text.Length - trimmed.Length })

/// Matches when a string starts with any of the specified sub-strings
let (|StartsWithAny|_|) (starts:seq<string>) (text:string) =
let (|StartsWithAny|_|) (starts:seq<string>) (text:string, n:MarkdownRange) =
if starts |> Seq.exists (text.StartsWith) then Some() else None

/// Matches when a string starts with the specified sub-string
let (|StartsWith|_|) (start:string) (text:string) =
if text.StartsWith(start) then Some(text.Substring(start.Length)) else None
let (|StartsWith|_|) (start:string) (text:string, n:MarkdownRange) =
if text.StartsWith(start) then Some(text.Substring(start.Length), { n with StartColumn = n.StartColumn + text.Length - start.Length }) else None

/// Matches when a string starts with the specified sub-string
/// The matched string is trimmed from all whitespace.
let (|StartsWithTrim|_|) (start:string) (text:string) =
if text.StartsWith(start) then Some(text.Substring(start.Length).Trim()) else None
let (|StartsWithTrim|_|) (start:string) (text:string, n:MarkdownRange) =
if text.StartsWith(start) then Some(text.Substring(start.Length).Trim(), { n with StartColumn = n.StartColumn + text.Length - start.Length }) else None

/// Matches when a string starts with the specified sub-string (ignoring whitespace at the start)
/// The matched string is trimmed from all whitespace.
let (|StartsWithNTimesTrimIgnoreStartWhitespace|_|) (start:string) (text:string) =
let (|StartsWithNTimesTrimIgnoreStartWhitespace|_|) (start:string) (text:string, n:MarkdownRange) =
if text.Contains(start) then
let beforeStart = text.Substring(0, text.IndexOf(start))
if String.IsNullOrWhiteSpace (beforeStart) then
Expand All @@ -67,10 +157,10 @@ module String =

/// Matches when a string starts with the given value and ends
/// with a given value (and returns the rest of it)
let (|StartsAndEndsWith|_|) (starts, ends) (s:string) =
let (|StartsAndEndsWith|_|) (starts, ends) (s:string, n:MarkdownRange) =
if s.StartsWith(starts) && s.EndsWith(ends) &&
s.Length >= starts.Length + ends.Length then
Some(s.Substring(starts.Length, s.Length - starts.Length - ends.Length))
Some(s.Substring(starts.Length, s.Length - starts.Length - ends.Length), { n with StartColumn = n.StartColumn + s.Length - starts.Length; EndColumn = n.EndColumn - s.Length + ends.Length })
else None

/// Matches when a string starts with the given value and ends
Expand All @@ -85,78 +175,43 @@ module String =
///
/// let (StartsWithRepeated "/\" (2, " abc")) = "/\/\ abc"
///
let (|StartsWithRepeated|_|) (repeated:string) (text:string) =
let (|StartsWithRepeated|_|) (repeated:string) (text:string, ln:MarkdownRange) =
let rec loop i =
if i = text.Length then i
elif text.[i] <> repeated.[i % repeated.Length] then i
else loop (i + 1)

let n = loop 0
if n = 0 || n % repeated.Length <> 0 then None
else Some(n/repeated.Length, text.Substring(n, text.Length - n))

/// Ignores everything until a end-line character is detected, returns the remaining string.
let (|SkipSingleLine|) (text:string) =
let rec tryEol eolList =
match eolList with
| h : string :: t ->
match text.IndexOf(h) with
| i when i < 0 -> tryEol t
| i ->
text.Substring (i + h.Length)
| _ ->
text
let result = tryEol [ "\r\n"; "\n" ]
let skipped = text.Substring(0, text.Length - result.Length)
if not <| String.IsNullOrWhiteSpace(skipped) then
FSharp.Formatting.Common.Log.warnf "skipped '%s' which contains non-whitespace character!" skipped
if result = text then
FSharp.Formatting.Common.Log.warnf "could not skip a line of %s, because no line-ending character was found!" text
result

else Some(n/repeated.Length, (text.Substring(n, text.Length - n), { ln with StartColumn = n }))

/// Matches when a string starts with a sub-string wrapped using the
/// opening and closing sub-string specified in the parameter.
/// For example "[aa]bc" is wrapped in [ and ] pair. Returns the wrapped
/// text together with the rest.
let (|StartsWithWrapped|_|) (starts:string, ends:string) (text:string) =
let (|StartsWithWrapped|_|) (starts:string, ends:string) (text:string, n:MarkdownRange) =
if text.StartsWith(starts) then
let id = text.IndexOf(ends, starts.Length)
if id >= 0 then
let wrapped = text.Substring(starts.Length, id - starts.Length)
let rest = text.Substring(id + ends.Length, text.Length - id - ends.Length)
Some(wrapped, rest)
Some(wrapped, (rest, { n with StartColumn = id + ends.Length }))
else None
else None

/// Matches when a string consists of some number of
/// complete repetitions of a specified sub-string.
let (|EqualsRepeated|_|) repeated = function
| StartsWithRepeated repeated (n, "") -> Some()
let (|EqualsRepeated|_|) (repeated, n:MarkdownRange) = function
| StartsWithRepeated repeated (n, ("", _)) -> Some()
| _ -> None

/// Given a list of lines indented with certan number of whitespace
/// characters (spaces), remove the spaces from the beginning of each line
/// and return the string as a list of lines
let removeSpaces lines =
let spaces =
lines
|> Seq.filter (String.IsNullOrWhiteSpace >> not)
|> Seq.map (fun line -> line |> Seq.takeWhile Char.IsWhiteSpace |> Seq.length)
|> fun xs -> if Seq.isEmpty xs then 0 else Seq.min xs
lines
|> Seq.map (fun line ->
if String.IsNullOrWhiteSpace(line) then ""
else line.Substring(spaces))


module List =
/// Matches a list if it starts with a sub-list that is delimited
/// using the specified delimiters. Returns a wrapped list and the rest.
let inline (|DelimitedWith|_|) startl endl input =
if List.startsWith startl input then
match List.partitionUntilEquals endl (List.skip startl.Length input) with
| Some(pre, post) -> Some(pre, List.skip endl.Length post)
| Some(pre, post) -> Some(pre, List.skip endl.Length post, startl.Length, endl.Length)
| None -> None
else None

Expand Down Expand Up @@ -197,8 +252,8 @@ module Lines =
/// Removes blank lines from the start and the end of a list
let (|TrimBlank|) lines =
lines
|> List.skipWhile String.IsNullOrWhiteSpace |> List.rev
|> List.skipWhile String.IsNullOrWhiteSpace |> List.rev
|> List.skipWhile (fun (s, n) -> String.IsNullOrWhiteSpace s) |> List.rev
|> List.skipWhile (fun (s, n) -> String.IsNullOrWhiteSpace s) |> List.rev

/// Matches when there are some lines at the beginning that are
/// either empty (or whitespace) or start with the specified string.
Expand All @@ -213,7 +268,7 @@ module Lines =
/// either empty (or whitespace) or start with at least 4 spaces (a tab counts as 4 spaces here).
/// Returns all such lines from the beginning until a different line and
/// the number of spaces the first line started with.
let (|TakeCodeBlock|_|) (input:string list) =
let (|TakeCodeBlock|_|) (input:(string * MarkdownRange) list) =
let spaceNum = 4
//match input with
//| h :: _ ->
Expand All @@ -225,24 +280,25 @@ module Lines =
let normalized = s.Replace("\t", " ")
normalized.Length >= spaceNum &&
normalized.Substring(0, spaceNum) = System.String(' ', spaceNum)
match List.partitionWhile (fun s ->
match List.partitionWhile (fun (s, n) ->
String.IsNullOrWhiteSpace s || startsWithSpaces s) input with
| matching, rest when matching <> [] && spaceNum >= 4 ->
Some(spaceNum, matching, rest)
| _ -> None

/// Removes whitespace lines from the beginning of the list
let (|TrimBlankStart|) = List.skipWhile (String.IsNullOrWhiteSpace)
let (|TrimBlankStart|) = List.skipWhile (fun (s:string, n:MarkdownRange) -> String.IsNullOrWhiteSpace s)

/// Trims all lines of the current paragraph
let (|TrimParagraphLines|) lines =
lines
// first remove all whitespace on the beginning of the line
|> List.map (fun (s:string) -> s.TrimStart())
|> List.map StringPosition.(|TrimStart|)
// Now remove all additional spaces at the end, but keep two spaces if existent
|> List.map (fun s ->
|> List.map (fun (s, n) ->
let endsWithTwoSpaces = s.EndsWith(" ")
s.TrimEnd([|' '|]) + if endsWithTwoSpaces then " " else "")
let trimmed = s.TrimEnd([|' '|]) + if endsWithTwoSpaces then " " else ""
(trimmed, { n with EndColumn = n.EndColumn - s.Length + trimmed.Length }))

/// Parameterized pattern that assigns the specified value to the
/// first component of a tuple. Usage:
Expand Down
6 changes: 3 additions & 3 deletions src/FSharp.CodeFormat/CodeFormatAgent.fs
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ module private Helpers =
state := nstate
yield! parseLine()
| None, nstate -> state := nstate }
yield n, parseLine() |> List.ofSeq ]
yield { StartLine = n; StartColumn = 0; EndLine = n; EndColumn = 0 }, parseLine() |> List.ofSeq ]

// Count the minimal number of spaces at the beginning of lines
// (so that we can remove spaces for indented text)
Expand Down Expand Up @@ -238,10 +238,10 @@ type CodeFormatAgent() =
processSnippetLine
checkResults
(categorizedSpans
|> Map.tryFind ((fst snippetLine) + 1)
|> Map.tryFind ((fst snippetLine).StartLine + 1)
|> function None -> [] | Some spans -> List.ofSeq spans)
lines
snippetLine)
((fst snippetLine).StartLine, snd snippetLine))

// --------------------------------------------------------------------------------------

Expand Down
Loading