We are now even closer to the spec

RELEASE_NOTES.md

@@ -1,3 +1,10 @@
+## 2.12.2 (30 December, 2015)
+ - Be compatible with the common-mark spec for 'Fenced code blocks' and 'Indented code blocks'.
+   See https://github.com/tpetricek/FSharp.Formatting/pull/343.
+   Please follow-up by adding support for more sections of the spec!
+   Just add the section to https://github.com/tpetricek/FSharp.Formatting/blob/master/tests/FSharp.Markdown.Tests/CommonMarkSpecTest.fs#L20
+   and fix the newly enabled tests.
+
 ## 2.12.1 (24 December, 2015)
  - update dependencies
  - Upgrade the CommandTool to F# 4 and bundle FSharp.Core with sigdata and optdata.

src/Common/StringParsing.fs

@@ -214,13 +214,13 @@ module Lines =
   /// Returns all such lines from the beginning until a different line and
   /// the number of spaces the first line started with.
   let (|TakeCodeBlock|_|) (input:string list) =
-    let spaceNum =
-      match input with
-      | h :: _ ->
-        let head = (input |> List.head).Replace("\t", "    ") |> Seq.toList
-        let spaces, _ = List.partitionWhile (fun s -> s = ' ') head
-        spaces.Length
-      | _ -> 0
+    let spaceNum = 4
+      //match input with
+      //| h :: _ ->
+      //  let head = (input |> List.head).Replace("\t", "    ") |> Seq.toList
+      //  let spaces, _ = List.partitionWhile (fun s -> s = ' ') head
+      //  spaces.Length
+      //| _ -> 0
     let startsWithSpaces (s:string) =
       let normalized = s.Replace("\t", "    ")
       normalized.Length >= spaceNum &&

src/FSharp.Markdown/HtmlFormatting.fs

@@ -60,7 +60,7 @@ type FormattingContext =
     ParagraphIndent : unit -> unit }
 
 let bigBreak (ctx:FormattingContext) () =
-  ctx.Writer.Write(ctx.Newline + ctx.Newline)
+  ctx.Writer.Write(ctx.Newline)
 let smallBreak (ctx:FormattingContext) () =
   ctx.Writer.Write(ctx.Newline)
 let noBreak (ctx:FormattingContext) () = ()
@@ -155,7 +155,11 @@ let formatAnchor (ctx:FormattingContext) (spans:MarkdownSpans) =
     |> String.concat "-"
     |> fun name -> if String.IsNullOrWhiteSpace name then "header" else name
     |> ctx.UniqueNameGenerator.GetName
-
+let withInner ctx f =
+  use sb = new StringWriter()
+  let newCtx = { ctx with Writer = sb }
+  f newCtx
+  sb.ToString()
 /// Write a MarkdownParagraph value to a TextWriter
 let rec formatParagraph (ctx:FormattingContext) paragraph =
   match paragraph with
@@ -230,15 +234,25 @@ let rec formatParagraph (ctx:FormattingContext) paragraph =
       ctx.Writer.Write("<" + tag + ">" + ctx.Newline)
       for body in items do
         ctx.Writer.Write("<li>")
-        body |> List.iterInterleaved 
-                  (formatParagraph { ctx with LineBreak = noBreak ctx }) 
-                  (fun () -> ctx.Writer.Write(ctx.Newline))
+        match body with
+        // Simple Paragraph
+        | [ Paragraph [MarkdownSpan.Literal s] ] when not (s.Contains(ctx.Newline)) ->
+          ctx.Writer.Write s
+        | _ ->
+          let inner =
+            withInner ctx (fun ctx ->
+              body |> List.iterInterleaved
+                    (formatParagraph { ctx with LineBreak = noBreak ctx })
+                    (fun () -> ctx.Writer.Write(ctx.Newline)))
+          let wrappedInner =
+            if inner.Contains(ctx.Newline) then ctx.Newline + inner + ctx.Newline else inner
+          ctx.Writer.Write(wrappedInner)
         ctx.Writer.Write("</li>" + ctx.Newline)
       ctx.Writer.Write("</" + tag + ">")
   | QuotedBlock(body) ->
       ctx.ParagraphIndent()
       ctx.Writer.Write("<blockquote>" + ctx.Newline)
-      formatParagraphs { ctx with ParagraphIndent = fun () -> ctx.ParagraphIndent(); ctx.Writer.Write("  ") } body
+      formatParagraphs { ctx with ParagraphIndent = fun () -> ctx.ParagraphIndent() (*; ctx.Writer.Write("  ")*) } body
       ctx.ParagraphIndent()
       ctx.Writer.Write("</blockquote>")
   | Span spans -> 

src/FSharp.Markdown/MarkdownParser.fs

@@ -290,12 +290,11 @@ let (|HorizontalRule|_|) (line:string) =
 
 /// Recognizes a code block - lines starting with four spaces (including blank)
 let (|NestedCodeBlock|_|) = function
-  | Lines.TakeStartingWithOrBlank "    " (Lines.TrimBlank lines, rest) when lines <> [] ->
+  | Lines.TakeCodeBlock (numspaces, Lines.TrimBlank lines, rest) when lines <> [] ->
       let code =
         [ for l in lines ->
-            if l.Length < 5 && String.IsNullOrWhiteSpace l then ""
-            elif l.Length > 4 then l.Substring(4, l.Length - 4)
-            else l ]
+            if String.IsNullOrEmpty l then ""
+            else trimSpaces 4 l ]
       Some(code @ [""], rest, "", "")
   | _ -> None
 
@@ -312,7 +311,7 @@ let (|FencedCodeBlock|_|) = function
         match [line] with
         // end cannot contain info string afterwards (see http://spec.commonmark.org/0.23/#example-104)
         // end must be indended with less then 4 spaces: http://spec.commonmark.org/0.23/#example-95
-        | String.StartsWithNTimesTrimIgnoreStartWhitespace start (n, i, h) :: _ when n >= num && i < 4 && not (h.Contains start) ->
+        | String.StartsWithNTimesTrimIgnoreStartWhitespace start (n, i, h) :: _ when n >= num && i < 4 && String.IsNullOrWhiteSpace h ->
           endStr <- String.replicate n start
           true
         | _ -> false)
@@ -342,7 +341,7 @@ let (|FencedCodeBlock|_|) = function
                 let after = hd.Substring(idx + endStr.Length)
                 code @ [""], (if String.IsNullOrWhiteSpace after then tl else after :: tl)
             else
-                code, tl
+                code @ [""], tl
         | _ ->
             code, rest
       Some (code, rest, langString, ignoredString)
@@ -368,7 +367,7 @@ let (|ListStart|_|) = function
       let li = item.Substring(2)
       let (String.TrimStartAndCount (startIndent2, spaces2, _)) = li
       let endIndent =
-        startIndent +
+        startIndent + 2 +
         // Handle case of code block
         if startIndent2 >= 5 then 1 else startIndent2
       Some(Unordered, startIndent, endIndent, li)
@@ -408,23 +407,30 @@ let (|ListItem|_|) prevSimple = function
             // the value 'more' will contain indented paragraphs
             (LinesUntilListOrUnindented (more, rest) as next)) ->
       let simple =
-        match next, rest with
-        | String.WhiteSpace::_, (ListStart _)::_ -> false
-        | (ListStart _)::_, _ -> true
-        | [], _ -> true
-        | String.WhiteSpace::[], _ -> true
-        | String.WhiteSpace::String.WhiteSpace::_, _ -> true
-        | _, String.Unindented::_ -> prevSimple
-        | _, _ -> false
+        match item with
+        | String.TrimStartAndCount (_, spaces, _) when spaces >= 4->
+          // Code Block
+          false
+        | _ ->
+          match next, rest with
+          | String.WhiteSpace::_, (ListStart _)::_ -> false
+          | (ListStart _)::_, _ -> true
+          | [], _ -> true
+          | [ String.WhiteSpace ], _ -> true
+          | String.WhiteSpace::String.WhiteSpace::_, _ -> true
+          | _, String.Unindented::_ -> prevSimple
+          | _, _ -> false
 
       let lines =
         [ yield item
           for line in continued do
             yield line.Trim()
           for line in more do
-            let trimmed = line.TrimStart()
-            if trimmed.Length >= line.Length - endIndent then yield trimmed
-            else yield line.Substring(endIndent) ]
+            let trimmed = trimSpaces endIndent line
+            yield trimmed ]
+            //let trimmed = line.TrimStart()
+            //if trimmed.Length >= line.Length - endIndent then yield trimmed
+            //else yield line.Substring(endIndent) ]
       Some(startIndent, (simple, kind, lines), rest)
   | _ -> None
 
@@ -567,14 +573,19 @@ let (|HtmlBlock|_|) = function
   | _ -> None
 
 /// Continues taking lines until a whitespace line or start of a blockquote
-let (|LinesUntilBlockquoteOrWhite|) =
-  List.partitionUntil (function
-    | BlockquoteStart _ | String.WhiteSpace -> true | _ -> false)
+let (|LinesUntilBlockquoteEnds|) input =
+  List.partitionUntilLookahead (fun next ->
+    match next with
+    | BlockquoteStart _ :: _
+    | Heading _
+    | String.WhiteSpace :: _ -> true
+    | _ ->
+      false) input
 
 /// Recognizes blockquote - continues taking paragraphs
 /// starting with '>' until there is something else
 let rec (|Blockquote|_|) = function
-  | BlockquoteStart(line)::LinesUntilBlockquoteOrWhite(continued, Lines.TrimBlankStart rest) ->
+  | BlockquoteStart(line)::LinesUntilBlockquoteEnds(continued, Lines.TrimBlankStart rest) ->
       let moreLines, rest =
         match rest with
         | Blockquote(lines, rest) -> lines, rest
@@ -615,7 +626,7 @@ let rec parseParagraphs (ctx:ParsingContext) lines = seq {
       yield CodeBlock(code |> String.concat ctx.Newline, langString, ignoredLine)
       yield! parseParagraphs ctx lines
   | Blockquote(body, Lines.TrimBlankStart rest) ->
-      yield QuotedBlock(parseParagraphs ctx body |> List.ofSeq)
+      yield QuotedBlock(parseParagraphs ctx (body @ [""]) |> List.ofSeq)
       yield! parseParagraphs ctx rest
   | EmacsTableBlock((headers, alignments, rows), Lines.TrimBlankStart rest)
   | PipeTableBlock((headers, alignments, rows), Lines.TrimBlankStart rest) ->

tests/Benchmarks/testfiles/mdtest-1.1/Links_reference_style.html

@@ -1,52 +1,28 @@
 <p>Foo <a href="/url/" title="Title">bar</a>.</p>
-
 <p>Foo <a href="/url/" title="Title">bar</a>.</p>
-
 <p>Foo <a href="/url/" title="Title">bar</a>.</p>
-
 <p>With <a href="/url/">embedded [brackets]</a>.</p>
-
 <p>Indented <a href="/url">once</a>.</p>
-
 <p>Indented <a href="/url">twice</a>.</p>
-
 <p>Indented <a href="/url">thrice</a>.</p>
-
 <p>Indented [four][] times.</p>
-
 <pre><code>[four]: /url
 </code></pre>
-
 <hr />
-
 <p><a href="foo">this</a> should work</p>
-
 <p>So should <a href="foo">this</a>.</p>
-
 <p>And <a href="foo">this</a>.</p>
-
 <p>And <a href="foo">this</a>.</p>
-
 <p>And <a href="foo">this</a>.</p>
-
 <p>But not [that] [].</p>
-
 <p>Nor [that][].</p>
-
 <p>Nor [that].</p>
-
 <p>[Something in brackets like <a href="foo">this</a> should work]</p>
-
 <p>[Same with <a href="foo">this</a>.]</p>
-
 <p>In this case, <a href="/somethingelse/">this</a> points to something else.</p>
-
 <p>Backslashing should suppress [this] and [this].</p>
-
 <hr />
-
 <p>Here's one where the <a href="/url/">link
 breaks</a> across lines.</p>
-
-<p>Here's another where the <a href="/url/">link 
+<p>Here's another where the <a href="/url/">link
 breaks</a> across lines, but with a line-ending space.</p>

tests/Benchmarks/testfiles/mdtest-1.1/Links_shortcut_references.html

@@ -1,9 +1,6 @@
 <p>This is the <a href="/simple">simple case</a>.</p>
-
 <p>This one has a <a href="/foo">line
 break</a>.</p>
-
-<p>This one has a <a href="/foo">line 
+<p>This one has a <a href="/foo">line
 break</a> with a line-ending space.</p>
-
 <p><a href="/that">this</a> and the <a href="/other">other</a></p>

tests/Benchmarks/testfiles/php-markdown/Code block in a list item.html

@@ -1,10 +1,9 @@
 <ul>
-<li><p>List Item:</p>
-
+<li>
+<p>List Item:</p>
 <pre><code>code block
 
 with a blank line
 </code></pre>
-
 <p>within a list item.</p></li>
 </ul>

tests/Benchmarks/testfiles/php-markdown/PHP-Specific Bugs.html

@@ -1,17 +1,11 @@
-<p>This tests for a bug where quotes escaped by PHP when using 
+<p>This tests for a bug where quotes escaped by PHP when using
 <code>preg_replace</code> with the <code>/e</code> modifier must be correctly unescaped
 (hence the <code>_UnslashQuotes</code> function found only in PHP Markdown).</p>
-
 <p>Headers below should appear exactly as they are typed (no backslash
 added or removed).</p>
-
 <h1>Header "quoted\" again \""</h1>
-
 <h2>Header "quoted\" again \""</h2>
-
 <h3>Header "quoted\" again \""</h3>
-
 <p>Test with tabs for <code>_Detab</code>:</p>
-
-<pre><code>Code    'block' with    some    "tabs"  and "quotes"
+<pre><code>Code	'block'	with	some	"tabs"	and	"quotes"
 </code></pre>

tests/FSharp.Literate.Tests/Tests.fs

@@ -217,7 +217,7 @@ let ``Correctly handles Norwegian letters in SQL code block (#249)`` () =
     Æøå"""
   let doc = Literate.ParseMarkdownString(content, formatAgent=getFormatAgent())
   let html = Literate.WriteHtml(doc)
-  html |> should contain ">Æøå<"
+  html |> should contain (sprintf ">Æøå%s<" System.Environment.NewLine)
 
 [<Test>]
 let ``Correctly handles code starting with whitespace`` () =
@@ -226,7 +226,7 @@ let ``Correctly handles code starting with whitespace`` () =
       inner"""
   let doc = Literate.ParseMarkdownString(content, formatAgent=getFormatAgent())
   let html = Literate.WriteHtml(doc)
-  html |> should contain ">  inner<"
+  html |> should contain ">  inner"
 
 [<Test>]
 let ``Correctly handles code which garbage after commands`` () =
@@ -236,16 +236,7 @@ let ``Correctly handles code which garbage after commands`` () =
       inner"""
   let doc = Literate.ParseMarkdownString(content, formatAgent=getFormatAgent())
   let html = Literate.WriteHtml(doc)
-  html |> should contain ">  inner<"
-
-[<Test>]
-let ``Correctly handles code after the commands`` () =
-  // This will work, but trigger a warning!
-  let content = """
-    [lang=unknown] let this be some code"""
-  let doc = Literate.ParseMarkdownString(content, formatAgent=getFormatAgent())
-  let html = Literate.WriteHtml(doc)
-  html |> should contain " let this be some code"
+  html |> should contain ">  inner"
 
 [<Test>]
 let ``Correctly handles apostrophes in JS code block (#213)`` () =