Follow CommonMark spec.

FSharp.Formatting.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 2013
-VisualStudioVersion = 12.0.40629.0
+# Visual Studio 14
+VisualStudioVersion = 14.0.23107.0
 MinimumVisualStudioVersion = 12.0.31101.0
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "project", "project", "{194BD478-0DB5-44F3-A6C2-1FC75D3F3294}"
 	ProjectSection(SolutionItems) = preProject

RELEASE_NOTES.md

@@ -1,3 +1,10 @@
+## 2.12.2 (30 December, 2015)
+ - Be compatible with the common-mark spec for 'Fenced code blocks' and 'Indented code blocks'.
+   See https://github.com/tpetricek/FSharp.Formatting/pull/343.
+   Please follow-up by adding support for more sections of the spec!
+   Just add the section to https://github.com/tpetricek/FSharp.Formatting/blob/master/tests/FSharp.Markdown.Tests/CommonMarkSpecTest.fs#L20
+   and fix the newly enabled tests.
+
 ## 2.12.1 (24 December, 2015)
  - update dependencies
  - Upgrade the CommandTool to F# 4 and bundle FSharp.Core with sigdata and optdata.

build.fsx

@@ -280,22 +280,38 @@ let commandToolStartInfo workingDirectory environmentVars args =
         setVar "FSI" fsiPath)
 
 /// Run the given buildscript with FAKE.exe
-let executeCommandToolWithOutput workingDirectory envArgs args =
+let executeWithOutput configStartInfo =
     let exitCode =
         ExecProcessWithLambdas
-            (commandToolStartInfo workingDirectory envArgs args)
+            configStartInfo
             TimeSpan.MaxValue false ignore ignore
     System.Threading.Thread.Sleep 1000
     exitCode
 
-// Documentation
-let buildDocumentationCommandTool args =
-    trace (sprintf "Building documentation (CommandTool), this could take some time, please wait...")
-    let exit = executeCommandToolWithOutput "." [] args
+let executeWithRedirect errorF messageF configStartInfo =
+    let exitCode =
+        ExecProcessWithLambdas
+            configStartInfo
+            TimeSpan.MaxValue true errorF messageF
+    System.Threading.Thread.Sleep 1000
+    exitCode
+
+let executeHelper executer traceMsg failMessage configStartInfo =
+    trace traceMsg
+    let exit = executer configStartInfo
     if exit <> 0 then
-        failwith "generating documentation failed"
+        failwith failMessage
     ()
 
+let execute = executeHelper executeWithOutput
+
+// Documentation
+let buildDocumentationCommandTool args =
+  execute
+    "Building documentation (CommandTool), this could take some time, please wait..."
+    "generating documentation failed"
+    (commandToolStartInfo "." [] args)
+
 let createArg argName arguments =
     (arguments : string seq)
     |> fun files -> String.Join("\" \"", files)
@@ -334,22 +350,13 @@ let commandToolLiterateArgument inDir outDir layoutRoots parameters =
 
     sprintf "literate --processDirectory %s %s %s %s" inDirArg outDirArg layoutRootsArgs replacementsArgs
 
-/// Run the given buildscript with FAKE.exe
-let executeFAKEWithOutput workingDirectory script fsiargs envArgs =
-    let exitCode =
-        ExecProcessWithLambdas
-            (fakeStartInfo script workingDirectory "" fsiargs envArgs)
-            TimeSpan.MaxValue false ignore ignore
-    System.Threading.Thread.Sleep 1000
-    exitCode
-
 // Documentation
 let buildDocumentationTarget fsiargs target =
-    trace (sprintf "Building documentation (%s), this could take some time, please wait..." target)
-    let exit = executeFAKEWithOutput "docs/tools" "generate.fsx" fsiargs ["target", target]
-    if exit <> 0 then
-        failwith "generating reference documentation failed"
-    ()
+    execute
+      (sprintf "Building documentation (%s), this could take some time, please wait..." target)
+      "generating reference documentation failed"
+      (fakeStartInfo "generate.fsx" "docs/tools" "" fsiargs ["target", target])
+
 
 let bootStrapDocumentationFiles () =
     // This is needed to bootstrap ourself (make sure we have the same environment while building as our users) ...
@@ -446,6 +453,54 @@ Target "Release" DoNothing
 
 Target "All" DoNothing
 
+#r "System.IO.Compression"
+#r "System.IO.Compression.FileSystem"
+Target "DownloadPython" (fun _ ->
+  if not isUnix then
+    let w = new System.Net.WebClient()
+    let zipFile = "temp"@@"cpython.zip"
+    if File.Exists zipFile then File.Delete zipFile
+    w.DownloadFile("https://www.python.org/ftp/python/3.5.1/python-3.5.1-embed-amd64.zip", zipFile)
+    let cpython = "temp"@@"CPython"
+    CleanDir cpython
+    System.IO.Compression.ZipFile.ExtractToDirectory(zipFile, cpython)
+    let cpythonStdLib = cpython@@"stdlib"
+    CleanDir cpythonStdLib
+    System.IO.Compression.ZipFile.ExtractToDirectory(cpython@@"python35.zip", cpythonStdLib)
+)
+
+Target "CreateTestJson" (fun _ ->
+    let targetPath = "temp/CommonMark"
+    CleanDir targetPath
+    Git.Repository.clone targetPath "https://github.com/jgm/CommonMark.git" "."
+
+    let pythonExe, stdLib =
+      if not isUnix then
+        System.IO.Path.GetFullPath ("temp"@@"CPython"@@"python.exe"),
+        System.IO.Path.GetFullPath ("temp"@@"CPython"@@"stdlib")
+      else "python", ""
+
+    let resultFile = "temp"@@"commonmark-tests.json"
+    if File.Exists resultFile then File.Delete resultFile
+    ( use fileStream = new StreamWriter(File.Open(resultFile, System.IO.FileMode.Create))
+      executeHelper
+        (executeWithRedirect traceError fileStream.WriteLine)
+        "Creating test json file, this could take some time, please wait..."
+        "generating documentation failed"
+        (fun info ->
+            info.FileName <- pythonExe
+            info.Arguments <- "test/spec_tests.py --dump-tests"
+            info.WorkingDirectory <- targetPath
+            let setVar k v =
+                info.EnvironmentVariables.[k] <- v
+            if not isUnix then
+              setVar "PYTHONPATH" stdLib
+            setVar "MSBuild" msBuildExe
+            setVar "GIT" Git.CommandHelper.gitPath
+            setVar "FSI" fsiPath))
+    File.Copy(resultFile, "tests"@@"commonmark_spec.json")
+)
+
 "Clean" ==> "AssemblyInfo" ==> "Build" ==> "BuildTests"
 "Build" ==> "MergeVSPowerTools" ==> "All"
 "BuildTests" ==> "RunTests" ==> "All"
@@ -461,4 +516,6 @@ Target "All" DoNothing
   ==> "CreateTag"
   ==> "Release"
 
+"DownloadPython" ==> "CreateTestJson"
+
 RunTargetOrDefault "All"

docs/content/development.md

@@ -0,0 +1,23 @@
+F# Formatting: Development Topics
+==================================
+
+Improve Common-Mark Test Coverage
+----------------
+See https://github.com/tpetricek/FSharp.Formatting/pull/343
+
+Update Common-Mark Test Suite
+----------------
+
+    ./build.sh CreateTestJson
+    git add -A
+    git commit -m "Update Common-Mark test suite"
+
+More information
+----------------
+
+The project is hosted on [GitHub](https://github.com/tpetricek/FSharp.Formatting) where you can 
+[report issues](https://github.com/tpetricek/FSharp.Formatting/issues), fork the project and submit pull requests.
+Thanks to [Gustavo Guerra](https://github.com/ovatsus) for a great build script and 
+[Steffen Forkmann](https://github.com/forki) for the great build tool [FAKE](https://github.com/fsharp/FAKE).
+The library is available under Apache 2.0. For more information see the 
+[License file](https://github.com/tpetricek/FSharp.Formatting/blob/master/LICENSE.md) in the GitHub repository.

paket.dependencies

@@ -1,6 +1,7 @@
 source http://nuget.org/api/v2
 redirects: on
 
+nuget FSharp.Data
 nuget FAKE
 nuget CommandLineParser
 nuget FSharp.Core

paket.lock

@@ -6,6 +6,8 @@ NUGET
     FAKE (4.11.3)
     FSharp.Compiler.Service (1.4.2)
     FSharp.Core (4.0.0.1)
+    FSharp.Data (2.2.5)
+      Zlib.Portable (>= 1.10.0) - framework: portable-net40+sl50+wp80+win80
     FSharpVSPowerTools.Core (2.2.0)
       FSharp.Compiler.Service (>= 1.4.2.0)
     ILRepack (2.0.9)
@@ -16,6 +18,7 @@ NUGET
     RazorEngine (3.7.6)
       Microsoft.AspNet.Razor (>= 3.0.0) - framework: >= net45
       Microsoft.AspNet.Razor (2.0.30506) - framework: net40
+    Zlib.Portable (1.11.0) - framework: portable-net40+sl50+wp80+win80
 GITHUB
   remote: matthid/Yaaf.FSharp.Scripting
   specs:

src/Common/StringParsing.fs

@@ -34,8 +34,9 @@ module String =
   /// Returns a string trimmed from the start together with 
   /// the number of skipped whitespace characters
   let (|TrimStartAndCount|) (text:string) = 
-    let trimmed = text.TrimStart()
-    text.Length - trimmed.Length, trimmed
+    let trimmed = text.TrimStart([|' '; '\t'|])
+    let len = text.Length - trimmed.Length
+    len, text.Substring(0, len).Replace("\t", "    ").Length, trimmed
 
   /// Matches when a string starts with any of the specified sub-strings
   let (|StartsWithAny|_|) (starts:seq<string>) (text:string) = 
@@ -48,6 +49,22 @@ module String =
   let (|StartsWithTrim|_|) (start:string) (text:string) = 
     if text.StartsWith(start) then Some(text.Substring(start.Length).Trim()) else None
 
+  /// Matches when a string starts with the specified sub-string (ignoring whitespace at the start)
+  /// The matched string is trimmed from all whitespace.
+  let (|StartsWithNTimesTrimIgnoreStartWhitespace|_|) (start:string) (text:string) =
+    if text.Contains(start) then
+      let beforeStart = text.Substring(0, text.IndexOf(start))
+      if String.IsNullOrWhiteSpace (beforeStart) then
+        let startAndRest = text.Substring(beforeStart.Length)
+        let startNum =
+          Seq.windowed start.Length startAndRest
+          |> Seq.map (fun chars -> System.String(chars))
+          |> Seq.takeWhile ((=) start)
+          |> Seq.length
+        Some(startNum, beforeStart.Length, text.Substring(beforeStart.Length + (start.Length * startNum)).Trim())
+      else None
+    else None
+
   /// Matches when a string starts with the given value and ends 
   /// with a given value (and returns the rest of it)
   let (|StartsAndEndsWith|_|) (starts, ends) (s:string) =
@@ -151,6 +168,13 @@ module List =
   /// using the specified delimiter. Returns a wrapped list and the rest.
   let inline (|Delimited|_|) str = (|DelimitedWith|_|) str str
 
+  let inline (|DelimitedNTimes|_|) str input =
+    let strs, items = List.partitionWhile (fun i -> i = str) input
+    match strs with
+    | h :: _ ->
+      (|Delimited|_|) (List.init strs.Length (fun _ -> str)) input
+    | _ -> None
+
   /// Matches a list if it starts with a bracketed list. Nested brackets
   /// are skipped (by counting opening and closing brackets) and can be 
   /// escaped using the '\' symbol.
@@ -185,9 +209,40 @@ module Lines =
     | matching, rest when matching <> [] -> Some(matching, rest)
     | _ -> None
 
+  /// Matches when there are some lines at the beginning that are
+  /// either empty (or whitespace) or start with at least 4 spaces (a tab counts as 4 spaces here).
+  /// Returns all such lines from the beginning until a different line and
+  /// the number of spaces the first line started with.
+  let (|TakeCodeBlock|_|) (input:string list) =
+    let spaceNum = 4
+      //match input with
+      //| h :: _ ->
+      //  let head = (input |> List.head).Replace("\t", "    ") |> Seq.toList
+      //  let spaces, _ = List.partitionWhile (fun s -> s = ' ') head
+      //  spaces.Length
+      //| _ -> 0
+    let startsWithSpaces (s:string) =
+      let normalized = s.Replace("\t", "    ")
+      normalized.Length >= spaceNum &&
+        normalized.Substring(0, spaceNum) = System.String(' ', spaceNum)
+    match List.partitionWhile (fun s ->
+            String.IsNullOrWhiteSpace s || startsWithSpaces s) input with
+    | matching, rest when matching <> [] && spaceNum >= 4 ->
+      Some(spaceNum, matching, rest)
+    | _ -> None
+
   /// Removes whitespace lines from the beginning of the list
   let (|TrimBlankStart|) = List.skipWhile (String.IsNullOrWhiteSpace)
 
+  /// Trims all lines of the current paragraph
+  let (|TrimParagraphLines|) lines =
+    lines
+    // first remove all whitespace on the beginning of the line
+    |> List.map (fun (s:string) -> s.TrimStart())
+    // Now remove all additional spaces at the end, but keep two spaces if existent
+    |> List.map (fun s ->
+      let endsWithTwoSpaces = s.EndsWith("  ")
+      s.TrimEnd([|' '|]) + if endsWithTwoSpaces then "  " else "")
 
 /// Parameterized pattern that assigns the specified value to the 
 /// first component of a tuple. Usage:

src/FSharp.Literate/Contexts.fs

@@ -1,34 +1,33 @@
 namespace FSharp.Literate
+
 open FSharp.CodeFormat
 
 /// Specifies a context that is passed to functions 
 /// that need to use the F# compiler
 type CompilerContext = 
   { // An instance of the F# code formatting agent
-    FormatAgent : CodeFormatAgent 
+    FormatAgent : CodeFormatAgent
     // F# interactive evaluator
     Evaluator : IFsiEvaluator option
     // Command line options for the F# compiler
     CompilerOptions : string option
     // Defined symbols for the F# compiler
     DefinedSymbols : string option }
 
-
 /// Defines the two possible output types from literate script: HTML and LaTeX.
 [<RequireQualifiedAccess>]
 type OutputKind = Html | Latex
 
-
 /// Specifies a context that is passed to functions that generate the output
 type ProcessingContext = 
   { // Path to the template file
     TemplateFile : string option
     // Short prefix code added to all HTML 'id' elements
-    Prefix : string 
+    Prefix : string
     // Additional replacements to be made in the template file
-    Replacements : list<string * string> 
+    Replacements : list<string * string>
     // Generate line numbers for F# snippets?
-    GenerateLineNumbers : bool 
+    GenerateLineNumbers : bool
     // Include the source file in the generated output as '{source}'
     IncludeSource : bool
     // Auto-generate anchors for headers