Merge pull request #874 from nhirschey/ipynb-input

Ipynb input

.config/dotnet-tools.json

@@ -13,6 +13,12 @@
       "commands": [
         "fsharp-analyzers"
       ]
+    },
+    "dotnet-repl": {
+      "version": "0.1.208",
+      "commands": [
+        "dotnet-repl"
+      ]
     }
   }
 }

Directory.Packages.props

@@ -18,6 +18,7 @@
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="Suave" Version="2.6.2" />
     <PackageVersion Include="System.Memory" Version="4.5.5" />
+    <PackageVersion Include="System.Text.Json" Version="8.0.0" />
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.7.0" />
     <PackageVersion Include="NUnit" Version="3.14.0" />
     <PackageVersion Include="FsUnit" Version="5.6.0" />

RELEASE_NOTES.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 20.0.0-alpha-014 - 2023-11-22
+
+### Added
+* Added the ability to use ipynb files as inputs [#874](https://github.com/fsprojects/FSharp.Formatting/pull/874)
+
+### Fixed
+* Fsx outputs no longer treat inline html as F# code. Inline html blocks are now enclosed inside literate comments. 
+
 ## 20.0.0-alpha-013 - 2023-11-21
 
 ### Added

docs/apidocs.fsx

@@ -3,7 +3,7 @@
 title: Generating API Docs
 category: Documentation
 categoryindex: 1
-index: 5
+index: 6
 ---
 *)
 (*** condition: prepare ***)

docs/content.fsx

@@ -2,7 +2,7 @@
 ---
 category: Documentation
 categoryindex: 1
-index: 3
+index: 5
 ---
 *)
 (*** condition: prepare ***)

docs/evaluation.fsx

@@ -2,7 +2,7 @@
 ---
 category: Documentation
 categoryindex: 1
-index: 6
+index: 7
 ---
 *)
 (*** condition: prepare ***)

docs/literate-notebook.ipynb

@@ -0,0 +1,129 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "dotnet_repl_cellExecutionStartTime": "2023-11-22T13:28:54.243692+00:00",
+    "dotnet_repl_cellExecutionEndTime": "2023-11-22T13:28:54.2640212+00:00"
+   },
+   "source": [
+    "---\n",
+    "category: Documentation\n",
+    "\n",
+    "categoryindex: 1\n",
+    "\n",
+    "index: 4\n",
+    "\n",
+    "---\n",
+    "\n",
+    "# Literate Notebooks\n",
+    "\n",
+    "Content may be created using  [.NET interactive](https://github.com/dotnet/interactive/tree/main) polyglot notebooks as the input file. Notebooks are processed by converting the notebook to a literate `.fsx` script and then passing the script through the script processing pipeline. Markdown notebook cells are passed through as comments surrounded by `(**` and `*)`, F# code cells are passed through as code, and non-F# code is passed through as markdown fenced code blocks between `(**` and `*)` comment markers. \n",
+    "\n",
+    "The `fsdocs` tool uses [dotnet-repl](https://github.com/jonsequitur/dotnet-repl) to evaluate polyglot notebooks. You need this tool to evaluate notebooks using `dotnet fsdocs --eval`. It can be installed into your local tool manifest using the command `dotnet tool install dotnet-repl`.\n",
+    "\n",
+    "F# Formatting tries to faithfully reproduce a notebook's native appearance when generating documents. Notebook cell outputs are passed through unchanged to preserve the notebook's html output. The below snippet demonstrates a notebook's html output for F# records, which differs from the output you would get with the same code inside a literate scripts.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "dotnet_repl_cellExecutionStartTime": "2023-11-22T13:28:54.265034+00:00",
+    "dotnet_repl_cellExecutionEndTime": "2023-11-22T13:28:56.2484876+00:00",
+    "dotnet_interactive": {
+     "language": "fsharp"
+    },
+    "polyglot_notebook": {
+     "kernelName": "fsharp"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<details open=\"open\" class=\"dni-treeview\"><summary><span class=\"dni-code-hint\"><code>{ Name = &quot;Alf&quot;\\n  Phone = &quot;(555) 555-5555&quot;\\n  ZipCode = &quot;90210&quot; }</code></span></summary><div><table><thead><tr></tr></thead><tbody><tr><td>Name</td><td><div class=\"dni-plaintext\"><pre>&quot;Alf&quot;\r\n",
+       "</pre></div></td></tr><tr><td>Phone</td><td><div class=\"dni-plaintext\"><pre>&quot;(555) 555-5555&quot;\r\n",
+       "</pre></div></td></tr><tr><td>ZipCode</td><td><div class=\"dni-plaintext\"><pre>&quot;90210&quot;\r\n",
+       "</pre></div></td></tr></tbody></table></div></details><style>\r\n",
+       ".dni-code-hint {\r\n",
+       "    font-style: italic;\r\n",
+       "    overflow: hidden;\r\n",
+       "    white-space: nowrap;\r\n",
+       "}\r\n",
+       ".dni-treeview {\r\n",
+       "    white-space: nowrap;\r\n",
+       "}\r\n",
+       ".dni-treeview td {\r\n",
+       "    vertical-align: top;\r\n",
+       "    text-align: start;\r\n",
+       "}\r\n",
+       "details.dni-treeview {\r\n",
+       "    padding-left: 1em;\r\n",
+       "}\r\n",
+       "table td {\r\n",
+       "    text-align: start;\r\n",
+       "}\r\n",
+       "table tr { \r\n",
+       "    vertical-align: top; \r\n",
+       "    margin: 0em 0px;\r\n",
+       "}\r\n",
+       "table tr td pre \r\n",
+       "{ \r\n",
+       "    vertical-align: top !important; \r\n",
+       "    margin: 0em 0px !important;\r\n",
+       "} \r\n",
+       "table th {\r\n",
+       "    text-align: start;\r\n",
+       "}\r\n",
+       "</style>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "type ContactCard =\n",
+    "    { Name: string\n",
+    "      Phone: string\n",
+    "      ZipCode: string }\n",
+    "\n",
+    "// Create a new record\n",
+    "{ Name = \"Alf\"; Phone = \"(555) 555-5555\"; ZipCode = \"90210\" }"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".NET (F#)",
+   "language": "F#",
+   "name": ".net-fsharp"
+  },
+  "language_info": {
+   "file_extension": ".fs",
+   "mimetype": "text/x-fsharp",
+   "name": "F#",
+   "pygments_lexer": "fsharp",
+   "version": "6.0"
+  },
+  "polyglot_notebook": {
+   "defaultKernelName": "fsharp",
+   "items": [
+    {
+     "name": "fsharp"
+    }
+   ]
+  },
+  "dotnet_interactive": {
+   "defaultKernelName": "fsharp",
+   "items": [
+    {
+     "name": "fsharp"
+    }
+   ]
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/styling.md

@@ -1,7 +1,7 @@
 ---
 category: Documentation
 categoryindex: 1
-index: 7
+index: 8
 ---
 
 # Customization and Styling

docs/users.md

@@ -1,7 +1,7 @@
 ---
 category: Documentation
 categoryindex: 1
-index: 8
+index: 9
 ---
 # Users of FSharp.Formatting
 

docs/zero-to-hero.md

@@ -1,7 +1,7 @@
 ---
 category: Documentation
 categoryindex: 1
-index: 9
+index: 10
 ---
 # From zero to hero: deploying to GitHub Pages
 

src/FSharp.Formatting.Literate/FSharp.Formatting.Literate.fsproj

@@ -16,6 +16,7 @@
     <Compile Include="Contexts.fs" />
     <Compile Include="ParseScript.fs" />
     <Compile Include="ParseMarkdown.fs" />
+    <Compile Include="ParsePynb.fs" />
     <Compile Include="Transformations.fs" />
     <Compile Include="Formatting.fs" />
     <Compile Include="Literate.fs" />
@@ -26,7 +27,6 @@
     <ProjectReference Include="..\FSharp.Formatting.Common\FSharp.Formatting.Common.fsproj" PrivateAssets="all" />
     <ProjectReference Include="..\FSharp.Formatting.Markdown\FSharp.Formatting.Markdown.fsproj" PrivateAssets="all" />
   </ItemGroup>
-
   <ItemGroup>
     <!-- ugly hack: inline p2p libraries in NuGet package
          workaround for https://github.com/NuGet/Home/issues/3891 -->
@@ -38,5 +38,6 @@
   <ItemGroup>
     <PackageReference Include="FSharp.Core" />
     <PackageReference Include="FSharp.Compiler.Service" />
+    <PackageReference Include="System.Text.Json" />
   </ItemGroup>
-</Project>
+</Project>

src/FSharp.Formatting.Literate/Literate.fs

@@ -223,6 +223,45 @@ type Literate private () =
         |> Transformations.formatCodeSnippets filePath ctx
         |> Transformations.evaluateCodeSnippets ctx
 
+    /// <summary>
+    /// Parse pynb string as literate document
+    /// </summary>
+    /// <param name="content"></param>
+    /// <param name="path">optional file path for debugging purposes</param>
+    /// <param name="definedSymbols"></param>
+    /// <param name="references"></param>
+    /// <param name="parseOptions">Defaults to MarkdownParseOptions.AllowYamlFrontMatter</param>
+    /// <param name="rootInputFolder"></param>
+    /// <param name="onError"></param>
+    static member ParsePynbString
+        (
+            content,
+            ?path,
+            ?definedSymbols,
+            ?references,
+            ?parseOptions,
+            ?rootInputFolder,
+            ?onError
+        ) =
+        let onError = defaultArg onError ignore
+        let ctx = parsingContext None None definedSymbols onError
+
+        let filePath =
+            match path with
+            | Some s -> s
+            | None ->
+                match rootInputFolder with
+                | None -> "C:\\script.fsx"
+                | Some r -> Path.Combine(r, "script.fsx")
+
+        let content = ParsePynb.pynbStringToFsx content
+
+        ParseScript(parseOptions, ctx)
+            .ParseAndCheckScriptFile(filePath, content, rootInputFolder, onError)
+        |> Transformations.generateReferences references
+        |> Transformations.formatCodeSnippets filePath ctx
+        |> Transformations.evaluateCodeSnippets ctx
+
     // ------------------------------------------------------------------------------------
     // Simple writing functions
     // ------------------------------------------------------------------------------------
@@ -513,6 +552,59 @@ type Literate private () =
         let docModel = Formatting.transformDocument filesWithFrontMatter doc output ctx
         docModel
 
+    /// Parse and transform a pynb document
+    static member internal ParseAndTransformPynbFile
+        (
+            input,
+            output,
+            outputKind,
+            prefix,
+            fscOptions,
+            lineNumbers,
+            references,
+            substitutions,
+            generateAnchors,
+            imageSaver,
+            rootInputFolder,
+            crefResolver,
+            mdlinkResolver,
+            onError,
+            filesWithFrontMatter: FrontMatterFile array
+        ) =
+
+        let parseOptions =
+            match outputKind with
+            | OutputKind.Fsx
+            | OutputKind.Pynb -> (MarkdownParseOptions.ParseCodeAsOther)
+            | _ -> MarkdownParseOptions.None
+
+        let fsx = ParsePynb.pynbToFsx input
+
+        let doc =
+            Literate.ParseScriptString(
+                fsx,
+                ?fscOptions = fscOptions,
+                ?references = references,
+                parseOptions = parseOptions,
+                ?rootInputFolder = rootInputFolder,
+                ?onError = onError
+            )
+
+        let ctx =
+            makeFormattingContext
+                outputKind
+                prefix
+                lineNumbers
+                generateAnchors
+                substitutions
+                crefResolver
+                mdlinkResolver
+                None
+
+        let doc = downloadImagesForDoc imageSaver doc
+        let docModel = Formatting.transformDocument filesWithFrontMatter doc output ctx
+        docModel
+
     /// Convert a markdown file into HTML or another output kind
     static member ConvertMarkdownFile
         (