diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index ddac3e9a5..1e4893205 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -13,6 +13,12 @@ "commands": [ "fsharp-analyzers" ] + }, + "dotnet-repl": { + "version": "0.1.208", + "commands": [ + "dotnet-repl" + ] } } } \ No newline at end of file diff --git a/Directory.Packages.props b/Directory.Packages.props index b152a5b61..dd72d1cd3 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -18,6 +18,7 @@ + diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 8874e4625..ec6ca06b3 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,13 @@ # Changelog +## 20.0.0-alpha-014 - 2023-11-22 + +### Added +* Added the ability to use ipynb files as inputs [#874](https://github.com/fsprojects/FSharp.Formatting/pull/874) + +### Fixed +* Fsx outputs no longer treat inline html as F# code. Inline html blocks are now enclosed inside literate comments. + ## 20.0.0-alpha-013 - 2023-11-21 ### Added diff --git a/docs/apidocs.fsx b/docs/apidocs.fsx index 5c3a1eae1..c2ea469d4 100644 --- a/docs/apidocs.fsx +++ b/docs/apidocs.fsx @@ -3,7 +3,7 @@ title: Generating API Docs category: Documentation categoryindex: 1 -index: 5 +index: 6 --- *) (*** condition: prepare ***) diff --git a/docs/content.fsx b/docs/content.fsx index e04a0e735..b686c6566 100644 --- a/docs/content.fsx +++ b/docs/content.fsx @@ -2,7 +2,7 @@ --- category: Documentation categoryindex: 1 -index: 3 +index: 5 --- *) (*** condition: prepare ***) diff --git a/docs/evaluation.fsx b/docs/evaluation.fsx index 99c603def..1fe4fbb5d 100644 --- a/docs/evaluation.fsx +++ b/docs/evaluation.fsx @@ -2,7 +2,7 @@ --- category: Documentation categoryindex: 1 -index: 6 +index: 7 --- *) (*** condition: prepare ***) diff --git a/docs/literate-notebook.ipynb b/docs/literate-notebook.ipynb new file mode 100644 index 000000000..143a5a83a --- /dev/null +++ b/docs/literate-notebook.ipynb @@ -0,0 +1,129 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "dotnet_repl_cellExecutionStartTime": "2023-11-22T13:28:54.243692+00:00", + "dotnet_repl_cellExecutionEndTime": "2023-11-22T13:28:54.2640212+00:00" + }, + "source": [ + "---\n", + "category: Documentation\n", + "\n", + "categoryindex: 1\n", + "\n", + "index: 4\n", + "\n", + "---\n", + "\n", + "# Literate Notebooks\n", + "\n", + "Content may be created using [.NET interactive](https://github.com/dotnet/interactive/tree/main) polyglot notebooks as the input file. Notebooks are processed by converting the notebook to a literate `.fsx` script and then passing the script through the script processing pipeline. Markdown notebook cells are passed through as comments surrounded by `(**` and `*)`, F# code cells are passed through as code, and non-F# code is passed through as markdown fenced code blocks between `(**` and `*)` comment markers. \n", + "\n", + "The `fsdocs` tool uses [dotnet-repl](https://github.com/jonsequitur/dotnet-repl) to evaluate polyglot notebooks. You need this tool to evaluate notebooks using `dotnet fsdocs --eval`. It can be installed into your local tool manifest using the command `dotnet tool install dotnet-repl`.\n", + "\n", + "F# Formatting tries to faithfully reproduce a notebook's native appearance when generating documents. Notebook cell outputs are passed through unchanged to preserve the notebook's html output. The below snippet demonstrates a notebook's html output for F# records, which differs from the output you would get with the same code inside a literate scripts.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_repl_cellExecutionStartTime": "2023-11-22T13:28:54.265034+00:00", + "dotnet_repl_cellExecutionEndTime": "2023-11-22T13:28:56.2484876+00:00", + "dotnet_interactive": { + "language": "fsharp" + }, + "polyglot_notebook": { + "kernelName": "fsharp" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
{ Name = "Alf"\\n Phone = "(555) 555-5555"\\n ZipCode = "90210" }
Name
"Alf"\r\n",
+       "
Phone
"(555) 555-5555"\r\n",
+       "
ZipCode
"90210"\r\n",
+       "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "type ContactCard =\n", + " { Name: string\n", + " Phone: string\n", + " ZipCode: string }\n", + "\n", + "// Create a new record\n", + "{ Name = \"Alf\"; Phone = \"(555) 555-5555\"; ZipCode = \"90210\" }" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".NET (F#)", + "language": "F#", + "name": ".net-fsharp" + }, + "language_info": { + "file_extension": ".fs", + "mimetype": "text/x-fsharp", + "name": "F#", + "pygments_lexer": "fsharp", + "version": "6.0" + }, + "polyglot_notebook": { + "defaultKernelName": "fsharp", + "items": [ + { + "name": "fsharp" + } + ] + }, + "dotnet_interactive": { + "defaultKernelName": "fsharp", + "items": [ + { + "name": "fsharp" + } + ] + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/docs/styling.md b/docs/styling.md index 738c71e42..d76e1f0d3 100644 --- a/docs/styling.md +++ b/docs/styling.md @@ -1,7 +1,7 @@ --- category: Documentation categoryindex: 1 -index: 7 +index: 8 --- # Customization and Styling diff --git a/docs/users.md b/docs/users.md index b43dc684e..0185298e3 100644 --- a/docs/users.md +++ b/docs/users.md @@ -1,7 +1,7 @@ --- category: Documentation categoryindex: 1 -index: 8 +index: 9 --- # Users of FSharp.Formatting diff --git a/docs/zero-to-hero.md b/docs/zero-to-hero.md index 2234a3407..aa58e9428 100644 --- a/docs/zero-to-hero.md +++ b/docs/zero-to-hero.md @@ -1,7 +1,7 @@ --- category: Documentation categoryindex: 1 -index: 9 +index: 10 --- # From zero to hero: deploying to GitHub Pages diff --git a/src/FSharp.Formatting.Literate/FSharp.Formatting.Literate.fsproj b/src/FSharp.Formatting.Literate/FSharp.Formatting.Literate.fsproj index fda3ea260..25e193476 100644 --- a/src/FSharp.Formatting.Literate/FSharp.Formatting.Literate.fsproj +++ b/src/FSharp.Formatting.Literate/FSharp.Formatting.Literate.fsproj @@ -16,6 +16,7 @@ + @@ -26,7 +27,6 @@ - @@ -38,5 +38,6 @@ + - + \ No newline at end of file diff --git a/src/FSharp.Formatting.Literate/Literate.fs b/src/FSharp.Formatting.Literate/Literate.fs index f36ea46f1..3723adc47 100644 --- a/src/FSharp.Formatting.Literate/Literate.fs +++ b/src/FSharp.Formatting.Literate/Literate.fs @@ -223,6 +223,45 @@ type Literate private () = |> Transformations.formatCodeSnippets filePath ctx |> Transformations.evaluateCodeSnippets ctx + /// + /// Parse pynb string as literate document + /// + /// + /// optional file path for debugging purposes + /// + /// + /// Defaults to MarkdownParseOptions.AllowYamlFrontMatter + /// + /// + static member ParsePynbString + ( + content, + ?path, + ?definedSymbols, + ?references, + ?parseOptions, + ?rootInputFolder, + ?onError + ) = + let onError = defaultArg onError ignore + let ctx = parsingContext None None definedSymbols onError + + let filePath = + match path with + | Some s -> s + | None -> + match rootInputFolder with + | None -> "C:\\script.fsx" + | Some r -> Path.Combine(r, "script.fsx") + + let content = ParsePynb.pynbStringToFsx content + + ParseScript(parseOptions, ctx) + .ParseAndCheckScriptFile(filePath, content, rootInputFolder, onError) + |> Transformations.generateReferences references + |> Transformations.formatCodeSnippets filePath ctx + |> Transformations.evaluateCodeSnippets ctx + // ------------------------------------------------------------------------------------ // Simple writing functions // ------------------------------------------------------------------------------------ @@ -513,6 +552,59 @@ type Literate private () = let docModel = Formatting.transformDocument filesWithFrontMatter doc output ctx docModel + /// Parse and transform a pynb document + static member internal ParseAndTransformPynbFile + ( + input, + output, + outputKind, + prefix, + fscOptions, + lineNumbers, + references, + substitutions, + generateAnchors, + imageSaver, + rootInputFolder, + crefResolver, + mdlinkResolver, + onError, + filesWithFrontMatter: FrontMatterFile array + ) = + + let parseOptions = + match outputKind with + | OutputKind.Fsx + | OutputKind.Pynb -> (MarkdownParseOptions.ParseCodeAsOther) + | _ -> MarkdownParseOptions.None + + let fsx = ParsePynb.pynbToFsx input + + let doc = + Literate.ParseScriptString( + fsx, + ?fscOptions = fscOptions, + ?references = references, + parseOptions = parseOptions, + ?rootInputFolder = rootInputFolder, + ?onError = onError + ) + + let ctx = + makeFormattingContext + outputKind + prefix + lineNumbers + generateAnchors + substitutions + crefResolver + mdlinkResolver + None + + let doc = downloadImagesForDoc imageSaver doc + let docModel = Formatting.transformDocument filesWithFrontMatter doc output ctx + docModel + /// Convert a markdown file into HTML or another output kind static member ConvertMarkdownFile ( diff --git a/src/FSharp.Formatting.Literate/ParsePynb.fs b/src/FSharp.Formatting.Literate/ParsePynb.fs new file mode 100644 index 000000000..cb25d9695 --- /dev/null +++ b/src/FSharp.Formatting.Literate/ParsePynb.fs @@ -0,0 +1,189 @@ +namespace FSharp.Formatting.Literate + +open System.IO +open System.Text.Json +open FSharp.Formatting.Templating +open FSharp.Formatting.PynbModel + +module internal ParsePynb = + + type ParsedCell = + | Code of + {| lang: string + source: string + outputs: string[] option |} + | Markdown of source: string + + member this.ToMarkdown() = + match this with + | Markdown source -> source + | Code code -> + let codeBlock = sprintf $"```{code.lang}\n{addLineEnd code.source}```" + + match code.outputs with + | None -> codeBlock + | Some outputs -> + let outputsString = outputs |> String.concat "\n" + sprintf $"{codeBlock}\n{outputsString}" + + member this.ToFsx() = + match this with + | Markdown source -> $"(**\n{source}\n*)" + | Code code when code.lang = "fsharp" -> + let codeBlock = addLineEnd code.source + + match code.outputs with + | None -> codeBlock + | Some outputs -> + let outputsString = outputs |> String.concat "\n" + sprintf $"{codeBlock}\n(**\n{outputsString}\n*)" + | Code _ -> $"(**\n{this.ToMarkdown()}\n*)" + + module Output = + let (|TextHtml|_|) (x: JsonElement) = + match x.TryGetProperty("text/html") with + | true, html -> + let html = + html.EnumerateArray() + |> Seq.map (fun x -> x.GetString().Replace("\r\n", "\n") |> addLineEnd) + |> String.concat "" + + Some $"

{html}

" + | _ -> None + + let (|TextPlain|_|) (x: JsonElement) = + match x.TryGetProperty("text/plain") with + | true, text -> + let text = text.EnumerateArray() |> Seq.map (fun x -> x.GetString()) |> String.concat "" + Some $"""
{text}
""" + | _ -> None + + let (|DisplayData|_|) (x: JsonElement) = + match x.TryGetProperty("output_type") with + | true, outputType -> + if outputType.GetString() = "display_data" then + match x.TryGetProperty("data") with + | true, TextHtml html -> html + | true, TextPlain text -> text + | true, s -> failwith $"unknown output {s}" + | false, _ -> failwith "no data property" + |> Some + else + None + | _ -> failwith "no output_type property" + + let (|Stream|_|) (x: JsonElement) = + match x.TryGetProperty("output_type") with + | true, outputType -> + if outputType.GetString() = "stream" then + let text = + match x.TryGetProperty("text") with + | true, xs -> xs.EnumerateArray() |> Seq.map (fun x -> x.GetString()) |> String.concat "" + | _ -> failwith "no text property" + + Some + $"""
{text}
""" + else + None + | _ -> failwith "no output_type property" + + let parse (output: JsonElement) = + match output with + | Stream stream -> stream + | DisplayData displayData -> displayData + | s -> failwith $"""unknown output {s.GetProperty("output_type").GetString()}""" + + let getSource (cell: JsonElement) = + let source = + match cell.TryGetProperty("source") with + | true, xs -> xs.EnumerateArray() + | _ -> failwith "no source" + + source |> Seq.map (fun x -> x.GetString()) |> String.concat "" + + let collectOutputs (cell: JsonElement) = + match cell.TryGetProperty("outputs") with + | true, outputs -> + let xs = outputs.EnumerateArray() + + if Seq.isEmpty xs then + None + else + xs |> Seq.map Output.parse |> Seq.toArray |> Some + | _ -> None + + let getCode (cell: JsonElement) = + let lang = + let metadata (elem: JsonElement) = + match elem.TryGetProperty("metadata") with + | false, _ -> failwith "Code cell does not have metadata" + | true, metadata -> metadata + + let languageInfo (metadata: JsonElement) = + match metadata.TryGetProperty("polyglot_notebook") with + | false, _ -> failwith "code cell does not have metadata.polyglot_notebook" + | true, language_info -> language_info + + let kernelName (languageInfo: JsonElement) = + match languageInfo.TryGetProperty("kernelName") with + | false, _ -> failwith "code cell does not have metadata.polyglot_notebook.kernelName" + | true, name -> name.GetString() + + cell |> metadata |> languageInfo |> kernelName + + let source = getSource cell + let outputs = collectOutputs cell + + Code + {| lang = lang + source = source + outputs = outputs |} + + + let parseCell (cell: JsonElement) = + let cell_type = + match cell.TryGetProperty("cell_type") with + | true, cellType -> cellType.GetString() + | _ -> failwith "no cell type" + + match cell_type with + | "markdown" -> + match getSource cell, collectOutputs cell with + | _, Some _ -> failwith $"Markdown should not have outputs" + | source, None -> Markdown source + | "code" -> getCode cell + | _ -> failwith $"unknown cell type {cell_type}" + + let pynbStringToMarkdown (ipynb: string) = + let json = JsonDocument.Parse(ipynb) + + json.RootElement.GetProperty("cells").EnumerateArray() + |> Seq.map (parseCell >> (fun x -> x.ToMarkdown())) + |> String.concat "\n" + + let pynbToMarkdown ipynbFile = + ipynbFile |> File.ReadAllText |> pynbStringToMarkdown + + + let pynbStringToFsx (ipynb: string) = + let json = JsonDocument.Parse(ipynb) + + json.RootElement.GetProperty("cells").EnumerateArray() + |> Seq.map (parseCell >> (fun x -> x.ToFsx())) + |> String.concat "\n" + + let pynbToFsx ipynbFile = + ipynbFile |> File.ReadAllText |> pynbStringToFsx + + let parseFrontMatter ipynbFile = + let json = JsonDocument.Parse(ipynbFile |> File.ReadAllText) + + json.RootElement.GetProperty("cells").EnumerateArray() + |> Seq.map parseCell + |> Seq.choose (fun cell -> + match cell with + | Code _ -> None + | Markdown source -> + let lines = source.Split([| '\n'; '\r' |], System.StringSplitOptions.RemoveEmptyEntries) + FrontMatterFile.ParseFromLines ipynbFile lines) + |> Seq.tryHead diff --git a/src/FSharp.Formatting.Markdown/FsxFormatting.fs b/src/FSharp.Formatting.Markdown/FsxFormatting.fs index 5c4e243f0..86195707a 100644 --- a/src/FSharp.Formatting.Markdown/FsxFormatting.fs +++ b/src/FSharp.Formatting.Markdown/FsxFormatting.fs @@ -6,12 +6,43 @@ module internal FSharp.Formatting.Markdown.FsxFormatting open MarkdownUtils +/// Like MarkdownUtils.isCode, but does not treat InlineHtmlBlock as code +let isFsxCode = + (function + | CodeBlock _ -> true + | _ -> false) + +/// Like MarkdownUtils.splitParagraphs, but does not treat InlineHtmlBlock as code +let splitFsxParagraphs paragraphs = + let firstCode = paragraphs |> List.tryFindIndex isFsxCode + + match firstCode with + | Some 0 -> + let code = paragraphs.[0] + let codeLines = getCode code + let otherParagraphs = paragraphs.[1..] + + // Collect the code output(s) that follows this cell if any + let codeOutput = otherParagraphs |> List.takeWhile isCodeOutput |> List.map getCodeOutput + + let otherParagraphs = otherParagraphs |> List.skipWhile isCodeOutput + + Choice1Of2(codeLines, codeOutput, getExecutionCount code), otherParagraphs + + | Some _ + | None -> + let markdownParagraphs = paragraphs |> List.takeWhile (isFsxCode >> not) + + let otherParagraphs = paragraphs |> List.skipWhile (isFsxCode >> not) + + Choice2Of2 markdownParagraphs, otherParagraphs + /// Write a list of MarkdownParagraph values to a TextWriter let rec formatParagraphs ctx paragraphs = match paragraphs with | [] -> [] | _ -> - let k, otherParagraphs = splitParagraphs paragraphs + let k, otherParagraphs = splitFsxParagraphs paragraphs let cell = match k with diff --git a/src/fsdocs-tool/BuildCommand.fs b/src/fsdocs-tool/BuildCommand.fs index 5b5cd2aa6..c6da4d81c 100644 --- a/src/fsdocs-tool/BuildCommand.fs +++ b/src/fsdocs-tool/BuildCommand.fs @@ -74,10 +74,11 @@ type internal DocContent let inputFileName = Path.GetFileName(inputFileFullPath) let isFsx = inputFileFullPath.EndsWith(".fsx", true, CultureInfo.InvariantCulture) let isMd = inputFileFullPath.EndsWith(".md", true, CultureInfo.InvariantCulture) + let isPynb = inputFileFullPath.EndsWith(".ipynb", true, CultureInfo.InvariantCulture) let ext = outputKind.Extension let outputFileRelativeToRoot = - if isFsx || isMd then + if isFsx || isMd || isPynb then let basename = Path.GetFileNameWithoutExtension(inputFileFullPath) Path.Combine(outputFolderRelativeToRoot, sprintf "%s.%s" basename ext) @@ -185,9 +186,11 @@ type internal DocContent if name.StartsWith('.') then printfn "skipping file %s" inputFileFullPath elif not (name.StartsWith("_template", StringComparison.Ordinal)) then - let isFsx = inputFileFullPath.EndsWith(".fsx", true, CultureInfo.InvariantCulture) + let isFsx = inputFileFullPath.EndsWith(".fsx", StringComparison.OrdinalIgnoreCase) - let isMd = inputFileFullPath.EndsWith(".md", true, CultureInfo.InvariantCulture) + let isMd = inputFileFullPath.EndsWith(".md", StringComparison.OrdinalIgnoreCase) + + let isPynb = inputFileFullPath.EndsWith(".ipynb", StringComparison.OrdinalIgnoreCase) // A _template.tex or _template.pynb is needed to generate those files match outputKind, template with @@ -220,7 +223,7 @@ type internal DocContent let templateChangeTime = match template with - | Some t when isFsx || isMd -> + | Some t when isFsx || isMd || isPynb -> try let fi = FileInfo(t) let input = fi.Directory.Name @@ -344,6 +347,99 @@ type internal DocContent template, outputFileFullPath ))) + elif isPynb then + printfn " preparing %s --> %s" inputFileFullPath outputFileRelativeToRoot + + let evaluateNotebook ipynbFile = + let args = + $"repl --run {ipynbFile} --default-kernel fsharp --exit-after-run --output-path {ipynbFile}" + + let psi = + ProcessStartInfo( + fileName = "dotnet", + arguments = args, + UseShellExecute = false, + CreateNoWindow = true + ) + + try + let p = Process.Start(psi) + p.WaitForExit() + with _ -> + let msg = + $"Failed to evaluate notebook {ipynbFile} using dotnet-repl\n" + + $"""try running "{args}" at the command line and inspect the error""" + + failwith msg + + let checkDotnetReplInstall () = + let failmsg = + "'dotnet-repl' is not installed. Please install it using 'dotnet tool install dotnet-repl'" + + try + let psi = + ProcessStartInfo( + fileName = "dotnet", + arguments = "tool list --local", + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardOutput = true + ) + + let p = Process.Start(psi) + let ol = p.StandardOutput.ReadToEnd() + p.WaitForExit() + psi.Arguments <- "tool list --global" + p.Start() |> ignore + let og = p.StandardOutput.ReadToEnd() + let output = $"{ol}\n{og}" + + if not (output.Contains("dotnet-repl")) then + failwith failmsg + + p.WaitForExit() + with _ -> + failwith failmsg + + if evaluate then + checkDotnetReplInstall () + printfn $" evaluating {inputFileFullPath} with dotnet-repl" + evaluateNotebook inputFileFullPath + + + let model = + Literate.ParseAndTransformPynbFile( + inputFileFullPath, + output = outputFileRelativeToRoot, + outputKind = outputKind, + prefix = None, + fscOptions = None, + lineNumbers = lineNumbers, + references = Some false, + substitutions = substitutions, + generateAnchors = Some true, + imageSaver = imageSaverOpt, + rootInputFolder = rootInputFolder, + crefResolver = crefResolver, + mdlinkResolver = mdlinkResolver, + onError = Some onError, + filesWithFrontMatter = filesWithFrontMatter + ) + + yield + ((if mainRun then + Some(inputFileFullPath, isOtherLang, model) + else + None), + (fun p -> + printfn " writing %s --> %s" inputFileFullPath outputFileRelativeToRoot + ensureDirectory (Path.GetDirectoryName(outputFileFullPath)) + + SimpleTemplating.UseFileAsSimpleTemplate( + p @ model.Substitutions, + template, + outputFileFullPath + ))) else if mainRun then yield @@ -558,6 +654,8 @@ type internal DocContent ParseScript.ParseFrontMatter(fileName) elif ext = ".md" then File.ReadLines fileName |> FrontMatterFile.ParseFromLines fileName + elif ext = ".ipynb" then + ParsePynb.parseFrontMatter fileName else None) |> Seq.sortBy (fun { Index = idx; CategoryIndex = cIdx } -> cIdx, idx) diff --git a/tests/FSharp.Literate.Tests/DocContentTests.fs b/tests/FSharp.Literate.Tests/DocContentTests.fs index 94a2f90fc..6cf6cf842 100644 --- a/tests/FSharp.Literate.Tests/DocContentTests.fs +++ b/tests/FSharp.Literate.Tests/DocContentTests.fs @@ -45,33 +45,48 @@ let ``Can build doc content`` () = // Check simple2.md --> simple2.html substititions let html1 = File.ReadAllText(rootOutputFolderAsGiven "simple1.html") let html2 = File.ReadAllText(rootOutputFolderAsGiven "simple2.html") + let html3 = File.ReadAllText(rootOutputFolderAsGiven "simple3.html") html1 |> shouldContainText """href="simple1.html">""" html1 |> shouldContainText """href="simple2.html">""" + html1 |> shouldContainText """href="simple3.html">""" html2 |> shouldContainText """href="simple1.html">""" html2 |> shouldContainText """href="simple2.html">""" + html2 |> shouldContainText """href="simple3.html">""" + html3 |> shouldContainText """href="simple1.html">""" + html3 |> shouldContainText """href="simple2.html">""" + html3 |> shouldContainText """href="simple3.html">""" // Check simple1.fsx --> simple1.ipynb substititions // Check simple2.md --> simple1.ipynb substititions let ipynb1 = File.ReadAllText(rootOutputFolderAsGiven "simple1.ipynb") let ipynb2 = File.ReadAllText(rootOutputFolderAsGiven "simple2.ipynb") + let ipynb3 = File.ReadAllText(rootOutputFolderAsGiven "simple3.ipynb") ipynb1 |> shouldContainText "simple2.ipynb" + ipynb1 |> shouldContainText "simple3.ipynb" ipynb2 |> shouldContainText "simple1.ipynb" + ipynb3 |> shouldContainText "simple1.ipynb" // Check fsx exists // Check simple1.fsx --> simple1.fsx substititions // Check simple2.md --> simple1.fsx substititions let fsx1 = File.ReadAllText(rootOutputFolderAsGiven "simple1.fsx") let fsx2 = File.ReadAllText(rootOutputFolderAsGiven "simple2.fsx") + let fsx3 = File.ReadAllText(rootOutputFolderAsGiven "simple3.fsx") fsx1 |> shouldContainText "simple2.fsx" + fsx1 |> shouldContainText "simple3.fsx" fsx2 |> shouldContainText "simple1.fsx" + fsx3 |> shouldContainText "simple1.fsx" // Check md contents // Check simple1.fsx --> simple1.md substititions // Check simple2.md --> simple1.md substititions let md1 = File.ReadAllText(rootOutputFolderAsGiven "simple1.md") let md2 = File.ReadAllText(rootOutputFolderAsGiven "simple2.md") + let md3 = File.ReadAllText(rootOutputFolderAsGiven "simple3.md") md1 |> shouldContainText "simple2.md" + md1 |> shouldContainText "simple3.md" md2 |> shouldContainText "simple1.md" + md3 |> shouldContainText "simple1.md" // Check in-folder1.fsx --> in-folder1.html substititions @@ -134,34 +149,48 @@ let ``Can build doc content using relative input path`` () = // Check simple2.md --> simple2.html substititions let html1 = File.ReadAllText(rootOutputFolderAsGiven "simple1.html") let html2 = File.ReadAllText(rootOutputFolderAsGiven "simple2.html") + let html3 = File.ReadAllText(rootOutputFolderAsGiven "simple3.html") html1 |> shouldContainText """href="simple1.html">""" html1 |> shouldContainText """href="simple2.html">""" + html1 |> shouldContainText """href="simple3.html">""" html2 |> shouldContainText """href="simple1.html">""" html2 |> shouldContainText """href="simple2.html">""" + html2 |> shouldContainText """href="simple3.html">""" + html3 |> shouldContainText """href="simple1.html">""" + html3 |> shouldContainText """href="simple2.html">""" + html3 |> shouldContainText """href="simple3.html">""" // Check simple1.fsx --> simple1.ipynb substititions // Check simple2.md --> simple1.ipynb substititions let ipynb1 = File.ReadAllText(rootOutputFolderAsGiven "simple1.ipynb") let ipynb2 = File.ReadAllText(rootOutputFolderAsGiven "simple2.ipynb") + let ipynb3 = File.ReadAllText(rootOutputFolderAsGiven "simple3.ipynb") ipynb1 |> shouldContainText "simple2.ipynb" + ipynb1 |> shouldContainText "simple3.ipynb" ipynb2 |> shouldContainText "simple1.ipynb" + ipynb3 |> shouldContainText "simple1.ipynb" // Check fsx exists // Check simple1.fsx --> simple1.fsx substititions // Check simple2.md --> simple1.fsx substititions let fsx1 = File.ReadAllText(rootOutputFolderAsGiven "simple1.fsx") let fsx2 = File.ReadAllText(rootOutputFolderAsGiven "simple2.fsx") + let fsx3 = File.ReadAllText(rootOutputFolderAsGiven "simple3.fsx") fsx1 |> shouldContainText "simple2.fsx" + fsx1 |> shouldContainText "simple3.fsx" fsx2 |> shouldContainText "simple1.fsx" + fsx3 |> shouldContainText "simple1.fsx" // Check md contents // Check simple1.fsx --> simple1.md substititions // Check simple2.md --> simple1.md substititions let md1 = File.ReadAllText(rootOutputFolderAsGiven "simple1.md") let md2 = File.ReadAllText(rootOutputFolderAsGiven "simple2.md") + let md3 = File.ReadAllText(rootOutputFolderAsGiven "simple3.md") md1 |> shouldContainText "simple2.md" + md1 |> shouldContainText "simple3.md" md2 |> shouldContainText "simple1.md" - + md3 |> shouldContainText "simple1.md" // Check in-folder1.fsx --> in-folder1.html substititions let f1html1 = File.ReadAllText(rootOutputFolderAsGiven "folder1" "in-folder1.html") @@ -225,3 +254,38 @@ let ``Parses frontmatter correctly `` () = twoTowersHtml |> shouldContainText "Previous" twoTowersHtml |> shouldContainText "Next" returnHtml |> shouldContainText "Previous" + + +(* Cannot get this test to evaluate the notebook +[] +let ``ipynb notebook evaluates`` () = + let rootOutputFolderAsGiven = __SOURCE_DIRECTORY__ "ipynb-eval-output" + let rootInputFolderAsGiven = __SOURCE_DIRECTORY__ "ipynb-eval" + + if Directory.Exists(rootOutputFolderAsGiven) then + Directory.Delete(rootOutputFolderAsGiven, true) + + let content = + DocContent( + rootOutputFolderAsGiven, + Map.empty, + lineNumbers = None, + evaluate = true, + substitutions = [], + saveImages = None, + watch = false, + root = "https://github.com", + crefResolver = (fun _ -> None), + onError = failwith + ) + + let docModels = content.Convert(rootInputFolderAsGiven, None, []) + let globals = [] + + for (_thing, action) in docModels do + action globals + + let ipynbOut = rootOutputFolderAsGiven "eval.html" |> File.ReadAllText + + ipynbOut |> shouldContainText "10007" +*) diff --git a/tests/FSharp.Literate.Tests/LiterateTests.fs b/tests/FSharp.Literate.Tests/LiterateTests.fs index 10b88cf65..bd13e0c72 100644 --- a/tests/FSharp.Literate.Tests/LiterateTests.fs +++ b/tests/FSharp.Literate.Tests/LiterateTests.fs @@ -1359,7 +1359,7 @@ let hello5 = 4 // Doc preparation code is not present in generated notebooks [] let ``Notebook output is exactly right`` () = - let md = + let doc = Literate.ParseScriptString( """ let hello = 1 @@ -1371,7 +1371,7 @@ let goodbye = 2 ||| MarkdownParseOptions.ParseNonCodeAsOther) ) - let pynb = Literate.ToPynb(md) + let pynb = Literate.ToPynb(doc) printfn "----" printfn "%s" pynb printfn "----" @@ -1433,6 +1433,299 @@ let goodbye = 2 pynb2 |> shouldEqual expected2 +[] +let ``pynb outputs passed to script correctly`` () = + + let input = + """{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "dotnet_repl_cellExecutionStartTime": "2023-11-22T09:25:48.0570832+00:00", + "dotnet_repl_cellExecutionEndTime": "2023-11-22T09:25:48.0798154+00:00" + }, + "source": [ + "words" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_repl_cellExecutionStartTime": "2023-11-22T09:25:48.081018+00:00", + "dotnet_repl_cellExecutionEndTime": "2023-11-22T09:25:50.1467326+00:00", + "dotnet_interactive": { + "language": "fsharp" + }, + "polyglot_notebook": { + "kernelName": "fsharp" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
{ Name = "Alf"\\n Phone = "(555) 555-5555"\\n ZipCode = "90210" }
Name
"Alf"\r\n",
+       "
Phone
"(555) 555-5555"\r\n",
+       "
ZipCode
"90210"\r\n",
+       "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "type ContactCard =\n", + " { Name: string\n", + " Phone: string\n", + " ZipCode: string }\n", + "\n", + "// Create a new record\n", + "{ Name = \"Alf\"; Phone = \"(555) 555-5555\"; ZipCode = \"90210\" }" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".NET (F#)", + "language": "F#", + "name": ".net-fsharp" + }, + "language_info": { + "file_extension": ".fs", + "mimetype": "text/x-fsharp", + "name": "F#", + "pygments_lexer": "fsharp", + "version": "6.0" + }, + "polyglot_notebook": { + "defaultKernelName": "fsharp", + "items": [ + { + "name": "fsharp" + } + ] + }, + "dotnet_interactive": { + "defaultKernelName": "fsharp", + "items": [ + { + "name": "fsharp" + } + ] + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}""" + + let doc = Literate.ParsePynbString(input, parseOptions = (MarkdownParseOptions.ParseCodeAsOther)) + + let fsx = Literate.ToFsx(doc) + printfn "----" + printfn "%s" fsx + printfn "----" + + let fsx2 = + fsx + .Replace("\r\n", "\n") + .Replace(" \n", "\n") + .Replace("\n\n*)", "\n*)") + .Replace("\n", "!") + + let expected = + """(** +words +*) +type ContactCard = + { Name: string + Phone: string + ZipCode: string } + +// Create a new record +{ Name = "Alf"; Phone = "(555) 555-5555"; ZipCode = "90210" } +(** +

{ Name = "Alf"\n Phone = "(555) 555-5555"\n ZipCode = "90210" }
Name
"Alf"
+
Phone
"(555) 555-5555"
+
ZipCode
"90210"
+
+

+*)""" + + let expected2 = expected.Replace("\r\n", "\n").Replace("\n", "!") + fsx2 |> shouldEqual expected2 + +[] +let ``md --> pynb --> md comes back the same`` () = + let mdIn = + """Heading +======= + +| Col1 | Col2 | +|:----:|------| +| Table with heading cell A1 | Table with heading cell B1 | +| Table with heading cell A2 | Table with heading cell B2 | + +```fsharp +let add a b = a + b +``` + +```csharp +``` + +```python +``` +""" + + let mdOut = + Literate.ParseMarkdownString( + mdIn, + parseOptions = + (MarkdownParseOptions.ParseCodeAsOther + ||| MarkdownParseOptions.ParseNonCodeAsOther) + ) + |> Literate.ToPynb + |> ParsePynb.pynbStringToMarkdown + + (mdOut.Trim()) |> shouldEqual (mdIn.Trim()) + +[] +let ``Notebook is converted to script exactly right`` () = + let doc = + Literate.ParsePynbString( + """ +{ + "cells": [ + { + "cell_type": "code", + "metadata": { + "dotnet_interactive": { + "language": "fsharp" + }, + "polyglot_notebook": { + "kernelName": "fsharp" + } + }, + "execution_count": null, "outputs": [], + "source": [ + "let hello = 1\n", + "\n", + "let goodbye = 2\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".NET (F#)", + "language": "F#", + "name": ".net-fsharp" + }, + "language_info": { + "file_extension": ".fs", + "mimetype": "text/x-fsharp", + "name": "polyglot-notebook", + "pygments_lexer": "fsharp" + }, + "polyglot_notebook": { + "kernelInfo": { + "defaultKernelName": "fsharp", + "items": [ + { + "aliases": [], + "languageName": "fsharp", + "name": "fsharp" + } + ] + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}""", + parseOptions = + (MarkdownParseOptions.ParseCodeAsOther + ||| MarkdownParseOptions.ParseNonCodeAsOther) + ) + + let fsx = Literate.ToFsx(doc) + printfn "----" + printfn "%s" fsx + printfn "----" + + let fsx2 = fsx.Replace("\r\n", "\n").Replace("\n", "!") + + let expected = + """let hello = 1 + +let goodbye = 2""" + + let expected2 = expected.Replace("\r\n", "\n").Replace("\n", "!") + + fsx2 |> shouldEqual expected2 [] let ``Script output is exactly right`` () = diff --git a/tests/FSharp.Literate.Tests/files/simple1.fsx b/tests/FSharp.Literate.Tests/files/simple1.fsx index 5b5382c62..13640da51 100644 --- a/tests/FSharp.Literate.Tests/files/simple1.fsx +++ b/tests/FSharp.Literate.Tests/files/simple1.fsx @@ -13,6 +13,7 @@ substitute-in-markdown: {{fsdocs-source-basename}} Another [hyperlink](simple1.fsx) Another [hyperlink](simple2.md) +And another [hyperlink](simple3.ipynb) *) let hello = "Code sample" diff --git a/tests/FSharp.Literate.Tests/files/simple2.md b/tests/FSharp.Literate.Tests/files/simple2.md index f59ee8d0c..42a19e991 100644 --- a/tests/FSharp.Literate.Tests/files/simple2.md +++ b/tests/FSharp.Literate.Tests/files/simple2.md @@ -7,3 +7,4 @@ With some [hyperlink](http://tomasp.net) Another [hyperlink](simple1.fsx) Yet another [hyperlink](simple2.md) +And another [hyperlink](simple3.ipynb) \ No newline at end of file diff --git a/tests/FSharp.Literate.Tests/files/simple3.ipynb b/tests/FSharp.Literate.Tests/files/simple3.ipynb new file mode 100644 index 000000000..514ebd4c3 --- /dev/null +++ b/tests/FSharp.Literate.Tests/files/simple3.ipynb @@ -0,0 +1,50 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Heading\n", + "=======\n", + "\n", + "With some [hyperlink](http://tomasp.net)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "fsharp" + }, + "polyglot_notebook": { + "kernelName": "fsharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "let hello = \"Code sample\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "Another [hyperlink](simple1.fsx)\n", + "Yet another [hyperlink](simple2.md)\n", + "And another [hyperlink](simple3.ipynb)" + ] + } + ], + "metadata": { + "language_info": { + "name": "csharp" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/FSharp.Literate.Tests/ipynb-eval/eval.ipynb b/tests/FSharp.Literate.Tests/ipynb-eval/eval.ipynb new file mode 100644 index 000000000..404856077 --- /dev/null +++ b/tests/FSharp.Literate.Tests/ipynb-eval/eval.ipynb @@ -0,0 +1,72 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "dotnet_repl_cellExecutionEndTime": "2023-11-17T18:25:09.2441915+00:00", + "dotnet_repl_cellExecutionStartTime": "2023-11-17T18:25:09.2190851+00:00" + }, + "source": [ + "Heading\n", + "=======\n", + "\n", + "With some [hyperlink](http://tomasp.net)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "fsharp" + }, + "dotnet_repl_cellExecutionEndTime": "2023-11-17T18:25:11.1373468+00:00", + "dotnet_repl_cellExecutionStartTime": "2023-11-17T18:25:09.2453568+00:00", + "polyglot_notebook": { + "kernelName": "fsharp" + }, + "vscode": { + "languageId": "fsharp" + } + }, + "outputs": [], + "source": [ + "let x = 1\n", + "let y = 10006\n", + "x + y" + ] + } + ], + "metadata": { + "dotnet_interactive": { + "defaultKernelName": "fsharp", + "items": [ + { + "name": "fsharp" + } + ] + }, + "kernelspec": { + "display_name": ".NET (F#)", + "language": "F#", + "name": ".net-fsharp" + }, + "language_info": { + "file_extension": ".fs", + "mimetype": "text/x-fsharp", + "name": "F#", + "pygments_lexer": "fsharp", + "version": "6.0" + }, + "polyglot_notebook": { + "defaultKernelName": "fsharp", + "items": [ + { + "name": "fsharp" + } + ] + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/FSharp.Literate.Tests/ipynb-eval/script.fsx b/tests/FSharp.Literate.Tests/ipynb-eval/script.fsx new file mode 100644 index 000000000..777a99b51 --- /dev/null +++ b/tests/FSharp.Literate.Tests/ipynb-eval/script.fsx @@ -0,0 +1,3 @@ + +10 + 24 +(***include-fsi-output***) \ No newline at end of file