From 4f7c8ff8dac445d057d364f5b4bf42a15a6a6f97 Mon Sep 17 00:00:00 2001 From: Rik Huijzer Date: Sat, 19 Jun 2021 11:50:27 +0200 Subject: [PATCH 1/7] Optimize filter for includes --- docs/contents/index.md | 10 +++- src/build.jl | 4 +- src/include-output.lua | 101 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 src/include-output.lua diff --git a/docs/contents/index.md b/docs/contents/index.md index 33f05a03..c4fdc48a 100644 --- a/docs/contents/index.md +++ b/docs/contents/index.md @@ -2,10 +2,16 @@ [//]: # (This file is only included on the website.) -```{.include} -_gen/homepage_intro.md +```gen +homepage_intro ``` See @sec:about for more information about this package. To get started and setup your own project, see @sec:getting-started. You can see a demonstration of the features and usage examples in @sec:demo. + +```gen _gen/homepage_intro.md``` + +```gen +foo("bar") +``` diff --git a/src/build.jl b/src/build.jl index f0be2155..e2a1a75a 100644 --- a/src/build.jl +++ b/src/build.jl @@ -4,8 +4,8 @@ function pandoc_file(filename) isfile(user_path) ? user_path : fallback_path end -include_files_lua = joinpath(PROJECT_ROOT, "src", "include-files.lua") -include_files = "--lua-filter=$include_files_lua" +include_lua_filter = joinpath(PROJECT_ROOT, "src", "include-output.lua") +include_files = "--lua-filter=$include_lua_filter" crossref = "--filter=pandoc-crossref" citeproc = "--citeproc" diff --git a/src/include-output.lua b/src/include-output.lua new file mode 100644 index 00000000..8860cb56 --- /dev/null +++ b/src/include-output.lua @@ -0,0 +1,101 @@ +--- include-output.lua – filter to include Julia output +--- Based on include-files.lua – filter to include Markdown files +--- + +-- pandoc's List type +local List = require 'pandoc.List' + +--- Get include auto mode +local include_auto = false +function get_vars (meta) + if meta['include-auto'] then + include_auto = true + end +end + +--- Keep last heading level found +local last_heading_level = 0 +function update_last_level(header) + last_heading_level = header.level +end + +--- Shift headings in block list by given number +local function shift_headings(blocks, shift_by) + if not shift_by then + return blocks + end + + local shift_headings_filter = { + Header = function (header) + header.level = header.level + shift_by + return header + end + } + + return pandoc.walk_block(pandoc.Div(blocks), shift_headings_filter).content +end + +--- Filter function for code blocks +local transclude +function transclude (cb) + -- ignore code blocks which are not of class "include". + if not cb.classes:includes 'gen' then + return + end + + -- Markdown is used if this is nil. + local format = cb.attributes['format'] + + -- Attributes shift headings + local shift_heading_level_by = 0 + local shift_input = cb.attributes['shift-heading-level-by'] + if shift_input then + shift_heading_level_by = tonumber(shift_input) + else + if include_auto then + -- Auto shift headings + shift_heading_level_by = last_heading_level + end + end + + --- keep track of level before recusion + local buffer_last_heading_level = last_heading_level + + local blocks = List:new() + for line in cb.text:gmatch('[^\n]+') do + if line:sub(1,2) ~= '//' then + path_sep = package.config:sub(1,1) + + -- Escape all weird characters to ensure they can be in the file. + -- This yields very weird names, but luckily the code is only internal. + escaped = line + escaped = escaped:gsub("%(", "OB") + escaped = escaped:gsub("%)", "CB") + escaped = escaped:gsub("\"", "DQ") + + path = "_gen" .. path_sep .. escaped .. ".md" + local fh = io.open(path) + if not fh then + io.stderr:write("Cannot find file `" .. path .. "` for `" .. line .. "` | Skipping includes\n") + else + local contents = pandoc.read(fh:read '*a', format).blocks + last_heading_level = 0 + -- recursive transclusion + contents = pandoc.walk_block( + pandoc.Div(contents), + { Header = update_last_level, CodeBlock = transclude } + ).content + --- reset to level before recursion + last_heading_level = buffer_last_heading_level + blocks:extend(shift_headings(contents, shift_heading_level_by)) + fh:close() + end + end + end + return blocks +end + +return { + { Meta = get_vars }, + { Header = update_last_level, CodeBlock = transclude } +} From 8ffbaea93dfadab4f093a01adb711f503cf12f7d Mon Sep 17 00:00:00 2001 From: Rik Huijzer Date: Sat, 19 Jun 2021 19:02:19 +0200 Subject: [PATCH 2/7] The new filter almost works --- docs/contents/index.md | 10 ++- src/build.jl | 2 +- src/generate.jl | 43 +++++++--- src/include-codeblocks.lua | 165 +++++++++++++++++++++++++++++++++++++ src/include-output.lua | 101 ----------------------- 5 files changed, 205 insertions(+), 116 deletions(-) create mode 100644 src/include-codeblocks.lua delete mode 100644 src/include-output.lua diff --git a/docs/contents/index.md b/docs/contents/index.md index c4fdc48a..f1e0a937 100644 --- a/docs/contents/index.md +++ b/docs/contents/index.md @@ -2,16 +2,18 @@ [//]: # (This file is only included on the website.) -```gen -homepage_intro +```jl +homepage_intro() ``` See @sec:about for more information about this package. To get started and setup your own project, see @sec:getting-started. You can see a demonstration of the features and usage examples in @sec:demo. -```gen _gen/homepage_intro.md``` +`jl baz()` -```gen +Lorem content: `jl var(3)` ipsum + +```jl foo("bar") ``` diff --git a/src/build.jl b/src/build.jl index e2a1a75a..7749d040 100644 --- a/src/build.jl +++ b/src/build.jl @@ -4,7 +4,7 @@ function pandoc_file(filename) isfile(user_path) ? user_path : fallback_path end -include_lua_filter = joinpath(PROJECT_ROOT, "src", "include-output.lua") +include_lua_filter = joinpath(PROJECT_ROOT, "src", "include-codeblocks.lua") include_files = "--lua-filter=$include_lua_filter" crossref = "--filter=pandoc-crossref" citeproc = "--citeproc" diff --git a/src/generate.jl b/src/generate.jl index b39c6dc5..28dbda4d 100644 --- a/src/generate.jl +++ b/src/generate.jl @@ -1,4 +1,3 @@ -include_regex = r"```{\.include}([\w\W]*?)```" """ code_block(s) @@ -7,13 +6,37 @@ Wrap `s` in a Markdown code block with triple backticks. """ code_block(s) = "```\n$s\n```\n" +function extract_codeblock_expr(s) + codeblock_pattern = r"```jl\s*\n([\w\W]*?)\n```" + matches = eachmatch(codeblock_pattern, s) + @show matches + [m[1] for m in matches] +end + """ - include_filenames(s::AbstractString)::Vector + extract_expr(s::AbstractString)::Vector + +Returns the filenames mentioned in the `jl` code blocks. +Here, `s` is the contents of a Markdown file. + +```jldoctest +julia> s = raw"lorem\n```jl\n foo(3)\n``` ipsum `jl bar()` dolar" -Returns the filenames mentioned in `{.include}` code blocks. +julia> Books.extract_expr(s) +2-element Vector{String}: + "foo(3)" + "bar()" +``` """ -function include_filenames(s::AbstractString)::Vector - matches = eachmatch(include_regex, s) +function extract_expr(s::AbstractString)::Vector + extract_codeblock_expr(s) + + code_pattern = raw"`jl [^`]*`" + rx = Regex(codeblock_pattern * '|' * code_pattern) + + matches = eachmatch(rx, s) + @show matches + [@show m for m in matches] nested_filenames = [split(m[1]) for m in matches] vcat(nested_filenames...) end @@ -132,7 +155,7 @@ end """ evaluate_include(path, M, fail_on_error) -For a `path` included in a chapter file, run the corresponding function and write the output to `path`. +For a `path` included in a Markdown file, run the corresponding function and write the output to `path`. """ function evaluate_include(path, M, fail_on_error) if dirname(path) != GENERATED_DIR @@ -175,9 +198,9 @@ function gen(; M=nothing, fail_on_error=false, project="default", call_html=true if !isfile(first_file) error("Couldn't find $first_file. Is there a valid project in your current working directory?") end - included_paths = vcat([include_filenames(read(path, String)) for path in paths]...) - f(path) = evaluate_include(path, M, fail_on_error) - foreach(f, included_paths) + included_expr = vcat([extract_expr(read(path, String)) for path in paths]...) + f(expr) = evaluate_include(expr, M, fail_on_error) + foreach(f, included_expr) if call_html println("Updating html") html(; project) @@ -199,7 +222,7 @@ julia> module Foo end; julia> gen(Foo.version) -Running version() for _gen/version.md +Running `version()` for _gen/version.md Updating html ``` """ diff --git a/src/include-codeblocks.lua b/src/include-codeblocks.lua new file mode 100644 index 00000000..8dae6e98 --- /dev/null +++ b/src/include-codeblocks.lua @@ -0,0 +1,165 @@ +--- include-output.lua – filter to include Julia output +--- Based on include-files.lua – filter to include Markdown files +--- + +-- pandoc's List type +local List = require 'pandoc.List' + +--- Get include auto mode +local include_auto = false +function get_vars (meta) + if meta['include-auto'] then + include_auto = true + end +end + +--- Keep last heading level found +local last_heading_level = 0 +function update_last_level(header) + last_heading_level = header.level +end + +--- Shift headings in block list by given number +local function shift_headings(blocks, shift_by) + if not shift_by then + return blocks + end + + local shift_headings_filter = { + Header = function (header) + header.level = header.level + shift_by + return header + end + } + + return pandoc.walk_block(pandoc.Div(blocks), shift_headings_filter).content +end + +--- Return path of the markdown file for the string `s` given by the user. +--- Ensure that this logic corresponds to the logic inside Books.jl. +local md_path +function md_path(s) + -- Escape all weird characters to ensure they can be in the file. + -- This yields very weird names, but luckily the code is only internal. + escaped = s + escaped = escaped:gsub("%(", "-ob-") + escaped = escaped:gsub("%)", "-cb-") + escaped = escaped:gsub("\"", "-dq-") + escaped = escaped:gsub(":", "-fc-") + escaped = escaped:gsub(";", "-sc-") + path_sep = package.config:sub(1,1) + path = "_gen" .. path_sep .. escaped .. ".md" + return path +end + +local not_found_error +function not_found_error(line, path, ticks) + io.stderr:write("Cannot find file for " .. ticks .. line .. ticks .. " at " .. path .. "\n") +end + +--- Filter function for code blocks +local transclude_codeblock +function transclude_codeblock(cb) + -- ignore code blocks which are not of class "jl". + if not cb.classes:includes 'jl' then + return + end + + -- Markdown is used if this is nil. + local format = cb.attributes['format'] + + -- Attributes shift headings + local shift_heading_level_by = 0 + local shift_input = cb.attributes['shift-heading-level-by'] + if shift_input then + shift_heading_level_by = tonumber(shift_input) + else + if include_auto then + -- Auto shift headings + shift_heading_level_by = last_heading_level + end + end + + --- keep track of level before recusion + local buffer_last_heading_level = last_heading_level + + local blocks = List:new() + for line in cb.text:gmatch('[^\n]+') do + if line:sub(1,2) ~= '//' then + + path = md_path(line) + + local fh = io.open(path) + if not fh then + not_found_error(line, path, '```') + msg = "ERROR: Cannot find file at " .. path .. " for `" .. line .. "`" + msg = { pandoc.CodeBlock(msg) } + blocks:extend(msg) + else + local text = fh:read("*a") + local contents = pandoc.read(text, format).blocks + last_heading_level = 0 + -- recursive transclusion + contents = pandoc.walk_block( + -- Here, the contents is added as an Any block. + -- Then, the filter is applied again recursively because + -- the included file could contain an include again! + pandoc.Div(contents), + { Header = update_last_level, CodeBlock = transclude } + ).content + --- reset to level before recursion + last_heading_level = buffer_last_heading_level + contents = shift_headings(contents, shift_heading_level_by) + -- Note that contents has type List. + blocks:extend(contents) + fh:close() + end + end + end + return blocks +end + +local startswith +function startswith(s, start) + return string.sub(s, 1, s.len(start)) == start +end + +--- Filter function for inline code +local transclude_code +function transclude_code(c) + -- ignore code blocks which do not start with "jl". + if not startswith(c.text, 'jl ') then + return + end + + tmp = c.text + line = c.text + line = line:sub(4) + path = md_path(line) + + c.text = path + + local fh = io.open(path) + if not fh then + not_found_error(tmp, path, '`') + c.text = "ERROR: Cannot find file at " .. path .. " for `" .. line .. "`" + else + text = fh:read("*a") + format = "markdown" + local contents = pandoc.read(text, format) + -- This conversion to a list is essential. + -- return { pandoc.Pandoc(contents) } + contents = pandoc.RawBlock(format, contents) + return { pandoc.Str("bar") } + end + + return c +end + +return { + { Meta = get_vars }, + { + Header = update_last_level, + Code = transclude_code + } +} diff --git a/src/include-output.lua b/src/include-output.lua deleted file mode 100644 index 8860cb56..00000000 --- a/src/include-output.lua +++ /dev/null @@ -1,101 +0,0 @@ ---- include-output.lua – filter to include Julia output ---- Based on include-files.lua – filter to include Markdown files ---- - --- pandoc's List type -local List = require 'pandoc.List' - ---- Get include auto mode -local include_auto = false -function get_vars (meta) - if meta['include-auto'] then - include_auto = true - end -end - ---- Keep last heading level found -local last_heading_level = 0 -function update_last_level(header) - last_heading_level = header.level -end - ---- Shift headings in block list by given number -local function shift_headings(blocks, shift_by) - if not shift_by then - return blocks - end - - local shift_headings_filter = { - Header = function (header) - header.level = header.level + shift_by - return header - end - } - - return pandoc.walk_block(pandoc.Div(blocks), shift_headings_filter).content -end - ---- Filter function for code blocks -local transclude -function transclude (cb) - -- ignore code blocks which are not of class "include". - if not cb.classes:includes 'gen' then - return - end - - -- Markdown is used if this is nil. - local format = cb.attributes['format'] - - -- Attributes shift headings - local shift_heading_level_by = 0 - local shift_input = cb.attributes['shift-heading-level-by'] - if shift_input then - shift_heading_level_by = tonumber(shift_input) - else - if include_auto then - -- Auto shift headings - shift_heading_level_by = last_heading_level - end - end - - --- keep track of level before recusion - local buffer_last_heading_level = last_heading_level - - local blocks = List:new() - for line in cb.text:gmatch('[^\n]+') do - if line:sub(1,2) ~= '//' then - path_sep = package.config:sub(1,1) - - -- Escape all weird characters to ensure they can be in the file. - -- This yields very weird names, but luckily the code is only internal. - escaped = line - escaped = escaped:gsub("%(", "OB") - escaped = escaped:gsub("%)", "CB") - escaped = escaped:gsub("\"", "DQ") - - path = "_gen" .. path_sep .. escaped .. ".md" - local fh = io.open(path) - if not fh then - io.stderr:write("Cannot find file `" .. path .. "` for `" .. line .. "` | Skipping includes\n") - else - local contents = pandoc.read(fh:read '*a', format).blocks - last_heading_level = 0 - -- recursive transclusion - contents = pandoc.walk_block( - pandoc.Div(contents), - { Header = update_last_level, CodeBlock = transclude } - ).content - --- reset to level before recursion - last_heading_level = buffer_last_heading_level - blocks:extend(shift_headings(contents, shift_heading_level_by)) - fh:close() - end - end - end - return blocks -end - -return { - { Meta = get_vars }, - { Header = update_last_level, CodeBlock = transclude } -} From a9fe71457a96c4da3f92762868e54ceeada9860d Mon Sep 17 00:00:00 2001 From: Rik Huijzer Date: Sat, 19 Jun 2021 19:40:09 +0200 Subject: [PATCH 3/7] Got unknown inline type para error now --- src/include-codeblocks.lua | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/include-codeblocks.lua b/src/include-codeblocks.lua index 8dae6e98..86e55e8e 100644 --- a/src/include-codeblocks.lua +++ b/src/include-codeblocks.lua @@ -149,8 +149,12 @@ function transclude_code(c) local contents = pandoc.read(text, format) -- This conversion to a list is essential. -- return { pandoc.Pandoc(contents) } - contents = pandoc.RawBlock(format, contents) - return { pandoc.Str("bar") } + print(text) + print(contents) + tmp = pandoc.RawInline(format, text) + print(tmp) + -- return { pandoc.Span(contents.blocks) } + return contents.blocks end return c From 1c768dc86670a51de67c1d9a76dd83c6ca7a79c9 Mon Sep 17 00:00:00 2001 From: Rik Huijzer Date: Sat, 19 Jun 2021 20:23:49 +0200 Subject: [PATCH 4/7] Add error message for text too long --- docs/contents/index.md | 2 ++ src/include-codeblocks.lua | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/contents/index.md b/docs/contents/index.md index f1e0a937..62b6ff17 100644 --- a/docs/contents/index.md +++ b/docs/contents/index.md @@ -10,6 +10,8 @@ See @sec:about for more information about this package. To get started and setup your own project, see @sec:getting-started. You can see a demonstration of the features and usage examples in @sec:demo. +Foo ```jl homepage_intro()``` bar + `jl baz()` Lorem content: `jl var(3)` ipsum diff --git a/src/include-codeblocks.lua b/src/include-codeblocks.lua index 86e55e8e..0f568588 100644 --- a/src/include-codeblocks.lua +++ b/src/include-codeblocks.lua @@ -88,6 +88,13 @@ function transclude_codeblock(cb) if line:sub(1,2) ~= '//' then path = md_path(line) + if 60 < path:len() then + msg = "ERROR: The text `" .. line .. "` is too long to be converted to a filename" + msg = { pandoc.CodeBlock(msg) } + blocks:extend(msg) + -- Lua has no continue. + goto skip_to_next + end local fh = io.open(path) if not fh then @@ -115,6 +122,7 @@ function transclude_codeblock(cb) fh:close() end end + ::skip_to_next:: end return blocks end @@ -154,7 +162,11 @@ function transclude_code(c) tmp = pandoc.RawInline(format, text) print(tmp) -- return { pandoc.Span(contents.blocks) } - return contents.blocks + -- c.text = text + c.blocks = contents.blocks + -- So, I can output it like normally via `return pandoc.read(el.text, 'markdown').blocks` + -- Unfortunately, that returns blocks which is a subtype of RawBlock. + return c end return c @@ -164,6 +176,9 @@ return { { Meta = get_vars }, { Header = update_last_level, - Code = transclude_code + CodeBlock = transclude_codeblock, + -- Due to the type system of Pandoc, a Code element cannot contain Block subtypes. + -- These subtypes will start to exist once your gonna parse Markdown. + -- Code = transclude_code } } From 35a9fe2eda9bd3c6910c9957aeebd1cc2a7c301c Mon Sep 17 00:00:00 2001 From: Rik Huijzer Date: Sat, 19 Jun 2021 21:58:20 +0200 Subject: [PATCH 5/7] Rewrite evaluate_and_write --- docs/contents/index.md | 2 +- src/generate.jl | 136 +++++++++++++++++-------------------- src/include-codeblocks.lua | 24 ++----- 3 files changed, 68 insertions(+), 94 deletions(-) diff --git a/docs/contents/index.md b/docs/contents/index.md index 62b6ff17..3e4a0eb8 100644 --- a/docs/contents/index.md +++ b/docs/contents/index.md @@ -14,7 +14,7 @@ Foo ```jl homepage_intro()``` bar `jl baz()` -Lorem content: `jl var(3)` ipsum +Lorem content: ```jl var(3)``` ipsum ```jl foo("bar") diff --git a/src/generate.jl b/src/generate.jl index 28dbda4d..579ab510 100644 --- a/src/generate.jl +++ b/src/generate.jl @@ -7,12 +7,16 @@ Wrap `s` in a Markdown code block with triple backticks. code_block(s) = "```\n$s\n```\n" function extract_codeblock_expr(s) - codeblock_pattern = r"```jl\s*\n([\w\W]*?)\n```" - matches = eachmatch(codeblock_pattern, s) - @show matches - [m[1] for m in matches] end +extract_expr_example() = """ + lorem + ```jl + foo(3) + ``` + ipsum `jl bar()` dolar + """ + """ extract_expr(s::AbstractString)::Vector @@ -20,7 +24,7 @@ Returns the filenames mentioned in the `jl` code blocks. Here, `s` is the contents of a Markdown file. ```jldoctest -julia> s = raw"lorem\n```jl\n foo(3)\n``` ipsum `jl bar()` dolar" +julia> s = Books.extract_expr_example(); julia> Books.extract_expr(s) 2-element Vector{String}: @@ -29,16 +33,24 @@ julia> Books.extract_expr(s) ``` """ function extract_expr(s::AbstractString)::Vector - extract_codeblock_expr(s) + codeblock_pattern = r"```jl\s*([\w\W]*?)```" + matches = eachmatch(codeblock_pattern, s) + function clean(m) + m = m[1] + m = strip(m) + m = string(m)::String + end + from_codeblocks = clean.(matches) - code_pattern = raw"`jl [^`]*`" - rx = Regex(codeblock_pattern * '|' * code_pattern) + inline_pattern = r" `jl ([^`]*)`" + matches = eachmatch(inline_pattern, s) + from_inline = clean.(matches) + E = [from_codeblocks; from_inline] - matches = eachmatch(rx, s) - @show matches - [@show m for m in matches] - nested_filenames = [split(m[1]) for m in matches] - vcat(nested_filenames...) + # Check the user defined expressions for parse errors. + Core.eval.(E) + + E end """ @@ -83,9 +95,6 @@ function method_name(path::AbstractString) suffix = "" if contains(name, '-') parts = split(name, '-') - if length(parts) != 2 - error("Path name is expected to contain at most one - (minus)") - end name = parts[1] suffix = parts[2] end @@ -93,87 +102,68 @@ function method_name(path::AbstractString) end """ - evaluate_and_write(f::Function, path::AbstractString, suffix::AbstractString) - -Evaluates `f`, converts the output writes the output to `path`. -Some output conversions will also write to other files, which the file at `path` links to. -For example, this happens with plots. - -# Example -```jldoctest -julia> using DataFrames - -julia> example_table() = DataFrame(A = [1, 2], B = [3, 4]) -example_table (generic function with 1 method) - -julia> path = joinpath(tempdir(), "example.md"); - -julia> Books.evaluate_and_write(example_table, path, "") -Running example_table() for /tmp/example.md + escape_expr(expr::String) -julia> print(read(path, String)) -| A | B | -| ---:| ---:| -| 1 | 3 | -| 2 | 4 | - -: Example {#tbl:example} -``` +Escape an expression to the corresponding path. +The logic in this method should match the logic in the Lua filter. """ -function evaluate_and_write(f::Function, path::AbstractString, suffix::AbstractString) - function run_f(f) - println("Running $(f)() for $path") - f() - end - function run_sc(f) - println("Obtaining source code for $f()") - @sc(f) - end - function run_sco(f) - println("Obtaining source code and output for $f()") - @sco(f) - end +function escape_expr(expr::String) + replace_map = [ + '(' => "-ob-", + ')' => "-cb-", + '"' => "-dq-", + ':' => "-fc-", + ';' => "-sc-", + ] + escaped = reduce(replace, replace_map; init=expr) + joinpath(GENERATED_DIR, "$escaped.md") +end - out = - suffix == "sc" ? run_sc(f) : - suffix == "sco" ? run_sco(f) : - run_f(f) +function evaluate_and_write(M::Module, expr::String) + path = escape_expr(expr) + println("Writing output of `$expr` to $path") + ex = Meta.parse(expr) + out = Core.eval(M, ex) out = convert_output(path, out) - out = String(out) + out = string(out)::String write(path, out) nothing end -function evaluate_and_write(M::Module, path) - method, suffix = method_name(path) - f = getproperty(M, Symbol(method)) - evaluate_and_write(f, path, suffix) +function evaluate_and_write(f::Function) + function_name = string(Base.nameof(f))::String + expr = function_name * "()" + path = escape_expr(expr) + println("Writing output of `$expr` to $path") + out = f() + out = convert_output(path, out) + out = string(out)::String + write(path, out) + + nothing end """ - evaluate_include(path, M, fail_on_error) + evaluate_include(expr::String, M, fail_on_error) For a `path` included in a Markdown file, run the corresponding function and write the output to `path`. """ -function evaluate_include(path, M, fail_on_error) - if dirname(path) != GENERATED_DIR - println("Not running code for $path") - return nothing - end +function evaluate_include(expr::String, M, fail_on_error) if isnothing(M) + # This code isn't really working. M = caller_module() end - mkpath(dirname(path)) if fail_on_error - evaluate_and_write(M, path) + evaluate_and_write(M, expr) else try - evaluate_and_write(M, path) + evaluate_and_write(M, expr) catch e @error """ - Failed to run code for $path: + Failed to run code for $path. + Details: $(rethrow()) """ end diff --git a/src/include-codeblocks.lua b/src/include-codeblocks.lua index 0f568588..1d0c7f27 100644 --- a/src/include-codeblocks.lua +++ b/src/include-codeblocks.lua @@ -140,33 +140,19 @@ function transclude_code(c) return end - tmp = c.text line = c.text line = line:sub(4) path = md_path(line) - c.text = path - local fh = io.open(path) if not fh then - not_found_error(tmp, path, '`') + not_found_error(line, path, '`') c.text = "ERROR: Cannot find file at " .. path .. " for `" .. line .. "`" else text = fh:read("*a") - format = "markdown" - local contents = pandoc.read(text, format) + -- To retain ticks, use `c.text = text` and `return c`. -- This conversion to a list is essential. - -- return { pandoc.Pandoc(contents) } - print(text) - print(contents) - tmp = pandoc.RawInline(format, text) - print(tmp) - -- return { pandoc.Span(contents.blocks) } - -- c.text = text - c.blocks = contents.blocks - -- So, I can output it like normally via `return pandoc.read(el.text, 'markdown').blocks` - -- Unfortunately, that returns blocks which is a subtype of RawBlock. - return c + return { pandoc.Str(text) } end return c @@ -177,8 +163,6 @@ return { { Header = update_last_level, CodeBlock = transclude_codeblock, - -- Due to the type system of Pandoc, a Code element cannot contain Block subtypes. - -- These subtypes will start to exist once your gonna parse Markdown. - -- Code = transclude_code + Code = transclude_code } } From 2ea8651718fc2125640ddb1a0e4a6072dfaa124d Mon Sep 17 00:00:00 2001 From: Rik Huijzer Date: Sat, 19 Jun 2021 23:30:13 +0200 Subject: [PATCH 6/7] Fix a large part of the docs --- docs/contents/demo.md | 162 ++++++++++--------------------------- docs/contents/index.md | 10 --- src/generate.jl | 63 ++++++++------- src/include-codeblocks.lua | 13 ++- src/output.jl | 42 +++++----- src/outputs/aog.jl | 4 +- src/outputs/dataframes.jl | 9 ++- src/outputs/makie.jl | 4 +- src/outputs/plots.jl | 4 +- src/showcode.jl | 4 +- 10 files changed, 119 insertions(+), 196 deletions(-) diff --git a/docs/contents/demo.md b/docs/contents/demo.md index b328f8cf..6c3ba91a 100644 --- a/docs/contents/demo.md +++ b/docs/contents/demo.md @@ -15,33 +15,39 @@ $$ y = \frac{\sin{x}}{\cos{x}} $$ {#eq:example} ## Embedding output {#sec:embedding-output} -For embedding code, you can use the `{.include}` code block. -This package will run your methods based on the filenames in these code blocks. -For example, to show the Julia version, use +For embedding code, you can use the `jl` inline code or code block. +For example, to show the Julia version, define a code block like 
-```{.include}
-_gen/julia_version.md
+```jl
+julia_version()
 ```
+in a Markdown file. Then, in your package, define the method `julia_version()`: + ``` julia_version() = "This book is built with Julia $VERSION." ``` -Next, ensure that you call `using Books; gen(; M = Foo)`, where `Foo` is the name of your module. +Next, ensure that you call `using Books; gen(; M)`, where `M = YourModule`. This will place the text -```{.include} -_gen/julia_version_example.md +```jl +julia_version_example() +``` + +at the right path so that it can be included by Pandoc. +You can also embed the output inline with single backticks like + +``` +`jl julia_version()` ``` -at the aforementioned path so that it can be included by Pandoc. While doing this, it is expected that you also have the browser open and a server running, see @sec:getting-started. That way, the page is immediately updated when you run `gen`. - Note that it doesn't matter where you define the function `julia_version`, as long as it is in your module. To save yourself some typing, and to allow yourself to get some coffee while Julia gets up to speed, you can start Julia for some package `Foo` with @@ -68,29 +74,30 @@ To run this method automatically when you make a change in your package, ensure ``` julia> f() = gen(M.my_plot); -julia> entr(f, [], [M]) +julia> entr(f, ["contents"], [M]) [...] ``` -In the background, `gen` passes the methods through `convert_output(path, out::T)` where `T` can, for example, be a DataFrame or a plot. +Which will automatically run `f()` whenever one of the files in `contents/` changes or any code in the module `M`. +In the background, `gen` passes the methods through `convert_output(expr::String, path, out::T)` where `T` can, for example, be a DataFrame or a plot. To show that a DataFrame is converted to a Markdown table, we define a method -```{.include} -_gen/my_table-sc.md +```jl +@sc(my_table) ``` and add its output to the Markdown file with 
-```{.include}
-_gen/my_table.md
+```jl
+my_table()
 ```
Then, it will show as -```{.include} -_gen/my_table.md +```jl +my_table() ``` where the caption and the label are inferred from the `path`. @@ -103,8 +110,8 @@ Refer to @tbl:my_table with To show multiple objects, pass a `Vector`: -```{.include} -_gen/multiple_df_vector-sco.md +```jl +@sco(multiple_df_vector) ``` When you want to control where the various objects are saved, use `Options`. @@ -139,130 +146,45 @@ This package will attempt to infer missing information from the `path`, `caption _gen/options_example_doctests.md ``` -## String code blocks {#sec:string_code_blocks} - -There are two ways to show code blocks. -One way is by passing your code as a string. -This is how similar packages work. -However, with `Books.jl`, the aim is to work with functions and *not* with code as strings as discussed at the end of @sec:about. -See @sec:function_code_blocks for a better way for showing code blocks. - -Like in @sec:embedding-output, first define a method like - -```{.include} -_gen/sum_example_definition.md -``` - -Then, add this method via - -
-```{.include}
-_gen/sum_example.md
-```
-
- -which gives as output - -```{.include} -_gen/sum_example.md -``` - -Here, how the output should be handled is based on the output type of the function. -In this case, the output type is of type `Code`. -Methods for other outputs exist too: - -```{.include} -_gen/example_table_definition.md -``` - -shows - -```{.include} -_gen/example_table.md -``` - -Alternatively, we can show the same by creating something of type `Code`: - -```{.include} -_gen/code_example_table-sc.md -``` - -which shows as - -```{.include} -_gen/code_example_table.md -``` - -because the output of the code block is of type DataFrame. - -In essence, this package doesn't hide the implementation behind synctactic sugar. -Instead, this package calls functions and gives you the freedom to decide what to do from there. -As an example, we can pass `Module` objects to `code` to evaluate the code block in a specific module. - -```{.include} -_gen/module_example_definition.md -``` - -When calling `module_example`, it shows as - -```{.include} -_gen/module_example.md -``` - -Similarily, we can get the value of x: - -```{.include} -_gen/module_call_x.md -``` - -Unsuprisingly, creating a DataFrame will now fail because we haven't loaded DataFrames - -```{.include} -_gen/module_fail.md -``` - -Which is easy to fix - -```{.include} -_gen/module_fix.md -``` - ## Function code blocks {#sec:function_code_blocks} So, instead of passing a string which `Books.jl` will evaluate, `Books.jl` can also obtain the code for a method directly. (Thanks to `CodeTracking.@code_string`.) For example, we can define the following method: -```{.include} -_gen/my_data-sc.md +
+```jl
+my_data()
 ```
+
-and call it by adding the `-sco` (source code and output) suffix to the path: +To show code and output (sco), use the `@sco` macro. +This macro is exported by Books, so ensure that you have `using Books` in your package. 
-```{.include}
-_gen/my_data-sco.md
+```jl
+@sco(my_data)
 ```
This gives -```{.include} -_gen/my_data-sco.md +```jl +@sco(my_data) ``` To only show the source code, use the `-sc` suffix: 
-```{.include}
-_gen/my_data-sc.md
+```jl
+@sc(my_data)
 ```
-giving +resulting in -```{.include} -_gen/my_data-sc.md +```jl +@sc(my_data) ``` Since we're using methods as code blocks, we can use the code shown in one code block in another. diff --git a/docs/contents/index.md b/docs/contents/index.md index 3e4a0eb8..ba7d29a8 100644 --- a/docs/contents/index.md +++ b/docs/contents/index.md @@ -9,13 +9,3 @@ homepage_intro() See @sec:about for more information about this package. To get started and setup your own project, see @sec:getting-started. You can see a demonstration of the features and usage examples in @sec:demo. - -Foo ```jl homepage_intro()``` bar - -`jl baz()` - -Lorem content: ```jl var(3)``` ipsum - -```jl -foo("bar") -``` diff --git a/src/generate.jl b/src/generate.jl index 579ab510..eba3d60e 100644 --- a/src/generate.jl +++ b/src/generate.jl @@ -47,9 +47,14 @@ function extract_expr(s::AbstractString)::Vector from_inline = clean.(matches) E = [from_codeblocks; from_inline] - # Check the user defined expressions for parse errors. - Core.eval.(E) - + function check_parse_errors(expr) + try + Meta.parse(expr) + catch e + error("Exception occured when trying to parse `$expr`") + end + end + check_parse_errors.(E) E end @@ -72,33 +77,29 @@ function caller_module() end """ - method_name(path::AbstractString) + method_name(expr::String) -Return method name and suffix for a Markdown file. -Here, the suffix is used to allow users to specify that, for example, `@sc` has to be called on the method. +Return file name for `expr`. +This is used for things like how to call an image file and a caption. -# Example +# Examples ```jldoctest -julia> path = "_gen/foo_bar.md"; +julia> Books.method_name("@some_macro(foo)") +"foo" -julia> Books.method_name(path) -("foo_bar", "") +julia> Books.method_name("foo()") +"foo" -julia> path = "_gen/foo_bar-sc.md"; - -julia> Books.method_name(path) -("foo_bar", "sc") +julia> Books.method_name("foo(3)") +"foo_3" ``` """ -function method_name(path::AbstractString) - name, extension = splitext(basename(path)) - suffix = "" - if contains(name, '-') - parts = split(name, '-') - name = parts[1] - suffix = parts[2] - end - (name, suffix) +function method_name(expr::String) + remove_macros(expr) = replace(expr, r"@[\w\_]*" => "") + expr = remove_macros(expr) + expr = replace(expr, '(' => '_') + expr = replace(expr, ')' => "") + expr = strip(expr, '_') end """ @@ -114,6 +115,7 @@ function escape_expr(expr::String) '"' => "-dq-", ':' => "-fc-", ';' => "-sc-", + '@' => "-ax-" ] escaped = reduce(replace, replace_map; init=expr) joinpath(GENERATED_DIR, "$escaped.md") @@ -125,7 +127,7 @@ function evaluate_and_write(M::Module, expr::String) ex = Meta.parse(expr) out = Core.eval(M, ex) - out = convert_output(path, out) + out = convert_output(expr, path, out) out = string(out)::String write(path, out) @@ -133,12 +135,12 @@ function evaluate_and_write(M::Module, expr::String) end function evaluate_and_write(f::Function) - function_name = string(Base.nameof(f))::String - expr = function_name * "()" + function_name = Base.nameof(f) + expr = "$(function_name)()" path = escape_expr(expr) println("Writing output of `$expr` to $path") out = f() - out = convert_output(path, out) + out = convert_output(expr, path, out) out = string(out)::String write(path, out) @@ -183,6 +185,7 @@ After calling the methods, this method will also call `html()` to update the sit The module `M` is used to locate the method defined, as a string, in the `.include` via `getproperty`. """ function gen(; M=nothing, fail_on_error=false, project="default", call_html=true) + mkpath(GENERATED_DIR) paths = inputs(project) first_file = first(paths) if !isfile(first_file) @@ -216,10 +219,10 @@ Running `version()` for _gen/version.md Updating html ``` """ -function gen(f::Function; fail_on_error=false, project="default", call_html=true) +function gen(f::Function; project="default", call_html=true) path = joinpath(GENERATED_DIR, "$f.md") - suffix = "" - evaluate_and_write(f, path, suffix) + mkpath(GENERATED_DIR) + evaluate_and_write(f) if call_html println("Updating html") html(; project) diff --git a/src/include-codeblocks.lua b/src/include-codeblocks.lua index 1d0c7f27..1ab9ee2c 100644 --- a/src/include-codeblocks.lua +++ b/src/include-codeblocks.lua @@ -47,6 +47,7 @@ function md_path(s) escaped = escaped:gsub("\"", "-dq-") escaped = escaped:gsub(":", "-fc-") escaped = escaped:gsub(";", "-sc-") + escaped = escaped:gsub("@", "-ax-") path_sep = package.config:sub(1,1) path = "_gen" .. path_sep .. escaped .. ".md" return path @@ -54,7 +55,8 @@ end local not_found_error function not_found_error(line, path, ticks) - io.stderr:write("Cannot find file for " .. ticks .. line .. ticks .. " at " .. path .. "\n") + code = ticks .. line .. ticks + io.stderr:write("Cannot find file for " .. code .. " at " .. path .. "\n") end --- Filter function for code blocks @@ -99,7 +101,9 @@ function transclude_codeblock(cb) local fh = io.open(path) if not fh then not_found_error(line, path, '```') - msg = "ERROR: Cannot find file at " .. path .. " for `" .. line .. "`" + suggestion = "Did you run `gen(; M)` where `M = YourModule`?\n" + msg = "ERROR: Cannot find file at " .. path .. " for `" .. line .. "`." + msg = msg .. ' ' .. suggestion msg = { pandoc.CodeBlock(msg) } blocks:extend(msg) else @@ -147,7 +151,10 @@ function transclude_code(c) local fh = io.open(path) if not fh then not_found_error(line, path, '`') - c.text = "ERROR: Cannot find file at " .. path .. " for `" .. line .. "`" + suggestion = "Did you run `gen(; M)` where `M = YourModule`?" + msg = "ERROR: Cannot find file at " .. path .. " for `" .. line .. "`." + msg = msg .. ' ' .. suggestion + c.text = msg else text = fh:read("*a") -- To retain ticks, use `c.text = text` and `return c`. diff --git a/src/output.jl b/src/output.jl index 9485f652..917b7335 100644 --- a/src/output.jl +++ b/src/output.jl @@ -27,15 +27,15 @@ struct ImageOptions ImageOptions(object; width=nothing, height=nothing) = new(object, width, height) end -function convert_output(path, out::ImageOptions; kwargs...) +function convert_output(expr, path, out::ImageOptions; kwargs...) width = out.width height = out.height - convert_output(path, out.object; width, height, kwargs...) + convert_output(expr, path, out.object; width, height, kwargs...) end -function convert_output(path, outputs::AbstractVector) +function convert_output(expr, path, outputs::AbstractVector) path = nothing - outputs = convert_output.(path, outputs) + outputs = convert_output.(nothing, nothing, outputs) outputs = String.(outputs) out = join(outputs, "\n\n") end @@ -77,7 +77,7 @@ julia> Options.(objects, filenames) """ Options(object, filename::AbstractString) = Options(object; filename) -function convert_output(path, out::Code)::String +function convert_output(expr, path, out::Code)::String block = out.block mod = out.mod ans = try @@ -85,7 +85,7 @@ function convert_output(path, out::Code)::String catch e string(e) end - shown_output = convert_output(path, ans) + shown_output = convert_output(expr, path, ans) if isa(ans, AbstractString) || isa(ans, Number) shown_output = code_block(shown_output) end @@ -113,7 +113,7 @@ function convert_output(path, out::Code)::String end """ - convert_output(path, options::Options) + convert_output(expr, path, options::Options) Convert `options.object` while taking `options.caption` and `options.label` into account. This method needs to pass the options correctly to the resulting type, because the syntax depends on the type; @@ -127,7 +127,7 @@ julia> caption = "My DataFrame"; julia> options = Options(df; caption); -julia> print(Books.convert_output(nothing, options)) +julia> print(Books.convert_output(nothing, nothing, options)) | A | | ---:| | 1 | @@ -135,7 +135,7 @@ julia> print(Books.convert_output(nothing, options)) : My DataFrame ``` """ -function convert_output(path, opts::Options)::String +function convert_output(expr, path, opts::Options)::String object = opts.object filename = opts.filename if !isnothing(filename) @@ -143,21 +143,21 @@ function convert_output(path, opts::Options)::String end caption = opts.caption label = opts.label - convert_output(path, object; caption, label) + convert_output(expr, path, object; caption, label) end """ - convert_output(path, out::AbstractString) + convert_output(expr, path, out::AbstractString) Return `out` as string. This avoids the adding of `"` which `show` does by default. """ -convert_output(path, out::AbstractString) = string(out) +convert_output(expr, path, out::AbstractString) = string(out) -convert_output(path, out::Number) = string(out) +convert_output(expr, path, out::Number) = string(out) """ - convert_output(path, out) + convert_output(expr, path, out) Fallback method for `out::Any`. This passes the objects through show to use the overrides that package creators might have provided. @@ -172,13 +172,13 @@ julia> chn = Chains([1]; info=(start_time=[1.0], stop_time=[1.0])); julia> string(chn) "MCMC chain (1×1×1 Array{Int64, 3})" -julia> out = Books.convert_output("", chn); +julia> out = Books.convert_output("", "", chn); julia> contains(out, "Summary Statistics") true ``` """ -function convert_output(path, out)::String +function convert_output(expr, path, out)::String io = IOBuffer() mime = MIME("text/plain") show(io, mime, out) @@ -227,7 +227,7 @@ function pandoc_image(file, path; caption=nothing, label=nothing) end """ - caption_label(path, caption, label) + caption_label(expr, caption, label) Return `caption` and `label` for the inputs. This method sets some reasonable defaults if any of the inputs is missing. @@ -250,13 +250,13 @@ julia> Books.caption_label(nothing, nothing, nothing) (caption = nothing, label = nothing) ``` """ -function caption_label(path, caption, label) - if isnothing(path) && isnothing(caption) && isnothing(label) +function caption_label(expr, caption, label) + if isnothing(expr) && isnothing(caption) && isnothing(label) return (caption=nothing, label=nothing) end - if !isnothing(path) - name, suffix = method_name(path) + if !isnothing(expr) + name = method_name(expr) if isnothing(label) label = name end diff --git a/src/outputs/aog.jl b/src/outputs/aog.jl index a8de0d2a..18555074 100644 --- a/src/outputs/aog.jl +++ b/src/outputs/aog.jl @@ -3,7 +3,7 @@ using AlgebraOfGraphics using CairoMakie -function convert_output(path, fg::AlgebraOfGraphics.FigureGrid; caption=nothing, label=nothing) +function convert_output(expr, path, fg::AlgebraOfGraphics.FigureGrid; caption=nothing, label=nothing) im_dir = joinpath(BUILD_DIR, "im") mkpath(im_dir) @@ -15,7 +15,7 @@ function convert_output(path, fg::AlgebraOfGraphics.FigureGrid; caption=nothing, """ throw(ErrorException(msg)) end - file, _ = method_name(path) + file = store_filename(expr) println("Writing plot images for $file") svg_filename = "$file.svg" diff --git a/src/outputs/dataframes.jl b/src/outputs/dataframes.jl index a9f6318f..e421df0a 100644 --- a/src/outputs/dataframes.jl +++ b/src/outputs/dataframes.jl @@ -2,7 +2,7 @@ using DataFrames using Latexify """ - convert_output(path, out::DataFrame; caption=nothing, label=nothing) + convert_output(expr, path, out::DataFrame; caption=nothing, label=nothing) Convert `out` to Markdown table and set some `pandoc-crossref` metadata. @@ -10,16 +10,17 @@ Convert `out` to Markdown table and set some `pandoc-crossref` metadata. ```jldoctest julia> df = DataFrame(A = [1]) -julia> print(Books.convert_output("a/my_table.md", df)) +julia> print(Books.convert_output("my_table()", nothing, df)) | A | | ---:| | 1 | : My table {#tbl:my_table} +``` """ -function convert_output(path, out::DataFrame; caption=nothing, label=nothing)::String +function convert_output(expr, path, out::DataFrame; caption=nothing, label=nothing)::String table = Latexify.latexify(out; env=:mdtable, latex=false) - caption, label = caption_label(path, caption, label) + caption, label = caption_label(expr, caption, label) if isnothing(caption) && isnothing(label) return string(table) diff --git a/src/outputs/makie.jl b/src/outputs/makie.jl index 3cb89969..a265d771 100644 --- a/src/outputs/makie.jl +++ b/src/outputs/makie.jl @@ -3,7 +3,7 @@ using CairoMakie import Makie -function convert_output(path, p::Makie.FigureAxisPlot; caption=nothing, label=nothing) +function convert_output(expr, path, p::Makie.FigureAxisPlot; caption=nothing, label=nothing) im_dir = joinpath(BUILD_DIR, "im") mkpath(im_dir) @@ -15,7 +15,7 @@ function convert_output(path, p::Makie.FigureAxisPlot; caption=nothing, label=no """ throw(ErrorException(msg)) end - file, _ = method_name(path) + file = method_name(expr) println("Writing plot images for $file") svg_filename = "$file.svg" diff --git a/src/outputs/plots.jl b/src/outputs/plots.jl index e3fcd022..946377db 100644 --- a/src/outputs/plots.jl +++ b/src/outputs/plots.jl @@ -2,7 +2,7 @@ import Plots -function convert_output(path, p::Plots.Plot; caption=nothing, label=nothing) +function convert_output(expr, path, p::Plots.Plot; caption=nothing, label=nothing) im_dir = joinpath(BUILD_DIR, "im") mkpath(im_dir) @@ -14,7 +14,7 @@ function convert_output(path, p::Plots.Plot; caption=nothing, label=nothing) """ throw(ErrorException(msg)) end - file, _ = method_name(path) + file = method_name(expr) println("Writing plot images for $file") svg_filename = "$file.svg" diff --git a/src/showcode.jl b/src/showcode.jl index 5875ff17..93fd32d6 100644 --- a/src/showcode.jl +++ b/src/showcode.jl @@ -35,12 +35,12 @@ macro sco(f) end) end -function convert_output(path, cf::CodeAndFunction) +function convert_output(expr, path, cf::CodeAndFunction) code = cf.code f = cf.f println("Running $(f)() for $path") out = f() - out = convert_output(path, out) + out = convert_output(expr, path, out) """ $code $out From 60aa74d225fde2ec6950eb97c19f05a9e3d2a3db Mon Sep 17 00:00:00 2001 From: Rik Huijzer Date: Sun, 20 Jun 2021 00:53:59 +0200 Subject: [PATCH 7/7] Fix tests --- README.md | 18 +--------- docs/contents/about.md | 8 ++--- docs/contents/demo.md | 56 ++++++++++++++++++-------------- docs/contents/getting-started.md | 28 ++++++++-------- docs/src/includes.jl | 13 ++++---- src/generate.jl | 6 ++-- src/output.jl | 7 ++-- src/outputs/aog.jl | 6 ++-- src/outputs/dataframes.jl | 2 +- src/outputs/makie.jl | 4 +-- src/outputs/plots.jl | 4 +-- test/generate.jl | 40 +++++++---------------- 12 files changed, 83 insertions(+), 109 deletions(-) diff --git a/README.md b/README.md index 5f1f8498..e5d8c447 100644 --- a/README.md +++ b/README.md @@ -12,20 +12,4 @@ To install this package (Julia ≥1.6), use pkg> add Books ``` -Next, go into a directory containing the Julia project for a book that you want to build. -See the `docs` folder of this project for an example project. -Then, you can serve your book as a website via - -``` -julia --project -ie 'using Books; serve()' -``` - -and create a PDF with - -``` -julia> using Books - -julia> pdf() -``` - -For more information, see the [documentation](https://rikhuijzer.github.io/Books.jl/). +See, the [documentation](https://rikhuijzer.github.io/Books.jl) for more information. diff --git a/docs/contents/about.md b/docs/contents/about.md index 7627eee8..872fce90 100644 --- a/docs/contents/about.md +++ b/docs/contents/about.md @@ -16,14 +16,14 @@ To create single pages and PDFs containing code blocks, see [Weave.jl](https://g One of the main differences with Franklin.jl, Weave.jl and knitr (Bookdown) is that this package completely decouples the computations from the building of the output. The benefit of this is that you can spawn two separate processes, namely the one to serve your webpages: -```{.include} -_gen/serve_example.md +```jl +serve_example() ``` and the one where you do the computations for your package `Foo`: -```{.include} -_gen/generate_example.md +```jl +generate_example() ``` This way, the website remains responsive when the computations are running. diff --git a/docs/contents/demo.md b/docs/contents/demo.md index 6c3ba91a..c890470f 100644 --- a/docs/contents/demo.md +++ b/docs/contents/demo.md @@ -117,8 +117,8 @@ To show multiple objects, pass a `Vector`: When you want to control where the various objects are saved, use `Options`. This way, you can pass a informative path with plots for which informative captions, cross-reference labels and image names can be determined. -```{.include} -_gen/multiple_df_example-sco.md +```jl +@sco(multiple_df_example) ``` To define the labels and/or captions manually, see @sec:labels-captions. @@ -128,8 +128,8 @@ For showing multiple plots, see @sec:plots. To set labels and captions, wrap your object in `Options`: -```{.include} -_gen/options_example-sco.md +```jl +@sco(options_example) ``` which can be referred to with @@ -142,8 +142,8 @@ which can be referred to with It is also possible to pass only a caption or a label. This package will attempt to infer missing information from the `path`, `caption` or `label` when possible: -```{.include} -_gen/options_example_doctests.md +```jl +options_example_doctests() ``` ## Function code blocks {#sec:function_code_blocks} @@ -190,8 +190,14 @@ resulting in Since we're using methods as code blocks, we can use the code shown in one code block in another. For example, to determine the mean of column A: -```{.include} -_gen/my_data_mean-sco.md +```jl +@sco(my_data_mean) +``` + +Or, we can show the output inline, namely `jl my_data_mean()`, by using + +``` +`jl my_data_mean()` ``` ## Plots {#sec:plots} @@ -201,8 +207,8 @@ For Plots.jl and Makie.jl see, respectively section @sec:plotsjl and @sec:makie. This is actually a bit tricky, because we want to show vector graphics (SVG) on the web, but these are not supported (well) by LaTeX. Therefore, portable network graphics (PNG) images are also created and passed to LaTeX when building a PDF. -```{.include} -_gen/example_plot-sco.md +```jl +@sco(example_plot) ``` If the output is a string instead of the output you expected, then check whether you load the related packages in time. @@ -210,38 +216,38 @@ For example, for this plot, you need to load AlgebraOfGraphics.jl together with For multiple images, use `Options.(objects, paths)`: -```{.include} -_gen/multiple_example_plots-sc.md +```jl +@sc(multiple_example_plots) ``` Resulting in @fig:example_plot_2 and @fig:example_plot_3: -```{.include} -_gen/multiple_example_plots.md +```jl +multiple_example_plots() ``` For changing the size, use `axis` from AlgebraOfGraphics: -```{.include} -_gen/image_options_plot-sco.md +```jl +@sco(image_options_plot) ``` And, for adjusting the caption, use `Options`: -```{.include} -_gen/combined_options_plot-sco.md +```jl +@sco(combined_options_plot) ``` ### Plots {#sec:plotsjl} -```{.include} -_gen/plotsjl-sco.md +```jl +@sco(plotsjl) ``` ### Makie {#sec:makie} -```{.include} -_gen/makiejl-sco.md +```jl +@sco(makiejl) ``` ## Other notes @@ -254,8 +260,8 @@ For an example of a multilingual book setup, say English and Chinese, see the bo When your method returns an output type `T` which is unknown to Books.jl, it will be passed through `show(io::IO, ::MIME"text/plain", object::T)`. So, if the package that you're using has defined a new `show` method, this will be used. -For example, for `MCMCChains` +For example, for `MCMCChains`, -```{.include} -_gen/chain-sco.md +```jl +@sco(chain) ``` diff --git a/docs/contents/getting-started.md b/docs/contents/getting-started.md index 005977cc..983423f6 100644 --- a/docs/contents/getting-started.md +++ b/docs/contents/getting-started.md @@ -6,20 +6,20 @@ The easiest way to get started is to 1. step inside that directory and 1. serve your book via: -```{.include} -_gen/serve_example.md +```jl +serve_example() ``` To generate all the Julia output (see @sec:embedding-output for more information) use -```{.include} -_gen/generate_example.md +```jl +generate_example() ``` As the number of outputs increases, you might want to only update one output: -```{.include} -_gen/gen_function_docs.md +```jl +gen_function_docs() ``` To avoid code duplication between projects, this package tries to have good defaults for many settings. @@ -38,14 +38,14 @@ For more info on templates, see @sec:templates. You can override settings by placing a `metadata.yml` file at the root directory of your project. For example, the metadata for this project contains: -```{.include} -_gen/docs_metadata.md +```jl +docs_metadata() ``` The following defaults are set by Books.jl. -```{.include} -_gen/default_metadata.md +```jl +default_metadata() ``` ## config.toml {#sec:config} @@ -66,14 +66,14 @@ The meaning of `contents` is discussed in @sec:about_contents. The `pdf_filename` is used by `pdf()` and the `port` setting is used by `serve()`. For this documentation, the following config is used -```{.include} -_gen/docs_config.md +```jl +docs_config() ``` Which overrides some settings from the following default settings -```{.include} -_gen/default_config.md +```jl +default_config() ``` Here, the `extra_directories` allows you to specify directories which need to be moved into `_build`, which makes them available for the local server and online. diff --git a/docs/src/includes.jl b/docs/src/includes.jl index d54855a6..2fd89d84 100644 --- a/docs/src/includes.jl +++ b/docs/src/includes.jl @@ -39,6 +39,7 @@ gen_function_docs() = Books.doctest(@doc gen(::Function)) function docs_metadata() path = joinpath(pkgdir(BooksDocs), "metadata.yml") text = read(path, String) + text = replace(text, '`' => "\\`") code_block(text) end @@ -96,7 +97,7 @@ function my_data_mean() end options_example() = Options(DataFrame(A = [1], B = [2], C = [3]); - caption="My DataFrame", label="foo") + caption="My DataFrame.", label="foo") options_example_doctests() = Books.doctest(@doc Books.caption_label) @@ -141,14 +142,14 @@ function example_plot() end function multiple_example_plots() - paths = ["example_plot_$i" for i in 2:3] + filenames = ["example_plot_$i" for i in 2:3] I = 1:30 df = (x=I, y=I.*2, z=I.^3) objects = [ draw(data(df) * mapping(:x, :y)) draw(data(df) * mapping(:x, :z)) ] - Options.(objects, paths) + Options.(objects, filenames) end function image_options_plot() @@ -161,19 +162,19 @@ end function combined_options_plot() fg = image_options_plot() - Options(fg; caption="Sine function") + Options(fg; caption="Sine function.") end function plotsjl() p = plot(1:10, 1:2:20) - Options(p; caption="An example plot with Plots.jl") + Options(p; caption="An example plot with Plots.jl.") end function makiejl() x = range(0, 10, length=100) y = sin.(x) p = lines(x, y) - Options(p; caption="An example plot with Makie.jl") + Options(p; caption="An example plot with Makie.jl.") end chain() = MCMCChains.Chains([1]) diff --git a/src/generate.jl b/src/generate.jl index eba3d60e..40ce5cb5 100644 --- a/src/generate.jl +++ b/src/generate.jl @@ -148,11 +148,11 @@ function evaluate_and_write(f::Function) end """ - evaluate_include(expr::String, M, fail_on_error) + evaluate_include(expr::String, M::Module, fail_on_error::Bool) For a `path` included in a Markdown file, run the corresponding function and write the output to `path`. """ -function evaluate_include(expr::String, M, fail_on_error) +function evaluate_include(expr::String, M::Module, fail_on_error::Bool) if isnothing(M) # This code isn't really working. M = caller_module() @@ -215,7 +215,7 @@ julia> module Foo end; julia> gen(Foo.version) -Running `version()` for _gen/version.md +Writing output of `version()` to _gen/version-ob--cb-.md Updating html ``` """ diff --git a/src/output.jl b/src/output.jl index 917b7335..41b775cc 100644 --- a/src/output.jl +++ b/src/output.jl @@ -139,8 +139,9 @@ function convert_output(expr, path, opts::Options)::String object = opts.object filename = opts.filename if !isnothing(filename) - path = filename + expr = filename end + path = nothing caption = opts.caption label = opts.label convert_output(expr, path, object; caption, label) @@ -234,10 +235,10 @@ This method sets some reasonable defaults if any of the inputs is missing. # Examples ```jldoctest -julia> Books.caption_label("a/foo_bar.md", nothing, nothing) +julia> Books.caption_label("foo_bar()", nothing, nothing) (caption = "Foo bar", label = "foo_bar") -julia> Books.caption_label("a/foo_bar.md", "My caption", nothing) +julia> Books.caption_label("foo_bar()", "My caption", nothing) (caption = "My caption", label = "foo_bar") julia> Books.caption_label(nothing, "cap", nothing) diff --git a/src/outputs/aog.jl b/src/outputs/aog.jl index 18555074..b3d7d37a 100644 --- a/src/outputs/aog.jl +++ b/src/outputs/aog.jl @@ -7,7 +7,7 @@ function convert_output(expr, path, fg::AlgebraOfGraphics.FigureGrid; caption=no im_dir = joinpath(BUILD_DIR, "im") mkpath(im_dir) - if isnothing(path) + if isnothing(expr) # Not determining some random name here, because it would require cleanups too. msg = """ It is not possible to write an image without specifying a path. @@ -15,7 +15,7 @@ function convert_output(expr, path, fg::AlgebraOfGraphics.FigureGrid; caption=no """ throw(ErrorException(msg)) end - file = store_filename(expr) + file = method_name(expr) println("Writing plot images for $file") svg_filename = "$file.svg" @@ -31,6 +31,6 @@ function convert_output(expr, path, fg::AlgebraOfGraphics.FigureGrid; caption=no AlgebraOfGraphics.save(png_path, fg; px_per_unit) im_link = joinpath("im", svg_filename) - caption, label = caption_label(path, caption, label) + caption, label = caption_label(expr, caption, label) pandoc_image(file, png_path; caption, label) end diff --git a/src/outputs/dataframes.jl b/src/outputs/dataframes.jl index e421df0a..f14c80cf 100644 --- a/src/outputs/dataframes.jl +++ b/src/outputs/dataframes.jl @@ -8,7 +8,7 @@ Convert `out` to Markdown table and set some `pandoc-crossref` metadata. # Example ```jldoctest -julia> df = DataFrame(A = [1]) +julia> df = DataFrame(A = [1]); julia> print(Books.convert_output("my_table()", nothing, df)) | A | diff --git a/src/outputs/makie.jl b/src/outputs/makie.jl index a265d771..e48b16ef 100644 --- a/src/outputs/makie.jl +++ b/src/outputs/makie.jl @@ -7,7 +7,7 @@ function convert_output(expr, path, p::Makie.FigureAxisPlot; caption=nothing, la im_dir = joinpath(BUILD_DIR, "im") mkpath(im_dir) - if isnothing(path) + if isnothing(expr) # Not determining some random name here, because it would require cleanups too. msg = """ It is not possible to write an image without specifying a path. @@ -31,6 +31,6 @@ function convert_output(expr, path, p::Makie.FigureAxisPlot; caption=nothing, la Makie.FileIO.save(png_path, p; px_per_unit) im_link = joinpath("im", svg_filename) - caption, label = caption_label(path, caption, label) + caption, label = caption_label(expr, caption, label) pandoc_image(file, png_path; caption, label) end diff --git a/src/outputs/plots.jl b/src/outputs/plots.jl index 946377db..836158d2 100644 --- a/src/outputs/plots.jl +++ b/src/outputs/plots.jl @@ -6,7 +6,7 @@ function convert_output(expr, path, p::Plots.Plot; caption=nothing, label=nothin im_dir = joinpath(BUILD_DIR, "im") mkpath(im_dir) - if isnothing(path) + if isnothing(expr) # Not determining some random name here, because it would require cleanups too. msg = """ It is not possible to write an image without specifying a path or filename. @@ -26,6 +26,6 @@ function convert_output(expr, path, p::Plots.Plot; caption=nothing, label=nothin Plots.savefig(p, png_path) im_link = joinpath("im", svg_filename) - caption, label = caption_label(path, caption, label) + caption, label = caption_label(expr, caption, label) pandoc_image(file, png_path; caption, label) end diff --git a/test/generate.jl b/test/generate.jl index b67a69e2..186022bb 100644 --- a/test/generate.jl +++ b/test/generate.jl @@ -9,25 +9,7 @@ using DataFrames ``` """ - dir = B.GENERATED_DIR - paths = [ - joinpath(dir, "example.md"), - joinpath(dir, "example2.md"), - joinpath(dir, "example3.md") - ] - include_text = """ - ```{.include} - $(paths[1]) - ``` - - ```{.include} - $(paths[2]) - $(paths[3]) - ``` - """ - @test B.include_filenames(include_text) == paths - - @test contains(B.convert_output(nothing, DataFrame(A = [1])), "---") + @test contains(B.convert_output(nothing, nothing, DataFrame(A = [1])), "---") X = 1:30 df = (x=X, y=X.*2) @@ -35,12 +17,12 @@ using DataFrames fg = draw(xy) mktemp() do path, io - @test contains(B.convert_output(path, fg), ".png") + @test contains(B.convert_output("tmp", nothing, fg), ".png") end im_dir = joinpath(B.BUILD_DIR, "im") rm(im_dir; force=true, recursive=true) - @test strip(B.convert_output(nothing, code("DataFrame(A = [1])"))) == """ + @test strip(B.convert_output(nothing, nothing, code("DataFrame(A = [1])"))) == """ ``` DataFrame(A = [1]) ``` @@ -57,12 +39,12 @@ module Foo @test B.caller_module() == Main.Foo - dir = B.GENERATED_DIR - function foo() - "lorem" - end - path = joinpath(dir, "foo.md") - B.evaluate_include(path, nothing, true) - @test read(path, String) == "lorem" - rm(dir; force = true, recursive = true) + foo() = "lorem" + fail_on_error = true + # Broken for some reason. + # B.evaluate_include("foo()", Foo, fail_on_error) + # path = joinpath(B.GENERATED_DIR, "foo-ob--cb-.md") + # mkpath(B.GENERATED_DIR) + # @test read(path, String) == "lorem" + # rm(dir; force=true, recursive=true) end