From ba352b50b215745395bf80a6ed8cb2c71f6c0cff Mon Sep 17 00:00:00 2001 From: Rik Huijzer Date: Mon, 21 Jun 2021 23:34:09 +0200 Subject: [PATCH 1/2] Add sc and sco (#128) --- docs/contents/demo.md | 53 ++++++++++++++++++++++ src/Books.jl | 2 +- src/generate.jl | 31 +++++++------ src/include-codeblocks.lua | 91 +++++++++++++++++-------------------- src/include-files.lua | 92 -------------------------------------- src/showcode.jl | 48 +++++++++++++++++++- 6 files changed, 156 insertions(+), 161 deletions(-) delete mode 100644 src/include-files.lua diff --git a/docs/contents/demo.md b/docs/contents/demo.md index 88297592..bc91c87f 100644 --- a/docs/contents/demo.md +++ b/docs/contents/demo.md @@ -138,6 +138,59 @@ This way, you can pass a informative path with plots for which informative capti To define the labels and/or captions manually, see @sec:labels-captions. For showing multiple plots, see @sec:plots. +Most things can be done via functions. +However, defining a struct is not possible, because `@sco` cannot locate the struct definition inside the module. +Therefore, it is also possible to pass code and specify that you want to evaluate and show code (sc) without showing the output: + +
+```jl
+sc("
+struct Point
+    x
+    y
+end
+")
+```
+
+ +```jl +sc(" +struct Point + x + y +end +") +``` + +and show code and output (sco). +For example, + +
+```jl
+sco("p = Point(1, 2)")
+```
+
+ +shows as + +```jl +sco("p = Point(1, 2)") +``` + +Note that this is starting to look a lot like R Markdown where the syntax would be something like + +
+```{r, results='hide'}
+x = rnorm(100)
+```
+
+ +I guess that there is no perfect way here. +The benefit of evaluating the user input directly, as Books.jl is doing, seems to be that it is more extensible if I'm not mistaken. +Possibly, the reasoning is that R Markdown needs to convert the output directly, whereas Julia's better type system allows for converting in much later stages, but I'm not sure. + +> **Tip**: After you run `gen(; M)` with the `Point` struct defined above, the struct will be available in your REPL. + ## Labels and captions {#sec:labels-captions} To set labels and captions, wrap your object in `Options`: diff --git a/src/Books.jl b/src/Books.jl index 0aad7371..0cf06051 100644 --- a/src/Books.jl +++ b/src/Books.jl @@ -31,7 +31,7 @@ include("generate.jl") export html, pdf, docx, build_all export code, ImageOptions, Options export code_block -export @sc, CodeAndFunction, @sco +export @sc, sc, CodeAndFunction, @sco, sco export gen export serve diff --git a/src/generate.jl b/src/generate.jl index ba60c93a..7be27991 100644 --- a/src/generate.jl +++ b/src/generate.jl @@ -14,6 +14,10 @@ extract_expr_example() = """ ```jl foo(3) ``` + ```jl + foo(3) + bar + ``` ipsum `jl bar()` dolar """ @@ -27,8 +31,9 @@ Here, `s` is the contents of a Markdown file. julia> s = Books.extract_expr_example(); julia> Books.extract_expr(s) -2-element Vector{String}: +3-element Vector{String}: "foo(3)" + "foo(3)\\nbar" "bar()" ``` """ @@ -49,7 +54,7 @@ function extract_expr(s::AbstractString)::Vector function check_parse_errors(expr) try - Meta.parse(expr) + Meta.parse("begin $expr end") catch e error("Exception occured when trying to parse `$expr`") end @@ -100,7 +105,6 @@ julia> Books.method_name("Options(foo(); caption='b')") function method_name(expr::String) remove_macros(expr) = replace(expr, r"@[\w\_]*" => "") expr = remove_macros(expr) - # These rewrites are not reversible, because they do not have to be. expr = replace(expr, '(' => '_') expr = replace(expr, ')' => "") expr = replace(expr, ';' => "_") @@ -119,23 +123,17 @@ Escape an expression to the corresponding path. The logic in this method should match the logic in the Lua filter. """ function escape_expr(expr::String) - replace_map = [ - '(' => "-ob-", - ')' => "-cb-", - '"' => "-dq-", - ':' => "-fc-", - ';' => "-sc-", - '@' => "-ax-" - ] - escaped = reduce(replace, replace_map; init=expr) + escaped = 60 < length(expr) ? expr[1:60] : expr + escaped = replace(escaped, r"([^a-zA-Z0-9]+)" => "_") joinpath(GENERATED_DIR, "$escaped.md") end function evaluate_and_write(M::Module, expr::String) path = escape_expr(expr) - println("Writing output of `$expr` to $path") + expr_info = replace(expr, '\n' => "\\n") + println("Writing output of `$expr_info` to $path") - ex = Meta.parse(expr) + ex = Meta.parse("begin $expr end") out = Core.eval(M, ex) out = convert_output(expr, path, out) out = string(out)::String @@ -148,7 +146,8 @@ function evaluate_and_write(f::Function) function_name = Base.nameof(f) expr = "$(function_name)()" path = escape_expr(expr) - println("Writing output of `$expr` to $path") + expr_info = replace(expr, '\n' => "\\n") + println("Writing output of `$expr_info` to $path") out = f() out = convert_output(expr, path, out) out = string(out)::String @@ -242,7 +241,7 @@ julia> module Foo julia> call_html = false; # To avoid Pandoc errors breaking this jldoctest. julia> gen(Foo.version; call_html) -Writing output of `version()` to _gen/version-ob--cb-.md +Writing output of `version()` to _gen/version_.md ``` """ function gen(f::Function; project="default", call_html=true) diff --git a/src/include-codeblocks.lua b/src/include-codeblocks.lua index 1b26034b..7d1c609f 100644 --- a/src/include-codeblocks.lua +++ b/src/include-codeblocks.lua @@ -36,19 +36,18 @@ local function shift_headings(blocks, shift_by) end --- Return path of the markdown file for the string `s` given by the user. +--- +--- We can just drop a lot of info because it will probably still be unique. +--- Otherwise, Julia can detect duplicate filenames and throw an error. +--- --- Ensure that this logic corresponds to the logic inside Books.jl. +--- local md_path function md_path(s) - -- Escape all weird characters to ensure they can be in the file. - -- This yields very weird names, but luckily the code is only internal. - escaped = s - escaped = escaped:gsub("%(", "-ob-") - escaped = escaped:gsub("%)", "-cb-") - escaped = escaped:gsub("\"", "-dq-") - escaped = escaped:gsub(":", "-fc-") - escaped = escaped:gsub(";", "-sc-") - escaped = escaped:gsub("@", "-ax-") - path_sep = package.config:sub(1,1) + escaped = string.sub(s, 1, 60) + escaped = escaped:gsub("([^a-zA-Z0-9]+)", "_") + -- Platform independent path separator. + path_sep = package.config:sub(1, 1) path = "_gen" .. path_sep .. escaped .. ".md" return path end @@ -86,48 +85,38 @@ function transclude_codeblock(cb) local buffer_last_heading_level = last_heading_level local blocks = List:new() - for line in cb.text:gmatch('[^\n]+') do - if line:sub(1,2) ~= '//' then - - path = md_path(line) - if 100 < path:len() then - msg = "ERROR: The text `" .. line .. "` is too long to be converted to a filename" - msg = { pandoc.CodeBlock(msg) } - blocks:extend(msg) - -- Lua has no continue. - goto skip_to_next - end - - local fh = io.open(path) - if not fh then - not_found_error(line, path, '```') - suggestion = "Did you run `gen(; M)` where `M = YourModule`?\n" - msg = "ERROR: Cannot find file at " .. path .. " for `" .. line .. "`." - msg = msg .. ' ' .. suggestion - msg = { pandoc.CodeBlock(msg) } - blocks:extend(msg) - else - local text = fh:read("*a") - local contents = pandoc.read(text, format).blocks - last_heading_level = 0 - -- recursive transclusion - contents = pandoc.walk_block( - -- Here, the contents is added as an Any block. - -- Then, the filter is applied again recursively because - -- the included file could contain an include again! - pandoc.Div(contents), - { Header = update_last_level, CodeBlock = transclude } - ).content - --- reset to level before recursion - last_heading_level = buffer_last_heading_level - contents = shift_headings(contents, shift_heading_level_by) - -- Note that contents has type List. - blocks:extend(contents) - fh:close() - end - end - ::skip_to_next:: + + + path = md_path(cb.text) + + local fh = io.open(path) + if not fh then + not_found_error(cb.text, path, '```') + suggestion = "Did you run `gen(; M)` where `M = YourModule`?\n" + msg = "ERROR: Cannot find file at " .. path .. " for `" .. cb.text .. "`." + msg = msg .. ' ' .. suggestion + msg = { pandoc.CodeBlock(msg) } + blocks:extend(msg) + else + local text = fh:read("*a") + local contents = pandoc.read(text, format).blocks + last_heading_level = 0 + -- recursive transclusion + contents = pandoc.walk_block( + -- Here, the contents is added as an Any block. + -- Then, the filter is applied again recursively because + -- the included file could contain an include again! + pandoc.Div(contents), + { Header = update_last_level, CodeBlock = transclude } + ).content + --- reset to level before recursion + last_heading_level = buffer_last_heading_level + contents = shift_headings(contents, shift_heading_level_by) + -- Note that contents has type List. + blocks:extend(contents) + fh:close() end + ::skip_to_next:: return blocks end diff --git a/src/include-files.lua b/src/include-files.lua deleted file mode 100644 index f685ec00..00000000 --- a/src/include-files.lua +++ /dev/null @@ -1,92 +0,0 @@ ---- include-files.lua – filter to include Markdown files ---- ---- Copyright: © 2019–2020 Albert Krewinkel ---- License: MIT – see LICENSE file for details - --- pandoc's List type -local List = require 'pandoc.List' - ---- Get include auto mode -local include_auto = false -function get_vars (meta) - if meta['include-auto'] then - include_auto = true - end -end - ---- Keep last heading level found -local last_heading_level = 0 -function update_last_level(header) - last_heading_level = header.level -end - ---- Shift headings in block list by given number -local function shift_headings(blocks, shift_by) - if not shift_by then - return blocks - end - - local shift_headings_filter = { - Header = function (header) - header.level = header.level + shift_by - return header - end - } - - return pandoc.walk_block(pandoc.Div(blocks), shift_headings_filter).content -end - ---- Filter function for code blocks -local transclude -function transclude (cb) - -- ignore code blocks which are not of class "include". - if not cb.classes:includes 'include' then - return - end - - -- Markdown is used if this is nil. - local format = cb.attributes['format'] - - -- Attributes shift headings - local shift_heading_level_by = 0 - local shift_input = cb.attributes['shift-heading-level-by'] - if shift_input then - shift_heading_level_by = tonumber(shift_input) - else - if include_auto then - -- Auto shift headings - shift_heading_level_by = last_heading_level - end - end - - --- keep track of level before recusion - local buffer_last_heading_level = last_heading_level - - local blocks = List:new() - for line in cb.text:gmatch('[^\n]+') do - if line:sub(1,2) ~= '//' then - local fh = io.open(line) - if not fh then - io.stderr:write("Cannot open file " .. line .. " | Skipping includes\n") - else - local contents = pandoc.read(fh:read '*a', format).blocks - last_heading_level = 0 - -- recursive transclusion - contents = pandoc.walk_block( - pandoc.Div(contents), - { Header = update_last_level, CodeBlock = transclude } - ).content - --- reset to level before recursion - last_heading_level = buffer_last_heading_level - blocks:extend(shift_headings(contents, shift_heading_level_by)) - fh:close() - end - end - end - return blocks -end - -return { - { Meta = get_vars }, - { Header = update_last_level, CodeBlock = transclude } -} diff --git a/src/showcode.jl b/src/showcode.jl index 93fd32d6..e6a5c0cc 100644 --- a/src/showcode.jl +++ b/src/showcode.jl @@ -38,7 +38,6 @@ end function convert_output(expr, path, cf::CodeAndFunction) code = cf.code f = cf.f - println("Running $(f)() for $path") out = f() out = convert_output(expr, path, out) """ @@ -46,3 +45,50 @@ function convert_output(expr, path, cf::CodeAndFunction) $out """ end + + +""" + CodeAndOutput(code::AbstractString) + +This struct is used by [`sco`](@ref). +""" +struct CodeAndOutput + code::AbstractString +end + +""" + eval_convert(expr::AbstractString) + +Evaluate `expr` and convert the output. +This should be evaluated inside the correct module since it is typically called +inside `Core.eval(M, ex)` in `generate.jl`. +""" +function eval_convert(expr::AbstractString, M) + ex = Meta.parse(expr) + out = Core.eval(M, ex) + out = convert_output(expr, nothing, out) +end + +""" + sco(expr::AbstractString; M=Main) + +Show code and output for `expr`. +""" +function sco(expr::AbstractString; M=Main) + out = eval_convert(expr, M) + code = code_block(lstrip(expr)) + """ + $code + $out + """ +end + +""" + sc(expr::AbstractString; M=Main) + +Show only code for `expr`, that is, evaluate `expr` but hide the output. +""" +function sc(expr::AbstractString; M=Main) + eval_convert(expr, M) + code_block(lstrip(expr)) +end From 647151601420fed3c469746af06eb8df2b5e8173 Mon Sep 17 00:00:00 2001 From: Rik Huijzer Date: Tue, 22 Jun 2021 17:45:49 +0200 Subject: [PATCH 2/2] Fix some problems found by JET (#129) --- src/Books.jl | 2 +- src/build.jl | 2 +- src/generate.jl | 12 ++++++------ src/html.jl | 24 ++++++++++++------------ 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/Books.jl b/src/Books.jl index 0cf06051..39c7ffe2 100644 --- a/src/Books.jl +++ b/src/Books.jl @@ -12,7 +12,7 @@ using Requires using pandoc_jll using pandoc_crossref_jll -const PROJECT_ROOT = pkgdir(Books) +const PROJECT_ROOT = string(pkgdir(Books))::String const GENERATED_DIR = "_gen" const DEFAULTS_DIR = joinpath(PROJECT_ROOT, "defaults") const BUILD_DIR = "_build" diff --git a/src/build.jl b/src/build.jl index 7749d040..ab03aeb3 100644 --- a/src/build.jl +++ b/src/build.jl @@ -117,7 +117,7 @@ function html(; project="default", extra_head="") copy_extra_directories(project) url_prefix = is_ci() ? ci_url_prefix(project) : "" c = config(project, "contents") - write_html_pages(url_prefix, c, pandoc_html(project), extra_head) + write_html_pages(url_prefix, pandoc_html(project), extra_head) end """ diff --git a/src/generate.jl b/src/generate.jl index 7be27991..881b5a39 100644 --- a/src/generate.jl +++ b/src/generate.jl @@ -41,7 +41,7 @@ function extract_expr(s::AbstractString)::Vector codeblock_pattern = r"```jl\s*([\w\W]*?)```" matches = eachmatch(codeblock_pattern, s) function clean(m) - m = m[1] + m = m[1]::SubString{String} m = strip(m) m = string(m)::String end @@ -157,11 +157,11 @@ function evaluate_and_write(f::Function) end """ - evaluate_include(expr::String, M::Module, fail_on_error::Bool) + evaluate_include(expr::String, M, fail_on_error::Bool) For a `path` included in a Markdown file, run the corresponding function and write the output to `path`. """ -function evaluate_include(expr::String, M::Module, fail_on_error::Bool) +function evaluate_include(expr::String, M, fail_on_error::Bool) if isnothing(M) # This code isn't really working. M = caller_module() @@ -192,7 +192,7 @@ function expand_path(p) end """ - gen(paths::Vector; M=nothing, fail_on_error=false, project="default") + gen(paths::Vector; M=Main, fail_on_error=false, project="default") Populate the files in `$(Books.GENERATED_DIR)/` by calling the required methods. These methods are specified by the filename and will output to that filename. @@ -202,7 +202,7 @@ Otherwise, specify another module `M`. After calling the methods, this method will also call `html()` to update the site when `call_html == true`. """ -function gen(paths::Vector; M=nothing, fail_on_error=false, project="default", call_html=true) +function gen(paths::Vector; M=Main, fail_on_error=false, project="default", call_html=true) mkpath(GENERATED_DIR) paths = [contains(dirname(p), "contents") ? p : expand_path(p) for p in paths] included_expr = vcat([extract_expr(read(path, String)) for path in paths]...) @@ -215,7 +215,7 @@ function gen(paths::Vector; M=nothing, fail_on_error=false, project="default", c end gen(path::String; kwargs...) = gen([path]; kwargs...) -function gen(; M=nothing, fail_on_error=false, project="default", call_html=true) +function gen(; M=Main, fail_on_error=false, project="default", call_html=true) paths = inputs(project) first_file = first(paths) if !isfile(first_file) diff --git a/src/html.jl b/src/html.jl index f7fefada..b7279583 100644 --- a/src/html.jl +++ b/src/html.jl @@ -2,14 +2,14 @@ import YAML import URIs """ - split_keepdelim(str::AbstractString, dlm::Regex) + split_keepdelim(str::AbstractString, delim::Regex) Split on regex while keeping the matches. Based on https://github.com/JuliaLang/julia/issues/20625#issuecomment-581585151. """ -function split_keepdelim(str::AbstractString, dlm::Regex) - dlm = string(dlm)[3:end-1] - rx = Regex("(?=$dlm)") +function split_keepdelim(str::AbstractString, delim::Regex) + delim = string(delim)[3:end-1] + rx = Regex("(?=$delim)") split(str, rx) end @@ -79,7 +79,7 @@ function section_infos(text) for line in lines m = match(numbered_rx, line) if !isnothing(m) - number, id = m.captures + number, id = m.captures line_end = split(line, '>')[end-1] text = line_end[nextind(line_end, 0, 2):prevind(line_end, end, 4)] tuple = (num = number, id = id, text = lstrip(text)) @@ -116,7 +116,7 @@ end function html_href(text, link, level) threshold = 33 if threshold < length(text) - shortened = text[1:threshold] + shortened = text[1:threshold]::String text = shortened * ".." end """$text""" @@ -139,7 +139,7 @@ end Menu including numbered sections. """ -function add_menu(splitted=split_html()) +function add_menu(splitted) head, bodies, foot = splitted data = pandoc_metadata() title = data["title"] @@ -224,7 +224,7 @@ function add_extra_head(head, extra_head::AbstractString) replace(head, before => after) end -function html_pages(chs=chapters(), h=pandoc_html(), extra_head="") +function html_pages(h, extra_head="") head, menu, bodies, foot = add_menu(split_html(h)) head = add_extra_head(head, extra_head) ids_texts = html_page_name.(bodies) @@ -247,7 +247,7 @@ function map_ids(names, pages) html = page matches = eachmatch(rx, html) for m in matches - capture = first(m.captures) + capture = first(m.captures)::SubString{String} if startswith(capture, "sec:") key = '#' * capture mapping[key] = name @@ -269,7 +269,7 @@ function fix_links(names, pages, url_prefix) updated_pages = [] function fix_page(name, page) function replace_match(s) - capture = first(match(rx, s).captures) + capture = first(match(rx, s).captures)::SubString{String} if startswith(capture, "#sec:") page_link = mapping[capture] return uncapture("$url_prefix/$page_link.html$capture") @@ -290,9 +290,9 @@ function fix_links(names, pages, url_prefix) (names, fixed_pages) end -function write_html_pages(url_prefix, chs=chapters(), h=pandoc_html(), extra_head="") +function write_html_pages(url_prefix, h::AbstractString, extra_head="") h = fix_image_urls(h, url_prefix) - names, pages = html_pages(chs, h, extra_head) + names, pages = html_pages(h, extra_head) names, pages = fix_links(names, pages, url_prefix) for (i, (name, page)) in enumerate(zip(names, pages)) name = i == 1 ? "index" : name