From a4d3408500d47aef94d76537eb50c12d40473712 Mon Sep 17 00:00:00 2001
From: Rik Huijzer <t.h.huijzer@rug.nl>
Date: Sun, 20 Jun 2021 01:07:50 +0200
Subject: [PATCH] Optimize filter for includes (#118)

---
 README.md                        |  18 +--
 docs/contents/about.md           |   8 +-
 docs/contents/demo.md            | 218 +++++++++++--------------------
 docs/contents/getting-started.md |  28 ++--
 docs/contents/index.md           |   4 +-
 docs/src/includes.jl             |  13 +-
 src/build.jl                     |   4 +-
 src/generate.jl                  | 208 +++++++++++++++--------------
 src/include-codeblocks.lua       | 175 +++++++++++++++++++++++++
 src/output.jl                    |  49 +++----
 src/outputs/aog.jl               |   8 +-
 src/outputs/dataframes.jl        |  11 +-
 src/outputs/makie.jl             |   8 +-
 src/outputs/plots.jl             |   8 +-
 src/showcode.jl                  |   4 +-
 test/generate.jl                 |  40 ++----
 16 files changed, 446 insertions(+), 358 deletions(-)
 create mode 100644 src/include-codeblocks.lua

diff --git a/README.md b/README.md
index 5f1f8498..e5d8c447 100644
--- a/README.md
+++ b/README.md
@@ -12,20 +12,4 @@ To install this package (Julia ≥1.6), use
 pkg> add Books
 ```
 
-Next, go into a directory containing the Julia project for a book that you want to build.
-See the `docs` folder of this project for an example project.
-Then, you can serve your book as a website via
-
-```
-julia --project -ie 'using Books; serve()'
-```
-
-and create a PDF with
-
-```
-julia> using Books
-
-julia> pdf()
-```
-
-For more information, see the [documentation](https://rikhuijzer.github.io/Books.jl/).
+See, the [documentation](https://rikhuijzer.github.io/Books.jl) for more information.
diff --git a/docs/contents/about.md b/docs/contents/about.md
index 7627eee8..872fce90 100644
--- a/docs/contents/about.md
+++ b/docs/contents/about.md
@@ -16,14 +16,14 @@ To create single pages and PDFs containing code blocks, see [Weave.jl](https://g
 One of the main differences with Franklin.jl, Weave.jl and knitr (Bookdown) is that this package completely decouples the computations from the building of the output.
 The benefit of this is that you can spawn two separate processes, namely the one to serve your webpages:
 
-```{.include}
-_gen/serve_example.md
+```jl
+serve_example()
 ```
 
 and the one where you do the computations for your package `Foo`:
 
-```{.include}
-_gen/generate_example.md
+```jl
+generate_example()
 ```
 
 This way, the website remains responsive when the computations are running.
diff --git a/docs/contents/demo.md b/docs/contents/demo.md
index b328f8cf..c890470f 100644
--- a/docs/contents/demo.md
+++ b/docs/contents/demo.md
@@ -15,33 +15,39 @@ $$ y = \frac{\sin{x}}{\cos{x}} $$ {#eq:example}
 
 ## Embedding output {#sec:embedding-output}
 
-For embedding code, you can use the `{.include}` code block.
-This package will run your methods based on the filenames in these code blocks.
-For example, to show the Julia version, use
+For embedding code, you can use the `jl` inline code or code block.
+For example, to show the Julia version, define a code block like
 
 <pre>
-```{.include}
-_gen/julia_version.md
+```jl
+julia_version()
 ```
 </pre>
 
+in a Markdown file.
 Then, in your package, define the method `julia_version()`:
+
 ```
 julia_version() = "This book is built with Julia $VERSION."
 ```
 
-Next, ensure that you call `using Books; gen(; M = Foo)`, where `Foo` is the name of your module.
+Next, ensure that you call `using Books; gen(; M)`, where `M = YourModule`.
 This will place the text
 
-```{.include}
-_gen/julia_version_example.md
+```jl
+julia_version_example()
+```
+
+at the right path so that it can be included by Pandoc.
+You can also embed the output inline with single backticks like
+
+```
+`jl julia_version()`
 ```
 
-at the aforementioned path so that it can be included by Pandoc.
 While doing this, it is expected that you also have the browser open and a server running, see @sec:getting-started.
 That way, the page is immediately updated when you run `gen`.
 
-
 Note that it doesn't matter where you define the function `julia_version`, as long as it is in your module.
 To save yourself some typing, and to allow yourself to get some coffee while Julia gets up to speed, you can start Julia for some package `Foo` with
 
@@ -68,29 +74,30 @@ To run this method automatically when you make a change in your package, ensure
 ```
 julia> f() = gen(M.my_plot);
 
-julia> entr(f, [], [M])
+julia> entr(f, ["contents"], [M])
 [...]
 ```
 
-In the background, `gen` passes the methods through `convert_output(path, out::T)` where `T` can, for example, be a DataFrame or a plot.
+Which will automatically run `f()` whenever one of the files in `contents/` changes or any code in the module `M`.
+In the background, `gen` passes the methods through `convert_output(expr::String, path, out::T)` where `T` can, for example, be a DataFrame or a plot.
 To show that a DataFrame is converted to a Markdown table, we define a method
 
-```{.include}
-_gen/my_table-sc.md
+```jl
+@sc(my_table)
 ```
 
 and add its output to the Markdown file with
 
 <pre>
-```{.include}
-_gen/my_table.md
+```jl
+my_table()
 ```
 </pre>
 
 Then, it will show as
 
-```{.include}
-_gen/my_table.md
+```jl
+my_table()
 ```
 
 where the caption and the label are inferred from the `path`.
@@ -103,15 +110,15 @@ Refer to @tbl:my_table with
 
 To show multiple objects, pass a `Vector`:
 
-```{.include}
-_gen/multiple_df_vector-sco.md
+```jl
+@sco(multiple_df_vector)
 ```
 
 When you want to control where the various objects are saved, use `Options`.
 This way, you can pass a informative path with plots for which informative captions, cross-reference labels and image names can be determined.
 
-```{.include}
-_gen/multiple_df_example-sco.md
+```jl
+@sco(multiple_df_example)
 ```
 
 To define the labels and/or captions manually, see @sec:labels-captions.
@@ -121,8 +128,8 @@ For showing multiple plots, see @sec:plots.
 
 To set labels and captions, wrap your object in `Options`:
 
-```{.include}
-_gen/options_example-sco.md
+```jl
+@sco(options_example)
 ```
 
 which can be referred to with
@@ -135,96 +142,8 @@ which can be referred to with
 It is also possible to pass only a caption or a label.
 This package will attempt to infer missing information from the `path`, `caption` or `label` when possible:
 
-```{.include}
-_gen/options_example_doctests.md
-```
-
-## String code blocks {#sec:string_code_blocks}
-
-There are two ways to show code blocks.
-One way is by passing your code as a string.
-This is how similar packages work.
-However, with `Books.jl`, the aim is to work with functions and *not* with code as strings as discussed at the end of @sec:about.
-See @sec:function_code_blocks for a better way for showing code blocks.
-
-Like in @sec:embedding-output, first define a method like
-
-```{.include}
-_gen/sum_example_definition.md
-```
-
-Then, add this method via
-
-<pre>
-```{.include}
-_gen/sum_example.md
-```
-</pre>
-
-which gives as output
-
-```{.include}
-_gen/sum_example.md
-```
-
-Here, how the output should be handled is based on the output type of the function.
-In this case, the output type is of type `Code`.
-Methods for other outputs exist too:
-
-```{.include}
-_gen/example_table_definition.md
-```
-
-shows
-
-```{.include}
-_gen/example_table.md
-```
-
-Alternatively, we can show the same by creating something of type `Code`:
-
-```{.include}
-_gen/code_example_table-sc.md
-```
-
-which shows as
-
-```{.include}
-_gen/code_example_table.md
-```
-
-because the output of the code block is of type DataFrame.
-
-In essence, this package doesn't hide the implementation behind synctactic sugar.
-Instead, this package calls functions and gives you the freedom to decide what to do from there.
-As an example, we can pass `Module` objects to `code` to evaluate the code block in a specific module.
-
-```{.include}
-_gen/module_example_definition.md
-```
-
-When calling `module_example`, it shows as
-
-```{.include}
-_gen/module_example.md
-```
-
-Similarily, we can get the value of x:
-
-```{.include}
-_gen/module_call_x.md
-```
-
-Unsuprisingly, creating a DataFrame will now fail because we haven't loaded DataFrames
-
-```{.include}
-_gen/module_fail.md
-```
-
-Which is easy to fix
-
-```{.include}
-_gen/module_fix.md
+```jl
+options_example_doctests()
 ```
 
 ## Function code blocks {#sec:function_code_blocks}
@@ -233,43 +152,52 @@ So, instead of passing a string which `Books.jl` will evaluate, `Books.jl` can a
 (Thanks to `CodeTracking.@code_string`.)
 For example, we can define the following method:
 
-```{.include}
-_gen/my_data-sc.md
+<pre>
+```jl
+my_data()
 ```
+</pre>
 
-and call it by adding the `-sco` (source code and output) suffix to the path:
+To show code and output (sco), use the `@sco` macro.
+This macro is exported by Books, so ensure that you have `using Books` in your package.
 
 <pre>
-```{.include}
-_gen/my_data-sco.md
+```jl
+@sco(my_data)
 ```
 </pre>
 
 This gives
 
-```{.include}
-_gen/my_data-sco.md
+```jl
+@sco(my_data)
 ```
 
 To only show the source code, use the `-sc` suffix:
 
 <pre>
-```{.include}
-_gen/my_data-sc.md
+```jl
+@sc(my_data)
 ```
 </pre>
 
-giving
+resulting in
 
-```{.include}
-_gen/my_data-sc.md
+```jl
+@sc(my_data)
 ```
 
 Since we're using methods as code blocks, we can use the code shown in one code block in another.
 For example, to determine the mean of column A:
 
-```{.include}
-_gen/my_data_mean-sco.md
+```jl
+@sco(my_data_mean)
+```
+
+Or, we can show the output inline, namely `jl my_data_mean()`, by using
+
+```
+`jl my_data_mean()`
 ```
 
 ## Plots {#sec:plots}
@@ -279,8 +207,8 @@ For Plots.jl and Makie.jl see, respectively section @sec:plotsjl and @sec:makie.
 This is actually a bit tricky, because we want to show vector graphics (SVG) on the web, but these are not supported (well) by LaTeX.
 Therefore, portable network graphics (PNG) images are also created and passed to LaTeX when building a PDF.
 
-```{.include}
-_gen/example_plot-sco.md
+```jl
+@sco(example_plot)
 ```
 
 If the output is a string instead of the output you expected, then check whether you load the related packages in time.
@@ -288,38 +216,38 @@ For example, for this plot, you need to load AlgebraOfGraphics.jl together with
 
 For multiple images, use `Options.(objects, paths)`:
 
-```{.include}
-_gen/multiple_example_plots-sc.md
+```jl
+@sc(multiple_example_plots)
 ```
 
 Resulting in @fig:example_plot_2 and @fig:example_plot_3:
 
-```{.include}
-_gen/multiple_example_plots.md
+```jl
+multiple_example_plots()
 ```
 
 For changing the size, use `axis` from AlgebraOfGraphics:
 
-```{.include}
-_gen/image_options_plot-sco.md
+```jl
+@sco(image_options_plot)
 ```
 
 And, for adjusting the caption, use `Options`:
 
-```{.include}
-_gen/combined_options_plot-sco.md
+```jl
+@sco(combined_options_plot)
 ```
 
 ### Plots {#sec:plotsjl}
 
-```{.include}
-_gen/plotsjl-sco.md
+```jl
+@sco(plotsjl)
 ```
 
 ### Makie {#sec:makie}
 
-```{.include}
-_gen/makiejl-sco.md
+```jl
+@sco(makiejl)
 ```
 
 ## Other notes
@@ -332,8 +260,8 @@ For an example of a multilingual book setup, say English and Chinese, see the bo
 
 When your method returns an output type `T` which is unknown to Books.jl, it will be passed through `show(io::IO, ::MIME"text/plain", object::T)`.
 So, if the package that you're using has defined a new `show` method, this will be used.
-For example, for `MCMCChains`
+For example, for `MCMCChains`,
 
-```{.include}
-_gen/chain-sco.md
+```jl
+@sco(chain)
 ```
diff --git a/docs/contents/getting-started.md b/docs/contents/getting-started.md
index 005977cc..983423f6 100644
--- a/docs/contents/getting-started.md
+++ b/docs/contents/getting-started.md
@@ -6,20 +6,20 @@ The easiest way to get started is to
 1. step inside that directory and
 1. serve your book via:
 
-```{.include}
-_gen/serve_example.md
+```jl
+serve_example()
 ```
 
 To generate all the Julia output (see @sec:embedding-output for more information) use
 
-```{.include}
-_gen/generate_example.md
+```jl
+generate_example()
 ```
 
 As the number of outputs increases, you might want to only update one output:
 
-```{.include}
-_gen/gen_function_docs.md
+```jl
+gen_function_docs()
 ```
 
 To avoid code duplication between projects, this package tries to have good defaults for many settings.
@@ -38,14 +38,14 @@ For more info on templates, see @sec:templates.
 You can override settings by placing a `metadata.yml` file at the root directory of your project.
 For example, the metadata for this project contains:
 
-```{.include}
-_gen/docs_metadata.md
+```jl
+docs_metadata()
 ```
 
 The following defaults are set by Books.jl.
 
-```{.include}
-_gen/default_metadata.md
+```jl
+default_metadata()
 ```
 
 ## config.toml {#sec:config}
@@ -66,14 +66,14 @@ The meaning of `contents` is discussed in @sec:about_contents.
 The `pdf_filename` is used by `pdf()` and the `port` setting is used by `serve()`.
 For this documentation, the following config is used
 
-```{.include}
-_gen/docs_config.md
+```jl
+docs_config()
 ```
 
 Which overrides some settings from the following default settings
 
-```{.include}
-_gen/default_config.md
+```jl
+default_config()
 ```
 
 Here, the `extra_directories` allows you to specify directories which need to be moved into `_build`, which makes them available for the local server and online.
diff --git a/docs/contents/index.md b/docs/contents/index.md
index 33f05a03..ba7d29a8 100644
--- a/docs/contents/index.md
+++ b/docs/contents/index.md
@@ -2,8 +2,8 @@
 
 [//]: # (This file is only included on the website.)
 
-```{.include}
-_gen/homepage_intro.md
+```jl
+homepage_intro()
 ```
 
 See @sec:about for more information about this package.
diff --git a/docs/src/includes.jl b/docs/src/includes.jl
index d54855a6..2fd89d84 100644
--- a/docs/src/includes.jl
+++ b/docs/src/includes.jl
@@ -39,6 +39,7 @@ gen_function_docs() = Books.doctest(@doc gen(::Function))
 function docs_metadata()
     path = joinpath(pkgdir(BooksDocs), "metadata.yml")
     text = read(path, String)
+    text = replace(text, '`' => "\\`")
     code_block(text)
 end
 
@@ -96,7 +97,7 @@ function my_data_mean()
 end
 
 options_example() = Options(DataFrame(A = [1], B = [2], C = [3]);
-                        caption="My DataFrame", label="foo")
+                        caption="My DataFrame.", label="foo")
 
 options_example_doctests() = Books.doctest(@doc Books.caption_label)
 
@@ -141,14 +142,14 @@ function example_plot()
 end
 
 function multiple_example_plots()
-    paths = ["example_plot_$i" for i in 2:3]
+    filenames = ["example_plot_$i" for i in 2:3]
     I = 1:30
     df = (x=I, y=I.*2, z=I.^3)
     objects = [
         draw(data(df) * mapping(:x, :y))
         draw(data(df) * mapping(:x, :z))
     ]
-    Options.(objects, paths)
+    Options.(objects, filenames)
 end
 
 function image_options_plot()
@@ -161,19 +162,19 @@ end
 
 function combined_options_plot()
     fg = image_options_plot()
-    Options(fg; caption="Sine function")
+    Options(fg; caption="Sine function.")
 end
 
 function plotsjl()
     p = plot(1:10, 1:2:20)
-    Options(p; caption="An example plot with Plots.jl")
+    Options(p; caption="An example plot with Plots.jl.")
 end
 
 function makiejl()
     x = range(0, 10, length=100)
     y = sin.(x)
     p = lines(x, y)
-    Options(p; caption="An example plot with Makie.jl")
+    Options(p; caption="An example plot with Makie.jl.")
 end
 
 chain() = MCMCChains.Chains([1])
diff --git a/src/build.jl b/src/build.jl
index f0be2155..7749d040 100644
--- a/src/build.jl
+++ b/src/build.jl
@@ -4,8 +4,8 @@ function pandoc_file(filename)
     isfile(user_path) ? user_path : fallback_path
 end
 
-include_files_lua = joinpath(PROJECT_ROOT, "src", "include-files.lua")
-include_files = "--lua-filter=$include_files_lua"
+include_lua_filter = joinpath(PROJECT_ROOT, "src", "include-codeblocks.lua")
+include_files = "--lua-filter=$include_lua_filter"
 crossref = "--filter=pandoc-crossref"
 citeproc = "--citeproc"
 
diff --git a/src/generate.jl b/src/generate.jl
index b39c6dc5..40ce5cb5 100644
--- a/src/generate.jl
+++ b/src/generate.jl
@@ -1,4 +1,3 @@
-include_regex = r"```{\.include}([\w\W]*?)```"
 
 """
     code_block(s)
@@ -7,15 +6,56 @@ Wrap `s` in a Markdown code block with triple backticks.
 """
 code_block(s) = "```\n$s\n```\n"
 
+function extract_codeblock_expr(s)
+end
+
+extract_expr_example() = """
+    lorem
+    ```jl
+    foo(3)
+    ```
+    ipsum `jl bar()` dolar
+    """
+
 """
-    include_filenames(s::AbstractString)::Vector
+    extract_expr(s::AbstractString)::Vector
 
-Returns the filenames mentioned in `{.include}` code blocks.
+Returns the filenames mentioned in the `jl` code blocks.
+Here, `s` is the contents of a Markdown file.
+
+```jldoctest
+julia> s = Books.extract_expr_example();
+
+julia> Books.extract_expr(s)
+2-element Vector{String}:
+ "foo(3)"
+ "bar()"
+```
 """
-function include_filenames(s::AbstractString)::Vector
-    matches = eachmatch(include_regex, s)
-    nested_filenames = [split(m[1]) for m in matches]
-    vcat(nested_filenames...)
+function extract_expr(s::AbstractString)::Vector
+    codeblock_pattern = r"```jl\s*([\w\W]*?)```"
+    matches = eachmatch(codeblock_pattern, s)
+    function clean(m)
+        m = m[1]
+        m = strip(m)
+        m = string(m)::String
+    end
+    from_codeblocks = clean.(matches)
+
+    inline_pattern = r" `jl ([^`]*)`"
+    matches = eachmatch(inline_pattern, s)
+    from_inline = clean.(matches)
+    E = [from_codeblocks; from_inline]
+
+    function check_parse_errors(expr)
+        try
+            Meta.parse(expr)
+        catch e
+            error("Exception occured when trying to parse `$expr`")
+        end
+    end
+    check_parse_errors.(E)
+    E
 end
 
 """
@@ -37,120 +77,95 @@ function caller_module()
 end
 
 """
-    method_name(path::AbstractString)
+    method_name(expr::String)
 
-Return method name and suffix for a Markdown file.
-Here, the suffix is used to allow users to specify that, for example, `@sc` has to be called on the method.
+Return file name for `expr`.
+This is used for things like how to call an image file and a caption.
 
-# Example
+# Examples
 ```jldoctest
-julia> path = "_gen/foo_bar.md";
-
-julia> Books.method_name(path)
-("foo_bar", "")
+julia> Books.method_name("@some_macro(foo)")
+"foo"
 
-julia> path = "_gen/foo_bar-sc.md";
+julia> Books.method_name("foo()")
+"foo"
 
-julia> Books.method_name(path)
-("foo_bar", "sc")
+julia> Books.method_name("foo(3)")
+"foo_3"
 ```
 """
-function method_name(path::AbstractString)
-    name, extension = splitext(basename(path))
-    suffix = ""
-    if contains(name, '-')
-        parts = split(name, '-')
-        if length(parts) != 2
-            error("Path name is expected to contain at most one - (minus)")
-        end
-        name = parts[1]
-        suffix = parts[2]
-    end
-    (name, suffix)
+function method_name(expr::String)
+    remove_macros(expr) = replace(expr, r"@[\w\_]*" => "")
+    expr = remove_macros(expr)
+    expr = replace(expr, '(' => '_')
+    expr = replace(expr, ')' => "")
+    expr = strip(expr, '_')
 end
 
 """
-    evaluate_and_write(f::Function, path::AbstractString, suffix::AbstractString)
+    escape_expr(expr::String)
 
-Evaluates `f`, converts the output writes the output to `path`.
-Some output conversions will also write to other files, which the file at `path` links to.
-For example, this happens with plots.
-
-# Example
-```jldoctest
-julia> using DataFrames
-
-julia> example_table() = DataFrame(A = [1, 2], B = [3, 4])
-example_table (generic function with 1 method)
-
-julia> path = joinpath(tempdir(), "example.md");
-
-julia> Books.evaluate_and_write(example_table, path, "")
-Running example_table() for /tmp/example.md
-
-julia> print(read(path, String))
-|   A |   B |
-| ---:| ---:|
-|   1 |   3 |
-|   2 |   4 |
-
-: Example {#tbl:example}
-```
+Escape an expression to the corresponding path.
+The logic in this method should match the logic in the Lua filter.
 """
-function evaluate_and_write(f::Function, path::AbstractString, suffix::AbstractString)
-    function run_f(f)
-        println("Running $(f)() for $path")
-        f()
-    end
-    function run_sc(f)
-        println("Obtaining source code for $f()")
-        @sc(f)
-    end
-    function run_sco(f)
-        println("Obtaining source code and output for $f()")
-        @sco(f)
-    end
+function escape_expr(expr::String)
+    replace_map = [
+        '(' => "-ob-",
+        ')' => "-cb-",
+        '"' => "-dq-",
+        ':' => "-fc-",
+        ';' => "-sc-",
+        '@' => "-ax-"
+    ]
+    escaped = reduce(replace, replace_map; init=expr)
+    joinpath(GENERATED_DIR, "$escaped.md")
+end
 
-    out =
-        suffix == "sc" ? run_sc(f) :
-        suffix == "sco" ? run_sco(f) :
-        run_f(f)
+function evaluate_and_write(M::Module, expr::String)
+    path = escape_expr(expr)
+    println("Writing output of `$expr` to $path")
 
-    out = convert_output(path, out)
-    out = String(out)
+    ex = Meta.parse(expr)
+    out = Core.eval(M, ex)
+    out = convert_output(expr, path, out)
+    out = string(out)::String
     write(path, out)
 
     nothing
 end
 
-function evaluate_and_write(M::Module, path)
-    method, suffix = method_name(path)
-    f = getproperty(M, Symbol(method))
-    evaluate_and_write(f, path, suffix)
+function evaluate_and_write(f::Function)
+    function_name = Base.nameof(f)
+    expr = "$(function_name)()"
+    path = escape_expr(expr)
+    println("Writing output of `$expr` to $path")
+    out = f()
+    out = convert_output(expr, path, out)
+    out = string(out)::String
+    write(path, out)
+
+    nothing
 end
 
 """
-    evaluate_include(path, M, fail_on_error)
+    evaluate_include(expr::String, M::Module, fail_on_error::Bool)
 
-For a `path` included in a chapter file, run the corresponding function and write the output to `path`.
+For a `path` included in a Markdown file, run the corresponding function and write the output to `path`.
 """
-function evaluate_include(path, M, fail_on_error)
-    if dirname(path) != GENERATED_DIR
-        println("Not running code for $path")
-        return nothing
-    end
+function evaluate_include(expr::String, M::Module, fail_on_error::Bool)
     if isnothing(M)
+        # This code isn't really working.
         M = caller_module()
     end
-    mkpath(dirname(path))
     if fail_on_error
-        evaluate_and_write(M, path)
+        evaluate_and_write(M, expr)
     else
         try
-            evaluate_and_write(M, path)
+            evaluate_and_write(M, expr)
         catch e
             @error """
-            Failed to run code for $path:
+            Failed to run code for $path.
+            Details:
             $(rethrow())
             """
         end
@@ -170,14 +185,15 @@ After calling the methods, this method will also call `html()` to update the sit
 The module `M` is used to locate the method defined, as a string, in the `.include` via `getproperty`.
 """
 function gen(; M=nothing, fail_on_error=false, project="default", call_html=true)
+    mkpath(GENERATED_DIR)
     paths = inputs(project)
     first_file = first(paths)
     if !isfile(first_file)
         error("Couldn't find $first_file. Is there a valid project in your current working directory?")
     end
-    included_paths = vcat([include_filenames(read(path, String)) for path in paths]...)
-    f(path) = evaluate_include(path, M, fail_on_error)
-    foreach(f, included_paths)
+    included_expr = vcat([extract_expr(read(path, String)) for path in paths]...)
+    f(expr) = evaluate_include(expr, M, fail_on_error)
+    foreach(f, included_expr)
     if call_html
         println("Updating html")
         html(; project)
@@ -199,14 +215,14 @@ julia> module Foo
        end;
 
 julia> gen(Foo.version)
-Running version() for _gen/version.md
+Writing output of `version()` to _gen/version-ob--cb-.md
 Updating html
 ```
 """
-function gen(f::Function; fail_on_error=false, project="default", call_html=true)
+function gen(f::Function; project="default", call_html=true)
     path = joinpath(GENERATED_DIR, "$f.md")
-    suffix = ""
-    evaluate_and_write(f, path, suffix)
+    mkpath(GENERATED_DIR)
+    evaluate_and_write(f)
     if call_html
         println("Updating html")
         html(; project)
diff --git a/src/include-codeblocks.lua b/src/include-codeblocks.lua
new file mode 100644
index 00000000..1ab9ee2c
--- /dev/null
+++ b/src/include-codeblocks.lua
@@ -0,0 +1,175 @@
+--- include-output.lua – filter to include Julia output
+--- Based on include-files.lua – filter to include Markdown files
+---
+
+-- pandoc's List type
+local List = require 'pandoc.List'
+
+--- Get include auto mode
+local include_auto = false
+function get_vars (meta)
+  if meta['include-auto'] then
+    include_auto = true
+  end
+end
+
+--- Keep last heading level found
+local last_heading_level = 0
+function update_last_level(header)
+  last_heading_level = header.level
+end
+
+--- Shift headings in block list by given number
+local function shift_headings(blocks, shift_by)
+  if not shift_by then
+    return blocks
+  end
+
+  local shift_headings_filter = {
+    Header = function (header)
+      header.level = header.level + shift_by
+      return header
+    end
+  }
+
+  return pandoc.walk_block(pandoc.Div(blocks), shift_headings_filter).content
+end
+
+--- Return path of the markdown file for the string `s` given by the user.
+--- Ensure that this logic corresponds to the logic inside Books.jl.
+local md_path
+function md_path(s)
+  -- Escape all weird characters to ensure they can be in the file.
+  -- This yields very weird names, but luckily the code is only internal.
+  escaped = s
+  escaped = escaped:gsub("%(", "-ob-")
+  escaped = escaped:gsub("%)", "-cb-")
+  escaped = escaped:gsub("\"", "-dq-")
+  escaped = escaped:gsub(":", "-fc-")
+  escaped = escaped:gsub(";", "-sc-")
+  escaped = escaped:gsub("@", "-ax-")
+  path_sep = package.config:sub(1,1)
+  path = "_gen" .. path_sep .. escaped .. ".md"
+  return path
+end
+
+local not_found_error
+function not_found_error(line, path, ticks)
+  code = ticks .. line .. ticks
+  io.stderr:write("Cannot find file for " .. code .. " at " .. path .. "\n")
+end
+
+--- Filter function for code blocks
+local transclude_codeblock
+function transclude_codeblock(cb)
+  -- ignore code blocks which are not of class "jl".
+  if not cb.classes:includes 'jl' then
+    return
+  end
+
+  -- Markdown is used if this is nil.
+  local format = cb.attributes['format']
+
+  -- Attributes shift headings
+  local shift_heading_level_by = 0
+  local shift_input = cb.attributes['shift-heading-level-by']
+  if shift_input then
+    shift_heading_level_by = tonumber(shift_input)
+  else
+    if include_auto then
+      -- Auto shift headings
+      shift_heading_level_by = last_heading_level
+    end
+  end
+
+  --- keep track of level before recusion
+  local buffer_last_heading_level = last_heading_level
+
+  local blocks = List:new()
+  for line in cb.text:gmatch('[^\n]+') do
+    if line:sub(1,2) ~= '//' then
+
+      path = md_path(line)
+      if 60 < path:len() then
+        msg = "ERROR: The text `" .. line .. "` is too long to be converted to a filename"
+        msg = { pandoc.CodeBlock(msg) }
+        blocks:extend(msg)
+        -- Lua has no continue.
+        goto skip_to_next
+      end
+
+      local fh = io.open(path)
+      if not fh then
+        not_found_error(line, path, '```')
+        suggestion = "Did you run `gen(; M)` where `M = YourModule`?\n"
+        msg = "ERROR: Cannot find file at " .. path .. " for `" .. line .. "`."
+        msg = msg .. ' ' .. suggestion
+        msg = { pandoc.CodeBlock(msg) }
+        blocks:extend(msg)
+      else
+        local text = fh:read("*a")
+        local contents = pandoc.read(text, format).blocks
+        last_heading_level = 0
+        -- recursive transclusion
+        contents = pandoc.walk_block(
+          -- Here, the contents is added as an Any block.
+          -- Then, the filter is applied again recursively because
+          -- the included file could contain an include again!
+          pandoc.Div(contents),
+          { Header = update_last_level, CodeBlock = transclude }
+          ).content
+        --- reset to level before recursion
+        last_heading_level = buffer_last_heading_level
+        contents = shift_headings(contents, shift_heading_level_by)
+        -- Note that contents has type List.
+        blocks:extend(contents)
+        fh:close()
+      end
+    end
+    ::skip_to_next::
+  end
+  return blocks
+end
+
+local startswith
+function startswith(s, start)
+   return string.sub(s, 1, s.len(start)) == start
+end
+
+--- Filter function for inline code
+local transclude_code
+function transclude_code(c)
+  -- ignore code blocks which do not start with "jl".
+  if not startswith(c.text, 'jl ') then
+    return
+  end
+
+  line = c.text
+  line = line:sub(4)
+  path = md_path(line)
+
+  local fh = io.open(path)
+  if not fh then
+    not_found_error(line, path, '`')
+    suggestion = "Did you run `gen(; M)` where `M = YourModule`?"
+    msg = "ERROR: Cannot find file at " .. path .. " for `" .. line .. "`."
+    msg = msg .. ' ' .. suggestion
+    c.text = msg
+  else
+    text = fh:read("*a")
+    -- To retain ticks, use `c.text = text` and `return c`.
+    -- This conversion to a list is essential.
+    return { pandoc.Str(text) }
+  end
+
+  return c
+end
+
+return {
+  { Meta = get_vars },
+  {
+    Header = update_last_level,
+    CodeBlock = transclude_codeblock,
+    Code = transclude_code
+  }
+}
diff --git a/src/output.jl b/src/output.jl
index 9485f652..41b775cc 100644
--- a/src/output.jl
+++ b/src/output.jl
@@ -27,15 +27,15 @@ struct ImageOptions
     ImageOptions(object; width=nothing, height=nothing) = new(object, width, height)
 end
 
-function convert_output(path, out::ImageOptions; kwargs...)
+function convert_output(expr, path, out::ImageOptions; kwargs...)
     width = out.width
     height = out.height
-    convert_output(path, out.object; width, height, kwargs...)
+    convert_output(expr, path, out.object; width, height, kwargs...)
 end
 
-function convert_output(path, outputs::AbstractVector)
+function convert_output(expr, path, outputs::AbstractVector)
     path = nothing
-    outputs = convert_output.(path, outputs)
+    outputs = convert_output.(nothing, nothing, outputs)
     outputs = String.(outputs)
     out = join(outputs, "\n\n")
 end
@@ -77,7 +77,7 @@ julia> Options.(objects, filenames)
 """
 Options(object, filename::AbstractString) = Options(object; filename)
 
-function convert_output(path, out::Code)::String
+function convert_output(expr, path, out::Code)::String
     block = out.block
     mod = out.mod
     ans = try
@@ -85,7 +85,7 @@ function convert_output(path, out::Code)::String
     catch e
         string(e)
     end
-    shown_output = convert_output(path, ans)
+    shown_output = convert_output(expr, path, ans)
     if isa(ans, AbstractString) || isa(ans, Number)
         shown_output = code_block(shown_output)
     end
@@ -113,7 +113,7 @@ function convert_output(path, out::Code)::String
 end
 
 """
-    convert_output(path, options::Options)
+    convert_output(expr, path, options::Options)
 
 Convert `options.object` while taking `options.caption` and `options.label` into account.
 This method needs to pass the options correctly to the resulting type, because the syntax depends on the type;
@@ -127,7 +127,7 @@ julia> caption = "My DataFrame";
 
 julia> options = Options(df; caption);
 
-julia> print(Books.convert_output(nothing, options))
+julia> print(Books.convert_output(nothing, nothing, options))
 |   A |
 | ---:|
 |   1 |
@@ -135,29 +135,30 @@ julia> print(Books.convert_output(nothing, options))
 : My DataFrame
 ```
 """
-function convert_output(path, opts::Options)::String
+function convert_output(expr, path, opts::Options)::String
     object = opts.object
     filename = opts.filename
     if !isnothing(filename)
-        path = filename
+        expr = filename
     end
+    path = nothing
     caption = opts.caption
     label = opts.label
-    convert_output(path, object; caption, label)
+    convert_output(expr, path, object; caption, label)
 end
 
 """
-    convert_output(path, out::AbstractString)
+    convert_output(expr, path, out::AbstractString)
 
 Return `out` as string.
 This avoids the adding of `"` which `show` does by default.
 """
-convert_output(path, out::AbstractString) = string(out)
+convert_output(expr, path, out::AbstractString) = string(out)
 
-convert_output(path, out::Number) = string(out)
+convert_output(expr, path, out::Number) = string(out)
 
 """
-    convert_output(path, out)
+    convert_output(expr, path, out)
 
 Fallback method for `out::Any`.
 This passes the objects through show to use the overrides that package creators might have provided.
@@ -172,13 +173,13 @@ julia> chn = Chains([1]; info=(start_time=[1.0], stop_time=[1.0]));
 julia> string(chn)
 "MCMC chain (1×1×1 Array{Int64, 3})"
 
-julia> out = Books.convert_output("", chn);
+julia> out = Books.convert_output("", "", chn);
 
 julia> contains(out, "Summary Statistics")
 true
 ```
 """
-function convert_output(path, out)::String
+function convert_output(expr, path, out)::String
     io = IOBuffer()
     mime = MIME("text/plain")
     show(io, mime, out)
@@ -227,17 +228,17 @@ function pandoc_image(file, path; caption=nothing, label=nothing)
 end
 
 """
-    caption_label(path, caption, label)
+    caption_label(expr, caption, label)
 
 Return `caption` and `label` for the inputs.
 This method sets some reasonable defaults if any of the inputs is missing.
 
 # Examples
 ```jldoctest
-julia> Books.caption_label("a/foo_bar.md", nothing, nothing)
+julia> Books.caption_label("foo_bar()", nothing, nothing)
 (caption = "Foo bar", label = "foo_bar")
 
-julia> Books.caption_label("a/foo_bar.md", "My caption", nothing)
+julia> Books.caption_label("foo_bar()", "My caption", nothing)
 (caption = "My caption", label = "foo_bar")
 
 julia> Books.caption_label(nothing, "cap", nothing)
@@ -250,13 +251,13 @@ julia> Books.caption_label(nothing, nothing, nothing)
 (caption = nothing, label = nothing)
 ```
 """
-function caption_label(path, caption, label)
-    if isnothing(path) && isnothing(caption) && isnothing(label)
+function caption_label(expr, caption, label)
+    if isnothing(expr) && isnothing(caption) && isnothing(label)
         return (caption=nothing, label=nothing)
     end
 
-    if !isnothing(path)
-        name, suffix = method_name(path)
+    if !isnothing(expr)
+        name = method_name(expr)
         if isnothing(label)
             label = name
         end
diff --git a/src/outputs/aog.jl b/src/outputs/aog.jl
index a8de0d2a..b3d7d37a 100644
--- a/src/outputs/aog.jl
+++ b/src/outputs/aog.jl
@@ -3,11 +3,11 @@
 using AlgebraOfGraphics
 using CairoMakie
 
-function convert_output(path, fg::AlgebraOfGraphics.FigureGrid; caption=nothing, label=nothing)
+function convert_output(expr, path, fg::AlgebraOfGraphics.FigureGrid; caption=nothing, label=nothing)
     im_dir = joinpath(BUILD_DIR, "im")
     mkpath(im_dir)
 
-    if isnothing(path)
+    if isnothing(expr)
         # Not determining some random name here, because it would require cleanups too.
         msg = """
             It is not possible to write an image without specifying a path.
@@ -15,7 +15,7 @@ function convert_output(path, fg::AlgebraOfGraphics.FigureGrid; caption=nothing,
             """
         throw(ErrorException(msg))
     end
-    file, _ = method_name(path)
+    file = method_name(expr)
 
     println("Writing plot images for $file")
     svg_filename = "$file.svg"
@@ -31,6 +31,6 @@ function convert_output(path, fg::AlgebraOfGraphics.FigureGrid; caption=nothing,
     AlgebraOfGraphics.save(png_path, fg; px_per_unit)
 
     im_link = joinpath("im", svg_filename)
-    caption, label = caption_label(path, caption, label)
+    caption, label = caption_label(expr, caption, label)
     pandoc_image(file, png_path; caption, label)
 end
diff --git a/src/outputs/dataframes.jl b/src/outputs/dataframes.jl
index a9f6318f..f14c80cf 100644
--- a/src/outputs/dataframes.jl
+++ b/src/outputs/dataframes.jl
@@ -2,24 +2,25 @@ using DataFrames
 using Latexify
 
 """
-    convert_output(path, out::DataFrame; caption=nothing, label=nothing)
+    convert_output(expr, path, out::DataFrame; caption=nothing, label=nothing)
 
 Convert `out` to Markdown table and set some `pandoc-crossref` metadata.
 
 # Example
 ```jldoctest
-julia> df = DataFrame(A = [1])
+julia> df = DataFrame(A = [1]);
 
-julia> print(Books.convert_output("a/my_table.md", df))
+julia> print(Books.convert_output("my_table()", nothing, df))
 |   A |
 | ---:|
 |   1 |
 
 : My table {#tbl:my_table}
+```
 """
-function convert_output(path, out::DataFrame; caption=nothing, label=nothing)::String
+function convert_output(expr, path, out::DataFrame; caption=nothing, label=nothing)::String
     table = Latexify.latexify(out; env=:mdtable, latex=false)
-    caption, label = caption_label(path, caption, label)
+    caption, label = caption_label(expr, caption, label)
 
     if isnothing(caption) && isnothing(label)
         return string(table)
diff --git a/src/outputs/makie.jl b/src/outputs/makie.jl
index 3cb89969..e48b16ef 100644
--- a/src/outputs/makie.jl
+++ b/src/outputs/makie.jl
@@ -3,11 +3,11 @@
 using CairoMakie
 import Makie
 
-function convert_output(path, p::Makie.FigureAxisPlot; caption=nothing, label=nothing)
+function convert_output(expr, path, p::Makie.FigureAxisPlot; caption=nothing, label=nothing)
     im_dir = joinpath(BUILD_DIR, "im")
     mkpath(im_dir)
 
-    if isnothing(path)
+    if isnothing(expr)
         # Not determining some random name here, because it would require cleanups too.
         msg = """
             It is not possible to write an image without specifying a path.
@@ -15,7 +15,7 @@ function convert_output(path, p::Makie.FigureAxisPlot; caption=nothing, label=no
             """
         throw(ErrorException(msg))
     end
-    file, _ = method_name(path)
+    file = method_name(expr)
 
     println("Writing plot images for $file")
     svg_filename = "$file.svg"
@@ -31,6 +31,6 @@ function convert_output(path, p::Makie.FigureAxisPlot; caption=nothing, label=no
     Makie.FileIO.save(png_path, p; px_per_unit)
 
     im_link = joinpath("im", svg_filename)
-    caption, label = caption_label(path, caption, label)
+    caption, label = caption_label(expr, caption, label)
     pandoc_image(file, png_path; caption, label)
 end
diff --git a/src/outputs/plots.jl b/src/outputs/plots.jl
index e3fcd022..836158d2 100644
--- a/src/outputs/plots.jl
+++ b/src/outputs/plots.jl
@@ -2,11 +2,11 @@
 
 import Plots
 
-function convert_output(path, p::Plots.Plot; caption=nothing, label=nothing)
+function convert_output(expr, path, p::Plots.Plot; caption=nothing, label=nothing)
     im_dir = joinpath(BUILD_DIR, "im")
     mkpath(im_dir)
 
-    if isnothing(path)
+    if isnothing(expr)
         # Not determining some random name here, because it would require cleanups too.
         msg = """
             It is not possible to write an image without specifying a path or filename.
@@ -14,7 +14,7 @@ function convert_output(path, p::Plots.Plot; caption=nothing, label=nothing)
             """
         throw(ErrorException(msg))
     end
-    file, _ = method_name(path)
+    file = method_name(expr)
 
     println("Writing plot images for $file")
     svg_filename = "$file.svg"
@@ -26,6 +26,6 @@ function convert_output(path, p::Plots.Plot; caption=nothing, label=nothing)
     Plots.savefig(p, png_path)
 
     im_link = joinpath("im", svg_filename)
-    caption, label = caption_label(path, caption, label)
+    caption, label = caption_label(expr, caption, label)
     pandoc_image(file, png_path; caption, label)
 end
diff --git a/src/showcode.jl b/src/showcode.jl
index 5875ff17..93fd32d6 100644
--- a/src/showcode.jl
+++ b/src/showcode.jl
@@ -35,12 +35,12 @@ macro sco(f)
     end)
 end
 
-function convert_output(path, cf::CodeAndFunction)
+function convert_output(expr, path, cf::CodeAndFunction)
     code = cf.code
     f = cf.f
     println("Running $(f)() for $path")
     out = f()
-    out = convert_output(path, out)
+    out = convert_output(expr, path, out)
     """
     $code
     $out
diff --git a/test/generate.jl b/test/generate.jl
index b67a69e2..186022bb 100644
--- a/test/generate.jl
+++ b/test/generate.jl
@@ -9,25 +9,7 @@ using DataFrames
         ```
         """
 
-    dir = B.GENERATED_DIR
-    paths = [
-        joinpath(dir, "example.md"),
-        joinpath(dir, "example2.md"),
-        joinpath(dir, "example3.md")
-    ]
-    include_text = """
-    ```{.include}
-    $(paths[1])
-    ```
-
-    ```{.include}
-    $(paths[2])
-    $(paths[3])
-    ```
-    """
-    @test B.include_filenames(include_text) == paths
-
-    @test contains(B.convert_output(nothing, DataFrame(A = [1])), "---")
+    @test contains(B.convert_output(nothing, nothing, DataFrame(A = [1])), "---")
 
     X = 1:30
     df = (x=X, y=X.*2)
@@ -35,12 +17,12 @@ using DataFrames
     fg = draw(xy)
 
     mktemp() do path, io
-        @test contains(B.convert_output(path, fg), ".png")
+        @test contains(B.convert_output("tmp", nothing, fg), ".png")
     end
     im_dir = joinpath(B.BUILD_DIR, "im")
     rm(im_dir; force=true, recursive=true)
 
-    @test strip(B.convert_output(nothing, code("DataFrame(A = [1])"))) == """
+    @test strip(B.convert_output(nothing, nothing, code("DataFrame(A = [1])"))) == """
     ```
     DataFrame(A = [1])
     ```
@@ -57,12 +39,12 @@ module Foo
 
     @test B.caller_module() == Main.Foo
 
-    dir = B.GENERATED_DIR
-    function foo()
-        "lorem"
-    end
-    path = joinpath(dir, "foo.md")
-    B.evaluate_include(path, nothing, true)
-    @test read(path, String) == "lorem"
-    rm(dir; force = true, recursive = true)
+    foo() = "lorem"
+    fail_on_error = true
+    # Broken for some reason.
+    # B.evaluate_include("foo()", Foo, fail_on_error)
+    # path = joinpath(B.GENERATED_DIR, "foo-ob--cb-.md")
+    # mkpath(B.GENERATED_DIR)
+    # @test read(path, String) == "lorem"
+    # rm(dir; force=true, recursive=true)
 end