Improved include file filter

- Raw includes
- Including/excluding on different output
  formats.
- Variable substitution in include paths.
- Documentation.
- Tests.

include-files/Makefile

@@ -1,15 +1,20 @@
 DIFF ?= diff --strip-trailing-cr -u
 
+.EXPORT_ALL_VARIABLES:
+
+SUBDIR_NAME=subdir
+
+
 test: sample.md file-a.md file-b.md file-c.md include-files.lua
-	@pandoc --lua-filter=include-files.lua --to=native $< \
+	@pandoc --lua-filter=include-files.lua -M subdir-name="subdir" --to=native $< \
 	    | $(DIFF) expected.native -
-	@pandoc --lua-filter=include-files.lua -M include-auto --to=native $< \
+	@pandoc --lua-filter=include-files.lua -M subdir-name="subdir" -M include-auto --to=native $< \
 	    | $(DIFF) expected-auto.native -
 
 expected.native: sample.md file-a.md file-b.md file-c.md include-files.lua
-	pandoc --lua-filter=include-files.lua --output $@ $<
+	pandoc --lua-filter=include-files.lua -M subdir-name="subdir" --output $@ $<
 
 expected-auto.native: sample.md file-a.md file-b.md file-c.md include-files.lua
-	pandoc --lua-filter=include-files.lua -M include-auto --output $@ $<
+	pandoc --lua-filter=include-files.lua -M subdir-name="subdir" -M include-auto --output $@ $<
 
 .PHONY: test

include-files/README.md

@@ -18,12 +18,12 @@ be convenient to modify the level of headers; a top-level header
 in an included file should be a second or third-level header in
 the final document.
 
-#### Manual shifting
+#### Manual Shifting
 
 Use the `shift-heading-level-by` attribute to control header
 shifting.
 
-#### Automatic shifting
+#### Automatic Shifting
 
 1. Add metadata `-M include-auto` to enable automatic shifting.
 2. Do not specify `shift-heading-level-by`
@@ -52,14 +52,56 @@ file-a.md
 Comment lines can be added in the include block by beginning a
 line with two `//` characters.
 
-### Different formats
+### Different Formats
 
 Files are assumed to be written in Markdown, but sometimes one
 will want to include files written in a different format. An
 alternative format can be specified via the `format` attribute.
-Only plain-text formats are accepted.
+Only plain-text formats are accepted. The default format for all includes
+can be set by the meta data variable `include-format`, which is useful
+if you want all your files to be parsed in the same way as your main document,
+e.g. with extensions.
 
-### Recursive transclusion
+### Filter by Formats
+
+Files can be included and excluded on different formats by
+using attribute `include-if-format=formatA;formatB;...`
+and `exclude-if-format=formatA;formatB;...`, e.g.
+
+````md
+```{.include include-if-format=native;html;latex}
+subdir/file-h.md
+```
+
+```{.include exclude-if-format=native;commonmark}
+subdir/file-i.md
+```
+````
+
+### Variable Substitution in Paths
+
+If attribute `var-replace` is used, the patterns `${meta:<meta-var>}` or `${env:<env-var>}`
+will be replaced by the corresponding meta data variable `<meta-var>` in the document or the
+environment variable `<env-var>`, e.g.
+
+````md
+```{.include .var-replace}
+${meta:subdir-name}/file-h.md
+${env:SUBDIR_NAME}/file-h.md
+```
+````
+
+### Raw Includes
+
+You can also include the files in a raw block by using `raw=true`, e.g.
+
+````md
+```{.include raw=true format=latex include-if-format=latex}
+subdir/file-h-latex.md
+```
+````
+
+### Recursive Transclusion
 
 Included files can in turn include other files. Note that all
 filenames must be relative to the directory from which they are
@@ -70,6 +112,12 @@ this case `b/c.md`. The full relative path will be automatically
 generated in the final document. The same goes for image paths and
 codeblock file paths using the `include-code-files` filter.
 
+### Missing Includes
+
+You can set the meta data variable `include-fail-if-read-error` to `true`
+such that any not found include will fail the convertion and error out
+immediateley.
+
 ## Example
 
 Let's assume we are writing a longer document, like a thesis.

include-files/expected-auto.native

@@ -23,6 +23,15 @@
 ,Header 2 ("source-include",[],[]) [Str "Source",Space,Str "include"]
 ,Para [Str "File",Space,Str "inclusion",Space,Str "codeblocks",Space,Str "for",Space,Str "use",Space,Str "with",Space,Str "include-code-files",Space,Str "will",Space,Str "be",SoftBreak,Str "updated",Space,Str "too."]
 ,CodeBlock ("",["c"],[("include","subdir/somecode.c")]) ""
+,Header 1 ("includeexclude-if-format",[],[]) [Str "Include/exclude",Space,Str "if",Space,Str "format"]
+,Para [Str "The",Space,Str "next",Space,Str "document",Space,Str "should",Space,Str "be",Space,Str "included",Space,Str "in",Space,Str "formats",Space,Code ("",[],[]) "native, html, latex",Str "."]
+,Header 2 ("some-other-stuff",[],[]) [Str "Some",Space,Str "other",Space,Str "stuff"]
+,Para [Str "The",Space,Str "next",Space,Str "document",Space,Str "should",Space,Str "not",Space,Str "be",Space,Str "included",Space,Str "in",Space,Str "formats",Space,Code ("",[],[]) "native, commonmark",Str "."]
+,Header 1 ("meta-and-env-var-replacement",[],[]) [Str "Meta",Space,Str "and",Space,Str "env",Space,Str "var",Space,Str "replacement"]
+,Header 2 ("some-other-stuff",[],[]) [Str "Some",Space,Str "other",Space,Str "stuff"]
+,Header 2 ("some-other-stuff",[],[]) [Str "Some",Space,Str "other",Space,Str "stuff"]
+,Header 1 ("raw-include",[],[]) [Str "Raw",Space,Str "include"]
+,RawBlock (Format "latex") "\\section{Some other stuff}\n"
 ,Header 1 ("appendix",[],[]) [Str "Appendix"]
 ,Para [Str "More",Space,Str "info",Space,Str "goes",Space,Str "here."]
 ,Header 2 ("questionaire",[],[]) [Str "Questionaire"]

include-files/expected.native

@@ -23,6 +23,15 @@
 ,Header 1 ("source-include",[],[]) [Str "Source",Space,Str "include"]
 ,Para [Str "File",Space,Str "inclusion",Space,Str "codeblocks",Space,Str "for",Space,Str "use",Space,Str "with",Space,Str "include-code-files",Space,Str "will",Space,Str "be",SoftBreak,Str "updated",Space,Str "too."]
 ,CodeBlock ("",["c"],[("include","subdir/somecode.c")]) ""
+,Header 1 ("includeexclude-if-format",[],[]) [Str "Include/exclude",Space,Str "if",Space,Str "format"]
+,Para [Str "The",Space,Str "next",Space,Str "document",Space,Str "should",Space,Str "be",Space,Str "included",Space,Str "in",Space,Str "formats",Space,Code ("",[],[]) "native, html, latex",Str "."]
+,Header 1 ("some-other-stuff",[],[]) [Str "Some",Space,Str "other",Space,Str "stuff"]
+,Para [Str "The",Space,Str "next",Space,Str "document",Space,Str "should",Space,Str "not",Space,Str "be",Space,Str "included",Space,Str "in",Space,Str "formats",Space,Code ("",[],[]) "native, commonmark",Str "."]
+,Header 1 ("meta-and-env-var-replacement",[],[]) [Str "Meta",Space,Str "and",Space,Str "env",Space,Str "var",Space,Str "replacement"]
+,Header 1 ("some-other-stuff",[],[]) [Str "Some",Space,Str "other",Space,Str "stuff"]
+,Header 1 ("some-other-stuff",[],[]) [Str "Some",Space,Str "other",Space,Str "stuff"]
+,Header 1 ("raw-include",[],[]) [Str "Raw",Space,Str "include"]
+,RawBlock (Format "latex") "\\section{Some other stuff}\n"
 ,Header 1 ("appendix",[],[]) [Str "Appendix"]
 ,Para [Str "More",Space,Str "info",Space,Str "goes",Space,Str "here."]
 ,Header 2 ("questionaire",[],[]) [Str "Questionaire"]

include-files/include-files.lua

@@ -9,15 +9,99 @@ PANDOC_VERSION:must_be_at_least '2.12'
 local List = require 'pandoc.List'
 local path = require 'pandoc.path'
 local system = require 'pandoc.system'
+local cs = PANDOC_STATE
 
---- Get include auto mode
+-- This is the codeblock-var-replace
+-- filter directly copied, since we
+-- cannot run Lua filters inside this filter
+-- https://github.com/jgm/pandoc/issues/6830
+-- We replace variables in include blocks.
+
+local sys = require 'pandoc.system'
+local utils = require 'pandoc.utils'
+-- local ut = require "module-lua.utils"
+
+-- Save env. variables
+local env = sys.environment()
+
+-- Save meta table and metadata
+local meta
+function save_meta (m)
+  meta = m
+end
+
+--- Replace variables in code blocks
+local metaMap
+local function var_replace_codeblocks (cb)
+  --- Replace variable with values from environment
+  --- and meta data (stringifing).
+  local function replace(what, var)
+    local repl = nil
+    if what == "env" then
+      repl = env[var]
+    elseif what == "meta" then
+      local v = metaMap[var]
+      if v then
+        repl = utils.stringify(v)
+      end
+    end
+
+    if repl == nil then
+      io.stderr:write("Could not replace variable in codeblock: '".. var .."'\n")
+    end
+
+    return repl
+  end
+
+  -- ignore code blocks which are not of class "var-replace".
+  if not cb.classes:includes 'var-replace' then
+    return
+  end
+
+  cb.text = cb.text:gsub("%${(%l+):([^}]+)}", replace)
+end
+
+--- Include/exclude by attribute
+--- `exclude-if-format='formatA;formatB;...'
+--- `include-if-format='formatA;formatB;...`
+--- Default: true
+local function is_included(cb)
+  local include = true
+  local exclude = false
+
+  if cb.attributes['include-if-format'] then
+    include = cb.attributes['include-if-format']:match(FORMAT) ~= nil
+  end
+
+  if cb.attributes['exclude-if-format'] then
+    exclude = cb.attributes['exclude-if-format']:match(FORMAT) ~= nil
+  end
+
+  return include == true and exclude == false
+end
+
+--- Get default settings
 local include_auto = false
+local default_format = nil
+local include_fail_if_read_error = false
+
 function get_vars (meta)
   if meta['include-auto'] then
     include_auto = true
   end
+
+  if meta['include-fail-if-read-error'] then
+    include_fail_if_read_error = true
+  end
+
+  -- If this is nil, markdown is used as a default format.
+  default_format = meta['include-format']
+
+  -- Save meta table for var_replace
+  metaMap = meta
 end
 
+
 --- Keep last heading level found
 local last_heading_level = 0
 function update_last_level(header)
@@ -62,8 +146,23 @@ function transclude (cb)
     return
   end
 
-  -- Markdown is used if this is nil.
+  -- Filter by includes and excludes
+  if not is_included(cb) then
+    return List{} -- remove block
+  end
+
+  -- Variable substitution
+  var_replace_codeblocks(cb)
+
   local format = cb.attributes['format']
+  if not format then
+    -- Markdown is used if this is nil.
+    format = default_format
+  end
+
+  -- Check if we include the file as raw inline
+  local raw = cb.attributes['raw']
+  raw = raw == "true"
 
   -- Attributes shift headings
   local shift_heading_level_by = 0
@@ -77,35 +176,63 @@ function transclude (cb)
     end
   end
 
-  --- keep track of level before recusion
+  --- Keep track of level before recursion
   local buffer_last_heading_level = last_heading_level
 
   local blocks = List:new()
   for line in cb.text:gmatch('[^\n]+') do
-    if line:sub(1,2) ~= '//' then
-      local fh = io.open(line)
-      if not fh then
-        io.stderr:write("Cannot open file " .. line .. " | Skipping includes\n")
+    if line:sub(1,2) == '//' then
+      goto skip_to_next
+    end
+
+    if cs.verbosity == "INFO" then
+      io.stderr:write(string.format("Including: [format: %s, raw: %s]\n - '%s'\n",
+                      format,
+                      tostring(raw), line))
+    end
+
+    local fh = io.open(line)
+    if not fh then
+      local cwd = system.get_working_directory()
+      local msg = "Cannot find include file: '" .. line .. "' in working dir: '" .. cwd .. "'"
+      if include_fail_if_read_error then
+        io.stderr:write(msg .. " | error\n")
+        error("Abort due to include failure")
       else
-        local contents = pandoc.read(fh:read '*a', format).blocks
-        last_heading_level = 0
-        -- recursive transclusion
-        contents = system.with_working_directory(
-            path.directory(line),
-            function ()
-              return pandoc.walk_block(
-                pandoc.Div(contents),
-                { Header = update_last_level, CodeBlock = transclude }
-              )
-            end).content
-        --- reset to level before recursion
-        last_heading_level = buffer_last_heading_level
-        blocks:extend(update_contents(contents, shift_heading_level_by,
-                                      path.directory(line)))
-        fh:close()
+        io.stderr:write(msg .. " | skipping include\n")
+        goto skip_to_next
       end
     end
+
+    -- Read the file
+    local text = fh:read('*a')
+    fh:close()
+
+    if raw then
+      -- Include as raw inline element
+      blocks:extend({pandoc.RawBlock(format, text)})
+    else
+      -- Inlcude as parsed AST
+      local contents = pandoc.read(text, format).blocks
+      last_heading_level = 0
+      -- Recursive transclusion
+      contents = system.with_working_directory(
+          path.directory(line),
+          function ()
+            return pandoc.walk_block(
+          pandoc.Div(contents),
+          { Header = update_last_level, CodeBlock = transclude }
+            )
+          end).content
+        --- Reset to level before recursion
+        last_heading_level = buffer_last_heading_level
+      blocks:extend(update_contents(contents, shift_heading_level_by,
+                                    path.directory(line)))
+    end
+
+    ::skip_to_next::
   end
+
   return blocks
 end
 

include-files/sample.md

@@ -37,6 +37,38 @@ file-f.md
 subdir/file-g.md
 ```
 
+# Include/exclude if format
+
+The next document should be included in formats `native, html, latex`.
+
+```{.include include-if-format=native;html;latex}
+subdir/file-h.md
+```
+
+The next document should not be included in formats `native, commonmark`.
+
+```{.include exclude-if-format=native;commonmark}
+subdir/file-i.md
+```
+
+# Meta and env var replacement
+
+```{.include .var-replace}
+// Replace meta data variable in path
+${meta:subdir-name}/file-h.md
+```
+
+```{.include .var-replace}
+// Replace envrionment variable in path
+${env:SUBDIR_NAME}/file-h.md
+```
+
+# Raw include
+
+```{.include raw=true format=latex}
+subdir/file-h-latex.md
+```
+
 # Appendix
 
 More info goes here.

include-files/subdir/file-h-latex.md

@@ -0,0 +1 @@
+\section{Some other stuff}

include-files/subdir/file-h.md

@@ -0,0 +1 @@
+# Some other stuff