End-to-end Ruby front matter, templates, and data files (#285)

* Major feature addition for Ruby Front Matter and raw templates

Also additional work to normalize Liquid & ERB template APIs

* Improve docs

* improve error messages, support Ruby data files

* Add to_json support for Resources

* Improve code quality and test rbfm

* Refactor front matter importing and add rbfm to layouts

* Revert and use regex capture

* Remove logging around rbfm

* Lots of Ruby files and rbfm documentation, couple of fixes

* Better to_json compat

* Fix Liquid error on website build

bridgetown-core/lib/bridgetown-core.rb

@@ -93,6 +93,7 @@ module Bridgetown
   # TODO: this is a poorly named, unclear class. Relocate to Utils:
   autoload :External,            "bridgetown-core/external"
   autoload :FrontmatterDefaults, "bridgetown-core/frontmatter_defaults"
+  autoload :FrontMatterImporter, "bridgetown-core/concerns/front_matter_importer"
   autoload :Hooks,               "bridgetown-core/hooks"
   autoload :Layout,              "bridgetown-core/layout"
   autoload :LayoutPlaceable,     "bridgetown-core/concerns/layout_placeable"

bridgetown-core/lib/bridgetown-core/collection.rb

@@ -78,7 +78,8 @@ def read # rubocop:todo Metrics/PerceivedComplexity, Metrics/CyclomaticComplexit
         if site.uses_resource?
           next if File.basename(file_path).starts_with?("_")
 
-          if label == "data" || Utils.has_yaml_header?(full_path)
+          if label == "data" || Utils.has_yaml_header?(full_path) ||
+              Utils.has_rbfm_header?(full_path)
             read_resource(full_path)
           else
             read_static_file(file_path, full_path)
@@ -256,7 +257,7 @@ def merge_data_resources
           sanitized_segment = sanitize_filename.(File.basename(segment, ".*"))
           hsh = nested.empty? ? data_contents : data_contents.dig(*nested)
           hsh[sanitized_segment] = if index == segments.length - 1
-                                     data_resource.data.array || data_resource.data
+                                     data_resource.data.rows || data_resource.data
                                    else
                                      {}
                                    end
@@ -279,6 +280,16 @@ def merge_environment_specific_metadata(data_contents)
       data_contents
     end
 
+    # Read in resource from repo path
+    # @param full_path [String]
+    def read_resource(full_path)
+      id = "repo://#{label}.collection/" + Addressable::URI.escape(
+        Pathname(full_path).relative_path_from(Pathname(site.source)).to_s
+      )
+      resource = Bridgetown::Model::Base.find(id).to_resource.read!
+      resources << resource if site.unpublished || resource.published?
+    end
+
     private
 
     def container
@@ -290,14 +301,6 @@ def read_document(full_path)
       docs << doc if site.unpublished || doc.published?
     end
 
-    def read_resource(full_path)
-      id = "file://#{label}.collection/" + Addressable::URI.escape(
-        Pathname(full_path).relative_path_from(Pathname(site.source)).to_s
-      )
-      resource = Bridgetown::Model::Base.find(id).to_resource.read!
-      resources << resource if site.unpublished || resource.published?
-    end
-
     def sort_docs!
       if metadata["sort_by"].is_a?(String)
         sort_docs_by_key!

bridgetown-core/lib/bridgetown-core/concerns/front_matter_importer.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Bridgetown
+  module FrontMatterImporter
+    # Requires klass#content and klass#front_matter_line_count accessors
+    def self.included(klass)
+      klass.include Bridgetown::Utils::RubyFrontMatterDSL
+    end
+
+    YAML_HEADER = %r!\A---\s*\n!.freeze
+    YAML_BLOCK = %r!\A(---\s*\n.*?\n?)^((---|\.\.\.)\s*$\n?)!m.freeze
+    RUBY_HEADER = %r!\A[~`#\-]{3,}(?:ruby|<%|{%)\s*\n!.freeze
+    RUBY_BLOCK =
+      %r!#{RUBY_HEADER.source}(.*?\n?)^((?:%>|%})?[~`#\-]{3,}\s*$\n?)!m.freeze
+
+    def read_front_matter(file_path) # rubocop:todo Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
+      file_contents = File.read(
+        file_path, **Bridgetown::Utils.merged_file_read_opts(Bridgetown::Current.site, {})
+      )
+      yaml_content = file_contents.match(YAML_BLOCK)
+      if !yaml_content && Bridgetown::Current.site.config.should_execute_inline_ruby?
+        ruby_content = file_contents.match(RUBY_BLOCK)
+      end
+
+      if yaml_content
+        self.content = yaml_content.post_match
+        self.front_matter_line_count = yaml_content[1].lines.size - 1
+        SafeYAML.load(yaml_content[1])
+      elsif ruby_content
+        # rbfm header + content underneath
+        self.content = ruby_content.post_match
+        self.front_matter_line_count = ruby_content[1].lines.size
+        process_ruby_data(ruby_content[1], file_path, 2)
+      elsif Bridgetown::Utils.has_rbfm_header?(file_path)
+        process_ruby_data(File.read(file_path).lines[1..-1].join("\n"), file_path, 2)
+      elsif is_a?(Layout)
+        self.content = file_contents
+        {}
+      else
+        yaml_data = SafeYAML.load_file(file_path)
+        yaml_data.is_a?(Array) ? { rows: yaml_data } : yaml_data
+      end
+    end
+
+    def process_ruby_data(rubycode, file_path, starting_line)
+      ruby_data = instance_eval(rubycode, file_path.to_s, starting_line)
+      ruby_data.is_a?(Array) ? { rows: ruby_data } : ruby_data.to_h
+    rescue StandardError => e
+      raise "Ruby code isn't returning an array, or object which responds to `to_h' (#{e.message})"
+    end
+  end
+end

bridgetown-core/lib/bridgetown-core/concerns/site/content.rb

@@ -173,9 +173,8 @@ def docs_to_write
       documents.select(&:write?)
     end
 
-    # Get all documents.
-    # @return [Array<Document>] an array of documents from the
-    # configuration
+    # Get all loaded resources.
+    # @return [Array<Bridgetown::Resource::Base>] an array of resources
     def resources
       collections.each_with_object(Set.new) do |(_, collection), set|
         set.merge(collection.resources)

bridgetown-core/lib/bridgetown-core/concerns/validatable.rb

@@ -3,10 +3,6 @@
 module Bridgetown
   # TODO: to be retired once the Resource engine is made official
   module Validatable
-    # FIXME: there should be ONE TRUE METHOD to read the YAML frontmatter
-    # in the entire project. Both this and the equivalent Document method
-    # should be extracted and generalized.
-    #
     # Read the YAML frontmatter.
     #
     # base - The String path to the dir containing the file.

bridgetown-core/lib/bridgetown-core/configuration.rb

@@ -74,15 +74,16 @@ class Configuration < HashWithDotAccess::Hash
       },
 
       "kramdown"             => {
-        "auto_ids"      => true,
-        "toc_levels"    => (1..6).to_a,
-        "entity_output" => "as_char",
-        "smart_quotes"  => "lsquo,rsquo,ldquo,rdquo",
-        "input"         => "GFM",
-        "hard_wrap"     => false,
-        "guess_lang"    => true,
-        "footnote_nr"   => 1,
-        "show_warnings" => false,
+        "auto_ids"                => true,
+        "toc_levels"              => (1..6).to_a,
+        "entity_output"           => "as_char",
+        "smart_quotes"            => "lsquo,rsquo,ldquo,rdquo",
+        "input"                   => "GFM",
+        "hard_wrap"               => false,
+        "guess_lang"              => true,
+        "footnote_nr"             => 1,
+        "show_warnings"           => false,
+        "include_extraction_tags" => false,
       },
     }.each_with_object(Configuration.new) { |(k, v), hsh| hsh[k] = v.freeze }.freeze
 

bridgetown-core/lib/bridgetown-core/converter.rb

@@ -53,6 +53,15 @@ def output_ext(_ext)
       ".html"
     end
 
+    def line_start(convertible)
+      if convertible.is_a?(Bridgetown::Resource::Base) &&
+          convertible.model.origin.respond_to?(:front_matter_line_count)
+        convertible.model.origin.front_matter_line_count + 4
+      else
+        1
+      end
+    end
+
     def inspect
       "#<#{self.class}#{self.class.extname_list ? " #{self.class.extname_list.join(", ")}" : nil}>"
     end

bridgetown-core/lib/bridgetown-core/converters/erb_templates.rb

@@ -115,6 +115,7 @@ def convert(content, convertible)
 
         erb_renderer = Tilt::ErubiTemplate.new(
           convertible.relative_path,
+          line_start(convertible),
           outvar: "@_erbout",
           bufval: "Bridgetown::OutputBuffer.new",
           engine_class: ERBEngine

bridgetown-core/lib/bridgetown-core/converters/markdown.rb

@@ -69,7 +69,7 @@ def convert(content, convertible = nil)
           end
         else
           output = @parser.convert(content)
-          if @parser.respond_to?(:extractions)
+          if convertible && @parser.respond_to?(:extractions)
             convertible.data.markdown_extractions = @parser.extractions
           end
           output

bridgetown-core/lib/bridgetown-core/converters/ruby_templates.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Bridgetown
+  module Converters
+    class RubyTemplates < Converter
+      priority :highest
+      input :rb
+
+      def convert(content, convertible)
+        erb_view = Bridgetown::ERBView.new(convertible)
+        erb_view.instance_eval(
+          content, convertible.relative_path.to_s, line_start(convertible)
+        ).to_s
+      end
+    end
+  end
+end

bridgetown-core/lib/bridgetown-core/drops/resource_drop.rb

@@ -14,6 +14,7 @@ class ResourceDrop < Drop
       def_delegator :@obj, :relative_path, :path
       def_delegators :@obj,
                      :id,
+                     :data,
                      :output,
                      :content,
                      :to_s,

bridgetown-core/lib/bridgetown-core/drops/unified_payload_drop.rb

@@ -6,6 +6,7 @@ class UnifiedPayloadDrop < Drop
       mutable true
 
       attr_accessor :page, :layout, :content, :paginator
+      alias_method :resource, :page
 
       def bridgetown
         BridgetownDrop.global

bridgetown-core/lib/bridgetown-core/layout.rb

@@ -3,8 +3,8 @@
 module Bridgetown
   class Layout
     include DataAccessible
+    include FrontMatterImporter
     include LiquidRenderable
-    include Validatable
 
     # Gets the Site object.
     attr_reader :site
@@ -27,8 +27,12 @@ class Layout
     attr_accessor :data
 
     # Gets/Sets the content of this layout.
+    # @return [String]
     attr_accessor :content
 
+    # @return [Integer]
+    attr_accessor :front_matter_line_count
+
     # Gets/Sets the current document (for layout-compatible converters)
     attr_accessor :current_document
 
@@ -54,9 +58,29 @@ def initialize(site, base, name, from_plugin: false)
         @path = site.in_source_dir(base, name)
       end
       @relative_path = @path.sub(@base_dir, "")
+      @ext = File.extname(name)
+
+      @data = read_front_matter(@path)&.with_dot_access
+    rescue SyntaxError => e
+      Bridgetown.logger.error "Error:",
+                              "Ruby Exception in #{e.message}"
+    rescue StandardError => e
+      handle_read_error(e)
+    ensure
+      @data ||= HashWithDotAccess::Hash.new
+    end
+
+    def handle_read_error(error)
+      if error.is_a? Psych::SyntaxError
+        Bridgetown.logger.warn "YAML Exception reading #{@path}: #{error.message}"
+      else
+        Bridgetown.logger.warn "Error reading file #{@path}: #{error.message}"
+      end
 
-      process(name)
-      read_yaml(base, name)
+      if site.config["strict_front_matter"] ||
+          error.is_a?(Bridgetown::Errors::FatalException)
+        raise error
+      end
     end
 
     # The inspect string for this document.
@@ -67,15 +91,6 @@ def inspect
       "#<#{self.class} #{@path}>"
     end
 
-    # Extract information from the layout filename.
-    #
-    # name - The String filename of the layout file.
-    #
-    # Returns nothing.
-    def process(name)
-      self.ext = File.extname(name)
-    end
-
     # Provide this Layout's data to a Hash suitable for use by Liquid.
     #
     # Returns the Hash representation of this Layout.

bridgetown-core/lib/bridgetown-core/model/origin.rb

@@ -35,4 +35,4 @@ def exists?
 end
 
 require "bridgetown-core/model/builder_origin"
-require "bridgetown-core/model/file_origin"
+require "bridgetown-core/model/repo_origin"