implement chef-server fetcher and reporter

Signed-off-by: Christoph Hartmann <chris@lollyrock.com>

.gitignore

@@ -38,11 +38,11 @@ doc/
 Berksfile.lock
 .kitchen
 .kitchen.local.yml
-vendor/
 .coverage/
 .zero-knife.rb
 
 #vagrant stuff
 .vagrant/
 .vagrant.d/
 .kitchen/
+brewinc-validator.pem

.rubocop.yml

@@ -7,6 +7,7 @@ AllCops:
   - 'test/**/*'
   - 'spec/**/*'
   - 'vendor/**/*'
+  - 'examples/chef-server/Vagrantfile'
 Documentation:
   Enabled: false
 AlignParameters:

attributes/default.rb

@@ -32,7 +32,7 @@
 # the token(access_token) expires in 12h after creation
 default['audit']['token'] = nil
 
-# set this insecure attribute to true if the compliance server uses self-signed ssl certificates
+# set this insecure attribute to true if the compliance server / chef server uses self-signed ssl certificates
 default['audit']['insecure'] = nil
 
 # Chef Compliance organization to post the report to. Defaults to Chef Server org if not defined

examples/chef-server/README.md

@@ -0,0 +1,29 @@
+# Example: Vagrant with Chef Server and Chef Compliance
+
+This directory contains a simple vagrant setup that expects you have a Chef Server already running.
+
+1.) Upload cookbook to Chef Server
+
+```
+mkdir cookbooks
+cd cookbooks
+git clone https://github.com/chef-cookbooks/audit.git
+cd ..
+chef exec knife cookbook upload audit -o ./cookbooks -c test-chef-server/knife.rb
+```
+
+2.) Adapt the chef Server settings in vagrant file:
+
+```
+chef.chef_server_url = 'https://192.168.33.101/organizations/brewinc'
+chef.validation_key_path = 'brewinc-validator.pem'
+chef.validation_client_name = 'brewinc-validator'
+```
+
+3.) Start node with chef-client
+
+```
+vagrant up
+# or if you have it already up
+vagrant provision
+```

examples/chef-server/Vagrantfile

@@ -0,0 +1,51 @@
+# encoding: utf-8
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+NODE_SCRIPT = <<EOF.freeze
+echo "Prepare Chef Client node"
+apt-get update
+# ensure the time is uptodate
+apt-get -y install ntp
+service ntp stop
+ntpdate -s time.nist.gov
+service ntp start
+EOF
+
+def set_hostname(server)
+  server.vm.provision 'shell', inline: "hostname #{server.vm.hostname}"
+end
+
+Vagrant.configure(2) do |config|
+
+  config.vm.define 'chef-client-node' do |server|
+    server.vm.box = 'bento/ubuntu-14.04'
+    server.vm.box_url = 'http://opscode-vm-bento.s3.amazonaws.com/vagrant/virtualbox/opscode_ubuntu-14.04_chef-provisionerless.box'
+    server.vm.hostname = 'audit-node'
+    server.vm.network 'private_network', ip: '192.168.33.102'
+    server.vm.synced_folder '.', '/vagrant'
+    config.vm.provision :shell, inline: NODE_SCRIPT.dup
+    set_hostname(server)
+
+    config.vm.provision :chef_client do |chef|
+      chef.chef_server_url = 'https://192.168.33.101/organizations/brewinc'
+      chef.validation_key_path = 'brewinc-validator.pem'
+      chef.validation_client_name = 'brewinc-validator'
+      chef.log_level = :debug
+      chef.add_recipe 'audit::default'
+      chef.json = {
+        audit: {
+          collector: "chef-server",
+          insecure: true,
+          profiles: [{
+            name: "linux",
+            compliance: "base/linux"
+          },{
+            name: "ssh",
+            compliance: "base/ssh"
+          }]
+        },
+      }
+    end
+  end
+end

examples/compliance/README.md

@@ -1,4 +1,4 @@
-# Example: Test-Kitchen
+# Example: Test-Kitchen with Chef Compliance
 
 This example demonstrates the usage of the audit cookbook with test-kitchen. In order to use it, we expect to have `COMPLIANCE_API` and `COMPLIANCE_ACCESSTOKEN` available as environment variables.
 

files/default/audit_report.rb

@@ -26,6 +26,9 @@ def report
         # load inspec, supermarket bundle and compliance bundle
         load_needed_dependencies
 
+        # detect if we run in a chef client with chef server
+        load_chef_fetcher if reporters.include?('chef-server')
+
         # iterate through reporters
         reporters.each do |reporter|
           # ensure authentication for Chef Compliance is in place, see libraries/compliance.rb
@@ -61,17 +64,31 @@ def load_needed_dependencies
         require 'bundles/inspec-compliance/target'
       end
 
+      # we expect inspec to be loaded before
+      def load_chef_fetcher
+        Chef::Log.debug "Load vendored ruby files from: #{cookbook_vendor_path}"
+        $LOAD_PATH.unshift(cookbook_vendor_path)
+        require 'chef-server/fetcher'
+      end
+
       # sets format to json-min when chef-compliance, json when chef-visibility
       def get_opts(reporter, quiet)
         format = reporter == 'chef-visibility' ? 'json' : 'json-min'
         output = quiet ? ::File::NULL : $stdout
+
         Chef::Log.warn "Format is #{format}"
-        { 'report' => true, 'format' => format, 'output' => output }
+        opts = {
+          'report' => true,
+          'format' => format,
+          'output' => output,
+          'logger' => Chef::Log, # Use chef-client log level for inspec run
+        }
+        opts
       end
 
       # run profiles and return report
       def call(opts, profiles)
-        Chef::Log.debug 'Initialize InSpec'
+        Chef::Log.info 'Initialize InSpec'
         Chef::Log.debug "Options are set to: #{opts}"
         runner = ::Inspec::Runner.new(opts)
 
@@ -98,6 +115,22 @@ def gather_nodeinfo
         }
       end
 
+      # this is a helper methods to extract the profiles we scan and hand this
+      # over to the reporter in addition to the `json-min` report. `json-min`
+      # reports do not include information about the source of the profiles
+      # TODO: should be available in inspec `json-min` reports out-of-the-box
+      # TODO: raise warning when not a compliance-known profile
+      def cc_profile_index(profiles)
+        cc_profiles = tests_for_runner(profiles).select { |profile| profile[:compliance] }.map { |profile| profile[:compliance] }.uniq.compact
+        cc_profiles.map { |profile|
+          owner, profile_id = profile.split('/')
+          {
+            owner: owner,
+            profile_id: profile_id,
+          }
+        }
+      end
+
       # send report to the collector (see libraries/collector_classes.rb)
       def send_report(reporter, server, user, profiles, report)
         Chef::Log.info "Reporting to #{reporter}"
@@ -110,31 +143,19 @@ def send_report(reporter, server, user, profiles, report)
           raise_if_unreachable = node['audit']['raise_if_unreachable']
           url = construct_url(server, File.join('/owners', user, 'inspec'))
           if server
-            # TODO: we should not send the profiles to the reporter, all the information
-            # should be available in inspec reports out-of-the-box
-            # TODO: raise warning when not a compliance-known profile
-            profiles = tests_for_runner(profiles).map { |profile| profile[:compliance] if profile[:compliance] }.uniq.compact
-            compliance_profiles = profiles.map { |profile|
-              owner, profile_id = profile.split('/')
-              {
-                owner: owner,
-                profile_id: profile_id,
-              }
-            }
-            Collector::ChefCompliance.new(url, gather_nodeinfo, raise_if_unreachable, compliance_profiles, report).send_report
+            Collector::ChefCompliance.new(url, gather_nodeinfo, raise_if_unreachable, cc_profile_index(profiles), report).send_report
           else
             Chef::Log.warn "'server' and 'token' properties required by inspec report collector #{reporter}. Skipping..."
           end
-
-        # elsif reporter == 'chef-server'
-        #   chef_url = server || base_chef_server_url
-        #   if chef_url
-        #     url = construct_url(chef_url + '/compliance/', File.join('organizations', user, 'inspec'))
-        #     Collector::ChefServer.new(url).send_report
-        #   else
-        #     Chef::Log.warn "unable to determine chef-server url required by inspec report collector '#{reporter}'. Skipping..."
-        #   end
-
+        elsif reporter == 'chef-server'
+          chef_url = server || base_chef_server_url
+          chef_org = Chef::Config[:chef_server_url].split('/').last
+          if chef_url
+            url = construct_url(chef_url + '/compliance/', File.join('organizations', chef_org, 'inspec'))
+            Collector::ChefServer.new(url, gather_nodeinfo, raise_if_unreachable, cc_profile_index(profiles), report).send_report
+          else
+            Chef::Log.warn "unable to determine chef-server url required by inspec report collector '#{reporter}'. Skipping..."
+          end
         elsif reporter == 'json-file'
           timestamp = Time.now.utc.to_s.tr(' ', '_')
           Collector::JsonFile.new(report, timestamp).send_report

files/default/vendor/chef-server/fetcher.rb

@@ -0,0 +1,100 @@
+# encoding: utf-8
+# author: Christoph Hartmann
+
+require 'uri'
+
+require 'bundles/inspec-compliance/target'
+require 'inspec/fetcher'
+require 'inspec/errors'
+
+# This class implements an InSpec fetcher for for Chef Server. The implementation
+# is based on the Chef Compliance fetcher and only adapts the calls to redirect
+# the requests via Chef Server.
+#
+# This implementation depends on chef-client runtime, therefore it is only executable
+# inside of a chef-client run
+module ChefServer
+  class Fetcher < Compliance::Fetcher
+    name 'chef-server'
+
+    # it positions itself before `compliance` fetcher
+    # only load it, if the Chef Server is integrated with Chef Compliance
+    priority 501
+
+    def self.resolve(target)
+      uri = if target.is_a?(String) && URI(target).scheme == 'compliance'
+              URI(target)
+            elsif target.respond_to?(:key?) && target.key?(:compliance)
+              URI("compliance://#{target[:compliance]}")
+            end
+
+      return nil if uri.nil?
+
+      profile = uri.host + uri.path
+      config = {
+        'insecure' => true,
+      }
+      new(target_url(profile, config), config)
+    rescue URI::Error => _e
+      nil
+    end
+
+    def self.chef_server_url_base
+      cs = URI(Chef::Config[:chef_server_url])
+      cs.path = ''
+      cs.to_s
+    end
+
+    def self.chef_server_org
+      Chef::Config[:chef_server_url].split('/').last
+    end
+
+    def self.target_url(profile, config)
+      o, p = profile.split('/')
+      reqpath ="organizations/#{chef_server_org}/owners/#{o}/compliance/#{p}/tar"
+
+      if config['insecure']
+        Chef::Config[:verify_api_cert] = false
+        Chef::Config[:ssl_verify_mode] = :verify_none
+      end
+
+      construct_url(chef_server_url_base + '/compliance/', reqpath)
+    end
+
+    #
+    # We want to save compliance: in the lockfile rather than url: to
+    # make sure we go back through the ComplianceAPI handling.
+    #
+    def resolved_source
+      { compliance: chef_server_url }
+    end
+
+    # Downloads archive to temporary file from Chef Compliance via Chef Server
+    def download_archive_to_temp
+      return @temp_archive_path if ! @temp_archive_path.nil?
+      Inspec::Log.debug("Fetching URL: #{@target}")
+
+      Chef::Config[:verify_api_cert] = false # FIXME
+      Chef::Config[:ssl_verify_mode] = :verify_none # FIXME
+
+      rest = Chef::ServerAPI.new(@target, Chef::Config)
+      archive = with_http_rescue do
+        rest.streaming_request(@target)
+      end
+      @archive_type = '.tar.gz'
+      Inspec::Log.debug("Archive stored at temporary location: #{archive.path}")
+      @temp_archive_path = archive.path
+    end
+
+    def to_s
+      'Chef Server/Compliance Profile Loader'
+    end
+
+    private
+
+    def chef_server_url
+      m = %r{^#{@config['server']}/owners/(?<owner>[^/]+)/compliance/(?<id>[^/]+)/tar$}.match(@target)
+      "#{m[:owner]}/#{m[:id]}"
+    end
+  end
+end

libraries/collector_classes.rb

@@ -212,7 +212,7 @@ def send_report
 
       Chef::Log.info "Report to Chef Compliance: #{@url}"
 
-      # TODO: use insecure option
+      # TODO: use secure option
       opts = { use_ssl: @url.scheme == 'https',
         verify_mode: OpenSSL::SSL::VERIFY_NONE,
       }
@@ -234,8 +234,11 @@ def enriched_report(report)
       # build report for chef compliance, it includes node data
       blob[:reports] = {}
       blob[:profiles] = {}
+      Chef::Log.info "Control Profile: #{profiles}"
       profiles.each { |profile|
-        namespace = @compliance_profiles.select { |entry| entry[:profile_id] == 'ssh' }
+        Chef::Log.info "Control Profil: #{profile}"
+        Chef::Log.info "Compliance Profils: #{@compliance_profiles}"
+        namespace = @compliance_profiles.select { |entry| entry[:profile_id] == profile }
         unless namespace.nil? && namespace.empty?
           Chef::Log.debug "Namespace for #{profile} is #{namespace[0][:owner]}"
           blob[:profiles][profile] = namespace[0][:owner]
@@ -254,23 +257,29 @@ def enriched_report(report)
   #
   # Used to send inspec reports to a Chef Compliance server via Chef Server
   #
-  class ChefServer
-    include ReportHelpers
-
+  class ChefServer < ChefCompliance
     @url = nil
 
-    def initialize(url)
+    def initialize(url, node_info, raise_if_unreachable, compliance_profiles, report)
       @url = url
+      @node_info = node_info
+      @raise_if_unreachable = raise_if_unreachable
+      @compliance_profiles = compliance_profiles
+      @report = report
     end
 
     def send_report
-      content = results
+      content = @report
+      json_report = enriched_report(JSON.parse(content))
+
+      # TODO: only disable if insecure option is set
       Chef::Config[:verify_api_cert] = false
       Chef::Config[:ssl_verify_mode] = :verify_none
+
       Chef::Log.info "Report to Chef Server: #{@url}"
       rest = Chef::ServerAPI.new(@url, Chef::Config)
       with_http_rescue do
-        rest.post(@url, content)
+        rest.post(@url, JSON.parse(json_report))
       end
     end
   end

libraries/helper.rb

@@ -102,4 +102,8 @@ def handle_reporters(reporters)
     return reporters if reporters.is_a? Array
     [reporters]
   end
+
+  def cookbook_vendor_path
+    File.expand_path('../../files/default/vendor', __FILE__)
+  end
 end