Merge pull request #99 from rodjek/sdk-256

(SDK-256) Acceptance tests for metadata validator behavior and output

lib/pdk/validators/metadata.rb

@@ -14,6 +14,10 @@ def self.cmd
         'metadata-json-lint'
       end
 
+      def self.spinner_text
+        _('Checking metadata.json')
+      end
+
       def self.parse_targets(_options)
         [File.join(PDK::Util.module_root, 'metadata.json')]
       end
@@ -25,19 +29,31 @@ def self.parse_options(_options, targets)
       end
 
       def self.parse_output(report, json_data, _targets)
-        return if json_data.empty?
-
-        json_data.delete('result')
-        json_data.keys.each do |type|
-          json_data[type].each do |offense|
-            report.add_event(
-              file:     'metadata.json',
-              source:   cmd,
-              message:  offense['msg'],
-              test:     offense['check'],
-              severity: type,
-              state:    :failure,
-            )
+        if json_data.empty?
+          report.add_event(
+            file:     'metadata.json',
+            source:   cmd,
+            state:    :passed,
+            severity: :ok,
+          )
+        else
+          json_data.delete('result')
+          json_data.keys.each do |type|
+            json_data[type].each do |offense|
+              # metadata-json-lint groups the offenses by type, so the type ends
+              # up being `warnings` or `errors`. We want to convert that to the
+              # singular noun for the event.
+              event_type = type[%r{\A(.+?)s?\Z}, 1]
+
+              report.add_event(
+                file:     'metadata.json',
+                source:   cmd,
+                message:  offense['msg'],
+                test:     offense['check'],
+                severity: event_type,
+                state:    :failure,
+              )
+            end
           end
         end
       end

spec/acceptance/support/it_generates_valid_junit_xml.rb

@@ -0,0 +1,6 @@
+RSpec.shared_examples_for :it_generates_valid_junit_xml do
+  its(:stdout) do
+    xsd = File.join(RSpec.configuration.fixtures_path, 'JUnit.xsd')
+    is_expected.to pass_validation(xsd)
+  end
+end

spec/acceptance/validate_metadata_spec.rb

@@ -0,0 +1,73 @@
+require 'spec_helper_acceptance'
+
+describe 'Running metadata validation' do
+  let(:spinner_text) { %r{checking metadata\.json}i }
+
+  context 'with a fresh module' do
+    include_context 'in a new module', 'foo'
+
+    describe command('pdk validate metadata') do
+      its(:exit_status) { is_expected.to eq(0) }
+      its(:stdout) { is_expected.to match(%r{\A\Z}) }
+      its(:stderr) { is_expected.to match(spinner_text) }
+    end
+
+    describe command('pdk validate metadata --format junit') do
+      its(:exit_status) { is_expected.to eq(0) }
+      its(:stderr) { is_expected.to match(spinner_text) }
+      it_behaves_like :it_generates_valid_junit_xml
+
+      its(:stdout) do
+        is_expected.to have_xpath('/testsuites/testsuite[@name="metadata-json-lint"]').with_attributes(
+          'failures' => '0',
+          'tests'    => '1',
+        )
+      end
+
+      its(:stdout) do
+        is_expected.to have_xpath('/testsuites/testsuite[@name="metadata-json-lint"]/testcase').with_attributes(
+          'classname' => 'metadata-json-lint',
+          'name'      => 'metadata.json',
+        )
+      end
+    end
+  end
+
+  context 'with a metadata violation' do
+    include_context 'in a new module', 'foo'
+
+    before(:all) do
+      metadata = JSON.parse(File.read('metadata.json'))
+      metadata['dependencies'].first['version_requirement'] = '>= 1.0.0'
+      File.open('metadata.json', 'w') do |f|
+        f.puts metadata.to_json
+      end
+    end
+
+    describe command('pdk validate metadata') do
+      its(:exit_status) { is_expected.not_to eq(0) }
+      its(:stdout) { is_expected.to match(%r{^metadata\.json:.+warning.+open ended dependency}) }
+      its(:stderr) { is_expected.to match(spinner_text) }
+    end
+
+    describe command('pdk validate metadata --format junit') do
+      its(:exit_status) { is_expected.not_to eq(0) }
+      its(:stderr) { is_expected.to match(spinner_text) }
+      it_behaves_like :it_generates_valid_junit_xml
+
+      its(:stdout) do
+        is_expected.to have_xpath('/testsuites/testsuite[@name="metadata-json-lint"]').with_attributes(
+          'failures' => '1',
+          'tests'    => '1',
+        )
+      end
+
+      its(:stdout) do
+        is_expected.to have_xpath('/testsuites/testsuite[@name="metadata-json-lint"]/testcase').with_attributes(
+          'classname' => 'metadata-json-lint.dependencies',
+          'name'      => 'metadata.json',
+        )
+      end
+    end
+  end
+end

spec/acceptance/validate_ruby_spec.rb

@@ -23,7 +23,7 @@
     describe command('pdk validate ruby --format junit') do
       its(:exit_status) { is_expected.to eq(0) }
       its(:stderr) { is_expected.to match(%r{checking ruby code style}i) }
-      its(:stdout) { is_expected.to pass_validation(junit_xsd) }
+      it_behaves_like :it_generates_valid_junit_xml
 
       its(:stdout) do
         is_expected.to have_xpath('/testsuites/testsuite[@name="rubocop"]').with_attributes(
@@ -96,7 +96,7 @@
     describe command('pdk validate ruby --format junit') do
       its(:exit_status) { is_expected.not_to eq(0) }
       its(:stderr) { is_expected.to match(%r{checking ruby code style}i) }
-      its(:stdout) { is_expected.to pass_validation(junit_xsd) }
+      it_behaves_like :it_generates_valid_junit_xml
 
       its(:stdout) do
         is_expected.to have_xpath('/testsuites/testsuite[@name="rubocop"]').with_attributes(