Integration test improvements (#2109)

* Improvements to integrations

Borrowed largely from elastic/elasticsearch-serverless-js#38

* Bump all the things to 8.12.0

* Split Dockerfile copy into two layers

* Fix test cron names

(cherry picked from commit 4aaf49b)

.buildkite/Dockerfile

@@ -10,5 +10,7 @@ RUN apt-get clean -y && \
 
 WORKDIR /usr/src/app
 
-COPY . .
+COPY package.json .
 RUN npm install --production=false
+
+COPY . .

.dockerignore

@@ -3,3 +3,5 @@ npm-debug.log
 test/benchmarks
 elasticsearch
 .git
+lib
+junit-output

catalog-info.yaml

@@ -42,9 +42,9 @@ spec:
         main_semi_daily:
           branch: 'main'
           cronline: '0 */12 * * *'
-        8_9_semi_daily:
-          branch: '8.9'
+        8_12_semi_daily:
+          branch: '8.12'
           cronline: '0 */12 * * *'
-        8_8_daily:
-          branch: '8.8'
+        8_11_daily:
+          branch: '8.11'
           cronline: '@daily'

package.json

@@ -93,4 +93,4 @@
     "coverage": false,
     "check-coverage": false
   }
-}
+}

test/integration/index.js

@@ -317,7 +317,7 @@ async function start ({ client, isXPack }) {
         if (name === 'setup' || name === 'teardown') continue
         if (options.test && !name.endsWith(options.test)) continue
 
-        const junitTestCase = junitTestSuite.testcase(name, `node_${process.version}/${cleanPath}`)
+        const junitTestCase = junitTestSuite.testcase(name, `node_${process.version}: ${cleanPath}`)
 
         stats.total += 1
         if (shouldSkip(isXPack, file, name)) {
@@ -336,6 +336,7 @@ async function start ({ client, isXPack }) {
           junitTestSuite.end()
           junitTestSuites.end()
           generateJunitXmlReport(junit, isXPack ? 'platinum' : 'free')
+          err.meta = JSON.stringify(err.meta ?? {}, null, 2)
           console.error(err)
 
           if (options.bail) {
@@ -374,6 +375,7 @@ async function start ({ client, isXPack }) {
   - Total: ${stats.total}
   - Skip: ${stats.skip}
   - Pass: ${stats.pass}
+  - Fail: ${stats.total - (stats.pass + stats.skip)}
   - Assertions: ${stats.assertions}
   `)
 }

test/integration/test-runner.js

@@ -593,13 +593,14 @@ function build (opts = {}) {
         const key = Object.keys(action.match)[0]
         match(
           // in some cases, the yaml refers to the body with an empty string
-          key === '$body' || key === ''
+          key.split('.')[0] === '$body' || key === ''
             ? response
             : delve(response, fillStashedValues(key)),
-          key === '$body'
+          key.split('.')[0] === '$body'
             ? action.match[key]
             : fillStashedValues(action.match)[key],
-          action.match
+          action.match,
+          response
         )
       }
 
@@ -608,7 +609,8 @@ function build (opts = {}) {
         const key = Object.keys(action.lt)[0]
         lt(
           delve(response, fillStashedValues(key)),
-          fillStashedValues(action.lt)[key]
+          fillStashedValues(action.lt)[key],
+          response
         )
       }
 
@@ -617,7 +619,8 @@ function build (opts = {}) {
         const key = Object.keys(action.gt)[0]
         gt(
           delve(response, fillStashedValues(key)),
-          fillStashedValues(action.gt)[key]
+          fillStashedValues(action.gt)[key],
+          response
         )
       }
 
@@ -626,7 +629,8 @@ function build (opts = {}) {
         const key = Object.keys(action.lte)[0]
         lte(
           delve(response, fillStashedValues(key)),
-          fillStashedValues(action.lte)[key]
+          fillStashedValues(action.lte)[key],
+          response
         )
       }
 
@@ -635,7 +639,8 @@ function build (opts = {}) {
         const key = Object.keys(action.gte)[0]
         gte(
           delve(response, fillStashedValues(key)),
-          fillStashedValues(action.gte)[key]
+          fillStashedValues(action.gte)[key],
+          response
         )
       }
 
@@ -648,7 +653,8 @@ function build (opts = {}) {
             : delve(response, fillStashedValues(key)),
           key === '$body'
             ? action.length[key]
-            : fillStashedValues(action.length)[key]
+            : fillStashedValues(action.length)[key],
+          response
         )
       }
 
@@ -657,7 +663,8 @@ function build (opts = {}) {
         const isTrue = fillStashedValues(action.is_true)
         is_true(
           delve(response, isTrue),
-          isTrue
+          isTrue,
+          response
         )
       }
 
@@ -666,7 +673,8 @@ function build (opts = {}) {
         const isFalse = fillStashedValues(action.is_false)
         is_false(
           delve(response, isFalse),
-          isFalse
+          isFalse,
+          response
         )
       }
     }
@@ -679,46 +687,67 @@ function build (opts = {}) {
  * Asserts that the given value is truthy
  * @param {any} the value to check
  * @param {string} an optional message
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function is_true (val, msg) {
-  assert.ok(val, `expect truthy value: ${msg} - value: ${JSON.stringify(val)}`)
+function is_true (val, msg, response) {
+  try {
+    assert.ok((typeof val === 'string' && val.toLowerCase() === 'true') || val, `expect truthy value: ${msg} - value: ${JSON.stringify(val)}`)
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
+  }
 }
 
 /**
  * Asserts that the given value is falsey
  * @param {any} the value to check
  * @param {string} an optional message
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function is_false (val, msg) {
-  assert.ok(!val, `expect falsey value: ${msg} - value: ${JSON.stringify(val)}`)
+function is_false (val, msg, response) {
+  try {
+    assert.ok((typeof val === 'string' && val.toLowerCase() === 'false') || !val, `expect falsey value: ${msg} - value: ${JSON.stringify(val)}`)
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
+  }
 }
 
 /**
  * Asserts that two values are the same
  * @param {any} the first value
  * @param {any} the second value
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function match (val1, val2, action) {
-  // both values are objects
-  if (typeof val1 === 'object' && typeof val2 === 'object') {
-    assert.deepEqual(val1, val2, typeof action === 'object' ? JSON.stringify(action) : action)
-  // the first value is the body as string and the second a pattern string
-  } else if (
-    typeof val1 === 'string' && typeof val2 === 'string' &&
-    val2.startsWith('/') && (val2.endsWith('/\n') || val2.endsWith('/'))
-  ) {
-    const regStr = val2
-      .replace(/(^|[^\\])#.*/g, '$1')
-      .replace(/(^|[^\\])\s+/g, '$1')
-      .slice(1, -1)
-    // 'm' adds the support for multiline regex
-    assert.match(val1, new RegExp(regStr, 'm'), `should match pattern provided: ${val2}, but got: ${val1}`)
-  // everything else
-  } else {
-    assert.equal(val1, val2, `should be equal: ${val1} - ${val2}, action: ${JSON.stringify(action)}`)
+function match (val1, val2, action, response) {
+  try {
+    // both values are objects
+    if (typeof val1 === 'object' && typeof val2 === 'object') {
+      assert.deepEqual(val1, val2, typeof action === 'object' ? JSON.stringify(action) : action)
+    // the first value is the body as string and the second a pattern string
+    } else if (
+      typeof val1 === 'string' && typeof val2 === 'string' &&
+      val2.startsWith('/') && (val2.endsWith('/\n') || val2.endsWith('/'))
+    ) {
+      const regStr = val2
+        .replace(/(^|[^\\])#.*/g, '$1')
+        .replace(/(^|[^\\])\s+/g, '$1')
+        .slice(1, -1)
+      // 'm' adds the support for multiline regex
+      assert.match(val1, new RegExp(regStr, 'm'), `should match pattern provided: ${val2}, but got: ${val1}: ${JSON.stringify(action)}`)
+    } else if (typeof val1 === 'string' && typeof val2 === 'string') {
+      // string comparison
+      assert.include(val1, val2, `should include pattern provided: ${val2}, but got: ${val1}: ${JSON.stringify(action)}`)
+    } else {
+      // everything else
+      assert.equal(val1, val2, `should be equal: ${val1} - ${val2}, action: ${JSON.stringify(action)}`)
+    }
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
   }
 }
 
@@ -727,62 +756,92 @@ function match (val1, val2, action) {
  * It also verifies that the two values are numbers
  * @param {any} the first value
  * @param {any} the second value
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function lt (val1, val2) {
-  ;[val1, val2] = getNumbers(val1, val2)
-  assert.ok(val1 < val2)
+function lt (val1, val2, response) {
+  try {
+    ;[val1, val2] = getNumbers(val1, val2)
+    assert.ok(val1 < val2)
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
+  }
 }
 
 /**
  * Asserts that the first value is greater than the second
  * It also verifies that the two values are numbers
  * @param {any} the first value
  * @param {any} the second value
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function gt (val1, val2) {
-  ;[val1, val2] = getNumbers(val1, val2)
-  assert.ok(val1 > val2)
+function gt (val1, val2, response) {
+  try {
+    ;[val1, val2] = getNumbers(val1, val2)
+    assert.ok(val1 > val2)
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
+  }
 }
 
 /**
  * Asserts that the first value is less than or equal the second
  * It also verifies that the two values are numbers
  * @param {any} the first value
  * @param {any} the second value
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function lte (val1, val2) {
-  ;[val1, val2] = getNumbers(val1, val2)
-  assert.ok(val1 <= val2)
+function lte (val1, val2, response) {
+  try {
+    ;[val1, val2] = getNumbers(val1, val2)
+    assert.ok(val1 <= val2)
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
+  }
 }
 
 /**
  * Asserts that the first value is greater than or equal the second
  * It also verifies that the two values are numbers
  * @param {any} the first value
  * @param {any} the second value
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
 */
-function gte (val1, val2) {
-  ;[val1, val2] = getNumbers(val1, val2)
-  assert.ok(val1 >= val2)
+function gte (val1, val2, response) {
+  try {
+    ;[val1, val2] = getNumbers(val1, val2)
+    assert.ok(val1 >= val2)
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
+  }
 }
 
 /**
  * Asserts that the given value has the specified length
  * @param {string|object|array} the object to check
  * @param {number} the expected length
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function length (val, len) {
-  if (typeof val === 'string' || Array.isArray(val)) {
-    assert.equal(val.length, len)
-  } else if (typeof val === 'object' && val !== null) {
-    assert.equal(Object.keys(val).length, len)
-  } else {
-    assert.fail(`length: the given value is invalid: ${val}`)
+function length (val, len, response) {
+  try {
+    if (typeof val === 'string' || Array.isArray(val)) {
+      assert.equal(val.length, len)
+    } else if (typeof val === 'object' && val !== null) {
+      assert.equal(Object.keys(val).length, len)
+    } else {
+      assert.fail(`length: the given value is invalid: ${val}`)
+    }
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
   }
 }
 
@@ -813,6 +872,10 @@ function length (val, len) {
  */
 function parseDo (action) {
   action = JSON.parse(JSON.stringify(action))
+
+  if (typeof action === 'string') action = {[action]: {}}
+  if (Array.isArray(action)) action = action[0]
+
   return Object.keys(action).reduce((acc, val) => {
     switch (val) {
       case 'catch':