Fix failing CI, local dev dockerfile setup, implement Centos 6 deprecation

.circleci/config.yml

@@ -2,21 +2,23 @@ version: 2
 jobs:
   kitchen-docker-tests:
     machine:
-      image: ubuntu-2004:202201-02
+      image: ubuntu-2004:2024.01.2
     resource_class: large
     environment:
       CHEF_LICENSE: accept # newer versions of Chef client need explicit license accept to run
       KITCHEN_LOCAL_YAML: kitchen.docker.yml
       RUBY_VERSION: '2.6.3' # ruby used to invoke kitchen, not the version used in the tests
+      DOCKER_BUILDKIT: 0  # Disable buildkit for compatibility with version of kitchen we use
+    shell: /bin/bash --login -eo pipefail
     steps:
       - checkout
       - run:
           name: Set Ruby version
           command: |
                     echo "Using $RUBY_VERSION"
-                    curl -sSL https://rvm.io/pkuczynski.asc | gpg --import -
-                    # Update RVM. Using this path instead of https://get.rvm.io because the later uses the letsencrypt cert that breaks openssl 1.0
-                    curl -sSL https://raw.githubusercontent.com/rvm/rvm/master/binscripts/rvm-installer | bash -s stable
+                    gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
+                    curl -sSL https://get.rvm.io | bash -s stable
+                    source /home/circleci/.rvm/scripts/rvm
                     rvm reload
                     rvm install $RUBY_VERSION
                     echo . $(rvm $RUBY_VERSION do rvm env --path) >> $BASH_ENV

Gemfile.lock

@@ -1,7 +1,7 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.3.4)
+    activesupport (6.0.6.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -78,6 +78,8 @@ GEM
       mixlib-config (>= 2.2.12, < 3.0)
       mixlib-shellout (~> 2.0)
       tomlrb (~> 1.2)
+    chef-utils (18.4.12)
+      concurrent-ruby
     chef-zero (14.0.17)
       ffi-yajl (~> 2.2)
       hashie (>= 2.0, < 4.0)
@@ -110,7 +112,7 @@ GEM
     cucumber-tag_expressions (1.1.1)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
-    diff-lcs (1.4.4)
+    diff-lcs (1.5.1)
     docile (1.3.2)
     ed25519 (1.2.4)
     equalizer (0.0.11)
@@ -134,8 +136,9 @@ GEM
     gherkin (5.1.0)
     gssapi (1.3.0)
       ffi (>= 1.0.1)
-    gyoku (1.3.1)
+    gyoku (1.4.0)
       builder (>= 2.1.2)
+      rexml (~> 3.0)
     hashie (3.6.0)
     highline (1.7.10)
     httpclient (2.8.3)
@@ -166,7 +169,7 @@ GEM
       multi_json (~> 1.14)
     mini_portile2 (2.8.5)
     minitar (0.9)
-    minitest (5.14.2)
+    minitest (5.22.3)
     mixlib-archive (1.0.7)
       mixlib-log
     mixlib-authentication (2.1.1)
@@ -194,8 +197,8 @@ GEM
       net-ssh (>= 2.6.5)
       net-ssh-gateway (>= 1.2.0)
     net-telnet (0.1.1)
-    nokogiri (1.15.5)
-      mini_portile2 (~> 2.8.2)
+    nokogiri (1.13.10)
+      mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
     nori (2.6.0)
     octokit (4.18.0)
@@ -226,7 +229,7 @@ GEM
     racc (1.7.3)
     rack (2.2.3)
     rainbow (3.0.0)
-    rake (13.0.1)
+    rake (13.2.1)
     rbnacl (4.0.2)
       ffi
     rbnacl-libsodium (1.0.16)
@@ -239,7 +242,7 @@ GEM
       rspec-mocks (~> 3.9.0)
     rspec-core (3.9.3)
       rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
+    rspec-expectations (3.9.4)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-its (1.3.0)
@@ -287,18 +290,19 @@ GEM
       net-ssh (>= 2.7)
       net-telnet (= 0.1.1)
       sfl
-    strings (0.2.0)
+    strings (0.2.1)
       strings-ansi (~> 0.2)
-      unicode-display_width (~> 1.5)
+      unicode-display_width (>= 1.5, < 3.0)
       unicode_utils (~> 1.4)
     strings-ansi (0.2.0)
     sync (0.5.0)
     syslog-logger (1.6.8)
     systemu (2.6.5)
     term-ansicolor (1.7.1)
       tins (~> 1.0)
-    test-kitchen (2.7.2)
+    test-kitchen (2.12.0)
       bcrypt_pbkdf (~> 1.0)
+      chef-utils (>= 16.4.35)
       ed25519 (~> 1.2)
       license-acceptance (>= 1.0.11, < 3.0)
       mixlib-install (~> 3.6)
@@ -317,7 +321,7 @@ GEM
     tomlrb (1.3.0)
     treetop (1.6.11)
       polyglot (~> 0.3)
-    tty-box (0.6.0)
+    tty-box (0.7.0)
       pastel (~> 0.8)
       strings (~> 0.2.0)
       tty-cursor (~> 0.7)
@@ -362,7 +366,7 @@ GEM
     wisper (2.0.1)
     wmi-lite (1.0.5)
     yaml (0.1.0)
-    zeitwerk (2.4.0)
+    zeitwerk (2.6.13)
 
 PLATFORMS
   ruby
@@ -385,10 +389,11 @@ DEPENDENCIES
   rake
   rbnacl (~> 4.0.2)
   rbnacl-libsodium (~> 1.0.16)
+  rspec-expectations (< 3.12.4)
   rubocop (~> 0.80.1)
   test-kitchen
   virtus
   yaml
 
 BUNDLED WITH
-   2.3.22
+   2.3.26

README.md

@@ -80,23 +80,6 @@ The following Opscode cookbooks are dependencies:
 
 5. Wait for the next scheduled `chef-client` run or trigger it manually.
 
-### Dockerized environment
-
-To build a Docker environment, use the files under `docker_test_env`:
-
-```
-cd docker_test_env
-docker build -t chef-datadog-container .
-```
-
-To run the container use:
-
-```
-docker run -d -v /dev/vboxdrv:/dev/vboxdrv --privileged=true chef-datadog-container
-```
-
-Then attach a console to the container or use the VScode remote-container feature to develop inside the container.
-
 #### Datadog attributes
 
 The following methods are available for adding your [Datadog API and application keys][4]:
@@ -396,6 +379,32 @@ To get the available versions of the integrations, see the integration-specific
 
 **Note**: For Chef Windows users, the `chef-client` must have read access to the `datadog.yaml` file when the `datadog-agent` binary available on the node is used by this resource.
 
+## Development
+
+### Dockerized environment
+
+To build a Docker environment with which to run kitchen tests, use the files under `docker_test_env`:
+
+```
+cd docker_test_env
+docker build -t chef-datadog-test-env .
+```
+
+To run the container use:
+
+```
+docker run -d -v /var/run/docker.sock:/var/run/docker.sock chef-datadog-test-env
+```
+
+Then attach a console to the container or use the VS Code remote-container feature to develop inside the container.
+
+To run kitchen-docker tests from within the container:
+
+```
+# Note: Also set KITCHEN_DOCKER_HOSTNAME=host.docker.internal if on MacOS or Windows
+# Run this under a login shell (otherwise `bundle` won't be found)
+KITCHEN_LOCAL_YAML=kitchen.docker.yml bundle exec rake circle
+```
 
 [1]: https://github.com/DataDog/chef-datadog/blob/master/attributes/default.rb
 [2]: https://github.com/DataDog/chef-datadog/releases/tag/v2.18.0

docker_test_env/Dockerfile

@@ -1,25 +1,59 @@
-# docker run --rm -it -v /dev/vboxdrv:/dev/vboxdrv
 ARG BASE_IMAGE=ubuntu:20.04
 FROM $BASE_IMAGE
 ARG TZ=America/New_York
-ARG RUBY_VERSION=2.6
+ARG RUBY_VERSION=2.6.3
 
 ENV TZ=$TZ
 ENV RUBY_VERSION=$RUBY_VERSION
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-RUN apt-get update && apt-get install -y wget virtualbox git zlib1g-dev curl
+RUN apt-get update && apt-get install -y wget git zlib1g-dev curl gpg
 
 # RVM
-RUN gpg --keyserver hkp://pool.sks-keyservers.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
+RUN gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
 RUN curl -sSL https://get.rvm.io | bash -s stable
-RUN /bin/bash -l -c "rvm requirements"
+
+RUN /bin/bash -l -c "rvm requirements" # || cat /usr/local/rvm/log/*/*.log && false
 RUN /bin/bash -l -c "rvm install ${RUBY_VERSION} && rvm cleanup all"
-RUN /bin/bash -l -c "gem install bundler --no-document"
+RUN /bin/bash -l -c "gem install bundler:2.3.26 --no-document"
+
+RUN /bin/bash -l -c "gem install nori:2.6.0 test-kitchen:2.7.2 octokit:4.18.0 semverse:3.0.0 chef:14.10.9 berkshelf:7.0.10 kitchen-vagrant:1.7.0 kitchen-docker:2.3.0"
+
+# build with --build-arg INSTALL_VAGRANT=1 to get vagrant support (not needed for kitchen-docker-tests)
+# requires `-v /dev/vboxdrv:/dev/vboxdrv` when running
+ARG INSTALL_VAGRANT
+RUN if [ -n "${INSTALL_VAGRANT}" ]; then \
+    apt-get install -y virtualbox && wget https://releases.hashicorp.com/vagrant/2.2.9/vagrant_2.2.9_x86_64.deb && dpkg -i vagrant_2.2.9_x86_64.deb && rm vagrant_2.2.9_x86_64.deb \
+    ; fi
+
+# Install docker. Requires `-v /var/run/docker.sock:/var/run/docker.sock` when running to use the host's docker daemon
+RUN \
+    apt-get update && \
+    apt-get install -y ca-certificates && \
+    install -m 0755 -d /etc/apt/keyrings && \
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc && \
+    chmod a+r /etc/apt/keyrings/docker.asc && \
+    # Add the repository to Apt sources:
+    echo \
+    "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+    $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+    tee /etc/apt/sources.list.d/docker.list > /dev/null && \
+    apt-get update && \
+    apt-get install -y docker-ce-cli
 
-RUN /bin/bash -l -c "gem install test-kitchen berkshelfbundler kitchen-vagrant"
-RUN wget https://releases.hashicorp.com/vagrant/2.2.9/vagrant_2.2.9_x86_64.deb && dpkg -i vagrant_2.2.9_x86_64.deb && rm vagrant_2.2.9_x86_64.deb
 COPY entrypoint.sh /usr/bin/entrypoint.sh
 RUN chmod 755 /usr/bin/entrypoint.sh
-ENTRYPOINT ["/usr/bin/entrypoint.sh"]
+
+# Apply patch kitchen-docker to make it possible to manually override the hostname for docker
+# with the `KITCHEN_DOCKER_HOSTNAME` environment variable, which is necessary when working on non-native docker.
+COPY docker.rb.patch /
+RUN patch -p1 -i /docker.rb.patch
+
+# Helpful defaults to run kitchen tests
+ENV DOCKER_BUILDKIT=0 \
+    CIRCLE_NODE_TOTAL=1 \
+    CIRCLE_NODE_INDEX=0 \
+    CHEF_LICENSE=accept
+
+ENTRYPOINT ["/bin/bash", "-l", "-c", "/usr/bin/entrypoint.sh"]

docker_test_env/docker.rb.patch

@@ -0,0 +1,11 @@
+--- /usr/local/rvm/gems/ruby-2.6.3/gems/kitchen-docker-2.3.0/lib/kitchen/driver/docker.rb	2024-04-09 10:54:16.032712772 -0400
++++ /docker.rb	2024-04-09 10:53:25.988909160 -0400
+@@ -103,6 +103,6 @@
+         state[:ssh_key] = config[:private_key]
+         state[:image_id] = build_image(state) unless state[:image_id]
+         state[:container_id] = run_container(state) unless state[:container_id]
+-        state[:hostname] = remote_socket? ? socket_uri.host : 'localhost'
++        state[:hostname] = ENV['KITCHEN_DOCKER_HOSTNAME'] || (remote_socket? ? socket_uri.host : 'localhost')
+         state[:port] = container_ssh_port(state)
+         wait_for_sshd(state[:hostname], nil, :port => state[:port]) if config[:wait_for_sshd]
+       end

recipes/_install-linux.rb

@@ -45,6 +45,21 @@
     action :upgrade
   end
 when 'rhel', 'fedora', 'amazon'
+  # Centos < 7 was deprecated on agent {6,7}.52
+  agent_major_version = Chef::Datadog.agent_major_version(node)
+  if agent_major_version.to_i >= 6 && platform_family?('rhel') && node['platform_version'].to_i < 7
+    agent_minor_version = Chef::Datadog.agent_minor_version(node)
+    if dd_agent_version && agent_minor_version && agent_minor_version >= 52
+      # Error out with a useful message when the version was pinned to an unsupported one
+      Chef::Log.error("Agent versions #{agent_major_version}.52 and above not supported by current OS (RHEL < 7 equivalent).")
+      raise
+    else
+      # Set an upper bound for the package when the version was left unpinned
+      # Bounds like this one need to go on the package name, they're not supported on the version field
+      dd_agent_flavor = "#{dd_agent_flavor} < 1:#{agent_major_version}.52.0-1"
+    end
+  end
+
   if (platform_family?('rhel')   && node['platform_version'].to_i >= 8)    ||
      (platform_family?('fedora') && node['platform_version'].to_i >= 28)   ||
      (platform_family?('amazon') && node['platform_version'].to_i >= 2022)

spec/dd-agent_spec.rb

@@ -40,6 +40,12 @@ def set_env_var(name, value)
   it_behaves_like 'rhellions datadog-agent'
 end
 
+shared_examples_for 'rhellions no version set rhel<7' do
+  it_behaves_like 'common linux resources'
+
+  it_behaves_like 'rhellions datadog-agent rhel<7'
+end
+
 shared_examples_for 'rhellions dnf no version set' do
   it_behaves_like 'common linux resources'
 
@@ -103,11 +109,11 @@ def set_env_var(name, value)
       it_behaves_like 'debianoids no version set'
     end
 
-    context 'on RedHat-family distro above 6.x' do
+    context 'on RedHat-family distro above 7.x' do
       cached(:chef_run) do
         ChefSpec::SoloRunner.new(
           :platform => 'centos',
-          :version => '6.9'
+          :version => '7.7'
         ) do |node|
           node.normal['datadog'] = { 'api_key' => 'somethingnotnil' }
           node.normal['languages'] = { 'python' => { 'version' => '2.6.2' } }
@@ -118,19 +124,19 @@ def set_env_var(name, value)
       it_behaves_like 'rhellions no version set'
     end
 
-    context 'on CentOS 5.11 distro' do
+    context 'on CentOS < 7 distro' do
       cached(:chef_run) do
         ChefSpec::SoloRunner.new(
           :platform => 'centos',
-          :version => '5.11'
+          :version => '6.10'
         ) do |node|
           node.normal['datadog'] = { 'api_key' => 'somethingnotnil' }
           node.normal['languages'] = { 'python' => { 'version' => '2.4.3' } }
         end.converge described_recipe
       end
 
       it_behaves_like 'repo recipe'
-      it_behaves_like 'rhellions no version set'
+      it_behaves_like 'rhellions no version set rhel<7'
     end
 
     context 'on Fedora distro' do
@@ -484,7 +490,7 @@ def set_env_var(name, value)
       cached(:chef_run) do
         ChefSpec::SoloRunner.new(
           :platform => 'redhat',
-          :version => '6.10'
+          :version => '7.7'
         ) do |node|
           node.normal['datadog'] = {
             'agent_major_version' => 6,
@@ -914,7 +920,7 @@ def set_env_var(name, value)
         cached(:chef_run) do
           ChefSpec::SoloRunner.new(
             platform: 'centos',
-            version: '6.9'
+            version: '7.7'
           ) do |node|
             node.normal['datadog'] = { 'api_key' => 'somethingnotnil' }
             node.normal['languages'] = { 'python' => { 'version' => '2.6.2' } }
@@ -959,7 +965,7 @@ def set_env_var(name, value)
         cached(:chef_run) do
           ChefSpec::SoloRunner.new(
             platform: 'centos',
-            version: '6.9'
+            version: '7.7'
           ) do |node|
             node.normal['datadog'] = {
               'api_key' => 'somethingnotnil',