From 8a66c9430f06e5f731c1b3c7472a6961808393f6 Mon Sep 17 00:00:00 2001 From: Matt Brictson Date: Fri, 14 Jun 2024 17:14:10 -0700 Subject: [PATCH 1/6] Replace Vagrant with Docker Compose --- .docker/Dockerfile | 6 ++ .docker/ubuntu_setup.sh | 22 +++++++ .gitignore | 1 - .rubocop_todo.yml | 7 -- CONTRIBUTING.md | 4 +- RELEASING.md | 2 +- Rakefile | 4 -- Vagrantfile | 24 ------- docker-compose.yml | 8 +++ test/boxes.json | 17 ----- .../backends/netssh_transfer_tests.rb | 2 +- test/functional/backends/test_netssh.rb | 2 +- ...sh_server_comes_up_for_functional_tests.rb | 24 ------- test/helper.rb | 45 +------------ test/support/docker_wrapper.rb | 50 +++++++++++++++ test/support/vagrant_wrapper.rb | 64 ------------------- test/unit/test_command_map.rb | 16 ++--- 17 files changed, 101 insertions(+), 197 deletions(-) create mode 100644 .docker/Dockerfile create mode 100755 .docker/ubuntu_setup.sh delete mode 100644 Vagrantfile create mode 100644 docker-compose.yml delete mode 100644 test/boxes.json delete mode 100644 test/functional/test_ssh_server_comes_up_for_functional_tests.rb create mode 100644 test/support/docker_wrapper.rb delete mode 100644 test/support/vagrant_wrapper.rb diff --git a/.docker/Dockerfile b/.docker/Dockerfile new file mode 100644 index 00000000..12d6d7b7 --- /dev/null +++ b/.docker/Dockerfile @@ -0,0 +1,6 @@ +FROM ubuntu:22.04 +WORKDIR /provision +COPY ./ubuntu_setup.sh ./ +RUN ./ubuntu_setup.sh +EXPOSE 22 +CMD ["/usr/sbin/sshd", "-D"] diff --git a/.docker/ubuntu_setup.sh b/.docker/ubuntu_setup.sh new file mode 100755 index 00000000..99b5e95b --- /dev/null +++ b/.docker/ubuntu_setup.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +export DEBIAN_FRONTEND=noninteractive +apt -y update + +# Create `deployer` user that can sudo without a password +apt-get -y install sudo +adduser --disabled-password deployer < /dev/null +echo "deployer:topsecret" | chpasswd +echo "deployer ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +# Install and configure sshd +apt-get -y install openssh-server +{ + echo "Port 22" + echo "PasswordAuthentication yes" + echo "ChallengeResponseAuthentication no" +} >> /etc/ssh/sshd_config +mkdir /var/run/sshd +chmod 0755 /var/run/sshd diff --git a/.gitignore b/.gitignore index 7b0b67f0..8207258a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,5 @@ bin/rake .bundle .yardoc -.vagrant* test/tmp Gemfile.lock diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 242f9414..a680666f 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -102,7 +102,6 @@ Layout/IndentHash: Exclude: - 'test/functional/backends/test_local.rb' - 'test/functional/backends/test_netssh.rb' - - 'test/support/vagrant_wrapper.rb' - 'test/unit/formatters/test_custom.rb' - 'test/unit/formatters/test_pretty.rb' - 'test/unit/test_mapping_interaction_handler.rb' @@ -445,12 +444,6 @@ Style/MethodName: Exclude: - 'test/unit/test_color.rb' -# Offense count: 1 -# Cop supports --auto-correct. -Style/MutableConstant: - Exclude: - - 'Vagrantfile' - # Offense count: 1 # Cop supports --auto-correct. # Configuration parameters: Strict. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 54710b5c..8e35b7a4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,8 +24,8 @@ using unsupported features. ## Tests -SSHKit has a unit test suite and a functional test suite. Some functional tests run against -[Vagrant](https://www.vagrantup.com/) VMs. If possible, you should make sure that the +SSHKit has a unit test suite and a functional test suite. Some functional tests run using +[Docker](https://docs.docker.com/get-docker/). If possible, you should make sure that the tests pass for each commit by running `rake` in the sshkit directory. This is in case we need to cherry pick commits or rebase. You should ensure the tests pass, (preferably on the minimum and maximum ruby version), before creating a PR. diff --git a/RELEASING.md b/RELEASING.md index 5f035192..348d879d 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -9,7 +9,7 @@ ## How to release 1. Run `bundle install` to make sure that you have all the gems necessary for testing and releasing. -2. **Ensure the tests are passing by running `rake test`.** If functional tests fail, ensure you have [Vagrant](https://www.vagrantup.com) installed and have started it with `vagrant up`. +2. **Ensure the tests are passing by running `rake test`.** If functional tests fail, ensure you have [Docker installed](https://docs.docker.com/get-docker/) and running. 3. Determine which would be the correct next version number according to [semver](http://semver.org/). 4. Update the version in `./lib/sshkit/version.rb`. 5. Commit the `version.rb` change with a message like "Preparing vX.Y.Z" diff --git a/Rakefile b/Rakefile index 1fea0323..8f5558e9 100644 --- a/Rakefile +++ b/Rakefile @@ -21,10 +21,6 @@ namespace :test do end -Rake::Task["test:functional"].enhance do - warn "Remember there are still some VMs running, kill them with `vagrant halt` if you are finished using them." -end - desc 'Run RuboCop lint checks' RuboCop::RakeTask.new(:lint) do |task| task.options = ['--lint'] diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index 13d20725..00000000 --- a/Vagrantfile +++ /dev/null @@ -1,24 +0,0 @@ -VAGRANTFILE_API_VERSION = "2" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = 'bento/ubuntu-22.10' - - config.vm.boot_timeout = 600 # seconds - config.ssh.insert_key = false - config.vm.provision "shell", inline: <<-SHELL - echo 'ClientAliveInterval 3' >> /etc/ssh/sshd_config - echo 'ClientAliveCountMax 3' >> /etc/ssh/sshd_config - echo 'MaxAuthTries 6' >> /etc/ssh/sshd_config - service ssh restart - SHELL - - json_config_path = File.join("test", "boxes.json") - list = File.open(json_config_path).read - list = JSON.parse(list) - - list.each do |vm| - config.vm.define vm["name"] do |web| - web.vm.network "forwarded_port", guest: 22, host: vm["port"] - end - end -end diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..4ce0f79f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,8 @@ +name: sshkit + +services: + ssh_server: + build: + context: .docker + ports: + - "2122:22" diff --git a/test/boxes.json b/test/boxes.json deleted file mode 100644 index 57793583..00000000 --- a/test/boxes.json +++ /dev/null @@ -1,17 +0,0 @@ -[ - { - "name": "one", - "port": 3001, - "user": "vagrant", - "password": "vagrant", - "hostname": "localhost" - }, - { - "name": "two", - "port": 3002 - }, - { - "name": "three", - "port": 3003 - } -] diff --git a/test/functional/backends/netssh_transfer_tests.rb b/test/functional/backends/netssh_transfer_tests.rb index 32562443..796d10e6 100644 --- a/test/functional/backends/netssh_transfer_tests.rb +++ b/test/functional/backends/netssh_transfer_tests.rb @@ -11,7 +11,7 @@ def setup end def a_host - VagrantWrapper.hosts['one'] + DockerWrapper.host end def test_upload_and_then_capture_file_contents diff --git a/test/functional/backends/test_netssh.rb b/test/functional/backends/test_netssh.rb index 3a2f03d6..ee7000ad 100644 --- a/test/functional/backends/test_netssh.rb +++ b/test/functional/backends/test_netssh.rb @@ -15,7 +15,7 @@ def setup end def a_host - VagrantWrapper.hosts['one'] + DockerWrapper.host end def test_simple_netssh diff --git a/test/functional/test_ssh_server_comes_up_for_functional_tests.rb b/test/functional/test_ssh_server_comes_up_for_functional_tests.rb deleted file mode 100644 index d1e42ab3..00000000 --- a/test/functional/test_ssh_server_comes_up_for_functional_tests.rb +++ /dev/null @@ -1,24 +0,0 @@ -require 'helper' - -module SSHKit - - class TestHost < FunctionalTest - - def host - @_host ||= Host.new('') - end - - def test_that_it_works - assert true - end - - def test_creating_a_user_gives_us_back_his_private_key_as_a_string - skip 'It is not safe to create an user for non vagrant envs' unless VagrantWrapper.running? - keys = create_user_with_key(:peter) - assert_equal [:one, :two, :three], keys.keys - assert keys.values.all? - end - - end - -end diff --git a/test/helper.rb b/test/helper.rb index f46bdfed..da44d2c3 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -28,51 +28,10 @@ def flush_connections end class FunctionalTest < Minitest::Test - def setup - unless VagrantWrapper.running? - warn "Vagrant VMs are not running. Please, start it manually with `vagrant up`" - end + require_relative "support/docker_wrapper" + DockerWrapper.start unless DockerWrapper.running? end - - private - - def create_user_with_key(username, password = :secret) - username, password = username.to_s, password.to_s - - keys = VagrantWrapper.hosts.collect do |_name, host| - Net::SSH.start(host.hostname, host.user, port: host.port, password: host.password) do |ssh| - - # Remove the user, make it again, force-generate a key for him - # short keys save us a few microseconds - ssh.exec!("sudo userdel -rf #{username}; true") # The `rescue nil` of the shell world - ssh.exec!("sudo useradd -m #{username}") - ssh.exec!("sudo echo y | ssh-keygen -b 1024 -f #{username} -N ''") - ssh.exec!("sudo chown vagrant:vagrant #{username}*") - ssh.exec!("sudo echo #{username}:#{password} | chpasswd") - - # Make the .ssh directory, change the ownership and the - ssh.exec!("sudo mkdir -p /home/#{username}/.ssh") - ssh.exec!("sudo chown #{username}:#{username} /home/#{username}/.ssh") - ssh.exec!("sudo chmod 700 /home/#{username}/.ssh") - - # Move the key to authorized keys and chown and chmod it - ssh.exec!("sudo cat #{username}.pub > /home/#{username}/.ssh/authorized_keys") - ssh.exec!("sudo chown #{username}:#{username} /home/#{username}/.ssh/authorized_keys") - ssh.exec!("sudo chmod 600 /home/#{username}/.ssh/authorized_keys") - - key = ssh.exec!("cat /home/vagrant/#{username}") - - # Clean Up Files - ssh.exec!("sudo rm #{username} #{username}.pub") - - key - end - end - - Hash[VagrantWrapper.hosts.collect { |n, _h| n.to_sym }.zip(keys)] - end - end # diff --git a/test/support/docker_wrapper.rb b/test/support/docker_wrapper.rb new file mode 100644 index 00000000..16f09672 --- /dev/null +++ b/test/support/docker_wrapper.rb @@ -0,0 +1,50 @@ +Minitest.after_run do + DockerWrapper.stop if DockerWrapper.running? +end + +module DockerWrapper + class << self + def host + SSHKit::Host.new( + user: "deployer", + hostname: "localhost", + port: "2122", + password: "topsecret", + ssh_options: host_verify_options + ) + end + + def running? + out, status = run_compose_command("ps --status running", false) + status.success? && out.include?("ssh_server") + end + + def start + run_compose_command("up -d") + end + + def stop + run_compose_command("down") + end + + private + + def run_compose_command(command, echo=true) + $stderr.puts "[docker compose] #{command}" if echo + stdout, stderr, status = Open3.capture3("docker compose #{command}") + + output = stdout + stderr + output.each_line { |line| $stderr.puts "[docker compose] #{line}" } if echo + + [output, status] + end + + def host_verify_options + if Net::SSH::Version::MAJOR >= 5 + { verify_host_key: :never } + else + { paranoid: false } + end + end + end +end diff --git a/test/support/vagrant_wrapper.rb b/test/support/vagrant_wrapper.rb deleted file mode 100644 index fbf3ae4d..00000000 --- a/test/support/vagrant_wrapper.rb +++ /dev/null @@ -1,64 +0,0 @@ -class VagrantWrapper - class << self - def hosts - @vm_hosts ||= begin - result = {} - - boxes = boxes_list - - unless running? - boxes.map! do |box| - box['user'] = ENV['USER'] - box['port'] = '22' - box - end - end - - boxes.each do |vm| - result[vm['name']] = vm_host(vm) - end - - result - end - end - - def running? - @running ||= begin - status = `#{vagrant_binary} status` - status.include?('running') - end - end - - def boxes_list - json_config_path = File.join('test', 'boxes.json') - boxes = File.open(json_config_path).read - JSON.parse(boxes) - end - - def vagrant_binary - 'vagrant' - end - - private - - def vm_host(vm) - host_options = { - user: vm['user'] || 'vagrant', - hostname: vm['hostname'] || 'localhost', - port: vm['port'] || '22', - password: vm['password'] || 'vagrant', - ssh_options: host_verify_options - } - - SSHKit::Host.new(host_options) - end - - def host_verify_options - if Net::SSH::Version::MAJOR >= 5 - { verify_host_key: :never } - else - { paranoid: false } - end - end - end -end diff --git a/test/unit/test_command_map.rb b/test/unit/test_command_map.rb index 06d37dfc..38d58e00 100644 --- a/test/unit/test_command_map.rb +++ b/test/unit/test_command_map.rb @@ -27,26 +27,26 @@ def test_setter_procs def test_prefix map = CommandMap.new - map.prefix[:rake].push("/home/vagrant/.rbenv/bin/rbenv exec") + map.prefix[:rake].push("/home/deployer/.rbenv/bin/rbenv exec") map.prefix[:rake].push("bundle exec") - assert_equal map[:rake], "/home/vagrant/.rbenv/bin/rbenv exec bundle exec rake" + assert_equal map[:rake], "/home/deployer/.rbenv/bin/rbenv exec bundle exec rake" end def test_prefix_procs map = CommandMap.new - map.prefix[:rake].push("/home/vagrant/.rbenv/bin/rbenv exec") + map.prefix[:rake].push("/home/deployer/.rbenv/bin/rbenv exec") map.prefix[:rake].push(proc{ "bundle exec" }) - assert_equal map[:rake], "/home/vagrant/.rbenv/bin/rbenv exec bundle exec rake" + assert_equal map[:rake], "/home/deployer/.rbenv/bin/rbenv exec bundle exec rake" end def test_prefix_unshift map = CommandMap.new map.prefix[:rake].push("bundle exec") - map.prefix[:rake].unshift("/home/vagrant/.rbenv/bin/rbenv exec") + map.prefix[:rake].unshift("/home/deployer/.rbenv/bin/rbenv exec") - assert_equal map[:rake], "/home/vagrant/.rbenv/bin/rbenv exec bundle exec rake" + assert_equal map[:rake], "/home/deployer/.rbenv/bin/rbenv exec bundle exec rake" end def test_indifferent_setter @@ -59,10 +59,10 @@ def test_indifferent_setter def test_indifferent_prefix map = CommandMap.new - map.prefix[:rake].push("/home/vagrant/.rbenv/bin/rbenv exec") + map.prefix[:rake].push("/home/deployer/.rbenv/bin/rbenv exec") map.prefix["rake"].push("bundle exec") - assert_equal map[:rake], "/home/vagrant/.rbenv/bin/rbenv exec bundle exec rake" + assert_equal map[:rake], "/home/deployer/.rbenv/bin/rbenv exec bundle exec rake" end def test_prefix_initialization_is_thread_safe From fa2b58336bf73d1d73e2dd472caf353d0a175155 Mon Sep 17 00:00:00 2001 From: Matt Brictson Date: Fri, 14 Jun 2024 17:14:20 -0700 Subject: [PATCH 2/6] Reenable functional tests in CI --- .github/workflows/ci.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05f50c14..27f0ac8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,3 +69,30 @@ jobs: bundler-cache: true - name: Run rubocop run: bundle exec rake lint + + functional: + runs-on: ubuntu-latest + strategy: + matrix: + ruby: ["2.0", "ruby"] + steps: + - uses: actions/checkout@v4 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true + - name: Run functional tests + run: bundle exec rake test:functional + + functional-all: + runs-on: ubuntu-latest + needs: [functional] + if: always() + steps: + - name: All tests ok + if: ${{ !(contains(needs.*.result, 'failure')) }} + run: exit 0 + - name: Some tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 From f4939ee26a8aca85a03fd4044259e3e71ace9e4c Mon Sep 17 00:00:00 2001 From: Matt Brictson Date: Thu, 20 Jun 2024 17:06:06 -0700 Subject: [PATCH 3/6] Stream docker compose output while building image --- test/support/docker_wrapper.rb | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/test/support/docker_wrapper.rb b/test/support/docker_wrapper.rb index 16f09672..cb63d30d 100644 --- a/test/support/docker_wrapper.rb +++ b/test/support/docker_wrapper.rb @@ -31,12 +31,20 @@ def stop def run_compose_command(command, echo=true) $stderr.puts "[docker compose] #{command}" if echo - stdout, stderr, status = Open3.capture3("docker compose #{command}") - - output = stdout + stderr - output.each_line { |line| $stderr.puts "[docker compose] #{line}" } if echo + Open3.popen2e("docker compose #{command}") do |stdin, outerr, wait_thread| + stdin.close + output = Thread.new { capture_stream(outerr, echo) } + [output.value, wait_thread.value] + end + end - [output, status] + def capture_stream(stream, echo=true) + buffer = +'' + while line = stream.gets + buffer << line + $stderr.puts("[docker compose] #{line}") if echo + end + buffer end def host_verify_options From a87b26379678e1597f10a6d06c872e78fbc49aba Mon Sep 17 00:00:00 2001 From: Matt Brictson Date: Thu, 20 Jun 2024 17:08:22 -0700 Subject: [PATCH 4/6] Fix RuboCop issue and Ruby 2 compat --- test/support/docker_wrapper.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/support/docker_wrapper.rb b/test/support/docker_wrapper.rb index cb63d30d..428503cb 100644 --- a/test/support/docker_wrapper.rb +++ b/test/support/docker_wrapper.rb @@ -39,8 +39,8 @@ def run_compose_command(command, echo=true) end def capture_stream(stream, echo=true) - buffer = +'' - while line = stream.gets + buffer = String.new + while (line = stream.gets) buffer << line $stderr.puts("[docker compose] #{line}") if echo end From 6b5f18dd618933876db6da84c169bdce05770270 Mon Sep 17 00:00:00 2001 From: Matt Brictson Date: Thu, 20 Jun 2024 17:14:56 -0700 Subject: [PATCH 5/6] Wait for Docker container to start, to avoid test flake --- test/helper.rb | 5 ++++- test/support/docker_wrapper.rb | 14 +++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/test/helper.rb b/test/helper.rb index da44d2c3..162fbdaf 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -30,7 +30,10 @@ def flush_connections class FunctionalTest < Minitest::Test def setup require_relative "support/docker_wrapper" - DockerWrapper.start unless DockerWrapper.running? + return if DockerWrapper.running? + + DockerWrapper.start + DockerWrapper.wait_for_ssh_server end end diff --git a/test/support/docker_wrapper.rb b/test/support/docker_wrapper.rb index 428503cb..5d6d8393 100644 --- a/test/support/docker_wrapper.rb +++ b/test/support/docker_wrapper.rb @@ -1,14 +1,18 @@ +require "socket" + Minitest.after_run do DockerWrapper.stop if DockerWrapper.running? end module DockerWrapper + SSH_SERVER_PORT = 2122 + class << self def host SSHKit::Host.new( user: "deployer", hostname: "localhost", - port: "2122", + port: SSH_SERVER_PORT, password: "topsecret", ssh_options: host_verify_options ) @@ -27,6 +31,14 @@ def stop run_compose_command("down") end + def wait_for_ssh_server(retries=3) + Socket.tcp("localhost", SSH_SERVER_PORT, connect_timeout: 1).close + rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT + retries -= 1 + sleep(2) && retry if retries.positive? + raise + end + private def run_compose_command(command, echo=true) From 37dd32589a26d3f05d90358f55e447033b3c03de Mon Sep 17 00:00:00 2001 From: Matt Brictson Date: Thu, 20 Jun 2024 17:19:19 -0700 Subject: [PATCH 6/6] A little extra delay, for good measure --- test/support/docker_wrapper.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/support/docker_wrapper.rb b/test/support/docker_wrapper.rb index 5d6d8393..2165e699 100644 --- a/test/support/docker_wrapper.rb +++ b/test/support/docker_wrapper.rb @@ -33,6 +33,7 @@ def stop def wait_for_ssh_server(retries=3) Socket.tcp("localhost", SSH_SERVER_PORT, connect_timeout: 1).close + sleep(1) rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT retries -= 1 sleep(2) && retry if retries.positive?