Skip to content

Commit

Permalink
Fixed end2end test concept and testcases (#510)
Browse files Browse the repository at this point in the history 
Details:

* Test: Added missing env.vars in the pytest invocation for end2end tests.

* Test: Added support for specifying 'hmc_auth.ca_certs' and 'hmc_auth.verify'
  from the 'hmc_verify_cert' parameter in the HMC definition file in
  end2end test cases for zhmc_partition and zhmc_user.

* Test: Made end2end testing compatible with zhmcclient.testutils support for
  HMC definition files.
  The HMC definition file is now by default ~/.hmc_hmc_definitions.yaml, and
  the env.var to override that is now TESTHMCFILE.

* Test: Added support for a TESTCASES env.var for filtering testcases with the
  pytest -k option.

* Fixed incorrect assumption about defaulting of optional module parameters:
  This is only done automatically when the module is invoked by ansible,
  but not when invoked by the end2end testcases. Changed the end2end testcases
  for zhmc_partition and zhmc_user to specify the optional parameters.

* Changed picking of partition to test in end2end testcase
  test_partition_facts() to be a random partition of each type.

Signed-off-by: Andreas Maier <maiera@de.ibm.com>
  • Loading branch information
@andy-maier
andy-maier authored Apr 11, 2022
1 parent 41d85cd commit 873349d
Show file tree
Hide file tree
Showing 5 changed files with 137 additions and 61 deletions.
40 changes: 24 additions & 16 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,16 +86,16 @@ test_py_files := \
$(wildcard $(test_dir)/*/*.py) \
$(wildcard $(test_dir)/*/*/*.py) \

# Directory with hmc_definitions.yaml file for end2end tests
default_test_hmc_dir := ../python-zhmcclient/tests
ifndef TESTHMCDIR
TESTHMCDIR := $(default_test_hmc_dir)
# Path name of default HMC definitions file used for end2end tests.
default_testhmcfile := ~/.zhmc_hmc_definitions.yaml
ifndef TESTHMCFILE
TESTHMCFILE := $(default_testhmcfile)
endif

# Nickname of test HMC in hmc_definitions.yaml file for end2end tests
default_test_hmc := default
# Default HMC nickname in HMC definitions file
default_testhmc := default
ifndef TESTHMC
TESTHMC := $(default_test_hmc)
TESTHMC := $(default_testhmc)
endif

# Flake8 options
Expand Down Expand Up @@ -144,7 +144,11 @@ dist_dependent_files := \
sphinx_opts := -v

# Pytest options
pytest_opts := $(TESTOPTS) -s
ifdef TESTCASES
pytest_opts := --color=yes -s $(TESTOPTS) -k "$(TESTCASES)"
else
pytest_opts := --color=yes -s $(TESTOPTS)
endif
ifeq ($(python_m_n_version),3.4)
pytest_cov_opts :=
else
Expand Down Expand Up @@ -172,14 +176,18 @@ help:
@echo ' linkcheck - Check links in documentation'
@echo ' all - Do all of the above'
@echo ' end2end - Run end2end tests'
@echo ' upload - Publish the collection to Ansible Galaxy'
@echo ' upload - Publish the collection to Ansible Galaxy and AutomationHub'
@echo ' clobber - Remove any produced files'
@echo 'Environment variables:'
@echo ' TESTHMC=... - Nickname of HMC to be used in end2end tests. Default: $(default_test_hmc)'
@echo ' TESTHMCDIR=... - Path name of directory with hmc_definitions.yaml file. Default: $(default_test_hmc_dir)'
@echo ' TESTOPTS=... - Additional options for py.test (e.g. "-k test_module.py")'
@echo ' PACKAGE_LEVEL="minimum" - Install minimum version of dependent Python packages'
@echo ' PACKAGE_LEVEL="latest" - Default: Install latest version of dependent Python packages'
@echo " TESTCASES=... - Testcase filter for pytest -k (e.g. 'test_func' or 'test_mod.py')"
@echo " TESTOPTS=... - Additional options for pytest (e.g. '-x')"
@echo " TESTHMC=... - HMC nickname in HMC definitions file used in end2end tests. Default: $(default_testhmc)"
@echo " TESTHMCFILE=... - Path name of HMC definitions file used in end2end tests. Default: $(default_testhmcfile)"
@echo " PACKAGE_LEVEL - Package level to be used for installing dependent Python"
@echo " packages in 'install' and 'develop' targets:"
@echo " latest - Latest package versions available on Pypi"
@echo " minimum - A minimum version as defined in minimum-constraints.txt"
@echo " Optional, defaults to 'latest'."
@echo ' PYTHON_CMD=... - Name of python command. Default: python'
@echo ' PIP_CMD=... - Name of pip command. Default: pip'
@echo ' GALAXY_TOKEN=... - Your Ansible Galaxy API token, required for upload (see https://galaxy.ansible.com/me/preferences)'
Expand Down Expand Up @@ -212,7 +220,7 @@ linkcheck: _check_version develop_$(pymn).done $(doc_rst_files)

.PHONY: test
test: _check_version develop_$(pymn).done
bash -c 'PYTHONWARNINGS=default ANSIBLE_LIBRARY=$(module_py_dir) PYTHONPATH=. pytest $(pytest_cov_opts) $(pytest_opts) $(test_dir)/unit $(test_dir)/function'
bash -c 'PYTHONWARNINGS=default ANSIBLE_LIBRARY=$(module_py_dir) PYTHONPATH=. TESTHMCFILE=$(TESTHMCFILE) TESTHMC=$(TESTHMC) pytest $(pytest_cov_opts) $(pytest_opts) $(test_dir)/unit $(test_dir)/function'
@echo '$@ done.'

.PHONY: check
Expand All @@ -239,7 +247,7 @@ endif

.PHONY: end2end
end2end: _check_version develop_$(pymn).done
bash -c 'PYTHONWARNINGS=default TESTHMCDIR=$(TESTHMCDIR) TESTHMC=$(TESTHMC) py.test -v $(pytest_opts) $(test_dir)/end2end'
bash -c 'PYTHONWARNINGS=default ANSIBLE_LIBRARY=$(module_py_dir) PYTHONPATH=. TESTHMCFILE=$(TESTHMCFILE) TESTHMC=$(TESTHMC) pytest -v $(pytest_opts) $(test_dir)/end2end'
@echo '$@ done.'

.PHONY: upload
Expand Down
16 changes: 16 additions & 0 deletions docs/source/release_notes.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,24 @@ Availability: `AutomationHub`_, `Galaxy`_, `GitHub`_
module, and made the module result in check mode consistent with non-check
mode. (issue #507)

* Test: Added missing env.vars in the pytest invocation for end2end tests.

* Test: Added missing optional module parameters in the end2end tests.

* Test: Added support for specifying 'hmc_auth.ca_certs' and 'hmc_auth.verify'
from the 'hmc_verify_cert' parameter in the HMC definition file in
end2end test cases for zhmc_partition and zhmc_user.

**Enhancements:**

* Test: Made end2end testing compatible with zhmcclient.testutils support for
HMC definition files.
The HMC definition file is now by default ~/.hmc_hmc_definitions.yaml, and
the env.var to override that is now TESTHMCFILE.

* Test: Added support for a TESTCASES env.var for filtering testcases with the
pytest -k option.

**Cleanup:**

**Known issues:**
Expand Down
44 changes: 34 additions & 10 deletions tests/end2end/test_zhmc_partition.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,23 +21,29 @@

import pytest
import mock
import random
import requests.packages.urllib3
from ansible.module_utils import six
import zhmcclient
from zhmcclient.testutils.hmc_definition_fixtures import hmc_definition, hmc_session # noqa: F401, E501

from plugins.modules import zhmc_partition
from .utils import mock_ansible_module, get_failure_msg

requests.packages.urllib3.disable_warnings()


# Partition properties that are not always present, nbut only under certain
# Partition properties that are not always present, but only under certain
# conditions.
PARTITION_CONDITIONAL_PROPS = (
'boot-ftp-password',
'boot-network-nic-name',
'boot-storage-hba-name',
'ssc-master-pw',
'ssc-boot-selection',
'ssc-host-name',
'ssc-ipv4-gateway',
'ssc-dns-servers',
'ssc-master-userid',
)


Expand Down Expand Up @@ -101,35 +107,53 @@ def assert_partition_props(partition_props):

@pytest.mark.parametrize(
"check_mode", [False, True])
@pytest.mark.parametrize(
"partition_type", ['ssc', 'linux'])
@mock.patch("plugins.modules.zhmc_partition.AnsibleModule", autospec=True)
def test_partition_facts(ansible_mod_cls, check_mode, hmc_session): # noqa: F811, E501
def test_partition_facts(
ansible_mod_cls, partition_type, check_mode, hmc_session): # noqa: F811, E501
"""
Test fact gathering on a partition.
"""

hd = hmc_session.hmc_definition

hmc_auth = dict(userid=hd.hmc_userid, password=hd.hmc_password)
if isinstance(hd.hmc_verify_cert, six.string_types):
hmc_auth['ca_certs'] = hd.hmc_verify_cert
elif isinstance(hd.hmc_verify_cert, bool):
hmc_auth['verify'] = hd.hmc_verify_cert

# Determine one of the CPCs in the HMC definition file to test
cpc_names = hd.cpcs.keys()
cpc_names = list(hd.cpcs.keys())
assert len(cpc_names) >= 1
cpc_name = cpc_names[0]

# Determine an existing partition to test. This also validates that
# the CPC defined in the HMC definition file actually exists.
client = zhmcclient.Client(hmc_session)
cpc = client.cpcs.find_by_name(cpc_name)

# Determine a random existing partition of the desired type to test.
partitions = cpc.partitions.list()
assert len(partitions) >= 1
partition = partitions[0] # Pick first partition in list

# Prepare module input parameters
typed_partitions = [p for p in partitions
if p.get_property('type') == partition_type]
if len(typed_partitions) == 0:
pytest.skip("CPC '{c}' has no partitions of type '{t}'".
format(c=cpc_name, t=partition_type))
partition = random.choice(typed_partitions)

# Prepare module input parameters (must be all required + optional)
params = {
'hmc_host': hd.hmc_host,
'hmc_auth': dict(userid=hd.hmc_userid, password=hd.hmc_password),
'hmc_auth': hmc_auth,
'cpc_name': cpc_name,
'name': partition.name,
'state': 'facts',
'properties': {},
'expand_storage_groups': False,
'expand_crypto_adapters': False,
'log_file': None,
'_faked_session': None,
}

# Prepare mocks for AnsibleModule object
Expand Down
90 changes: 57 additions & 33 deletions tests/end2end/test_zhmc_user.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,11 @@
import uuid
import pytest
import mock
import random
import requests.packages.urllib3
from collections import OrderedDict
from pprint import pformat
from ansible.module_utils import six
import zhmcclient
from zhmcclient.testutils.hmc_definition_fixtures import hmc_definition, hmc_session # noqa: F401, E501

Expand Down Expand Up @@ -207,48 +209,58 @@ def assert_user_props(user_props, expand):
@pytest.mark.parametrize(
"expand", [False, True])
@mock.patch("plugins.modules.zhmc_user.AnsibleModule", autospec=True)
def test_user_facts(ansible_mod_cls, expand, check_mode, hmc_session): # noqa: F811, E501
def test_user_facts(
ansible_mod_cls, expand, check_mode, hmc_session): # noqa: F811, E501
"""
Test fact gathering on all users of the HMC.
"""

hd = hmc_session.hmc_definition

hmc_auth = dict(userid=hd.hmc_userid, password=hd.hmc_password)
if isinstance(hd.hmc_verify_cert, six.string_types):
hmc_auth['ca_certs'] = hd.hmc_verify_cert
elif isinstance(hd.hmc_verify_cert, bool):
hmc_auth['verify'] = hd.hmc_verify_cert

# Determine an existing user to test.
client = zhmcclient.Client(hmc_session)
console = client.consoles.console

# Pick a random existing user to test
users = console.users.list()
assert len(users) >= 1

for user in users:

# Prepare module input parameters
params = {
'hmc_host': hd.hmc_host,
'hmc_auth': dict(userid=hd.hmc_userid, password=hd.hmc_password),
'name': user.name,
'state': 'facts',
'expand': expand,
'log_file': LOG_FILE,
}

# Prepare mocks for AnsibleModule object
mod_obj = mock_ansible_module(ansible_mod_cls, params, check_mode)

# Exercise the code to be tested
with pytest.raises(SystemExit) as exc_info:
zhmc_user.main()
exit_code = exc_info.value.args[0]

# Assert module exit code
assert exit_code == 0, \
"Module unexpectedly failed with this message:\n{0}". \
format(get_failure_msg(mod_obj))

# Assert module output
changed, user_props = get_module_output(mod_obj)
assert changed is False
assert_user_props(user_props, expand)
user = random.choice(users)

# Prepare module input parameters (must be all required + optional)
params = {
'hmc_host': hd.hmc_host,
'hmc_auth': hmc_auth,
'name': user.name,
'state': 'facts',
'properties': {},
'expand': expand,
'log_file': LOG_FILE,
'_faked_session': None,
}

# Prepare mocks for AnsibleModule object
mod_obj = mock_ansible_module(ansible_mod_cls, params, check_mode)

# Exercise the code to be tested
with pytest.raises(SystemExit) as exc_info:
zhmc_user.main()
exit_code = exc_info.value.args[0]

# Assert module exit code
assert exit_code == 0, \
"Module unexpectedly failed with this message:\n{0}". \
format(get_failure_msg(mod_obj))

# Assert module output
changed, user_props = get_module_output(mod_obj)
assert changed is False
assert_user_props(user_props, expand)


USER_ABSENT_PRESENT_TESTCASES = [
Expand Down Expand Up @@ -335,8 +347,16 @@ def test_user_absent_present(
"""
Test the zhmc_user module with all combinations of absent & present state.
"""
expand = False # Expansion is tested elsewhere

hd = hmc_session.hmc_definition

hmc_auth = dict(userid=hd.hmc_userid, password=hd.hmc_password)
if isinstance(hd.hmc_verify_cert, six.string_types):
hmc_auth['ca_certs'] = hd.hmc_verify_cert
elif isinstance(hd.hmc_verify_cert, bool):
hmc_auth['verify'] = hd.hmc_verify_cert

expand = False # Expansion is tested elsewhere
client = zhmcclient.Client(hmc_session)
console = client.consoles.console

Expand Down Expand Up @@ -368,16 +388,20 @@ def test_user_absent_present(

try:

# Prepare module input parameters (must be all required + optional)
params = {
'hmc_host': hd.hmc_host,
'hmc_auth': dict(userid=hd.hmc_userid, password=hd.hmc_password),
'hmc_auth': hmc_auth,
'name': user_name,
'state': input_state,
'expand': expand,
'log_file': LOG_FILE,
'_faked_session': None,
}
if input_props is not None:
params['properties'] = input_props
else:
params['properties'] = {}

mod_obj = mock_ansible_module(ansible_mod_cls, params, check_mode)

Expand Down
8 changes: 6 additions & 2 deletions tests/end2end/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,13 @@

def mock_ansible_module(ansible_mod_cls, params, check_mode):
"""
Prepare mocks for AnsibleModule object.
Prepare the mocked AnsibleModule object for the end2end test.
Note: Since this is a mocked object, the argument_spec defined in the
module is not applied, and the params must be the defaulted set of
all parameters in the module's argument_spec.
"""
mod_obj = ansible_mod_cls.return_value
mod_obj = ansible_mod_cls.return_value # the mocked object
mod_obj.params = params
mod_obj.check_mode = check_mode
mod_obj.fail_json.configure_mock(side_effect=SystemExit(1))
Expand Down

0 comments on commit 873349d

Please sign in to comment.