Fixed end2end test concept and testcases

Makefile

@@ -86,16 +86,16 @@ test_py_files := \
     $(wildcard $(test_dir)/*/*.py) \
     $(wildcard $(test_dir)/*/*/*.py) \
 
-# Directory with hmc_definitions.yaml file for end2end tests
-default_test_hmc_dir := ../python-zhmcclient/tests
-ifndef TESTHMCDIR
-  TESTHMCDIR := $(default_test_hmc_dir)
+# Path name of default HMC definitions file used for end2end tests.
+default_testhmcfile := ~/.zhmc_hmc_definitions.yaml
+ifndef TESTHMCFILE
+  TESTHMCFILE := $(default_testhmcfile)
 endif
 
-# Nickname of test HMC in hmc_definitions.yaml file for end2end tests
-default_test_hmc := default
+# Default HMC nickname in HMC definitions file
+default_testhmc := default
 ifndef TESTHMC
-  TESTHMC := $(default_test_hmc)
+  TESTHMC := $(default_testhmc)
 endif
 
 # Flake8 options
@@ -144,7 +144,11 @@ dist_dependent_files := \
 sphinx_opts := -v
 
 # Pytest options
-pytest_opts := $(TESTOPTS) -s
+ifdef TESTCASES
+  pytest_opts := --color=yes -s $(TESTOPTS) -k "$(TESTCASES)"
+else
+  pytest_opts := --color=yes -s $(TESTOPTS)
+endif
 ifeq ($(python_m_n_version),3.4)
   pytest_cov_opts :=
 else
@@ -172,14 +176,18 @@ help:
 	@echo '  linkcheck  - Check links in documentation'
 	@echo '  all        - Do all of the above'
 	@echo '  end2end    - Run end2end tests'
-	@echo '  upload     - Publish the collection to Ansible Galaxy'
+	@echo '  upload     - Publish the collection to Ansible Galaxy and AutomationHub'
 	@echo '  clobber    - Remove any produced files'
 	@echo 'Environment variables:'
-	@echo '  TESTHMC=... - Nickname of HMC to be used in end2end tests. Default: $(default_test_hmc)'
-	@echo '  TESTHMCDIR=... - Path name of directory with hmc_definitions.yaml file. Default: $(default_test_hmc_dir)'
-	@echo '  TESTOPTS=... - Additional options for py.test (e.g. "-k test_module.py")'
-	@echo '  PACKAGE_LEVEL="minimum" - Install minimum version of dependent Python packages'
-	@echo '  PACKAGE_LEVEL="latest" - Default: Install latest version of dependent Python packages'
+	@echo "  TESTCASES=... - Testcase filter for pytest -k (e.g. 'test_func' or 'test_mod.py')"
+	@echo "  TESTOPTS=... - Additional options for pytest (e.g. '-x')"
+	@echo "  TESTHMC=... - HMC nickname in HMC definitions file used in end2end tests. Default: $(default_testhmc)"
+	@echo "  TESTHMCFILE=... - Path name of HMC definitions file used in end2end tests. Default: $(default_testhmcfile)"
+	@echo "  PACKAGE_LEVEL - Package level to be used for installing dependent Python"
+	@echo "      packages in 'install' and 'develop' targets:"
+	@echo "        latest - Latest package versions available on Pypi"
+	@echo "        minimum - A minimum version as defined in minimum-constraints.txt"
+	@echo "      Optional, defaults to 'latest'."
 	@echo '  PYTHON_CMD=... - Name of python command. Default: python'
 	@echo '  PIP_CMD=... - Name of pip command. Default: pip'
 	@echo '  GALAXY_TOKEN=... - Your Ansible Galaxy API token, required for upload (see https://galaxy.ansible.com/me/preferences)'
@@ -212,7 +220,7 @@ linkcheck: _check_version develop_$(pymn).done $(doc_rst_files)
 
 .PHONY: test
 test: _check_version develop_$(pymn).done
-	bash -c 'PYTHONWARNINGS=default ANSIBLE_LIBRARY=$(module_py_dir) PYTHONPATH=. pytest $(pytest_cov_opts) $(pytest_opts) $(test_dir)/unit $(test_dir)/function'
+	bash -c 'PYTHONWARNINGS=default ANSIBLE_LIBRARY=$(module_py_dir) PYTHONPATH=. TESTHMCFILE=$(TESTHMCFILE) TESTHMC=$(TESTHMC) pytest $(pytest_cov_opts) $(pytest_opts) $(test_dir)/unit $(test_dir)/function'
 	@echo '$@ done.'
 
 .PHONY: check
@@ -239,7 +247,7 @@ endif
 
 .PHONY:	end2end
 end2end: _check_version develop_$(pymn).done
-	bash -c 'PYTHONWARNINGS=default TESTHMCDIR=$(TESTHMCDIR) TESTHMC=$(TESTHMC) py.test -v $(pytest_opts) $(test_dir)/end2end'
+	bash -c 'PYTHONWARNINGS=default ANSIBLE_LIBRARY=$(module_py_dir) PYTHONPATH=. TESTHMCFILE=$(TESTHMCFILE) TESTHMC=$(TESTHMC) pytest -v $(pytest_opts) $(test_dir)/end2end'
 	@echo '$@ done.'
 
 .PHONY: upload

docs/source/release_notes.rst

@@ -52,8 +52,24 @@ Availability: `AutomationHub`_, `Galaxy`_, `GitHub`_
   module, and made the module result in check mode consistent with non-check
   mode. (issue #507)
 
+* Test: Added missing env.vars in the pytest invocation for end2end tests.
+
+* Test: Added missing optional module parameters in the end2end tests.
+
+* Test: Added support for specifying 'hmc_auth.ca_certs' and 'hmc_auth.verify'
+  from the 'hmc_verify_cert' parameter in the HMC definition file in
+  end2end test cases for zhmc_partition and zhmc_user.
+
 **Enhancements:**
 
+* Test: Made end2end testing compatible with zhmcclient.testutils support for
+  HMC definition files.
+  The HMC definition file is now by default ~/.hmc_hmc_definitions.yaml, and
+  the env.var to override that is now TESTHMCFILE.
+
+* Test: Added support for a TESTCASES env.var for filtering testcases with the
+  pytest -k option.
+
 **Cleanup:**
 
 **Known issues:**

tests/end2end/test_zhmc_partition.py

@@ -21,23 +21,29 @@
 
 import pytest
 import mock
+import random
 import requests.packages.urllib3
+from ansible.module_utils import six
 import zhmcclient
 from zhmcclient.testutils.hmc_definition_fixtures import hmc_definition, hmc_session  # noqa: F401, E501
-
 from plugins.modules import zhmc_partition
 from .utils import mock_ansible_module, get_failure_msg
 
 requests.packages.urllib3.disable_warnings()
 
 
-# Partition properties that are not always present, nbut only under certain
+# Partition properties that are not always present, but only under certain
 # conditions.
 PARTITION_CONDITIONAL_PROPS = (
     'boot-ftp-password',
     'boot-network-nic-name',
     'boot-storage-hba-name',
     'ssc-master-pw',
+    'ssc-boot-selection',
+    'ssc-host-name',
+    'ssc-ipv4-gateway',
+    'ssc-dns-servers',
+    'ssc-master-userid',
 )
 
 
@@ -101,35 +107,53 @@ def assert_partition_props(partition_props):
 
 @pytest.mark.parametrize(
     "check_mode", [False, True])
+@pytest.mark.parametrize(
+    "partition_type", ['ssc', 'linux'])
 @mock.patch("plugins.modules.zhmc_partition.AnsibleModule", autospec=True)
-def test_partition_facts(ansible_mod_cls, check_mode, hmc_session):  # noqa: F811, E501
+def test_partition_facts(
+        ansible_mod_cls, partition_type, check_mode, hmc_session):  # noqa: F811, E501
     """
     Test fact gathering on a partition.
     """
 
     hd = hmc_session.hmc_definition
 
+    hmc_auth = dict(userid=hd.hmc_userid, password=hd.hmc_password)
+    if isinstance(hd.hmc_verify_cert, six.string_types):
+        hmc_auth['ca_certs'] = hd.hmc_verify_cert
+    elif isinstance(hd.hmc_verify_cert, bool):
+        hmc_auth['verify'] = hd.hmc_verify_cert
+
     # Determine one of the CPCs in the HMC definition file to test
-    cpc_names = hd.cpcs.keys()
+    cpc_names = list(hd.cpcs.keys())
     assert len(cpc_names) >= 1
     cpc_name = cpc_names[0]
 
-    # Determine an existing partition to test. This also validates that
-    # the CPC defined in the HMC definition file actually exists.
     client = zhmcclient.Client(hmc_session)
     cpc = client.cpcs.find_by_name(cpc_name)
+
+    # Determine a random existing partition of the desired type to test.
     partitions = cpc.partitions.list()
     assert len(partitions) >= 1
-    partition = partitions[0]  # Pick first partition in list
-
-    # Prepare module input parameters
+    typed_partitions = [p for p in partitions
+                        if p.get_property('type') == partition_type]
+    if len(typed_partitions) == 0:
+        pytest.skip("CPC '{c}' has no partitions of type '{t}'".
+                    format(c=cpc_name, t=partition_type))
+    partition = random.choice(typed_partitions)
+
+    # Prepare module input parameters (must be all required + optional)
     params = {
         'hmc_host': hd.hmc_host,
-        'hmc_auth': dict(userid=hd.hmc_userid, password=hd.hmc_password),
+        'hmc_auth': hmc_auth,
         'cpc_name': cpc_name,
         'name': partition.name,
         'state': 'facts',
+        'properties': {},
+        'expand_storage_groups': False,
+        'expand_crypto_adapters': False,
         'log_file': None,
+        '_faked_session': None,
     }
 
     # Prepare mocks for AnsibleModule object

tests/end2end/test_zhmc_user.py

@@ -22,9 +22,11 @@
 import uuid
 import pytest
 import mock
+import random
 import requests.packages.urllib3
 from collections import OrderedDict
 from pprint import pformat
+from ansible.module_utils import six
 import zhmcclient
 from zhmcclient.testutils.hmc_definition_fixtures import hmc_definition, hmc_session  # noqa: F401, E501
 
@@ -207,48 +209,58 @@ def assert_user_props(user_props, expand):
 @pytest.mark.parametrize(
     "expand", [False, True])
 @mock.patch("plugins.modules.zhmc_user.AnsibleModule", autospec=True)
-def test_user_facts(ansible_mod_cls, expand, check_mode, hmc_session):  # noqa: F811, E501
+def test_user_facts(
+        ansible_mod_cls, expand, check_mode, hmc_session):  # noqa: F811, E501
     """
     Test fact gathering on all users of the HMC.
     """
 
     hd = hmc_session.hmc_definition
 
+    hmc_auth = dict(userid=hd.hmc_userid, password=hd.hmc_password)
+    if isinstance(hd.hmc_verify_cert, six.string_types):
+        hmc_auth['ca_certs'] = hd.hmc_verify_cert
+    elif isinstance(hd.hmc_verify_cert, bool):
+        hmc_auth['verify'] = hd.hmc_verify_cert
+
     # Determine an existing user to test.
     client = zhmcclient.Client(hmc_session)
     console = client.consoles.console
+
+    # Pick a random existing user to test
     users = console.users.list()
     assert len(users) >= 1
-
-    for user in users:
-
-        # Prepare module input parameters
-        params = {
-            'hmc_host': hd.hmc_host,
-            'hmc_auth': dict(userid=hd.hmc_userid, password=hd.hmc_password),
-            'name': user.name,
-            'state': 'facts',
-            'expand': expand,
-            'log_file': LOG_FILE,
-        }
-
-        # Prepare mocks for AnsibleModule object
-        mod_obj = mock_ansible_module(ansible_mod_cls, params, check_mode)
-
-        # Exercise the code to be tested
-        with pytest.raises(SystemExit) as exc_info:
-            zhmc_user.main()
-        exit_code = exc_info.value.args[0]
-
-        # Assert module exit code
-        assert exit_code == 0, \
-            "Module unexpectedly failed with this message:\n{0}". \
-            format(get_failure_msg(mod_obj))
-
-        # Assert module output
-        changed, user_props = get_module_output(mod_obj)
-        assert changed is False
-        assert_user_props(user_props, expand)
+    user = random.choice(users)
+
+    # Prepare module input parameters (must be all required + optional)
+    params = {
+        'hmc_host': hd.hmc_host,
+        'hmc_auth': hmc_auth,
+        'name': user.name,
+        'state': 'facts',
+        'properties': {},
+        'expand': expand,
+        'log_file': LOG_FILE,
+        '_faked_session': None,
+    }
+
+    # Prepare mocks for AnsibleModule object
+    mod_obj = mock_ansible_module(ansible_mod_cls, params, check_mode)
+
+    # Exercise the code to be tested
+    with pytest.raises(SystemExit) as exc_info:
+        zhmc_user.main()
+    exit_code = exc_info.value.args[0]
+
+    # Assert module exit code
+    assert exit_code == 0, \
+        "Module unexpectedly failed with this message:\n{0}". \
+        format(get_failure_msg(mod_obj))
+
+    # Assert module output
+    changed, user_props = get_module_output(mod_obj)
+    assert changed is False
+    assert_user_props(user_props, expand)
 
 
 USER_ABSENT_PRESENT_TESTCASES = [
@@ -335,8 +347,16 @@ def test_user_absent_present(
     """
     Test the zhmc_user module with all combinations of absent & present state.
     """
-    expand = False  # Expansion is tested elsewhere
+
     hd = hmc_session.hmc_definition
+
+    hmc_auth = dict(userid=hd.hmc_userid, password=hd.hmc_password)
+    if isinstance(hd.hmc_verify_cert, six.string_types):
+        hmc_auth['ca_certs'] = hd.hmc_verify_cert
+    elif isinstance(hd.hmc_verify_cert, bool):
+        hmc_auth['verify'] = hd.hmc_verify_cert
+
+    expand = False  # Expansion is tested elsewhere
     client = zhmcclient.Client(hmc_session)
     console = client.consoles.console
 
@@ -368,16 +388,20 @@ def test_user_absent_present(
 
     try:
 
+        # Prepare module input parameters (must be all required + optional)
         params = {
             'hmc_host': hd.hmc_host,
-            'hmc_auth': dict(userid=hd.hmc_userid, password=hd.hmc_password),
+            'hmc_auth': hmc_auth,
             'name': user_name,
             'state': input_state,
             'expand': expand,
             'log_file': LOG_FILE,
+            '_faked_session': None,
         }
         if input_props is not None:
             params['properties'] = input_props
+        else:
+            params['properties'] = {}
 
         mod_obj = mock_ansible_module(ansible_mod_cls, params, check_mode)
 

tests/end2end/utils.py

@@ -23,9 +23,13 @@
 
 def mock_ansible_module(ansible_mod_cls, params, check_mode):
     """
-    Prepare mocks for AnsibleModule object.
+    Prepare the mocked AnsibleModule object for the end2end test.
+
+    Note: Since this is a mocked object, the argument_spec defined in the
+    module is not applied, and the params must be the defaulted set of
+    all parameters in the module's argument_spec.
     """
-    mod_obj = ansible_mod_cls.return_value
+    mod_obj = ansible_mod_cls.return_value  # the mocked object
     mod_obj.params = params
     mod_obj.check_mode = check_mode
     mod_obj.fail_json.configure_mock(side_effect=SystemExit(1))