Merge branch 'enhancement/vd-e2e-tests' into 5074-replace-macos-packa…

…ges-by-npm

CHANGELOG.md

@@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file.
 ### Added
 
 - Add support for the installation/uninstallation of npm packages ([#5092](https://github.com/wazuh/wazuh-qa/pull/5092)) \- (Tests)
+- Add alert.json file to Vulnerability Detector E2E test report ([#5147](https://github.com/wazuh/wazuh-qa/pull/5147)) \- (Framework)
 - Add documentation about markers for system tests ([#5080](https://github.com/wazuh/wazuh-qa/pull/5080)) \- (Documentation)
 - Add AWS Custom Buckets Integration tests ([#4675](https://github.com/wazuh/wazuh-qa/pull/4675)) \- (Framework + Tests)
 - Add Vulnerability Detector end to end tests ([#4878](https://github.com/wazuh/wazuh-qa/pull/4878)) \- (Framework + Tests)
@@ -50,6 +51,8 @@ All notable changes to this project will be documented in this file.
 
 ### Fixed
 
+- Fix Vulnerability Detector E2E tests by adding description to all tests ([#5151](https://github.com/wazuh/wazuh-qa/pull/5151)) \- (Tests)
+- Fix parser for non package vulnerabilities ([#5146](https://github.com/wazuh/wazuh-qa/pull/5146)) \- (Framework)
 - Fix remote_operations_handler functions to Vulnerability Detector E2E tests ([#5155](https://github.com/wazuh/wazuh-qa/pull/5155)) \- (Framework)
 - Fix test_shutdown_message system test ([#5087](https://github.com/wazuh/wazuh-qa/pull/5087)) \- (Tests)
 - Include timeout to test_authd system tests ([#5083](https://github.com/wazuh/wazuh-qa/pull/5083)) \- (Tests)

deps/wazuh_testing/wazuh_testing/end_to_end/logs.py

@@ -52,10 +52,29 @@ def get_hosts_logs(host_manager: HostManager, host_group: str = 'all') -> Dict[s
     - host_manager (HostManager): An instance of the HostManager class for managing remote hosts.
     - host_group (str, optional): The name of the host group where the files will be truncated.
       Default is 'all'.
+
+    Returns:
+    - host_logs (Dict[str, str]): Dictionary containing the logs from the ossec.log file of each host
     """
     host_logs = {}
     for host in host_manager.get_group_hosts(host_group):
         host_os_name = host_manager.get_host_variables(host)['os_name']
         host_logs[host] = host_manager.get_file_content(host, logs_filepath_os[host_os_name])
 
     return host_logs
+
+def get_hosts_alerts(host_manager: HostManager) -> Dict[str, str]:
+    """
+    Get the alerts in the alert.json file from the specified host group.
+
+    Parameters:
+    - host_manager (HostManager): An instance of the HostManager class for managing remote hosts.
+
+    Returns:
+    - host_alerts (Dict[str, str]): Dictionary containing the alerts from the alert.json file of each manager
+    """
+    host_alerts = {}
+    for host in host_manager.get_group_hosts("manager"):
+        host_alerts[host] = host_manager.get_file_content(host, ALERTS_JSON_PATH)
+
+    return host_alerts

deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py

@@ -259,8 +259,8 @@ def check_vuln_state_consistency(vulnerabilities_alerts, vulnerabilities_states)
     if vulnerabilities_states.keys() != vulnerabilities_alerts.keys():
         logging.critical("The number of agents is not the same between alerts and states")
 
-    agents_in_alerts_states = [agent for agent in vulnerabilities_states.keys() \
-            if agent in vulnerabilities_alerts.keys()]
+    agents_in_alerts_states = [agent for agent in vulnerabilities_states.keys()
+                               if agent in vulnerabilities_alerts.keys()]
 
     alerts_not_in_states = []
     states_not_in_alerts = []
@@ -305,14 +305,23 @@ def get_vulnerabilities_from_states(vulnerabilities_states: List) -> List:
         try:
             vulnerability = Vulnerability(
                 cve=state_vulnerability['_source']['vulnerability']['id'],
-                package_name=state_vulnerability['_source']['package']['name'],
-                package_version=state_vulnerability['_source']['package']['version'],
-                type=state_vulnerability['_source']['pacakge']['type'] if 'type' in state_vulnerability['_source']['vulnerability'] else None,
-                architecture=state_vulnerability['_source']['package']['architecture'] if 'architecture' in state_vulnerability['_source']['vulnerability'] else None
+                package_name=(state_vulnerability['_source']['package']['name']
+                              if 'package' in state_vulnerability['_source']
+                              and 'name' in state_vulnerability['_source']['package'] else None),
+                package_version=(state_vulnerability['_source']['package']['version']
+                                 if 'package' in state_vulnerability['_source']
+                                 and 'version' in state_vulnerability['_source']['package'] else None),
+                type=(state_vulnerability['_source']['package']['type']
+                      if 'package' in state_vulnerability['_source']
+                      and 'type' in state_vulnerability['_source']['package'] else None),
+                architecture=(state_vulnerability['_source']['package']['architecture']
+                              if 'package' in state_vulnerability['_source']
+                              and 'architecture' in state_vulnerability['_source']['package'] else None)
             )
             vulnerabilities.append(vulnerability)
         except KeyError:
             logging.error(f"Error parsing vulnerability: {state_vulnerability}")
+            raise KeyError
 
     vulnerabilities = sorted(vulnerabilities, key=lambda x: (x.cve, x.package_name, x.package_version, x.architecture))
 

tests/end_to_end/test_vulnerability_detector/conftest.py

@@ -42,7 +42,7 @@ def test_example(host_manager):
 
 from wazuh_testing.tools.system import HostManager
 from wazuh_testing.end_to_end.remote_operations_handler import launch_parallel_operations
-from wazuh_testing.end_to_end.logs import get_hosts_logs
+from wazuh_testing.end_to_end.logs import get_hosts_logs, get_hosts_alerts
 
 
 STYLE_PATH = os.path.join(os.path.dirname(__file__), '../../../deps/wazuh_testing/wazuh_testing/reporting/style.css')
@@ -61,17 +61,25 @@ def collect_e2e_environment_data(test_name, host_manager) -> None:
     """
     logging.info("Collecting environment data")
     environment_logs = get_hosts_logs(host_manager)
+    environment_alerts = get_hosts_alerts(host_manager)
 
     current_dir = os.path.dirname(__file__)
     vulnerability_detector_logs_dir = os.path.join(current_dir, "logs")
     tests_evidences_directory = os.path.join(str(vulnerability_detector_logs_dir), str(test_name))
 
     for host in environment_logs.keys():
         logging.info(f"Collecting logs for {host}")
-        host_logs_name_evidence = host + "_ossec.log"
-        evidence_file = os.path.join(tests_evidences_directory, host_logs_name_evidence)
-        with open(evidence_file, 'w') as evidence_file:
-            evidence_file.write(environment_logs[host])
+        host_logs_name_evidence = host + "_ossec.log"        
+        evidence_log_file = os.path.join(tests_evidences_directory, host_logs_name_evidence)
+        with open(evidence_log_file, 'w') as evidence_log_file:
+            evidence_log_file.write(environment_logs[host])
+
+    for host in environment_alerts.keys():
+        logging.info(f"Collecting alerts for {host}")           
+        host_alerts_name_evidence = host + "_alert.json"
+        evidence_alert_file = os.path.join(tests_evidences_directory, host_alerts_name_evidence)
+        with open(evidence_alert_file, 'w') as evidence_alert_file:
+            evidence_alert_file.write(environment_alerts[host])
 
 
 def collect_evidences(test_name, evidences) -> None:

tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py

@@ -21,7 +21,7 @@
         - test_syscollector_first_scan_index: Validates that the Vulnerability Detector detects vulnerabilities within the environment in the first scan in the index.
         - test_syscollector_second_scan: Validates the initiation of the second Syscollector scans across all agents in the environment.
         - tests_syscollector_first_second_scan_consistency_index: Ensure the consistency of the agent's vulnerabilities between the first and second scans in index.
-    - TestScanSyscollectorCases: Validates the Vulnerability Detector's ability to detect new vulnerabilities in the environment.
+    - TestScanSyscollectorCases: Validates the Vulnerability Detector's ability to detect new vulnerabilities in the environment for each of the defined cases.
 
 Issue: https://github.com/wazuh/wazuh-qa/issues/4369
 
@@ -556,7 +556,58 @@ def get_results(self):
 
     @pytest.mark.parametrize('preconditions, body, teardown', complete_list, ids=list_ids)
     def test_vulnerability_detector_scans_cases(self, setup_vulnerability_tests, request, preconditions, body, teardown, setup,
-                                                host_manager, get_results):
+                                                host_manager, get_results):        
+        """
+        description: Validates the Vulnerability Detector's ability to detect new vulnerabilities in the environment for each of the defined cases.
+
+        This test evaluates the effectiveness of the Vulnerability Detector in real-world scenarios, focusing on the installation, removal, 
+        or upgrade of various vulnerable and non-vulnerable packages in the environment. It ensures that all agents generate the expected 
+        vulnerabilities and associated alerts.
+
+        tier: 0
+
+        parameters:
+            - setup_vulnerability_tests:
+                type: fixture
+                brief: Setup the environment to proceed with the testing
+            - request: pytest request object
+            - preconditions:
+                type: fixture
+                brief: The preconditions within the test cases, if any
+            - body:
+                type: fixture
+                brief: The body of the test case, which contains the tasks to be executed
+            - teardown:
+                type: fixture
+                brief: The teardown within the test cases, if any
+            - setup: 
+                type: fixture
+                brief: Test setup results, to check if the hosts are setup correctly
+            - host_manager:
+                type: fixture
+                brief: Get the host manager of the environment            
+            - get_results: fixture to get the results of global class tests
+
+        assertions:
+            - Verify that all the hosts are properly setup.
+            - Verify whether vulnerabilities remain, appear or disappear, and whether alerts appear.
+
+        cases:
+            - install_package
+            - remove_package
+            - upgrade_package_maintain_vulnerability
+            - upgrade_package_maintain_add_vulnerability
+            - upgrade_package_remove_vulnerability
+            - upgrade_package_nonvulnerable_to_nonvulnerable
+            - upgrade_package_nonvulnerable_to_vulnerable
+            - install_package_non_vulnerable
+            - remove_non_vulnerable_packge
+
+        tags:
+            - syscollector
+            - vulnerability_detector
+        """
+
         test_name = request.node.name
 
         setup_results = setup