Fix the iptables issues (#16491)

* Fix the iptables issues

The fileset of the iptables test didn't skip the @timestamp field, so
I have included the iptables.log data to the list of dataset to remove.

I also changed the assertion slighly to be relevant on the order of the
expectation and instead of dropping the JSON output in the log we will
display the difference as a diff.

Fix issues with #15980

filebeat/tests/system/test_modules.py

@@ -9,6 +9,7 @@
 import json
 import logging
 from parameterized import parameterized
+from deepdiff import DeepDiff
 
 
 def load_fileset_test_cases():
@@ -188,20 +189,18 @@ def _test_expected_events(self, test_file, objects):
         assert len(expected) == len(objects), "expected {} events to compare but got {}".format(
             len(expected), len(objects))
 
-        for ev in expected:
-            found = False
-            for obj in objects:
+        for idx in range(len(expected)):
+            ev = expected[idx]
+            obj = objects[idx]
 
-                # Flatten objects for easier comparing
-                obj = self.flatten_object(obj, {}, "")
-                clean_keys(obj)
+            # Flatten objects for easier comparing
+            obj = self.flatten_object(obj, {}, "")
+            clean_keys(obj)
+            clean_keys(ev)
 
-                if ev == obj:
-                    found = True
-                    break
+            d = DeepDiff(ev, obj, ignore_order=True)
 
-            assert found, "The following expected object was not found:\n {}\nSearched in: \n{}".format(
-                pretty_json(ev), pretty_json(objects))
+            assert len(d) == 0, "The following expected object doesn't match:\n Diff: \n {}".format(d, obj)
 
 
 def clean_keys(obj):
@@ -216,7 +215,7 @@ def clean_keys(obj):
         delete_key(obj, key)
 
     # Remove timestamp for comparison where timestamp is not part of the log line
-    if obj["event.dataset"] in ["icinga.startup", "redis.log", "haproxy.log", "system.auth", "system.syslog"]:
+    if obj["event.dataset"] in ["icinga.startup", "redis.log", "haproxy.log", "system.auth", "system.syslog", 'iptables.log']:
         delete_key(obj, "@timestamp")
 
 

libbeat/tests/system/requirements.txt

@@ -30,3 +30,4 @@ websocket-client==0.47.0
 parameterized==0.6.1
 jsondiff==1.1.2
 semver==2.8.1
+deepdiff==3.3.0