Add sophos xg integration (#479)

* Add sophos xg integration

* Add tcp and udp system tests

* Sync with latest changes from beats

* Apply suggestions

packages/sophos/_dev/build/docs/README.md

@@ -0,0 +1,33 @@
+# Sophos Integration
+
+The Sophos integration collects and parses logs from Sophos Products.
+
+Currently it accepts logs in syslog format or from a file for the following devices:
+
+- `utm` dataset: supports Astaro Security Gateway logs.
+- `xg` dataset: supports Sophos XG SFOS logs.
+
+To configure a remote syslog destination, please reference the [SophosXG/SFOS Documentation](https://community.sophos.com/kb/en-us/123184).
+
+The syslog format choosen should be `Default`.
+
+## Compatibility
+
+This module has been tested against SFOS version 17.5.x and 18.0.x.
+Versions above this are expected to work but have not been tested.
+
+## Logs
+
+### Utm log
+
+The `utm` dataset collects Astaro Security Gateway logs.
+
+{{fields "utm"}}
+
+### XG log
+
+This is the Sophos `xg` dataset.
+
+{{event "xg"}}
+
+{{fields "xg"}}

packages/sophos/_dev/deploy/docker/docker-compose.yml

@@ -1,6 +1,6 @@
 version: '2.3'
 services:
-  sophos-utm-logfile:
+  sophos-logfile:
     image: alpine
     volumes:
       - ./sample_logs:/sample_logs:ro
@@ -18,3 +18,15 @@ services:
       - ./sample_logs:/sample_logs:ro
     entrypoint: /bin/bash
     command: -c "/stream log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9549 -p=tcp /sample_logs/sophos-utm-*.log"
+  sophos-xg-udp:
+    image: akroh/stream:v0.2.0
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    entrypoint: /bin/bash
+    command: -c "/stream log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9549 -p=udp /sample_logs/sophos-xg*.log"
+  sophos-xg-tcp:
+    image: akroh/stream:v0.2.0
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    entrypoint: /bin/bash
+    command: -c "/stream log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9549 -p=tcp /sample_logs/sophos-xg*.log"

packages/sophos/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.2.0"
+  changes:
+    - description: Add XG data stream
+      type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      link: https://github.com/elastic/package-storage/pull/400
 - version: "0.1.0"
   changes:
     - description: initial release

packages/sophos/data_stream/utm/_dev/test/system/test-logfile-config.yml

@@ -1,4 +1,4 @@
-service: sophos-utm-logfile
+service: sophos-logfile
 input: logfile
 data_stream:
   vars:

packages/sophos/data_stream/xg/_dev/test/pipeline/test-sophos-xg.log-config.json

@@ -0,0 +1,5 @@
+{
+    "dynamic_fields": {
+        "event.ingested": ".*"
+    }
+}