Merge pull request wazuh#24712 from wazuh/merge-4.10.0-into-master

Merge 4.10.0 into master
thereisnosun · Jul 22, 2024 · d039550 · d039550
2 parents 3b44b3b + 0711d0e
commit d039550
Show file tree

Hide file tree

Showing 200 changed files with 3,760 additions and 675 deletions.
diff --git a/.github/workflows/macos-build-tests.yml b/.github/workflows/macos-build-tests.yml
@@ -4,32 +4,24 @@ on:
   pull_request:
     paths:
       - "src/**"
+      - ".github/workflows/macos-build-tests.yml"
 
 jobs:
-  build-bigsur:
-    runs-on: macos-11
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v3
-      - name: Build wazuh agent for macOS 11
-        run: |
-          make deps -C src TARGET=agent -j2
-          make -C src TARGET=agent -j2
-  build-monterey:
-    runs-on: macos-12
+  build-ventura:
+    runs-on: macos-13
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v3
-      - name: Build wazuh agent for macOS 12
+      - name: Build wazuh agent for macOS 13
         run: |
-          make deps -C src TARGET=agent -j2
-          make -C src TARGET=agent -j2
-  build-ventura:
-    runs-on: macos-13
+          make deps -C src TARGET=agent -j4
+          make -C src TARGET=agent -j4
+  build-sonoma:
+    runs-on: macos-14
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v3
-      - name: Build wazuh agent for macOS 13
+      - name: Build wazuh agent for macOS 14
         run: |
-          make deps -C src TARGET=agent -j2
-          make -C src TARGET=agent -j2
+          make deps -C src TARGET=agent -j3
+          make -C src TARGET=agent -j3
diff --git a/.github/workflows/macos-syscollector-tests.yml b/.github/workflows/macos-syscollector-tests.yml
@@ -11,25 +11,25 @@ on:
 
 jobs:
   build:
-    runs-on: macos-11
+    runs-on: macos-13
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v3
       # Build wazuh agent for macOS.
       - name: Build wazuh agent for macOS
         run: |
           rm -rf src/VERSION
-          make deps -C src TARGET=agent -j2
-          make -C src build_syscollector TARGET=agent -j2
+          make deps -C src TARGET=agent -j4
+          make -C src build_syscollector TARGET=agent -j4
       - name: Install dependencies
         run: |
           brew install wget
           pip3 install -r src/data_provider/qa/requirements.txt
       - name: Install macports package manager
         run: |
-          wget https://github.com/macports/macports-base/releases/download/v2.8.1/MacPorts-2.8.1-11-BigSur.pkg
-          sudo installer -pkg MacPorts-2.8.1-11-BigSur.pkg -target /
-          rm -rf MacPorts-2.8.1-11-BigSur.pkg
+          wget https://github.com/macports/macports-base/releases/download/v2.8.1/MacPorts-2.8.1-13-Ventura.pkg
+          sudo installer -pkg MacPorts-2.8.1-13-Ventura.pkg -target /
+          rm -rf MacPorts-2.8.1-13-Ventura.pkg
       - name: Install port
         run: |
           sudo /opt/local/bin/port -b install nano

diff --git a/.github/workflows/macos-unit-tests.yml b/.github/workflows/macos-unit-tests.yml
@@ -4,43 +4,22 @@ on:
   pull_request:
     paths:
       - "src/**"
+      - ".github/workflows/macos-unit-tests.yml"
 
 jobs:
-  build-bigsur:
-    runs-on: macos-11
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v3
-      - name: Install cmocka and lcov
-        run: |
-          brew install cmocka lcov
-      - name: Build wazuh agent for macOS 11 with tests flags
-        run: |
-          make deps -C src TARGET=agent -j3
-          LIBRARY_PATH=/usr/local/lib make -C src TARGET=agent -j3 DEBUG=1 TEST=1
-      - name: Run wazuh unit tests for macOS 11
-        run: |
-          cd src/data_provider/build
-          ctest -V
-          cd ../../shared_modules/dbsync/build
-          ctest -V
-          cd ../../rsync/build
-          ctest -V
-          cd ../../../wazuh_modules/syscollector/build
-          ctest -V
-  build-monterey:
-    runs-on: macos-12
+  build-ventura:
+    runs-on: macos-13
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v3
       - name: Install cmocka and lcov
         run: |
           brew install cmocka lcov
-      - name: Build wazuh agent for macOS 12 with tests flags
+      - name: Build wazuh agent for macOS 13 with tests flags
         run: |
-          make deps -C src TARGET=agent -j3
-          LIBRARY_PATH=/usr/local/lib make -C src TARGET=agent -j3 DEBUG=1 TEST=1
-      - name: Run wazuh unit tests for macOS 12
+          make deps -C src TARGET=agent -j4
+          LIBRARY_PATH=/usr/local/lib make -C src TARGET=agent -j4 DEBUG=1 TEST=1
+      - name: Run wazuh unit tests for macOS 13
         run: |
           cd src/data_provider/build
           ctest -V
@@ -50,19 +29,19 @@ jobs:
           ctest -V
           cd ../../../wazuh_modules/syscollector/build
           ctest -V
-  build-ventura:
-    runs-on: macos-13
+  build-sonoma:
+    runs-on: macos-14
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v3
       - name: Install cmocka and lcov
         run: |
           brew install cmocka lcov
-      - name: Build wazuh agent for macOS 13 with tests flags
+      - name: Build wazuh agent for macOS 14 with tests flags
         run: |
           make deps -C src TARGET=agent -j3
-          LIBRARY_PATH=/usr/local/lib make -C src TARGET=agent -j3 DEBUG=1 TEST=1
-      - name: Run wazuh unit tests for macOS 13
+          C_INCLUDE_PATH=$C_INCLUDE_PATH:/opt/homebrew/include LIBRARY_PATH=/opt/homebrew/lib make -C src TARGET=agent -j3 DEBUG=1 TEST=1
+      - name: Run wazuh unit tests for macOS 14
         run: |
           cd src/data_provider/build
           ctest -V

diff --git a/.github/workflows/packages-filebeat.yml b/.github/workflows/packages-filebeat.yml
@@ -67,7 +67,6 @@ jobs:
           cp -r _meta alerts archives module.yml wazuh/
           sudo chown -R root:root wazuh
           sudo chmod 755 wazuh
-          sudo chmod 755 wazuh/_meta
           sudo chmod 755 wazuh/alerts
           sudo chmod 755 wazuh/alerts/config
           sudo chmod 755 wazuh/alerts/ingest
@@ -96,9 +95,9 @@ jobs:
           sudo tar -xzvf ${{ env.FILEBEAT_TAR_NAME }}
           for file in ./wazuh/*; do
             if [ -d "$file" ]; then
-              if [ "$(stat -L -c "%a %G %U" "$file")" != "$DIR_PERMS" ]; then echo "Wrong permissions for $file. Expected: $DIR_PERMS. Currently: "$(stat -L -c "%a %G %U" "$file")"."; exit 1; fi
+              if [ "$(stat -L -c "%a %G %U" "$file")" != "$DIR_PERMS" ] && [[ "$file" != *"_meta" ]]; then echo "Wrong permissions for $file. Expected: $DIR_PERMS. Currently: $(stat -L -c "%a %G %U" "$file")."; exit 1; fi
             elif [ -f "$file" ]; then
-              if [ "$(stat -L -c "%a %G %U" "$file")" != "$FILE_PERMS" ]; then echo "Wrong permissions for $file. Expected: $FILE_PERMS. Currently: "$(stat -L -c "%a %G %U" "$file")"."; exit 1; fi
+              if [ "$(stat -L -c "%a %G %U" "$file")" != "$FILE_PERMS" ]; then echo "Wrong permissions for $file. Expected: $FILE_PERMS. Currently: $(stat -L -c "%a %G %U" "$file")."; exit 1; fi
             fi
           done
           if [ $(diff -r wazuh wazuh2) ]; then exit 1; fi

diff --git a/.github/workflows/packages-retag-images.yml b/.github/workflows/packages-retag-images.yml
@@ -63,7 +63,7 @@ jobs:
 
       - name: Run retag script
         run: |
-          if [ "${{ inputs.old_version }}" != "none" ] && [ "${{ inputs.new_version }}" != "none" ]; then
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
             new_version=${{ inputs.new_version }}
             old_version=${{ inputs.old_version }}
           else

diff --git a/.github/workflows/wazuh-keystore-tests.yml b/.github/workflows/wazuh-keystore-tests.yml
@@ -0,0 +1,37 @@
+name: Wazuh Keystore
+
+on:
+  workflow_dispatch:
+  pull_request:
+  # Pull request events
+    types: [synchronize, opened, reopened, ready_for_review]
+  # Path filtering
+    paths:
+      - ".github/workflows/wazuh-keystore-tests.yml"
+      - "src/shared_modules/keystore/**"
+      - "src/shared_modules/utils/**"
+
+jobs:
+  wazuh-keystore-qa:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Compile keystore
+        run: |
+          make deps TARGET=server -j2 -C src/
+          make libwazuhext.so TARGET=server -j2 -C src/
+          mkdir -p src/shared_modules/keystore/build
+          cmake -S src/shared_modules/keystore -B  src/shared_modules/keystore/build -DSRC_FOLDER=$(pwd)/src
+          cmake --build src/shared_modules/keystore/build -j2
+
+      - name: Install dependencies
+        run: |
+          pip install -r src/shared_modules/keystore/qa/requirements.txt
+
+      - name: Run tests
+        run: |
+          python -m pytest -vv src/shared_modules/keystore/qa/ --log-cli-level=DEBUG
diff --git a/.gitignore b/.gitignore
@@ -70,6 +70,7 @@ src/wazuh-reportd
 src/wazuh-syscheckd
 src/wazuh-integratord
 src/wazuh-keystore
+src/wazuh-keystore-testtool
 
 # Active responses
 src/firewall-drop

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,7 @@ All notable changes to this project will be documented in this file.
 
 ## [v5.0.0]
 
+## [v4.10.0]
 
 ## [v4.9.0]
 
@@ -11,21 +12,36 @@ All notable changes to this project will be documented in this file.
 #### Added
 
 - The manager now supports alert forwarding to Fluentd. ([#17306](https://github.com/wazuh/wazuh/pull/17306))
+- Added missing functionality for vulnerability scanner translations. ([#23518](https://github.com/wazuh/wazuh/issues/23518))
+- Improved performance for vulnerability scanner translations. ([#23722](https://github.com/wazuh/wazuh/pull/23722))
+- Enhanced vulnerability scanner logging to be more expressive. ([#24536](https://github.com/wazuh/wazuh/pull/24536))
+- The manager now supports alert forwarding to Fluentd. ([#17306](https://github.com/wazuh/wazuh/pull/17306))
+- Added the HAProxy helper to manage load balancer configuration and automatically balance agents. ([#23513](https://github.com/wazuh/wazuh/pull/23513))
+- Added helper to manage HAProxy configuration and automatically balance agents. ([#23513](https://github.com/wazuh/wazuh/pull/23513))
+- Added a validation to avoid killing processes from external services. ([#23222](https://github.com/wazuh/wazuh/pull/23222))
+- Enabled ceritificates validation in the requests to the HAProxy helper using the default CA bundle. ([#23996](https://github.com/wazuh/wazuh/pull/23996))
 
 #### Fixed
 
 - Fixed compilation issue for local installation. ([#20505](https://github.com/wazuh/wazuh/pull/20505))
+- Fixed malformed JSON error in wazuh-analysisd. ([#16666](https://github.com/wazuh/wazuh/pull/16666))
+- Fixed a warning when uninstalling the Wazuh manager if the VD feed is missing. ([#24375](https://github.com/wazuh/wazuh/pull/24375))
+- Ensured vulnerability detection scanner log messages end with a period. ([#24393](https://github.com/wazuh/wazuh/pull/24393))
 
 #### Changed
 
 - Changed error messages about `recv()` messages from wazuh-db to debug logs. ([#20285](https://github.com/wazuh/wazuh/pull/20285))
+- Sanitized the `integrations` directory code. ([#21195](https://github.com/wazuh/wazuh/pull/21195))
+- Modified multiple cluster commands to be asynchronous. ([#22640](https://github.com/wazuh/wazuh/pull/22640))
 
 ### Agent
 
 #### Added
 
 - Added debug logging in FIM to detect invalid report change registry values. Thanks to Zafer Balkan (@zbalkan). ([#21690](https://github.com/wazuh/wazuh/pull/21690))
 - Added Amazon Linux 1 and 2023 support for the installation script. ([#21287](https://github.com/wazuh/wazuh/pull/21287))
+- Added Journald support in Logcollector. ([#23137](https://github.com/wazuh/wazuh/pull/23137))
+- Added support for Amazon Security Hub via AWS SQS. ([#23203](https://github.com/wazuh/wazuh/pull/23203))
 
 #### Fixed
 
@@ -37,10 +53,56 @@ All notable changes to this project will be documented in this file.
 - Fixed a crash in the agent's Rootcheck component when using `<ignore>`. ([#22588](https://github.com/wazuh/wazuh/pull/22588))
 - Fixed command wodle to support UTF-8 characters on windows agent. ([#19146](https://github.com/wazuh/wazuh/pull/19146))
 - Fixed Windows agent to delete wazuh-agent.state file when stopped. ([#20425](https://github.com/wazuh/wazuh/pull/20425))
+- Fixed Windows Agent 4.8.0 permission errors on Windows 11 after upgrade. ([#20727](https://github.com/wazuh/wazuh/pull/20727))
+- Fixed Syscollector not checking if there's a scan in progress before starting a new one. ([#22440](https://github.com/wazuh/wazuh/pull/22440))
+- Fixed alerts are created when syscheck diff DB is full. ([#16487](https://github.com/wazuh/wazuh/pull/16487))
+- Fixed Wazuh deb uninstallation to remove non-config files. ([#2195](https://github.com/wazuh/wazuh/pull/2195))
+- Fixed improper Windows agent ACL on non-default installation directory. ([#23273](https://github.com/wazuh/wazuh/pull/23273))
+- Fixed socket configuration of an agent is displayed. ([#17664](https://github.com/wazuh/wazuh/pull/17664))
+- Fixed wazuh-modulesd printing child process not found error. ([#18494](https://github.com/wazuh/wazuh/pull/18494))
+- Fixed issue with an agent starting automatically without reason. ([#23848](https://github.com/wazuh/wazuh/pull/23848))
+- Fixed GET /syscheck to properly report size for files larger than 2GB. ([#17415](https://github.com/wazuh/wazuh/pull/17415))
+- Fixed error in packages generation centos 7. ([#24412](https://github.com/wazuh/wazuh/pull/24412))
+- Fixed Wazuh deb uninstallation to remove non-config files from the installation directory. ([#2195](https://github.com/wazuh/wazuh/issues/2195))
+- Fixed Azure auditLogs/signIns status parsing (thanks to @Jmnis for the contribution). ([#22392](https://github.com/wazuh/wazuh/pull/22392))
+- Fixed how the S3 object keys with special characters are handled in the Custom Logs Buckets integration. ([#22621](https://github.com/wazuh/wazuh/pull/22621))
 
 #### Changed
 
 - The directory /boot has been removed from the default FIM settings for AIX. ([#19753](https://github.com/wazuh/wazuh/pull/19753))
+- Refactored and modularized the Azure integration code. ([#20624](https://github.com/wazuh/wazuh/pull/20624))
+- Improved logging of errors in Azure and AWS modules. ([#16314](https://github.com/wazuh/wazuh/issues/16314))
+
+#### Removed
+- Dropped support for Python 3.7 in cloud integrations. ([#22583](https://github.com/wazuh/wazuh/pull/22583))
+
+### RESTful API
+
+#### Added
+- Added support in the Wazuh API to parse `journald` configurations from the `ossec.conf` file. ([#23094](https://github.com/wazuh/wazuh/pull/23094))
+- Added user-agent to the CTI service request. ([#24360](https://github.com/wazuh/wazuh/pull/24360))
+
+#### Changed
+
+- Merged group files endpoints into one (`GET /groups/{group_id}/files/{filename}`) that uses the `raw` parameter to receive plain text data. ([#21653](https://github.com/wazuh/wazuh/pull/21653))
+- Removed the hardcoded fields returned by the `GET /agents/outdated` endpoint and added the select parameter to the specification. ([#22388](https://github.com/wazuh/wazuh/pull/22388))
+- Updated the regex used to validate CDB lists. ([#22423](https://github.com/wazuh/wazuh/pull/22423))
+- Changed the default value for empty fields in the `GET /agents/stats/distinct` endpoint response. ([#22413](https://github.com/wazuh/wazuh/pull/22413))
+- Changed the Wazuh API endpoint responses when receiving the `Expect` header. ([#22380](https://github.com/wazuh/wazuh/pull/22380))
+- Enhanced Authorization header values decoding errors to avoid showing the stack trace and fail gracefully. ([#22745](https://github.com/wazuh/wazuh/pull/22745))
+- Updated the format of the fields that can be N/A in the API specification. ([#22908](https://github.com/wazuh/wazuh/pull/22908))
+- Updated the WAZUH API specification to conform with the current endpoint requests and responses. ([#22954](https://github.com/wazuh/wazuh/pull/22954))
+- Replaced the used aiohttp server with uvicorn. ([#23199](https://github.com/wazuh/wazuh/pull/23199))
+    - Changed the `PUT /groups/{group_id}/configuration` endpoint response error code when uploading an empty file.
+    - Changed the `GET, PUT and DELETE /lists/files/{filename}` endpoints response status code when an invalid file is used.
+    - Changed the `PUT /manager/configuration` endpoint response status code when uploading a file with invalid content-type.
+
+#### Fixed
+- Improved XML validation to match the Wazuh internal XML validator. ([#20507](https://github.com/wazuh/wazuh/pull/20507))
+- Fixed bug in `GET /groups`. ([#22428](https://github.com/wazuh/wazuh/pull/22428))
+
+#### Removed
+- Removed the `cache` configuration option from the Wazuh API. ([#22416](https://github.com/wazuh/wazuh/pull/22416))
 
 ### Ruleset
 
@@ -57,13 +119,48 @@ All notable changes to this project will be documented in this file.
 #### Changed
 
 - Upgraded external OpenSSL library dependency version to 3.0. ([#20778](https://github.com/wazuh/wazuh/pull/20778))
+- Migrated QA framework. ([#17427](https://github.com/wazuh/wazuh/issues/17427))
+- Improved WPKs. ([#21152](https://github.com/wazuh/wazuh/issues/21152))
+- Migrated and adapted Wazuh subsystem repositories as part of Wazuh packages redesign. ([#23508](https://github.com/wazuh/wazuh/pull/23508))
+- Upgraded external connexion library dependency version to 3.0.5 and its related interdependencies. ([#22680](https://github.com/wazuh/wazuh/pull/22680))
 
 #### Fixed
 
 - Fixed a buffer overflow hazard in HMAC internal library. ([#19794](https://github.com/wazuh/wazuh/pull/19794))
 
 ## [v4.8.1]
 
+### Manager
+
+#### Added
+
+- Added dedicated RSA keys for keystore encryption. ([#24357](https://github.com/wazuh/wazuh/pull/24357))
+
+#### Fixed
+
+- Fixed bug in `upgrade_agent` CLI where it would sometimes raise an unhandled exception. ([#24341](https://github.com/wazuh/wazuh/pull/24341))
+- Changed keystore cipher algorithm to remove reuse of sslmanager.cert and sslmanager.key. ([#24509](https://github.com/wazuh/wazuh/pull/24509))
+
+### Agent
+
+#### Fixed
+
+- Fixed incorrect macOS agent name retrieval. ([#23989](https://github.com/wazuh/wazuh/pull/23989))
+
+### RESTful API
+
+#### Changed
+
+- Changed `GET /manager/version/check` endpoint response to always show the `uuid` field. ([#24173](https://github.com/wazuh/wazuh/pull/24173))
+
+### Other
+
+#### Changed
+
+- Upgraded external Jinja2 library dependency version to 3.1.4. ([#24108](https://github.com/wazuh/wazuh/pull/24108))
+- Upgraded external requests library dependency version to 2.32.2. ([#23925](https://github.com/wazuh/wazuh/pull/23925))
+
+
 ## [v4.8.0]
 
 ### Manager
@@ -127,7 +224,6 @@ All notable changes to this project will be documented in this file.
 - Introduced an `auto` option for the ssl_protocol setting in the API configuration. This enables automatic negotiation of the TLS certificate to be used. ([#20420](https://github.com/wazuh/wazuh/pull/20420))
 - Added API indexer protection to allow uploading new configuration files if the `<indexer>` section is not modified. ([#22727](https://github.com/wazuh/wazuh/pull/22727))
 
-
 #### Fixed
 
 - Fixed a warning from SQLAlchemy involving detached Roles instances in RBAC. ([#20527](https://github.com/wazuh/wazuh/pull/20527))

diff --git a/api/api/alogging.py b/api/api/alogging.py
@@ -259,9 +259,10 @@ def custom_logging(user, remote, method, path, query,
 
     if path == '/events' and logger.level >= 20:
         # If log level is info simplify the messages for the /events requests.
-        events = body.get('events', [])
-        body = {'events': len(events)}
-        json_info['body'] = body
+        if isinstance(body, dict):
+            events = body.get('events', [])
+            body = {'events': len(events)}
+            json_info['body'] = body
 
     log_info += f'with parameters {json.dumps(query)} and body '\
                 f'{json.dumps(body)} done in {elapsed_time:.3f}s: {status}'