resolved merge conflicts

.buildkite/pipeline.schedule-daily.yml

@@ -21,7 +21,7 @@ steps:
       env:
         SERVERLESS: "false"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 7.17.23-SNAPSHOT
+        STACK_VERSION: 7.17.24-SNAPSHOT
     depends_on:
       - step: "check"
         allow_failure: false

.github/CODEOWNERS

@@ -119,6 +119,7 @@
 /packages/bitwarden @elastic/security-service-integrations
 /packages/bluecoat @elastic/sec-deployment-and-devices
 /packages/box_events @elastic/security-service-integrations
+/packages/canva @elastic/security-service-integrations
 /packages/carbon_black_cloud @elastic/security-service-integrations
 /packages/carbonblack_edr @elastic/security-service-integrations
 /packages/cassandra @elastic/obs-infraobs-integrations
@@ -172,6 +173,7 @@
 /packages/etcd @elastic/obs-infraobs-integrations
 /packages/f5 @elastic/security-service-integrations
 /packages/f5_bigip @elastic/security-service-integrations
+/packages/falco @elastic/security-service-integrations
 /packages/fim @elastic/sec-linux-platform
 /packages/fireeye @elastic/security-service-integrations
 /packages/fleet_server @elastic/fleet
@@ -317,6 +319,7 @@
 /packages/sophos @elastic/sec-deployment-and-devices
 /packages/sophos_central @elastic/security-service-integrations
 /packages/spring_boot @elastic/obs-infraobs-integrations
+/packages/spycloud @elastic/security-service-integrations
 /packages/sql_input @elastic/obs-infraobs-integrations
 /packages/squid @elastic/sec-deployment-and-devices
 /packages/stan @elastic/obs-infraobs-integrations

packages/amazon_security_lake/_dev/build/docs/README.md

@@ -53,7 +53,6 @@ The Amazon Security Lake integration collects logs from both [Third-party servic
 5. The integration currently only supports collecting logs via AWS S3.
 6. While adding the integration, you have to configure the following details:
    - bucket arn
-   - collect logs via S3 Bucket toggled on
    - role ARN
    - external id
 

packages/amazon_security_lake/changelog.yml

@@ -4,6 +4,11 @@
     - description: Updated to support OCSF v1.1.0. with major pipeline rework and dynamic template support.
       type: enhancement
       link: https://github.com/elastic/integrations/pull/10405
+- version: "1.4.1"
+  changes:
+    - description: "Remove confusing documentation remaining from previous change."
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10799
 - version: "1.4.0"
   changes:
     - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."

packages/amazon_security_lake/docs/README.md

@@ -53,7 +53,6 @@ The Amazon Security Lake integration collects logs from both [Third-party servic
 5. The integration currently only supports collecting logs via AWS S3.
 6. While adding the integration, you have to configure the following details:
    - bucket arn
-   - collect logs via S3 Bucket toggled on
    - role ARN
    - external id
 

packages/azure_network_watcher_vnet/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.2.2"
+  changes:
+    - description: Fix field guards for MAC address processors.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10777
 - version: "0.2.1"
   changes:
     - description: Remove reference to a Kibana version from the README.

packages/azure_network_watcher_vnet/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -106,11 +106,11 @@ processors:
       field: json.macAddress
       tag: uppercase_mac_address
       ignore_missing: true
-      if: ctx.json?.macAddress !='' && ctx.json.macAddress != null
+      if: ctx.json?.macAddress != null && ctx.json.macAddress != ''
       target_field: azure_network_watcher_vnet.log.mac_address
   - gsub:
       field: azure_network_watcher_vnet.log.mac_address
-      if: ctx.azure_network_watcher_vnet?.log?.mac_address != '' && ctx.azure_network_watcher_vnet.log.mac_address!= null
+      if: ctx.azure_network_watcher_vnet?.log?.mac_address != null && ctx.azure_network_watcher_vnet.log.mac_address!= ''
       tag: gsub_mac_address
       pattern: '(..)(?!$)'
       replacement: '$1-'

packages/azure_network_watcher_vnet/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.1.2
 name: azure_network_watcher_vnet
 title: Azure Network Watcher VNet
-version: "0.2.1"
+version: "0.2.2"
 description: Collect logs from Azure Network Watcher VNet with Elastic Agent.
 type: integration
 categories:

packages/canva/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: "git@v8.11.0"

packages/canva/_dev/build/docs/README.md

@@ -0,0 +1,104 @@
+# Canva
+
+[Canva](https://www.canva.com/) is an online graphic design platform used for creating social media graphics, presentations, posters, documents, and other visual content. Canva provides [Audit logs](https://www.canva.dev/docs/audit-logs/) that contain records of user activies in Canva, such as installing a [Canva App](https://www.canva.com/your-apps/), [exporting a design](https://www.canva.com/help/download-or-purchase/) for download, or a user changing their [account settings](https://www.canva.com/help/account-settings/). These logs can be useful for compliance audits, monitoring for unauthorized activity, and other matters that require details about the creation, access, and deletion of data in Canva.
+
+**NOTE**:
+- Audit logs are available for organizations that use Canva Enterprise.
+- Canva starts generating Audit logs when an organization upgrades their account to Canva Enterprise and will start logging events for a brand once it joins the Canva Enterprise account.
+
+The Canva integration can be used in two different modes to collect data:
+- **AWS S3 polling mode** - Canva writes data to S3 and Elastic Agent polls the S3 bucket by listing its contents and reading new files.
+- **AWS S3 SQS mode** - Canva writes data to S3, S3 pushes a new object notification to SQS, Elastic Agent receives the notification from SQS, and then reads the S3 object. Multiple Agents can be used in this mode.
+
+## Data streams
+
+The Canva integration collects Audit logs in the **Audit** data stream.
+
+**Audit** contains the information about the user activies in Canva. The user changing account settings, installing Canva app, managing teams, and groups information can be logged through the Audit logs.
+
+## Requirements
+
+- Elastic Agent must be installed.
+- You can install only one Elastic Agent per host.
+- Elastic Agent is required to stream data from the S3 bucket and ship the data to Elastic, where the events will then be processed via the integration's ingest pipelines.
+
+### Installing and managing an Elastic Agent:
+
+You have a few options for installing and managing an Elastic Agent:
+
+### Install a Fleet-managed Elastic Agent (recommended):
+
+With this approach, you install Elastic Agent and use Fleet in Kibana to define, configure, and manage your agents in a central location. We recommend using Fleet management because it makes the management and upgrade of your agents considerably easier.
+
+### Install Elastic Agent in standalone mode (advanced users):
+
+With this approach, you install Elastic Agent and manually configure the agent locally on the system where it’s installed. You are responsible for managing and upgrading the agents. This approach is reserved for advanced users only.
+
+### Install Elastic Agent in a containerized environment:
+
+You can run Elastic Agent inside a container, either with Fleet Server or standalone. Docker images for all versions of Elastic Agent are available from the Elastic Docker registry, and we provide deployment manifests for running on Kubernetes.
+
+There are some minimum requirements for running Elastic Agent and for more information, refer to the link [here](https://www.elastic.co/guide/en/fleet/current/elastic-agent-installation.html).
+
+## Setup
+
+### To stream data from Canva to the AWS S3 Bucket:
+
+- Follow the instructions [here](https://www.canva.dev/docs/audit-logs/setup/) to forward your Audit log data from Canva to the AWS S3 bucket.
+- Canva adds events to your S3 bucket every minute as a gzipped archive containing JSONL content and requires PutObject permission on the S3 bucket.
+- It store the files in hourly folders, in the format orgId/yyyy/MM/dd/HH.
+
+### To collect data from AWS S3 Bucket, follow the below steps:
+
+- Create an Amazon S3 bucket. Refer to the instructions [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html).
+- The default value of the "Bucket List Prefix" should be empty. However, the user can set the parameter "Bucket List Prefix" according to the requirement.
+
+### To collect data from AWS SQS, follow the below steps:
+
+1. If data forwarding to an AWS S3 Bucket hasn't been configured, then first set up an AWS S3 Bucket as mentioned in the above documentation.
+2. To set up an SQS queue, follow "Step 1: Create an Amazon SQS queue" mentioned in the [Amazon Documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ways-to-add-notification-config-to-bucket.html).
+  - While creating an SQS Queue, please provide the same bucket ARN that has been generated after creating an AWS S3 Bucket.
+3. Set up event notifications for an S3 bucket. Follow the instructions [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/enable-event-notifications.html).
+  - Users have to set the prefix parameter the same as the S3 Bucket List Prefix as created earlier. (for example, `log/` for a log data stream.)
+  - Select the event type as s3:ObjectCreated:*, select the destination type SQS Queue, and select the queue that has been created in Step 2.
+
+**Note**:
+  - Credentials for the above AWS S3 and SQS input types should be configured using the instructions [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html#aws-credentials-config).
+  - Data collection via AWS S3 Bucket and AWS SQS are mutually exclusive in this case.
+
+### Enabling the integration in Elastic:
+
+1. In Kibana go to Management > Integrations
+2. In "Search for integrations" search bar, type Canva
+3. Click on the "Canva" integration from the search results.
+4. Click on the Add Canva Integration button to add the integration.
+5. While adding the integration, if you want to collect logs via AWS S3, then you have to put the following details:
+   - Collect logs via S3 Bucket toggled on
+   - Access Key ID
+   - Secret Access Key
+   - Bucket ARN
+   - Session Token
+
+   or if you want to collect logs via AWS SQS, then you have to put the following details:
+   - Collect logs via S3 Bucket toggled off
+   - Queue URL
+   - Secret Access Key
+   - Access Key ID
+   - Session Token
+
+6. Save the integration.
+
+**NOTE**:
+There are other input combination options available for the AWS S3 and AWS SQS, please check [here](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html).
+
+## Logs Reference
+
+### Audit
+
+This is the `Audit` dataset.
+
+#### Example
+
+{{event "audit"}}
+
+{{fields "audit"}}

packages/canva/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.1.0"
+  changes:
+    - description: Initial release.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10742

packages/canva/data_stream/audit/_dev/deploy/tf/env.yml

@@ -0,0 +1,9 @@
+version: '2.3'
+services:
+  terraform:
+    environment:
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}
+      - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE}
+      - AWS_REGION=${AWS_REGION:-us-east-1}

packages/canva/data_stream/audit/_dev/deploy/tf/files/test-audit.log

@@ -0,0 +1 @@
+{"id":"3849ef51-ca85-4028-bae3-1b8de3ee5738","timestamp":1704070800123,"actor":{"type":"USER","user":{"id":"UXoqDbwwSbQ","display_name":"JaneDoe","email":"jane.doe@example.com"},"team":{"id":"BXeFatjDhdR","display_name":"AcmeCorporation"},"organization":{"id":"OXtgecafZvh"},"details":{"type":"SCIM"}},"target":{"target_type":"USER","user":{"id":"UXoqDbwwSbQ","display_name":"JaneDoe","email":"jane.doe@example.com"},"team":{"id":"BXeFatjDhdR","display_name":"AcmeCorporation"},"organization":{"id":"abc"},"owner":{"type":"USER","user":{"id":"UXoqDbwwSbQ","display_name":"JaneDoe","email":"jane.doe@example.com"},"team":{"id":"BXeFatjDhdR","display_name":"AcmeCorporation"},"organization":{"id":"abc"}},"resource_type":"DESIGN","id":"abc123","name":"abc"},"action":{"type":"REMOVE_TEAM_FROM_ORGANIZATION","display_name":"Marketing","first_name":"string","last_name":"string","email":"alex.doe@example.com","email_verified":true,"phone_number":"string","country_code":"string","locale":"string","managing_entity":{"type":"TEAM","team":{"id":"BXeFatjDhdR","display_name":"AcmeCorporation"},"organization":{"id":"Abc11233"}},"saml_accounts":[{"idp_issuer":"string","name_id":"string"}],"oauth_accounts":[{"platform":"string","external_user_id":"string"}],"totp_mfa_enabled":true,"sms_mfa_enabled":true,"reason":{"type":"SAML_JIT_PROVISIONING"},"changed_fields":"ADDRESS","login_type":"PASSWORD","oauth_platform":"APPLE","user_scope":"CURRENT_USER","session_scope":"CURRENT_SESSION","app_id":"string","app_version":"string","app_name":"string","permissions":["DESIGN_CONTENT_READ"],"old_permissions":["DESIGN_CONTENT_READ"],"new_permissions":["DESIGN_CONTENT_READ"],"output_type":"PDF","create_type":"CREATE","title":"Myawesomedesign","original_design_id":"DAGKs37VOUl","design_type":"Presentation(16:9)","view_type":"VIEW_IN_EDITOR","changes":[{"type":"CREATE_DESIGN_ACCESS_INVITE","token_prefix":"ZMrbBHL2","recipient":"ash.doe@example.com","access":{"read":true,"write":true}},{"type":"REDEEM_DESIGN_ACCESS_INVITE","token_prefix":"ZMrbBHL2","recipient":"ash.doe@example.com","user":{"id":"UXoqDbwwSbQ","display_name":"JaneDoe","email":"jane.doe@example.com"}},{"type":"DELETE_DESIGN_ACCESS_INVITE","token_prefix":"ZMrbBHL2","recipient":"ash.doe@example.com"},{"type":"UPDATE_DESIGN_OWNER","old_owner":{"id":"UXoqDbwwSbQ","display_name":"JaneDoe","email":"jane.doe@example.com"},"new_owner":{"id":"UXqwwoQDSbb","display_name":"AshDoe","email":"ash.doe@example.com"}},{"type":"CREATE_DESIGN_ACCESS_RESTRICTION"},{"type":"GRANT_USER_DESIGN_ACCESS","access":{"read":true,"write":true},"user":{"id":"UXoqDbwwSbQ","display_name":"JaneDoe","email":"jane.doe@example.com"}},{"type":"REVOKE_USER_DESIGN_ACCESS","access":{"read":true,"write":true},"user":{"id":"UXoqDbwwSbQ","display_name":"JaneDoe","email":"jane.doe@example.com"}},{"type":"UPDATE_USER_DESIGN_ACCESS","old_access":{"read":true,"write":false},"new_access":{"read":true,"write":true},"user":{"id":"UXoqDbwwSbQ","display_name":"JaneDoe","email":"jane.doe@example.com"}},{"type":"GRANT_GROUP_DESIGN_ACCESS","access":{"read":true,"write":true},"group":"GADkBZ48E04"},{"type":"REVOKE_GROUP_DESIGN_ACCESS","access":{"read":true,"write":true},"group":"GADkBZ48E04"},{"type":"UPDATE_GROUP_DESIGN_ACCESS","old_access":{"read":true,"write":false},"new_access":{"read":true,"write":true},"group":"GADkBZ48E04"},{"type":"GRANT_TEAM_DESIGN_ACCESS","access":{"read":true,"write":true},"team":{"id":"BXeFatjDhdR","display_name":"AcmeCorporation"}},{"type":"REVOKE_TEAM_DESIGN_ACCESS","access":{"read":true,"write":true},"team":{"id":"BXeFatjDhdR","display_name":"AcmeCorporation"}},{"type":"UPDATE_TEAM_DESIGN_ACCESS","old_access":{"read":true,"write":false},"new_access":{"read":true,"write":true},"team":{"id":"BXeFatjDhdR","display_name":"AcmeCorporation"}},{"type":"GRANT_ORGANIZATION_DESIGN_ACCESS","access":{"read":true,"write":true},"organization":{"id":"OXtgecafZvh"}},{"type":"REVOKE_ORGANIZATION_DESIGN_ACCESS","access":{"read":true,"write":true},"organization":{"id":"OXtgecafZvh"}},{"type":"UPDATE_ORGANIZATION_DESIGN_ACCESS","old_access":{"read":true,"write":false},"new_access":{"read":true,"write":true},"organization":{"id":"OXtgecafZvh"}},{"type":"GRANT_DESIGN_LINK_ACCESS","access":{"read":true,"write":true},"owning_team_only":true},{"type":"REVOKE_DESIGN_LINK_ACCESS","access":{"read":true,"write":true},"owning_team_only":true},{"type":"UPDATE_DESIGN_LINK_ACCESS","old_link_role":{"access":{"read":true,"write":false},"owning_team_only":true},"new_link_role":{"access":{"read":true,"write":true},"owning_team_only":false}}],"description":"TheAcmeCorporationmarketinggroup.","old_display_name":"Marketing","new_display_name":"Growth","user":{"id":"UXoqDbwwSbQ","display_name":"JaneDoe","email":"jane.doe@example.com"},"role":"ADMIN","new_role":"ADMIN","old_role":"ADMIN","team_address":{"street1":"110Kippaxstreet","city":"SurryHills","subdivision":"AU-NSW","country_code":"AU","postcode":2010},"approval_status":"PENDING","emails":["ash.doe@example.com","alex.doe@example.com"],"report_type":"USER","start_timestamp":1709751447000,"end_timestamp":1720292247000,"old_name":"UntitledCorporation","new_name":"AcmeCorporation","default_team_id":"BXeFatjDhdR","default_team_policy":"ADMIN_AND_UP","team":{"id":"BXeFatjDhdR","display_name":"AcmeCorporation"}},"outcome":{"result":"PERMITTED","details":{"type":"RESOURCE_CREATED","resource_id":"DXWEBartcNg","resource_type":"DESIGN","user_id":"ac343"}},"context":{"ip_address":"81.2.69.142","session":"abc111","request_id":"fafas","device_id":"Ddb44"}}

packages/canva/data_stream/audit/_dev/deploy/tf/main.tf

@@ -0,0 +1,57 @@
+provider "aws" {
+  region = "us-east-1"
+  default_tags {
+    tags = {
+      environment  = var.ENVIRONMENT
+      repo         = var.REPO
+      branch       = var.BRANCH
+      build        = var.BUILD_ID
+      created_date = var.CREATED_DATE
+    }
+  }
+}
+
+resource "aws_s3_bucket" "bucket" {
+  bucket = "elastic-package-canva-bucket-${var.TEST_RUN_ID}"
+}
+
+resource "aws_sqs_queue" "queue" {
+  name       = "elastic-package-canva-queue-${var.TEST_RUN_ID}"
+  policy     = <<POLICY
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": "*",
+      "Action": "sqs:SendMessage",
+      "Resource": "arn:aws:sqs:*:*:elastic-package-canva-queue-${var.TEST_RUN_ID}",
+      "Condition": {
+        "ArnEquals": { "aws:SourceArn": "${aws_s3_bucket.bucket.arn}" }
+      }
+    }
+  ]
+}
+POLICY
+}
+
+resource "aws_s3_bucket_notification" "bucket_notification" {
+  bucket = aws_s3_bucket.bucket.id
+
+  queue {
+    queue_arn = aws_sqs_queue.queue.arn
+    events    = ["s3:ObjectCreated:*"]
+  }
+}
+
+resource "aws_s3_object" "object" {
+  bucket = aws_s3_bucket.bucket.id
+  key    = "audit.log"
+  source = "./files/test-audit.log"
+
+  depends_on = [aws_sqs_queue.queue]
+}
+
+output "queue_url" {
+  value = aws_sqs_queue.queue.url
+}

packages/canva/data_stream/audit/_dev/deploy/tf/variables.tf

@@ -0,0 +1,26 @@
+variable "BRANCH" {
+  description = "Branch name or pull request for tagging purposes"
+  default     = "unknown-branch"
+}
+
+variable "BUILD_ID" {
+  description = "Build ID in the CI for tagging purposes"
+  default     = "unknown-build"
+}
+
+variable "CREATED_DATE" {
+  description = "Creation date in epoch time for tagging purposes"
+  default     = "unknown-date"
+}
+
+variable "ENVIRONMENT" {
+  default = "unknown-environment"
+}
+
+variable "REPO" {
+  default = "unknown-repo-name"
+}
+
+variable "TEST_RUN_ID" {
+  default = "detached"
+}