diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f4c76a9cb312a..045f39df6246d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,8 +1,8 @@ docs/ @vectordotdev/ux-team -lib/file-source/ @spencergilbert @vectordotdev/integrations-team -lib/k8s-e2e-tests/ @spencergilbert @vectordotdev/integrations-team -lib/k8s-test-framework/ @spencergilbert @vectordotdev/integrations-team -lib/opentelemetry-proto/ @spencergilbert @vectordotdev/integrations-team +lib/file-source/ @vectordotdev/integrations-team +lib/k8s-e2e-tests/ @vectordotdev/integrations-team +lib/k8s-test-framework/ @vectordotdev/integrations-team +lib/opentelemetry-proto/ @vectordotdev/integrations-team lib/vector-common/ @vectordotdev/core-team lib/vector-config/ @vectordotdev/core-team lib/vector-config-common/ @vectordotdev/core-team @@ -15,39 +15,39 @@ src/internal_telemetry/ @vectordotdev/core-team src/sinks/ @vectordotdev/integrations-team src/sinks/amqp/ @StephenWakely @vectordotdev/integrations-team src/sinks/appsignal/ @neuronull @vectordotdev/integrations-team -src/sinks/aws_cloudwatch_logs/ @spencergilbert @vectordotdev/integrations-team -src/sinks/aws_cloudwatch_metrics/ @spencergilbert @vectordotdev/integrations-team -src/sinks/aws_kinesis/ @spencergilbert @vectordotdev/integrations-team # sink_aws_kinesis_firehose,sink_aws_kinesis_stream -src/sinks/aws_s3/ @spencergilbert @vectordotdev/integrations-team -src/sinks/aws_sqs/ @spencergilbert @vectordotdev/integrations-team -src/sinks/axiom.rs @spencergilbert @vectordotdev/integrations-team +src/sinks/aws_cloudwatch_logs/ @vectordotdev/integrations-team +src/sinks/aws_cloudwatch_metrics/ @vectordotdev/integrations-team +src/sinks/aws_kinesis/ @vectordotdev/integrations-team # sink_aws_kinesis_firehose,sink_aws_kinesis_stream +src/sinks/aws_s3/ @vectordotdev/integrations-team +src/sinks/aws_sqs/ @vectordotdev/integrations-team +src/sinks/axiom.rs @vectordotdev/integrations-team src/sinks/azure_blob/ @dsmith3197 @vectordotdev/integrations-team src/sinks/azure_monitor_logs.rs @dsmith3197 @vectordotdev/integrations-team src/sinks/blackhole/ @dsmith3197 @vectordotdev/integrations-team src/sinks/clickhouse/ @dsmith3197 @vectordotdev/integrations-team src/sinks/console/ @dsmith3197 @vectordotdev/integrations-team -src/sinks/databend/ @spencergilbert @vectordotdev/integrations-team +src/sinks/databend/ @vectordotdev/integrations-team src/sinks/datadog_events/ @neuronull @vectordotdev/integrations-team src/sinks/datadog_logs/ @neuronull @vectordotdev/integrations-team src/sinks/datadog_metrics/ @neuronull @vectordotdev/integrations-team src/sinks/datadog_traces/ @neuronull @vectordotdev/integrations-team -src/sinks/elasticsearch/ @spencergilbert @vectordotdev/integrations-team -src/sinks/file/ @spencergilbert @vectordotdev/integrations-team +src/sinks/elasticsearch/ @vectordotdev/integrations-team +src/sinks/file/ @vectordotdev/integrations-team src/sinks/gcp/ @StephenWakely @vectordotdev/integrations-team # sink_gcp_chronicle_unstructured,sink_gcp_cloud_storage,sink_gcp_pubsub,sink_gcp_stackdriver_logs,sink_gcp_stackdriver_metrics -src/sinks/honeycomb.rs @spencergilbert @vectordotdev/integrations-team +src/sinks/honeycomb.rs @vectordotdev/integrations-team src/sinks/http.rs @neuronull @vectordotdev/integrations-team src/sinks/humio/ @StephenWakely @vectordotdev/integrations-team # sink_humio_logs,sink_humio_metrics src/sinks/influxdb/ @dsmith3197 @vectordotdev/integrations-team # sink_influxdb_logs,sink_influxdb_metrics src/sinks/kafka/ @dsmith3197 @vectordotdev/integrations-team src/sinks/logdna.rs @neuronull @vectordotdev/integrations-team -src/sinks/loki/ @spencergilbert @vectordotdev/integrations-team +src/sinks/loki/ @vectordotdev/integrations-team src/sinks/nats.rs @StephenWakely @vectordotdev/integrations-team src/sinks/new_relic/ @dsmith3197 @vectordotdev/integrations-team # sink_newrelix,sink_newrelic_logs src/sinks/papertrail.rs @StephenWakely @vectordotdev/integrations-team src/sinks/prometheus/ @StephenWakely @vectordotdev/integrations-team # sink_prometheus_exporter,sink_prometheus_remote_write src/sinks/pulsar.rs @dsmith3197 @vectordotdev/integrations-team src/sinks/redis.rs @StephenWakely @vectordotdev/integrations-team -src/sinks/sematext/ @spencergilbert @vectordotdev/integrations-team # sink_sematext_logs,sink_sematext_metrics +src/sinks/sematext/ @vectordotdev/integrations-team # sink_sematext_logs,sink_sematext_metrics src/sinks/socket.rs @neuronull @vectordotdev/integrations-team src/sinks/splunk_hec/ @StephenWakely @vectordotdev/integrations-team # sink_splunk_hec_logs,sink_splunk_hec_metrics src/sinks/statsd.rs @neuronull @vectordotdev/integrations-team @@ -57,34 +57,34 @@ src/source_sender/ @vectordotdev/core-team src/sources/ @vectordotdev/integrations-team src/sources/amqp.rs @StephenWakely @vectordotdev/integrations-team src/sources/apache_metrics/ @dsmith3197 @vectordotdev/integrations-team -src/sources/aws_ecs_metrics/ @spencergilbert @vectordotdev/integrations-team -src/sources/aws_kinesis_firehose/ @spencergilbert @vectordotdev/integrations-team -src/sources/aws_s3/ @spencergilbert @vectordotdev/integrations-team -src/sources/aws_sqs/ @spencergilbert @vectordotdev/integrations-team +src/sources/aws_ecs_metrics/ @vectordotdev/integrations-team +src/sources/aws_kinesis_firehose/ @vectordotdev/integrations-team +src/sources/aws_s3/ @vectordotdev/integrations-team +src/sources/aws_sqs/ @vectordotdev/integrations-team src/sources/datadog_agent/ @neuronull @vectordotdev/integrations-team src/sources/demo_logs.rs @StephenWakely @vectordotdev/integrations-team src/sources/dnstap/ @StephenWakely @vectordotdev/integrations-team -src/sources/docker_logs/ @spencergilbert @vectordotdev/integrations-team +src/sources/docker_logs/ @vectordotdev/integrations-team src/sources/eventstoredb_metrics/ @dsmith3197 @vectordotdev/integrations-team src/sources/exec/ @dsmith3197 @vectordotdev/integrations-team -src/sources/file.rs @spencergilbert @vectordotdev/integrations-team +src/sources/file.rs @vectordotdev/integrations-team src/sources/file_descriptors/ @dsmith3197 @vectordotdev/integrations-team # source_file_descriptor,source_stdin src/sources/fluent/ @neuronull @vectordotdev/integrations-team src/sources/gcp_pubsub.rs @StephenWakely @vectordotdev/integrations-team -src/sources/heroku_logs.rs @spencergilbert @vectordotdev/integrations-team +src/sources/heroku_logs.rs @vectordotdev/integrations-team src/sources/host_metrics/ @dsmith3197 @vectordotdev/integrations-team src/sources/http_client/ @neuronull @vectordotdev/integrations-team src/sources/http_server.rs @neuronull @vectordotdev/integrations-team src/sources/internal_logs.rs @neuronull @vectordotdev/integrations-team src/sources/internal_metrics.rs @neuronull @vectordotdev/integrations-team -src/sources/journald.rs @spencergilbert @vectordotdev/integrations-team +src/sources/journald.rs @vectordotdev/integrations-team src/sources/kafka.rs @dsmith3197 @vectordotdev/integrations-team -src/sources/kubernetes_logs/ @spencergilbert @vectordotdev/integrations-team +src/sources/kubernetes_logs/ @vectordotdev/integrations-team src/sources/logstash.rs @neuronull @vectordotdev/integrations-team src/sources/mongodb_metrics/ @dsmith3197 @vectordotdev/integrations-team src/sources/nats.rs @StephenWakely @vectordotdev/integrations-team src/sources/nginx_metrics/ @dsmith3197 @vectordotdev/integrations-team -src/sources/opentelemetry/ @spencergilbert @vectordotdev/integrations-team +src/sources/opentelemetry/ @vectordotdev/integrations-team src/sources/postgresql_metrics.rs @dsmith3197 @vectordotdev/integrations-team src/sources/prometheus/ @StephenWakely @vectordotdev/integrations-team # source_prometheus_remote_write,source_prometheus_scrape src/sources/redis/ @StephenWakely @vectordotdev/integrations-team diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 435dd1bb3ca24..967072ef5d300 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -23,6 +23,9 @@ updates: clap: patterns: - "clap*" + csv: + patterns: + - "csv*" phf: patterns: - "phf*" diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index 653d3395bdfda..7bf03d0c2a8a0 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -56,8 +56,14 @@ on: value: ${{ jobs.int_tests.outputs.clickhouse }} databend: value: ${{ jobs.int_tests.outputs.databend }} - datadog: - value: ${{ jobs.int_tests.outputs.datadog }} + datadog-agent: + value: ${{ jobs.int_tests.outputs.datadog-agent }} + datadog-logs: + value: ${{ jobs.int_tests.outputs.datadog-logs }} + datadog-metrics: + value: ${{ jobs.int_tests.outputs.datadog-metrics }} + datadog-traces: + value: ${{ jobs.int_tests.outputs.datadog-traces }} dnstap: value: ${{ jobs.int_tests.outputs.dnstap }} docker-logs: @@ -143,9 +149,6 @@ jobs: - "Makefile" - "rust-toolchain.toml" - "vdev/**" - deny: - - 'deny.toml' - - "vdev/**" dependencies: - ".cargo/**" - 'Cargo.toml' @@ -189,7 +192,10 @@ jobs: azure: ${{ steps.filter.outputs.azure }} clickhouse: ${{ steps.filter.outputs.clickhouse }} databend: ${{ steps.filter.outputs.databend }} - datadog: ${{ steps.filter.outputs.datadog }} + datadog-agent: ${{ steps.filter.outputs.datadog-agent }} + datadog-logs: ${{ steps.filter.outputs.datadog-logs }} + datadog-metrics: ${{ steps.filter.outputs.datadog-metrics }} + datadog-traces: ${{ steps.filter.outputs.datadog-traces }} dnstap: ${{ steps.filter.outputs.dnstap }} docker-logs: ${{ steps.filter.outputs.docker-logs }} elasticsearch: ${{ steps.filter.outputs.elasticsearch }} diff --git a/.github/workflows/comment-trigger.yml b/.github/workflows/comment-trigger.yml index 59ec98530a7ba..6083e9db6ac4a 100644 --- a/.github/workflows/comment-trigger.yml +++ b/.github/workflows/comment-trigger.yml @@ -9,6 +9,7 @@ # /ci-run-all : runs all of the below # /ci-run-cli : runs CLI - Linux # /ci-run-misc : runs Miscellaneous - Linux +# /ci-run-deny : runs Deny - Linux # /ci-run-component-features : runs Component Features - Linux # /ci-run-cross : runs Cross # /ci-run-unit-mac : runs Unit - Mac @@ -50,6 +51,7 @@ jobs: github.event.issue.pull_request && ( contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-cli') || contains(github.event.comment.body, '/ci-run-misc') + || contains(github.event.comment.body, '/ci-run-deny') || contains(github.event.comment.body, '/ci-run-component-features') || contains(github.event.comment.body, '/ci-run-cross') || contains(github.event.comment.body, '/ci-run-unit-mac') @@ -67,7 +69,7 @@ jobs: private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }} - name: Get PR comment author id: comment - uses: tspascoal/get-user-teams-membership@v2 + uses: tspascoal/get-user-teams-membership@v3 with: username: ${{ github.actor }} team: 'Vector' @@ -89,6 +91,12 @@ jobs: uses: ./.github/workflows/misc.yml secrets: inherit + deny: + needs: validate + if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-deny') + uses: ./.github/workflows/deny.yml + secrets: inherit + component-features: needs: validate if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-component-features') diff --git a/.github/workflows/deny.yml b/.github/workflows/deny.yml new file mode 100644 index 0000000000000..f0040b68fb639 --- /dev/null +++ b/.github/workflows/deny.yml @@ -0,0 +1,74 @@ +# Deny - Linux +# +# Checks for security vulnerabilities or license incompatibilities +# +# Runs on: +# - scheduled UTC midnight +# - on PR comment (see comment-trigger.yml) +# - on demand from github actions UI + +name: Deny - Linux + +on: + workflow_call: + workflow_dispatch: + schedule: + # At midnight UTC + - cron: '0 0 * * *' + +jobs: + test-deny: + runs-on: ubuntu-latest + env: + CARGO_INCREMENTAL: 0 + steps: + - name: (PR comment) Get PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: xt0rted/pull-request-comment-branch@v2 + id: comment-branch + + - name: (PR comment) Set latest commit status as pending + if: ${{ github.event_name == 'issue_comment' }} + uses: myrotvorets/set-commit-status-action@v2.0.0 + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Deny - Linux + status: pending + + - name: (PR comment) Checkout PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: actions/checkout@v3 + with: + ref: ${{ steps.comment-branch.outputs.head_ref }} + + - name: Checkout branch + if: ${{ github.event_name != 'issue_comment' }} + uses: actions/checkout@v3 + + - uses: actions/cache@v3 + name: Cache Cargo registry + index + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo- + + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: bash scripts/environment/prepare.sh + - run: echo "::add-matcher::.github/matchers/rust.json" + - name: Check cargo deny advisories/licenses + run: make check-deny + + - name: (PR comment) Set latest commit status as ${{ job.status }} + uses: myrotvorets/set-commit-status-action@v2.0.0 + if: always() && github.event_name == 'issue_comment' + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Deny - Linux + status: ${{ job.status }} diff --git a/.github/workflows/gardener_open_pr.yml b/.github/workflows/gardener_open_pr.yml index 3ee4e0acc40d8..c047b196330d8 100644 --- a/.github/workflows/gardener_open_pr.yml +++ b/.github/workflows/gardener_open_pr.yml @@ -19,7 +19,7 @@ jobs: with: app_id: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }} private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }} - - uses: tspascoal/get-user-teams-membership@v2 + - uses: tspascoal/get-user-teams-membership@v3 id: checkVectorMember with: username: ${{ github.actor }} diff --git a/.github/workflows/integration-comment.yml b/.github/workflows/integration-comment.yml index fdb0eb5e25e45..3f85c7c8ed614 100644 --- a/.github/workflows/integration-comment.yml +++ b/.github/workflows/integration-comment.yml @@ -55,7 +55,7 @@ jobs: private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }} - name: Get PR comment author id: comment - uses: tspascoal/get-user-teams-membership@v2 + uses: tspascoal/get-user-teams-membership@v3 with: username: ${{ github.actor }} team: 'Vector' @@ -387,7 +387,7 @@ jobs: private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }} - name: Validate issue comment if: github.event_name == 'issue_comment' - uses: tspascoal/get-user-teams-membership@v2 + uses: tspascoal/get-user-teams-membership@v3 with: username: ${{ github.actor }} team: 'Vector' diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 01a2a7dabca3a..709f2091fe0b1 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -60,7 +60,10 @@ jobs: || needs.changes.outputs.azure == 'true' || needs.changes.outputs.clickhouse == 'true' || needs.changes.outputs.databend == 'true' - || needs.changes.outputs.datadog == 'true' + || needs.changes.outputs.datadog-agent == 'true' + || needs.changes.outputs.datadog-logs == 'true' + || needs.changes.outputs.datadog-metrics == 'true' + || needs.changes.outputs.datadog-traces == 'true' || needs.changes.outputs.dnstap == 'true' || needs.changes.outputs.docker-logs == 'true' || needs.changes.outputs.elasticsearch == 'true' @@ -166,7 +169,7 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh databend - - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') && + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-agent == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: datadog-agent uses: nick-fields/retry@v2 @@ -175,7 +178,7 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-agent - - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') && + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-logs == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: datadog-logs uses: nick-fields/retry@v2 @@ -184,7 +187,7 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-logs - - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') && + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-metrics == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: datadog-metrics uses: nick-fields/retry@v2 @@ -193,7 +196,7 @@ jobs: max_attempts: 3 command: bash scripts/ci-integration-test.sh datadog-metrics - - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') && + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-traces == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: datadog-traces uses: nick-fields/retry@v2 diff --git a/.github/workflows/protobuf.yml b/.github/workflows/protobuf.yml index bfe0066480ec6..b0c21c12bb4b9 100644 --- a/.github/workflows/protobuf.yml +++ b/.github/workflows/protobuf.yml @@ -20,7 +20,7 @@ jobs: # Run `git checkout` - uses: actions/checkout@v3 # Install the `buf` CLI - - uses: bufbuild/buf-setup-action@v1.26.1 + - uses: bufbuild/buf-setup-action@v1.27.0 # Perform breaking change detection against the `master` branch - uses: bufbuild/buf-breaking-action@v1.1.3 with: diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 387750b55ddf9..e350c8cec6e46 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -362,7 +362,7 @@ jobs: - compute-metadata steps: - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.0.0 + uses: aws-actions/configure-aws-credentials@v4.0.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -394,7 +394,7 @@ jobs: docker load --input baseline-image.tar - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.0.0 + uses: aws-actions/configure-aws-credentials@v4.0.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -432,7 +432,7 @@ jobs: docker load --input comparison-image.tar - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.0.0 + uses: aws-actions/configure-aws-credentials@v4.0.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -478,7 +478,7 @@ jobs: ref: ${{ needs.compute-metadata.outputs.comparison-sha }} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.0.0 + uses: aws-actions/configure-aws-credentials@v4.0.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -597,7 +597,7 @@ jobs: - uses: actions/checkout@v3 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.0.0 + uses: aws-actions/configure-aws-credentials@v4.0.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -688,7 +688,7 @@ jobs: ref: ${{ needs.compute-metadata.outputs.comparison-sha }} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.0.0 + uses: aws-actions/configure-aws-credentials@v4.0.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dda403b3b6030..30a430fd1348b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -94,10 +94,6 @@ jobs: if: needs.changes.outputs.source == 'true' run: make check-events - - name: Check cargo deny advisories/licenses - if: needs.changes.outputs.dependencies == 'true' || needs.changes.outputs.deny == 'true' - run: make check-deny - - name: Check that the 3rd-party license file is up to date if: needs.changes.outputs.dependencies == 'true' run: make check-licenses diff --git a/.github/workflows/workload_checks.yml b/.github/workflows/workload_checks.yml index 76d336ebe2a99..b91e05dc95ad8 100644 --- a/.github/workflows/workload_checks.yml +++ b/.github/workflows/workload_checks.yml @@ -96,7 +96,7 @@ jobs: - uses: actions/checkout@v3 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.0.0 + uses: aws-actions/configure-aws-credentials@v4.0.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} diff --git a/Cargo.lock b/Cargo.lock index 538e992cd56ae..4d83969e7a658 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -354,7 +354,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88903cb14723e4d4003335bb7f8a14f27691649105346a0f0957466c096adfe6" dependencies = [ "anstyle 1.0.0", - "bstr 1.6.2", + "bstr 1.7.0", "doc-comment", "predicates", "predicates-core", @@ -524,7 +524,7 @@ dependencies = [ "async-graphql-parser 5.0.10", "darling 0.14.2", "proc-macro-crate 1.2.1", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", "thiserror", @@ -540,10 +540,10 @@ dependencies = [ "async-graphql-parser 6.0.0", "darling 0.20.3", "proc-macro-crate 1.2.1", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "strum", - "syn 2.0.37", + "syn 2.0.38", "thiserror", ] @@ -719,9 +719,9 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -741,9 +741,9 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -758,9 +758,9 @@ version = "0.1.73" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -1554,7 +1554,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49fb8528abca6895a5ada33d62aedd538a5c33e77068256483b44a3230270163" dependencies = [ "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -1678,7 +1678,7 @@ dependencies = [ "borsh-derive-internal", "borsh-schema-derive-internal", "proc-macro-crate 0.1.5", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "syn 1.0.109", ] @@ -1688,7 +1688,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61820b4c5693eafb998b1e67485423c923db4a75f72585c247bdee32bad81e7b" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -1699,7 +1699,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c76cdbfa13def20d1f8af3ae7b3c6771f06352a74221d8851262ac384c122b8e" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -1738,12 +1738,12 @@ dependencies = [ [[package]] name = "bstr" -version = "1.6.2" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c2f7349907b712260e64b0afe2f84692af14a454be26187d9df565c7f69266a" +checksum = "c79ad7fb2dd38f3dabd76b09c6a5a20c038fc0213ef1e9afd30eb777f120f019" dependencies = [ "memchr", - "regex-automata 0.3.9", + "regex-automata 0.4.1", "serde", ] @@ -1769,7 +1769,7 @@ version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13e576ebe98e605500b3c8041bb888e966653577172df6dd97398714eb30b9bf" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -1848,7 +1848,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8245dd5f576a41c3b76247b54c15b0e43139ceeb4f732033e15be7c005176" dependencies = [ "darling 0.14.2", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -2108,9 +2108,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" dependencies = [ "heck 0.4.1", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2569,9 +2569,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.2.2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" dependencies = [ "csv-core", "itoa", @@ -2581,9 +2581,9 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" dependencies = [ "memchr", ] @@ -2635,9 +2635,9 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2661,7 +2661,7 @@ dependencies = [ "cc", "codespan-reporting", "once_cell", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "scratch", "syn 1.0.109", @@ -2679,7 +2679,7 @@ version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a08a6e2fcc370a089ad3b4aaf54db3b1b4cee38ddabce5896b33eb693275f470" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -2722,7 +2722,7 @@ checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "strsim 0.10.0", "syn 1.0.109", @@ -2736,7 +2736,7 @@ checksum = "a784d2ccaf7c98501746bf0be29b2022ba41fd62a2e622af997a03e9f972859f" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "strsim 0.10.0", "syn 1.0.109", @@ -2750,10 +2750,10 @@ checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "strsim 0.10.0", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2786,7 +2786,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2877,7 +2877,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -2888,9 +2888,9 @@ version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53e0efad4403bfc52dc201159c4b842a246a14b98c64b55dfd0f2d89729dfeb8" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2900,7 +2900,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case 0.4.0", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "rustc_version 0.4.0", "syn 1.0.109", @@ -3147,7 +3147,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21cdad81446a7f7dc43f6a77409efeb9733d2fa65553efef6018ef257c959b73" dependencies = [ "heck 0.4.1", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -3159,9 +3159,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a" dependencies = [ "heck 0.4.1", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -3171,9 +3171,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f33313078bb8d4d05a2733a94ac4c2d8a0df9a2b84424ebf4f33bfc224a890e" dependencies = [ "once_cell", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -3191,9 +3191,9 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f95e2801cd355d4a1a3e3953ce6ee5ae9603a5c833455343a8bfe3f44d418246" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -3308,7 +3308,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f47da3a72ec598d9c8937a7ebca8962a5c7a1f28444e38c2b33c771ba3f55f05" dependencies = [ "proc-macro-error", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -3364,7 +3364,7 @@ checksum = "e825f6987101665dea6ec934c09ec6d721de7bc1bf92248e1d5810c8cd636b77" name = "file-source" version = "0.1.0" dependencies = [ - "bstr 1.6.2", + "bstr 1.7.0", "bytes 1.5.0", "chrono", "crc", @@ -3416,7 +3416,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4c81935e123ab0741c4c4f0d9b8377e5fb21d3de7e062fa4b1263b1fbcba1ea" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -3597,9 +3597,9 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -3776,7 +3776,7 @@ dependencies = [ "graphql-parser", "heck 0.4.1", "lazy_static", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "serde", "serde_json", @@ -3790,7 +3790,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00bda454f3d313f909298f626115092d348bc231025699f557b27e248475f48c" dependencies = [ "graphql_client_codegen", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "syn 1.0.109", ] @@ -5007,9 +5007,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a83fb7698b3643a0e34f9ae6f2e8f0178c0fd42f8b59d493aa271ff3a5bf21" +checksum = "1efa59af2ddfad1854ae27d75009d538d0998b4b2fd47083e743ac1a10e46c60" [[package]] name = "lru-cache" @@ -5158,9 +5158,9 @@ checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memmap2" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a5a03cefb0d953ec0be133036f14e109412fa594edc2f77227249db66cc3ed" +checksum = "deaba38d7abf1d4cca21cc89e932e542ba2b9258664d2a9ef0e61512039c9375" dependencies = [ "libc", ] @@ -5200,9 +5200,9 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -5235,7 +5235,7 @@ dependencies = [ "metrics", "num_cpus", "ordered-float 3.9.1", - "quanta", + "quanta 0.11.1", "radix_trie", "sketches-ddsketch", ] @@ -5298,7 +5298,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c3a7a7ff4481ec91b951a733390211a8ace1caba57266ccb5f4d4966704e560" dependencies = [ - "bstr 1.6.2", + "bstr 1.7.0", "mlua-sys", "mlua_derive", "num-traits", @@ -5328,10 +5328,10 @@ dependencies = [ "itertools 0.11.0", "once_cell", "proc-macro-error", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "regex", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -5711,7 +5711,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" dependencies = [ "proc-macro-crate 1.2.1", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -5723,9 +5723,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96667db765a921f7b295ffee8b60472b686a51d4f21c2ee4ffdb94c7013b65a6" dependencies = [ "proc-macro-crate 1.2.1", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -5735,9 +5735,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56ea360eafe1022f7cc56cd7b869ed57330fb2453d0c7831d99b74c65d2f5597" dependencies = [ "proc-macro-crate 1.2.1", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -5918,7 +5918,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -6172,9 +6172,9 @@ checksum = "d1511785c5e98d79a05e8a6bc34b4ac2168a0e3e92161862030ad84daa223141" dependencies = [ "pest", "pest_meta", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -6260,9 +6260,9 @@ version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -6496,7 +6496,7 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c142c0e46b57171fe0c528bee8c5b7569e80f0c17e377cd0e30ea57dbc11bb51" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "syn 1.0.109", ] @@ -6506,8 +6506,8 @@ version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" dependencies = [ - "proc-macro2 1.0.67", - "syn 2.0.37", + "proc-macro2 1.0.69", + "syn 2.0.38", ] [[package]] @@ -6551,7 +6551,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", "version_check", @@ -6563,7 +6563,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "version_check", ] @@ -6591,9 +6591,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] @@ -6691,7 +6691,7 @@ dependencies = [ "prost 0.12.1", "prost-types 0.12.1", "regex", - "syn 2.0.37", + "syn 2.0.38", "tempfile", "which", ] @@ -6704,7 +6704,7 @@ checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", "itertools 0.10.5", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -6717,9 +6717,9 @@ checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" dependencies = [ "anyhow", "itertools 0.11.0", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -6769,7 +6769,7 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -6829,7 +6829,23 @@ dependencies = [ "libc", "mach2", "once_cell", - "raw-cpuid", + "raw-cpuid 10.6.0", + "wasi 0.11.0+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + +[[package]] +name = "quanta" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4966e46e3f559f8681bf8203b5e0e2421ebf6de5dd8b59f2ec8fa91acc028a19" +dependencies = [ + "crossbeam-utils", + "libc", + "mach2", + "once_cell", + "raw-cpuid 11.0.1", "wasi 0.11.0+wasi-snapshot-preview1", "web-sys", "winapi", @@ -6878,7 +6894,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b22a693222d716a9587786f37ac3f6b4faedb5b80c23914e7303ff5a1d8016e9" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -6898,7 +6914,7 @@ version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", ] [[package]] @@ -7039,6 +7055,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "raw-cpuid" +version = "11.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" +dependencies = [ + "bitflags 2.4.0", +] + [[package]] name = "raw-window-handle" version = "0.5.0" @@ -7207,6 +7232,12 @@ dependencies = [ "regex-syntax 0.7.5", ] +[[package]] +name = "regex-automata" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465c6fc0621e4abc4187a2bda0937bfd4f722c2730b29562e19689ea796c9a4b" + [[package]] name = "regex-lite" version = "0.1.0" @@ -7242,9 +7273,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.20" +version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ "base64 0.21.4", "bytes 1.5.0", @@ -7270,6 +7301,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "system-configuration", "tokio", "tokio-native-tls", "tokio-rustls 0.24.1", @@ -7335,7 +7367,7 @@ version = "0.7.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff26ed6c7c4dfc2aa9480b86a60e3c7233543a270a680e10758a507c5a4ce476" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -7429,12 +7461,12 @@ checksum = "d428f8247852f894ee1be110b375111b586d4fa431f6c46e64ba5a0dcccbe605" dependencies = [ "cfg-if", "glob", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "regex", "relative-path", "rustc_version 0.4.0", - "syn 2.0.37", + "syn 2.0.38", "unicode-ident", ] @@ -7483,7 +7515,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.19", + "semver 1.0.20", ] [[package]] @@ -7749,9 +7781,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" dependencies = [ "serde", ] @@ -7816,9 +7848,9 @@ version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -7827,9 +7859,9 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330f01ce65a3a5fe59a60c82f3c9a024b573b8a6e875bd233fe5f934e71d54e3" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -7879,9 +7911,9 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -7939,7 +7971,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" dependencies = [ "darling 0.13.4", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -7951,9 +7983,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e6be15c453eb305019bfa438b1593c731f36a289a7853f7707ee29e870b3b3c" dependencies = [ "darling 0.20.3", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -8212,7 +8244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" dependencies = [ "heck 0.4.1", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -8354,7 +8386,7 @@ checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" dependencies = [ "heck 0.3.3", "proc-macro-error", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -8375,10 +8407,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad8d03b598d3d0fff69bf533ee3ef19b8eeb342729596df84bcc7e1f96ec4059" dependencies = [ "heck 0.4.1", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "rustversion", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -8414,18 +8446,18 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.37" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "unicode-ident", ] @@ -8459,6 +8491,27 @@ dependencies = [ "nom", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "take_mut" version = "0.2.2" @@ -8583,9 +8636,9 @@ version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -8684,9 +8737,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.32.0" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ "backtrace", "bytes 1.5.0", @@ -8729,9 +8782,9 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -8980,7 +9033,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07" dependencies = [ "prettyplease 0.1.21", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "prost-build 0.11.9", "quote 1.0.33", "syn 1.0.109", @@ -8993,10 +9046,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d021fc044c18582b9a2408cd0dd05b1596e3ecdb5c4df822bb0183545683889" dependencies = [ "prettyplease 0.2.12", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "prost-build 0.12.1", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -9098,7 +9151,7 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -9337,7 +9390,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89851716b67b937e393b3daa8423e67ddfc4bbbf1654bcf05488e95e0828db0c" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "syn 1.0.109", ] @@ -9357,9 +9410,9 @@ version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -9387,9 +9440,9 @@ version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfc13d450dc4a695200da3074dacf43d449b968baee95e341920e47f61a3b40f" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -9754,7 +9807,7 @@ dependencies = [ "roaring", "rstest", "seahash", - "semver 1.0.19", + "semver 1.0.20", "serde", "serde-toml-merge", "serde_bytes", @@ -9944,11 +9997,11 @@ dependencies = [ "convert_case 0.6.0", "darling 0.20.3", "once_cell", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "serde", "serde_json", - "syn 2.0.37", + "syn 2.0.38", "tracing 0.1.37", ] @@ -9957,11 +10010,11 @@ name = "vector-config-macros" version = "0.1.0" dependencies = [ "darling 0.20.3", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", "serde", "serde_derive_internals", - "syn 2.0.37", + "syn 2.0.38", "vector-config", "vector-config-common", ] @@ -10009,7 +10062,7 @@ dependencies = [ "prost 0.12.1", "prost-build 0.12.1", "prost-types 0.12.1", - "quanta", + "quanta 0.12.0", "quickcheck", "quickcheck_macros", "rand 0.8.5", @@ -10221,7 +10274,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d257817081c7dffcdbab24b9e62d2def62e2ff7d00b1c20062551e6cccc145ff" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", ] @@ -10322,9 +10375,9 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", "wasm-bindgen-shared", ] @@ -10356,9 +10409,9 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.69", "quote 1.0.33", - "syn 2.0.37", + "syn 2.0.38", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/Cargo.toml b/Cargo.toml index 4f8ff8cacce5d..f12588fa5a3d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -118,7 +118,7 @@ members = [ ] [workspace.dependencies] -vrl = { version = "0.7.0", default-features = false, features = ["cli", "test", "test_framework", "arbitrary", "compiler", "value", "diagnostic", "path", "parser", "stdlib", "datadog", "core"] } +vrl = { version = "0.7.0", features = ["cli"] } pin-project = { version = "1.1.3", default-features = false } @@ -151,7 +151,7 @@ loki-logproto = { path = "lib/loki-logproto", optional = true } async-stream = { version = "0.3.5", default-features = false } async-trait = { version = "0.1.73", default-features = false } futures = { version = "0.3.28", default-features = false, features = ["compat", "io-compat"], package = "futures" } -tokio = { version = "1.32.0", default-features = false, features = ["full"] } +tokio = { version = "1.33.0", default-features = false, features = ["full"] } tokio-openssl = { version = "0.6.3", default-features = false } tokio-stream = { version = "0.1.14", default-features = false, features = ["net", "sync", "time"] } tokio-util = { version = "0.7", default-features = false, features = ["io", "time"] } @@ -257,7 +257,7 @@ chrono = { version = "0.4.31", default-features = false, features = ["serde"] } cidr-utils = { version = "0.5.11", default-features = false } clap = { version = "4.4.6", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } colored = { version = "2.0.4", default-features = false } -csv = { version = "1.2", default-features = false } +csv = { version = "1.3", default-features = false } derivative = { version = "2.2.0", default-features = false } dirs-next = { version = "2.0.0", default-features = false, optional = true } dyn-clone = { version = "1.0.14", default-features = false } @@ -288,7 +288,7 @@ k8s-openapi = { version = "0.18.0", default-features = false, features = ["api", kube = { version = "0.82.0", default-features = false, features = ["client", "openssl-tls", "runtime"], optional = true } listenfd = { version = "1.0.1", default-features = false, optional = true } logfmt = { version = "0.0.2", default-features = false, optional = true } -lru = { version = "0.11.1", default-features = false, optional = true } +lru = { version = "0.12.0", default-features = false, optional = true } maxminddb = { version = "0.23.0", default-features = false, optional = true } md-5 = { version = "0.10", default-features = false, optional = true } mongodb = { version = "2.7.0", default-features = false, features = ["tokio-runtime"], optional = true } @@ -311,7 +311,7 @@ redis = { version = "0.23.3", default-features = false, features = ["connection- regex = { version = "1.9.6", default-features = false, features = ["std", "perf"] } roaring = { version = "0.10.2", default-features = false, optional = true } seahash = { version = "4.1.0", default-features = false } -semver = { version = "1.0.19", default-features = false, features = ["serde", "std"], optional = true } +semver = { version = "1.0.20", default-features = false, features = ["serde", "std"], optional = true } smallvec = { version = "1", default-features = false, features = ["union", "serde"] } snafu = { version = "0.7.5", default-features = false, features = ["futures"] } snap = { version = "1.1.0", default-features = false, optional = true } @@ -370,7 +370,7 @@ reqwest = { version = "0.11", features = ["json"] } rstest = {version = "0.18.2"} tempfile = "3.8.0" test-generator = "0.3.1" -tokio = { version = "1.32.0", features = ["test-util"] } +tokio = { version = "1.33.0", features = ["test-util"] } tokio-test = "0.4.3" tower-test = "0.4.0" vector-core = { path = "lib/vector-core", default-features = false, features = ["vrl", "test"] } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 8aaddcf107047..01a4864d0480a 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -539,6 +539,7 @@ syn,https://github.com/dtolnay/syn,MIT OR Apache-2.0,David Tolnay syslog,https://github.com/Geal/rust-syslog,MIT,contact@geoffroycouprie.com syslog_loose,https://github.com/FungusHumungus/syslog-loose,MIT,Stephen Wakely +system-configuration,https://github.com/mullvad/system-configuration-rs,MIT OR Apache-2.0,Mullvad VPN take_mut,https://github.com/Sgeo/take_mut,MIT,Sgeo tap,https://github.com/myrrlyn/tap,MIT,"Elliott Linder , myrrlyn " tcp-stream,https://github.com/amqp-rs/tcp-stream,BSD-2-Clause,Marc-Antoine Perennou diff --git a/lib/codecs/src/encoding/format/native_json.rs b/lib/codecs/src/encoding/format/native_json.rs index 854bba9d97ec2..94fd6b3c14527 100644 --- a/lib/codecs/src/encoding/format/native_json.rs +++ b/lib/codecs/src/encoding/format/native_json.rs @@ -52,7 +52,8 @@ impl Encoder for NativeJsonSerializer { #[cfg(test)] mod tests { use bytes::BytesMut; - use vector_core::event::{LogEvent, Value}; + use vector_core::buckets; + use vector_core::event::{LogEvent, Metric, MetricKind, MetricValue, Value}; use vrl::btreemap; use super::*; @@ -84,4 +85,25 @@ mod tests { assert_eq!(bytes.freeze(), serde_json::to_string(&json).unwrap()); } + + #[test] + fn serialize_aggregated_histogram() { + let histogram_event = Event::from(Metric::new( + "histogram", + MetricKind::Absolute, + MetricValue::AggregatedHistogram { + count: 1, + sum: 1.0, + buckets: buckets!(f64::NEG_INFINITY => 0 ,2.0 => 1, f64::INFINITY => 0), + }, + )); + + let mut serializer = NativeJsonSerializer::new(); + let mut bytes = BytesMut::new(); + serializer + .encode(histogram_event.clone(), &mut bytes) + .unwrap(); + let json = serializer.to_json_value(histogram_event).unwrap(); + assert_eq!(bytes.freeze(), serde_json::to_string(&json).unwrap()); + } } diff --git a/lib/codecs/tests/native_json.rs b/lib/codecs/tests/native_json.rs new file mode 100644 index 0000000000000..dcc08bdaa187b --- /dev/null +++ b/lib/codecs/tests/native_json.rs @@ -0,0 +1,86 @@ +use bytes::BytesMut; +use codecs::decoding::format::Deserializer; +use codecs::encoding::format::Serializer; +use codecs::{NativeJsonDeserializerConfig, NativeJsonSerializerConfig}; +use vector_core::buckets; +use vector_core::config::LogNamespace; +use vector_core::event::{Event, Metric}; +use vector_core::event::{MetricKind, MetricValue}; + +fn assert_roundtrip( + input_event: Event, + serializer: &mut dyn Serializer, + deserializer: &dyn Deserializer, + expected_json_value: serde_json::Value, +) { + let mut bytes_mut = BytesMut::new(); + serializer + .encode(input_event.clone(), &mut bytes_mut) + .unwrap(); + let bytes = bytes_mut.freeze(); + let events = deserializer.parse(bytes, LogNamespace::Vector).unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0], input_event); + + let json_value = serde_json::to_value(input_event.as_metric()).unwrap(); + assert_eq!(json_value, expected_json_value); +} + +#[test] +fn histogram_metric_roundtrip() { + let histogram_event = Event::from(Metric::new( + "histogram", + MetricKind::Absolute, + MetricValue::AggregatedHistogram { + count: 1, + sum: 1.0, + buckets: buckets!( + f64::NEG_INFINITY => 10 , + f64::MIN => 10, 1.5 => 10, + f64::MAX => 10, + f64::INFINITY => 10), + }, + )); + + let expected_json_value = serde_json::from_str( + r#" + { + "aggregated_histogram": { + "buckets": [ + { + "count": 10, + "upper_limit": "-inf" + }, + { + "count": 10, + "upper_limit": -1.7976931348623157e308 + }, + { + "count": 10, + "upper_limit": 1.5 + }, + { + "count": 10, + "upper_limit": 1.7976931348623157e308 + }, + { + "count": 10, + "upper_limit": "inf" + } + ], + "count": 1, + "sum": 1.0 + }, + "kind": "absolute", + "name": "histogram" + }"#, + ) + .unwrap(); + + assert_roundtrip( + histogram_event, + &mut NativeJsonSerializerConfig.build(), + &NativeJsonDeserializerConfig::default().build(), + expected_json_value, + ) +} diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index e854b24677148..78315eb49ec5c 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -19,7 +19,7 @@ vector-config-common = { path = "../vector-config-common", default-features = fa vector-config-macros = { path = "../vector-config-macros", default-features = false } [dependencies.bstr] -version = "1.6" +version = "1.7" default-features = false features = [] @@ -69,7 +69,7 @@ default-features = false features = [] [dependencies.tokio] -version = "1.32.0" +version = "1.33.0" default-features = false features = ["full"] diff --git a/lib/k8s-e2e-tests/Cargo.toml b/lib/k8s-e2e-tests/Cargo.toml index fd88c11372297..22f25151b0244 100644 --- a/lib/k8s-e2e-tests/Cargo.toml +++ b/lib/k8s-e2e-tests/Cargo.toml @@ -12,9 +12,9 @@ futures = "0.3" k8s-openapi = { version = "0.16.0", default-features = false, features = ["v1_19"] } k8s-test-framework = { version = "0.1", path = "../k8s-test-framework" } regex = "1" -reqwest = { version = "0.11.20", features = ["json"] } +reqwest = { version = "0.11.22", features = ["json"] } serde_json = "1" -tokio = { version = "1.32.0", features = ["full"] } +tokio = { version = "1.33.0", features = ["full"] } indoc = "2.0.4" env_logger = "0.10" tracing = { version = "0.1", features = ["log"] } diff --git a/lib/k8s-test-framework/Cargo.toml b/lib/k8s-test-framework/Cargo.toml index 74b76b3bc903b..46227bab9c74a 100644 --- a/lib/k8s-test-framework/Cargo.toml +++ b/lib/k8s-test-framework/Cargo.toml @@ -11,5 +11,5 @@ license = "MPL-2.0" k8s-openapi = { version = "0.16.0", default-features = false, features = ["v1_19"] } serde_json = "1" tempfile = "3" -tokio = { version = "1.32.0", features = ["full"] } +tokio = { version = "1.33.0", features = ["full"] } log = "0.4" diff --git a/lib/vector-api-client/Cargo.toml b/lib/vector-api-client/Cargo.toml index d5f65276c8fbe..9ab51d786c4f5 100644 --- a/lib/vector-api-client/Cargo.toml +++ b/lib/vector-api-client/Cargo.toml @@ -18,14 +18,14 @@ anyhow = { version = "1.0.75", default-features = false, features = ["std"] } # Tokio / Futures async-trait = { version = "0.1", default-features = false } futures = { version = "0.3", default-features = false, features = ["compat", "io-compat"] } -tokio = { version = "1.32.0", default-features = false, features = ["macros", "rt", "sync"] } +tokio = { version = "1.33.0", default-features = false, features = ["macros", "rt", "sync"] } tokio-stream = { version = "0.1.14", default-features = false, features = ["sync"] } # GraphQL graphql_client = { version = "0.13.0", default-features = false, features = ["graphql_query_derive"] } # HTTP / WebSockets -reqwest = { version = "0.11.20", default-features = false, features = ["json"] } +reqwest = { version = "0.11.22", default-features = false, features = ["json"] } tokio-tungstenite = { version = "0.20.1", default-features = false, features = ["connect", "rustls"] } # External libs diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index 6c074661d8eaf..9a169bfbaea2c 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -16,7 +16,7 @@ crossbeam-queue = { version = "0.3.8", default-features = false, features = ["st crossbeam-utils = { version = "0.8.16", default-features = false } fslock = { version = "0.2.1", default-features = false, features = ["std"] } futures = { version = "0.3.28", default-features = false, features = ["std"] } -memmap2 = { version = "0.8.0", default-features = false } +memmap2 = { version = "0.9.0", default-features = false } metrics = "0.21.1" num-traits = { version = "0.2.16", default-features = false } pin-project.workspace = true @@ -24,7 +24,7 @@ rkyv = { version = "0.7.40", default-features = false, features = ["size_32", "s serde = { version = "1.0.188", default-features = false, features = ["derive"] } snafu = { version = "0.7.5", default-features = false, features = ["std"] } tokio-util = { version = "0.7.0", default-features = false } -tokio = { version = "1.32.0", default-features = false, features = ["rt", "macros", "rt-multi-thread", "sync", "fs", "io-util", "time"] } +tokio = { version = "1.33.0", default-features = false, features = ["rt", "macros", "rt-multi-thread", "sync", "fs", "io-util", "time"] } tracing = { version = "0.1.34", default-features = false, features = ["attributes"] } vector-config = { path = "../vector-config", default-features = false } vector-config-common = { path = "../vector-config-common", default-features = false } diff --git a/lib/vector-buffers/src/lib.rs b/lib/vector-buffers/src/lib.rs index 7031848daafb7..4b5334674ffa3 100644 --- a/lib/vector-buffers/src/lib.rs +++ b/lib/vector-buffers/src/lib.rs @@ -147,7 +147,10 @@ where T: Send + 'static, { #[cfg(tokio_unstable)] - return tokio::task::Builder::new().name(_name).spawn(task); + return tokio::task::Builder::new() + .name(_name) + .spawn(task) + .expect("tokio task should spawn"); #[cfg(not(tokio_unstable))] tokio::spawn(task) diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index 53c12c2b9890a..9f3d2b1346e25 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -60,7 +60,7 @@ serde = { version = "1.0.188", optional = true, features = ["derive"] } smallvec = { version = "1", default-features = false } snafu = { version = "0.7", optional = true } stream-cancel = { version = "0.8.1", default-features = false } -tokio = { version = "1.32.0", default-features = false, features = ["macros", "time"] } +tokio = { version = "1.33.0", default-features = false, features = ["macros", "time"] } tracing = { version = "0.1.34", default-features = false } vrl.workspace = true vector-config = { path = "../vector-config" } @@ -69,6 +69,6 @@ vector-config-macros = { path = "../vector-config-macros" } [dev-dependencies] futures = { version = "0.3.28", default-features = false, features = ["async-await", "std"] } -tokio = { version = "1.32.0", default-features = false, features = ["rt", "time"] } +tokio = { version = "1.33.0", default-features = false, features = ["rt", "time"] } quickcheck = "1" quickcheck_macros = "1" diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 3383f270624eb..eef903cd49ca9 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -38,7 +38,7 @@ pin-project.workspace = true proptest = { version = "1.3", optional = true } prost-types = { version = "0.12", default-features = false } prost = { version = "0.12", default-features = false, features = ["std"] } -quanta = { version = "0.11.1", default-features = false } +quanta = { version = "0.12.0", default-features = false } regex = { version = "1.9.6", default-features = false, features = ["std", "perf"] } ryu = { version = "1", default-features = false } serde = { version = "1.0.188", default-features = false, features = ["derive", "rc"] } @@ -47,7 +47,7 @@ serde_with = { version = "3.3.0", default-features = false, features = ["std", " smallvec = { version = "1", default-features = false, features = ["serde", "const_generics"] } snafu = { version = "0.7.5", default-features = false } socket2 = { version = "0.5.4", default-features = false } -tokio = { version = "1.32.0", default-features = false, features = ["net"] } +tokio = { version = "1.33.0", default-features = false, features = ["net"] } tokio-openssl = { version = "0.6.3", default-features = false } tokio-stream = { version = "0.1", default-features = false, features = ["time"], optional = true } tokio-util = { version = "0.7.0", default-features = false, features = ["time"] } diff --git a/lib/vector-core/src/event/metadata.rs b/lib/vector-core/src/event/metadata.rs index a8dd2e79e6bec..bb15033a7fea1 100644 --- a/lib/vector-core/src/event/metadata.rs +++ b/lib/vector-core/src/event/metadata.rs @@ -88,25 +88,19 @@ pub struct DatadogMetricOriginMetadata { } impl DatadogMetricOriginMetadata { - /// Replaces the originating product ID with `product`. - #[must_use] - pub fn with_product(mut self, product: u32) -> Self { - self.product = Some(product); - self - } - - /// Replaces the originating category ID with `category`. - #[must_use] - pub fn with_category(mut self, category: u32) -> Self { - self.category = Some(category); - self - } - - /// Replaces the originating service ID with `service`. + /// Creates a new `DatadogMetricOriginMetadata`. + /// + /// When Vector sends out metrics containing the Origin metadata, it should do so with all of + /// the fields defined. The edge case where the Origin metadata is created within a component + /// and does not initially contain all of the metadata fields, is in the `log_to_metric` + /// transform. #[must_use] - pub fn with_service(mut self, service: u32) -> Self { - self.service = Some(service); - self + pub fn new(product: Option, category: Option, service: Option) -> Self { + Self { + product, + category, + service, + } } /// Returns the originating product ID, if it exists. diff --git a/lib/vector-core/src/event/metric/value.rs b/lib/vector-core/src/event/metric/value.rs index 6e27e61f3848d..a260f72e8fa34 100644 --- a/lib/vector-core/src/event/metric/value.rs +++ b/lib/vector-core/src/event/metric/value.rs @@ -1,12 +1,19 @@ use core::fmt; use std::collections::BTreeSet; +use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; + use vector_common::byte_size_of::ByteSizeOf; use vector_config::configurable_component; -use super::{samples_to_buckets, write_list, write_word}; use crate::{float_eq, metrics::AgentDDSketch}; +use super::{samples_to_buckets, write_list, write_word}; + +const INFINITY: &str = "inf"; +const NEG_INFINITY: &str = "-inf"; +const NAN: &str = "NaN"; + /// Metric value. #[configurable_component] #[derive(Clone, Debug)] @@ -597,14 +604,62 @@ impl ByteSizeOf for Sample { } } +/// Custom serialization function which converts special `f64` values to strings. +/// Non-special values are serialized as numbers. +#[allow(clippy::trivially_copy_pass_by_ref)] +fn serialize_f64(value: &f64, serializer: S) -> Result +where + S: Serializer, +{ + if value.is_infinite() { + serializer.serialize_str(if *value > 0.0 { INFINITY } else { NEG_INFINITY }) + } else if value.is_nan() { + serializer.serialize_str(NAN) + } else { + serializer.serialize_f64(*value) + } +} + +/// Custom deserialization function for handling special f64 values. +fn deserialize_f64<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + struct UpperLimitVisitor; + + impl<'de> de::Visitor<'de> for UpperLimitVisitor { + type Value = f64; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a number or a special string value") + } + + fn visit_f64(self, value: f64) -> Result { + Ok(value) + } + + fn visit_str(self, value: &str) -> Result { + match value { + NAN => Ok(f64::NAN), + INFINITY => Ok(f64::INFINITY), + NEG_INFINITY => Ok(f64::NEG_INFINITY), + _ => Err(E::custom("unsupported string value")), + } + } + } + + deserializer.deserialize_any(UpperLimitVisitor) +} + /// A histogram bucket. /// /// Histogram buckets represent the `count` of observations where the value of the observations does /// not exceed the specified `upper_limit`. -#[configurable_component] -#[derive(Clone, Copy, Debug)] +#[configurable_component(no_deser, no_ser)] +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] pub struct Bucket { /// The upper limit of values in the bucket. + #[serde(serialize_with = "serialize_f64", deserialize_with = "deserialize_f64")] pub upper_limit: f64, /// The number of values tracked in this bucket. diff --git a/lib/vector-core/src/event/proto/metadata.rs b/lib/vector-core/src/event/proto/metadata.rs index 6b7ed014ef6ed..47eac72e45576 100644 --- a/lib/vector-core/src/event/proto/metadata.rs +++ b/lib/vector-core/src/event/proto/metadata.rs @@ -38,19 +38,11 @@ impl From for super::proto_types::DatadogOriginMeta impl From for DatadogMetricOriginMetadata { fn from(value: super::proto_types::DatadogOriginMetadata) -> Self { - let mut metadata = Self::default(); - - if let Some(product) = value.origin_product { - metadata = metadata.with_product(product); - } - if let Some(category) = value.origin_category { - metadata = metadata.with_category(category); - } - if let Some(service) = value.origin_service { - metadata = metadata.with_service(service); - } - - metadata + Self::new( + value.origin_product, + value.origin_category, + value.origin_service, + ) } } diff --git a/lib/vector-vrl/tests/Cargo.toml b/lib/vector-vrl/tests/Cargo.toml index 672a60522618c..73e6d9e5df72f 100644 --- a/lib/vector-vrl/tests/Cargo.toml +++ b/lib/vector-vrl/tests/Cargo.toml @@ -7,7 +7,7 @@ publish = false [dependencies] enrichment = { path = "../../enrichment" } -vrl.workspace = true +vrl = { version = "0.7.0", features = ["test", "test_framework"] } vector-vrl-functions = { path = "../../vector-vrl/functions" } ansi_term = "0.12" diff --git a/regression/cases/otlp_http_to_blackhole/experiment.yaml b/regression/cases/otlp_http_to_blackhole/experiment.yaml new file mode 100644 index 0000000000000..dd1d95b86d9b5 --- /dev/null +++ b/regression/cases/otlp_http_to_blackhole/experiment.yaml @@ -0,0 +1,2 @@ +optimization_goal: ingress_throughput +erratic: true diff --git a/regression/cases/syslog_regex_logs2metric_ddmetrics/vector/vector.toml b/regression/cases/syslog_regex_logs2metric_ddmetrics/vector/vector.toml index b8a0cb4fbf558..b04171757296c 100644 --- a/regression/cases/syslog_regex_logs2metric_ddmetrics/vector/vector.toml +++ b/regression/cases/syslog_regex_logs2metric_ddmetrics/vector/vector.toml @@ -27,7 +27,7 @@ type = "log_to_metric" inputs = ["remap"] [[transforms.log2metric.metrics]] - type = "gauge" + type = "counter" field = "procid" tags.hostname = "{{ hostname }}" tags.facility = "{{ facility }}" diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index 6ef0e034346f9..ff01e4b4d5f69 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -18,7 +18,7 @@ if ! cargo deny --version >& /dev/null ; then rustup run stable cargo install cargo-deny --force --locked fi if ! dd-rust-license-tool --help >& /dev/null ; then - rustup run stable cargo install dd-rust-license-tool --version 1.0.1 --force --locked + rustup run stable cargo install dd-rust-license-tool --version 1.0.2 --force --locked fi # Currently fixing this to version 0.30 since version 0.31 has introduced diff --git a/scripts/integration/datadog-metrics/test.yaml b/scripts/integration/datadog-metrics/test.yaml index 237008a0d2551..a45e55a9797f2 100644 --- a/scripts/integration/datadog-metrics/test.yaml +++ b/scripts/integration/datadog-metrics/test.yaml @@ -1,7 +1,7 @@ features: - datadog-metrics-integration-tests -test_filter: '::datadog::metrics::' +test_filter: '::datadog::metrics::integration_tests' runner: env: diff --git a/scripts/integration/datadog-traces/test.yaml b/scripts/integration/datadog-traces/test.yaml index 31c4c0f97ef11..e518c8d3c1d93 100644 --- a/scripts/integration/datadog-traces/test.yaml +++ b/scripts/integration/datadog-traces/test.yaml @@ -19,6 +19,6 @@ matrix: paths: - "src/common/datadog.rs" - "src/internal_events/datadog_*" -- "src/sinks/datadog/**" +- "src/sinks/datadog/traces/**" - "src/sinks/util/**" - "scripts/integration/datadog-traces/**" diff --git a/src/enrichment_tables/geoip.rs b/src/enrichment_tables/geoip.rs index 3a46bf17d8fac..c9289c2c2198e 100644 --- a/src/enrichment_tables/geoip.rs +++ b/src/enrichment_tables/geoip.rs @@ -6,14 +6,16 @@ //! [geolite]: https://dev.maxmind.com/geoip/geoip2/geolite2/#Download_Access use std::{collections::BTreeMap, fs, net::IpAddr, sync::Arc, time::SystemTime}; -use enrichment::{Case, Condition, IndexHandle, Table}; use maxminddb::{ geoip2::{City, ConnectionType, Isp}, MaxMindDBError, Reader, }; -use vector_config::configurable_component; +use ordered_float::NotNan; use vrl::value::Value; +use enrichment::{Case, Condition, IndexHandle, Table}; +use vector_config::configurable_component; + use crate::config::{EnrichmentTableConfig, GenerateConfig}; // MaxMind GeoIP database files have a type field we can use to recognize specific @@ -180,10 +182,19 @@ impl Geoip { let location = data.location.as_ref(); add_field!("timezone", location.and_then(|location| location.time_zone)); - add_field!("latitude", location.and_then(|location| location.latitude)); + add_field!( + "latitude", + location + .and_then(|location| location.latitude) + .map(|latitude| Value::Float( + NotNan::new(latitude).expect("latitude cannot be Nan") + )) + ); add_field!( "longitude", - location.and_then(|location| location.longitude) + location + .and_then(|location| location.longitude) + .map(|longitude| NotNan::new(longitude).expect("longitude cannot be Nan")) ); add_field!( "metro_code", diff --git a/src/lib.rs b/src/lib.rs index a6568cf3523a8..b133205c844b9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -223,7 +223,10 @@ where T: Send + 'static, { #[cfg(tokio_unstable)] - return tokio::task::Builder::new().name(_name).spawn(task); + return tokio::task::Builder::new() + .name(_name) + .spawn(task) + .expect("tokio task should spawn"); #[cfg(not(tokio_unstable))] tokio::spawn(task) diff --git a/src/sinks/datadog/metrics/encoder.rs b/src/sinks/datadog/metrics/encoder.rs index 9f3f45dbdaf49..f399b24c27caa 100644 --- a/src/sinks/datadog/metrics/encoder.rs +++ b/src/sinks/datadog/metrics/encoder.rs @@ -415,12 +415,25 @@ fn generate_sketch_metadata( origin_product_value: u32, ) -> Option { generate_origin_metadata(maybe_pass_through, maybe_source_type, origin_product_value).map( - |origin| ddmetric_proto::Metadata { - origin: Some(ddmetric_proto::Origin { - origin_product: origin.product().expect("OriginProduct should be set"), - origin_category: origin.category().expect("OriginCategory should be set"), - origin_service: origin.service().expect("OriginService should be set"), - }), + |origin| { + if origin.product().is_none() + || origin.category().is_none() + || origin.service().is_none() + { + warn!( + message = "Generated sketch origin metadata should have each field set.", + product = origin.product(), + category = origin.category(), + service = origin.service() + ); + } + ddmetric_proto::Metadata { + origin: Some(ddmetric_proto::Origin { + origin_product: origin.product().unwrap_or_default(), + origin_category: origin.category().unwrap_or_default(), + origin_service: origin.service().unwrap_or_default(), + }), + } }, ) } @@ -626,12 +639,11 @@ fn generate_origin_metadata( // - `log_to_metric` transform set the OriginService in the EventMetadata when it creates // the new metric. if let Some(pass_through) = maybe_pass_through { - Some( - DatadogMetricOriginMetadata::default() - .with_product(pass_through.product().unwrap_or(origin_product_value)) - .with_category(pass_through.category().unwrap_or(ORIGIN_CATEGORY_VALUE)) - .with_service(pass_through.service().unwrap_or(no_value)), - ) + Some(DatadogMetricOriginMetadata::new( + pass_through.product().or(Some(origin_product_value)), + pass_through.category().or(Some(ORIGIN_CATEGORY_VALUE)), + pass_through.service().or(Some(no_value)), + )) // No metadata has been set upstream } else { @@ -640,10 +652,11 @@ fn generate_origin_metadata( // In order to preserve consistent behavior, we intentionally don't set origin metadata // for the case where the Datadog Agent did not set it. source_type_to_service(source_type).map(|origin_service_value| { - DatadogMetricOriginMetadata::default() - .with_product(origin_product_value) - .with_category(ORIGIN_CATEGORY_VALUE) - .with_service(origin_service_value) + DatadogMetricOriginMetadata::new( + Some(origin_product_value), + Some(ORIGIN_CATEGORY_VALUE), + Some(origin_service_value), + ) }) }) } @@ -1073,10 +1086,7 @@ mod tests { let service = 9; let event_metadata = EventMetadata::default().with_origin_metadata( - DatadogMetricOriginMetadata::default() - .with_product(product) - .with_category(category) - .with_service(service), + DatadogMetricOriginMetadata::new(Some(product), Some(category), Some(service)), ); let counter = get_simple_counter_with_metadata(event_metadata); diff --git a/src/sinks/datadog/metrics/integration_tests.rs b/src/sinks/datadog/metrics/integration_tests.rs index 458cc5be987f3..99645780b65ca 100644 --- a/src/sinks/datadog/metrics/integration_tests.rs +++ b/src/sinks/datadog/metrics/integration_tests.rs @@ -1,4 +1,5 @@ use bytes::Bytes; +use chrono::{SubsecRound, Utc}; use flate2::read::ZlibDecoder; use futures::{channel::mpsc::Receiver, stream, StreamExt}; use hyper::StatusCode; @@ -22,28 +23,25 @@ use crate::{ }, }; -enum ApiStatus { - OK, - // Forbidden, -} +fn generate_metric_events() -> Vec { + let timestamp = Utc::now().trunc_subsecs(3); + let events: Vec<_> = (0..10) + .map(|index| { + let ts = timestamp + (std::time::Duration::from_secs(2) * index); + Event::Metric( + Metric::new( + format!("counter_{}", thread_rng().gen::()), + MetricKind::Incremental, + MetricValue::Counter { + value: index as f64, + }, + ) + .with_timestamp(Some(ts)), + ) + }) + .collect(); -fn test_server( - addr: std::net::SocketAddr, - api_status: ApiStatus, -) -> ( - futures::channel::mpsc::Receiver<(http::request::Parts, Bytes)>, - stream_cancel::Trigger, - impl std::future::Future>, -) { - let status = match api_status { - ApiStatus::OK => StatusCode::OK, - // ApiStatus::Forbidden => StatusCode::FORBIDDEN, - }; - - // NOTE: we pass `Trigger` out to the caller even though this suite never - // uses it as it's being dropped cancels the stream machinery here, - // indicating failures that might not be valid. - build_test_server_status(addr, status) + events } /// Starts a test sink with random metrics running into it @@ -55,10 +53,7 @@ fn test_server( /// Testers may set `http_status` and `batch_status`. The first controls what /// status code faked HTTP responses will have, the second acts as a check on /// the `Receiver`'s status before being returned to the caller. -async fn start_test( - api_status: ApiStatus, - batch_status: BatchStatus, -) -> (Vec, Receiver<(http::request::Parts, Bytes)>) { +async fn start_test() -> (Vec, Receiver<(http::request::Parts, Bytes)>) { let config = indoc! {r#" default_api_key = "atoken" default_namespace = "foo" @@ -73,25 +68,18 @@ async fn start_test( let (sink, _) = config.build(cx).await.unwrap(); - let (rx, _trigger, server) = test_server(addr, api_status); + let (rx, _trigger, server) = build_test_server_status(addr, StatusCode::OK); tokio::spawn(server); - let (batch, receiver) = BatchNotifier::new_with_receiver(); - let events: Vec<_> = (0..10) - .map(|index| { - Event::Metric(Metric::new( - format!("counter_{}", thread_rng().gen::()), - MetricKind::Absolute, - MetricValue::Counter { - value: index as f64, - }, - )) - }) - .collect(); + let (batch, mut receiver) = BatchNotifier::new_with_receiver(); + + let events = generate_metric_events(); + let stream = map_event_batch_stream(stream::iter(events.clone()), Some(batch)); sink.run(stream).await.unwrap(); - assert_eq!(receiver.await, batch_status); + + assert_eq!(receiver.try_recv(), Ok(BatchStatus::Delivered)); (events, rx) } @@ -110,68 +98,96 @@ fn decompress_payload(payload: Vec) -> std::io::Result> { /// were delivered and then asserts that every message is able to be /// deserialized. async fn smoke() { - let (expected, rx) = start_test(ApiStatus::OK, BatchStatus::Delivered).await; + let (expected, rx) = start_test().await; let output = rx.take(expected.len()).collect::>().await; - for val in output.iter() { - assert_eq!( - val.0.headers.get("Content-Type").unwrap(), - "application/json" - ); - assert_eq!(val.0.headers.get("DD-API-KEY").unwrap(), "atoken"); - assert!(val.0.headers.contains_key("DD-Agent-Payload")); - - let compressed_payload = val.1.to_vec(); - let payload = decompress_payload(compressed_payload).unwrap(); - let payload = std::str::from_utf8(&payload).unwrap(); - let payload: serde_json::Value = serde_json::from_str(payload).unwrap(); - - let series = payload - .as_object() - .unwrap() - .get("series") - .unwrap() - .as_array() - .unwrap(); - assert!(!series.is_empty()); - - // check metrics are sorted by name, which helps HTTP compression - let metric_names: Vec = series - .iter() - .map(|value| { - value - .as_object() - .unwrap() - .get("metric") - .unwrap() - .as_str() - .unwrap() - .to_string() - }) - .collect(); - let mut sorted_names = metric_names.clone(); - sorted_names.sort(); - assert_eq!(metric_names, sorted_names); - - let entry = series.first().unwrap().as_object().unwrap(); - assert_eq!( - entry.get("metric").unwrap().as_str().unwrap(), - "foo.counter" - ); - assert_eq!(entry.get("type").unwrap().as_str().unwrap(), "count"); - let points = entry - .get("points") - .unwrap() - .as_array() - .unwrap() - .first() - .unwrap() - .as_array() - .unwrap(); - assert_eq!(points.len(), 2); - assert_eq!(points.get(1).unwrap().as_f64().unwrap(), 1.0); - } + assert!(output.len() == 1, "Should have received a response"); + + let val = output.first().unwrap(); + + assert_eq!( + val.0.headers.get("Content-Type").unwrap(), + "application/json" + ); + assert_eq!(val.0.headers.get("DD-API-KEY").unwrap(), "atoken"); + assert!(val.0.headers.contains_key("DD-Agent-Payload")); + + let compressed_payload = val.1.to_vec(); + let payload = decompress_payload(compressed_payload).unwrap(); + let payload = std::str::from_utf8(&payload).unwrap(); + let payload: serde_json::Value = serde_json::from_str(payload).unwrap(); + + let series = payload + .as_object() + .unwrap() + .get("series") + .unwrap() + .as_array() + .unwrap(); + assert!(!series.is_empty()); + + // check metrics are sorted by name, which helps HTTP compression + let metric_names: Vec = series + .iter() + .map(|value| { + value + .as_object() + .unwrap() + .get("metric") + .unwrap() + .as_str() + .unwrap() + .to_string() + }) + .collect(); + let mut sorted_names = metric_names.clone(); + sorted_names.sort(); + assert_eq!(metric_names, sorted_names); + + let entry = series.first().unwrap().as_object().unwrap(); + assert!(entry + .get("metric") + .unwrap() + .as_str() + .unwrap() + .starts_with("foo.counter_"),); + assert_eq!(entry.get("type").unwrap().as_str().unwrap(), "count"); + let points = entry + .get("points") + .unwrap() + .as_array() + .unwrap() + .first() + .unwrap() + .as_array() + .unwrap(); + assert_eq!(points.len(), 2); + + // validate that all values were received + let all_values: f64 = series + .iter() + .map(|entry| { + entry + .as_object() + .unwrap() + .get("points") + .unwrap() + .as_array() + .unwrap() + .first() + .unwrap() + .as_array() + .unwrap() + .get(1) + .unwrap() + .as_f64() + .unwrap() + }) + .sum(); + + // the input values are [0..10) + assert_eq!(all_values, 45.0); } async fn run_sink() { @@ -186,17 +202,9 @@ async fn run_sink() { let (sink, _) = config.build(cx).await.unwrap(); let (batch, receiver) = BatchNotifier::new_with_receiver(); - let events: Vec<_> = (0..10) - .map(|index| { - Event::Metric(Metric::new( - "counter", - MetricKind::Absolute, - MetricValue::Counter { - value: index as f64, - }, - )) - }) - .collect(); + + let events = generate_metric_events(); + let stream = map_event_batch_stream(stream::iter(events.clone()), Some(batch)); sink.run(stream).await.unwrap(); diff --git a/src/sinks/datadog/metrics/sink.rs b/src/sinks/datadog/metrics/sink.rs index 5ceefc3c487d2..fccfe040fdd95 100644 --- a/src/sinks/datadog/metrics/sink.rs +++ b/src/sinks/datadog/metrics/sink.rs @@ -8,7 +8,6 @@ use futures_util::{ StreamExt, }; use tower::Service; -use vector_common::finalization::EventFinalizers; use vector_core::{ event::{Event, Metric, MetricValue}, partition::Partitioner, @@ -23,8 +22,8 @@ use super::{ use crate::{ internal_events::DatadogMetricsEncodingError, sinks::util::{ - buffer::metrics::sort::sort_for_compression, buffer::metrics::{AggregatedSummarySplitter, MetricSplitter}, + request_builder::default_request_builder_concurrency_limit, SinkBuilderExt, }, }; @@ -103,15 +102,18 @@ where // Aggregate counters with identical timestamps, otherwise identical counters (same // series and same timestamp, when rounded to whole seconds) will be dropped in a // last-write-wins situation when they hit the DD metrics intake. - .map(|((api_key, endpoint), metrics)| { - let collapsed_metrics = collapse_counters_by_series_and_timestamp(metrics); - ((api_key, endpoint), collapsed_metrics) - }) - // Sort metrics by name, which significantly improves HTTP compression. - .map(|((api_key, endpoint), mut metrics)| { - sort_for_compression(&mut metrics); - ((api_key, endpoint), metrics) - }) + // + // This also sorts metrics by name, which significantly improves HTTP compression. + .concurrent_map( + default_request_builder_concurrency_limit(), + |((api_key, endpoint), metrics)| { + Box::pin(async move { + let collapsed_metrics = + sort_and_collapse_counters_by_series_and_timestamp(metrics); + ((api_key, endpoint), collapsed_metrics) + }) + }, + ) // We build our requests "incrementally", which means that for a single batch of metrics, we might generate // N requests to send them all, as Datadog has API-level limits on payload size, so we keep adding metrics // to a request until we reach the limit, and then create a new request, and so on and so forth, until all @@ -159,142 +161,98 @@ where } } -fn collapse_counters_by_series_and_timestamp(mut metrics: Vec) -> Vec { - // NOTE: Astute observers may recognize that this behavior could also be achieved by using - // `Vec::dedup_by`, but the clincher is that `dedup_by` requires a sorted vector to begin with. - // - // This function is designed to collapse duplicate counters even if the metrics are unsorted, - // which leads to a measurable boost in performance, being nearly 35% faster than `dedup_by` - // when the inputs are sorted, and up to 50% faster when the inputs are unsorted. - // - // These numbers are based on sorting a newtype wrapper around the metric instead of the metric - // itself, which does involve allocating a string in our tests. _However_, sorting the `Metric` - // directly is not possible without a customized `PartialOrd` implementation, as some of the - // nested fields containing `f64` values makes it underivable, and I'm not 100% sure that we - // could/would want to have a narrowly-focused impl of `PartialOrd` on `Metric` to fit this use - // case (metric type -> metric name -> metric timestamp, nothing else) vs being able to sort - // metrics by name first, etc. Then there's the potential issue of the reordering of fields - // changing the ordering behavior of `Metric`... and it just felt easier to write this tailored - // algorithm for the use case at hand. - let mut idx = 0; +/// Collapses counters by series and timestamp, leaving all other metrics unmodified. +/// The return value is sorted by metric series, which is desirable for compression. A sorted vector +/// tends to compress better than a random ordering by 2-3x (JSON encoded, deflate algorithm). +/// +/// Note that the time complexity of this function is O(n log n) and the space complexity is O(1). +/// If needed, we can trade space for time by using a HashMap, which would be O(n) time and O(n) space. +fn sort_and_collapse_counters_by_series_and_timestamp(mut metrics: Vec) -> Vec { let now_ts = Utc::now().timestamp(); - // For each metric, see if it's a counter. If so, we check the rest of the metrics - // _after_ it to see if they share the same series _and_ timestamp, when converted - // to a Unix timestamp. If they match, we take that counter's value and merge it - // with our "current" counter metric, and then drop the secondary one from the - // vector. - // - // For any non-counter, we simply ignore it and leave it as-is. - while idx < metrics.len() { - let curr_idx = idx; - let counter_ts = match metrics[curr_idx].value() { - MetricValue::Counter { .. } => metrics[curr_idx] - .data() - .timestamp() - .map(|dt| dt.timestamp()) - .unwrap_or(now_ts), - // If it's not a counter, we can skip it. - _ => { - idx += 1; - continue; - } - }; - - let mut accumulated_value = 0.0; - let mut accumulated_finalizers = EventFinalizers::default(); - - // Now go through each metric _after_ the current one to see if it matches the - // current metric: is a counter, with the same name and timestamp. If it is, we - // accumulate its value and then remove it. - // - // Otherwise, we skip it. - let mut is_disjoint = false; - let mut had_match = false; - let mut inner_idx = curr_idx + 1; - while inner_idx < metrics.len() { - let mut should_advance = true; - if let MetricValue::Counter { value } = metrics[inner_idx].value() { - let other_counter_ts = metrics[inner_idx] - .data() - .timestamp() - .map(|dt| dt.timestamp()) - .unwrap_or(now_ts); - if metrics[curr_idx].series() == metrics[inner_idx].series() - && counter_ts == other_counter_ts - { - had_match = true; - - // Collapse this counter by accumulating its value, and its - // finalizers, and removing it from the original vector of metrics. - accumulated_value += *value; - - let mut old_metric = metrics.swap_remove(inner_idx); - accumulated_finalizers.merge(old_metric.metadata_mut().take_finalizers()); - should_advance = false; - } else { - // We hit a counter that _doesn't_ match, but we can't just skip - // it because we also need to evaluate it against all the - // counters that come after it, so we only increment the index - // for this inner loop. - // - // As well, we mark ourselves to stop incrementing the outer - // index if we find more counters to accumulate, because we've - // hit a disjoint counter here. While we may be continuing to - // shrink the count of remaining metrics from accumulating, - // we have to ensure this counter we just visited is visited by - // the outer loop. - is_disjoint = true; - } - } - - if should_advance { - inner_idx += 1; - - if !is_disjoint { - idx += 1; - } - } + // Sort by series and timestamp which is required for the below dedupe to behave as desired. + // This also tends to compress better than a random ordering by 2-3x (JSON encoded, deflate algorithm). + // Note that `sort_unstable_by_key` would be simpler but results in lifetime errors without cloning. + metrics.sort_unstable_by(|a, b| { + ( + a.value().as_name(), + a.series(), + a.timestamp().map(|dt| dt.timestamp()).unwrap_or(now_ts), + ) + .cmp(&( + a.value().as_name(), + b.series(), + b.timestamp().map(|dt| dt.timestamp()).unwrap_or(now_ts), + )) + }); + + // Aggregate counters that share the same series and timestamp. + // While `coalesce` is semantically more fitting here than `dedupe_by`, we opt for the latter because + // they share the same functionality and `dedupe_by`'s implementation is more optimized, doing the + // operation in place. + metrics.dedup_by(|left, right| { + if left.series() != right.series() { + return false; } - // If we had matches during the accumulator phase, update our original counter. - if had_match { - let metric = metrics.get_mut(curr_idx).expect("current index must exist"); - match metric.value_mut() { - MetricValue::Counter { value } => { - *value += accumulated_value; - metric - .metadata_mut() - .merge_finalizers(accumulated_finalizers); - } - _ => unreachable!("current index must represent a counter"), - } + let left_ts = left.timestamp().map(|dt| dt.timestamp()).unwrap_or(now_ts); + let right_ts = right.timestamp().map(|dt| dt.timestamp()).unwrap_or(now_ts); + if left_ts != right_ts { + return false; } - idx += 1; - } + // Only aggregate counters. All other types can be skipped. + if let ( + MetricValue::Counter { value: left_value }, + MetricValue::Counter { value: right_value }, + ) = (left.value(), right.value_mut()) + { + // NOTE: The docs for `dedup_by` specify that if `left`/`right` are equal, then + // `left` is the element that gets removed. + *right_value += left_value; + right + .metadata_mut() + .merge_finalizers(left.metadata_mut().take_finalizers()); + + true + } else { + false + } + }); metrics } #[cfg(test)] mod tests { + use std::{collections::HashSet, time::Duration}; + use chrono::{DateTime, Utc}; use proptest::prelude::*; - use vector_core::event::{Metric, MetricKind, MetricValue}; + use vector_core::{ + event::{Metric, MetricKind, MetricValue}, + metric_tags, + }; - use super::collapse_counters_by_series_and_timestamp; + use super::sort_and_collapse_counters_by_series_and_timestamp; fn arb_collapsible_metrics() -> impl Strategy> { let ts = Utc::now(); any::>().prop_map(move |values| { + let mut unique_metrics = HashSet::new(); values .into_iter() .map(|(id, value)| { let name = format!("{}-{}", value.as_name(), id); Metric::new(name, MetricKind::Incremental, value).with_timestamp(Some(ts)) }) + // Filter out duplicates other than counters. We do this to prevent false positives. False positives would occur + // because we don't collapse other metric types and we can't sort metrics by their values. + .filter(|metric| { + matches!(metric.value(), MetricValue::Counter { .. }) + || unique_metrics.insert(metric.series().clone()) + }) .collect() }) } @@ -315,7 +273,7 @@ mod tests { fn collapse_no_metrics() { let input = Vec::new(); let expected = input.clone(); - let actual = collapse_counters_by_series_and_timestamp(input); + let actual = sort_and_collapse_counters_by_series_and_timestamp(input); assert_eq!(expected, actual); } @@ -324,7 +282,7 @@ mod tests { fn collapse_single_metric() { let input = vec![create_counter("basic", 42.0)]; let expected = input.clone(); - let actual = collapse_counters_by_series_and_timestamp(input); + let actual = sort_and_collapse_counters_by_series_and_timestamp(input); assert_eq!(expected, actual); } @@ -333,7 +291,7 @@ mod tests { fn collapse_identical_metrics_gauge() { let input = vec![create_gauge("basic", 42.0), create_gauge("basic", 42.0)]; let expected = input.clone(); - let actual = collapse_counters_by_series_and_timestamp(input); + let actual = sort_and_collapse_counters_by_series_and_timestamp(input); assert_eq!(expected, actual); @@ -348,7 +306,7 @@ mod tests { create_gauge("basic", gauge_value), ]; let expected = input.clone(); - let actual = collapse_counters_by_series_and_timestamp(input); + let actual = sort_and_collapse_counters_by_series_and_timestamp(input); assert_eq!(expected, actual); } @@ -368,7 +326,91 @@ mod tests { let expected_counter_value = input.len() as f64 * counter_value; let expected = vec![create_counter("basic", expected_counter_value)]; - let actual = collapse_counters_by_series_and_timestamp(input); + let actual = sort_and_collapse_counters_by_series_and_timestamp(input); + + assert_eq!(expected, actual); + } + + #[test] + fn collapse_identical_metrics_counter_unsorted() { + let gauge_value = 1.0; + let counter_value = 42.0; + let input = vec![ + create_gauge("gauge", gauge_value), + create_counter("basic", counter_value), + create_counter("basic", counter_value), + create_counter("basic", counter_value), + create_gauge("gauge", gauge_value), + create_counter("basic", counter_value), + create_counter("basic", counter_value), + create_counter("basic", counter_value), + create_counter("basic", counter_value), + ]; + + let expected_counter_value = (input.len() - 2) as f64 * counter_value; + let expected = vec![ + create_counter("basic", expected_counter_value), + create_gauge("gauge", gauge_value), + create_gauge("gauge", gauge_value), + ]; + let actual = sort_and_collapse_counters_by_series_and_timestamp(input); + + assert_eq!(expected, actual); + } + + #[test] + fn collapse_identical_metrics_multiple_timestamps() { + let ts_1 = Utc::now() - Duration::from_secs(5); + let ts_2 = ts_1 - Duration::from_secs(5); + let counter_value = 42.0; + let input = vec![ + create_counter("basic", counter_value), + create_counter("basic", counter_value).with_timestamp(Some(ts_1)), + create_counter("basic", counter_value).with_timestamp(Some(ts_2)), + create_counter("basic", counter_value), + create_counter("basic", counter_value).with_timestamp(Some(ts_2)), + create_counter("basic", counter_value).with_timestamp(Some(ts_1)), + create_counter("basic", counter_value), + ]; + + let expected = vec![ + create_counter("basic", counter_value * 2.).with_timestamp(Some(ts_2)), + create_counter("basic", counter_value * 2.).with_timestamp(Some(ts_1)), + create_counter("basic", counter_value * 3.), + ]; + let actual = sort_and_collapse_counters_by_series_and_timestamp(input); + + assert_eq!(expected, actual); + } + + #[test] + fn collapse_identical_metrics_with_tags() { + let counter_value = 42.0; + let input = vec![ + create_counter("basic", counter_value).with_tags(Some(metric_tags!("a" => "a"))), + create_counter("basic", counter_value).with_tags(Some(metric_tags!( + "a" => "a", + "b" => "b", + ))), + create_counter("basic", counter_value), + create_counter("basic", counter_value).with_tags(Some(metric_tags!( + "b" => "b", + "a" => "a", + ))), + create_counter("basic", counter_value), + create_counter("basic", counter_value), + create_counter("basic", counter_value).with_tags(Some(metric_tags!("a" => "a"))), + ]; + + let expected = vec![ + create_counter("basic", counter_value * 3.), + create_counter("basic", counter_value * 2.).with_tags(Some(metric_tags!("a" => "a"))), + create_counter("basic", counter_value * 2.).with_tags(Some(metric_tags!( + "a" => "a", + "b" => "b", + ))), + ]; + let actual = sort_and_collapse_counters_by_series_and_timestamp(input); assert_eq!(expected, actual); } @@ -419,8 +461,7 @@ mod tests { expected_output.sort_by_cached_key(MetricCollapseSort::from_metric); expected_output.dedup_by(collapse_dedup_fn); - let mut actual_output = collapse_counters_by_series_and_timestamp(input); - actual_output.sort_by_cached_key(MetricCollapseSort::from_metric); + let actual_output = sort_and_collapse_counters_by_series_and_timestamp(input); prop_assert_eq!(expected_output, actual_output); } diff --git a/src/sinks/util/buffer/metrics/mod.rs b/src/sinks/util/buffer/metrics/mod.rs index 877cdc9c4bcc1..e66b3c2364140 100644 --- a/src/sinks/util/buffer/metrics/mod.rs +++ b/src/sinks/util/buffer/metrics/mod.rs @@ -1,5 +1,3 @@ -pub mod sort; - use std::cmp::Ordering; use vector_core::event::metric::{Metric, MetricValue, Sample}; diff --git a/src/sinks/util/buffer/metrics/sort.rs b/src/sinks/util/buffer/metrics/sort.rs deleted file mode 100644 index feaa563493789..0000000000000 --- a/src/sinks/util/buffer/metrics/sort.rs +++ /dev/null @@ -1,67 +0,0 @@ -use crate::event::Metric; - -/// Sorts metrics in an order that is likely to achieve good compression. -pub fn sort_for_compression(metrics: &mut [Metric]) { - // This just sorts by series today. This tends to compress better than a random ordering by - // 2-3x (JSON encoded, deflate algorithm) - metrics.sort_unstable_by(|a, b| a.series().cmp(b.series())) -} - -#[cfg(test)] -mod test { - use crate::event::MetricValue; - use rand::prelude::SliceRandom; - use rand::thread_rng; - use vector_core::event::{Metric, MetricKind}; - use vector_core::metric_tags; - - // This just ensures the sorting does not change. `sort_for_compression` relies on - // the default `PartialOrd` on `MetricSeries`. - #[test] - fn test_compression_order() { - let sorted_metrics = vec![ - Metric::new( - "metric_1", - MetricKind::Absolute, - MetricValue::Gauge { value: 0.0 }, - ), - Metric::new( - "metric_2", - MetricKind::Incremental, - MetricValue::Gauge { value: 0.0 }, - ), - Metric::new( - "metric_3", - MetricKind::Absolute, - MetricValue::Gauge { value: 0.0 }, - ) - .with_tags(Some(metric_tags!("z" => "z"))), - Metric::new( - "metric_4", - MetricKind::Absolute, - MetricValue::Gauge { value: 0.0 }, - ) - .with_tags(Some(metric_tags!("a" => "a"))), - Metric::new( - "metric_4", - MetricKind::Absolute, - MetricValue::Gauge { value: 0.0 }, - ) - .with_tags(Some(metric_tags!( - "a" => "a", - "b" => "b", - ))), - Metric::new( - "metric_4", - MetricKind::Absolute, - MetricValue::Gauge { value: 0.0 }, - ) - .with_tags(Some(metric_tags!("b" => "b"))), - ]; - - let mut rand_metrics = sorted_metrics.clone(); - rand_metrics.shuffle(&mut thread_rng()); - super::sort_for_compression(&mut rand_metrics); - assert_eq!(sorted_metrics, rand_metrics); - } -} diff --git a/src/sinks/util/builder.rs b/src/sinks/util/builder.rs index c51bf405dc4d1..617697ea84ec1 100644 --- a/src/sinks/util/builder.rs +++ b/src/sinks/util/builder.rs @@ -82,13 +82,13 @@ pub trait SinkBuilderExt: Stream { /// /// If the spawned future panics, the panic will be carried through and resumed on the task /// calling the stream. - fn concurrent_map(self, limit: Option, f: F) -> ConcurrentMap + fn concurrent_map(self, limit: NonZeroUsize, f: F) -> ConcurrentMap where Self: Sized, F: Fn(Self::Item) -> Pin + Send + 'static>> + Send + 'static, T: Send + 'static, { - ConcurrentMap::new(self, limit, f) + ConcurrentMap::new(self, Some(limit), f) } /// Constructs a [`Stream`] which transforms the input into a request suitable for sending to @@ -114,7 +114,7 @@ pub trait SinkBuilderExt: Stream { { let builder = Arc::new(builder); - self.concurrent_map(Some(limit), move |input| { + self.concurrent_map(limit, move |input| { let builder = Arc::clone(&builder); Box::pin(async move { diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index e4b146b933852..ee1c998c13ced 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -244,12 +244,11 @@ fn get_event_metadata(metadata: Option<&Metadata>) -> EventMetadata { origin.origin_category, origin.origin_service, ); - EventMetadata::default().with_origin_metadata( - DatadogMetricOriginMetadata::default() - .with_product(origin.origin_product) - .with_category(origin.origin_category) - .with_service(origin.origin_service), - ) + EventMetadata::default().with_origin_metadata(DatadogMetricOriginMetadata::new( + Some(origin.origin_product), + Some(origin.origin_category), + Some(origin.origin_service), + )) }) } diff --git a/src/sources/datadog_agent/traces.rs b/src/sources/datadog_agent/traces.rs index bd9099c13a2ab..c5daa9a4b2910 100644 --- a/src/sources/datadog_agent/traces.rs +++ b/src/sources/datadog_agent/traces.rs @@ -6,11 +6,12 @@ use futures::future; use http::StatusCode; use ordered_float::NotNan; use prost::Message; -use vector_common::internal_event::{CountByteSize, InternalEventHandle as _}; -use vector_core::EstimatedJsonEncodedSizeOf; use vrl::event_path; use warp::{filters::BoxedFilter, path, path::FullPath, reply::Response, Filter, Rejection, Reply}; +use vector_common::internal_event::{CountByteSize, InternalEventHandle as _}; +use vector_core::EstimatedJsonEncodedSizeOf; + use crate::{ event::{Event, TraceEvent, Value}, sources::{ @@ -150,8 +151,14 @@ fn handle_dd_trace_payload_v1( trace_event.insert(&source.log_schema_host_key, hostname.clone()); trace_event.insert(event_path!("env"), env.clone()); trace_event.insert(event_path!("agent_version"), agent_version.clone()); - trace_event.insert(event_path!("target_tps"), target_tps); - trace_event.insert(event_path!("error_tps"), error_tps); + trace_event.insert( + event_path!("target_tps"), + Value::Float(NotNan::new(target_tps).expect("target_tps cannot be Nan")), + ); + trace_event.insert( + event_path!("error_tps"), + Value::Float(NotNan::new(error_tps).expect("error_tps cannot be Nan")), + ); if let Some(Value::Object(span_tags)) = trace_event.get_mut(event_path!("tags")) { span_tags.extend(tags.clone()); } else { diff --git a/src/transforms/log_to_metric.rs b/src/transforms/log_to_metric.rs index 0571d715c6cc6..f6ba686e09de2 100644 --- a/src/transforms/log_to_metric.rs +++ b/src/transforms/log_to_metric.rs @@ -268,9 +268,11 @@ fn to_metric(config: &MetricConfig, event: &Event) -> Result