From 1b713aa5129b4d7351c4544d14dce75ccc00fe64 Mon Sep 17 00:00:00 2001 From: Matt Broadstone Date: Thu, 7 Nov 2024 13:55:22 -0800 Subject: [PATCH] 7.0.16 release maintenance (#28907) GitOrigin-RevId: 6170c0d97374089f5cad91aa4a03b1a690ad6276 --- .github/CODEOWNERS | 4 +- SConstruct | 20 - buildscripts/package_test.py | 24 +- buildscripts/setup_spawnhost_coredump | 17 +- buildscripts/win/mongodb.natvis | 14 +- ...kports_required_for_multiversion_tests.yml | 8 + etc/evergreen.yml | 12 +- .../variants/atlas.yml | 2 +- .../variants/misc_release.yml | 10 +- .../variants/ninja.yml | 6 +- .../variants/task_generation.yml | 4 + etc/pip/components/platform.req | 3 +- etc/system_perf.yml | 1241 +---------------- etc/system_perf_yml_components/tasks.yml | 252 ---- etc/system_perf_yml_components/variants.yml | 407 ------ .../variants/task_generation.yml | 15 + evergreen/do_jepsen_setup/install_jepsen.sh | 2 +- evergreen/prelude_venv.sh | 2 +- .../{documents.js => documents/basics.js} | 0 .../documents/subpipeline_validation.js | 215 +++ jstests/auth/speculative-auth-replset.js | 3 +- ...ternal_transactions_sharded_from_mongod.js | 14 - ...m_moveChunk_refine_collection_shard_key.js | 1 + jstests/core/hostinfo.js | 6 +- .../project/projection_with_hashed_index.js | 48 + jstests/libs/analyze_plan.js | 13 + jstests/libs/query_stats_utils.js | 4 +- .../catalog_snapshot_consistency.js | 38 + jstests/noPassthrough/lookup_profile.js | 22 + .../query_stats_agg_key_change_stream.js | 217 ++- ...agg_key_change_stream_passthrough_shard.js | 93 ++ ...erver_status_metrics_lifespan_histogram.js | 27 - .../shell_exhaust_query.js | 21 + .../list_databases_for_all_tenants.js | 66 +- .../libs/query_sampling_util.js | 5 - .../persist_sampled_diffs.js | 3 +- .../persist_sampled_read_queries.js | 3 +- .../persist_sampled_write_queries.js | 5 +- .../sharding/max_time_ms_connection_pool.js | 6 +- jstests/sharding/refresh_sessions.js | 11 +- ...eletes_not_targeting_orphaned_documents.js | 16 +- .../verify_sessions_expiration_sharded.js | 4 + site_scons/site_tools/oom_auto_retry.py | 26 +- src/mongo/bson/bsonelement.cpp | 4 +- src/mongo/bson/bsonobj.cpp | 4 +- .../db/auth/authorization_manager_impl.cpp | 1 + src/mongo/db/catalog/collection_catalog.cpp | 60 + src/mongo/db/catalog/collection_catalog.h | 23 + src/mongo/db/commands/create_indexes_cmd.cpp | 6 +- src/mongo/db/commands/generic_servers.cpp | 8 +- src/mongo/db/commands/generic_servers.idl | 1 + src/mongo/db/commands/list_collections.cpp | 4 + src/mongo/db/commands/list_databases.cpp | 27 +- .../list_databases_for_all_tenants.cpp | 25 +- .../set_cluster_parameter_invocation.cpp | 5 + src/mongo/db/db_raii.cpp | 5 + src/mongo/db/exec/projection.cpp | 11 +- src/mongo/db/exec/sbe/sbe_test.cpp | 20 + src/mongo/db/exec/sbe/values/value.cpp | 14 +- src/mongo/db/ftdc/ftdc_system_stats_linux.cpp | 8 +- .../document_source_change_stream.cpp | 8 +- .../pipeline/document_source_change_stream.h | 25 + .../document_source_change_stream.idl | 10 - ...nt_source_change_stream_add_post_image.cpp | 7 +- ...ment_source_change_stream_add_post_image.h | 8 +- ...ent_source_change_stream_add_pre_image.cpp | 8 +- ...ument_source_change_stream_add_pre_image.h | 7 +- ..._source_change_stream_check_invalidate.cpp | 5 +- ...nt_source_change_stream_check_invalidate.h | 7 +- ...ource_change_stream_check_resumability.cpp | 7 +- ..._source_change_stream_check_resumability.h | 4 +- ...ce_change_stream_check_topology_change.cpp | 2 +- ...urce_change_stream_check_topology_change.h | 7 +- ...nge_stream_ensure_resume_token_present.cpp | 2 +- ...hange_stream_ensure_resume_token_present.h | 2 +- ...e_change_stream_handle_topology_change.cpp | 4 +- ...rce_change_stream_handle_topology_change.h | 5 +- ...ument_source_change_stream_oplog_match.cpp | 17 +- ...ocument_source_change_stream_oplog_match.h | 9 +- .../document_source_change_stream_test.cpp | 206 +-- ...ocument_source_change_stream_transform.cpp | 13 +- .../document_source_change_stream_transform.h | 9 +- ...ource_change_stream_unwind_transaction.cpp | 16 +- ..._source_change_stream_unwind_transaction.h | 4 +- .../db/pipeline/document_source_lookup.cpp | 3 +- .../db/pipeline/document_source_match.cpp | 18 + src/mongo/db/pipeline/document_source_match.h | 38 + .../pipeline/document_source_union_with.cpp | 26 +- src/mongo/db/pipeline/resume_token.cpp | 30 +- .../optimizer/interval_simplify_test.cpp | 2 +- .../db/query/query_stats/query_stats.cpp | 2 +- src/mongo/db/query/sbe_plan_cache.cpp | 2 +- src/mongo/db/query/util/memory_util.cpp | 2 +- .../repl/topology_version_observer_test.cpp | 8 + .../resharding_coordinator_commit_monitor.cpp | 25 +- .../resharding_coordinator_commit_monitor.h | 5 +- ...arding_coordinator_commit_monitor_test.cpp | 1 + .../resharding_coordinator_service.cpp | 3 +- .../db/s/resharding/resharding_metrics.cpp | 4 +- .../s/resharding/resharding_metrics_test.cpp | 18 + .../resharding_server_parameters.idl | 15 + ...act_structured_encryption_data_command.cpp | 27 +- src/mongo/db/service_context.cpp | 25 +- src/mongo/db/session/SConscript | 1 + src/mongo/db/session/sessions_collection.cpp | 36 +- .../db/session/sessions_server_parameters.idl | 63 + src/mongo/db/storage/SConscript | 3 + src/mongo/db/storage/backup_block.cpp | 30 +- src/mongo/db/storage/backup_block.h | 13 +- src/mongo/db/storage/backup_block_test.cpp | 76 + src/mongo/db/storage/collection_markers.cpp | 34 +- src/mongo/db/storage/collection_markers.h | 17 +- .../execution_control/throughput_probing.cpp | 11 +- .../storage/oplog_cap_maintainer_thread.cpp | 82 +- .../db/storage/oplog_cap_maintainer_thread.h | 3 +- src/mongo/db/storage/record_store.h | 6 +- src/mongo/db/storage/storage_engine.h | 2 + .../wiredtiger_begin_transaction_block.cpp | 45 +- .../wiredtiger/wiredtiger_kv_engine.cpp | 30 +- .../storage/wiredtiger/wiredtiger_kv_engine.h | 17 + .../wiredtiger/wiredtiger_parameters.idl | 8 + .../wiredtiger/wiredtiger_record_store.cpp | 27 +- .../wiredtiger/wiredtiger_record_store.h | 4 +- .../db/storage/wiredtiger/wiredtiger_util.cpp | 8 + ...ster_parameter_synchronization_helpers.cpp | 17 +- ...luster_parameter_synchronization_helpers.h | 6 +- .../cluster_server_parameter_op_observer.cpp | 4 +- src/mongo/rpc/op_legacy_integration_test.cpp | 12 + .../s/collection_routing_info_targeter.cpp | 2 +- src/mongo/s/service_entry_point_mongos.cpp | 10 +- src/mongo/s/service_entry_point_mongos.h | 3 +- src/mongo/s/write_ops/batch_write_op.cpp | 50 +- src/mongo/s/write_ops/batch_write_op_test.cpp | 118 ++ src/mongo/shell/query.js | 1 + .../transport/asio/asio_transport_layer.cpp | 7 +- src/mongo/transport/service_executor_bm.cpp | 2 +- src/mongo/transport/session_workflow_bm.cpp | 2 +- src/mongo/transport/transport_layer.h | 2 +- src/mongo/util/exception_filter_win32.cpp | 5 +- src/mongo/util/processinfo.h | 11 +- src/mongo/util/processinfo_linux.cpp | 5 + src/mongo/util/processinfo_test.cpp | 4 +- .../mongo_sources/jscustomallocator_oom.h | 2 +- src/third_party/wiredtiger/dist/stat_data.py | 1 + src/third_party/wiredtiger/import.data | 2 +- .../wiredtiger/src/btree/bt_compact.c | 15 +- src/third_party/wiredtiger/src/include/stat.h | 1 + .../wiredtiger/src/include/wiredtiger.in | 248 ++-- .../wiredtiger/src/session/session_compact.c | 3 + src/third_party/wiredtiger/src/support/stat.c | 3 + src/third_party/wiredtiger/src/txn/txn_ckpt.c | 4 +- .../test/evergreen/setup_spawn_host.sh | 80 ++ .../wiredtiger/test/suite/test_compact12.py | 142 ++ 153 files changed, 2390 insertions(+), 2670 deletions(-) delete mode 100644 etc/system_perf_yml_components/tasks.yml delete mode 100644 etc/system_perf_yml_components/variants.yml create mode 100644 etc/system_perf_yml_components/variants/task_generation.yml rename jstests/aggregation/sources/{documents.js => documents/basics.js} (100%) create mode 100644 jstests/aggregation/sources/documents/subpipeline_validation.js create mode 100644 jstests/core/query/project/projection_with_hashed_index.js create mode 100644 jstests/noPassthrough/catalog_snapshot_consistency.js create mode 100644 jstests/noPassthrough/lookup_profile.js create mode 100644 jstests/noPassthrough/queryStats/query_stats_agg_key_change_stream_passthrough_shard.js create mode 100644 jstests/noPassthroughWithMongod/shell_exhaust_query.js create mode 100644 src/mongo/db/session/sessions_server_parameters.idl create mode 100644 src/mongo/db/storage/backup_block_test.cpp create mode 100644 src/third_party/wiredtiger/test/evergreen/setup_spawn_host.sh create mode 100644 src/third_party/wiredtiger/test/suite/test_compact12.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 704e066d84469..eb44d2a81be25 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,8 +2,10 @@ * @10gen/server-release # Exclude some test files and READMEs from the backport approvals -/jstests/**/* /etc/backports_required_for_multiversion_tests.yml /etc/*.suppressions /README.md **/README.md + +# Exclude all the tests under "jstests" directories from backport approvals +**/jstests/**/* diff --git a/SConstruct b/SConstruct index 07579349b83b2..3fbd62fe0a04e 100644 --- a/SConstruct +++ b/SConstruct @@ -2002,26 +2002,6 @@ if env.get('ENABLE_OOM_RETRY'): else: env['OOM_RETRY_ATTEMPTS'] = 10 env['OOM_RETRY_MAX_DELAY_SECONDS'] = 120 - - if env.ToolchainIs('clang', 'gcc'): - env['OOM_RETRY_MESSAGES'] = [ - ': out of memory', - 'virtual memory exhausted: Cannot allocate memory', - ': fatal error: Killed signal terminated program cc1', - # TODO: SERVER-77322 remove this non memory related ICE. - r'during IPA pass: cp.+g\+\+: internal compiler error', - 'ld terminated with signal 9', - ] - elif env.ToolchainIs('msvc'): - env['OOM_RETRY_MESSAGES'] = [ - 'LNK1102: out of memory', - 'C1060: compiler is out of heap space', - 'c1xx : fatal error C1063: INTERNAL COMPILER ERROR', - r'LNK1171: unable to load mspdbcore\.dll', - "LNK1201: error writing to program database ''", - ] - env['OOM_RETRY_RETURNCODES'] = [1102] - env.Tool('oom_auto_retry') if env.ToolchainIs('clang'): diff --git a/buildscripts/package_test.py b/buildscripts/package_test.py index b8dc5c6721753..854001694f4a4 100644 --- a/buildscripts/package_test.py +++ b/buildscripts/package_test.py @@ -90,13 +90,13 @@ 'rhel55': None, 'rhel57': None, 'rhel62': None, - 'rhel70': ('centos:7', "yum", + 'rhel70': ('registry.access.redhat.com/ubi7/ubi', "yum", frozenset(["rh-python38.x86_64", "wget", "pkgconfig", "systemd", "procps", "file"]), "/opt/rh/rh-python38/root/usr/bin/python3"), - 'rhel71': ('centos:7', "yum", + 'rhel71': ('registry.access.redhat.com/ubi7/ubi', "yum", frozenset(["rh-python38.x86_64", "wget", "pkgconfig", "systemd", "procps", "file"]), "/opt/rh/rh-python38/root/usr/bin/python3"), - 'rhel72': ('centos:7', "yum", + 'rhel72': ('registry.access.redhat.com/ubi7/ubi', "yum", frozenset(["rh-python38.x86_64", "wget", "pkgconfig", "systemd", "procps", "file"]), "/opt/rh/rh-python38/root/usr/bin/python3"), 'rhel8': ('almalinux:8', "yum", @@ -227,12 +227,6 @@ def run_test(test: Test, client: DockerClient) -> Tuple[Test, Result]: commands: List[str] = ["export PYTHONIOENCODING=UTF-8"] if test.os_name.startswith('rhel'): - if test.os_name.startswith('rhel7'): - # RHEL 7 needs the SCL installed for Python 3 - commands += [ - "yum -y install centos-release-scl", - "yum-config-manager --enable centos-sclo-rh", - ] # RHEL distros need EPEL for Compass dependencies commands += [ "yum -y install yum-utils epel-release", @@ -258,9 +252,15 @@ def run_test(test: Test, client: DockerClient) -> Tuple[Test, Result]: container: Container | None = None try: image = get_image(test, client) - container = client.containers.run(image, command=f"bash -c \"{join_commands(commands)}\"", - auto_remove=True, detach=True, - volumes=[f'{test_external_root}:{test_docker_root}']) + container = client.containers.run( + image, command=f"bash -c \"{join_commands(commands)}\"", auto_remove=True, detach=True, + volumes=[ + f'{test_external_root}:{test_docker_root}', + '/etc/pki/entitlement/:/run/secrets/etc-pki-entitlement', + '/etc/rhsm:/run/secrets/rhsm', + '/etc/yum.repos.d/redhat.repo:/run/secrets/redhat.repo', + '/etc/yum.repos.d/redhat-rhsm.repo:/run/secrets/redhat-rhsm.repo' + ]) for log in container.logs(stream=True): result["log_raw"] += log.decode('UTF-8') # This is pretty verbose, lets run this way for a while and we can delete this if it ends up being too much diff --git a/buildscripts/setup_spawnhost_coredump b/buildscripts/setup_spawnhost_coredump index 7a3a31825885d..07524ef6e900c 100755 --- a/buildscripts/setup_spawnhost_coredump +++ b/buildscripts/setup_spawnhost_coredump @@ -26,25 +26,35 @@ if [[ "${machine}" = "Cygwin" ]]; then echo "----------------------" echo -e "\n=> Setting _NT_SOURCE_PATH environment variable for debuggers to pick up source files." - SRC_DIR_HASH=$(readlink -f /cygdrive/z/data/mci/source-*) - SRC_DIR="${SRC_DIR_HASH}/src" + SRC_DIR_HASH=$(find /cygdrive/z/data/mci -name 'source-*' | head -n 1 | xargs -I% basename %) + SRC_DIR="Z:\data\mci\\${SRC_DIR_HASH}\src" echo "Source Path: [${SRC_DIR}]" set -x; setx _NT_SOURCE_PATH "${SRC_DIR}" { set +x; } 2>/dev/null echo -e "\n=> Setting _NT_SYMBOL_PATH environment variable for debuggers to pick up the symbols." + DBG_SYMBOLS_HASH=$(find /cygdrive/z/data/mci -name 'artifacts-*dist_test_debug' | head -n 1 | xargs -I% basename %) + DBG_SYMBOLS_WINPATH="\"Z:\data\mci\\${DBG_SYMBOLS_HASH}\extracted_symbols\dist-test\bin;srv*;\"" DBG_ARCHIVE_PARENT=$(readlink -f /cygdrive/z/data/mci/artifacts-*dist_test_debug) DBG_ARCHIVE=$(readlink -f ${DBG_ARCHIVE_PARENT}/debugsymbols-*.zip) DBG_ARCHIVE_TARGET_PARENT="${DBG_ARCHIVE_PARENT}/extracted_symbols" DBG_ARCHIVE_TARGET="${DBG_ARCHIVE_TARGET_PARENT}/dist-test/bin" echo "Symbols Dir: [${DBG_ARCHIVE_TARGET}]" + echo -e "\n=> Copying .natvis file to Visual Studio's expected directories (System for 2017/2019 and User)." + set -x; + cp "/cygdrive/z/data/mci/${SRC_DIR_HASH}/buildscripts/win/mongodb.natvis" /cygdrive/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2017/Professional/Common7/Packages/Debugger/Visualizers + cp "/cygdrive/z/data/mci/${SRC_DIR_HASH}/buildscripts/win/mongodb.natvis" /cygdrive/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2019/Professional/Common7/Packages/Debugger/Visualizers + mkdir "${USERPROFILE}/Documents/Visual Studio 2022/Visualizers" + cp "/cygdrive/z/data/mci/${SRC_DIR_HASH}/buildscripts/win/mongodb.natvis" "${USERPROFILE}/Documents/Visual Studio 2022/Visualizers" + { set +x; } 2>/dev/null + echo -e "\n=> Extracting Symbol files." set -x; mkdir -p ${DBG_ARCHIVE_TARGET_PARENT} unzip -n ${DBG_ARCHIVE} -d ${DBG_ARCHIVE_TARGET_PARENT} - setx _NT_SYMBOL_PATH "${DBG_ARCHIVE_TARGET};srv*;" + setx _NT_SYMBOL_PATH "${DBG_SYMBOLS_WINPATH}" { set +x; } 2>/dev/null echo -e "\n=> Extracting Core Dumps to Desktop." @@ -93,6 +103,7 @@ if [[ "${machine}" = "Cygwin" ]]; then if [[ -z $(ls ${COREDUMP_ARCHIVE_TARGET}/dump* 2>/dev/null) ]]; then echo "No core dumps found." fi + { set +x; } 2>/dev/null echo "Copied to Desktop." } &> ${out_dir} diff --git a/buildscripts/win/mongodb.natvis b/buildscripts/win/mongodb.natvis index 047b4ccb747e8..bee3119c152e6 100644 --- a/buildscripts/win/mongodb.natvis +++ b/buildscripts/win/mongodb.natvis @@ -1,14 +1,16 @@ - + @@ -63,16 +65,16 @@ - + Code+4,[*(int32_t*)(Code)-1]s8 *(mongo::BSONObjData*)(Code+*(int32_t*)(Code)+4) - diff --git a/etc/backports_required_for_multiversion_tests.yml b/etc/backports_required_for_multiversion_tests.yml index fdd239a90aca4..739eff13e2968 100644 --- a/etc/backports_required_for_multiversion_tests.yml +++ b/etc/backports_required_for_multiversion_tests.yml @@ -553,6 +553,10 @@ last-continuous: ticket: SERVER-93099 - test_file: jstests/core/timeseries/timeseries_update_mixed_schema_bucket.js ticket: SERVER-93099 + - test_file: jstests/sharding/refresh_sessions.js + ticket: SERVER-94635 + - test_file: jstests/core/query/project/projection_with_hashed_index.js + ticket: SERVER-91757 suites: null last-lts: all: @@ -1158,4 +1162,8 @@ last-lts: ticket: SERVER-93099 - test_file: jstests/core/timeseries/timeseries_update_mixed_schema_bucket.js ticket: SERVER-93099 + - test_file: jstests/sharding/refresh_sessions.js + ticket: SERVER-94635 + - test_file: jstests/core/query/project/projection_with_hashed_index.js + ticket: SERVER-91757 suites: null diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 10d789f6ba3ee..1536cd2d75fdb 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -285,7 +285,7 @@ variables: - name: unittest_shell_hang_analyzer_gen - name: test_packages distros: - - ubuntu2004-package + - rhel94-large-packagetest - name: vector_search - name: vector_search_auth - name: vector_search_ssl @@ -794,7 +794,7 @@ buildvariants: display_name: "* macOS DEBUG" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter run_on: - - macos-1100 + - macos-11 expansions: &macos-debug-expansions compile_variant: *macos-debug-suggested test_flags: --excludeWithAnyTags=incompatible_with_macos --enableEnterpriseTests=off @@ -830,7 +830,7 @@ buildvariants: display_name: "Enterprise macOS Via Rosetta 2" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - macos-1100-arm64 + - macos-11-arm64 expansions: compile_variant: *enterprise-macos-rosetta-2 test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm @@ -861,7 +861,7 @@ buildvariants: display_name: "Enterprise macOS DEBUG" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - macos-1100 + - macos-11 expansions: compile_variant: *enterprise-macos test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm @@ -892,7 +892,7 @@ buildvariants: - name: &enterprise-macos-arm64 enterprise-macos-arm64 display_name: "~ Enterprise macOS arm64" run_on: - - macos-1100-arm64 + - macos-11-arm64 expansions: compile_variant: *enterprise-macos-arm64 test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm @@ -1623,7 +1623,7 @@ buildvariants: cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. stepback: false run_on: - - macos-1100 + - macos-11 expansions: num_scons_link_jobs_available: 0.1 compile_env: DEVELOPER_DIR=/Applications/Xcode13.app diff --git a/etc/evergreen_yml_components/variants/atlas.yml b/etc/evergreen_yml_components/variants/atlas.yml index bfd5eb8ad80a0..70b01c527a878 100644 --- a/etc/evergreen_yml_components/variants/atlas.yml +++ b/etc/evergreen_yml_components/variants/atlas.yml @@ -70,7 +70,7 @@ buildvariants: - name: unittest_shell_hang_analyzer_gen - name: test_packages distros: - - ubuntu2004-package + - rhel94-large-packagetest - name: vector_search - name: vector_search_auth - name: vector_search_ssl diff --git a/etc/evergreen_yml_components/variants/misc_release.yml b/etc/evergreen_yml_components/variants/misc_release.yml index 8e247371482e1..ed8f4b7beb378 100644 --- a/etc/evergreen_yml_components/variants/misc_release.yml +++ b/etc/evergreen_yml_components/variants/misc_release.yml @@ -1047,7 +1047,7 @@ buildvariants: - name: .stitch - name: test_packages distros: - - ubuntu2004-package + - rhel94-large-packagetest - name: selinux_rhel7_org - name: .publish - name: generate_buildid_to_debug_symbols_mapping @@ -2558,7 +2558,7 @@ buildvariants: display_name: macOS cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - macos-1100 + - macos-11 expansions: test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_external_data_source,requires_latch_analyzer --enableEnterpriseTests=off push_path: osx @@ -2594,7 +2594,7 @@ buildvariants: display_name: macOS arm64 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - macos-1100-arm64 + - macos-11-arm64 expansions: test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_external_data_source,requires_latch_analyzer --enableEnterpriseTests=off push_path: osx @@ -2630,7 +2630,7 @@ buildvariants: display_name: Enterprise macOS cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - macos-1100 + - macos-11 expansions: test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm,requires_external_data_source,requires_latch_analyzer additional_package_targets: >- @@ -2668,7 +2668,7 @@ buildvariants: display_name: Enterprise macOS arm64 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - macos-1100-arm64 + - macos-11-arm64 expansions: test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm,requires_external_data_source,requires_latch_analyzer additional_package_targets: >- diff --git a/etc/evergreen_yml_components/variants/ninja.yml b/etc/evergreen_yml_components/variants/ninja.yml index 15992d040e17e..85b57f9912880 100644 --- a/etc/evergreen_yml_components/variants/ninja.yml +++ b/etc/evergreen_yml_components/variants/ninja.yml @@ -18,7 +18,7 @@ buildvariants: display_name: "Ninja Build: macOS Enterprise" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - macos-1100 + - macos-11 expansions: compile_env: DEVELOPER_DIR=/Applications/Xcode13.app compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx.vars @@ -31,7 +31,7 @@ buildvariants: display_name: "Ninja Build Profiles: macOS" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - macos-1100 + - macos-11 expansions: compile_env: DEVELOPER_DIR=/Applications/Xcode13.app compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --libc++ @@ -45,7 +45,7 @@ buildvariants: display_name: "Ninja Build Profiles: macOS ARM" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - macos-1100-arm64 + - macos-11-arm64 expansions: compile_env: DEVELOPER_DIR=/Applications/Xcode13.app compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --libc++ diff --git a/etc/evergreen_yml_components/variants/task_generation.yml b/etc/evergreen_yml_components/variants/task_generation.yml index 4c5c81f4e2c88..1f7a0294a28f2 100644 --- a/etc/evergreen_yml_components/variants/task_generation.yml +++ b/etc/evergreen_yml_components/variants/task_generation.yml @@ -1,4 +1,8 @@ # Build variant to generate tasks for evergreen versions. +# +# Updates to this file may also need to appear in etc/system_perf_yml_components/variants/task_generation.yml, +# which is the same but excludes resmoke task generation tasks. +# buildvariants: - name: generate-tasks-for-version diff --git a/etc/pip/components/platform.req b/etc/pip/components/platform.req index e14765ddf0afd..a7fbf382d16a4 100644 --- a/etc/pip/components/platform.req +++ b/etc/pip/components/platform.req @@ -2,7 +2,8 @@ pypiwin32>=223; sys_platform == "win32" and python_version > "3" pywin32>=225; sys_platform == "win32" and python_version > "3" -cryptography == 2.3; platform_machine == "s390x" or platform_machine == "ppc64le" # Needed for oauthlib to use RSAAlgorithm # Version locked - see SERVER-36618 +cryptography == 2.3; (platform_machine == "s390x" and platform_release < "5.14.0-362.8.1.el9_3.s390x") or (platform_machine == "ppc64le" and platform_release < "5.14.0-362.13.1.el9_3.ppc64le") +cryptography == 36.0.2; (platform_machine == "s390x" and platform_release >= "5.14.0-362.8.1.el9_3.s390x") or (platform_machine == "ppc64le" and platform_release >= "5.14.0-362.13.1.el9_3.ppc64le") cryptography == 36.0.2; platform_machine != "s390x" and platform_machine != "ppc64le" mongo-ninja-python == 1.11.1.5; (platform_machine == "x86_64" or platform_machine == "aarch64") and sys_platform == "linux" diff --git a/etc/system_perf.yml b/etc/system_perf.yml index 0eaa3c2e95a7f..92bdeace22904 100755 --- a/etc/system_perf.yml +++ b/etc/system_perf.yml @@ -4,10 +4,16 @@ exec_timeout_secs: &exec_timeout_secs 21600 timeout_secs: &timeout_secs 7200 include: - - filename: etc/evergreen_yml_components/variants/task_generation.yml - filename: etc/evergreen_yml_components/definitions.yml - - filename: etc/system_perf_yml_components/tasks.yml - - filename: etc/system_perf_yml_components/variants.yml + # This is a substitute task_generation.yml for sys-perf that doesn't + # generate resmoke tasks. + - filename: etc/system_perf_yml_components/variants/task_generation.yml + - filename: evergreen/system_perf/7.0/variants.yml + module: dsi + - filename: evergreen/system_perf/shared_tasks.yml + module: dsi + - filename: evergreen/system_perf/7.0/genny_tasks.yml + module: genny ## Parameters for parameterized builds (see https://github.com/evergreen-ci/evergreen/wiki/Parameterized-Builds) parameters: @@ -39,20 +45,6 @@ variables: - PrivateWorkloads - flamegraph - _schedule_variant_auto_tasks_task: &schedule_variant_auto_tasks_task - name: schedule_variant_auto_tasks - activate: false - depends_on: - - name: schedule_global_auto_tasks - variant: task_generation - - _schedule_patch_auto_tasks_task: &schedule_patch_auto_tasks_task - name: schedule_patch_auto_tasks - activate: false - depends_on: - - name: schedule_global_auto_tasks - variant: task_generation - _shared_compile_expansions: &shared_compile_expansions platform: linux project_dir: dsi @@ -104,7 +96,7 @@ modules: owner: mongodb-labs repo: YCSB prefix: ${workdir}/src - branch: production + branch: main - name: py-tpcc owner: mongodb-labs repo: py-tpcc @@ -139,61 +131,6 @@ timeout: functions: - ### - # Same in every DSI project - f_dsi_pre_run: - - command: manifest.load - f_dsi_post_run: - - command: shell.exec - params: - script: ./src/dsi/run-dsi post_run - - command: perf.send - params: - file: ./build/CedarReports/cedar_report.json - aws_key: ${terraform_key} - aws_secret: ${terraform_secret} - bucket: genny-metrics - region: us-east-1 - prefix: ${task_id}_${execution} - - command: attach.results - params: - file_location: ./build/EvergreenResultsJson/results.json - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: ./build/Artifacts/DSIArtifacts.tgz - remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/dsi-artifacts-${task_name}-${build_id}-${execution}.tgz - bucket: mciuploads - permissions: public-read - content_type: application/x-gzip - display_name: DSI Artifacts - Execution ${execution} - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: ./build/Documentation/index.html - remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/${task_name}-${build_id}-index.html - bucket: mciuploads - permissions: public-read - content_type: text/html - display_name: Documentation - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: bootstrap.yml - remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/bootstrap-${task_name}-${build_id}-${execution}.yml - bucket: mciuploads - permissions: public-read - content_type: text/plain - display_name: Task Bootstrap Config - f_dsi_timeout: - - command: shell.exec - params: - script: ./src/dsi/run-dsi on_timeout - ### - f_other_post_ops: - command: shell.exec params: @@ -220,43 +157,6 @@ functions: - command: shell.exec params: {script: "echo"} - f_generate_all_variant_auto_tasks: - - command: git.get_project - params: - directory: *src_dir - revisions: - dsi: ${dsi_rev} - genny: ${genny_rev} - linkbench: ${linkbench_rev} - linkbench2: ${linkbench2_rev} - tsbs: ${tsbs_rev} - workloads: ${workloads_rev} - mongo-perf: ${mongo-perf_rev} - YCSB: ${YCSB_rev} - py-tpcc: ${py-tpcc_rev} - PrivateWorkloads: ${PrivateWorkloads_rev} - - command: expansions.write - params: - file: ./expansions.yml - - command: shell.exec - params: - # The script below makes sure that if we generate all tasks in a patch, we will only generate - # them and not run all of them, since activating all tasks is the default behavior for task - # generation in a patch. - script: | - pushd ./src/mongo - ~/evergreen evaluate ./etc/system_perf.yml > evaluated_project_configuration.yml - popd - if [ "${is_patch|false}" = "true" ]; then - ./src/genny/run-genny auto-tasks-all --project-file ./src/mongo/evaluated_project_configuration.yml --no-activate - else - ./src/genny/run-genny auto-tasks-all --project-file ./src/mongo/evaluated_project_configuration.yml - fi - - command: generate.tasks - params: - files: - - build/TaskJSON/Tasks.json - ### # Package the supplementary DSI data (mongo tools, feature flag list, # server params list, jstests, and buildscripts) @@ -432,109 +332,7 @@ functions: display_name: supplementary-data.tgz ### - ## Schedule Tasks ## - f_schedule_tasks: - - command: git.get_project - params: - directory: *src_dir - clone_depth: 1000 - revisions: - dsi: ${dsi_rev} - genny: ${genny_rev} - linkbench: ${linkbench_rev} - linkbench2: ${linkbench2_rev} - tsbs: ${tsbs_rev} - workloads: ${workloads_rev} - mongo-perf: ${mongo-perf_rev} - YCSB: ${YCSB_rev} - py-tpcc: ${py-tpcc_rev} - PrivateWorkloads: ${PrivateWorkloads_rev} - - command: expansions.write - params: - file: ./expansions.yml - - command: shell.exec - params: - script: ./src/dsi/run-dsi schedule_tasks --tasks=${tasks} - - command: generate.tasks - params: - files: - - build/TaskJSON/Tasks.json - - tasks: - ### - # Same in every DSI project - - name: generate_all_variant_auto_tasks - priority: 5 - commands: - - func: f_generate_all_variant_auto_tasks - - name: schedule_global_auto_tasks - patch_only: true - priority: 5 - commands: - - func: f_schedule_tasks - vars: - tasks: all_tasks - - name: schedule_variant_auto_tasks - patch_only: true - priority: 5 - commands: - - func: f_schedule_tasks - vars: - tasks: variant_tasks - - name: schedule_patch_auto_tasks - patch_only: true - priority: 5 - commands: - - func: f_schedule_tasks - vars: - tasks: patch_tasks - - name: smoke_test - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: short - - name: canaries_only - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: canaries - - name: smoke_test_ssl - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: short - mongodb_setup: replica-ssl - infrastructure_provisioning: replica - - name: smoke_test_standalone_auth - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: short - mongodb_setup: standalone-auth - infrastructure_provisioning: single - - name: smoke_test_replset_auth - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: short - mongodb_setup: replica-auth - infrastructure_provisioning: replica - - name: smoke_test_shard_lite_auth - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: short - mongodb_setup: shard-lite-auth - infrastructure_provisioning: shard-lite - ### - - name: package_supplementary_data depends_on: - name: version_expansions_gen @@ -557,1026 +355,7 @@ tasks: - func: "package supplementary data" - func: "upload supplementary data" - - name: linkbench - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "linkbench" - - - name: linkbench_stepdowns - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "linkbench_stepdowns" - - - name: linkbench_rolling_restarts - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "linkbench_rolling_restarts" - - - name: linkbench_non_retryable_writes_stepdowns - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "linkbench_non_retryable_writes_stepdowns" - - - name: linkbench_non_retryable_writes_rolling_restarts - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "linkbench_non_retryable_writes_rolling_restarts" - - - name: linkbench2 - priority: 5 - exec_timeout_secs: 43200 # 12 hours - commands: - - func: f_dsi_run_workload - vars: - test_control: "linkbench2" - additional_tfvars: "tags: {expire-on-delta: 12}" - - - name: tsbs_load - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_load" - - - name: tsbs_query - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_query" - - - name: tsbs_query_finance - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_query_finance" - - - name: tsbs_query_high_cardinality - priority: 5 - exec_timeout_secs: 432000 # 5 days - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_query_high_cardinality" - - - name: tsbs_query_sharded - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_query_sharded" - - - name: tsbs_query_finance_sharded - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_query_finance_sharded" - - - name: tsbs_query_sharded_balancer - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_query_sharded_balancer" - - - name: tsbs_query_finance_sharded_balancer - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_query_finance_sharded_balancer" - - - name: tsbs_query_manual_bucketing - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_query_manual_bucketing" - - - name: tsbs-query-genny - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_query_genny" - test_control_params: | - {task_name: tsbs_query_genny, - config_filename: ./src/genny/dist/etc/genny/workloads/query/TimeseriesTsbsQuery.yml} - - - name: tsbs-query-optimizations - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_query_genny" - test_control_params: | - {task_name: tsbs_query_optimizations, - config_filename: ./src/genny/dist/etc/genny/workloads/query/TimeseriesTsbsOptimizations.yml} - - - name: tpcc - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tpcc" - - - name: tpcc_majority - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tpcc_majority" - - - name: industry_benchmarks - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb" - - - name: ycsb_60GB - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-60GB" - - - name: ycsb.load - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb.load" - - - name: ycsb_60GB.long - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-60GB.long" - - - name: industry_benchmarks_secondary_reads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-secondary-reads" - - - name: industry_benchmarks_w1 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-w1.2023-02" - - - name: industry_benchmarks_stepdowns - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb_stepdowns" - - - name: industry_benchmarks_rolling_restarts - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb_rolling_restarts" - - - name: industry_benchmarks_non_retryable_writes_stepdowns - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb_non_retryable_writes_stepdowns" - - - name: industry_benchmarks_non_retryable_writes_rolling_restarts - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb_non_retryable_writes_rolling_restarts" - - - name: crud_workloads_w1 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "crud_workloads_w1.2023-02" - - - name: crud_workloads_majority - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "crud_workloads_majority" - - - name: cursor_manager - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "cursor_manager" - - # unscheduling according to SERVER-78888 - # - name: mixed_workloads - # priority: 5 - # commands: - # - func: f_dsi_run_workload - # vars: - # test_control: "mixed_workloads" - - - name: mixed_workloads_genny_stepdowns - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "mixed_workloads_genny_stepdowns" - - - name: mixed_workloads_genny_rolling_restarts - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "mixed_workloads_genny_rolling_restarts" - - - name: big_update_10k - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "BigUpdate10k" - - - name: misc_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "misc_workloads" - - - - name: map_reduce_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "map_reduce_workloads" - - - name: genny_canaries - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "genny_canaries" - - - name: retryable_writes_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "retryable_writes" - - - name: snapshot_reads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "snapshot_reads" - - - name: secondary_reads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "secondary_reads" - - - name: bestbuy_agg - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_agg" - - - name: bestbuy_agg_merge_same_db - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_agg_merge_same_db" - - - name: bestbuy_agg_merge_different_db - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_agg_merge_different_db" - - - name: bestbuy_agg_merge_target_hashed - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_agg_merge_target_hashed" - - - name: bestbuy_agg_merge_wordcount - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_agg_merge_wordcount" - - - name: bestbuy_query - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_query" - - - name: bestbuy_4_analytics - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_analytics" - test_control_params: | - {scale: 4, - columnstore: false} - - - name: bestbuy_4_analytics_columnstore - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_analytics" - test_control_params: | - {scale: 4, - columnstore: true} - - # Named Pipes single concurrent query benchmarks - - name: external_data_source_1 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - mongodb_setup: "external_data_source" - test_control: "external_data_source_1" - - # Named Pipes four concurrent query benchmarks - - name: external_data_source_4 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - mongodb_setup: "external_data_source" - test_control: "external_data_source_4" - - - - name: tpch_1_normalized - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tpch" - test_control_params: | - {scale: 1, - schema: normalized} - - - name: tpch_1_denormalized - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tpch" - test_control_params: | - {scale: 1, - schema: denormalized} - - - name: tpch_10_normalized - priority: 5 - exec_timeout_secs: 43200 # 12 hours - commands: - - func: f_dsi_run_workload - vars: - test_control: "tpch" - test_control_params: | - {scale: 10, - schema: normalized} - - - name: tpch_10_denormalized - priority: 5 - exec_timeout_secs: 43200 # 12 hours - commands: - - func: f_dsi_run_workload - vars: - test_control: "tpch" - test_control_params: | - {scale: 10, - schema: denormalized} - - - name: ssb_column_store_comparison - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ssb_column_store_index" - test_control_params: | - {scale: 5} - - name: column_store_tpch_10_denormalized - priority: 5 - exec_timeout_secs: 43200 # 12 hours - commands: - - func: f_dsi_run_workload - vars: - test_control: "column_store_tpch" - test_control_params: | - {scale: 10, - schema: denormalized, - columnstore: true} - - name: column_store_tpch_10_denormalized_unindexed - priority: 5 - exec_timeout_secs: 43200 # 12 hours - commands: - - func: f_dsi_run_workload - vars: - test_control: "column_store_tpch" - test_control_params: | - {scale: 10, - schema: denormalized, - columnstore: false} - - - name: mixed_workloads_genny_rate_limited_high_value - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: mixed_workloads_genny_rate_limited - auto_workload_path: ./src/genny/dist/etc/genny/workloads/scale/MixedWorkloadsGennyRateLimited.yml - - name: load_test_high_value - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: load_test - auto_workload_path: ./src/genny/dist/etc/genny/workloads/scale/LoadTest.yml - - name: majority_reads10_k_threads_high_value - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: majority_reads10_k_threads - auto_workload_path: ./src/genny/dist/etc/genny/workloads/scale/MajorityReads10KThreads.yml - - name: large_indexed_ins_high_value - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: large_indexed_ins - auto_workload_path: ./src/genny/dist/etc/genny/workloads/scale/LargeIndexedIns.yml - - name: expressive_queries_high_value - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: expressive_queries - auto_workload_path: ./src/genny/dist/etc/genny/workloads/query/ExpressiveQueries.yml - - name: time_series_sort_high_value - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: time_series_sort - auto_workload_path: ./src/genny/dist/etc/genny/workloads/query/TimeSeriesSort.yml - - name: medical_workload_diagnosis_50_50_high_value - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: medical_workload_diagnosis_50_50 - auto_workload_path: ./src/genny/dist/etc/genny/workloads/encrypted/medical_workload-diagnosis-50-50.yml - - name: ycsb_like_queryable_encrypt1_cfdefault_high_value - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: ycsb_like_queryable_encrypt1_cfdefault - auto_workload_path: ./src/genny/dist/etc/genny/workloads/encrypted/YCSBLikeQueryableEncrypt1Cfdefault.yml - - name: filter_with_complex_logical_expression_high_value - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: filter_with_complex_logical_expression - auto_workload_path: ./src/genny/dist/etc/genny/workloads/query/FilterWithComplexLogicalExpression.yml - - name: array_traversal_high_value - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: array_traversal - auto_workload_path: ./src/genny/dist/etc/genny/workloads/query/ArrayTraversal.yml - -# TODO PERF-3094: Remove these charts_events tasks. - - name: column_store_index_charts_events_1G - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "charts_events" - test_control_params: | - {scale: 1} - - - name: column_store_index_charts_events_10G - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "charts_events" - test_control_params: | - {scale: 10} - -# TODO PERF-3094: Remove this task. - - name: bestbuy_4_inserts - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_4_inserts" - - - name: non_sharded_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "non_sharded" - - - name: mongos_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "mongos" - - - name: mongos_large_catalog_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "mongos_large_catalog" - - - name: move_chunk_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "move_chunk" - - - name: move_chunk_waiting_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "move_chunk_waiting" - - - name: move_chunk_large_chunk_map_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "move_chunk_large_chunk_map" - - - name: refine_shard_key_transaction_stress - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "refine_shard_key_transaction_stress" - - - name: secondary_performance - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - # Unfortunately the dash/underscore style is different for mongodb_setup and test_control - test_control: "secondary_performance" - mongodb_setup: "secondary-performance" - - - name: initialsync - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "initialsync" - - - name: initialsync-fcbis - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "initialsync" - mongodb_setup: "replica-2node-fcbis" - - - name: initialsync-logkeeper - priority: 5 - exec_timeout_secs: 43200 # 12 hours - commands: - - func: f_dsi_run_workload - timeout_secs: 43200 # 12 hours - vars: - test_control: "initialsync-logkeeper" - - - name: initialsync-logkeeper-fcbis - priority: 5 - exec_timeout_secs: 43200 # 12 hours - commands: - - func: f_dsi_run_workload - timeout_secs: 43200 # 12 hours - vars: - test_control: "initialsync-logkeeper" - mongodb_setup: "initialsync-logkeeper-fcbis" - - # The following two initial sync logkeeper automation tasks are only used in the commented-out - # "Linux ReplSet Initial Sync LogKeeper Snapshot Update" variant below and are only intended to be - # run in patch builds to update FCV for logkeeper datasets. - - - name: initialsync-logkeeper-snapshot-update - priority: 5 - exec_timeout_secs: 216000 # 2.5 days - commands: - - func: f_dsi_run_workload - vars: - test_control: "initialsync-logkeeper-snapshot-update" - - - name: initialsync-large - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "initialsync-large" - - - name: initialsync-large-fcbis - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "initialsync-large" - mongodb_setup: "replica-2node-fcbis" - - - name: change_streams_latency - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "change_streams_latency" - - - name: change_streams_preimage_throughput - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "change_streams_preimage_throughput" - - - name: change_streams_preimage_latency - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "change_streams_preimage_latency" - - - name: change_streams_listen_throughput - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "change_streams_listen_throughput" - - - name: change_streams_multi_mongos - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "change_streams_multi_mongos" - - - name: genny_execution_UserAcquisition - commands: - - func: f_dsi_run_workload - vars: - test_control: auto_genny_workload - auto_workload_path: ./src/genny/dist/etc/genny/workloads/execution/UserAcquisition.yml - - name: genny_scale_InsertRemove - commands: - - func: f_dsi_run_workload - vars: - test_control: auto_genny_workload - auto_workload_path: ./src/genny/dist/etc/genny/workloads/scale/InsertRemove.yml - - name: query_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: query, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: query_read_commands_large_dataset - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: query_large_dataset, - include_filter_2: regression, - exclude_filter: none, - threads: "1 4", - read_cmd: 'true', - share_dataset: 'true'} - - name: big_collection - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: query, - include_filter_2: getmore, - exclude_filter: none, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: views-query - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: query_identityview, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: views-aggregation - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: aggregation_identityview, - include_filter_2: regression, - exclude_filter: none, - threads: "1", - read_cmd: 'true'} - - name: where_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: where, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: update_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: update, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: insert_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: insert, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: compound_wildcard_index_write_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: compound-wildcard-insert compound-wildcard-remove compound-wildcard-update, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: compound_wildcard_index_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: compound-wildcard-query, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: wildcard-index-read_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: wildcard_read, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: wildcard-index-write_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: wildcard_write, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: geo_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: geo, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: misc_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: command multi remove mixed, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: misc_custom_filter_default_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - mongodb_setup: mongo-perf-standalone-custom-filter-default.2023-02 - test_control_params: | - {include_filter_1: command multi remove mixed, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: misc_custom_filter_slow_or_sample_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - mongodb_setup: mongo-perf-standalone-custom-filter-slow-or-sample.2023-02 - test_control_params: | - {include_filter_1: command multi remove mixed, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: misc_custom_filter_complex_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - mongodb_setup: mongo-perf-standalone-custom-filter-complex.2023-02 - test_control_params: | - {include_filter_1: command multi remove mixed, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: misc_custom_filter_whole_doc_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - mongodb_setup: mongo-perf-standalone-custom-filter-whole-doc.2023-02 - test_control_params: | - {include_filter_1: command multi remove mixed, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: misc_slowms_everything_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - mongodb_setup: mongo-perf-standalone-slowms-everything.2023-02 - test_control_params: | - {include_filter_1: command multi remove mixed, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: singleThreaded_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: single_threaded, - include_filter_2: core regression, - exclude_filter: none, - threads: "1", - read_cmd: 'true'} - - name: aggregation_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: aggregation, - include_filter_2: regression, - exclude_filter: js, - threads: "1", - read_cmd: 'true'} - - name: aggregation_read_commands_large_dataset - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: aggregation_large_dataset, - include_filter_2: regression, - exclude_filter: js, - threads: "1 4", - read_cmd: 'true', - share_dataset: 'true'} - - name: agg-query-comparison_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: agg_query_comparison, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: pipeline-updates - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: pipeline-updates, - include_filter_2: regression, - exclude_filter: none, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: javascript - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: js, - include_filter_2: aggregation, - exclude_filter: none, - threads: "1 2 4 8", - read_cmd: 'true'} - buildvariants: - - name: task_generation - display_name: Task Generation - cron: "0 0 1 1 *" # Every year starting 1/1 at 00:00 - modules: *modules - expansions: - project_dir: dsi - run_on: - - amazon2-build - tasks: - - name: generate_all_variant_auto_tasks - activate: true - - name: schedule_global_auto_tasks - - name: amazon2-x86-compile display_name: "* Compile for Amazon Linux 2 x86" expansions: diff --git a/etc/system_perf_yml_components/tasks.yml b/etc/system_perf_yml_components/tasks.yml deleted file mode 100644 index 112de86cab244..0000000000000 --- a/etc/system_perf_yml_components/tasks.yml +++ /dev/null @@ -1,252 +0,0 @@ -functions: - f_dsi_run_workload: &dsi_run_func - - command: timeout.update - params: - exec_timeout_secs: ${exec_timeout_secs_override} - timeout_secs: ${timeout_secs_override} - - command: git.get_project - params: - directory: src/mongo - clone_depth: 1000 - revisions: - dsi: ${dsi_rev} - genny: ${genny_rev} - linkbench: ${linkbench_rev} - linkbench2: ${linkbench2_rev} - tsbs: ${tsbs_rev} - workloads: ${workloads_rev} - YCSB: ${YCSB_rev} - flamegraph: ${flamegraph_rev} - PrivateWorkloads: ${PrivateWorkloads_rev} - - command: expansions.write - params: - file: ./expansions.yml - redacted: true - - command: shell.exec - params: - script: ./src/dsi/run-dsi run_workload - - command: shell.exec - type: system - params: - script: ./src/dsi/run-dsi determine_failure -m SYSTEM - - command: shell.exec - type: setup - params: - script: ./src/dsi/run-dsi determine_failure -m SETUP - - command: shell.exec - type: test - params: - script: ./src/dsi/run-dsi determine_failure -m TEST - - f_run_dsi_workload: *dsi_run_func # Do not use this function. It is deprecated. - - ## DSI_SELFTEST ## - dsi_selftest_setup_tests: - - command: git.get_project - params: - directory: src/dsi - dsi_selftest_check_python_formatting: - - command: shell.exec - type: test - params: - script: ./src/dsi/run-dsi selftest testscripts/check-format-python.sh - dsi_selftest_lint_python_scripts: - - command: shell.exec - type: test - params: - script: ./src/dsi/run-dsi selftest testscripts/lint-python.sh - dsi_selftest_lint_yml: - - command: shell.exec - type: test - params: - script: ./src/dsi/run-dsi selftest testscripts/lint-yml.sh - dsi_selftest_pytest: - - command: shell.exec - type: test - params: - script: ./src/dsi/run-dsi selftest testscripts/pytest.sh - dsi_selftest_mypy: - - command: shell.exec - type: test - params: - script: ./src/dsi/run-dsi selftest testscripts/mypy.sh - dsi_selftest_shellcheck: - - command: shell.exec - type: test - params: - script: ./src/dsi/run-dsi selftest testscripts/lint-shell.sh - dsi_selftest_e2e: - - command: shell.exec - type: test - params: - script: ./src/dsi/run-dsi e2e_test - -tasks: - ### - # Same in every DSI project - - name: renew_ssl_cert - commands: - - command: git.get_project - params: - directory: src/mongo - revisions: - dsi: ${dsi_rev} - # Run the script to generate ssl cert files - - command: shell.exec - params: - script: AWS_ACCESS_KEY_ID=${terraform_key} AWS_SECRET_ACCESS_KEY=${terraform_secret} ./src/dsi/run-dsi generate_ssl_cert - # Upload files for further DSI usage - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: member.pem - # path to the remote file is intended to be static - remote_file: dsi/ssl/member.pem - bucket: mciuploads - # the visibility has to be public for consumption by DSI - permissions: public-read - content_type: text/plain - display_name: member.pem - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: root.crt - # path to the remote file is intended to be static - remote_file: dsi/ssl/root.crt - bucket: mciuploads - # the visibility has to be public for consumption by DSI - permissions: public-read - content_type: text/plain - display_name: root.crt - - - name: ycsb.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb.2023-09" - - - name: ycsb_60GB.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-60GB.2023-09" - - - name: ycsb_60GB.long.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-60GB.long.2023-09" - - - name: ycsb_secondary_reads.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-secondary-reads.2023-09" - - - name: ycsb_w1.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-w1.2023-09" - - - name: ycsb_stepdowns.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb_stepdowns.2023-09" - - - name: ycsb_rolling_restarts.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb_rolling_restarts.2023-09" - - - name: ycsb_non_retryable_writes_stepdowns.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb_non_retryable_writes_stepdowns.2023-09" - - - name: ycsb_non_retryable_writes_rolling_restarts.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb_non_retryable_writes_rolling_restarts.2023-09" - - - name: locust_bulk_insert - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "locust_bulk_insert" - - - name: tsbs_query_fixed_bucketing - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_query_fixed_bucketing" - - - name: tsbs-expression-query - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_query_genny" - test_control_params: | - {task_name: tsbs_expression_query, - config_filename: ./src/genny/dist/etc/genny/workloads/query/TimeseriesTsbsExpressionQuery.yml} - - - name: startup - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "startup" - - - name: dbcheck - priority: 5 - exec_timeout_secs: 43200 # 12 hours - commands: - - func: f_dsi_run_workload - timeout_secs: 43200 # 12 hours - vars: - test_control: "dbcheck" - additional_tfvars: "tags: {expire-on-delta: 12}" # increase host expiration to 12 hours. - - - name: genny_resharding_withIndexes - exec_timeout_secs: 172800 # 2 days - commands: - - func: f_dsi_run_workload - vars: - test_control: auto_genny_workload - auto_workload_path: ./src/genny/dist/etc/genny/workloads/sharding/ReshardCollectionWithIndexes.yml - - - name: tie-breaking-heuristics - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: tie-breaking, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - - name: stream_workloads - commands: - - func: f_dsi_run_workload - vars: - test_control: streams.2023-10 diff --git a/etc/system_perf_yml_components/variants.yml b/etc/system_perf_yml_components/variants.yml deleted file mode 100644 index 4fa6b857c8522..0000000000000 --- a/etc/system_perf_yml_components/variants.yml +++ /dev/null @@ -1,407 +0,0 @@ -variables: -- &amazon2_x86_compile_variant_dependency - depends_on: - - name: archive_dist_test - variant: amazon2-x86-compile - - name: package_supplementary_data - variant: amazon2-x86-compile - -- &amazon_linux2_arm64_compile_variant_dependency - depends_on: - - name: archive_dist_test - variant: amazon2-arm64-compile - - name: package_supplementary_data - variant: amazon2-arm64-compile - -- &amazon_linux2_arm64_mongocrypt_compile_variant_dependency - depends_on: - - name: archive_dist_test - variant: amazon2-x86-compile - - name: package_supplementary_data - variant: amazon2-arm64-compile - - name: crypt_create_lib - variant: amazon2-arm64-mongocrypt-shlib-compile - -- &schedule_variant_auto_tasks_task - name: schedule_variant_auto_tasks - activate: false - depends_on: - - name: schedule_global_auto_tasks - variant: task_generation - -- &schedule_patch_auto_tasks_task - name: schedule_patch_auto_tasks - activate: false - depends_on: - - name: schedule_global_auto_tasks - variant: task_generation - -- modules: &perf_modules - - dsi - - genny - - workloads - - linkbench - - linkbench2 - - tsbs - - mongo-perf - - YCSB - - PrivateWorkloads - - py-tpcc - - flamegraph - -buildvariants: -- <<: *amazon_linux2_arm64_compile_variant_dependency - name: perf-atlas-M60-real.arm.aws.2023-11 - display_name: PERF M60-Atlas ReplSet ARM AWS 2023-11 - cron: "0 0 * * 0,4" # 00:00 on Sunday, Thursday - modules: *perf_modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: atlas - canaries: none - atlas_setup: M60-repl - use_custom_build: true - infrastructure_provisioning: workload_client_arm.2023-04 - infrastructure_provisioning_release: 2023-09 - workload_setup: 2022-11 - platform: linux - project_dir: &perf_project_dir dsi - storageEngine: wiredTiger - compile_variant: amazon2-arm64-compile - use_supplementary_data: true - run_on: - - "rhel94-perf-atlas-large" - tasks: - - *schedule_patch_auto_tasks_task - - *schedule_variant_auto_tasks_task - - name: ycsb.2023-09 - - name: ycsb_60GB.2023-09 - - name: tpcc - - name: tpcc_majority - - name: linkbench - - name: linkbench2 - -- <<: *amazon2_x86_compile_variant_dependency - name: perf-atlas-M60-real.intel.azure.2023-11 - display_name: PERF M60-Atlas ReplSet Intel Azure 2023-11 - cron: "0 0 * * 0,4" # 00:00 on Sunday, Thursday - modules: *perf_modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: atlas - canaries: none - atlas_setup: M60-repl-azure - use_custom_build_azure: true - compile_variant: amazon2-x86-compile - use_supplementary_data: true - infrastructure_provisioning: workload_client_intel.2023-11 - infrastructure_provisioning_release: 2023-09 - workload_setup: 2022-11 - platform: linux - project_dir: *perf_project_dir - storageEngine: wiredTiger - run_on: - - "rhel94-perf-atlas-large" - tasks: # Cannot use *3nodetasks because secondary_performance uses a special mongodb setup - - *schedule_patch_auto_tasks_task - - *schedule_variant_auto_tasks_task - - name: ycsb.2023-09 - - name: ycsb_60GB.2023-09 - - name: tpcc - - name: tpcc_majority - - name: linkbench - - name: linkbench2 - -- <<: *amazon_linux2_arm64_compile_variant_dependency - name: perf-3-shard.arm.aws.2023-11 - display_name: PERF 3-Shard Cluster ARM AWS 2023-11 - cron: "0 0 * * 4" # 00:00 on Thursday - modules: *perf_modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: shard - infrastructure_provisioning_release: 2023-09 - infrastructure_provisioning: shard - workload_setup: 2022-11 - platform: linux - project_dir: *perf_project_dir - authentication: enabled - storageEngine: wiredTiger - compile_variant: amazon2-arm64-compile - use_supplementary_data: true - run_on: - - "rhel94-perf-shard" - tasks: - - *schedule_patch_auto_tasks_task - - *schedule_variant_auto_tasks_task - - name: ycsb.2023-09 - - name: ycsb_w1.2023-09 - - name: crud_workloads_majority - - name: crud_workloads_w1 - - name: misc_workloads - - name: map_reduce_workloads - - name: smoke_test - - name: mongos_workloads - - name: mongos_large_catalog_workloads - - name: move_chunk_workloads - - name: change_streams_latency - - name: change_streams_listen_throughput - - name: change_streams_multi_mongos - - name: tsbs_query_sharded - - name: tsbs_query_finance_sharded - - name: tsbs_query_sharded_balancer - - name: tsbs_query_finance_sharded_balancer - -- <<: *amazon_linux2_arm64_compile_variant_dependency - name: perf-3-node-replSet.arm.aws.2023-11 - display_name: PERF 3-Node ReplSet ARM AWS 2023-11 - cron: "0 0 * * 1,2,3,4,5,6" # Everyday except Sunday at 00:00 - modules: *perf_modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: replica - infrastructure_provisioning_release: 2023-09 - infrastructure_provisioning: replica - workload_setup: 2022-11 - platform: linux - project_dir: *perf_project_dir - authentication: enabled - storageEngine: wiredTiger - compile_variant: amazon2-arm64-compile - use_supplementary_data: true - run_on: - - "rhel94-perf-replset" - tasks: - - *schedule_patch_auto_tasks_task - - *schedule_variant_auto_tasks_task - - name: ycsb.2023-09 - - name: ycsb_w1.2023-09 - - name: ycsb_60GB.2023-09 - - name: ycsb.load - - name: ycsb_60GB.long.2023-09 - - name: ycsb_secondary_reads.2023-09 - - name: crud_workloads_majority - - name: crud_workloads_w1 - - name: misc_workloads - - name: map_reduce_workloads - - name: refine_shard_key_transaction_stress - - name: smoke_test - - name: secondary_performance # Uses a special 2 node mongodb setup - - name: non_sharded_workloads - - name: bestbuy_agg - - name: bestbuy_agg_merge_different_db - - name: bestbuy_agg_merge_same_db - - name: bestbuy_agg_merge_wordcount - - name: bestbuy_query - - name: change_streams_preimage_throughput - - name: change_streams_latency - - name: change_streams_preimage_latency - - name: change_streams_listen_throughput - - name: snapshot_reads - - name: secondary_reads - - name: tpcc - - name: tpcc_majority - - name: tpch_1_normalized - - name: tpch_1_denormalized - # TODO: Enable in SERVER-66572. - # - name: tpch_10_normalized - # - name: tpch_10_denormalized - - name: linkbench - - name: linkbench2 - - name: tsbs_load - - name: tsbs_query - - name: tsbs_query_finance - - name: tsbs_query_manual_bucketing - - name: tsbs-query-genny - - name: tsbs-query-optimizations - - name: tsbs-expression-query - - name: big_update_10k - - name: mixed_workloads_genny_rate_limited_high_value - - name: load_test_high_value - - name: majority_reads10_k_threads_high_value - - name: large_indexed_ins_high_value - - name: expressive_queries_high_value - - name: time_series_sort_high_value - -- <<: *amazon2_x86_compile_variant_dependency - name: perf-3-node-replSet-intel.intel.aws.2023-11 - display_name: PERF 3-Node ReplSet Intel AWS 2023-11 - cron: "0 0 * * 1,2,3,4,5,6" # Everyday except Sunday at 00:00 - modules: *perf_modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: replica - infrastructure_provisioning_release: 2023-09 - infrastructure_provisioning: replica-intel.2023-11 - workload_setup: 2022-11 - platform: linux - project_dir: *perf_project_dir - authentication: enabled - storageEngine: wiredTiger - compile_variant: amazon2-x86-compile - use_supplementary_data: true - run_on: - - "rhel94-perf-replset" - tasks: - - *schedule_patch_auto_tasks_task - - *schedule_variant_auto_tasks_task - - name: ycsb.2023-09 - - name: ycsb_60GB.2023-09 - - name: crud_workloads_majority - - name: smoke_test - - name: linkbench - - name: linkbench2 - - name: mixed_workloads_genny_rate_limited_high_value - - name: tsbs-expression-query - -# On PERF-730 we changed the initial sync tests to use two nodes instead of three. To avoid -# losing history, the name remains unchanged, but the display_name reflects the change to 2-Node. -- <<: *amazon_linux2_arm64_compile_variant_dependency - name: perf-2-node-replSet-initialsync.arm.aws.2023-11 - display_name: PERF 2-Node ReplSet Initial Sync ARM AWS 2023-11 - cron: "0 0 * * 4" # 00:00 on Thursday - modules: *perf_modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: replica-2node - infrastructure_provisioning_release: 2023-09 - infrastructure_provisioning: replica-2node - workload_setup: 2022-11 - platform: linux - authentication: disabled - storageEngine: wiredTiger - compile_variant: amazon2-arm64-compile - use_supplementary_data: true - project_dir: *perf_project_dir - run_on: - - "rhel94-perf-replset" - tasks: - - *schedule_patch_auto_tasks_task - - *schedule_variant_auto_tasks_task - - name: initialsync-large - - name: initialsync-large-fcbis - -- <<: *amazon_linux2_arm64_mongocrypt_compile_variant_dependency - name: perf-shard-lite-fle.arm.aws.2023-11 - display_name: PERF Shard Lite FLE ARM AWS 2023-11 - cron: "0 0 * * 0,4" # 00:00 on Sunday,Thursday - modules: *perf_modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: shard-lite-fle - infrastructure_provisioning_release: 2023-09 - infrastructure_provisioning: shard-lite - workload_setup: 2022-11 - platform: linux - project_dir: *perf_project_dir - authentication: enabled - storageEngine: wiredTiger - compile_variant: amazon2-arm64-compile - use_supplementary_data: true - shlib_compile_variant: amazon2-arm64-mongocrypt-shlib-compile - mongocrypt_shlib_required: true - run_on: - - "rhel94-perf-shard-lite" - tasks: - - *schedule_patch_auto_tasks_task - - *schedule_variant_auto_tasks_task - - name: medical_workload_diagnosis_50_50_high_value - - name: ycsb_like_queryable_encrypt1_cfdefault_high_value - -- <<: *amazon_linux2_arm64_compile_variant_dependency - name: perf-mongo-perf-standalone.arm.aws.2023-11 - display_name: PERF Mongo-Perf Standalone inMemory ARM AWS 2023-11 - cron: &linux-microbench-cron "0 0 * * *" # Everyday at 00:00 - modules: *perf_modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: mongo-perf-standalone.2023-02 - infrastructure_provisioning_release: 2023-09 - infrastructure_provisioning: workload_client_mongod_combined.2023-01 - workload_setup: 2022-11 - use_scons_cache: true - platform: linux - canaries: none - storageEngine: inMemory - project_dir: *perf_project_dir - compile_variant: amazon2-arm64-compile - use_supplementary_data: true - run_on: - - "rhel94-perf-microbenchmarks" - tasks: - - name: big_collection - - name: genny_scale_InsertRemove - - name: genny_execution_UserAcquisition - - name: aggregation_read_commands - - name: aggregation_read_commands_large_dataset - - name: agg-query-comparison_read_commands - - name: query_read_commands - - name: query_read_commands_large_dataset - - name: views-aggregation - - name: views-query - - name: where_read_commands - - name: update_read_commands - - name: insert_read_commands - - name: wildcard-index-read_read_commands - - name: wildcard-index-write_read_commands - - name: geo_read_commands - - name: misc_read_commands - - name: misc_custom_filter_default_read_commands - - name: misc_custom_filter_slow_or_sample_read_commands - - name: misc_custom_filter_complex_read_commands - - name: misc_custom_filter_whole_doc_read_commands - - name: misc_slowms_everything_read_commands - - name: singleThreaded_read_commands - - name: pipeline-updates - - name: javascript - - name: compound_wildcard_index_write_commands - - name: compound_wildcard_index_read_commands - -- <<: *amazon2_x86_compile_variant_dependency - name: perf-mongo-perf-standalone.intel.aws.2023-11 - display_name: PERF Mongo-Perf Standalone inMemory Intel AWS 2023-11 - cron: *linux-microbench-cron - modules: *perf_modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: mongo-perf-standalone.2023-02 - infrastructure_provisioning_release: 2023-09 - infrastructure_provisioning: workload_client_mongod_combined_intel.2023-11 - workload_setup: 2022-11 - use_scons_cache: true - platform: linux - canaries: none - storageEngine: inMemory - project_dir: *perf_project_dir - compile_variant: amazon2-x86-compile - use_supplementary_data: true - run_on: - - "rhel94-perf-microbenchmarks" - tasks: - - name: big_collection - - name: genny_scale_InsertRemove - - name: genny_execution_UserAcquisition - - name: aggregation_read_commands - - name: aggregation_read_commands_large_dataset - - name: agg-query-comparison_read_commands - - name: query_read_commands - - name: query_read_commands_large_dataset - - name: views-aggregation - - name: views-query - - name: where_read_commands - - name: update_read_commands - - name: insert_read_commands - - name: wildcard-index-read_read_commands - - name: wildcard-index-write_read_commands - - name: geo_read_commands - - name: misc_read_commands - - name: misc_custom_filter_default_read_commands - - name: misc_custom_filter_slow_or_sample_read_commands - - name: misc_custom_filter_complex_read_commands - - name: misc_custom_filter_whole_doc_read_commands - - name: misc_slowms_everything_read_commands - - name: singleThreaded_read_commands - - name: pipeline-updates - - name: javascript - - name: compound_wildcard_index_write_commands - - name: compound_wildcard_index_read_commands diff --git a/etc/system_perf_yml_components/variants/task_generation.yml b/etc/system_perf_yml_components/variants/task_generation.yml new file mode 100644 index 0000000000000..5b4e291d49075 --- /dev/null +++ b/etc/system_perf_yml_components/variants/task_generation.yml @@ -0,0 +1,15 @@ +# Replacement for generate-tasks-for-version from evergreen_yml_components. +# +# This is similar to generate-tasks-for-version in evergreen_yml_components, +# but doesn't include actual task generation, which does nothing +# for sys-perf, and breaks when it tries to process the sys-perf project. +# + +buildvariants: + - name: generate-tasks-for-version + display_name: "! Generate tasks for evergreen version" + activate: true + run_on: + - rhel80-medium + tasks: + - name: version_expansions_gen diff --git a/evergreen/do_jepsen_setup/install_jepsen.sh b/evergreen/do_jepsen_setup/install_jepsen.sh index e92b684935e06..3c2a6c0f08aa2 100755 --- a/evergreen/do_jepsen_setup/install_jepsen.sh +++ b/evergreen/do_jepsen_setup/install_jepsen.sh @@ -1,7 +1,7 @@ set -o errexit cd src -git clone --branch=v0.2.0-jepsen-mongodb-master --depth=1 git@github.com:10gen/jepsen.git jepsen-mongodb +git clone --branch=v0.3.0-jepsen-mongodb-master --depth=1 git@github.com:10gen/jepsen.git jepsen-mongodb cd jepsen-mongodb lein install diff --git a/evergreen/prelude_venv.sh b/evergreen/prelude_venv.sh index f0d3225aa2704..a0851406f57cb 100644 --- a/evergreen/prelude_venv.sh +++ b/evergreen/prelude_venv.sh @@ -43,5 +43,5 @@ function activate_venv { fi export PIP_CACHE_DIR=${workdir}/pip_cache - echo "python set to $(which $python)" + echo "python set to $(which $python) and python version: $($python --version)" } diff --git a/jstests/aggregation/sources/documents.js b/jstests/aggregation/sources/documents/basics.js similarity index 100% rename from jstests/aggregation/sources/documents.js rename to jstests/aggregation/sources/documents/basics.js diff --git a/jstests/aggregation/sources/documents/subpipeline_validation.js b/jstests/aggregation/sources/documents/subpipeline_validation.js new file mode 100644 index 0000000000000..7d4ae87584f2c --- /dev/null +++ b/jstests/aggregation/sources/documents/subpipeline_validation.js @@ -0,0 +1,215 @@ +// Tests the validation logic for combinations of "collectionless" stages like $documents with or +// around sub-pipelines. For the cases that should be legal, we mostly just care the the command +// succeeds. However, we will use 'resultsEq' to test correct semantics while we are here, gaining +// more coverage. +// TODO SERVER-94226 consider extending this test to cases like $currentOp and $queryStats as well. +// This test uses stages like $documents which are not permitted inside a $facet stage. +// @tags: [do_not_wrap_aggregations_in_facets] +(function() { +"use strict"; +load("jstests/aggregation/extras/utils.js"); // For 'resultsEq.' +load("jstests/libs/fixture_helpers.js"); // For 'isMongos.' + +const coll = db[jsTestName()]; +coll.drop(); + +const targetCollForMerge = db["target_coll"]; +targetCollForMerge.drop(); +assert.commandWorked(coll.insert({_id: 0, arr: [{}, {}]})); + +{ + // Tests for an aggregation over a collection (i.e. {aggregate: "collName"} commands) with a + // $documents stage used in a sub-pipeline. Each of these cases should be legal, which is most + // of the value of the assertion. We will use 'resultsEq' to test correct semantics while we are + // here. + + // $lookup. + assert(resultsEq(coll.aggregate([ + {$lookup: { + let: {documents: "$arr"}, + pipeline: [ + {$documents: "$$documents"}, + ], + as: "duplicated" + }}, + ]).toArray(), [{_id: 0, arr: [{}, {}], duplicated: [{}, {}]}])); + + // $unionWith. + assert(resultsEq(coll.aggregate([ + { + $unionWith: { + pipeline: [ + {$documents: [{_id: "gen"}]}, + ], + } + }, + ]) + .toArray(), + [{_id: 0, arr: [{}, {}]}, {_id: "gen"}])); + + // Both, and more nesting. + assert(resultsEq(coll.aggregate([{ + $unionWith: { + coll: coll.getName(), + pipeline: [{ + $lookup: { + pipeline: [ + {$documents: []}, + {$unionWith: {coll: coll.getName(), pipeline: []}} + ], + as: "nest" + } + }] + } + }]) + .toArray(), + [ + {_id: 0, arr: [{}, {}]}, + {_id: 0, arr: [{}, {}], nest: [{_id: 0, arr: [{}, {}]}]} + ])); +} + +{ + // Tests for a db-level aggregate (i.e. {aggregate: 1} commands) with sub-pipelines on regular + // collections. + + // $facet + assert(resultsEq(db.aggregate([ + { + $documents: [ + {x: 1, y: 1, val: 1}, + {x: 2, y: 2, val: 1}, + {x: 3, y: 1, val: 2}, + {x: 2, y: 2, val: 1} + ] + }, + { + $facet: { + sumByX: [{$group: {_id: "$x", sum: {$sum: "$val"}}}], + sumByY: [{$group: {_id: "$y", sum: {$sum: "$val"}}}] + } + } + ]).toArray(), + [{ + sumByX: [{_id: 1, sum: 1}, {_id: 2, sum: 2}, {_id: 3, sum: 2}], + sumByY: [{_id: 1, sum: 3}, {_id: 2, sum: 2}] + }])); + + if (!FixtureHelpers.isMongos(db)) { + // This doesn't work on mongos in v7.0 and earlier - waiting for SERVER-65534. + + // $lookup. + assert(resultsEq(db.aggregate([ + {$documents: [{x: 1, arr: [{x: 2}]}, {y: 1, arr: []}]}, + {$lookup: { + let: {documents: "$arr"}, + pipeline: [ + {$documents: "$$documents"}, + ], + as: "duplicated" + }}, + ]).toArray(), + [ + {x: 1, arr: [{x: 2}], duplicated: [{x: 2}]}, + {y: 1, arr: [], duplicated: []} + ])); + + // $merge. + assert.doesNotThrow(() => db.aggregate([ + { + $documents: [ + {_id: 2, x: "foo"}, + {_id: 4, x: "bar"}, + ] + }, + { + $merge: { + into: targetCollForMerge.getName(), + on: "_id", + whenMatched: [{$set: {x: {$setUnion: ["$x", "$$new.x"]}}}] + } + } + ])); + assert(resultsEq(targetCollForMerge.find({}, {_id: 1}).toArray(), [{_id: 2}, {_id: 4}])); + + // $unionWith + assert(resultsEq(db.aggregate([ + {$documents: [{_id: 2}, {_id: 4}]}, + {$unionWith: {coll: coll.getName(), pipeline: []}} + ]).toArray(), + [{_id: 2}, {_id: 4}, {_id: 0, arr: [{}, {}]}])); + + // All of the above, plus nesting. + const results = + db.aggregate([ + {$documents: [{_id: "first"}]}, + { + $unionWith: { + pipeline: [ + {$documents: [{_id: "uw"}]}, + {$unionWith: {pipeline: [{$documents: [{_id: "uw_2"}]}]}}, + { + $facet: { + allTogether: + [{$group: {_id: null, all: {$addToSet: "$_id"}}}], + countEach: [{$group: {_id: "$_id", count: {$sum: 1}}}], + } + }, + { + $lookup: + {pipeline: [{$documents: [{x: "lu1"}, {x: "lu2"}]}], as: "xs"} + }, + {$set: {xs: {$map: {input: "$xs", in : "$$this.x"}}}} + ] + }, + }, + ]).toArray(); + assert(resultsEq(results, + [ + {_id: "first"}, + { + allTogether: [{_id: null, all: ["uw", "uw_2"]}], + countEach: [{_id: "uw", count: 1}, {_id: "uw_2", count: 1}], + xs: ["lu1", "lu2"] + } + ]), + results); + } +} + +// Test for invalid combinations. + +// To use $documents inside a $lookup, there must not be a "from" argument. +// As of SERVER-94144, this does not throw on 7.0 and older branches. +assert.doesNotThrow( + () => coll.aggregate([{$lookup: {from: "foo", pipeline: [{$documents: []}], as: "lustuff"}}])); +assert.doesNotThrow( + () => coll.aggregate([ + {$lookup: { + from: "foo", + let: {docs: "$arr"}, + pipeline: [ + {$documents: "$$docs"}, + {$lookup: { + from: "foo", + let: {x: "$x", y: "$y"}, + pipeline: [ + {$match: {$expr: {$and: [ + {$eq: ["$x", "$$x"]}, + {$eq: ["$y", "$$y"]} + ]}}} + ], + as: "doesnt_matter" + }} + ], + as: "lustuff" + }}])); + +// To use $documents inside a $unionWith, there must not be a "coll" argument. +// As of SERVER-94144, this does not throw on 7.0 and older branches. +assert.doesNotThrow( + () => coll.aggregate([{$unionWith: {coll: "foo", pipeline: [{$documents: []}]}}])); + +// Cannot use $documents inside of $facet. +assert.throwsWithCode(() => coll.aggregate([{$facet: {test: [{$documents: []}]}}]), 40600); +})(); diff --git a/jstests/auth/speculative-auth-replset.js b/jstests/auth/speculative-auth-replset.js index 0bb392ac60543..f30d949948e25 100644 --- a/jstests/auth/speculative-auth-replset.js +++ b/jstests/auth/speculative-auth-replset.js @@ -27,8 +27,7 @@ function countAuthInLog(conn) { } } else if (entry.id === kAuthenticationFailedLogId) { // Authentication can fail legitimately because the secondary abandons the connection - // during shutdown - if we do encounter an authentication failure in the log, make sure - // that it is only of this type, fail anything else + // during shutdown. assert.eq(entry.attr.result, ErrorCodes.AuthenticationAbandoned); } else { // Irrelevant. diff --git a/jstests/concurrency/fsm_workloads/internal_transactions_sharded_from_mongod.js b/jstests/concurrency/fsm_workloads/internal_transactions_sharded_from_mongod.js index f2ae41e3ac5c0..ed3bd8f6d1c27 100644 --- a/jstests/concurrency/fsm_workloads/internal_transactions_sharded_from_mongod.js +++ b/jstests/concurrency/fsm_workloads/internal_transactions_sharded_from_mongod.js @@ -112,20 +112,6 @@ var $config = extendWorkload($config, function($config, $super) { } }; - $config.teardown = function teardown(db, collName, cluster) { - $super.teardown.apply(this, arguments); - - // If a shard node that is acting as a router for an internal transaction is - // killed/terminated/stepped down or the transaction's session is killed while running a - // non-retryable transaction, the transaction would be left in-progress since nothing - // would aborted it. Such dangling transactions can cause the CheckReplDBHash hook to hang - // as the fsyncLock command requires taking the global S lock and it cannot do that while - // there is an in-progress transaction. - if (TestData.runningWithShardStepdowns || this.retryOnKilledSession) { - this.killAllSessions(cluster); - } - }; - $config.states.init = function init(db, collName, connCache) { const retryableErrorMessages = [ "The server is in quiesce mode and will shut down", diff --git a/jstests/concurrency/fsm_workloads/random_moveChunk_refine_collection_shard_key.js b/jstests/concurrency/fsm_workloads/random_moveChunk_refine_collection_shard_key.js index c397518ecc53d..2c7e591964866 100644 --- a/jstests/concurrency/fsm_workloads/random_moveChunk_refine_collection_shard_key.js +++ b/jstests/concurrency/fsm_workloads/random_moveChunk_refine_collection_shard_key.js @@ -66,6 +66,7 @@ var $config = extendWorkload($config, function($config, $super) { // The refienCollectionCoordinator interrupt all migrations by setting `allowMigration` // to false ErrorCodes.Interrupted, + ErrorCodes.OrphanedRangeCleanUpFailed, ]; return (err.code && codes.includes(err.code)) || (err.message && diff --git a/jstests/core/hostinfo.js b/jstests/core/hostinfo.js index de085496fee42..07d2f6c52909a 100644 --- a/jstests/core/hostinfo.js +++ b/jstests/core/hostinfo.js @@ -34,7 +34,11 @@ if (hostinfo.os.type != "") { assert.neq(hostinfo.system.currentTime, "" || null, "Missing Current Time"); assert.neq(hostinfo.system.cpuAddrSize, "" || null || 0, "Missing CPU Address Size"); assert.neq(hostinfo.system.memSizeMB, "" || null, "Missing Memory Size"); - assert.neq(hostinfo.system.numCores, "" || null || 0, "Missing Number of Cores"); + assert.neq(hostinfo.system.numCores, "" || null || 0, "Missing Number of Logical Cores"); + // Check that numCoresAvailableToProcess != -1 as that indicates syscall failure. + assert.neq(hostinfo.system.numCoresAvailableToProcess, + "" || null || -1, + "Missing Number of Cores Available To Process"); assert.neq( hostinfo.system.numPhysicalCores, "" || null || 0, "Missing Number of Physical Cores"); assert.neq(hostinfo.system.numCpuSockets, "" || null || 0, "Missing Number of CPU Sockets"); diff --git a/jstests/core/query/project/projection_with_hashed_index.js b/jstests/core/query/project/projection_with_hashed_index.js new file mode 100644 index 0000000000000..998e3da9ab28e --- /dev/null +++ b/jstests/core/query/project/projection_with_hashed_index.js @@ -0,0 +1,48 @@ +/** + * Confirm that a hashed index field does not prevent the index prefix field to be used for covered + * projection and to produce correct result. + * @tags: [ + * # Explain may return incomplete results if interrupted by a stepdown. + * does_not_support_stepdowns, + * ] + */ + +load("jstests/aggregation/extras/utils.js"); +load("jstests/libs/analyze_plan.js"); + +const coll = db[jsTestName()]; +coll.drop(); + +assert.commandWorked( + coll.insertMany([{_id: 1, a: {b: 5}}, {_id: 2, a: {b: 2, c: 1}}, {_id: 3, a: {b: 0}}])); + +// Confirm that the hashed index scan produces the same results as the collection scan. +const resultsCollScan = coll.find({}, {_id: 0, 'a.b': 1}).sort({'a.b': 1}); +assert.commandWorked(coll.createIndex({'a.b': 1, a: 'hashed'})); +const resultsIndexScan = coll.find({}, {_id: 0, 'a.b': 1}).sort({'a.b': 1}); +assert(orderedArrayEq(resultsCollScan, resultsIndexScan)); + +// Check that the index with hashed field is used in the plan. +const explain = coll.find({}, {_id: 0, 'a.b': 1}).sort({'a.b': 1}).explain(); + +const plan = getWinningPlanFromExplain(explain); +const project = getPlanStage(plan, "PROJECTION_DEFAULT"); +assert.neq(project, null, explain); +const ixScan = getPlanStage(plan, "IXSCAN"); +assert.eq(ixScan.indexName, "a.b_1_a_hashed", explain); + +const fetch = getPlanStage(plan, "FETCH"); +const shards = getShardsFromExplain(explain); +if (shards) { + // In sharded environment if a sharding_filter stage is added, a FETCH stage is also added on + // top of the index scan. Otherwise covered projection is used without a fetch. + const shardingFilter = getPlanStage(plan, "SHARDING_FILTER"); + if (shardingFilter) { + assert.neq(fetch, null, plan); + } else { + assert.eq(fetch, null, plan); + } +} else { + // In non-sharded environment covered projection is used without a FETCH stage. + assert.eq(fetch, null, plan); +} diff --git a/jstests/libs/analyze_plan.js b/jstests/libs/analyze_plan.js index a54b5548f7309..78a53c556ee76 100644 --- a/jstests/libs/analyze_plan.js +++ b/jstests/libs/analyze_plan.js @@ -41,6 +41,19 @@ function getRejectedPlan(rejectedPlan) { return rejectedPlan.hasOwnProperty("queryPlan") ? rejectedPlan.queryPlan : rejectedPlan; } +/** + * Help function to extract shards from explain in sharded environment. Returns null for + * non-sharded plans. + */ +function getShardsFromExplain(explain) { + if (explain.hasOwnProperty("queryPlanner") && + explain.queryPlanner.hasOwnProperty("winningPlan")) { + return explain.queryPlanner.winningPlan.shards; + } + + return null; +} + /** * Returns a sub-element of the 'cachedPlan' explain output which represents a query plan. */ diff --git a/jstests/libs/query_stats_utils.js b/jstests/libs/query_stats_utils.js index c64b28424411e..b33d430e9c6a0 100644 --- a/jstests/libs/query_stats_utils.js +++ b/jstests/libs/query_stats_utils.js @@ -447,8 +447,8 @@ function checkChangeStreamEntry({queryStatsEntry, db, collectionName, numExecs, assert.eq(collectionName, queryStatsEntry.key.queryShape.cmdNs.coll); // Confirm entry is a change stream request. - let stringifiedPipeline = JSON.stringify(queryStatsEntry.key.queryShape.pipeline, null, 0); - assert(stringifiedPipeline.includes("_internalChangeStream")); + const pipelineShape = queryStatsEntry.key.queryShape.pipeline; + assert(pipelineShape[0].hasOwnProperty("$changeStream"), pipelineShape); // TODO SERVER-76263 Support reporting 'collectionType' on a sharded cluster. if (!FixtureHelpers.isMongos(db)) { diff --git a/jstests/noPassthrough/catalog_snapshot_consistency.js b/jstests/noPassthrough/catalog_snapshot_consistency.js new file mode 100644 index 0000000000000..7c5811c747873 --- /dev/null +++ b/jstests/noPassthrough/catalog_snapshot_consistency.js @@ -0,0 +1,38 @@ +/** + * Test local catalog for commands like listDatabases, focusing on consistency with the durable + * storage snapshot. + */ +load('jstests/libs/fail_point_util.js'); // For configureFailPoint(), + // kDefaultWaitForFailPointTimeout() + +let replTest = new ReplSetTest({ + name: jsTestName(), + nodes: 1, +}); +replTest.startSet(); +replTest.initiate(); +let mongod = replTest.getPrimary(); + +const slowPublishDb = "catalog_snapshot_consistency_slow_publish_db"; +const slowPublishColl = "coll"; + +// List database should reflect an implicitly created database that has been committed but not +// published into the local catalog yet. Use a failpoint to hang before publishing the catalog, +// simulating a slow catalog publish. +const failPoint = configureFailPoint(mongod, + "hangBeforePublishingCatalogUpdates", + {collectionNS: slowPublishDb + '.' + slowPublishColl}); +const waitDbCreate = startParallelShell(`{ + db.getSiblingDB('${slowPublishDb}')['${slowPublishColl}'].createIndex({a:1}); +}`, mongod.port); +failPoint.wait(); + +let cmdRes = assert.commandWorked( + mongod.adminCommand({listDatabases: 1, filter: {$expr: {$eq: ["$name", slowPublishDb]}}})); +assert.eq(1, cmdRes.databases.length, tojson(cmdRes)); +assert.eq(slowPublishDb, cmdRes.databases[0].name, tojson(cmdRes)); + +failPoint.off(); +waitDbCreate(); + +replTest.stopSet(); diff --git a/jstests/noPassthrough/lookup_profile.js b/jstests/noPassthrough/lookup_profile.js new file mode 100644 index 0000000000000..7327cdcbfdb8a --- /dev/null +++ b/jstests/noPassthrough/lookup_profile.js @@ -0,0 +1,22 @@ +// Tests that lookups on local capped collections acquire a snapshot on the capped collection +// correctly. Tests the scenario fixed by SERVER-91203 no longer causes a crash. + +let rst = new ReplSetTest({nodes: {n0: {profile: "0"}}}); +rst.startSet(); +rst.initiate(); + +const dbName = "test"; +const collName = "foo"; + +let testDB = rst.getPrimary().getDB(dbName); +let testColl = testDB.getCollection(collName); + +testColl.insert({a: 1}); + +testDB.setProfilingLevel(2); + +const pipeline = + [{$lookup: {from: 'system.profile', localField: 'key', foreignField: 'key', as: 'results'}}]; +testColl.aggregate(pipeline).toArray(); + +rst.stopSet(); diff --git a/jstests/noPassthrough/queryStats/query_stats_agg_key_change_stream.js b/jstests/noPassthrough/queryStats/query_stats_agg_key_change_stream.js index 1ef3ae40a7bec..3e108e3ca4057 100644 --- a/jstests/noPassthrough/queryStats/query_stats_agg_key_change_stream.js +++ b/jstests/noPassthrough/queryStats/query_stats_agg_key_change_stream.js @@ -1,64 +1,19 @@ /** * This test confirms that query stats store key fields for an aggregate command are properly nested - * and none are missing when running a change stream query with $_passthroughToShard. - * @tags: [ - * requires_sharding, + * and none are missing. It also validates the exact pipeline in the query shape. + * @tags: [ * uses_change_streams, + * requires_replication, + * requires_sharding, + * requires_fcv_70 * ] */ -load("jstests/libs/query_stats_utils.js"); +load("jstests/libs/query_stats_utils.js"); // For runCommandAndValidateQueryStats. +load("jstests/libs/collection_drop_recreate.js"); // For assertDropAndRecreateCollection. const dbName = jsTestName(); const collName = "coll"; -// $_passthroughToShard is only possible on a sharded cluster. -const st = new ShardingTest({ - shards: 2, - mongos: 1, - config: 1, - rs: {nodes: 1}, - other: { - mongosOptions: { - setParameter: { - internalQueryStatsRateLimit: -1, - } - } - } -}); - -const sdb = st.s0.getDB(dbName); -assert.commandWorked(sdb.dropDatabase()); - -sdb.setProfilingLevel(0, -1); -st.shard0.getDB(dbName).setProfilingLevel(0, -1); - -// Shard the relevant collections. -assert.commandWorked(st.s.adminCommand({enableSharding: dbName, primaryShard: st.shard0.name})); -// Shard the collection on {_id: 1}, split at {_id: 0} and move the empty upper chunk to -// shard1. -st.shardColl(collName, {_id: 1}, {_id: 0}, {_id: 0}, dbName); - -const shardId = st.shard0.shardName; -let coll = sdb[collName]; - -const aggregateCommandObj = { - aggregate: coll.getName(), - pipeline: [{"$changeStream": {}}], - allowDiskUse: false, - cursor: {batchSize: 2}, - maxTimeMS: 50 * 1000, - bypassDocumentValidation: false, - readConcern: {level: "majority"}, - collation: {locale: "en_US", strength: 2}, - hint: {"v": 1}, - comment: "", - let : {}, - apiDeprecationErrors: false, - apiVersion: "1", - apiStrict: false, - $_passthroughToShard: {shard: shardId} -}; - const queryShapeAggregateFields = ["cmdNs", "command", "pipeline", "allowDiskUse", "collation", "let"]; @@ -77,17 +32,153 @@ const queryStatsAggregateKeyFields = [ "hint", "readConcern", "cursor.batchSize", - "$_passthroughToShard", - "$_passthroughToShard.shard" ]; -assert.commandWorked(coll.createIndex({v: 1})); -runCommandAndValidateQueryStats({ - coll: coll, - commandName: "aggregate", - commandObj: aggregateCommandObj, - shapeFields: queryShapeAggregateFields, - keyFields: queryStatsAggregateKeyFields -}); +const testCases = [ + // Default fields. + { + pipeline: [{"$changeStream": {}}], + expectedShapifiedPipeline: [{ + "$changeStream": { + startAtOperationTime: "?timestamp", + fullDocument: "default", + fullDocumentBeforeChange: "off" + } + }] + }, + // Non default field values. + { + pipeline: [{ + "$changeStream": { + fullDocument: "updateLookup", + fullDocumentBeforeChange: "required", + showExpandedEvents: true, + } + }], + expectedShapifiedPipeline: [{ + "$changeStream": { + startAtOperationTime: "?timestamp", + fullDocument: "updateLookup", + fullDocumentBeforeChange: "required", + showExpandedEvents: true, + } + }], + }, + // $changeStream followed by a $match. $changeStream internally creates another $match stage + // which shouldn't appear in the query shape, but a $match in the user specified pipeline should + // appear in the query shape. + { + pipeline: [{$changeStream: {}}, {$match: {a: "field"}}], + expectedShapifiedPipeline: [ + { + "$changeStream": { + startAtOperationTime: "?timestamp", + fullDocument: "default", + fullDocumentBeforeChange: "off" + } + }, + {$match: {a: {$eq: "?string"}}} + ] + } +]; + +function assertPipelineField(conn, expectedPipeline) { + const entry = getLatestQueryStatsEntry(conn, {collName: collName}); + const statsPipeline = getValueAtPath(entry, "key.queryShape.pipeline"); + assert.eq(statsPipeline, expectedPipeline); +} + +function validateResumeTokenQueryShape(conn, coll) { + // Start a change stream. + const changeStream = coll.watch([]); + + // Going to create an invalid event by checking a change stream on a dropped collection. + assert.commandWorked(coll.insert({_id: 1})); + assert(coll.drop()); + assert.soon(() => changeStream.hasNext()); + changeStream.next(); + const invalidateResumeToken = changeStream.getResumeToken(); + + // Resume the change stream using 'startAfter' field. + coll.watch([], {startAfter: invalidateResumeToken}); + assert.commandWorked(coll.insert({_id: 2})); + + const expectedShapifiedPipeline = [{ + "$changeStream": { + resumeAfter: {_data: "?string"}, + fullDocument: "default", + fullDocumentBeforeChange: "off" + } + }]; + assertPipelineField(conn, expectedShapifiedPipeline); +} + +function validateChangeStreamAggKey(conn) { + const db = conn.getDB("test"); + assertDropAndRecreateCollection(db, collName); + + // Change streams with 'startAfter' or 'resumeAfter' are only executed after a certain event and + // require re-parsing a resume token. To validate the query shape of these pipelines, we have to + // execute the events to register the pipeline. + validateResumeTokenQueryShape(conn, db[collName]); + + // Validate the key for the rest of the pipelines. + testCases.forEach(input => { + const pipeline = input.pipeline; + const aggCmdObj = { + aggregate: collName, + pipeline: pipeline, + allowDiskUse: false, + cursor: {batchSize: 2}, + maxTimeMS: 50 * 1000, + bypassDocumentValidation: false, + readConcern: {level: "majority"}, + collation: {locale: "en_US", strength: 2}, + hint: {"v": 1}, + comment: "", + let : {}, + apiDeprecationErrors: false, + apiVersion: "1", + apiStrict: false, + }; + + runCommandAndValidateQueryStats({ + coll: db[collName], + commandName: "aggregate", + commandObj: aggCmdObj, + shapeFields: queryShapeAggregateFields, + keyFields: queryStatsAggregateKeyFields + }); + assertPipelineField(conn, input.expectedShapifiedPipeline); + }); +} + +{ + // Test on a sharded cluster. + const st = new ShardingTest({ + mongos: 1, + shards: 2, + config: 1, + rs: {nodes: 1, setParameter: {writePeriodicNoops: true, periodicNoopIntervalSecs: 1}}, + mongosOptions: { + setParameter: { + internalQueryStatsRateLimit: -1, + } + }, + }); + validateChangeStreamAggKey(st.s); + st.stop(); +} + +{ + // Test the non-sharded case. + const rst = new ReplSetTest({nodes: 2}); + rst.startSet({setParameter: {internalQueryStatsRateLimit: -1}}); + rst.initiate(); + rst.getPrimary().getDB("admin").setLogLevel(3, "queryStats"); -st.stop(); + // Only aggregations run on replica sets have the '$readPreference' field in the key. + queryStatsAggregateKeyFields.push("$readPreference"); + validateChangeStreamAggKey(rst.getPrimary()); + rst.stopSet(); +} diff --git a/jstests/noPassthrough/queryStats/query_stats_agg_key_change_stream_passthrough_shard.js b/jstests/noPassthrough/queryStats/query_stats_agg_key_change_stream_passthrough_shard.js new file mode 100644 index 0000000000000..b82c3ba5fd342 --- /dev/null +++ b/jstests/noPassthrough/queryStats/query_stats_agg_key_change_stream_passthrough_shard.js @@ -0,0 +1,93 @@ +/** + * This test confirms that query stats store key fields for an aggregate command are properly nested + * and none are missing when running a change stream query with $_passthroughToShard. + * @tags: [ + * requires_sharding, + * uses_change_streams, + * ] + */ + +load("jstests/libs/query_stats_utils.js"); +const dbName = jsTestName(); +const collName = "coll"; + +// $_passthroughToShard is only possible on a sharded cluster. +const st = new ShardingTest({ + shards: 2, + mongos: 1, + config: 1, + rs: {nodes: 1}, + other: { + mongosOptions: { + setParameter: { + internalQueryStatsRateLimit: -1, + } + } + } +}); + +const sdb = st.s0.getDB(dbName); +assert.commandWorked(sdb.dropDatabase()); + +sdb.setProfilingLevel(0, -1); +st.shard0.getDB(dbName).setProfilingLevel(0, -1); + +// Shard the relevant collections. +assert.commandWorked(st.s.adminCommand({enableSharding: dbName, primaryShard: st.shard0.name})); +// Shard the collection on {_id: 1}, split at {_id: 0} and move the empty upper chunk to +// shard1. +st.shardColl(collName, {_id: 1}, {_id: 0}, {_id: 0}, dbName); + +const shardId = st.shard0.shardName; +let coll = sdb[collName]; + +const aggregateCommandObj = { + aggregate: coll.getName(), + pipeline: [{"$changeStream": {}}], + allowDiskUse: false, + cursor: {batchSize: 2}, + maxTimeMS: 50 * 1000, + bypassDocumentValidation: false, + readConcern: {level: "majority"}, + collation: {locale: "en_US", strength: 2}, + hint: {"v": 1}, + comment: "", + let : {}, + apiDeprecationErrors: false, + apiVersion: "1", + apiStrict: false, + $_passthroughToShard: {shard: shardId} +}; + +const queryShapeAggregateFields = + ["cmdNs", "command", "pipeline", "allowDiskUse", "collation", "let"]; + +// The outer fields not nested inside queryShape. +const queryStatsAggregateKeyFields = [ + "queryShape", + "cursor", + "maxTimeMS", + "bypassDocumentValidation", + "comment", + "apiDeprecationErrors", + "apiVersion", + "apiStrict", + "collectionType", + "client", + "hint", + "readConcern", + "cursor.batchSize", + "$_passthroughToShard", + "$_passthroughToShard.shard" +]; +assert.commandWorked(coll.createIndex({v: 1})); + +runCommandAndValidateQueryStats({ + coll: coll, + commandName: "aggregate", + commandObj: aggregateCommandObj, + shapeFields: queryShapeAggregateFields, + keyFields: queryStatsAggregateKeyFields +}); + +st.stop(); \ No newline at end of file diff --git a/jstests/noPassthroughWithMongod/cursor_server_status_metrics_lifespan_histogram.js b/jstests/noPassthroughWithMongod/cursor_server_status_metrics_lifespan_histogram.js index b80ce4dcc0f74..ce6988a748945 100644 --- a/jstests/noPassthroughWithMongod/cursor_server_status_metrics_lifespan_histogram.js +++ b/jstests/noPassthroughWithMongod/cursor_server_status_metrics_lifespan_histogram.js @@ -22,14 +22,6 @@ function getNumCursorsLessThan30Seconds() { return db.serverStatus().metrics.cursor.lifespan.lessThan30Seconds; } -function getNumCursorsLessThan1Minute() { - return db.serverStatus().metrics.cursor.lifespan.lessThan1Minute; -} - -function getNumCursorsLessThan10Minutes() { - return db.serverStatus().metrics.cursor.lifespan.lessThan10Minutes; -} - for (let i = 0; i < 40; i++) { coll.insert({a: i, b: "field b"}); } @@ -38,8 +30,6 @@ const initialNumCursorsLt1s = getNumCursorsLessThan1Second(); const initialNumCursorsLt5s = getNumCursorsLessThan5Seconds(); const initialNumCursorsLt15s = getNumCursorsLessThan15Seconds(); const initialNumCursorsLt30s = getNumCursorsLessThan30Seconds(); -const initialNumCursorsLt1m = getNumCursorsLessThan1Minute(); -const initialNumCursorsLt10m = getNumCursorsLessThan10Minutes(); // Since we aren't guaranteed perfect timings, the checks in this test have been relaxed to window // sizes of 30s. For example, a cursor that is expected to die in under 5s may actually take longer @@ -65,21 +55,4 @@ for (let i = 0; i < 3; i++) { } assert.eq(cursorsDeadSinceStartLt30Seconds(), 4); - -const cursorLt1Minute = coll.find().batchSize(2); -const cursorLt10Minutes = coll.aggregate([], {cursor: {batchSize: 2}}); -cursorLt1Minute.next(); -cursorLt10Minutes.next(); - -sleep(31000); // Sleep for 31 s. -while (cursorLt1Minute.hasNext()) { - cursorLt1Minute.next(); -} -assert.eq(getNumCursorsLessThan1Minute() - initialNumCursorsLt1m, 1); - -sleep(30000); // Sleep another 30s, so the total should be greater than 1m and less than 10m. -while (cursorLt10Minutes.hasNext()) { - cursorLt10Minutes.next(); -} -assert.eq(getNumCursorsLessThan10Minutes() - initialNumCursorsLt10m, 1); }()); \ No newline at end of file diff --git a/jstests/noPassthroughWithMongod/shell_exhaust_query.js b/jstests/noPassthroughWithMongod/shell_exhaust_query.js new file mode 100644 index 0000000000000..95ae9d04f1b81 --- /dev/null +++ b/jstests/noPassthroughWithMongod/shell_exhaust_query.js @@ -0,0 +1,21 @@ +/** + * Ensure that the shell correctly handles exhaust queries + */ + +const coll = db.shell_exhaust_queries; + +// Ensure that read concern is not allowed +db.getMongo().setReadConcern('majority'); +assert.throws(() => coll.find().addOption(DBQuery.Option.exhaust).itcount()); +db.getMongo().setReadConcern(null); + +// Ensure that collation is not allowed +assert.throws( + () => coll.find().collation({locale: "simple"}).addOption(DBQuery.Option.exhaust).itcount()); + +// Ensure that "allowDiskUse" is not allowed +assert.throws(() => coll.find().allowDiskUse(true).addOption(DBQuery.Option.exhaust).itcount()); + +// Ensure that read preference is handled correctly +db.getMongo().setReadPref('secondary'); +assert.eq(0, coll.find().addOption(DBQuery.Option.exhaust).itcount()); diff --git a/jstests/serverless/list_databases_for_all_tenants.js b/jstests/serverless/list_databases_for_all_tenants.js index f24be8b53ade8..8e6fa08dc70fe 100644 --- a/jstests/serverless/list_databases_for_all_tenants.js +++ b/jstests/serverless/list_databases_for_all_tenants.js @@ -4,7 +4,11 @@ (function() { "use strict"; -load('jstests/aggregation/extras/utils.js'); // For arrayEq() +load('jstests/aggregation/extras/utils.js'); // For arrayEq() +load('jstests/libs/fail_point_util.js'); // For configureFailPoint(), + // kDefaultWaitForFailPointTimeout() +load('jstests/libs/parallel_shell_helpers.js'); // For funWithArgs() +load('jstests/replsets/libs/tenant_migration_util.js'); // For makeTenantDB() // Given the output from the listDatabasesForAllTenants command, ensures that the total size // reported is the sum of the individual db sizes. @@ -96,6 +100,65 @@ function runTestCheckMultitenantDatabases(primary, numDBs) { return tenantIds; } +// Check that a delay in publishing the database creation to the in-memory catalog doesn't prevent +// the database from being visible. +function runTestCheckSlowPublishMultitenantDb(primary) { + const adminDB = primary.getDB("admin"); + const tokenConn = new Mongo(primary.host); + + let kTenant = ObjectId(); + + // Create a user for kTenant and then set the security token on the connection. + assert.commandWorked(primary.getDB('$external').runCommand({ + createUser: "slowPublishTenant", + '$tenant': kTenant, + roles: [{role: 'readWriteAnyDatabase', db: 'admin'}] + })); + + let token = _createSecurityToken({user: "slowPublishTenant", db: '$external', tenant: kTenant}); + tokenConn._setSecurityToken(token); + + const slowPublishDb = "slow_publish_multitenant_db"; + const slowPublishColl = "coll"; + + // List database should reflect an implicitly created database that has been committed but not + // published into the local catalog yet. Use a failpoint to hang before publishing the catalog, + // simulating a slow catalog publish. + assert.commandWorked(adminDB.runCommand({ + configureFailPoint: "hangBeforePublishingCatalogUpdates", + mode: "alwaysOn", + data: {tenant: kTenant, collectionNS: slowPublishDb + '.' + slowPublishColl} + })); + const shellFn = (token, dbName, collName) => { + let shellConn = db.getSiblingDB("admin").getMongo(); + shellConn._setSecurityToken(token); + db.getSiblingDB(dbName)[collName].createIndex({a: 1}); + }; + const waitDbCreate = startParallelShell( + funWithArgs(shellFn, token, slowPublishDb, slowPublishColl), primary.port); + assert.commandWorked(adminDB.runCommand({ + waitForFailPoint: "hangBeforePublishingCatalogUpdates", + timesEntered: 1, + maxTimeMS: kDefaultWaitForFailPointTimeout + })); + + // use to verify that the database entry is correct + const expectedDatabase = [{"name": slowPublishDb, "tenantId": kTenant, "empty": true}]; + + let cmdRes = assert.commandWorked(adminDB.runCommand( + {listDatabasesForAllTenants: 1, filter: {$expr: {$eq: ["$name", slowPublishDb]}}})); + assert.eq(1, cmdRes.databases.length); + verifySizeSum(cmdRes); + verifyDatabaseEntries(cmdRes, expectedDatabase); + + assert.commandWorked(adminDB.runCommand( + {configureFailPoint: "hangBeforePublishingCatalogUpdates", mode: "off"})); + waitDbCreate(); + + // Reset token + tokenConn._setSecurityToken(undefined); +} + // Test correctness of filter and nameonly options function runTestCheckCmdOptions(primary, tenantIds) { const adminDB = primary.getDB("admin"); @@ -253,6 +316,7 @@ function runTestsWithMultiTenancySupport() { const numDBs = 5; const tenantIds = runTestCheckMultitenantDatabases(primary, numDBs); runTestCheckCmdOptions(primary, tenantIds); + runTestCheckSlowPublishMultitenantDb(primary); runTestInvalidCommands(primary); rst.stopSet(); diff --git a/jstests/sharding/analyze_shard_key/libs/query_sampling_util.js b/jstests/sharding/analyze_shard_key/libs/query_sampling_util.js index 21888a4f2da1d..cadae9074e328 100644 --- a/jstests/sharding/analyze_shard_key/libs/query_sampling_util.js +++ b/jstests/sharding/analyze_shard_key/libs/query_sampling_util.js @@ -12,10 +12,6 @@ var QuerySamplingUtil = (function() { return listCollectionRes.cursor.firstBatch[0].info.uuid; } - function generateRandomString(length = 5) { - return extractUUIDFromObject(UUID()).substring(0, length); - } - function generateRandomCollation() { return {locale: "en_US", strength: AnalyzeShardKeyUtil.getRandInteger(1, 5)}; } @@ -434,7 +430,6 @@ var QuerySamplingUtil = (function() { return { getCollectionUuid, - generateRandomString, generateRandomCollation, makeCmdObjIgnoreSessionInfo, waitForActiveSamplingReplicaSet, diff --git a/jstests/sharding/analyze_shard_key/persist_sampled_diffs.js b/jstests/sharding/analyze_shard_key/persist_sampled_diffs.js index 71330d456fe23..c6fb2cc968513 100644 --- a/jstests/sharding/analyze_shard_key/persist_sampled_diffs.js +++ b/jstests/sharding/analyze_shard_key/persist_sampled_diffs.js @@ -9,6 +9,7 @@ "use strict"; load("jstests/libs/config_shard_util.js"); +load("jstests/libs/uuid_util.js"); load("jstests/sharding/analyze_shard_key/libs/query_sampling_util.js"); const testCases = []; @@ -112,7 +113,7 @@ function testDiffs(rst, testCase, expectSampling) { // If running on the config server, use "config" as the database name since it is illegal to // create a user database on the config server. const dbName = rst.isConfigRS ? "config" : "testDb"; - const collName = "testColl-" + QuerySamplingUtil.generateRandomString(); + const collName = "testColl-" + extractUUIDFromObject(UUID()); const ns = dbName + "." + collName; const primary = rst.getPrimary(); diff --git a/jstests/sharding/analyze_shard_key/persist_sampled_read_queries.js b/jstests/sharding/analyze_shard_key/persist_sampled_read_queries.js index 661bdbd5a46e8..6c96d0267393e 100644 --- a/jstests/sharding/analyze_shard_key/persist_sampled_read_queries.js +++ b/jstests/sharding/analyze_shard_key/persist_sampled_read_queries.js @@ -8,6 +8,7 @@ "use strict"; load("jstests/libs/config_shard_util.js"); +load("jstests/libs/uuid_util.js"); load("jstests/sharding/analyze_shard_key/libs/query_sampling_util.js"); const supportedTestCases = [ @@ -38,7 +39,7 @@ function testReadCmd(rst, cmdOpts, testCase) { // If running on the config server, use "config" as the database name since it is illegal to // create a user database on the config server. const dbName = rst.isConfigRS ? "config" : "testDb"; - const collName = "testColl-" + cmdOpts.cmdName + "-" + QuerySamplingUtil.generateRandomString(); + const collName = "testColl-" + cmdOpts.cmdName + "-" + extractUUIDFromObject(UUID()); const ns = dbName + "." + collName; const primary = rst.getPrimary(); diff --git a/jstests/sharding/analyze_shard_key/persist_sampled_write_queries.js b/jstests/sharding/analyze_shard_key/persist_sampled_write_queries.js index 87accc46197b3..7e03fa717d7f1 100644 --- a/jstests/sharding/analyze_shard_key/persist_sampled_write_queries.js +++ b/jstests/sharding/analyze_shard_key/persist_sampled_write_queries.js @@ -8,6 +8,7 @@ "use strict"; load("jstests/libs/config_shard_util.js"); +load("jstests/libs/uuid_util.js"); load("jstests/sharding/analyze_shard_key/libs/query_sampling_util.js"); const supportedTestCases = [ @@ -27,7 +28,7 @@ function testWriteCmd(rst, cmdOpts, testCase) { // If running on the config server, use "config" as the database name since it is illegal to // create a user database on the config server. const dbName = rst.isConfigRS ? "config" : "testDb"; - const collName = "testColl-" + cmdOpts.cmdName + "-" + QuerySamplingUtil.generateRandomString(); + const collName = "testColl-" + cmdOpts.cmdName + "-" + extractUUIDFromObject(UUID()); const ns = dbName + "." + collName; const primary = rst.getPrimary(); @@ -199,7 +200,7 @@ function testFindAndModifyCmd(rst, testCases) { function testInsertCmd(rst) { const dbName = "testDb"; - const collName = "testColl-insert-" + QuerySamplingUtil.generateRandomString(); + const collName = "testColl-insert-" + extractUUIDFromObject(UUID()); const primary = rst.getPrimary(); const db = primary.getDB(dbName); // Verify that no mongods support persisting sampled insert queries. Specifically, "sampleId" diff --git a/jstests/sharding/max_time_ms_connection_pool.js b/jstests/sharding/max_time_ms_connection_pool.js index f4b5422c76089..54ae36b8e0db0 100644 --- a/jstests/sharding/max_time_ms_connection_pool.js +++ b/jstests/sharding/max_time_ms_connection_pool.js @@ -1,10 +1,12 @@ /** - * * Tests the rewrite of NetworkInterfaceExceededTimeLimit exception coming from * `executor/connection_pool.cpp` into MaxTimeMSError when MaxTimeMS option is set for a given * sharding command. * - * @tags: [requires_fcv_61] + * @tags: [ + * requires_fcv_61, + * does_not_support_stepdowns, + * ] */ (function() { diff --git a/jstests/sharding/refresh_sessions.js b/jstests/sharding/refresh_sessions.js index c6d229707ca56..dec07890b387c 100644 --- a/jstests/sharding/refresh_sessions.js +++ b/jstests/sharding/refresh_sessions.js @@ -8,8 +8,15 @@ var sessionsDb = "config"; var refresh = {refreshLogicalSessionCacheNow: 1}; var startSession = {startSession: 1}; -// Create a cluster with 1 shard. -var cluster = new ShardingTest({shards: 2}); +var cluster = new ShardingTest({ + mongos: [{setParameter: {sessionWriteConcernTimeoutSystemMillis: 0, sessionMaxBatchSize: 500}}], + shards: 2, + rs: {setParameter: {sessionWriteConcernTimeoutSystemMillis: 0, sessionMaxBatchSize: 500}}, + other: { + configOptions: + {setParameter: {sessionWriteConcernTimeoutSystemMillis: 0, sessionMaxBatchSize: 500}} + } +}); // Test that we can refresh without any sessions, as a sanity check. { diff --git a/jstests/sharding/ttl_deletes_not_targeting_orphaned_documents.js b/jstests/sharding/ttl_deletes_not_targeting_orphaned_documents.js index 10ec9c91e314c..7590dce0a739b 100644 --- a/jstests/sharding/ttl_deletes_not_targeting_orphaned_documents.js +++ b/jstests/sharding/ttl_deletes_not_targeting_orphaned_documents.js @@ -21,14 +21,12 @@ const testDB = st.s.getDB('test'); const coll = testDB[jsTest.name()]; const collName = coll.getFullName(); +// Shard a collection on _id:1 so that the initial chunk will reside on the primary shard (shard0) assert.commandWorked( st.s.adminCommand({enableSharding: dbName, primaryShard: st.shard0.shardName})); assert.commandWorked(st.s.adminCommand({shardCollection: collName, key: {_id: 1}})); -// Initialize TTL index: delete documents with field `a: ` after 20 seconds -assert.commandWorked(coll.createIndex({a: 1}, {expireAfterSeconds: 20})); - -// Insert documents that are going to be deleted in 20 seconds +// Insert documents that are going to be deleted by the TTL index created later on const currTime = new Date(); var bulk = coll.initializeUnorderedBulkOp(); const nDocs = 100; @@ -37,16 +35,20 @@ for (let i = 0; i < nDocs; i++) { } assert.commandWorked(bulk.execute()); -// Move all documents on other shards +// Move all documents to the other shard (shard1) but keep a chunk on shard0 to create the TTL index +assert.commandWorked(st.s.adminCommand({split: collName, middle: {_id: -1}})); assert.commandWorked( st.s.adminCommand({moveChunk: collName, find: {_id: 0}, to: st.shard1.shardName})); -// Verify that TTL index worked properly on owned documents +// Initialize TTL index: delete documents with field `a: ` older than 1 second +assert.commandWorked(coll.createIndex({a: 1}, {expireAfterSeconds: 1})); + +// Verify that TTL index worked properly on owned documents on shard1 assert.soon(function() { return coll.countDocuments({}) == 0; }, "Failed to move all documents", 60000 /* 60 seconds */, 5000 /* 5 seconds */); -// Verify that TTL index did not delete orphaned documents +// Verify that TTL index did not delete orphaned documents on shard0 assert.eq(nDocs, st.rs0.getPrimary().getCollection(collName).countDocuments({})); st.stop(); diff --git a/jstests/sharding/verify_sessions_expiration_sharded.js b/jstests/sharding/verify_sessions_expiration_sharded.js index 0abcba21aca47..bb37f798b37f0 100644 --- a/jstests/sharding/verify_sessions_expiration_sharded.js +++ b/jstests/sharding/verify_sessions_expiration_sharded.js @@ -127,6 +127,10 @@ for (let i = 0; i < 3; i++) { lastUseValues[j] = sessionsCollectionArray[j].lastUse; } } + + // Date_t has the granularity of milliseconds, so we have to make sure we don't run this loop + // faster than that. + sleep(10); } // 3. Verify that letting sessions expire (simulated by manual deletion) will kill their diff --git a/site_scons/site_tools/oom_auto_retry.py b/site_scons/site_tools/oom_auto_retry.py index 7ff457d2798d6..1fed4cace4ea7 100644 --- a/site_scons/site_tools/oom_auto_retry.py +++ b/site_scons/site_tools/oom_auto_retry.py @@ -32,6 +32,10 @@ from typing import Callable, List, Dict +# Note: The auto-retry settings are prefixed w/ "OOM", but since it's an unconditional retry, +# it's not really OOM-specific. We're keeping the OOM prefix to make the code change simpler. +# (This custom retry logic will go away once the build is fully Bazelified). + def command_spawn_func(sh: str, escape: Callable[[str], str], cmd: str, args: List, env: Dict, target: List, source: List): @@ -39,11 +43,6 @@ def command_spawn_func(sh: str, escape: Callable[[str], str], cmd: str, args: Li success = False build_env = target[0].get_build_env() - oom_messages = [ - re.compile(msg, re.MULTILINE | re.DOTALL) - for msg in build_env.get('OOM_RETRY_MESSAGES', []) - ] - oom_returncodes = [int(returncode) for returncode in build_env.get('OOM_RETRY_RETURNCODES', [])] max_retries = build_env.get('OOM_RETRY_ATTEMPTS', 10) oom_max_retry_delay = build_env.get('OOM_RETRY_MAX_DELAY_SECONDS', 120) @@ -63,16 +62,13 @@ def command_spawn_func(sh: str, escape: Callable[[str], str], cmd: str, args: Li except subprocess.CalledProcessError as exc: print(f"{os.path.basename(__file__)} captured error:") print(exc.stdout) - if any([re.findall(oom_message, exc.stdout) for oom_message in oom_messages]) or any( - [oom_returncode == exc.returncode for oom_returncode in oom_returncodes]): - retries += 1 - retry_delay = int((time.time() - start_time) + - oom_max_retry_delay * random.random()) - print(f"Ran out of memory while trying to build {target[0]}", ) - if retries <= max_retries: - print(f"trying again in {retry_delay} seconds with retry attempt {retries}") - time.sleep(retry_delay) - continue + retries += 1 + retry_delay = int((time.time() - start_time) + oom_max_retry_delay * random.random()) + print(f"Failed while trying to build {target[0]}", ) + if retries <= max_retries: + print(f"trying again in {retry_delay} seconds with retry attempt {retries}") + time.sleep(retry_delay) + continue # There was no OOM error or no more OOM retries left return exc.returncode diff --git a/src/mongo/bson/bsonelement.cpp b/src/mongo/bson/bsonelement.cpp index 99666165e3969..258f2d7d032a8 100644 --- a/src/mongo/bson/bsonelement.cpp +++ b/src/mongo/bson/bsonelement.cpp @@ -1023,7 +1023,7 @@ bool BSONObj::coerceVector(std::vector* out) const { /** * Types used to represent BSONElement memory in the Visual Studio debugger */ -#if defined(_MSC_VER) && defined(_DEBUG) +#if defined(_MSC_VER) struct BSONElementData { char type; char name; @@ -1039,6 +1039,6 @@ struct BSONElementDBRefType { } bsonElementDBPointerType; struct BSONElementCodeWithScopeType { } bsonElementCodeWithScopeType; -#endif // defined(_MSC_VER) && defined(_DEBUG) +#endif // defined(_MSC_VER) } // namespace mongo diff --git a/src/mongo/bson/bsonobj.cpp b/src/mongo/bson/bsonobj.cpp index c18cea72cec23..54736ffbeeab3 100644 --- a/src/mongo/bson/bsonobj.cpp +++ b/src/mongo/bson/bsonobj.cpp @@ -904,7 +904,7 @@ BSONArrayIteratorSorted::BSONArrayIteratorSorted(const BSONArray& array) /** * Types used to represent BSONObj and BSONArray memory in the Visual Studio debugger */ -#if defined(_MSC_VER) && defined(_DEBUG) +#if defined(_MSC_VER) struct BSONObjData { int32_t size; } bsonObjDataInstance; @@ -912,6 +912,6 @@ struct BSONObjData { struct BSONArrayData { int32_t size; } bsonObjArrayInstance; -#endif // defined(_MSC_VER) && defined(_DEBUG) +#endif // defined(_MSC_VER) } // namespace mongo diff --git a/src/mongo/db/auth/authorization_manager_impl.cpp b/src/mongo/db/auth/authorization_manager_impl.cpp index 40295852468e0..f9ebdea91293d 100644 --- a/src/mongo/db/auth/authorization_manager_impl.cpp +++ b/src/mongo/db/auth/authorization_manager_impl.cpp @@ -262,6 +262,7 @@ AuthorizationManagerImpl::AuthorizationManagerImpl( AuthorizationManagerImpl::~AuthorizationManagerImpl() = default; std::unique_ptr AuthorizationManagerImpl::makeAuthorizationSession() { + invariant(_externalState != nullptr); return std::make_unique( _externalState->makeAuthzSessionExternalState(this), AuthorizationSessionImpl::InstallMockForTestingOrAuthImpl{}); diff --git a/src/mongo/db/catalog/collection_catalog.cpp b/src/mongo/db/catalog/collection_catalog.cpp index e973a38290fb9..2b386aa80f2c3 100644 --- a/src/mongo/db/catalog/collection_catalog.cpp +++ b/src/mongo/db/catalog/collection_catalog.cpp @@ -51,6 +51,11 @@ namespace mongo { + +// Failpoint which causes to hang after wuow commits, before publishing the catalog updates on a +// given namespace. +MONGO_FAIL_POINT_DEFINE(hangBeforePublishingCatalogUpdates); + namespace { // Sentinel id for marking a catalogId mapping range as unknown. Must use an invalid RecordId. static RecordId kUnknownRangeMarkerId = RecordId::minLong(); @@ -300,6 +305,21 @@ class CollectionCatalog::PublishCatalogUpdates final : public RecoveryUnit::Chan // Create catalog write jobs for all updates registered in this WriteUnitOfWork auto entries = _uncommittedCatalogUpdates.releaseEntries(); for (auto&& entry : entries) { + hangBeforePublishingCatalogUpdates.executeIf( + [&](const BSONObj& data) { + LOGV2( + 9089303, "hangBeforePublishingCatalogUpdates enabled", logAttrs(entry.nss)); + hangBeforePublishingCatalogUpdates.pauseWhileSet(); + }, + [&](const BSONObj& data) { + const auto tenantField = data.getField("tenant"); + const auto tenantId = tenantField.ok() + ? boost::optional(TenantId::parseFromBSON(tenantField)) + : boost::none; + const auto fpNss = + NamespaceStringUtil::deserialize(tenantId, data["collectionNS"].str()); + return fpNss.isEmpty() || entry.nss == fpNss; + }); switch (entry.action) { case UncommittedCatalogUpdates::Entry::Action::kWritableCollection: { writeJobs.push_back([collection = std::move(entry.collection), @@ -2039,6 +2059,46 @@ std::set CollectionCatalog::getAllTenants() const { return ret; } +std::vector CollectionCatalog::getAllConsistentDbNames( + OperationContext* opCtx) const { + return getAllConsistentDbNamesForTenant(opCtx, boost::none); +} + +std::vector CollectionCatalog::getAllConsistentDbNamesForTenant( + OperationContext* opCtx, boost::optional tenantId) const { + // The caller must have an active storage snapshot + tassert(9089300, + "cannot get database list consistent to a snapshot without an active snapshot", + opCtx->recoveryUnit()->isActive()); + + // First get the dbnames that are not pending commit + std::vector ret = getAllDbNamesForTenant(tenantId); + stdx::unordered_set visitedDBs(ret.begin(), ret.end()); + auto insertSortedIfUnique = [&ret, &visitedDBs](DatabaseName dbname) { + auto [_, isNewDB] = visitedDBs.emplace(dbname); + if (isNewDB) { + ret.insert(std::lower_bound(ret.begin(), ret.end(), dbname), dbname); + } + }; + + // Now iterate over uncommitted list and validate against the storage snapshot. + // Only consider databases we have not seen so far. + auto readTimestamp = opCtx->recoveryUnit()->getPointInTimeReadTimestamp(opCtx); + tassert(9089301, + "point in time catalog lookup for a database list is not supported", + RecoveryUnit::ReadSource::kNoTimestamp == + opCtx->recoveryUnit()->getTimestampReadSource()); + for (auto const& [ns, coll] : _pendingCommitNamespaces) { + if (!visitedDBs.contains(ns.dbName())) { + if (establishConsistentCollection(opCtx, ns, readTimestamp)) { + insertSortedIfUnique(ns.dbName()); + } + } + } + + return ret; +} + void CollectionCatalog::setAllDatabaseProfileFilters(std::shared_ptr filter) { auto dbProfileSettingsWriter = _databaseProfileSettings.transient(); for (const auto& [dbName, settings] : _databaseProfileSettings) { diff --git a/src/mongo/db/catalog/collection_catalog.h b/src/mongo/db/catalog/collection_catalog.h index afce07e3692df..631e08b0f451a 100644 --- a/src/mongo/db/catalog/collection_catalog.h +++ b/src/mongo/db/catalog/collection_catalog.h @@ -570,6 +570,29 @@ class CollectionCatalog { */ std::set getAllTenants() const; + /** + * This function gets all the database names. The result is sorted in alphabetical ascending + * order. The returned list is consistent with the storage snapshot. + * + * Callers of this method must hold an active storage snapshot. This method takes a global lock + * in MODE_IS. + * + * Unlike DatabaseHolder::getNames(), this does not return databases that are empty. + */ + std::vector getAllConsistentDbNames(OperationContext* opCtx) const; + + /** + * This function gets all the database names associated with tenantId. The result is sorted in + * alphabetical ascending order. The returned list is consistent with the storage snapshot. + * + * Callers of this method must hold an active storage snapshot. This method takes a global lock + * in MODE_IS. + * + * Unlike DatabaseHolder::getNames(), this does not return databases that are empty. + */ + std::vector getAllConsistentDbNamesForTenant( + OperationContext* opCtx, boost::optional tenantId) const; + /** * Updates the profile filter on all databases with non-default settings. */ diff --git a/src/mongo/db/commands/create_indexes_cmd.cpp b/src/mongo/db/commands/create_indexes_cmd.cpp index 7a3e695e5f625..a76e35e83f1e9 100644 --- a/src/mongo/db/commands/create_indexes_cmd.cpp +++ b/src/mongo/db/commands/create_indexes_cmd.cpp @@ -169,7 +169,8 @@ void validateTTLOptions(OperationContext* opCtx, } const auto clusteredAndCapped = [&](LockMode mode) { - AutoGetCollection collection(opCtx, ns, mode); + AutoGetCollection collection( + opCtx, ns, mode, AutoGetCollection::Options{}.expectedUUID(cmd.getCollectionUUID())); if (collection) { const auto c = collection.getCollection().get(); if (c->getClusteredInfo() && c->isCapped()) { @@ -190,7 +191,8 @@ void validateTTLOptions(OperationContext* opCtx, void checkEncryptedFieldIndexRestrictions(OperationContext* opCtx, const NamespaceString& ns, const CreateIndexesCommand& cmd) { - AutoGetCollection collection(opCtx, ns, MODE_IS); + AutoGetCollection collection( + opCtx, ns, MODE_IS, AutoGetCollection::Options{}.expectedUUID(cmd.getCollectionUUID())); if (!collection) { return; } diff --git a/src/mongo/db/commands/generic_servers.cpp b/src/mongo/db/commands/generic_servers.cpp index a565f4924f586..c653a077745c1 100644 --- a/src/mongo/db/commands/generic_servers.cpp +++ b/src/mongo/db/commands/generic_servers.cpp @@ -157,7 +157,13 @@ HostInfoReply HostInfoCmd::Invocation::typedRun(OperationContext*) { system.setCpuAddrSize(static_cast(p.getAddrSize())); system.setMemSizeMB(static_cast(p.getSystemMemSizeMB())); system.setMemLimitMB(static_cast(p.getMemSizeMB())); - system.setNumCores(static_cast(p.getNumAvailableCores())); + system.setNumCores(static_cast(p.getNumLogicalCores())); + const auto num_cores_avl_to_process = p.getNumCoresAvailableToProcess(); + // Adding the num cores available to process only if API returns successfully ie. value >=0 + if (num_cores_avl_to_process >= 0) { + system.setNumCoresAvailableToProcess(static_cast(num_cores_avl_to_process)); + } + system.setNumPhysicalCores(static_cast(p.getNumPhysicalCores())); system.setNumCpuSockets(static_cast(p.getNumCpuSockets())); system.setCpuArch(p.getArch()); diff --git a/src/mongo/db/commands/generic_servers.idl b/src/mongo/db/commands/generic_servers.idl index 33b54dc465132..691adaec59d8d 100644 --- a/src/mongo/db/commands/generic_servers.idl +++ b/src/mongo/db/commands/generic_servers.idl @@ -65,6 +65,7 @@ structs: memSizeMB: long memLimitMB: long numCores: int + numCoresAvailableToProcess: int numPhysicalCores: int numCpuSockets: int cpuArch: string diff --git a/src/mongo/db/commands/list_collections.cpp b/src/mongo/db/commands/list_collections.cpp index 544e6e222eaa5..b4b8aef0c8425 100644 --- a/src/mongo/db/commands/list_collections.cpp +++ b/src/mongo/db/commands/list_collections.cpp @@ -348,6 +348,10 @@ class CmdListCollections : public ListCollectionsCmdVersion1Gen(); auto root = std::make_unique(expCtx.get(), ws.get()); auto readTimestamp = opCtx->recoveryUnit()->getPointInTimeReadTimestamp(opCtx); + tassert(9089302, + "point in time catalog lookup for a collection list is not supported", + RecoveryUnit::ReadSource::kNoTimestamp == + opCtx->recoveryUnit()->getTimestampReadSource()); if (DatabaseHolder::get(opCtx)->dbExists(opCtx, dbName)) { if (auto collNames = _getExactNameMatches(matcher.get())) { diff --git a/src/mongo/db/commands/list_databases.cpp b/src/mongo/db/commands/list_databases.cpp index 3ad19b1d236ff..eb47fe39b2b64 100644 --- a/src/mongo/db/commands/list_databases.cpp +++ b/src/mongo/db/commands/list_databases.cpp @@ -117,23 +117,26 @@ class CmdListDatabases final : public ListDatabasesCmdVersion1Gen filter = list_databases::getFilter(cmd, opCtx, ns()); std::vector dbNames; - StorageEngine* storageEngine = getGlobalServiceContext()->getStorageEngine(); { - Lock::GlobalLock lk(opCtx, MODE_IS); + // Read lock free through a consistent in-memory catalog and storage snapshot. + AutoReadLockFree lockFreeReadBlock(opCtx); + auto catalog = CollectionCatalog::get(opCtx); + CurOpFailpointHelpers::waitWhileFailPointEnabled( &hangBeforeListDatabases, opCtx, "hangBeforeListDatabases", []() {}); - - dbNames = storageEngine->listDatabases(cmd.getDbName().tenantId()); + dbNames = + catalog->getAllConsistentDbNamesForTenant(opCtx, cmd.getDbName().tenantId()); } std::vector items; - int64_t totalSize = list_databases::setReplyItems(opCtx, - dbNames, - items, - storageEngine, - nameOnly, - filter, - false /* setTenantId */, - authorizedDatabases); + int64_t totalSize = + list_databases::setReplyItems(opCtx, + dbNames, + items, + getGlobalServiceContext()->getStorageEngine(), + nameOnly, + filter, + false /* setTenantId */, + authorizedDatabases); ListDatabasesReply reply(items); if (!nameOnly) { diff --git a/src/mongo/db/commands/list_databases_for_all_tenants.cpp b/src/mongo/db/commands/list_databases_for_all_tenants.cpp index c7574aef1b4b2..451c66a76ed35 100644 --- a/src/mongo/db/commands/list_databases_for_all_tenants.cpp +++ b/src/mongo/db/commands/list_databases_for_all_tenants.cpp @@ -112,22 +112,23 @@ class CmdListDatabasesForAllTenants final : public TypedCommand filter = list_databases::getFilter(cmd, opCtx, ns()); std::vector dbNames; - StorageEngine* storageEngine = getGlobalServiceContext()->getStorageEngine(); { - Lock::GlobalLock lk(opCtx, MODE_IS); - dbNames = storageEngine->listDatabases(); + // Read lock free through a consistent in-memory catalog and storage snapshot. + AutoReadLockFree lockFreeReadBlock(opCtx); + auto catalog = CollectionCatalog::get(opCtx); + dbNames = catalog->getAllConsistentDbNames(opCtx); } std::vector items; - int64_t totalSize = list_databases::setReplyItems(opCtx, - dbNames, - items, - storageEngine, - nameOnly, - filter, - true /* setTenantId */, - false /* authorizedDatabases*/); - + int64_t totalSize = + list_databases::setReplyItems(opCtx, + dbNames, + items, + getGlobalServiceContext()->getStorageEngine(), + nameOnly, + filter, + true /* setTenantId */, + false /* authorizedDatabases*/); Reply reply(items); if (!nameOnly) { reply.setTotalSize(totalSize); diff --git a/src/mongo/db/commands/set_cluster_parameter_invocation.cpp b/src/mongo/db/commands/set_cluster_parameter_invocation.cpp index ddc6a21fccc32..d14cd88361db6 100644 --- a/src/mongo/db/commands/set_cluster_parameter_invocation.cpp +++ b/src/mongo/db/commands/set_cluster_parameter_invocation.cpp @@ -156,6 +156,11 @@ StatusWith ClusterParameterDBClientService::updateParameterOnDisk( return Status(ErrorCodes::FailedToParse, errmsg); } + auto responseStatus = response.toStatus(); + if (!responseStatus.isOK()) { + return responseStatus; + } + return response.getNModified() > 0 || response.getN() > 0; } diff --git a/src/mongo/db/db_raii.cpp b/src/mongo/db/db_raii.cpp index 796f4c9fe6160..02bfc79958e9d 100644 --- a/src/mongo/db/db_raii.cpp +++ b/src/mongo/db/db_raii.cpp @@ -1130,6 +1130,11 @@ ConsistentCatalogAndSnapshot getConsistentCatalogAndSnapshot( // openCollection is eventually called to construct a Collection object from the durable // catalog. establishCappedSnapshotIfNeeded(opCtx, catalogBeforeSnapshot, nsOrUUID); + if (resolvedSecondaryNamespaces) { + for (const auto& secondaryNss : *resolvedSecondaryNamespaces) { + establishCappedSnapshotIfNeeded(opCtx, catalogBeforeSnapshot, {secondaryNss}); + } + } openSnapshot(opCtx, nss.isOplog()); diff --git a/src/mongo/db/exec/projection.cpp b/src/mongo/db/exec/projection.cpp index 9cec5eac22b68..3794e3feff3f8 100644 --- a/src/mongo/db/exec/projection.cpp +++ b/src/mongo/db/exec/projection.cpp @@ -96,7 +96,8 @@ auto rehydrateIndexKey(const BSONObj& keyPattern, const BSONObj& dehydratedKey) BSONObjIterator valueIter{dehydratedKey}; while (keyIter.more() && valueIter.more()) { - auto fieldName = keyIter.next().fieldNameStringData(); + const auto& keyElt = keyIter.next(); + auto fieldName = keyElt.fieldNameStringData(); auto value = valueIter.next(); // Skip the $** index virtual field, as it's not part of the actual index key. @@ -104,6 +105,14 @@ auto rehydrateIndexKey(const BSONObj& keyPattern, const BSONObj& dehydratedKey) continue; } + // Skip hashed index fields. Rehydrating of index keys is used for covered projections. + // Rehydrating of hashed field value is pointless on its own. The query planner dependency + // analysis should make sure that a covered projection can only be generated for non-hashed + // fields. + if (keyElt.type() == mongo::String && keyElt.valueStringData() == IndexNames::HASHED) { + continue; + } + md.setNestedField(fieldName, Value{value}); } diff --git a/src/mongo/db/exec/sbe/sbe_test.cpp b/src/mongo/db/exec/sbe/sbe_test.cpp index c92649647f4e9..ba7bed9178416 100644 --- a/src/mongo/db/exec/sbe/sbe_test.cpp +++ b/src/mongo/db/exec/sbe/sbe_test.cpp @@ -227,6 +227,26 @@ TEST(SBEValues, HashCompound) { obj2->push_back("b"_sd, value::TypeTags::NumberDouble, value::bitcastFrom(-6.0)); obj2->push_back("c"_sd, value::TypeTags::NumberDouble, value::bitcastFrom(-7.0)); + ASSERT_EQUALS(value::hashValue(tag1, val1), value::hashValue(tag2, val2)); + + value::releaseValue(tag1, val1); + value::releaseValue(tag2, val2); + } + + { + auto [tag1, val1] = value::makeNewArraySet(); + auto set1 = value::getArraySetView(val1); + set1->push_back(value::TypeTags::NumberInt32, value::bitcastFrom(-5)); + set1->push_back(value::TypeTags::NumberInt32, value::bitcastFrom(-6)); + set1->push_back(value::TypeTags::NumberInt32, value::bitcastFrom(-7)); + + auto [tag2, val2] = value::makeNewArraySet(); + auto set2 = value::getArraySetView(val2); + set2->push_back(value::TypeTags::NumberDouble, value::bitcastFrom(-7.0)); + set2->push_back(value::TypeTags::NumberDouble, value::bitcastFrom(-6.0)); + set2->push_back(value::TypeTags::NumberDouble, value::bitcastFrom(-5.0)); + + ASSERT_EQUALS(value::hashValue(tag1, val1), value::hashValue(tag2, val2)); value::releaseValue(tag1, val1); diff --git a/src/mongo/db/exec/sbe/values/value.cpp b/src/mongo/db/exec/sbe/values/value.cpp index 3f2b1bd7f638c..79731f5abcf7f 100644 --- a/src/mongo/db/exec/sbe/values/value.cpp +++ b/src/mongo/db/exec/sbe/values/value.cpp @@ -510,7 +510,6 @@ std::size_t hashValue(TypeTags tag, Value val, const CollatorInterface* collator case TypeTags::ksValue: return getKeyStringView(val)->hash(); case TypeTags::Array: - case TypeTags::ArraySet: case TypeTags::bsonArray: { auto arr = ArrayEnumerator{tag, val}; auto res = hashInit(); @@ -524,6 +523,19 @@ std::size_t hashValue(TypeTags tag, Value val, const CollatorInterface* collator return res; } + case TypeTags::ArraySet: { + size_t size = getArraySetView(val)->size(); + std::vector valueHashes; + valueHashes.reserve(size); + for (ArrayEnumerator arr{tag, val}; !arr.atEnd(); arr.advance()) { + auto [elemTag, elemVal] = arr.getViewOfValue(); + valueHashes.push_back(hashValue(elemTag, elemVal)); + } + // TODO SERVER-92666 Implement a more efficient hashing algorithm + std::sort(valueHashes.begin(), valueHashes.end()); + return std::accumulate( + valueHashes.begin(), valueHashes.end(), hashInit(), &hashCombine); + } case TypeTags::Object: case TypeTags::bsonObject: { auto obj = ObjectEnumerator{tag, val}; diff --git a/src/mongo/db/ftdc/ftdc_system_stats_linux.cpp b/src/mongo/db/ftdc/ftdc_system_stats_linux.cpp index 7832f3f026f98..c8c08d7cebf6b 100644 --- a/src/mongo/db/ftdc/ftdc_system_stats_linux.cpp +++ b/src/mongo/db/ftdc/ftdc_system_stats_linux.cpp @@ -107,7 +107,13 @@ class LinuxSystemMetricsCollector final : public SystemMetricsCollector { // Include the number of cpus to simplify client calculations ProcessInfo p; - subObjBuilder.append("num_cpus", static_cast(p.getNumAvailableCores())); + subObjBuilder.append("num_logical_cores", static_cast(p.getNumLogicalCores())); + const auto num_cores_avlbl_to_process = p.getNumCoresAvailableToProcess(); + // Adding the num cores available to process only if API is successful ie. value >=0 + if (num_cores_avlbl_to_process >= 0) { + subObjBuilder.append("num_cores_available_to_process", + static_cast(num_cores_avlbl_to_process)); + } processStatusErrors( procparser::parseProcStatFile("/proc/stat"_sd, kCpuKeys, &subObjBuilder), diff --git a/src/mongo/db/pipeline/document_source_change_stream.cpp b/src/mongo/db/pipeline/document_source_change_stream.cpp index 60063c79dab0f..21e9ef0c9fb35 100644 --- a/src/mongo/db/pipeline/document_source_change_stream.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream.cpp @@ -354,7 +354,7 @@ std::list> DocumentSourceChangeStream::_bui // If 'showExpandedEvents' is NOT set, add a filter that returns only classic change events. if (!spec.getShowExpandedEvents()) { - stages.push_back(DocumentSourceMatch::create( + stages.push_back(DocumentSourceInternalChangeStreamMatch::create( change_stream_filter::getMatchFilterForClassicOperationTypes(), expCtx)); } return stages; @@ -371,6 +371,12 @@ void DocumentSourceChangeStream::assertIsLegalSpecification( (replCoord && replCoord->getReplicationMode() == repl::ReplicationCoordinator::Mode::modeReplSet)); + // We will not validate user specified options when we are not expecting to execute queries, + // such as during $queryStats. + if (!expCtx->mongoProcessInterface->isExpectedToExecuteQueries()) { + return; + } + // If 'allChangesForCluster' is true, the stream must be opened on the 'admin' database with // {aggregate: 1}. uassert(ErrorCodes::InvalidOptions, diff --git a/src/mongo/db/pipeline/document_source_change_stream.h b/src/mongo/db/pipeline/document_source_change_stream.h index 236be5e3994c0..88b07bf1558d4 100644 --- a/src/mongo/db/pipeline/document_source_change_stream.h +++ b/src/mongo/db/pipeline/document_source_change_stream.h @@ -342,4 +342,29 @@ class LiteParsedDocumentSourceChangeStreamInternal final } }; +/** + * A DocumentSource class for all internal change stream stages. This class is useful for + * shared logic between all of the internal change stream stages. For internally created match + * stages see 'DocumentSourceInternalChangeStreamMatch'. + */ +class DocumentSourceInternalChangeStreamStage : public DocumentSource { +public: + DocumentSourceInternalChangeStreamStage(StringData stageName, + const boost::intrusive_ptr& expCtx) + : DocumentSource(stageName, expCtx) {} + + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const override { + if (opts.literalPolicy != LiteralSerializationPolicy::kUnchanged || + opts.transformIdentifiers) { + // Stages made internally by 'DocumentSourceChangeStream' should not be serialized for + // query stats. For query stats we will serialize only the user specified $changeStream + // stage. + return Value(); + } + return doSerialize(opts); + } + + virtual Value doSerialize(const SerializationOptions& opts) const = 0; +}; + } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_change_stream.idl b/src/mongo/db/pipeline/document_source_change_stream.idl index d0fbe4babea33..99369bb7b4d46 100644 --- a/src/mongo/db/pipeline/document_source_change_stream.idl +++ b/src/mongo/db/pipeline/document_source_change_stream.idl @@ -203,51 +203,41 @@ structs: strict: true description: A document used to specify the $_internalChangeStreamCheckInvalidate stage of an aggregation pipeline. - query_shape_component: true fields: startAfterInvalidate: type: resumeToken optional: true - query_shape: custom DocumentSourceChangeStreamCheckResumabilitySpec: strict: true description: A document used to specify the $_internalChangeStreamCheckResumability stage of an aggregation pipeline. - query_shape_component: true fields: resumeToken: type: resumeToken optional: false - query_shape: custom DocumentSourceChangeStreamAddPreImageSpec: strict: true description: A document used to specify the $_internalChangeStreamAddPreImage stage of an aggregation pipeline. - query_shape_component: true fields: fullDocumentBeforeChange: type: FullDocumentBeforeChangeMode - query_shape: parameter DocumentSourceChangeStreamAddPostImageSpec: strict: true description: A document used to specify the $_internalChangeStreamAddPostImage stage of an aggregation pipeline. - query_shape_component: true fields: fullDocument: type: FullDocumentMode - query_shape: parameter DocumentSourceChangeStreamHandleTopologyChangeSpec: strict: true description: A document used to specify the $_internalChangeStreamHandleTopologyChange stage of an aggregation pipeline. - query_shape_component: true fields: originalAggregateCommand: type: object optional: true - query_shape: literal diff --git a/src/mongo/db/pipeline/document_source_change_stream_add_post_image.cpp b/src/mongo/db/pipeline/document_source_change_stream_add_post_image.cpp index 7edad502fd132..740474acc2273 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_add_post_image.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_add_post_image.cpp @@ -209,14 +209,13 @@ boost::optional DocumentSourceChangeStreamAddPostImage::lookupLatestPo pExpCtx, nss, *resumeTokenData.uuid, documentKey, std::move(readConcern)); } -Value DocumentSourceChangeStreamAddPostImage::serialize(const SerializationOptions& opts) const { +Value DocumentSourceChangeStreamAddPostImage::doSerialize(const SerializationOptions& opts) const { return opts.verbosity ? Value(Document{ {DocumentSourceChangeStream::kStageName, Document{{"stage"_sd, kStageName}, {kFullDocumentFieldName, FullDocumentMode_serializer(_fullDocumentMode)}}}}) - : Value(Document{ - {kStageName, - DocumentSourceChangeStreamAddPostImageSpec(_fullDocumentMode).toBSON(opts)}}); + : Value(Document{{kStageName, + DocumentSourceChangeStreamAddPostImageSpec(_fullDocumentMode).toBSON()}}); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_change_stream_add_post_image.h b/src/mongo/db/pipeline/document_source_change_stream_add_post_image.h index c778090634ca8..a2e3af33646d0 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_add_post_image.h +++ b/src/mongo/db/pipeline/document_source_change_stream_add_post_image.h @@ -38,7 +38,8 @@ namespace mongo { * Part of the change stream API machinery used to look up the post-image of a document. Uses the * "documentKey" field of the input to look up the new version of the document. */ -class DocumentSourceChangeStreamAddPostImage final : public DocumentSource { +class DocumentSourceChangeStreamAddPostImage final + : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamAddPostImage"_sd; static constexpr StringData kFullDocumentFieldName = @@ -116,7 +117,7 @@ class DocumentSourceChangeStreamAddPostImage final : public DocumentSource { void addVariableRefs(std::set* refs) const final {} - Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; const char* getSourceName() const final { return kStageName.rawData(); @@ -125,7 +126,8 @@ class DocumentSourceChangeStreamAddPostImage final : public DocumentSource { private: DocumentSourceChangeStreamAddPostImage(const boost::intrusive_ptr& expCtx, const FullDocumentModeEnum fullDocumentMode) - : DocumentSource(kStageName, expCtx), _fullDocumentMode(fullDocumentMode) { + : DocumentSourceInternalChangeStreamStage(kStageName, expCtx), + _fullDocumentMode(fullDocumentMode) { tassert(5842300, "the 'fullDocument' field cannot be 'default'", _fullDocumentMode != FullDocumentModeEnum::kDefault); diff --git a/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.cpp b/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.cpp index ed5984d699c30..f51ab89b827cf 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.cpp @@ -139,16 +139,16 @@ boost::optional DocumentSourceChangeStreamAddPreImage::lookupPreImage( return preImageField.getDocument().getOwned(); } -Value DocumentSourceChangeStreamAddPreImage::serialize(const SerializationOptions& opts) const { +Value DocumentSourceChangeStreamAddPreImage::doSerialize(const SerializationOptions& opts) const { return opts.verbosity ? Value(Document{ {DocumentSourceChangeStream::kStageName, Document{{"stage"_sd, "internalAddPreImage"_sd}, {"fullDocumentBeforeChange"_sd, FullDocumentBeforeChangeMode_serializer(_fullDocumentBeforeChangeMode)}}}}) - : Value(Document{{kStageName, - DocumentSourceChangeStreamAddPreImageSpec(_fullDocumentBeforeChangeMode) - .toBSON(opts)}}); + : Value(Document{ + {kStageName, + DocumentSourceChangeStreamAddPreImageSpec(_fullDocumentBeforeChangeMode).toBSON()}}); } std::string DocumentSourceChangeStreamAddPreImage::makePreImageNotFoundErrorMsg( diff --git a/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.h b/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.h index 5bf4aa4d066f4..64952df5a7c14 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.h +++ b/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.h @@ -40,7 +40,7 @@ namespace mongo { * The identifier of pre-image is in "preImageId" field of the incoming document. The pre-image is * set to "fullDocumentBeforeChange" field of the returned document. */ -class DocumentSourceChangeStreamAddPreImage final : public DocumentSource { +class DocumentSourceChangeStreamAddPreImage final : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamAddPreImage"_sd; static constexpr StringData kFullDocumentBeforeChangeFieldName = @@ -67,7 +67,8 @@ class DocumentSourceChangeStreamAddPreImage final : public DocumentSource { DocumentSourceChangeStreamAddPreImage(const boost::intrusive_ptr& expCtx, FullDocumentBeforeChangeModeEnum mode) - : DocumentSource(kStageName, expCtx), _fullDocumentBeforeChangeMode(mode) { + : DocumentSourceInternalChangeStreamStage(kStageName, expCtx), + _fullDocumentBeforeChangeMode(mode) { // This stage should never be created with FullDocumentBeforeChangeMode::kOff. invariant(_fullDocumentBeforeChangeMode != FullDocumentBeforeChangeModeEnum::kOff); } @@ -109,7 +110,7 @@ class DocumentSourceChangeStreamAddPreImage final : public DocumentSource { void addVariableRefs(std::set* refs) const final {} - Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; const char* getSourceName() const final { return kStageName.rawData(); diff --git a/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.cpp b/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.cpp index 50d77bcf1bee4..94fa9310beeae 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.cpp @@ -181,7 +181,8 @@ DocumentSource::GetNextResult DocumentSourceChangeStreamCheckInvalidate::doGetNe return nextInput; } -Value DocumentSourceChangeStreamCheckInvalidate::serialize(const SerializationOptions& opts) const { +Value DocumentSourceChangeStreamCheckInvalidate::doSerialize( + const SerializationOptions& opts) const { BSONObjBuilder builder; if (opts.verbosity) { BSONObjBuilder sub(builder.subobjStart(DocumentSourceChangeStream::kStageName)); @@ -192,7 +193,7 @@ Value DocumentSourceChangeStreamCheckInvalidate::serialize(const SerializationOp if (_startAfterInvalidate) { spec.setStartAfterInvalidate(ResumeToken(*_startAfterInvalidate)); } - builder.append(DocumentSourceChangeStreamCheckInvalidate::kStageName, spec.toBSON(opts)); + builder.append(DocumentSourceChangeStreamCheckInvalidate::kStageName, spec.toBSON()); return Value(builder.obj()); } diff --git a/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.h b/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.h index 34ee2ad6c82e5..ce575abf42bc8 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.h +++ b/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.h @@ -39,7 +39,8 @@ namespace mongo { * "invalidate" entry for commands that should invalidate the change stream (e.g. collection drop * for a single-collection change stream). It is not intended to be created by the user. */ -class DocumentSourceChangeStreamCheckInvalidate final : public DocumentSource { +class DocumentSourceChangeStreamCheckInvalidate final + : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamCheckInvalidate"_sd; @@ -64,7 +65,7 @@ class DocumentSourceChangeStreamCheckInvalidate final : public DocumentSource { return boost::none; } - Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; void addVariableRefs(std::set* refs) const final {} @@ -81,7 +82,7 @@ class DocumentSourceChangeStreamCheckInvalidate final : public DocumentSource { */ DocumentSourceChangeStreamCheckInvalidate(const boost::intrusive_ptr& expCtx, boost::optional startAfterInvalidate) - : DocumentSource(kStageName, expCtx), + : DocumentSourceInternalChangeStreamStage(kStageName, expCtx), _startAfterInvalidate(std::move(startAfterInvalidate)) { invariant(!_startAfterInvalidate || _startAfterInvalidate->fromInvalidate == ResumeTokenData::kFromInvalidate); diff --git a/src/mongo/db/pipeline/document_source_change_stream_check_resumability.cpp b/src/mongo/db/pipeline/document_source_change_stream_check_resumability.cpp index bc3e2012a8694..685f52f0f3fbd 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_check_resumability.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_check_resumability.cpp @@ -133,7 +133,8 @@ DocumentSourceChangeStreamCheckResumability::compareAgainstClientResumeToken( DocumentSourceChangeStreamCheckResumability::DocumentSourceChangeStreamCheckResumability( const intrusive_ptr& expCtx, ResumeTokenData token) - : DocumentSource(getSourceName(), expCtx), _tokenFromClient(std::move(token)) {} + : DocumentSourceInternalChangeStreamStage(getSourceName(), expCtx), + _tokenFromClient(std::move(token)) {} intrusive_ptr DocumentSourceChangeStreamCheckResumability::create(const intrusive_ptr& expCtx, @@ -211,7 +212,7 @@ DocumentSource::GetNextResult DocumentSourceChangeStreamCheckResumability::doGet MONGO_UNREACHABLE; } -Value DocumentSourceChangeStreamCheckResumability::serialize( +Value DocumentSourceChangeStreamCheckResumability::doSerialize( const SerializationOptions& opts) const { BSONObjBuilder builder; if (opts.verbosity) { @@ -223,7 +224,7 @@ Value DocumentSourceChangeStreamCheckResumability::serialize( builder.append( kStageName, DocumentSourceChangeStreamCheckResumabilitySpec(ResumeToken(_tokenFromClient)) - .toBSON(opts)); + .toBSON()); } return Value(builder.obj()); } diff --git a/src/mongo/db/pipeline/document_source_change_stream_check_resumability.h b/src/mongo/db/pipeline/document_source_change_stream_check_resumability.h index 18e6dfa1f3026..239b2fe874f11 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_check_resumability.h +++ b/src/mongo/db/pipeline/document_source_change_stream_check_resumability.h @@ -59,7 +59,7 @@ namespace mongo { * - Otherwise we cannot resume, as we do not know if there were any events between the resume token * and the first matching document in the oplog. */ -class DocumentSourceChangeStreamCheckResumability : public DocumentSource { +class DocumentSourceChangeStreamCheckResumability : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamCheckResumability"_sd; @@ -90,7 +90,7 @@ class DocumentSourceChangeStreamCheckResumability : public DocumentSource { return boost::none; } - Value serialize(const SerializationOptions& opts = SerializationOptions{}) const override; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const override; void addVariableRefs(std::set* refs) const final {} diff --git a/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.cpp b/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.cpp index ad2d8a4f3e27b..c5995ba54414e 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.cpp @@ -87,7 +87,7 @@ DocumentSource::GetNextResult DocumentSourceChangeStreamCheckTopologyChange::doG return nextInput; } -Value DocumentSourceChangeStreamCheckTopologyChange::serialize( +Value DocumentSourceChangeStreamCheckTopologyChange::doSerialize( const SerializationOptions& opts) const { if (opts.verbosity) { return Value(DOC(DocumentSourceChangeStream::kStageName diff --git a/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.h b/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.h index bc8ef02331c26..211aee0bc1309 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.h +++ b/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.h @@ -45,7 +45,8 @@ namespace mongo { * that previously may not have held any data for the collection being watched, and they contain the * information necessary for the mongoS to include the new shard in the merged change stream. */ -class DocumentSourceChangeStreamCheckTopologyChange final : public DocumentSource { +class DocumentSourceChangeStreamCheckTopologyChange final + : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamCheckTopologyChange"_sd; @@ -67,14 +68,14 @@ class DocumentSourceChangeStreamCheckTopologyChange final : public DocumentSourc return boost::none; } - Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; void addVariableRefs(std::set* refs) const final {} private: DocumentSourceChangeStreamCheckTopologyChange( const boost::intrusive_ptr& expCtx) - : DocumentSource(kStageName, expCtx) {} + : DocumentSourceInternalChangeStreamStage(kStageName, expCtx) {} GetNextResult doGetNext() final; }; diff --git a/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.cpp b/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.cpp index 0ae0ba95d7ed6..b5c24a36af8f7 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.cpp @@ -153,7 +153,7 @@ DocumentSource::GetNextResult DocumentSourceChangeStreamEnsureResumeTokenPresent } } -Value DocumentSourceChangeStreamEnsureResumeTokenPresent::serialize( +Value DocumentSourceChangeStreamEnsureResumeTokenPresent::doSerialize( const SerializationOptions& opts) const { BSONObjBuilder builder; if (opts.verbosity) { diff --git a/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.h b/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.h index 8fc70392337a0..0482e128379f2 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.h +++ b/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.h @@ -55,7 +55,7 @@ class DocumentSourceChangeStreamEnsureResumeTokenPresent final const boost::intrusive_ptr& expCtx, const DocumentSourceChangeStreamSpec& spec); - Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; private: /** diff --git a/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.cpp b/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.cpp index 28d7670c32fa7..f2d6b1e38902d 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.cpp @@ -129,7 +129,7 @@ DocumentSourceChangeStreamHandleTopologyChange::create( DocumentSourceChangeStreamHandleTopologyChange::DocumentSourceChangeStreamHandleTopologyChange( const boost::intrusive_ptr& expCtx) - : DocumentSource(kStageName, expCtx) {} + : DocumentSourceInternalChangeStreamStage(kStageName, expCtx) {} StageConstraints DocumentSourceChangeStreamHandleTopologyChange::constraints( Pipeline::SplitState) const { @@ -269,7 +269,7 @@ BSONObj DocumentSourceChangeStreamHandleTopologyChange::replaceResumeTokenInComm return newCmd.freeze().toBson(); } -Value DocumentSourceChangeStreamHandleTopologyChange::serialize( +Value DocumentSourceChangeStreamHandleTopologyChange::doSerialize( const SerializationOptions& opts) const { if (opts.verbosity) { return Value(DOC(DocumentSourceChangeStream::kStageName diff --git a/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.h b/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.h index e3dc5ce197aed..fc3fee21a5212 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.h +++ b/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.h @@ -46,7 +46,8 @@ namespace mongo { * the first time. When this event is detected, this stage will establish a new cursor on that * shard and add it to the cursors being merged. */ -class DocumentSourceChangeStreamHandleTopologyChange final : public DocumentSource { +class DocumentSourceChangeStreamHandleTopologyChange final + : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = change_stream_constants::stage_names::kHandleTopologyChange; @@ -64,7 +65,7 @@ class DocumentSourceChangeStreamHandleTopologyChange final : public DocumentSour return kStageName.rawData(); } - Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; StageConstraints constraints(Pipeline::SplitState) const final; diff --git a/src/mongo/db/pipeline/document_source_change_stream_oplog_match.cpp b/src/mongo/db/pipeline/document_source_change_stream_oplog_match.cpp index 31e056c328d06..a0db243ab9023 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_oplog_match.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_oplog_match.cpp @@ -96,8 +96,8 @@ std::unique_ptr buildOplogMatchFilter( DocumentSourceChangeStreamOplogMatch::DocumentSourceChangeStreamOplogMatch( Timestamp clusterTime, const boost::intrusive_ptr& expCtx) - : DocumentSourceMatch(change_stream_filter::buildOplogMatchFilter(expCtx, clusterTime), - expCtx) { + : DocumentSourceInternalChangeStreamMatch( + change_stream_filter::buildOplogMatchFilter(expCtx, clusterTime), expCtx) { _clusterTime = clusterTime; expCtx->tailableMode = TailableModeEnum::kTailableAndAwaitData; } @@ -205,7 +205,7 @@ Pipeline::SourceContainer::iterator DocumentSourceChangeStreamOplogMatch::doOpti return nextChangeStreamStageItr; } -Value DocumentSourceChangeStreamOplogMatch::serialize(const SerializationOptions& opts) const { +Value DocumentSourceChangeStreamOplogMatch::doSerialize(const SerializationOptions& opts) const { BSONObjBuilder builder; if (opts.verbosity) { BSONObjBuilder sub(builder.subobjStart(DocumentSourceChangeStream::kStageName)); @@ -215,13 +215,10 @@ Value DocumentSourceChangeStreamOplogMatch::serialize(const SerializationOptions sub.done(); } else { BSONObjBuilder sub(builder.subobjStart(kStageName)); - if (opts.literalPolicy != LiteralSerializationPolicy::kUnchanged || - opts.transformIdentifiers) { - sub.append(DocumentSourceChangeStreamOplogMatchSpec::kFilterFieldName, - getMatchExpression()->serialize(opts)); - } else { - DocumentSourceChangeStreamOplogMatchSpec(_predicate).serialize(&sub); - } + + // 'SerializationOptions' are not required here, since serialization for explain and query + // stats occur before this function call. + DocumentSourceChangeStreamOplogMatchSpec(_predicate).serialize(&sub); sub.done(); } return Value(builder.obj()); diff --git a/src/mongo/db/pipeline/document_source_change_stream_oplog_match.h b/src/mongo/db/pipeline/document_source_change_stream_oplog_match.h index f6e8772ba8ca4..6138fdaae3187 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_oplog_match.h +++ b/src/mongo/db/pipeline/document_source_change_stream_oplog_match.h @@ -36,7 +36,7 @@ namespace mongo { * A custom subclass of DocumentSourceMatch which is used to generate a $match stage to be applied * on the oplog. The stage requires itself to be the first stage in the pipeline. */ -class DocumentSourceChangeStreamOplogMatch final : public DocumentSourceMatch { +class DocumentSourceChangeStreamOplogMatch final : public DocumentSourceInternalChangeStreamMatch { public: static constexpr StringData kStageName = "$_internalChangeStreamOplogMatch"_sd; @@ -45,7 +45,7 @@ class DocumentSourceChangeStreamOplogMatch final : public DocumentSourceMatch { DocumentSourceChangeStreamOplogMatch(const DocumentSourceChangeStreamOplogMatch& other, const boost::intrusive_ptr& newExpCtx) - : DocumentSourceMatch(other, newExpCtx) { + : DocumentSourceInternalChangeStreamMatch(other, newExpCtx) { _clusterTime = other._clusterTime; _optimizedEndOfPipeline = other._optimizedEndOfPipeline; } @@ -74,7 +74,7 @@ class DocumentSourceChangeStreamOplogMatch final : public DocumentSourceMatch { StageConstraints constraints(Pipeline::SplitState pipeState) const final; - Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + Value doSerialize(const SerializationOptions& opts) const final; protected: Pipeline::SourceContainer::iterator doOptimizeAt(Pipeline::SourceContainer::iterator itr, @@ -88,7 +88,8 @@ class DocumentSourceChangeStreamOplogMatch final : public DocumentSourceMatch { */ DocumentSourceChangeStreamOplogMatch(BSONObj filter, const boost::intrusive_ptr& expCtx) - : DocumentSourceMatch(std::move(filter), expCtx), _optimizedEndOfPipeline(true) { + : DocumentSourceInternalChangeStreamMatch(std::move(filter), expCtx), + _optimizedEndOfPipeline(true) { expCtx->tailableMode = TailableModeEnum::kTailableAndAwaitData; } diff --git a/src/mongo/db/pipeline/document_source_change_stream_test.cpp b/src/mongo/db/pipeline/document_source_change_stream_test.cpp index 3cd512376431b..b0476dc6dd0f9 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_test.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_test.cpp @@ -95,15 +95,23 @@ using DSChangeStream = DocumentSourceChangeStream; // Deterministic values used for testing const UUID testConstUuid = UUID::parse("6948DF80-14BD-4E04-8842-7668D9C001F5").getValue(); +class ExecutableStubMongoProcessInterface : public StubMongoProcessInterface { + bool isExpectedToExecuteQueries() override { + return true; + } +}; + class ChangeStreamStageTestNoSetup : public AggregationContextFixture { public: ChangeStreamStageTestNoSetup() : ChangeStreamStageTestNoSetup(nss) {} explicit ChangeStreamStageTestNoSetup(NamespaceString nsString) - : AggregationContextFixture(nsString) {} + : AggregationContextFixture(nsString) { + getExpCtx()->mongoProcessInterface = + std::make_unique(); + }; }; -struct MockMongoInterface final : public StubMongoProcessInterface { - +struct MockMongoInterface final : public ExecutableStubMongoProcessInterface { // Used by operations which need to obtain the oplog's UUID. static const UUID& oplogUuid() { static const UUID* oplog_uuid = new UUID(UUID::gen()); @@ -4476,7 +4484,7 @@ TEST_F(MultiTokenFormatVersionTest, CanResumeFromV1HighWaterMark) { ASSERT_FALSE(next.isAdvanced()); } -TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamAddPostImage) { +TEST_F(ChangeStreamStageTestNoSetup, DocumentSourceChangeStreamAddPostImageEmptyForQueryStats) { auto spec = DocumentSourceChangeStreamSpec(); spec.setFullDocument(FullDocumentModeEnum::kUpdateLookup); @@ -4485,12 +4493,13 @@ TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamAddPostImag ASSERT_BSONOBJ_EQ_AUTO( // NOLINT R"({"$_internalChangeStreamAddPostImage":{"fullDocument":"updateLookup"}})", docSource->serialize().getDocument().toBson()); - ASSERT_BSONOBJ_EQ_AUTO( // NOLINT - R"({"$_internalChangeStreamAddPostImage":{"fullDocument":"updateLookup"}})", - redact(*docSource)); + + auto opts = SerializationOptions{ + .literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); } -TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamAddPreImage) { +TEST_F(ChangeStreamStageTestNoSetup, DocumentSourceChangeStreamAddPreImageEmptyForQueryStats) { auto docSource = DocumentSourceChangeStreamAddPreImage{ getExpCtx(), FullDocumentBeforeChangeModeEnum::kWhenAvailable}; @@ -4501,12 +4510,14 @@ TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamAddPreImage } })", docSource.serialize().getDocument().toBson()); - ASSERT_BSONOBJ_EQ_AUTO( // NOLINT - R"({"$_internalChangeStreamAddPreImage":{"fullDocumentBeforeChange":"whenAvailable"}})", - redact(docSource)); + + + auto opts = SerializationOptions{ + .literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource.serialize(opts).missing()); } -TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamCheckInvalidate) { +TEST_F(ChangeStreamStageTestNoSetup, DocumentSourceChangeStreamCheckInvalidateEmptyForQueryStats) { DocumentSourceChangeStreamSpec spec; spec.setResumeAfter( ResumeToken::parse(makeResumeToken(Timestamp(), @@ -4526,24 +4537,14 @@ TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamCheckInvali } })", docSource->serialize().getDocument().toBson()); - ASSERT_BSONOBJ_EQ_AUTO( // NOLINT - R"({"$_internalChangeStreamCheckInvalidate":{"startAfterInvalidate":{ - "_data": "?string" - }}})", - redact(*docSource)); - ASSERT_BSONOBJ_EQ_AUTO( // NOLINT - R"({"$_internalChangeStreamCheckInvalidate":{"startAfterInvalidate":{ - "_data": "820000000000000000292904" - }}})", - docSource - ->serialize(SerializationOptions{ - .literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue}) - .getDocument() - .toBson()); + auto opts = SerializationOptions{ + .literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); } -TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamCheckResumability) { +TEST_F(ChangeStreamStageTestNoSetup, + DocumentSourceChangeStreamCheckResumabilityEmptyForQueryStats) { DocumentSourceChangeStreamSpec spec; spec.setResumeAfter( ResumeToken::parse(makeResumeToken(Timestamp(), @@ -4562,39 +4563,27 @@ TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamCheckResuma } })", docSource->serialize().getDocument().toBson()); - ASSERT_BSONOBJ_EQ_AUTO( // NOLINT - R"({"$_internalChangeStreamCheckResumability":{ - "resumeToken": { - "_data": "?string" - } - }})", - redact(*docSource)); - ASSERT_BSONOBJ_EQ_AUTO( // NOLINT - R"({"$_internalChangeStreamCheckResumability":{ - "resumeToken": { - "_data": "820000000000000000292904" - } - }})", - docSource - ->serialize(SerializationOptions{ - .literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue}) - .getDocument() - .toBson()); + auto opts = SerializationOptions{ + .literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); } -TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamCheckTopologyChange) { +TEST_F(ChangeStreamStageTestNoSetup, + DocumentSourceChangeStreamCheckTopologyChangeEmptyForQueryStats) { auto docSource = DocumentSourceChangeStreamCheckTopologyChange::create(getExpCtx()); ASSERT_BSONOBJ_EQ_AUTO( // NOLINT R"({"$_internalChangeStreamCheckTopologyChange":{}})", docSource->serialize().getDocument().toBson()); - ASSERT_BSONOBJ_EQ_AUTO( // NOLINT - R"({"$_internalChangeStreamCheckTopologyChange":{}})", - redact(*docSource)); + + auto opts = SerializationOptions{ + .literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); } -TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamEnsureResumeTokenPresent) { +TEST_F(ChangeStreamStageTestNoSetup, + DocumentSourceChangeStreamEnsureResumeTokenPresentEmptyForQueryStats) { DocumentSourceChangeStreamSpec spec; spec.setResumeAfter( ResumeToken::parse(makeResumeToken(Timestamp(), @@ -4613,26 +4602,23 @@ TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamEnsureResum } })", docSource->serialize().getDocument().toBson()); - ASSERT_BSONOBJ_EQ_AUTO( // NOLINT - R"({ - "$_internalChangeStreamEnsureResumeTokenPresent": { - "resumeToken": { - "_data": "?string" - } - } - })", - redact(*docSource)); + + auto opts = SerializationOptions{ + .literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); } -TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamHandleTopologyChange) { +TEST_F(ChangeStreamStageTestNoSetup, + DocumentSourceChangeStreamHandleTopologyChangeEmptyForQueryStats) { auto docSource = DocumentSourceChangeStreamHandleTopologyChange::create(getExpCtx()); ASSERT_BSONOBJ_EQ_AUTO( // NOLINT R"({"$_internalChangeStreamHandleTopologyChange":{}})", docSource->serialize().getDocument().toBson()); - ASSERT_BSONOBJ_EQ_AUTO( // NOLINT - R"({"$_internalChangeStreamHandleTopologyChange":{}})", - redact(*docSource)); + + auto opts = SerializationOptions{ + .literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); } TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamSplitLargeEvent) { @@ -4674,9 +4660,10 @@ TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamTransform) } })", docSource->serialize().getDocument().toBson()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT R"({ - "$_internalChangeStreamTransform": { + "$changeStream": { "resumeAfter": { "_data": "?string" }, @@ -4688,9 +4675,9 @@ TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamTransform) ASSERT_BSONOBJ_EQ_AUTO( // NOLINT R"({ - "$_internalChangeStreamTransform": { + "$changeStream": { "resumeAfter": { - "_data": "820000000000000000292904" + "_data": "8200000000000000002B0229296E04" }, "fullDocument": "default", "fullDocumentBeforeChange": "off" @@ -4703,6 +4690,64 @@ TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamTransform) .toBson()); } + +TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamTransformMoreFields) { + DocumentSourceChangeStreamSpec spec; + spec.setStartAfter( + ResumeToken::parse(makeResumeToken(Timestamp(), + testConstUuid, + BSON("_id" << 1 << "x" << 2), + DocumentSourceChangeStream::kInsertOpType))); + spec.setFullDocument(FullDocumentModeEnum::kRequired); + spec.setFullDocumentBeforeChange(FullDocumentBeforeChangeModeEnum::kWhenAvailable); + spec.setShowExpandedEvents(true); + + auto docSource = DocumentSourceChangeStreamTransform::create(getExpCtx(), spec); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalChangeStreamTransform": { + "startAfter": { + "_data": "8200000000000000002B042C0100296E5A10046948DF8014BD4E0488427668D9C001F5463C6F7065726174696F6E54797065003C696E736572740046646F63756D656E744B657900461E5F6964002B021E78002B04000004" + }, + "fullDocument": "required", + "fullDocumentBeforeChange": "whenAvailable", + "showExpandedEvents": true + + } + })", + docSource->serialize().getDocument().toBson()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$changeStream": { + "startAfter": { + "_data": "?string" + }, + "fullDocument": "required", + "fullDocumentBeforeChange": "whenAvailable", + "showExpandedEvents": true + } + })", + redact(*docSource)); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$changeStream": { + "startAfter": { + "_data": "8200000000000000002B0229296E04" + }, + "fullDocument": "required", + "fullDocumentBeforeChange": "whenAvailable", + "showExpandedEvents": true + } + })", + docSource + ->serialize(SerializationOptions{ + .literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue}) + .getDocument() + .toBson()); +} + // For DocumentSource types which contain an arbitrarily internal // MatchExpression, we don't want match the entire structure. This // assertion allows us to check some basic structure. @@ -4747,21 +4792,16 @@ void assertRedactedMatchExpressionContainsOperatorsAndRedactedFieldPaths(BSONEle ASSERT(redactedFieldPaths > 0); } -TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamUnwindTransaction) { +TEST_F(ChangeStreamStageTestNoSetup, + DocumentSourceChangeStreamUnwindTransactionEmptyForQueryStats) { auto docSource = DocumentSourceChangeStreamUnwindTransaction::create(getExpCtx()); - auto redacted = redact(*docSource); - // First, check the outermost structure. - BSONElement el = redacted.getField("$_internalChangeStreamUnwindTransaction"_sd); - ASSERT(el); - el = el.Obj().getField("filter"); - ASSERT(el); - el = el.Obj().firstElement(); - - assertRedactedMatchExpressionContainsOperatorsAndRedactedFieldPaths(el); + auto opts = SerializationOptions{ + .literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); } -TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamOplogMatch) { +TEST_F(ChangeStreamStageTestNoSetup, DocumentSourceChangeStreamOplogMatchEmptyForQueryStats) { DocumentSourceChangeStreamSpec spec; spec.setResumeAfter( ResumeToken::parse(makeResumeToken(Timestamp(), @@ -4771,15 +4811,9 @@ TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamOplogMatch) auto docSource = DocumentSourceChangeStreamOplogMatch::create(getExpCtx(), spec); - auto redacted = redact(*docSource); - // First, check the outermost structure. - BSONElement el = redacted.getField("$_internalChangeStreamOplogMatch"_sd); - ASSERT(el); - el = el.Obj().getField("filter"); - ASSERT(el); - el = el.Obj().firstElement(); - - assertRedactedMatchExpressionContainsOperatorsAndRedactedFieldPaths(el); + auto opts = SerializationOptions{ + .literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); } } // namespace diff --git a/src/mongo/db/pipeline/document_source_change_stream_transform.cpp b/src/mongo/db/pipeline/document_source_change_stream_transform.cpp index 2575bd7075cf4..af30e18b94e0a 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_transform.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_transform.cpp @@ -75,7 +75,8 @@ DocumentSourceChangeStreamTransform::createFromBson( DocumentSourceChangeStreamTransform::DocumentSourceChangeStreamTransform( const boost::intrusive_ptr& expCtx, DocumentSourceChangeStreamSpec spec) - : DocumentSource(DocumentSourceChangeStreamTransform::kStageName, expCtx), + : DocumentSourceInternalChangeStreamStage(DocumentSourceChangeStreamTransform::kStageName, + expCtx), _changeStreamSpec(std::move(spec)), _transformer(expCtx, _changeStreamSpec), _isIndependentOfAnyCollection(expCtx->ns.isCollectionlessAggregateNS()) { @@ -206,8 +207,14 @@ Value DocumentSourceChangeStreamTransform::serialize(const SerializationOptions& {"options"_sd, _changeStreamSpec.toBSON(opts)}}}}); } - return Value(Document{ - {DocumentSourceChangeStreamTransform::kStageName, _changeStreamSpec.toBSON(opts)}}); + // Internal change stream stages are not serialized for query stats. Query stats uses this stage + // to serialize the user specified stage, and therefore if serializing for query stats, we + // should use the '$changeStream' stage name. + auto stageName = + (opts.literalPolicy != LiteralSerializationPolicy::kUnchanged || opts.transformIdentifiers) + ? DocumentSourceChangeStream::kStageName + : DocumentSourceChangeStreamTransform::kStageName; + return Value(Document{{stageName, _changeStreamSpec.toBSON(opts)}}); } DepsTracker::State DocumentSourceChangeStreamTransform::getDependencies(DepsTracker* deps) const { diff --git a/src/mongo/db/pipeline/document_source_change_stream_transform.h b/src/mongo/db/pipeline/document_source_change_stream_transform.h index 05342e4d164c7..44faa9dcce99f 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_transform.h +++ b/src/mongo/db/pipeline/document_source_change_stream_transform.h @@ -34,7 +34,7 @@ namespace mongo { -class DocumentSourceChangeStreamTransform : public DocumentSource { +class DocumentSourceChangeStreamTransform : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamTransform"_sd; @@ -58,6 +58,13 @@ class DocumentSourceChangeStreamTransform : public DocumentSource { Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + /** + * This function should never be called, since this DocumentSource has its own serialize method. + */ + Value doSerialize(const SerializationOptions& opts) const final { + MONGO_UNREACHABLE; + } + StageConstraints constraints(Pipeline::SplitState pipeState) const final; boost::optional distributedPlanLogic() final { diff --git a/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.cpp b/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.cpp index 09584bbd2d527..0b9858de2c992 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.cpp @@ -106,7 +106,7 @@ DocumentSourceChangeStreamUnwindTransaction::createFromBson( DocumentSourceChangeStreamUnwindTransaction::DocumentSourceChangeStreamUnwindTransaction( const BSONObj& filter, const boost::intrusive_ptr& expCtx) - : DocumentSource(kStageName, expCtx) { + : DocumentSourceInternalChangeStreamStage(kStageName, expCtx) { rebuild(filter); } @@ -128,7 +128,7 @@ StageConstraints DocumentSourceChangeStreamUnwindTransaction::constraints( ChangeStreamRequirement::kChangeStreamStage); } -Value DocumentSourceChangeStreamUnwindTransaction::serialize( +Value DocumentSourceChangeStreamUnwindTransaction::doSerialize( const SerializationOptions& opts) const { tassert(7481400, "expression has not been initialized", _expression); @@ -141,14 +141,10 @@ Value DocumentSourceChangeStreamUnwindTransaction::serialize( return Value(DOC(DocumentSourceChangeStream::kStageName << builder.obj())); } - Value spec; - if (opts.literalPolicy != LiteralSerializationPolicy::kUnchanged || opts.transformIdentifiers) { - spec = Value(DOC(DocumentSourceChangeStreamUnwindTransactionSpec::kFilterFieldName - << _expression->serialize(opts))); - } else { - spec = Value(DocumentSourceChangeStreamUnwindTransactionSpec(_filter).toBSON()); - } - return Value(Document{{kStageName, spec}}); + // 'SerializationOptions' are not required here, since serialization for explain and query + // stats occur before this function call. + return Value(Document{ + {kStageName, Value{DocumentSourceChangeStreamUnwindTransactionSpec{_filter}.toBSON()}}}); } DepsTracker::State DocumentSourceChangeStreamUnwindTransaction::getDependencies( diff --git a/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.h b/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.h index b4c6b457607a6..867ddb9e675d2 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.h +++ b/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.h @@ -41,7 +41,7 @@ namespace mongo { * output, but all other entries pass through unmodified. Note that the namespace filter applies * only to unwound transaction operations, not to any other entries. */ -class DocumentSourceChangeStreamUnwindTransaction : public DocumentSource { +class DocumentSourceChangeStreamUnwindTransaction : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamUnwindTransaction"_sd; @@ -57,7 +57,7 @@ class DocumentSourceChangeStreamUnwindTransaction : public DocumentSource { DocumentSource::GetModPathsReturn getModifiedPaths() const final; - Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; StageConstraints constraints(Pipeline::SplitState pipeState) const final; diff --git a/src/mongo/db/pipeline/document_source_lookup.cpp b/src/mongo/db/pipeline/document_source_lookup.cpp index e74ce0c3c50da..fec2817f21126 100644 --- a/src/mongo/db/pipeline/document_source_lookup.cpp +++ b/src/mongo/db/pipeline/document_source_lookup.cpp @@ -1093,7 +1093,8 @@ void DocumentSourceLookUp::serializeToArray(std::vector& array, } if (opts.transformIdentifiers || opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) { - return Pipeline::parse(*_userPipeline, _fromExpCtx)->serializeToBson(opts); + return Pipeline::parse(*_userPipeline, _fromExpCtx, lookupPipeValidator) + ->serializeToBson(opts); } if (opts.serializeForQueryAnalysis) { // If we are in query analysis, encrypted fields will have been marked in the diff --git a/src/mongo/db/pipeline/document_source_match.cpp b/src/mongo/db/pipeline/document_source_match.cpp index 8dbce175cb157..4599533a2f9ef 100644 --- a/src/mongo/db/pipeline/document_source_match.cpp +++ b/src/mongo/db/pipeline/document_source_match.cpp @@ -582,4 +582,22 @@ void DocumentSourceMatch::rebuild(BSONObj filter) { getDependencies(&_dependencies); } +Value DocumentSourceInternalChangeStreamMatch::serialize(const SerializationOptions& opts) const { + if (opts.literalPolicy != LiteralSerializationPolicy::kUnchanged || opts.transformIdentifiers) { + // Stages made internally by 'DocumentSourceChangeStream' should not be serialized for + // query stats. For query stats we will serialize only the user specified $changeStream + // stage. + return Value(); + } + return doSerialize(opts); +} + +intrusive_ptr +DocumentSourceInternalChangeStreamMatch::create(BSONObj filter, + const intrusive_ptr& expCtx) { + intrusive_ptr internalMatch( + new DocumentSourceInternalChangeStreamMatch(filter, expCtx)); + return internalMatch; +} + } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_match.h b/src/mongo/db/pipeline/document_source_match.h index f986516f61ea7..39955e9ee7da6 100644 --- a/src/mongo/db/pipeline/document_source_match.h +++ b/src/mongo/db/pipeline/document_source_match.h @@ -233,4 +233,42 @@ class DocumentSourceMatch : public DocumentSource { DepsTracker _dependencies; }; +/** + * A DocumentSource class for all internal change stream stages that are also match stages. This + * currently handles parsing for query stats. + */ +class DocumentSourceInternalChangeStreamMatch : public DocumentSourceMatch { +public: + DocumentSourceInternalChangeStreamMatch(std::unique_ptr expr, + const boost::intrusive_ptr& expCtx) + : DocumentSourceMatch(std::move(expr), expCtx) {} + + static boost::intrusive_ptr create( + BSONObj filter, const boost::intrusive_ptr& expCtx); + + /** + * Must override the serialize method, since internal change stream stages are serialized + * differently than match stages. This function mirrors + * DocumentSourceInternalChangeStreamStage::serialize and was added because this class cannot + * inherit from both DocumentSourceInternalChangeStreamStage and DocumentSourceMatch. + */ + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + + virtual Value doSerialize(const SerializationOptions& opts) const { + return DocumentSourceMatch::serialize(opts); + }; + +protected: + DocumentSourceInternalChangeStreamMatch(const BSONObj& query, + const boost::intrusive_ptr& expCtx) + : DocumentSourceMatch(query, expCtx) {} + + DocumentSourceInternalChangeStreamMatch( + const DocumentSourceInternalChangeStreamMatch& other, + const boost::intrusive_ptr& newExpCtx) + : DocumentSourceMatch( + other.serialize().getDocument().toBson().firstElement().embeddedObject(), + newExpCtx ? newExpCtx : other.pExpCtx) {} +}; + } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_union_with.cpp b/src/mongo/db/pipeline/document_source_union_with.cpp index 2d4e632edb1e2..eaba82d1c1800 100644 --- a/src/mongo/db/pipeline/document_source_union_with.cpp +++ b/src/mongo/db/pipeline/document_source_union_with.cpp @@ -53,21 +53,21 @@ REGISTER_DOCUMENT_SOURCE(unionWith, AllowedWithApiStrict::kAlways); namespace { +void validatorCallback(const Pipeline& pipeline) { + const auto& sources = pipeline.getSources(); + std::for_each(sources.begin(), sources.end(), [](auto& src) { + uassert(31441, + str::stream() << src->getSourceName() + << " is not allowed within a $unionWith's sub-pipeline", + src->constraints().isAllowedInUnionPipeline()); + }); +} + std::unique_ptr buildPipelineFromViewDefinition( const boost::intrusive_ptr& expCtx, ExpressionContext::ResolvedNamespace resolvedNs, std::vector currentPipeline) { - auto validatorCallback = [](const Pipeline& pipeline) { - const auto& sources = pipeline.getSources(); - std::for_each(sources.begin(), sources.end(), [](auto& src) { - uassert(31441, - str::stream() << src->getSourceName() - << " is not allowed within a $unionWith's sub-pipeline", - src->constraints().isAllowedInUnionPipeline()); - }); - }; - MakePipelineOptions opts; opts.attachCursorSource = false; // Only call optimize() here if we actually have a pipeline to resolve in the view definition. @@ -370,7 +370,9 @@ Value DocumentSourceUnionWith::serialize(const SerializationOptions& opts) const std::move(_pushedDownStages.begin(), _pushedDownStages.end(), std::back_inserter(recoveredPipeline)); - pipeCopy = Pipeline::parse(recoveredPipeline, _pipeline->getContext()).release(); + pipeCopy = + Pipeline::parse(recoveredPipeline, _pipeline->getContext(), validatorCallback) + .release(); } else { // The plan does not require reading from the sub-pipeline, so just include the // serialization in the explain output. @@ -403,7 +405,7 @@ Value DocumentSourceUnionWith::serialize(const SerializationOptions& opts) const auto serializedPipeline = [&]() -> std::vector { if (opts.transformIdentifiers || opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) { - return Pipeline::parse(_userPipeline, _pipeline->getContext()) + return Pipeline::parse(_userPipeline, _pipeline->getContext(), validatorCallback) ->serializeToBson(opts); } return _pipeline->serializeToBson(opts); diff --git a/src/mongo/db/pipeline/resume_token.cpp b/src/mongo/db/pipeline/resume_token.cpp index 1cc9a502edf39..360bfd90aec81 100644 --- a/src/mongo/db/pipeline/resume_token.cpp +++ b/src/mongo/db/pipeline/resume_token.cpp @@ -299,21 +299,21 @@ ResumeTokenData ResumeToken::getData() const { } Document ResumeToken::toDocument(const SerializationOptions& options) const { - /* - * This is our default resume token for the representative query shape. - * We use a high water mark token, otherwise a resume event is expected when reparsing. - * When serializing the "_typeBits", we purposely avoid serializing with SerializationOptions, - * as this will result in mistakenly add '?undefined' to the Document. - * The serialization of the Document will typically exclude the "_typeBits" if they - * were unset, which is the case for "kDefaultToken". - */ - static const auto kDefaultToken = makeHighWaterMarkToken(Timestamp(), 0); - return Document{{kDataFieldName, - options.serializeLiteral(_hexKeyString, Value(kDefaultToken._hexKeyString))}, - {kTypeBitsFieldName, - options.literalPolicy != LiteralSerializationPolicy::kToDebugTypeString - ? options.serializeLiteral(_typeBits, kDefaultToken._typeBits) - : kDefaultToken._typeBits}}; + // This is our default resume token for the representative query shape. + static const auto kDefaultTokenQueryStats = makeHighWaterMarkToken(Timestamp(), 1); + + return Document{ + {kDataFieldName, + options.serializeLiteral(_hexKeyString, Value(kDefaultTokenQueryStats._hexKeyString))}, + + // When serializing with 'kToDebugTypeString' 'serializeLiteral' will return an + // incorrect result. Therefore, we prefer to always exclude '_typeBits' when serializing + // the debug string by passing an empty value, since '_typeBits' is rarely set and will + // always be either missing or of type BinData. + {kTypeBitsFieldName, + options.literalPolicy == LiteralSerializationPolicy::kToDebugTypeString + ? Value() + : options.serializeLiteral(_typeBits, kDefaultTokenQueryStats._typeBits)}}; } BSONObj ResumeToken::toBSON(const SerializationOptions& options) const { diff --git a/src/mongo/db/query/optimizer/interval_simplify_test.cpp b/src/mongo/db/query/optimizer/interval_simplify_test.cpp index d6ccfcd1a55a4..9823a6153456c 100644 --- a/src/mongo/db/query/optimizer/interval_simplify_test.cpp +++ b/src/mongo/db/query/optimizer/interval_simplify_test.cpp @@ -869,7 +869,7 @@ void testIntervalFuzz(const uint64_t seed, PseudoRandom& threadLocalRNG) { // Number of bits held in the bitset. This include MinKey and MaxKey, so it must be at least two. static constexpr int bitsetSize = 11; -static const size_t numThreads = ProcessInfo::getNumCores(); +static const size_t numThreads = ProcessInfo::getNumLogicalCores(); TEST_F(IntervalIntersection, IntervalPermutations) { // Number of permutations is bitsetSize^4 * 2^4 * 2 diff --git a/src/mongo/db/query/query_stats/query_stats.cpp b/src/mongo/db/query/query_stats/query_stats.cpp index cf4d23fbbc3d4..d558efc5d2fc4 100644 --- a/src/mongo/db/query/query_stats/query_stats.cpp +++ b/src/mongo/db/query/query_stats/query_stats.cpp @@ -153,7 +153,7 @@ ServiceContext::ConstructorActionRegisterer queryStatsStoreManagerRegisterer{ // number of cores. The size needs to be cast to a double since we want to round up the // number of partitions, and therefore need to avoid int division. size_t numPartitions = std::ceil(double(size) / (16 * 1024 * 1024)); - auto numLogicalCores = ProcessInfo::getNumCores(); + auto numLogicalCores = ProcessInfo::getNumLogicalCores(); if (numPartitions < numLogicalCores) { numPartitions = numLogicalCores; } diff --git a/src/mongo/db/query/sbe_plan_cache.cpp b/src/mongo/db/query/sbe_plan_cache.cpp index 2129554a2d914..eb379e6e673ce 100644 --- a/src/mongo/db/query/sbe_plan_cache.cpp +++ b/src/mongo/db/query/sbe_plan_cache.cpp @@ -86,7 +86,7 @@ ServiceContext::ConstructorActionRegisterer planCacheRegisterer{ } auto& globalPlanCache = sbePlanCacheDecoration(serviceCtx); globalPlanCache = - std::make_unique(cappedCacheSize, ProcessInfo::getNumCores()); + std::make_unique(cappedCacheSize, ProcessInfo::getNumLogicalCores()); }}; } // namespace diff --git a/src/mongo/db/query/util/memory_util.cpp b/src/mongo/db/query/util/memory_util.cpp index dbd4f23bee109..3503240884c27 100644 --- a/src/mongo/db/query/util/memory_util.cpp +++ b/src/mongo/db/query/util/memory_util.cpp @@ -95,7 +95,7 @@ size_t getRequestedMemSizeInBytes(const MemorySize& memSize) { size_t planCacheSize = convertToSizeInBytes(memSize); uassert(5968001, "Cache size must be at least 1KB * number of cores", - planCacheSize >= 1024 * ProcessInfo::getNumCores()); + planCacheSize >= 1024 * ProcessInfo::getNumLogicalCores()); return planCacheSize; } diff --git a/src/mongo/db/repl/topology_version_observer_test.cpp b/src/mongo/db/repl/topology_version_observer_test.cpp index c42547030911d..008ec5df87c7b 100644 --- a/src/mongo/db/repl/topology_version_observer_test.cpp +++ b/src/mongo/db/repl/topology_version_observer_test.cpp @@ -42,6 +42,7 @@ #include "mongo/db/repl/replication_coordinator_test_fixture.h" #include "mongo/db/repl/topology_version_observer.h" #include "mongo/unittest/barrier.h" +#include "mongo/unittest/log_test.h" #include "mongo/unittest/unittest.h" #include "mongo/util/assert_util.h" #include "mongo/util/clock_source.h" @@ -120,6 +121,9 @@ class TopologyVersionObserverTest : public ReplCoordTest { const Milliseconds sleepTime = Milliseconds(100); std::unique_ptr observer; + + unittest::MinimumLoggedSeverityGuard severityGuard{logv2::LogComponent::kDefault, + logv2::LogSeverity::Debug(4)}; }; @@ -142,11 +146,15 @@ TEST_F(TopologyVersionObserverTest, UpdateCache) { auto electionTimeoutWhen = getReplCoord()->getElectionTimeout_forTest(); simulateSuccessfulV1ElectionWithoutExitingDrainMode(electionTimeoutWhen, opCtx.get()); + auto sleepCounter = 0; // Wait for the observer to update its cache while (observer->getCached()->getTopologyVersion()->getCounter() == cachedResponse->getTopologyVersion()->getCounter()) { sleepFor(sleepTime); + // Make sure the test doesn't wait here for longer than 15 seconds. + ASSERT_LTE(sleepCounter++, 150); } + LOGV2(9326401, "Observer topology incremented after successful election"); auto newResponse = observer->getCached(); ASSERT(newResponse && newResponse->getTopologyVersion()); diff --git a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.cpp b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.cpp index 68f640d13b62c..7dc313a79c56d 100644 --- a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.cpp +++ b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.cpp @@ -93,6 +93,7 @@ CoordinatorCommitMonitor::CoordinatorCommitMonitor( std::vector recipientShards, CoordinatorCommitMonitor::TaskExecutorPtr executor, CancellationToken cancelToken, + int delayBeforeInitialQueryMillis, Milliseconds maxDelayBetweenQueries) : _metrics{std::move(metrics)}, _ns(std::move(ns)), @@ -100,11 +101,12 @@ CoordinatorCommitMonitor::CoordinatorCommitMonitor( _executor(std::move(executor)), _cancelToken(std::move(cancelToken)), _threshold(Milliseconds(gRemainingReshardingOperationTimeThresholdMillis.load())), + _delayBeforeInitialQueryMillis(Milliseconds(delayBeforeInitialQueryMillis)), _maxDelayBetweenQueries(maxDelayBetweenQueries) {} SemiFuture CoordinatorCommitMonitor::waitUntilRecipientsAreWithinCommitThreshold() const { - return _makeFuture() + return _makeFuture(_delayBeforeInitialQueryMillis) .onError([](Status status) { if (ErrorCodes::isCancellationError(status.code()) || ErrorCodes::isInterruption(status.code())) { @@ -198,9 +200,16 @@ CoordinatorCommitMonitor::queryRemainingOperationTimeForRecipients() const { return {minRemainingTime, maxRemainingTime}; } -ExecutorFuture CoordinatorCommitMonitor::_makeFuture() const { +ExecutorFuture CoordinatorCommitMonitor::_makeFuture(Milliseconds delayBetweenQueries) const { return ExecutorFuture(_executor) - .then([this] { return queryRemainingOperationTimeForRecipients(); }) + // Start waiting so that we have a more time to calculate a more realistic remaining time + // estimate. + .then([this, anchor = shared_from_this(), delayBetweenQueries] { + return _executor->sleepFor(delayBetweenQueries, _cancelToken) + .then([this, anchor = std::move(anchor)] { + return queryRemainingOperationTimeForRecipients(); + }); + }) .onError([this](Status status) { if (_cancelToken.isCanceled()) { // Do not retry on cancellation errors. @@ -235,12 +244,10 @@ ExecutorFuture CoordinatorCommitMonitor::_makeFuture() const { // The following ensures that the monitor would never sleep for more than a predefined // maximum delay between querying recipient shards. Thus, it can handle very large, // and potentially inaccurate estimates of the remaining operation time. - auto sleepTime = std::min(remainingTimes.max - _threshold, _maxDelayBetweenQueries); - return _executor->sleepFor(sleepTime, _cancelToken) - .then([this, anchor = std::move(anchor)] { - // We are not canceled yet, so schedule new queries against recipient shards. - return _makeFuture(); - }); + auto delayBetweenQueries = + std::min(remainingTimes.max - _threshold, _maxDelayBetweenQueries); + + return _makeFuture(delayBetweenQueries); }); } diff --git a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.h b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.h index 7f396c627c16b..484c6095ab5c0 100644 --- a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.h +++ b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.h @@ -74,6 +74,7 @@ class CoordinatorCommitMonitor : public std::enable_shared_from_this recipientShards, TaskExecutorPtr executor, CancellationToken cancelToken, + int delayBeforeInitialQueryMillis, Milliseconds maxDelayBetweenQueries = kMaxDelayBetweenQueries); SemiFuture waitUntilRecipientsAreWithinCommitThreshold() const; @@ -90,7 +91,7 @@ class CoordinatorCommitMonitor : public std::enable_shared_from_this _makeFuture() const; + ExecutorFuture _makeFuture(Milliseconds delayBetweenQueries) const; static constexpr auto kDiagnosticLogLevel = 0; static constexpr auto kMaxDelayBetweenQueries = Seconds(30); @@ -102,6 +103,8 @@ class CoordinatorCommitMonitor : public std::enable_shared_from_thistoken(), + 0, Milliseconds(0)); _commitMonitor->setNetworkExecutorForTest(executor()); } diff --git a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp index a95d4f886135a..20caaf3b4c421 100644 --- a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp +++ b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp @@ -2035,7 +2035,8 @@ void ReshardingCoordinator::_startCommitMonitor( _coordinatorDoc.getSourceNss(), resharding::extractShardIdsFromParticipantEntries(_coordinatorDoc.getRecipientShards()), **executor, - _ctHolder->getCommitMonitorToken()); + _ctHolder->getCommitMonitorToken(), + resharding::gReshardingDelayBeforeRemainingOperationTimeQueryMillis.load()); _commitMonitorQuiesced = _commitMonitor->waitUntilRecipientsAreWithinCommitThreshold() .thenRunOn(**executor) diff --git a/src/mongo/db/s/resharding/resharding_metrics.cpp b/src/mongo/db/s/resharding/resharding_metrics.cpp index 7d19538153017..ac5de55a0e901 100644 --- a/src/mongo/db/s/resharding/resharding_metrics.cpp +++ b/src/mongo/db/s/resharding/resharding_metrics.cpp @@ -180,10 +180,10 @@ boost::optional ReshardingMetrics::getRecipientHighEstimateRemaini getStartFor(TimedPhase::kApplying).has_value(), getBytesWrittenCount(), getApproxBytesToScanCount(), - getElapsed(TimedPhase::kCloning, getClockSource()).value_or(Seconds{0}), + getElapsed(TimedPhase::kCloning, getClockSource()).value_or(Seconds{0}), getOplogEntriesApplied(), getOplogEntriesFetched(), - getElapsed(TimedPhase::kApplying, getClockSource()).value_or(Seconds{0})); + getElapsed(TimedPhase::kApplying, getClockSource()).value_or(Seconds{0})); } std::unique_ptr ReshardingMetrics::makeInstance(UUID instanceId, diff --git a/src/mongo/db/s/resharding/resharding_metrics_test.cpp b/src/mongo/db/s/resharding/resharding_metrics_test.cpp index 0d6f78bab7d72..ea97c371a7864 100644 --- a/src/mongo/db/s/resharding/resharding_metrics_test.cpp +++ b/src/mongo/db/s/resharding/resharding_metrics_test.cpp @@ -714,5 +714,23 @@ TEST_F(ReshardingMetricsTest, OnStateTransitionInformsCumulativeMetrics) { }); } +TEST_F(ReshardingMetricsTest, RecipientReportsRemainingTimeLowElapsed) { + auto metrics = createInstanceMetrics(getClockSource(), UUID::gen(), Role::kRecipient); + const auto& clock = getClockSource(); + constexpr auto timeSpentCloning = Seconds(20); + constexpr auto timeSpentApplying = Milliseconds(50); + metrics->onOplogEntriesFetched(500000); + metrics->setStartFor(TimedPhase::kCloning, clock->now()); + + clock->advance(timeSpentCloning); + metrics->setEndFor(TimedPhase::kCloning, clock->now()); + metrics->setStartFor(TimedPhase::kApplying, clock->now()); + clock->advance(timeSpentApplying); + metrics->onOplogEntriesApplied(300000); + + auto report = metrics->getHighEstimateRemainingTimeMillis(); + ASSERT_NE(report, Milliseconds{0}); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/s/resharding/resharding_server_parameters.idl b/src/mongo/db/s/resharding/resharding_server_parameters.idl index ea5703cf34d1f..3cd4aa67b935d 100644 --- a/src/mongo/db/s/resharding/resharding_server_parameters.idl +++ b/src/mongo/db/s/resharding/resharding_server_parameters.idl @@ -151,6 +151,21 @@ server_parameters: validator: gte: 0 + reshardingDelayBeforeRemainingOperationTimeQueryMillis: + description: >- + Initial delay before querying for remaining operation time from recipient shards. + The delay allows for applying more oplog entries before calculating time remaining, giving + a more accurate value. + Note we will have this delay every time we happen to have a failover occur. + set_at: [startup, runtime] + cpp_vartype: AtomicWord + cpp_varname: gReshardingDelayBeforeRemainingOperationTimeQueryMillis + default: + expr: 0 + validator: + gte: 0 + redact: false + reshardingCriticalSectionTimeoutMillis: description: >- The upper limit on how long to wait to hear back from recipient shards reaching strict diff --git a/src/mongo/db/s/shardsvr_compact_structured_encryption_data_command.cpp b/src/mongo/db/s/shardsvr_compact_structured_encryption_data_command.cpp index a6967e48abb77..05c69ec0f8fed 100644 --- a/src/mongo/db/s/shardsvr_compact_structured_encryption_data_command.cpp +++ b/src/mongo/db/s/shardsvr_compact_structured_encryption_data_command.cpp @@ -124,16 +124,27 @@ class _shardsvrCompactStructuredEncryptionDataCommand final uassertStatusOK(EncryptedStateCollectionsNamespaces::createFromDataCollection( *(baseColl.getCollection().get()))); - AutoGetCollection ecocColl(opCtx, namespaces.ecocNss, MODE_IX); - AutoGetCollection ecocTempColl(opCtx, namespaces.ecocRenameNss, MODE_IX); - CompactStructuredEncryptionDataState compact; - if (ecocColl.getCollection()) { - compact.setEcocUuid(ecocColl->uuid()); - } - if (ecocTempColl.getCollection()) { - compact.setEcocRenameUuid(ecocTempColl->uuid()); + // To avoid deadlock, IX locks for ecocRenameNss and ecocNss must be acquired in the + // same order they'd be acquired during renameCollection (ascending ResourceId order). + // Providing ecocRenameNss as a secondary to ecocNss in AutoGetCollection ensures the + // locks for both namespaces are acquired in correct order. + { + AutoGetCollection ecocColl( + opCtx, + namespaces.ecocNss, + MODE_IX, + AutoGetCollection::Options{}.secondaryNssOrUUIDs({namespaces.ecocRenameNss})); + if (ecocColl.getCollection()) { + compact.setEcocUuid(ecocColl->uuid()); + } + auto catalog = CollectionCatalog::get(opCtx); + auto ecocTempColl = CollectionPtr( + catalog->lookupCollectionByNamespace(opCtx, namespaces.ecocRenameNss)); + if (ecocTempColl) { + compact.setEcocRenameUuid(ecocTempColl->uuid()); + } } compact.setShardingDDLCoordinatorMetadata( diff --git a/src/mongo/db/service_context.cpp b/src/mongo/db/service_context.cpp index 62f6a32a92c0b..b266de52658e2 100644 --- a/src/mongo/db/service_context.cpp +++ b/src/mongo/db/service_context.cpp @@ -250,6 +250,16 @@ ServiceContext::UniqueOperationContext ServiceContext::makeOperationContext(Clie } }); + // If there's an egress transport layer and it can produce a + // nonnull `BatonHandle`, let it do so. Otherwise, we'll use a + // `DefaultBaton` as a fallback. + opCtx->setBaton([&]() -> BatonHandle { + if (_transportLayer) + if (auto baton = _transportLayer->makeBaton(opCtx.get())) + return baton; + return std::make_shared(opCtx.get()); + }()); + // We must prevent changing the storage engine while setting a new opCtx on the client. auto sharedStorageChangeToken = _storageChangeLk.acquireSharedStorageChangeToken(); @@ -265,12 +275,6 @@ ServiceContext::UniqueOperationContext ServiceContext::makeOperationContext(Clie opCtx->setRecoveryUnit(std::make_unique(), WriteUnitOfWork::RecoveryUnitState::kNotInUnitOfWork); } - // The baton must be attached before attaching to a client - if (_transportLayer) { - _transportLayer->makeBaton(opCtx.get()); - } else { - makeBaton(opCtx.get()); - } ScopeGuard batonGuard([&] { opCtx->getBaton()->detach(); }); @@ -519,13 +523,4 @@ void ServiceContext::ServiceContextDeleter::operator()(ServiceContext* service) delete service; } -BatonHandle ServiceContext::makeBaton(OperationContext* opCtx) const { - invariant(!opCtx->getBaton()); - - auto baton = std::make_shared(opCtx); - opCtx->setBaton(baton); - - return baton; -} - } // namespace mongo diff --git a/src/mongo/db/session/SConscript b/src/mongo/db/session/SConscript index 8e7d243c93bf2..6082102863952 100644 --- a/src/mongo/db/session/SConscript +++ b/src/mongo/db/session/SConscript @@ -50,6 +50,7 @@ env.Library( target='sessions_collection', source=[ 'sessions_collection.cpp', + 'sessions_server_parameters.idl', ], LIBDEPS=[ '$BUILD_DIR/mongo/base', diff --git a/src/mongo/db/session/sessions_collection.cpp b/src/mongo/db/session/sessions_collection.cpp index 111f029288fa2..e908c2656a227 100644 --- a/src/mongo/db/session/sessions_collection.cpp +++ b/src/mongo/db/session/sessions_collection.cpp @@ -42,26 +42,13 @@ #include "mongo/db/repl/read_concern_args.h" #include "mongo/db/session/logical_session_id.h" #include "mongo/db/session/logical_session_id_helpers.h" +#include "mongo/db/session/sessions_server_parameters_gen.h" #include "mongo/rpc/get_status_from_command_result.h" +#include "mongo/util/duration.h" namespace mongo { namespace { -// This batch size is chosen to ensure that we don't form requests larger than the 16mb limit. -// Especially for refreshes, the updates we send include the full user name (user@db), and user -// names can be quite large (we enforce a max 10k limit for usernames used with sessions). -// -// At 1000 elements, a 16mb payload gives us a budget of 16000 bytes per user, which we should -// comfortably be able to stay under, even with 10k user names. -constexpr size_t kMaxBatchSize = 1000; - -// Used to refresh or remove items from the session collection with write -// concern majority -const WriteConcernOptions kMajorityWriteConcern{WriteConcernOptions::kMajority, - WriteConcernOptions::SyncMode::UNSET, - WriteConcernOptions::kWriteConcernTimeoutSystem}; - - BSONObj lsidQuery(const LogicalSessionId& lsid) { return BSON(LogicalSessionRecord::kIdFieldName << lsid.toBSON()); } @@ -106,7 +93,7 @@ void runBulkGeneric(TFactory makeT, AddLineFn addLine, SendFn sendBatch, const C for (const auto& item : items) { addLine(*thing, item); - if (++i >= kMaxBatchSize) { + if (++i >= std::size_t(mongo::gSessionMaxBatchSize.load())) { sendLocalBatch(); setupBatch(); @@ -194,7 +181,14 @@ SessionsCollection::FindBatchFn SessionsCollection::makeFindFnForCommand(const N void SessionsCollection::_doRefresh(const NamespaceString& ns, const std::vector& sessions, SendBatchFn send) { - auto init = [ns](BSONObjBuilder* batch) { + + // Used to refresh items from the session collection with write concern majority + const WriteConcernOptions kMajorityWriteConcern{ + WriteConcernOptions::kMajority, + WriteConcernOptions::SyncMode::UNSET, + Milliseconds(mongo::gSessionWriteConcernTimeoutSystemMillis.load())}; + + auto init = [ns, kMajorityWriteConcern](BSONObjBuilder* batch) { batch->append("update", ns.coll()); batch->append("ordered", false); batch->append(WriteConcernOptions::kWriteConcernField, kMajorityWriteConcern.toBSON()); @@ -211,7 +205,13 @@ void SessionsCollection::_doRefresh(const NamespaceString& ns, void SessionsCollection::_doRemove(const NamespaceString& ns, const std::vector& sessions, SendBatchFn send) { - auto init = [ns](BSONObjBuilder* batch) { + // Used to remove items from the session collection with write concern majority + const WriteConcernOptions kMajorityWriteConcern{ + WriteConcernOptions::kMajority, + WriteConcernOptions::SyncMode::UNSET, + Milliseconds(mongo::gSessionWriteConcernTimeoutSystemMillis.load())}; + + auto init = [ns, kMajorityWriteConcern](BSONObjBuilder* batch) { batch->append("delete", ns.coll()); batch->append("ordered", false); batch->append(WriteConcernOptions::kWriteConcernField, kMajorityWriteConcern.toBSON()); diff --git a/src/mongo/db/session/sessions_server_parameters.idl b/src/mongo/db/session/sessions_server_parameters.idl new file mode 100644 index 0000000000000..19f8bb175a151 --- /dev/null +++ b/src/mongo/db/session/sessions_server_parameters.idl @@ -0,0 +1,63 @@ +# Copyright (C) 2024-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# . +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# + +# Server parameters for configuring the refresh of the session colelction. + +global: + cpp_namespace: "mongo" + +imports: + - "mongo/db/basic_types.idl" + +server_parameters: + sessionWriteConcernTimeoutSystemMillis: + description: Controls the write concern timeout (in milliseconds) for the refresh or removal of items from the session collection. + set_at: [startup, runtime] + cpp_vartype: AtomicWord + cpp_varname: gSessionWriteConcernTimeoutSystemMillis + default: 60000 + validator: + gte: 0 + redact: false + + sessionMaxBatchSize: + description: >- + Controls the maximum batch size (number of elements) for the sessions' refresh. + This batch size is chosen to ensure that we don't form requests larger than the 16mb limit. + Especially for refreshes, the updates we send include the full user name (user@db), and user + names can be quite large (we enforce a max 10k limit for usernames used with sessions). + At a default of 1000 elements, a 16mb payload gives us a budget of 16000 bytes per user, which we should + comfortably be able to stay under, even with 10k user names. so we do not form requests larger than the 16mb limit. + set_at: [startup, runtime] + cpp_vartype: AtomicWord + cpp_varname: gSessionMaxBatchSize + default: 1000 + validator: + gte: 100 + lte: 10000 + redact: false diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript index ff9cb2cc9466a..c4e1dcb856742 100644 --- a/src/mongo/db/storage/SConscript +++ b/src/mongo/db/storage/SConscript @@ -175,6 +175,7 @@ env.Library( '$BUILD_DIR/mongo/db/server_base', '$BUILD_DIR/mongo/db/service_context', '$BUILD_DIR/mongo/db/shard_role', + '$BUILD_DIR/mongo/db/storage/record_store_base', '$BUILD_DIR/mongo/util/background_job', ], ) @@ -555,6 +556,7 @@ env.Library( env.CppUnitTest( target='db_storage_test', source=[ + 'backup_block_test.cpp', 'collection_markers_test.cpp', 'external_record_store_test.cpp', 'disk_space_monitor_test.cpp', @@ -592,6 +594,7 @@ env.CppUnitTest( '$BUILD_DIR/mongo/executor/network_interface_factory', '$BUILD_DIR/mongo/executor/network_interface_mock', '$BUILD_DIR/mongo/util/periodic_runner_factory', + 'backup_block', 'disk_space_monitor', 'flow_control', 'flow_control_parameters', diff --git a/src/mongo/db/storage/backup_block.cpp b/src/mongo/db/storage/backup_block.cpp index 357290caeb352..9b8dc29f20975 100644 --- a/src/mongo/db/storage/backup_block.cpp +++ b/src/mongo/db/storage/backup_block.cpp @@ -49,22 +49,38 @@ const std::set kRequiredMDBFiles = {"_mdb_catalog.wt", "sizeStorer. } // namespace +namespace details { + +std::string extractIdentFromPath(const boost::filesystem::path& dbpath, + const boost::filesystem::path& identAbsolutePath) { + // Remove the dbpath prefix to the identAbsolutePath. + boost::filesystem::path identWithExtension = boost::filesystem::relative( + identAbsolutePath, boost::filesystem::path(storageGlobalParams.dbpath)); + + // Remove the file extension and convert to generic form (i.e. replace "\" with "/" + // on windows, no-op on unix). + return boost::filesystem::change_extension(identWithExtension, "").generic_string(); +} + +} // namespace details + BackupBlock::BackupBlock(OperationContext* opCtx, - std::string filePath, + std::string fileAbsolutePath, const IdentToNamespaceAndUUIDMap& identToNamespaceAndUUIDMap, boost::optional checkpointTimestamp, std::uint64_t offset, std::uint64_t length, std::uint64_t fileSize) - : _filePath(filePath), _offset(offset), _length(length), _fileSize(fileSize) { - boost::filesystem::path path(filePath); - _filenameStem = path.stem().string(); + : _fileAbsolutePath(fileAbsolutePath), _offset(offset), _length(length), _fileSize(fileSize) { + boost::filesystem::path absolutePath(fileAbsolutePath); + _ident = details::extractIdentFromPath(boost::filesystem::path(storageGlobalParams.dbpath), + absolutePath); _initialize(opCtx, identToNamespaceAndUUIDMap, checkpointTimestamp); } bool BackupBlock::isRequired() const { // Extract the filename from the path. - boost::filesystem::path path(_filePath); + boost::filesystem::path path(_fileAbsolutePath); const std::string filename = path.filename().string(); // Check whether this is a required WiredTiger file. @@ -121,7 +137,7 @@ void BackupBlock::_initialize(OperationContext* opCtx, } // Fetch the latest values for the ident. - auto it = identToNamespaceAndUUIDMap.find(_filenameStem); + auto it = identToNamespaceAndUUIDMap.find(_ident); if (it != identToNamespaceAndUUIDMap.end()) { _uuid = it->second.second; _setNamespaceString(it->second.first); @@ -134,7 +150,7 @@ void BackupBlock::_initialize(OperationContext* opCtx, // Check if the ident had a different value at the checkpoint timestamp. If so, we want to use // that instead as that will be the ident's value when restoring from the backup. boost::optional> historicalEntry = - HistoricalIdentTracker::get(opCtx).lookup(_filenameStem, checkpointTimestamp.value()); + HistoricalIdentTracker::get(opCtx).lookup(_ident, checkpointTimestamp.value()); if (historicalEntry) { _uuid = historicalEntry->second; _setNamespaceString(historicalEntry->first); diff --git a/src/mongo/db/storage/backup_block.h b/src/mongo/db/storage/backup_block.h index fd96b8e5e2e76..30a640190a6ce 100644 --- a/src/mongo/db/storage/backup_block.h +++ b/src/mongo/db/storage/backup_block.h @@ -39,6 +39,11 @@ namespace mongo { +namespace details { +std::string extractIdentFromPath(const boost::filesystem::path& dbpath, + const boost::filesystem::path& identAbsolutePath); +} + /** * Represents the file blocks returned by the storage engine during both full and incremental * backups. In the case of a full backup, each block is an entire file with offset=0 and @@ -57,7 +62,7 @@ class BackupBlock final { stdx::unordered_map>; explicit BackupBlock(OperationContext* opCtx, - std::string filePath, + std::string fileAbsolutePath, const IdentToNamespaceAndUUIDMap& identToNamespaceAndUUIDMap, boost::optional checkpointTimestamp, std::uint64_t offset = 0, @@ -67,7 +72,7 @@ class BackupBlock final { ~BackupBlock() = default; std::string filePath() const { - return _filePath; + return _fileAbsolutePath; } std::string ns() const { @@ -110,12 +115,12 @@ class BackupBlock final { boost::optional checkpointTimestamp); void _setNamespaceString(const NamespaceString& nss); - const std::string _filePath; + const std::string _fileAbsolutePath; const std::uint64_t _offset; const std::uint64_t _length; const std::uint64_t _fileSize; - std::string _filenameStem; + std::string _ident; NamespaceString _nss; boost::optional _uuid; }; diff --git a/src/mongo/db/storage/backup_block_test.cpp b/src/mongo/db/storage/backup_block_test.cpp new file mode 100644 index 0000000000000..d98a1e63650c3 --- /dev/null +++ b/src/mongo/db/storage/backup_block_test.cpp @@ -0,0 +1,76 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include + +#include "mongo/db/storage/backup_block.h" +#include "mongo/unittest/unittest.h" + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kTest + + +namespace mongo { +namespace details { +namespace { + +TEST(BackupBlockTest, ExtractIdentFromPath) { + boost::filesystem::path dbpath = "/data/db"; + boost::filesystem::path identAbsolutePathDefault = + "/data/db/collection-9-11733751379908443489.wt"; + std::string identDefault = "collection-9-11733751379908443489"; + + ASSERT_EQ(details::extractIdentFromPath(dbpath, identAbsolutePathDefault), identDefault); + + boost::filesystem::path identAbsolutePathDirectoryPerDb = + "/data/db/test/collection-9-11733751379908443489.wt"; + std::string identDirectoryPerDb = "test/collection-9-11733751379908443489"; + + ASSERT_EQ(details::extractIdentFromPath(dbpath, identAbsolutePathDirectoryPerDb), + identDirectoryPerDb); + + boost::filesystem::path identAbsolutePathWiredTigerDirectoryForIndexes = + "/data/db/collection/9-11733751379908443489.wt"; + std::string identWiredTigerDirectoryForIndexes = "collection/9-11733751379908443489"; + + ASSERT_EQ(details::extractIdentFromPath(dbpath, identAbsolutePathWiredTigerDirectoryForIndexes), + identWiredTigerDirectoryForIndexes); + + boost::filesystem::path identAbsolutePathDirectoryPerDbAndWiredTigerDirectoryForIndexes = + "/data/db/test/collection/9-11733751379908443489.wt"; + std::string identDirectoryPerDbWiredTigerDirectoryForIndexes = + "test/collection/9-11733751379908443489"; + + ASSERT_EQ(details::extractIdentFromPath( + dbpath, identAbsolutePathDirectoryPerDbAndWiredTigerDirectoryForIndexes), + identDirectoryPerDbWiredTigerDirectoryForIndexes); +} + +} // namespace +} // namespace details +} // namespace mongo diff --git a/src/mongo/db/storage/collection_markers.cpp b/src/mongo/db/storage/collection_markers.cpp index bb5a5474d136d..f60dd69662d30 100644 --- a/src/mongo/db/storage/collection_markers.cpp +++ b/src/mongo/db/storage/collection_markers.cpp @@ -64,27 +64,33 @@ void CollectionTruncateMarkers::kill() { } -void CollectionTruncateMarkers::awaitHasExcessMarkersOrDead(OperationContext* opCtx) { +bool CollectionTruncateMarkers::awaitHasExcessMarkersOrDead(OperationContext* opCtx) { // Wait until kill() is called or there are too many collection markers. stdx::unique_lock lock(_collectionMarkersReclaimMutex); - while (!_isDead) { - { - MONGO_IDLE_THREAD_BLOCK; - stdx::lock_guard lk(_markersMutex); - if (_hasExcessMarkers(opCtx)) { - const auto& oldestMarker = _markers.front(); - invariant(oldestMarker.lastRecord.isValid()); + MONGO_IDLE_THREAD_BLOCK; + auto isWaitConditionSatisfied = opCtx->waitForConditionOrInterruptFor( + _reclaimCv, lock, Seconds(gCollectionTruncationCheckPeriodSeconds), [this, opCtx] { + if (_isDead) { + return true; + } + + if (auto marker = peekOldestMarkerIfNeeded(opCtx)) { + invariant(marker->lastRecord.isValid()); LOGV2_DEBUG(7393215, 2, "Collection has excess markers", - "lastRecord"_attr = oldestMarker.lastRecord, - "wallTime"_attr = oldestMarker.wallTime); - return; + "lastRecord"_attr = marker->lastRecord, + "wallTime"_attr = marker->wallTime); + return true; } - } - _reclaimCv.wait_for(lock, stdx::chrono::seconds{gCollectionTruncationCheckPeriodSeconds}); - } + + return false; + }); + + // Return true only when we have detected excess markers, not because the record store + // is being destroyed (_isDead) or we timed out waiting on the condition variable. + return !(_isDead || !isWaitConditionSatisfied); } boost::optional diff --git a/src/mongo/db/storage/collection_markers.h b/src/mongo/db/storage/collection_markers.h index 05ce25909186c..09258c14fa89b 100644 --- a/src/mongo/db/storage/collection_markers.h +++ b/src/mongo/db/storage/collection_markers.h @@ -29,6 +29,8 @@ #pragma once +#include + #include #include "mongo/db/record_id.h" @@ -53,7 +55,7 @@ class OperationContext; // If these requirements hold then this class can be used to compute and maintain up-to-date markers // for ranges of deletions. These markers will be expired and returned to the deleter whenever the // implementation defined '_hasExcessMarkers' returns true. -class CollectionTruncateMarkers { +class CollectionTruncateMarkers : public std::enable_shared_from_this { public: /** Markers represent "waypoints" of the collection that contain information between the current * marker and the previous one. @@ -111,7 +113,18 @@ class CollectionTruncateMarkers { */ void kill(); - void awaitHasExcessMarkersOrDead(OperationContext* opCtx); + /** + * Waits for expired markers. See _hasExcessMarkers(). + * Returns true if expired markers are present. + * Otherwise, returns false. This could be due to reaching an implementation defined + * deadline for the wait, or if we are shutting down this CollectionTruncateMarkers + * instance. + * This operation may throw an exception if interrupted. + * Storage engines supporting oplog truncate markers must implement this function. + */ + bool awaitHasExcessMarkersOrDead(OperationContext* opCtx); + + virtual ~CollectionTruncateMarkers() = default; boost::optional peekOldestMarkerIfNeeded(OperationContext* opCtx) const; diff --git a/src/mongo/db/storage/execution_control/throughput_probing.cpp b/src/mongo/db/storage/execution_control/throughput_probing.cpp index 890546f50eb4d..5456f7e3618e3 100644 --- a/src/mongo/db/storage/execution_control/throughput_probing.cpp +++ b/src/mongo/db/storage/execution_control/throughput_probing.cpp @@ -85,11 +85,12 @@ ThroughputProbing::ThroughputProbing(ServiceContext* svcCtx, TicketHolder* writeTicketHolder, Milliseconds interval) : TicketHolderMonitor(svcCtx, readTicketHolder, writeTicketHolder, interval), - _stableConcurrency(gInitialConcurrency - ? gInitialConcurrency - : std::clamp(static_cast(ProcessInfo::getNumCores() * 2), - gMinConcurrency * 2, - gMaxConcurrency.load() * 2)), + _stableConcurrency( + gInitialConcurrency + ? gInitialConcurrency + : std::clamp(static_cast(ProcessInfo::getNumLogicalCores() * 2), + gMinConcurrency * 2, + gMaxConcurrency.load() * 2)), _timer(svcCtx->getTickSource()) { _resetConcurrency(); } diff --git a/src/mongo/db/storage/oplog_cap_maintainer_thread.cpp b/src/mongo/db/storage/oplog_cap_maintainer_thread.cpp index 2388800265dab..cbc5102d5358f 100644 --- a/src/mongo/db/storage/oplog_cap_maintainer_thread.cpp +++ b/src/mongo/db/storage/oplog_cap_maintainer_thread.cpp @@ -37,6 +37,8 @@ #include "mongo/db/catalog_raii.h" #include "mongo/db/client.h" #include "mongo/db/service_context.h" +#include "mongo/db/storage/collection_markers.h" +#include "mongo/db/storage/record_store.h" #include "mongo/logv2/log.h" #include "mongo/util/assert_util.h" #include "mongo/util/exit.h" @@ -60,35 +62,55 @@ OplogCapMaintainerThread* OplogCapMaintainerThread::get(ServiceContext* serviceC return &getMaintainerThread(serviceCtx); } -bool OplogCapMaintainerThread::_deleteExcessDocuments() { - if (!getGlobalServiceContext()->getStorageEngine()) { +bool OplogCapMaintainerThread::_deleteExcessDocuments(OperationContext* opCtx) { + if (!opCtx->getServiceContext()->getStorageEngine()) { LOGV2_DEBUG(22240, 2, "OplogCapMaintainerThread: no global storage engine yet"); return false; } - const ServiceContext::UniqueOperationContext opCtx = cc().makeOperationContext(); - - // Maintaining the Oplog cap is crucial to the stability of the server so that we don't let the - // oplog grow unbounded. We mark the operation as having immediate priority to skip ticket - // acquisition and flow control. + // Maintaining the Oplog cap is crucial to the stability of the server so that we don't let + // the oplog grow unbounded. We mark the operation as having immediate priority to skip + // ticket acquisition and flow control. ScopedAdmissionPriorityForLock priority(opCtx->lockState(), AdmissionContext::Priority::kImmediate); try { - // A Global IX lock should be good enough to protect the oplog truncation from - // interruptions such as restartCatalog. PBWM, database lock or collection lock is not - // needed. This improves concurrency if oplog truncation takes long time. - AutoGetOplog oplogWrite(opCtx.get(), OplogAccessMode::kWrite); - const auto& oplog = oplogWrite.getCollection(); - if (!oplog) { - LOGV2_DEBUG(4562600, 2, "oplog collection does not exist"); + std::shared_ptr oplogTruncateMarkers; + { + // A Global IX lock should be good enough to protect the oplog truncation from + // interruptions such as restartCatalog. PBWM, database lock or collection lock is + // not needed. This improves concurrency if oplog truncation takes long time. + AutoGetOplog oplogWrite(opCtx, OplogAccessMode::kWrite); + const auto& oplog = oplogWrite.getCollection(); + if (!oplog) { + LOGV2_DEBUG(4562600, 2, "oplog collection does not exist"); + return false; + } + auto rs = oplog->getRecordStore(); + + // Create another reference to the oplog truncate markers while holding a lock on + // the collection to prevent it from being destructed. + oplogTruncateMarkers = rs->getCollectionTruncateMarkers(); + invariant(oplogTruncateMarkers); + } + + if (!oplogTruncateMarkers->awaitHasExcessMarkersOrDead(opCtx)) { + // Oplog went away or we timed out waiting for oplog space to reclaim. return false; } - auto rs = oplog->getRecordStore(); - if (!rs->yieldAndAwaitOplogDeletionRequest(opCtx.get())) { - return false; // Oplog went away. + + { + // Oplog state could have changed while yielding. Reacquire global lock + // and refresh oplog state to ensure we have a valid pointer. + AutoGetOplog oplogWrite(opCtx, OplogAccessMode::kWrite); + const auto& oplog = oplogWrite.getCollection(); + if (!oplog) { + LOGV2_DEBUG(9064300, 2, "oplog collection does not exist"); + return false; + } + auto rs = oplog->getRecordStore(); + rs->reclaimOplog(opCtx); } - rs->reclaimOplog(opCtx.get()); } catch (const ExceptionFor& e) { LOGV2_DEBUG(5929700, 1, @@ -97,6 +119,10 @@ bool OplogCapMaintainerThread::_deleteExcessDocuments() { "error"_attr = e.toStatus()); } catch (const DBException& ex) { if (!opCtx->checkForInterruptNoAssert().isOK()) { + LOGV2_DEBUG(9064301, + 1, + "Oplog cap maintainer thread was interrupted, but can safely continue", + "error"_attr = ex); return false; } @@ -114,12 +140,28 @@ void OplogCapMaintainerThread::run() { ThreadClient tc(_name, getGlobalServiceContext()); while (!globalInShutdownDeprecated()) { + auto opCtx = tc->makeOperationContext(); + if (MONGO_unlikely(hangOplogCapMaintainerThread.shouldFail())) { LOGV2(5095500, "Hanging the oplog cap maintainer thread due to fail point"); - hangOplogCapMaintainerThread.pauseWhileSet(); + try { + hangOplogCapMaintainerThread.pauseWhileSet(opCtx.get()); + } catch (DBException& ex) { + auto interruptStatus = opCtx->checkForInterruptNoAssert(); + if (!interruptStatus.isOK()) { + LOGV2(9064302, + "Stop hanging the oplog cap maintainer thread due to interrupted fail " + "point wait", + "ex"_attr = ex, + "interruptStatus"_attr = interruptStatus, + "shutdown"_attr = globalInShutdownDeprecated()); + continue; + } + throw; + } } - if (!_deleteExcessDocuments() && !globalInShutdownDeprecated()) { + if (!_deleteExcessDocuments(opCtx.get()) && !globalInShutdownDeprecated()) { sleepmillis(1000); // Back off in case there were problems deleting. } } diff --git a/src/mongo/db/storage/oplog_cap_maintainer_thread.h b/src/mongo/db/storage/oplog_cap_maintainer_thread.h index e322a1e6a9e2e..baea76c415829 100644 --- a/src/mongo/db/storage/oplog_cap_maintainer_thread.h +++ b/src/mongo/db/storage/oplog_cap_maintainer_thread.h @@ -32,6 +32,7 @@ #include #include "mongo/db/namespace_string.h" +#include "mongo/db/operation_context.h" #include "mongo/util/background.h" namespace mongo { @@ -60,7 +61,7 @@ class OplogCapMaintainerThread : public BackgroundJob { /** * Returns true iff there was an oplog to delete from. */ - bool _deleteExcessDocuments(); + bool _deleteExcessDocuments(OperationContext* opCtx); std::string _name = std::string("OplogCapMaintainerThread-") + NamespaceString::kRsOplogNamespace.toString(); diff --git a/src/mongo/db/storage/record_store.h b/src/mongo/db/storage/record_store.h index 2477c429034e3..89f4bf7af9243 100644 --- a/src/mongo/db/storage/record_store.h +++ b/src/mongo/db/storage/record_store.h @@ -46,6 +46,7 @@ namespace mongo { class Collection; class CollectionPtr; +class CollectionTruncateMarkers; class MAdvise; class OperationContext; @@ -668,11 +669,12 @@ class RecordStore { } /** - * Returns false if the oplog was dropped while waiting for a deletion request. + * Returns a shared reference to the oplog truncate markers to allow the caller to wait + * for a deletion request. * This should only be called if StorageEngine::supportsOplogTruncateMarkers() is true. * Storage engines supporting oplog truncate markers must implement this function. */ - virtual bool yieldAndAwaitOplogDeletionRequest(OperationContext* opCtx) { + virtual std::shared_ptr getCollectionTruncateMarkers() { MONGO_UNREACHABLE; } diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h index 00eac79661a76..8c9932bbf095c 100644 --- a/src/mongo/db/storage/storage_engine.h +++ b/src/mongo/db/storage/storage_engine.h @@ -200,6 +200,8 @@ class StorageEngine { /** * List the databases stored in this storage engine. + * This function doesn't return databases whose creation has committed durably but hasn't been + * published yet in the CollectionCatalog. */ virtual std::vector listDatabases( boost::optional tenantId = boost::none) const = 0; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_begin_transaction_block.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_begin_transaction_block.cpp index aa484470e43d5..77ed75ea60c93 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_begin_transaction_block.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_begin_transaction_block.cpp @@ -45,6 +45,40 @@ namespace mongo { + +namespace { + +// There are a few delicate restore scenarios where untimestamped writes are still required. +bool allowUntimestampedWrites() { + // Magic restore may need to perform untimestamped writes on timestamped tables as a part of + // the server automated restore procedure. + if (storageGlobalParams.magicRestore) { + return true; + } + + if (!gAllowUnsafeUntimestampedWrites) { + return false; + } + + // Ignore timestamps in selective restore mode. + if (storageGlobalParams.restore) { + return true; + } + + // We can safely ignore setting this configuration option when recovering from the + // oplog as standalone because: + // 1. Replaying oplog entries write with a timestamp. + // 2. The instance is put in read-only mode after oplog application has finished. + if (getReplSetMemberInStandaloneMode(getGlobalServiceContext()) && + !repl::ReplSettings::shouldRecoverFromOplogAsStandalone()) { + return true; + } + + return false; +} + +} // namespace + using namespace fmt::literals; WiredTigerBeginTxnBlock::WiredTigerBeginTxnBlock( @@ -73,15 +107,8 @@ WiredTigerBeginTxnBlock::WiredTigerBeginTxnBlock( } builder << "),"; } - if (allowUntimestampedWrite != RecoveryUnit::UntimestampedWriteAssertionLevel::kEnforce) { - builder << "no_timestamp=true,"; - } else if (MONGO_unlikely(gAllowUnsafeUntimestampedWrites && - getReplSetMemberInStandaloneMode(getGlobalServiceContext()) && - !repl::ReplSettings::shouldRecoverFromOplogAsStandalone())) { - // We can safely ignore setting this configuration option when recovering from the oplog as - // standalone because: - // 1. Replaying oplog entries write with a timestamp. - // 2. The instance is put in read-only mode after oplog application has finished. + if (allowUntimestampedWrite != RecoveryUnit::UntimestampedWriteAssertionLevel::kEnforce || + MONGO_unlikely(allowUntimestampedWrites())) { builder << "no_timestamp=true,"; } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index f257050b10a7e..7d2a8c3d74eb9 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -1037,30 +1037,6 @@ std::deque getUniqueFiles(const std::vector& files, return result; } -/** - * Normalizes ident names with and without 'directoryPerDb' and 'wiredTigerDirectoryForIndexes' - * mode. - * - * The durable catalog can return idents in four forms: - * - // - * - directoryPerDb + wiredTigerDirectoryForIndexes - * - / - * - directoryPerDb - * - / - * - wiredTigerDirectoryForIndexes - * - - * - default, no options enabled - * - * ident_identifier: - - * ident_name: - - * - * This function trims the leading directory names leaving only the ident's unique identifier. - */ -inline std::string getIdentStem(const std::string& ident) { - boost::filesystem::path identPath(ident); - return identPath.stem().string(); -} - class StreamingCursorImpl : public StorageEngine::StreamingCursor { public: StreamingCursorImpl() = delete; @@ -1319,15 +1295,15 @@ WiredTigerKVEngine::beginNonBlockingBackup(OperationContext* opCtx, for (const DurableCatalog::EntryIdentifier& e : catalogEntries) { // Populate the collection ident with its namespace and UUID. UUID uuid = catalog->getMetaData(opCtx, e.catalogId)->options.uuid.value(); - std::string collectionIdent = getIdentStem(e.ident); + std::string collectionIdent = e.ident; _wtBackup.identToNamespaceAndUUIDMap.emplace(collectionIdent, std::make_pair(e.nss, uuid)); // Populate the collection's index idents with the collection's namespace and UUID. std::vector idxIdents = catalog->getIndexIdents(opCtx, e.catalogId); for (const std::string& idxIdentFull : idxIdents) { - std::string idxIdent = getIdentStem(idxIdentFull); - _wtBackup.identToNamespaceAndUUIDMap.emplace(idxIdent, std::make_pair(e.nss, uuid)); + _wtBackup.identToNamespaceAndUUIDMap.emplace(idxIdentFull, + std::make_pair(e.nss, uuid)); } } } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h index 6a1445c8e986c..5cee8896fdd2f 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h @@ -59,6 +59,23 @@ class WiredTigerSessionCache; class WiredTigerSizeStorer; class WiredTigerEngineRuntimeConfigParameter; +/** + * With the absolute path to an ident and the parent dbpath, return the ident. + * + * Note that the ident can have 4 different forms depending on the combination + * of server parameters present (directoryperdb / wiredTigerDirectoryForIndexes). + * With any one of these server parameters enabled, a directory could be included + * in the returned ident. + * See the unit test WiredTigerKVEngineTest::ExtractIdentFromPath for example usage. + * + * Note (2) idents use unix-style separators (always, see + * durable_catalog.cpp:generateUniqueIdent) but ident paths are platform-dependant. + * This method returns the unix-style "/" separators always. + */ +std::string extractIdentFromPath(const boost::filesystem::path& dbpath, + const boost::filesystem::path& identAbsolutePath); + + Status validateExtraDiagnostics(const std::vector& value, const boost::optional& tenantId); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.idl b/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.idl index 5b72b8166663e..566ea3dc4036c 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.idl +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.idl @@ -196,6 +196,14 @@ server_parameters: cpp_varname: gWiredTigerSkipTableLoggingChecksOnStartup default: false + wiredTigerSkipTableLoggingChecksDuringValidation: + description: Skips table logging setting checks during validation + set_at: startup + cpp_vartype: 'bool' + cpp_varname: gWiredTigerSkipTableLoggingChecksDuringValidation + default: false + redact: false + wiredTigerStressConfig: description: >- Encourage more interesting races in WiredTiger. diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp index e2fa64e8f26bd..e3b6d948661d0 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp @@ -786,31 +786,8 @@ Timestamp WiredTigerRecordStore::getPinnedOplog() const { return _kvEngine->getPinnedOplog(); } -bool WiredTigerRecordStore::yieldAndAwaitOplogDeletionRequest(OperationContext* opCtx) { - // Create another reference to the oplog truncate markers while holding a lock on the collection - // to prevent it from being destructed. - std::shared_ptr oplogTruncateMarkers = _oplogTruncateMarkers; - - Locker* locker = opCtx->lockState(); - Locker::LockSnapshot snapshot; - - // Release any locks before waiting on the condition variable. It is illegal to access any - // methods or members of this record store after this line because it could be deleted. - locker->saveLockStateAndUnlock(&snapshot); - - // The top-level locks were freed, so also release any potential low-level (storage engine) - // locks that might be held. - WiredTigerRecoveryUnit* recoveryUnit = (WiredTigerRecoveryUnit*)opCtx->recoveryUnit(); - recoveryUnit->abandonSnapshot(); - recoveryUnit->beginIdle(); - - // Wait for an oplog deletion request, or for this record store to have been destroyed. - oplogTruncateMarkers->awaitHasExcessMarkersOrDead(opCtx); - - // Reacquire the locks that were released. - locker->restoreLockState(opCtx, snapshot); - - return !oplogTruncateMarkers->isDead(); +std::shared_ptr WiredTigerRecordStore::getCollectionTruncateMarkers() { + return _oplogTruncateMarkers->shared_from_this(); } void WiredTigerRecordStore::reclaimOplog(OperationContext* opCtx) { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h index 8094a98c7804f..fb84929275800 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h @@ -215,6 +215,8 @@ class WiredTigerRecordStore : public RecordStore { Status updateOplogSize(OperationContext* opCtx, long long newOplogSize) final; + std::shared_ptr getCollectionTruncateMarkers() override; + const std::string& getURI() const { return _uri; } @@ -248,8 +250,6 @@ class WiredTigerRecordStore : public RecordStore { bool isOpHidden_forTest(const RecordId& id) const; - bool yieldAndAwaitOplogDeletionRequest(OperationContext* opCtx) override; - /** * Attempts to truncate oplog entries before the pinned oplog timestamp. Truncation will occur * if the oplog is at capacity and the maximum retention time has elapsed. diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp index fda525191a4f0..2706bb8219179 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp @@ -880,6 +880,14 @@ void WiredTigerUtil::validateTableLogging(OperationContext* opCtx, bool& valid, std::vector& errors, std::vector& warnings) { + if (gWiredTigerSkipTableLoggingChecksDuringValidation) { + LOGV2(9264500, + "Skipping validation of table log settings due to usage of " + "'wiredTigerSkipTableLoggingChecksDuringValidation'", + "ident"_attr = uri); + return; + } + logv2::DynamicAttributes attrs; if (indexName) { attrs.add("index", indexName); diff --git a/src/mongo/idl/cluster_parameter_synchronization_helpers.cpp b/src/mongo/idl/cluster_parameter_synchronization_helpers.cpp index 22a8b45e107d5..fed60d534e9b5 100644 --- a/src/mongo/idl/cluster_parameter_synchronization_helpers.cpp +++ b/src/mongo/idl/cluster_parameter_synchronization_helpers.cpp @@ -32,6 +32,7 @@ #include "mongo/base/string_data.h" #include "mongo/db/audit.h" #include "mongo/db/catalog_raii.h" +#include "mongo/db/concurrency/locker.h" #include "mongo/db/multitenancy_gen.h" #include "mongo/logv2/log.h" @@ -43,9 +44,7 @@ constexpr auto kIdField = "_id"_sd; constexpr auto kCPTField = "clusterParameterTime"_sd; constexpr auto kOplog = "oplog"_sd; -void validateParameter(OperationContext* opCtx, - BSONObj doc, - const boost::optional& tenantId) { +void validateParameter(BSONObj doc, const boost::optional& tenantId) { auto nameElem = doc[kIdField]; uassert(ErrorCodes::OperationFailed, "Validate with invalid parameter name", @@ -97,6 +96,12 @@ void updateParameter(OperationContext* opCtx, uassertStatusOK(sp->validate(doc, tenantId)); + // Callback handlers of the ServerParameters take operation context and are allowed to acquire + // Shared and ExclusiveLock (ResourceMutex). These mutex acquisitions are not allowed to throw + // and not allowed to do any blocking work either. Placing this ULG here covers the first + // condition. + UninterruptibleLockGuard ulg(opCtx->lockState()); // NOLINT (ResourceMutex acquisition) + BSONObjBuilder oldValueBob; sp->append(opCtx, &oldValueBob, name.toString(), tenantId); audit::logUpdateCachedClusterParameter(opCtx->getClient(), oldValueBob.obj(), doc, tenantId); @@ -112,6 +117,12 @@ void clearParameter(OperationContext* opCtx, return; } + // Callback handlers of the ServerParameters take operation context and are allowed to acquire + // Shared and ExclusiveLock (ResourceMutex). These mutex acquisitions are not allowed to throw + // and not allowed to do any blocking work either. Placing this ULG here covers the first + // condition. + UninterruptibleLockGuard ulg(opCtx->lockState()); // NOLINT (ResourceMutex acquisition) + BSONObjBuilder oldValueBob; sp->append(opCtx, &oldValueBob, sp->name(), tenantId); diff --git a/src/mongo/idl/cluster_parameter_synchronization_helpers.h b/src/mongo/idl/cluster_parameter_synchronization_helpers.h index eb1751b2bc4df..ed81a5d4f9dda 100644 --- a/src/mongo/idl/cluster_parameter_synchronization_helpers.h +++ b/src/mongo/idl/cluster_parameter_synchronization_helpers.h @@ -35,9 +35,7 @@ namespace mongo { namespace cluster_parameters { -void validateParameter(OperationContext* opCtx, - BSONObj doc, - const boost::optional& tenantId); +void validateParameter(BSONObj doc, const boost::optional& tenantId); void updateParameter(OperationContext* opCtx, BSONObj doc, @@ -91,7 +89,7 @@ void doLoadAllTenantParametersFromDisk(OperationContext* opCtx, for (auto doc = cursor->next(); doc; doc = cursor->next()) { try { auto data = doc.get().data.toBson(); - validateParameter(opCtx, data, tenantId); + validateParameter(data, tenantId); onEntry(opCtx, data, mode, tenantId); } catch (const DBException& ex) { failures.push_back(ex.toStatus()); diff --git a/src/mongo/idl/cluster_server_parameter_op_observer.cpp b/src/mongo/idl/cluster_server_parameter_op_observer.cpp index b0cd8614c740c..69a417b9df6ca 100644 --- a/src/mongo/idl/cluster_server_parameter_op_observer.cpp +++ b/src/mongo/idl/cluster_server_parameter_op_observer.cpp @@ -68,7 +68,7 @@ void ClusterServerParameterOpObserver::onInserts(OperationContext* opCtx, for (auto it = first; it != last; ++it) { auto& doc = it->doc; auto tenantId = coll->ns().dbName().tenantId(); - cluster_parameters::validateParameter(opCtx, doc, tenantId); + cluster_parameters::validateParameter(doc, tenantId); opCtx->recoveryUnit()->onCommit( [doc, tenantId](OperationContext* opCtx, boost::optional) { cluster_parameters::updateParameter(opCtx, doc, kOplog, tenantId); @@ -84,7 +84,7 @@ void ClusterServerParameterOpObserver::onUpdate(OperationContext* opCtx, } auto tenantId = args.coll->ns().dbName().tenantId(); - cluster_parameters::validateParameter(opCtx, updatedDoc, tenantId); + cluster_parameters::validateParameter(updatedDoc, tenantId); opCtx->recoveryUnit()->onCommit( [updatedDoc, tenantId](OperationContext* opCtx, boost::optional) { cluster_parameters::updateParameter(opCtx, updatedDoc, kOplog, tenantId); diff --git a/src/mongo/rpc/op_legacy_integration_test.cpp b/src/mongo/rpc/op_legacy_integration_test.cpp index 55a342fa858c8..b0408d0ed018b 100644 --- a/src/mongo/rpc/op_legacy_integration_test.cpp +++ b/src/mongo/rpc/op_legacy_integration_test.cpp @@ -207,6 +207,18 @@ TEST(OpLegacy, UnsupportedReadOps) { ExceptionForCat); } +TEST(OpLegacy, InvalidNs) { + auto conn = getIntegrationTestConnection(); + + auto msg = makeMessage(dbQuery, [&](BufBuilder& b) { + b.appendNum(0); + b.appendStrBytes("nonullbyte"); + }); + // Since our request is not able to be parsed, we don't receive a response from the server. + Message ignore; + ASSERT_THROWS(conn->call(msg, ignore), DBException); +} + TEST(OpLegacy, GenericCommandViaOpQuery) { auto conn = getIntegrationTestConnection(); diff --git a/src/mongo/s/collection_routing_info_targeter.cpp b/src/mongo/s/collection_routing_info_targeter.cpp index aa52d6a9dc2ea..c55a6e6fbe4ab 100644 --- a/src/mongo/s/collection_routing_info_targeter.cpp +++ b/src/mongo/s/collection_routing_info_targeter.cpp @@ -715,7 +715,7 @@ void CollectionRoutingInfoTargeter::noteStaleShardResponse(OperationContext* opC Grid::get(opCtx) ->catalogCache() ->invalidateShardOrEntireCollectionEntryForShardedCollection( - _nss, staleInfo.getVersionWanted(), endpoint.shardName); + _nss, boost::none, endpoint.shardName); } _lastError = LastErrorType::kStaleShardVersion; diff --git a/src/mongo/s/service_entry_point_mongos.cpp b/src/mongo/s/service_entry_point_mongos.cpp index 4368e11724542..5278ab5b77e27 100644 --- a/src/mongo/s/service_entry_point_mongos.cpp +++ b/src/mongo/s/service_entry_point_mongos.cpp @@ -200,9 +200,17 @@ Future HandleRequest::run() { } Future ServiceEntryPointMongos::handleRequestImpl(OperationContext* opCtx, - const Message& message) noexcept { + const Message& message) try { auto hr = std::make_shared(opCtx, message); return hr->run(); +} catch (const DBException& ex) { + auto status = ex.toStatus(); + LOGV2(9431602, "Failed to handle request", "error"_attr = redact(status)); + return status; +} catch (...) { + auto error = exceptionToStatus(); + LOGV2_FATAL( + 9431601, "Request handling produced unhandled exception", "error"_attr = redact(error)); } Future ServiceEntryPointMongos::handleRequest(OperationContext* opCtx, diff --git a/src/mongo/s/service_entry_point_mongos.h b/src/mongo/s/service_entry_point_mongos.h index 42645ac236fe0..77f5a9e91bffc 100644 --- a/src/mongo/s/service_entry_point_mongos.h +++ b/src/mongo/s/service_entry_point_mongos.h @@ -45,8 +45,7 @@ class ServiceEntryPointMongos final : public ServiceEntryPointImpl { public: using ServiceEntryPointImpl::ServiceEntryPointImpl; - static Future handleRequestImpl(OperationContext* opCtx, - const Message& request) noexcept; + static Future handleRequestImpl(OperationContext* opCtx, const Message& request); Future handleRequest(OperationContext* opCtx, const Message& request) noexcept override; diff --git a/src/mongo/s/write_ops/batch_write_op.cpp b/src/mongo/s/write_ops/batch_write_op.cpp index a67992bbb4b34..247250fea49ae 100644 --- a/src/mongo/s/write_ops/batch_write_op.cpp +++ b/src/mongo/s/write_ops/batch_write_op.cpp @@ -76,26 +76,12 @@ BSONObj upgradeWriteConcern(const BSONObj& origWriteConcern) { return newWriteConcern.obj(); } -/** - * Helper to determine whether a number of targeted writes require a new targeted batch. - */ -bool isNewBatchRequiredOrdered(const std::vector>& writes, - const TargetedBatchMap& batchMap) { - for (auto&& write : writes) { - if (batchMap.find(write->endpoint.shardName) == batchMap.end()) { - return true; - } - } - - return false; -} - /** * Helper to determine whether a shard is already targeted with a different shardVersion, which * necessitates a new batch. This happens when a batch write includes a multi target write and * a single target write. */ -bool isNewBatchRequiredUnordered( +bool wasShardAlreadyTargetedWithDifferentShardVersion( const NamespaceString& nss, const std::vector>& writes, const std::map>& nsShardIdMap, @@ -123,6 +109,37 @@ bool isNewBatchRequiredUnordered( return false; } +/** + * Helper to determine whether a number of targeted writes require a new targeted batch. + */ +bool isNewBatchRequiredOrdered( + const NamespaceString& nss, + const std::vector>& writes, + const TargetedBatchMap& batchMap, + const std::map>& nsShardIdMap, + const std::map>& nsEndpointMap) { + // If this write targets a different shard, it needs to go in a different batch. + for (auto&& write : writes) { + if (batchMap.find(write->endpoint.shardName) == batchMap.end()) { + return true; + } + } + + // If we already targeted this shard with a different shard version, then we also need a new + // batch. + return wasShardAlreadyTargetedWithDifferentShardVersion( + nss, writes, nsShardIdMap, nsEndpointMap); +} + +bool isNewBatchRequiredUnordered( + const NamespaceString& nss, + const std::vector>& writes, + const std::map>& nsShardIdMap, + const std::map>& nsEndpointMap) { + return wasShardAlreadyTargetedWithDifferentShardVersion( + nss, writes, nsShardIdMap, nsEndpointMap); +} + /** * Helper to determine whether a number of targeted writes require a new targeted batch. */ @@ -375,7 +392,8 @@ StatusWith targetWriteOps(OperationContext* opCtx, // these targeted writes to any other endpoints. if (ordered && !batchMap.empty()) { dassert(batchMap.size() == 1u); - if (isNewBatchRequiredOrdered(writes, batchMap)) { + if (isNewBatchRequiredOrdered( + targeter.getNS(), writes, batchMap, nsShardIdMap, nsEndpointMap)) { writeOp.cancelWrites(nullptr); break; } diff --git a/src/mongo/s/write_ops/batch_write_op_test.cpp b/src/mongo/s/write_ops/batch_write_op_test.cpp index 438c36e97ef9e..b3c375808960e 100644 --- a/src/mongo/s/write_ops/batch_write_op_test.cpp +++ b/src/mongo/s/write_ops/batch_write_op_test.cpp @@ -1617,6 +1617,124 @@ TEST_F(BatchWriteOpTest, MultiOpTwoWCErrors) { ASSERT(clientResponse.isWriteConcernErrorSet()); } +// Ordered batch consisting of a multi=true delete that broadcasts to both shards and a multi=false +// delete targeted to one shard. Tests that when the first operation fails with a retryable error on +// one shard but succeeds on the other, the second operation in the batch must go in a separate +// batch because it uses a different ShardVersion (ShardVersion::IGNORE for the first op vs an +// actual ShardVersion for the second). +TEST_F(BatchWriteOpTest, MultiDeleteAndTargetedDeleteOrderedStaleConfig) { + NamespaceString nss = NamespaceString::createNamespaceString_forTest("foo.bar"); + CollectionGeneration collGeneration = {OID::gen(), Timestamp(1000, 1)}; + ShardEndpoint endpoint1( + ShardId("shard1"), + ShardVersionFactory::make(ChunkVersion(collGeneration, {1, 0}), boost::none), + boost::none); + ShardEndpoint endpoint2( + ShardId("shard2"), + ShardVersionFactory::make(ChunkVersion(collGeneration, {1, 1}), boost::none), + boost::none); + + // Note: Shard key is {x: 1} + auto targeter = initTargeterSplitRange(nss, endpoint1, endpoint2); + + BatchedCommandRequest request([&] { + write_ops::UpdateCommandRequest updateOp(nss); + // Two operations: The first one is multi=true and broadcasts to both shards. The second one + // is multi=false and is targeted to shard2. + updateOp.setUpdates({buildUpdate(BSON("x" << GTE << -10 << LT << 10), true), + buildUpdate(BSON("x" << 10), false)}); + return updateOp; + }()); + + BatchWriteOp batchOp(_opCtx, request); + + // Since ordered=true, the multi=true operation will be executed alone in the first batch. + std::map> targeted; + ASSERT_OK(batchOp.targetBatch(targeter, false, &targeted)); + ASSERT(!batchOp.isFinished()); + ASSERT_EQUALS(2u, targeted.size()); + + // Check that both shards were targeted, with ShardVersion::IGNORED. + for (const auto& endpoint : {endpoint1, endpoint2}) { + ASSERT_TRUE(targeted.contains(endpoint.shardName)); + const auto& targetedWriteBatch = targeted[endpoint.shardName]; + ASSERT_EQ(1u, targetedWriteBatch->getNumOps()); + + const auto& writeOp = targetedWriteBatch->getWrites().front(); + ASSERT_EQ(0, writeOp->writeOpRef.first); + ASSERT_TRUE(writeOp->endpoint.shardVersion); + ASSERT_TRUE(ShardVersion::isPlacementVersionIgnored(*writeOp->endpoint.shardVersion)); + } + + // Make shard1 succeed but shard2 fail with StaleConfig. + { + BatchedCommandResponse resp1; + buildResponse(0, &resp1); + batchOp.noteBatchResponse(*targeted[endpoint1.shardName], resp1, nullptr); + + BatchedCommandResponse resp2; + resp2.setStatus(Status::OK()); + resp2.addToErrDetails(write_ops::WriteError( + 0, + Status(StaleConfigInfo( + nss, + *targeted[endpoint2.shardName]->getWrites().front()->endpoint.shardVersion, + boost::none, + endpoint2.shardName), + "Stale error"))); + batchOp.noteBatchResponse(*targeted[endpoint2.shardName], resp2, nullptr); + } + + // StaleConfig is retryable so the operation should not be finished yet. Retry and check that + // only the shard that failed (shard2) is re-targeted, also with ShardVersion::IGNORED. + { + targeted.clear(); + ASSERT(!batchOp.isFinished()); + ASSERT_OK(batchOp.targetBatch(targeter, false, &targeted)); + ASSERT_EQUALS(1u, targeted.size()); + ASSERT_TRUE(targeted.contains(endpoint2.shardName)); + // Even though the second operation (targeted) also targets shard2, it must not execute yet + // because it must not use ShardVersion::IGNORED. + const auto& targetedWriteBatch = targeted[endpoint2.shardName]; + ASSERT_EQ(1u, targetedWriteBatch->getNumOps()); + const auto& writeOp = targetedWriteBatch->getWrites().front(); + ASSERT_EQ(0, writeOp->writeOpRef.first); + ASSERT_TRUE(writeOp->endpoint.shardVersion); + ASSERT_TRUE(ShardVersion::isPlacementVersionIgnored(*writeOp->endpoint.shardVersion)); + + + // Shard responds success. + BatchedCommandResponse resp2; + buildResponse(0, &resp2); + batchOp.noteBatchResponse(*targeted[endpoint2.shardName], resp2, nullptr); + } + + // The first operation has now completed. The second operation still has not been dispatched. + // Dispatch it now. Check that only shard2 is targeted, and that an actual ShardVersion is + // attached this time (not ShardVersion::IGNORED, since this is a targeted operation). + { + ASSERT(!batchOp.isFinished()); + targeted.clear(); + ASSERT_OK(batchOp.targetBatch(targeter, false, &targeted)); + ASSERT_EQUALS(1u, targeted.size()); + ASSERT_TRUE(targeted.contains(endpoint2.shardName)); + const auto& targetedWriteBatch = targeted[endpoint2.shardName]; + const auto& writeOp = targetedWriteBatch->getWrites().front(); + ASSERT_EQ(1, writeOp->writeOpRef.first); + ASSERT_EQ(endpoint2.shardVersion, writeOp->endpoint.shardVersion); + + // Shard responds success. + BatchedCommandResponse resp2; + buildResponse(0, &resp2); + batchOp.noteBatchResponse(*targeted[endpoint2.shardName], resp2, nullptr); + } + + ASSERT(batchOp.isFinished()); + BatchedCommandResponse clientResponse; + batchOp.buildClientResponse(&clientResponse); + ASSERT(clientResponse.getOk()); +} + TEST_F(BatchWriteOpTest, AttachingStmtIds) { NamespaceString nss = NamespaceString::createNamespaceString_forTest("foo.bar"); ShardEndpoint endpoint(ShardId("shard"), diff --git a/src/mongo/shell/query.js b/src/mongo/shell/query.js index a80ef5311d88e..de0a96a51032d 100644 --- a/src/mongo/shell/query.js +++ b/src/mongo/shell/query.js @@ -150,6 +150,7 @@ DBQuery.prototype._exec = function() { let readPreference = {}; if (findCmd["$readPreference"]) { readPreference = findCmd["$readPreference"]; + delete findCmd["$readPreference"]; } findCmd["$db"] = this._db.getName(); diff --git a/src/mongo/transport/asio/asio_transport_layer.cpp b/src/mongo/transport/asio/asio_transport_layer.cpp index 3511edf3256a3..2de8ae4544a2f 100644 --- a/src/mongo/transport/asio/asio_transport_layer.cpp +++ b/src/mongo/transport/asio/asio_transport_layer.cpp @@ -1685,12 +1685,7 @@ AsioTransportLayer::createTransientSSLContext(const TransientSSLParams& transien #ifdef __linux__ BatonHandle AsioTransportLayer::makeBaton(OperationContext* opCtx) const { - invariant(!opCtx->getBaton()); - - auto baton = std::make_shared(opCtx); - opCtx->setBaton(baton); - - return baton; + return std::dynamic_pointer_cast(std::make_shared(opCtx)); } #endif diff --git a/src/mongo/transport/service_executor_bm.cpp b/src/mongo/transport/service_executor_bm.cpp index 5130e580ac4b3..2a3383911281a 100644 --- a/src/mongo/transport/service_executor_bm.cpp +++ b/src/mongo/transport/service_executor_bm.cpp @@ -50,7 +50,7 @@ const auto kMaxThreads = 1; const auto kMaxChainSize = 1; #else /** 2x to benchmark the case of more threads than cores for curiosity's sake. */ -const auto kMaxThreads = 2 * ProcessInfo::getNumCores(); +const auto kMaxThreads = 2 * ProcessInfo::getNumLogicalCores(); const auto kMaxChainSize = 64; #endif diff --git a/src/mongo/transport/session_workflow_bm.cpp b/src/mongo/transport/session_workflow_bm.cpp index f9a85ad1d170e..ac87981fc9a4a 100644 --- a/src/mongo/transport/session_workflow_bm.cpp +++ b/src/mongo/transport/session_workflow_bm.cpp @@ -325,7 +325,7 @@ class SessionWorkflowBm : public benchmark::Fixture { const auto kMaxThreads = 1; #else /** 2x to benchmark the case of more threads than cores for curiosity's sake. */ -const auto kMaxThreads = 2 * ProcessInfo::getNumCores(); +const auto kMaxThreads = 2 * ProcessInfo::getNumLogicalCores(); #endif BENCHMARK_DEFINE_F(SessionWorkflowBm, Loop)(benchmark::State& state) { diff --git a/src/mongo/transport/transport_layer.h b/src/mongo/transport/transport_layer.h index 0c18900475cb4..91bca6b125b14 100644 --- a/src/mongo/transport/transport_layer.h +++ b/src/mongo/transport/transport_layer.h @@ -132,7 +132,7 @@ class TransportLayer { virtual ReactorHandle getReactor(WhichReactor which) = 0; virtual BatonHandle makeBaton(OperationContext* opCtx) const { - return opCtx->getServiceContext()->makeBaton(opCtx); + return {}; } std::shared_ptr getWireSpec() const { diff --git a/src/mongo/util/exception_filter_win32.cpp b/src/mongo/util/exception_filter_win32.cpp index 54ca6717400cc..07b89561112dd 100644 --- a/src/mongo/util/exception_filter_win32.cpp +++ b/src/mongo/util/exception_filter_win32.cpp @@ -105,10 +105,11 @@ void doMinidumpWithException(struct _EXCEPTION_POINTERS* exceptionInfo) { MINIDUMP_TYPE miniDumpType = #ifdef MONGO_CONFIG_DEBUG_BUILD - MiniDumpWithFullMemory; + static_cast(MiniDumpWithFullMemory | MiniDumpIgnoreInaccessibleMemory); #else static_cast(MiniDumpNormal | MiniDumpWithIndirectlyReferencedMemory | - MiniDumpWithProcessThreadData); + MiniDumpWithProcessThreadData | MiniDumpWithThreadInfo | + MiniDumpWithUnloadedModules | MiniDumpWithTokenInformation); #endif LOGV2(23132, "Writing minidump diagnostic file {dumpName}", diff --git a/src/mongo/util/processinfo.h b/src/mongo/util/processinfo.h index 509a9d5ccf1fe..2476b5c03ccfd 100644 --- a/src/mongo/util/processinfo.h +++ b/src/mongo/util/processinfo.h @@ -101,7 +101,7 @@ class ProcessInfo { /** * Get the number of (logical) CPUs */ - static unsigned getNumCores() { + static unsigned getNumLogicalCores() { return sysInfo().numCores; } @@ -124,7 +124,14 @@ class ProcessInfo { * If that information is not available, get the total number of CPUs. */ static unsigned long getNumAvailableCores() { - return ProcessInfo::getNumCoresForProcess().value_or(ProcessInfo::getNumCores()); + return ProcessInfo::getNumCoresForProcess().value_or(ProcessInfo::getNumLogicalCores()); + } + + /** + * Get the number of cores available for process or return the errorValue. + */ + static long getNumCoresAvailableToProcess(long errorValue = -1) { + return ProcessInfo::getNumCoresForProcess().value_or(errorValue); } /** diff --git a/src/mongo/util/processinfo_linux.cpp b/src/mongo/util/processinfo_linux.cpp index ac802777bda23..a89b95703fdb5 100644 --- a/src/mongo/util/processinfo_linux.cpp +++ b/src/mongo/util/processinfo_linux.cpp @@ -666,6 +666,11 @@ boost::optional ProcessInfo::getNumCoresForProcess() { #endif } + auto ec = lastSystemError(); + LOGV2(8366600, + "sched_getaffinity failed to collect cpu_set info", + "error"_attr = errorMessage(ec)); + return boost::none; } diff --git a/src/mongo/util/processinfo_test.cpp b/src/mongo/util/processinfo_test.cpp index 050835f97ac9e..14cf3c3d336db 100644 --- a/src/mongo/util/processinfo_test.cpp +++ b/src/mongo/util/processinfo_test.cpp @@ -52,11 +52,11 @@ TEST(ProcessInfo, GetNumAvailableCores) { defined(_WIN32) unsigned long numAvailCores = ProcessInfo::getNumAvailableCores(); ASSERT_GREATER_THAN(numAvailCores, 0u); - ASSERT_LESS_THAN_OR_EQUALS(numAvailCores, ProcessInfo::getNumCores()); + ASSERT_LESS_THAN_OR_EQUALS(numAvailCores, ProcessInfo::getNumLogicalCores()); #endif } TEST(ProcessInfo, GetNumCoresReturnsNonZeroNumberOfProcessors) { - ASSERT_GREATER_THAN(ProcessInfo::getNumCores(), 0u); + ASSERT_GREATER_THAN(ProcessInfo::getNumLogicalCores(), 0u); } } // namespace mongo_test diff --git a/src/third_party/mozjs/mongo_sources/jscustomallocator_oom.h b/src/third_party/mozjs/mongo_sources/jscustomallocator_oom.h index 18351e9787056..ac09b1028b4fc 100644 --- a/src/third_party/mozjs/mongo_sources/jscustomallocator_oom.h +++ b/src/third_party/mozjs/mongo_sources/jscustomallocator_oom.h @@ -59,7 +59,7 @@ namespace oom { // like. Testing worker threads is not supported. const ThreadType FirstThreadTypeToTest = THREAD_TYPE_MAIN; const ThreadType LastThreadTypeToTest = THREAD_TYPE_WASM_GENERATOR_TIER2; -# SERVER-64574 TODO: Should last thread type be changed here for testing + extern bool InitThreadType(void); extern void SetThreadType(ThreadType); extern JS_PUBLIC_API uint32_t GetThreadType(void); diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index dadd6f5c0ca18..5c97453cb6c9c 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -541,6 +541,7 @@ def __init__(self, name, desc, flags=''): SessionOpStat('session_table_alter_trigger_checkpoint', 'table alter triggering checkpoint calls', 'no_clear,no_scale'), SessionOpStat('session_table_compact_fail', 'table compact failed calls', 'no_clear,no_scale'), SessionOpStat('session_table_compact_fail_cache_pressure', 'table compact failed calls due to cache pressure', 'no_clear,no_scale'), + SessionOpStat('session_table_compact_passes', 'table compact passes', 'no_scale'), SessionOpStat('session_table_compact_running', 'table compact running', 'no_clear,no_scale'), SessionOpStat('session_table_compact_skipped', 'table compact skipped as process would not reduce file size', 'no_clear,no_scale'), SessionOpStat('session_table_compact_success', 'table compact successful calls', 'no_clear,no_scale'), diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 1f7aab7a6e528..a97b3b6586c36 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-7.0", - "commit": "eb70c851b7d807e7fb56dd08eba609f5dc02f6cf" + "commit": "79cf40eef56b1262479405266dc14d4489d88af3" } diff --git a/src/third_party/wiredtiger/src/btree/bt_compact.c b/src/third_party/wiredtiger/src/btree/bt_compact.c index 8d01449c62b7e..817707d3a57fa 100644 --- a/src/third_party/wiredtiger/src/btree/bt_compact.c +++ b/src/third_party/wiredtiger/src/btree/bt_compact.c @@ -165,23 +165,26 @@ __compact_page(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) WT_REF_LOCK(session, ref, &previous_state); /* - * Skip deleted pages but consider them progress (the on-disk block is discarded by the next - * checkpoint). + * Don't bother rewriting deleted pages but also don't skip. The on-disk block is discarded by + * the next checkpoint. */ - if (previous_state == WT_REF_DELETED) + if (previous_state == WT_REF_DELETED && ref->page_del == NULL) *skipp = false; /* - * If it's on-disk, get a copy of the address and ask the block manager to rewrite the block if + * If it's on disk, get a copy of the address and ask the block manager to rewrite the block if * it's useful. This is safe because we're holding the WT_REF locked, so nobody can read the - * page giving eviction a chance to modify the address. + * page giving eviction a chance to modify the address. Note that a deleted ref that is not + * globally visible is still on disk. * * In this path, we are holding the WT_REF lock across two OS buffer cache I/Os (the read of the * original block and the write of the new block), plus whatever overhead that entails. It's not * ideal, we could release the lock, but then we'd have to deal with the block having been read * into memory while we were moving it. */ - if (previous_state == WT_REF_DISK && __wt_ref_addr_copy(session, ref, ©)) { + if ((previous_state == WT_REF_DISK || + (previous_state == WT_REF_DELETED && ref->page_del != NULL)) && + __wt_ref_addr_copy(session, ref, ©)) { bm = S2BT(session)->bm; addr_size = copy.size; WT_ERR(bm->compact_page_rewrite(bm, session, copy.addr, &addr_size, skipp)); diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 96446b27c1245..3e4349297058d 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -820,6 +820,7 @@ struct __wt_connection_stats { int64_t session_table_alter_skip; int64_t session_table_compact_fail; int64_t session_table_compact_fail_cache_pressure; + int64_t session_table_compact_passes; int64_t session_table_compact_running; int64_t session_table_compact_skipped; int64_t session_table_compact_success; diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 06f025b80c472..662f3dab871d0 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -6537,318 +6537,320 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1479 /*! session: table compact failed calls due to cache pressure */ #define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL_CACHE_PRESSURE 1480 +/*! session: table compact passes */ +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_PASSES 1481 /*! session: table compact running */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_RUNNING 1481 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_RUNNING 1482 /*! session: table compact skipped as process would not reduce file size */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SKIPPED 1482 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SKIPPED 1483 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1483 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1484 /*! session: table compact timeout */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_TIMEOUT 1484 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_TIMEOUT 1485 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1485 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1486 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1486 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1487 /*! session: table create with import failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_IMPORT_FAIL 1487 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_IMPORT_FAIL 1488 /*! session: table create with import successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_IMPORT_SUCCESS 1488 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_IMPORT_SUCCESS 1489 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1489 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1490 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1490 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1491 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1491 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1492 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1492 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1493 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1493 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1494 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1494 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1495 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1495 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1496 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1496 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1497 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1497 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1498 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1498 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1499 /*! session: tiered operations dequeued and processed */ -#define WT_STAT_CONN_TIERED_WORK_UNITS_DEQUEUED 1499 +#define WT_STAT_CONN_TIERED_WORK_UNITS_DEQUEUED 1500 /*! session: tiered operations removed without processing */ -#define WT_STAT_CONN_TIERED_WORK_UNITS_REMOVED 1500 +#define WT_STAT_CONN_TIERED_WORK_UNITS_REMOVED 1501 /*! session: tiered operations scheduled */ -#define WT_STAT_CONN_TIERED_WORK_UNITS_CREATED 1501 +#define WT_STAT_CONN_TIERED_WORK_UNITS_CREATED 1502 /*! session: tiered storage local retention time (secs) */ -#define WT_STAT_CONN_TIERED_RETENTION 1502 +#define WT_STAT_CONN_TIERED_RETENTION 1503 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1503 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1504 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1504 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1505 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1505 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1506 /*! thread-yield: application thread snapshot refreshed for eviction */ -#define WT_STAT_CONN_APPLICATION_EVICT_SNAPSHOT_REFRESHED 1506 +#define WT_STAT_CONN_APPLICATION_EVICT_SNAPSHOT_REFRESHED 1507 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1507 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1508 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1508 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1509 /*! * thread-yield: connection close blocked waiting for transaction state * stabilization */ -#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1509 +#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1510 /*! thread-yield: connection close yielded for lsm manager shutdown */ -#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1510 +#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1511 /*! thread-yield: data handle lock yielded */ -#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1511 +#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1512 /*! * thread-yield: get reference for page index and slot time sleeping * (usecs) */ -#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1512 +#define WT_STAT_CONN_PAGE_INDEX_SLOT_REF_BLOCKED 1513 /*! thread-yield: page access yielded due to prepare state change */ -#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1513 +#define WT_STAT_CONN_PREPARED_TRANSITION_BLOCKED_PAGE 1514 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1514 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1515 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1515 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1516 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1516 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1517 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1517 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1518 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1518 +#define WT_STAT_CONN_PAGE_SLEEP 1519 /*! * thread-yield: page delete rollback time sleeping for state change * (usecs) */ -#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1519 +#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1520 /*! thread-yield: page reconciliation yielded due to child modification */ -#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1520 +#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1521 /*! transaction: Number of prepared updates */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES 1521 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES 1522 /*! transaction: Number of prepared updates committed */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COMMITTED 1522 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_COMMITTED 1523 /*! transaction: Number of prepared updates repeated on the same key */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_KEY_REPEATED 1523 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_KEY_REPEATED 1524 /*! transaction: Number of prepared updates rolled back */ -#define WT_STAT_CONN_TXN_PREPARED_UPDATES_ROLLEDBACK 1524 +#define WT_STAT_CONN_TXN_PREPARED_UPDATES_ROLLEDBACK 1525 /*! * transaction: a reader raced with a prepared transaction commit and * skipped an update or updates */ -#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_COMMIT 1525 +#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_COMMIT 1526 /*! transaction: checkpoint has acquired a snapshot for its transaction */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SNAPSHOT_ACQUIRED 1526 +#define WT_STAT_CONN_TXN_CHECKPOINT_SNAPSHOT_ACQUIRED 1527 /*! transaction: number of times overflow removed value is read */ -#define WT_STAT_CONN_TXN_READ_OVERFLOW_REMOVE 1527 +#define WT_STAT_CONN_TXN_READ_OVERFLOW_REMOVE 1528 /*! transaction: oldest pinned transaction ID rolled back for eviction */ -#define WT_STAT_CONN_TXN_ROLLBACK_OLDEST_PINNED 1528 +#define WT_STAT_CONN_TXN_ROLLBACK_OLDEST_PINNED 1529 /*! transaction: prepared transactions */ -#define WT_STAT_CONN_TXN_PREPARE 1529 +#define WT_STAT_CONN_TXN_PREPARE 1530 /*! transaction: prepared transactions committed */ -#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1530 +#define WT_STAT_CONN_TXN_PREPARE_COMMIT 1531 /*! transaction: prepared transactions currently active */ -#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1531 +#define WT_STAT_CONN_TXN_PREPARE_ACTIVE 1532 /*! transaction: prepared transactions rolled back */ -#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1532 +#define WT_STAT_CONN_TXN_PREPARE_ROLLBACK 1533 /*! transaction: query timestamp calls */ -#define WT_STAT_CONN_TXN_QUERY_TS 1533 +#define WT_STAT_CONN_TXN_QUERY_TS 1534 /*! transaction: race to read prepared update retry */ -#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1534 +#define WT_STAT_CONN_TXN_READ_RACE_PREPARE_UPDATE 1535 /*! transaction: rollback to stable calls */ -#define WT_STAT_CONN_TXN_RTS 1535 +#define WT_STAT_CONN_TXN_RTS 1536 /*! * transaction: rollback to stable history store keys that would have * been swept in non-dryrun mode */ -#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS_DRYRUN 1536 +#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS_DRYRUN 1537 /*! * transaction: rollback to stable history store records with stop * timestamps older than newer records */ -#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1537 +#define WT_STAT_CONN_TXN_RTS_HS_STOP_OLDER_THAN_NEWER_START 1538 /*! transaction: rollback to stable inconsistent checkpoint */ -#define WT_STAT_CONN_TXN_RTS_INCONSISTENT_CKPT 1538 +#define WT_STAT_CONN_TXN_RTS_INCONSISTENT_CKPT 1539 /*! transaction: rollback to stable keys removed */ -#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1539 +#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED 1540 /*! transaction: rollback to stable keys restored */ -#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1540 +#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED 1541 /*! * transaction: rollback to stable keys that would have been removed in * non-dryrun mode */ -#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED_DRYRUN 1541 +#define WT_STAT_CONN_TXN_RTS_KEYS_REMOVED_DRYRUN 1542 /*! * transaction: rollback to stable keys that would have been restored in * non-dryrun mode */ -#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED_DRYRUN 1542 +#define WT_STAT_CONN_TXN_RTS_KEYS_RESTORED_DRYRUN 1543 /*! transaction: rollback to stable pages visited */ -#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1543 +#define WT_STAT_CONN_TXN_RTS_PAGES_VISITED 1544 /*! transaction: rollback to stable restored tombstones from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1544 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES 1545 /*! transaction: rollback to stable restored updates from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES 1545 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES 1546 /*! transaction: rollback to stable skipping delete rle */ -#define WT_STAT_CONN_TXN_RTS_DELETE_RLE_SKIPPED 1546 +#define WT_STAT_CONN_TXN_RTS_DELETE_RLE_SKIPPED 1547 /*! transaction: rollback to stable skipping stable rle */ -#define WT_STAT_CONN_TXN_RTS_STABLE_RLE_SKIPPED 1547 +#define WT_STAT_CONN_TXN_RTS_STABLE_RLE_SKIPPED 1548 /*! transaction: rollback to stable sweeping history store keys */ -#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1548 +#define WT_STAT_CONN_TXN_RTS_SWEEP_HS_KEYS 1549 /*! * transaction: rollback to stable tombstones from history store that * would have been restored in non-dryrun mode */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES_DRYRUN 1549 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_TOMBSTONES_DRYRUN 1550 /*! transaction: rollback to stable tree walk skipping pages */ -#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1550 +#define WT_STAT_CONN_TXN_RTS_TREE_WALK_SKIP_PAGES 1551 /*! transaction: rollback to stable updates aborted */ -#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1551 +#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED 1552 /*! * transaction: rollback to stable updates from history store that would * have been restored in non-dryrun mode */ -#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES_DRYRUN 1552 +#define WT_STAT_CONN_TXN_RTS_HS_RESTORE_UPDATES_DRYRUN 1553 /*! transaction: rollback to stable updates removed from history store */ -#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1553 +#define WT_STAT_CONN_TXN_RTS_HS_REMOVED 1554 /*! * transaction: rollback to stable updates that would have been aborted * in non-dryrun mode */ -#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED_DRYRUN 1554 +#define WT_STAT_CONN_TXN_RTS_UPD_ABORTED_DRYRUN 1555 /*! * transaction: rollback to stable updates that would have been removed * from history store in non-dryrun mode */ -#define WT_STAT_CONN_TXN_RTS_HS_REMOVED_DRYRUN 1555 +#define WT_STAT_CONN_TXN_RTS_HS_REMOVED_DRYRUN 1556 /*! transaction: sessions scanned in each walk of concurrent sessions */ -#define WT_STAT_CONN_TXN_SESSIONS_WALKED 1556 +#define WT_STAT_CONN_TXN_SESSIONS_WALKED 1557 /*! transaction: set timestamp calls */ -#define WT_STAT_CONN_TXN_SET_TS 1557 +#define WT_STAT_CONN_TXN_SET_TS 1558 /*! transaction: set timestamp durable calls */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1558 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE 1559 /*! transaction: set timestamp durable updates */ -#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1559 +#define WT_STAT_CONN_TXN_SET_TS_DURABLE_UPD 1560 /*! transaction: set timestamp oldest calls */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1560 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST 1561 /*! transaction: set timestamp oldest updates */ -#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1561 +#define WT_STAT_CONN_TXN_SET_TS_OLDEST_UPD 1562 /*! transaction: set timestamp stable calls */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE 1562 +#define WT_STAT_CONN_TXN_SET_TS_STABLE 1563 /*! transaction: set timestamp stable updates */ -#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1563 +#define WT_STAT_CONN_TXN_SET_TS_STABLE_UPD 1564 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1564 +#define WT_STAT_CONN_TXN_BEGIN 1565 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1565 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1566 /*! * transaction: transaction checkpoint currently running for history * store file */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING_HS 1566 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING_HS 1567 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1567 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1568 /*! * transaction: transaction checkpoint history store file duration * (usecs) */ -#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1568 +#define WT_STAT_CONN_TXN_HS_CKPT_DURATION 1569 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1569 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1570 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1570 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1571 /*! * transaction: transaction checkpoint most recent duration for gathering * all handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1571 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION 1572 /*! * transaction: transaction checkpoint most recent duration for gathering * applied handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1572 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_APPLY 1573 /*! * transaction: transaction checkpoint most recent duration for gathering * skipped handles (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1573 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_DURATION_SKIP 1574 /*! transaction: transaction checkpoint most recent handles applied */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1574 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_APPLIED 1575 /*! transaction: transaction checkpoint most recent handles skipped */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1575 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_SKIPPED 1576 /*! transaction: transaction checkpoint most recent handles walked */ -#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1576 +#define WT_STAT_CONN_TXN_CHECKPOINT_HANDLE_WALKED 1577 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1577 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1578 /*! transaction: transaction checkpoint prepare currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1578 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RUNNING 1579 /*! transaction: transaction checkpoint prepare max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1579 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MAX 1580 /*! transaction: transaction checkpoint prepare min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1580 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_MIN 1581 /*! transaction: transaction checkpoint prepare most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1581 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_RECENT 1582 /*! transaction: transaction checkpoint prepare total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1582 +#define WT_STAT_CONN_TXN_CHECKPOINT_PREP_TOTAL 1583 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1583 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1584 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1584 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1585 /*! transaction: transaction checkpoint stop timing stress active */ -#define WT_STAT_CONN_TXN_CHECKPOINT_STOP_STRESS_ACTIVE 1585 +#define WT_STAT_CONN_TXN_CHECKPOINT_STOP_STRESS_ACTIVE 1586 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1586 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1587 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1587 +#define WT_STAT_CONN_TXN_CHECKPOINT 1588 /*! transaction: transaction checkpoints due to obsolete pages */ -#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1588 +#define WT_STAT_CONN_TXN_CHECKPOINT_OBSOLETE_APPLIED 1589 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1589 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1590 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1590 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1591 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1591 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1592 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1592 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1593 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1593 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1594 /*! transaction: transaction range of timestamps currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1594 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP 1595 /*! transaction: transaction range of timestamps pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1595 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_CHECKPOINT 1596 /*! * transaction: transaction range of timestamps pinned by the oldest * active read timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1596 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_READER 1597 /*! * transaction: transaction range of timestamps pinned by the oldest * timestamp */ -#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1597 +#define WT_STAT_CONN_TXN_PINNED_TIMESTAMP_OLDEST 1598 /*! transaction: transaction read timestamp of the oldest active reader */ -#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1598 +#define WT_STAT_CONN_TXN_TIMESTAMP_OLDEST_ACTIVE_READ 1599 /*! transaction: transaction rollback to stable currently running */ -#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1599 +#define WT_STAT_CONN_TXN_ROLLBACK_TO_STABLE_RUNNING 1600 /*! transaction: transaction walk of concurrent sessions */ -#define WT_STAT_CONN_TXN_WALK_SESSIONS 1600 +#define WT_STAT_CONN_TXN_WALK_SESSIONS 1601 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1601 +#define WT_STAT_CONN_TXN_COMMIT 1602 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1602 +#define WT_STAT_CONN_TXN_ROLLBACK 1603 /*! transaction: update conflicts */ -#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1603 +#define WT_STAT_CONN_TXN_UPDATE_CONFLICT 1604 /*! * @} diff --git a/src/third_party/wiredtiger/src/session/session_compact.c b/src/third_party/wiredtiger/src/session/session_compact.c index 1d8c83cad5363..6a39bd840f678 100644 --- a/src/third_party/wiredtiger/src/session/session_compact.c +++ b/src/third_party/wiredtiger/src/session/session_compact.c @@ -251,6 +251,8 @@ __compact_worker(WT_SESSION_IMPL *session) * clearly more than we need); quit if we make no progress. */ for (loop = 0; loop < 100; ++loop) { + WT_STAT_CONN_SET(session, session_table_compact_passes, loop); + /* Step through the list of files being compacted. */ for (another_pass = false, i = 0; i < session->op_handle_next; ++i) { /* Skip objects where there's no more work. */ @@ -306,6 +308,7 @@ __compact_worker(WT_SESSION_IMPL *session) err: session->compact_state = WT_COMPACT_NONE; + WT_STAT_CONN_SET(session, session_table_compact_passes, 0); return (ret); } diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 81040dc3b725c..db6ab6c0df208 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -1815,6 +1815,7 @@ static const char *const __stats_connection_desc[] = { "session: table alter unchanged and skipped", "session: table compact failed calls", "session: table compact failed calls due to cache pressure", + "session: table compact passes", "session: table compact running", "session: table compact skipped as process would not reduce file size", "session: table compact successful calls", @@ -2465,6 +2466,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) /* not clearing session_table_alter_skip */ /* not clearing session_table_compact_fail */ /* not clearing session_table_compact_fail_cache_pressure */ + stats->session_table_compact_passes = 0; /* not clearing session_table_compact_running */ /* not clearing session_table_compact_skipped */ /* not clearing session_table_compact_success */ @@ -3154,6 +3156,7 @@ __wt_stat_connection_aggregate(WT_CONNECTION_STATS **from, WT_CONNECTION_STATS * to->session_table_compact_fail += WT_STAT_READ(from, session_table_compact_fail); to->session_table_compact_fail_cache_pressure += WT_STAT_READ(from, session_table_compact_fail_cache_pressure); + to->session_table_compact_passes += WT_STAT_READ(from, session_table_compact_passes); to->session_table_compact_running += WT_STAT_READ(from, session_table_compact_running); to->session_table_compact_skipped += WT_STAT_READ(from, session_table_compact_skipped); to->session_table_compact_success += WT_STAT_READ(from, session_table_compact_success); diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 536ce921c324d..0f319ced88d46 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -1215,8 +1215,10 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__checkpoint_apply_to_dhandles(session, cfg, __wt_checkpoint_sync)); /* Sync the history store file. */ - if (F_ISSET(hs_dhandle, WT_DHANDLE_OPEN)) + if (F_ISSET(hs_dhandle, WT_DHANDLE_OPEN)) { WT_WITH_DHANDLE(session, hs_dhandle, ret = __wt_checkpoint_sync(session, NULL)); + WT_ERR(ret); + } time_stop_fsync = __wt_clock(session); fsync_duration_usecs = WT_CLOCKDIFF_US(time_stop_fsync, time_start_fsync); diff --git a/src/third_party/wiredtiger/test/evergreen/setup_spawn_host.sh b/src/third_party/wiredtiger/test/evergreen/setup_spawn_host.sh new file mode 100644 index 0000000000000..795c1c728206d --- /dev/null +++ b/src/third_party/wiredtiger/test/evergreen/setup_spawn_host.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# +# The script is used to setup the environment for a spawn host. Currently the script +# unzips the test artifacts, prepend the toolchain path to the PATH environment and exports the +# LD_LIBRARY_PATH. + +# Once setup, the script will notify the user who spawned the host that it is finished. +# +cd $HOME || exit 1 +TOOLCHAIN_ROOT=/opt/mongodbtoolchain/v4 + +# Communicate to users that logged in before the script started that nothing is ready. +wall "The setup_spawn_host script has just started setting up the debugging environment." + +# Make a directory on the larger volume. Soft-link it under the home directory. +mkdir -p /data/wiredtiger +ln -s /data/wiredtiger . +cd wiredtiger || exit 1 + +# Find the test artifacts. +WT_ARCHIVE=$(ls /data/mci/artifacts-*/*.tgz | grep -v "compile" | head -n 1 2>/dev/null) +if [[ -n $WT_ARCHIVE ]]; then + tar --wildcards -xzf $WT_ARCHIVE +fi + +# Setup the gdb environment if there are core dumps present in the artefacts. +GDB_CORE_DUMP=$(find ${HOME}/wiredtiger/cmake_build -name "*.core" | head -n 1 2>/dev/null) +if [[ -n $GDB_CORE_DUMP ]]; then + # Read the CMakeCache txt file to find the old source directory and fix up the expected path to + # the correct path on the new machine. + OLD_WT_PATH=$(grep "WiredTiger_SOURCE_DIR" ${HOME}/wiredtiger/cmake_build/CMakeCache.txt | sed -e 's/[^\/]*//') + + # GDB debug symbols includes a path to the source code we originally compiled. However since we + # compiled on a different machine and copied the artefacts to this spawn host, the file paths + # have changed. substitute-path command tells gdb the new location of the source code. Therefore + # set up the shared library paths and add another source directory to look under using + # substitute path. + cat >> ~/.gdbinit << EOF +set solib-search-path ${HOME}/wiredtiger/cmake_build/lang/python:${HOME}/wiredtiger/cmake_build:${HOME}/wiredtiger/TCMALLOC_LIB/lib +set substitute-path ${OLD_WT_PATH} ${HOME}/wiredtiger +set print pretty on +EOF + +fi + +# Install CMake into the machine. +. test/evergreen/find_cmake.sh + +cat >> ~/.profile << EOF +# Prepend the toolchain to the PATH environment and export the TCMALLOC and WT library into the +# library path. +export PATH="${TOOLCHAIN_ROOT}/bin:\$PATH" +export LD_LIBRARY_PATH="${HOME}/wiredtiger/cmake_build:${HOME}/wiredtiger/TCMALLOC_LIB/lib:\$LD_LIBRARY_PATH" +EOF + +echo 'if [ -f ~/.profile ]; then +. ~/.profile +fi' >> ~/.bash_profile + +# Send a Slack notification as the very last thing the setup_spawn_host script does. +ssh_user=$(whoami) +evg_credentials_pathname=~/.evergreen.yml +evg_binary_pathname=evergreen + +wall "The setup_spawn_host script has completed, please relogin to ensure the right environment variables are set." + +slack_user=$(awk '{if ($1 == "user:") print $2}' "$evg_credentials_pathname") +# Refer to the https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html +# documentation for more information on the AWS instance metadata endpoints. +aws_metadata_svc="http://169.254.169.254" +aws_token=$(curl -s -X PUT "$aws_metadata_svc/latest/api/token" -H 'X-aws-ec2-metadata-token-ttl-seconds: 60') +ssh_host=$(curl -s -H "X-aws-ec2-metadata-token: $aws_token" "$aws_metadata_svc/latest/meta-data/public-hostname") +slack_message="The setup_spawn_host script has finished setting things up. Please run "'```'"ssh $ssh_user@$ssh_host"'```'" to log in." + +# The Evergreen spawn host is expected to be provisioned with the user's .evergreen.yml credentials. +# But in case something unexpected happens we don't want the setup_spawn_host script itself +# to error. +if [[ -n "${slack_user}" ]]; then + "$evg_binary_pathname" --config "$evg_credentials_pathname" notify slack -t "@$slack_user" -m "$slack_message" +fi diff --git a/src/third_party/wiredtiger/test/suite/test_compact12.py b/src/third_party/wiredtiger/test/suite/test_compact12.py new file mode 100644 index 0000000000000..7c88040363b82 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_compact12.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wttest +from wiredtiger import stat + +kilobyte = 1024 + +# test_compact12.py +# This test creates: +# +# - One table with the first 1/4 of keys deleted and obsolete and the last 10th of the keys +# removed with fast truncate but not yet obsolete. +# +# It checks that: +# +# - Compaction correctly rewrites pages in WT_REF_DELETED state but are still on disk. +class test_compact12(wttest.WiredTigerTestCase): + create_params = 'key_format=i,value_format=S,allocation_size=4KB,leaf_page_max=32KB,leaf_value_max=16MB' + conn_config = 'cache_size=100MB,statistics=(all),verbose=[compact:4]' + uri_prefix = 'table:test_compact12' + + # This test uses 10 times as many keys as the implementation on the 8.0 branch. + # This is because `free_space_target` is not available on the 7.0 and earlier so we need to make ensure + # enough free space is produced in the data files to trigger compaction. + table_numkv = 100 * 1000 + value_size = kilobyte # The value should be small enough so that we don't create overflow pages. + + def get_size(self, uri): + stat_cursor = self.session.open_cursor('statistics:' + uri, None, 'statistics=(all)') + size = stat_cursor[stat.dsrc.block_size][2] + stat_cursor.close() + return size + + def get_fast_truncated_pages(self): + stat_cursor = self.session.open_cursor('statistics:', None, None) + pages = stat_cursor[stat.conn.rec_page_delete_fast][2] + stat_cursor.close() + return pages + + def truncate(self, uri, key1, key2): + lo_cursor = self.session.open_cursor(uri) + hi_cursor = self.session.open_cursor(uri) + lo_cursor.set_key(key1) + hi_cursor.set_key(key2) + self.session.truncate(None, lo_cursor, hi_cursor, None) + lo_cursor.close() + hi_cursor.close() + + def populate(self, uri, num_keys): + c = self.session.open_cursor(uri, None) + for k in range(num_keys // 10 * 9): + self.session.begin_transaction() + c[k] = ('%07d' % k) + '_' + 'a' * self.value_size + self.session.commit_transaction(f'commit_timestamp={self.timestamp_str(2)}') + + for k in range(num_keys // 10 * 9, num_keys): + self.session.begin_transaction() + c[k] = ('%07d' % k) + '_' + 'b' * self.value_size + self.session.commit_transaction(f'commit_timestamp={self.timestamp_str(2)}') + c.close() + + def test_compact12_truncate(self): + if 'tiered' in self.hook_names: + self.skipTest("Tiered tables do not support compaction") + self.conn.set_timestamp(f'oldest_timestamp={self.timestamp_str(1)}') + + # Create and populate a table. + uri = self.uri_prefix + self.session.create(uri, self.create_params) + + self.populate(uri, self.table_numkv) + + # Write to disk. + self.session.checkpoint() + + # Remove 1/4 of the keys at timestamp 3. + c = self.session.open_cursor(uri, None) + for i in range(self.table_numkv // 4): + self.session.begin_transaction() + c.set_key(i) + c.remove() + self.session.commit_transaction(f'commit_timestamp={self.timestamp_str(3)}') + c.close() + + # Reopen connection to ensure everything is on disk. + self.reopen_conn() + + # Move the oldest timestamp forward to make the previously removed keys obsolete. + self.conn.set_timestamp(f'oldest_timestamp={self.timestamp_str(4)}') + + # Fast truncate the last tenth of a file at timestamp 5. + self.session.begin_transaction() + self.truncate(uri, self.table_numkv // 10 * 9, self.table_numkv) + self.session.commit_transaction(f'commit_timestamp={self.timestamp_str(5)}') + + self.session.checkpoint() + + self.assertGreater(self.get_fast_truncated_pages(), 0) + + # Check the size of the table. + size_before_compact = self.get_size(uri) + + self.session.compact(uri) + + # Ensure compact has moved the fast truncated pages at the end of the file. + # We should have recovered at least 1/4 of the file. + size_after_compact = self.get_size(uri) + space_recovered = size_before_compact - size_after_compact + self.assertGreater(space_recovered, size_before_compact // 4) + + # Ignore compact verbose messages used for debugging. + self.ignoreStdoutPatternIfExists('WT_VERB_COMPACT') + self.ignoreStderrPatternIfExists('WT_VERB_COMPACT') + +if __name__ == '__main__': + wttest.run()