indexing-querying.json

{
  "name": "indexing-querying",
  "description": "Indexes half the data for NYC Taxis, then concurrently queries OpenSearch while indexing again.",
  "default": false,
  "schedule": [
    {
      "operation": {
        "operation-type": "put-settings",
        "body": {
          "transient": {
            "search.default_search_timeout": "{{default_search_timeout | default(-1)}}"
          }
        }
      }
    },
    {
      "operation": "delete-index"
    },
    {
      "operation": {
        "operation-type": "create-index",
        "settings": {{index_settings | default({}) | tojson}}
      }
    },
    {
      "name": "check-cluster-health",
      "operation": {
        "operation-type": "cluster-health",
        "index": "pmc",
        "request-params": {
          "wait_for_status": "{{cluster_health | default('green')}}",
          "wait_for_no_relocating_shards": "true"
        },
        "retry-until-success": true
      }
    },
    {
      "name": "initial-index-append",
      "operation": {
        "operation-type": "bulk",
        "bulk-size": {{bulk_size | default(500)}},
        "ingest-percentage": 50
      },
      "warmup-time-period": 30,
      "clients": {{bulk_indexing_clients | default(8)}},
      "ignore-response-error-level": "{{error_level | default('non-fatal')}}"
    },
    {
      "name": "refresh-after-index",
      "operation": "refresh"
    },
    {
      "operation": {
        "operation-type": "force-merge",
{%- if force_merge_max_num_segments is defined %}
        "max-num-segments": {{ force_merge_max_num_segments | tojson }},
{%- endif %}
        "request-timeout": 7200
      }
    },
    {
      "name": "refresh-after-force-merge",
      "operation": "refresh"
    },
    {
      "name": "wait-until-merges-finish",
      "operation": {
        "operation-type": "index-stats",
        "index": "_all",
        "condition": {
          "path": "_all.total.merges.current",
          "expected-value": 0
        },
        "retry-until-success": true,
        "include-in-reporting": false
      }
    },
    {
      "parallel": {
        "tasks": [
          {
            "name": "concurrent-index-append",
            "operation": {
              "operation-type": "bulk",
              "bulk-size": {{bulk_size | default(500)}},
              "ingest-percentage": 50
            },
            "clients": {{bulk_indexing_clients | default(8)}},
            "ignore-response-error-level": "{{error_level | default('non-fatal')}}"
          },
          {
            "operation": "default",
            "warmup-iterations": 500,
            "iterations": 200,
            "target-throughput": {{ default_target_throughput or target_throughput | default(20) | tojson }},
            "clients": {{ default_search_clients or search_clients | default(1) }}
          },
          {
            "operation": "term",
            "warmup-iterations": 500,
            "iterations": 200,
            "target-throughput": {{ term_target_throughput or target_throughput | default(20) | tojson }},
            "clients": {{ term_search_clients or search_clients | default(1) }}
          },
          {
            "operation": "phrase",
            "warmup-iterations": 500,
            "iterations": 200,
            "target-throughput": {{ phrase_target_throughput or target_throughput | default(20) | tojson }},
            "clients": {{ phrase_search_clients or search_clients | default(1) }}
          },
          {
            "operation": "articles_monthly_agg_uncached",
            "warmup-iterations": 500,
            "iterations": 200,
            "target-throughput": {{ articles_monthly_agg_uncached_target_throughput or target_throughput | default(20) | tojson }},
            "clients": {{ articles_monthly_agg_uncached_search_clients or search_clients | default(1) }}
          },
          {
            "operation": "articles_monthly_agg_cached",
            "warmup-iterations": 500,
            "iterations": 200,
            "target-throughput": {{ articles_monthly_agg_cached_target_throughput or target_throughput | default(20) | tojson }},
            "clients": {{ articles_monthly_agg_cached_search_clients or search_clients | default(1) }}
          },
          {
            "operation": "scroll",
            "warmup-iterations": 50,
            "iterations": 100,
            "target-throughput": {{ scroll_target_throughput or target_throughput | default(0.5) | tojson }},
            "clients": {{ scroll_search_clients or search_clients | default(1) }}
          }
        ]
      }
    }
  ]
}