From 4d5d55490a8300f5e15dbd3d64be76690fb5c7f5 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Mon, 22 Apr 2024 16:55:31 -0400 Subject: [PATCH] Add infra support and helper function to create real datasource (#1146) * Add infra support and helper function to create real datasource Signed-off-by: Derek Ho * Remove logs from previous change Signed-off-by: Derek Ho * Add test that uses the helper function to verify functionality Signed-off-by: Derek Ho * Change to request and add it to appropriate workflows Signed-off-by: Derek Ho * Add basic auth remote cluster Signed-off-by: Derek Ho * Call function Signed-off-by: Derek Ho * Move action Signed-off-by: Derek Ho * Update action and usage in docs Signed-off-by: Derek Ho * Checkout before access Signed-off-by: Derek Ho * Add relative path Signed-off-by: Derek Ho * Fix spelling Signed-off-by: Derek Ho * Remove -d Signed-off-by: Derek Ho * Move port logic Signed-off-by: Derek Ho * Cat log Signed-off-by: Derek Ho * Allow modification Signed-off-by: Derek Ho * Duplicate download Signed-off-by: Derek Ho * Change directory Signed-off-by: Derek Ho * Change name Signed-off-by: Derek Ho * Remove extra chmod Signed-off-by: Derek Ho * Within test directory Signed-off-by: Derek Ho * Fix Signed-off-by: Derek Ho * Fix for windows, add helper function, show usage Signed-off-by: Derek Ho * Remove pwd Signed-off-by: Derek Ho * Move to temp dire Signed-off-by: Derek Ho * Address PR feedback and try to spin up an instance with security Signed-off-by: Derek Ho * Small action change and documentation add Signed-off-by: Derek Ho --------- Signed-off-by: Derek Ho (cherry picked from commit df38e30f871a450ec25fc69fcf463015d89457ba) --- .github/actions/start-opensearch/action.yml | 195 ++++++++++++++++++ .../release-e2e-workflow-template-windows.yml | 13 ++ .../release-e2e-workflow-template.yml | 13 ++ .github/workflows/release-signoff-chrome.yml | 11 + DEVELOPER_GUIDE.md | 38 ++++ cypress.json | 4 + .../1_create_datasource.spec.js | 6 + .../commands.js | 50 +++++ 8 files changed, 330 insertions(+) create mode 100644 .github/actions/start-opensearch/action.yml diff --git a/.github/actions/start-opensearch/action.yml b/.github/actions/start-opensearch/action.yml new file mode 100644 index 000000000..2425cc2b9 --- /dev/null +++ b/.github/actions/start-opensearch/action.yml @@ -0,0 +1,195 @@ +name: 'Launch OpenSearch' +description: 'Downloads OpenSearch and runs it' + +inputs: + opensearch-version: + description: 'The version of OpenSearch that should be used, e.g "3.0.0"' + required: true + + plugins: + description: 'A comma separated list of plugins to install. Leave empty to not install any. Each entry should be a full path prefixed with `file: `, for example: `file:$(pwd)/my-plugin.zip`' + required: false + + security-enabled: + description: 'Whether security is enabled' + required: true + + admin-password: + description: 'The admin password uses for the cluster' + required: false + + security_config_file: + description: 'Path to a security config file to replace the default. Leave empty if security is not enabled or using the default config' + required: false + + port: + description: 'Port to run OpenSearch. Leave empty to use the default config (9200)' + required: false + default: '9200' + +runs: + using: "composite" + steps: + + # Configure longpath names if on Windows + - name: Enable Longpaths if on Windows + if: ${{ runner.os == 'Windows' }} + run: git config --system core.longpaths true + shell: pwsh + + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 11 + + - name: Create temporary directory + run: | + mkdir ${{ inputs.port }} + shell: bash + + # Download OpenSearch + - name: Download OpenSearch for Windows + uses: peternied/download-file@v2 + if: ${{ runner.os == 'Windows' }} + with: + url: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${{ inputs.opensearch-version }}/latest/windows/x64/zip/dist/opensearch/opensearch-${{ inputs.opensearch-version }}-windows-x64.zip + + + - name: Download OpenSearch for Linux + uses: peternied/download-file@v2 + if: ${{ runner.os == 'Linux' }} + with: + url: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${{ inputs.opensearch-version }}/latest/linux/x64/tar/dist/opensearch/opensearch-${{ inputs.opensearch-version }}-linux-x64.tar.gz + + # Extract downloaded zip + - name: Extract downloaded tar + if: ${{ runner.os == 'Linux' }} + run: | + tar -xzf opensearch-*.tar.gz -C ${{ inputs.port }} && mv ${{ inputs.port }}/opensearch-${{ inputs.opensearch-version }} ${{ inputs.port }}/opensearch-${{inputs.opensearch-version}}-SNAPSHOT${{ inputs.port }} + rm -f opensearch-*.tar.gz + shell: bash + + - name: Extract downloaded zip + if: ${{ runner.os == 'Windows' }} + run: | + Expand-Archive -Path "opensearch-${{ inputs.opensearch-version }}-windows-x64.zip" -DestinationPath "temp" + Move-Item -Path "temp/*" -Destination "${{ inputs.port }}/opensearch-${{ inputs.opensearch-version }}-SNAPSHOT${{ inputs.port }}" + Remove-Item -Path "temp" -Recurse + Remove-Item -Path "opensearch-${{ inputs.opensearch-version }}-windows-x64.zip.1" + shell: pwsh + + # Setup security if it's enabled + - name: Setup security demo configuration + if: ${{ runner.os == 'Linux' && inputs.security-enabled == 'true' }} + run: | + echo "running linux security demo configuration setup" + export OPENSEARCH_INITIAL_ADMIN_PASSWORD=${{ inputs.admin-password }} + chmod +x ./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT${{ inputs.port }}/plugins/opensearch-security/tools/install_demo_configuration.sh + opensearch_version="${{ inputs.opensearch-version }}" + opensearch_major_version=$(echo "$opensearch_version" | awk -F'.' '{print $1}') + opensearch_minor_version=$(echo "$opensearch_version" | awk -F'.' '{print $2}') + if [ "$opensearch_major_version" -lt 2 ] || ([ "$opensearch_major_version" -eq 2 ] && [ "$opensearch_minor_version" -lt 12 ]); then + echo "Running the command without -t option (OpenSearch version is $opensearch_version)" + /bin/bash -c "yes | ./opensearch-${opensearch_version}-SNAPSHOT${{ inputs.port }}/plugins/opensearch-security/tools/install_demo_configuration.sh" + else + echo "Running the command with -t option (OpenSearch version is $opensearch_version)" + /bin/bash -c "yes | ./opensearch-${opensearch_version}-SNAPSHOT${{ inputs.port }}/plugins/opensearch-security/tools/install_demo_configuration.sh -t" + fi + echo "plugins.security.unsupported.restapi.allow_securityconfig_modification: true" >> ./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT${{ inputs.port }}/config/opensearch.yml + shell: bash + working-directory: ${{ inputs.port }} + + - name: Setup security demo configuration for Windows + if: ${{ runner.os == 'Windows' && inputs.security-enabled == 'true' }} + run: | + echo "running windows security demo configuration setup" + export OPENSEARCH_INITIAL_ADMIN_PASSWORD=${{ inputs.admin-password }} + chmod +x ./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT${{ inputs.port }}/plugins/opensearch-security/tools/install_demo_configuration.bat + opensearch_version="${{ inputs.opensearch-version }}" + opensearch_major_version=$(echo "$opensearch_version" | cut -d'.' -f1) + opensearch_minor_version=$(echo "$opensearch_version" | cut -d'.' -f2) + if [ "$opensearch_major_version" -lt 2 ] || ([ "$opensearch_major_version" -eq 2 ] && [ "$opensearch_minor_version" -lt 12 ]); then + echo "Running the command without -t option (OpenSearch version is $opensearch_version)" + /bin/bash -c "yes | ./opensearch-${opensearch_version}-SNAPSHOT${{ inputs.port }}/plugins/opensearch-security/tools/install_demo_configuration.bat -y -i -s" + else + echo "Running the command with -t option (OpenSearch version is $opensearch_version)" + /bin/bash -c "yes | ./opensearch-${opensearch_version}-SNAPSHOT${{ inputs.port }}/plugins/opensearch-security/tools/install_demo_configuration.bat -t -y -i -s" + fi + echo "plugins.security.unsupported.restapi.allow_securityconfig_modification: true" >> ./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT${{ inputs.port }}/config/opensearch.yml + shell: bash + working-directory: ${{ inputs.port }} + + # Disable security if it's disabled + - name: Disable security + if: ${{ inputs.security-enabled == 'false' }} + run: | + echo "Remove OpenSearch Security" + echo "plugins.security.disabled: true" >> ./opensearch-${{inputs.opensearch-version}}-SNAPSHOT${{ inputs.port }}/config/opensearch.yml + shell: bash + working-directory: ${{ inputs.port }} + + - name: Use more space + run: | + echo '' >> ./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT${{ inputs.port }}/config/opensearch.yml + echo "cluster.routing.allocation.disk.threshold_enabled: false" >> ./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT${{ inputs.port }}/config/opensearch.yml + echo "http.port: ${{ inputs.port }}" >> ./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT${{ inputs.port }}/config/opensearch.yml + shell: bash + working-directory: ${{ inputs.port }} + + # Run OpenSearch + - name: Run OpenSearch with plugin on Linux + if: ${{ runner.os == 'Linux'}} + run: /bin/bash -c "./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT${{ inputs.port }}/bin/opensearch &" + shell: bash + working-directory: ${{ inputs.port }} + + - name: Run OpenSearch with plugin on Windows + if: ${{ runner.os == 'Windows'}} + run: start .\opensearch-${{ inputs.opensearch-version }}-SNAPSHOT${{ inputs.port }}\bin\opensearch.bat + shell: pwsh + working-directory: ${{ inputs.port }} + + # Give the OpenSearch process some time to boot up before sending any requires, might need to increase the default time! + - name: Sleep while OpenSearch starts + uses: peternied/action-sleep@v1 + with: + seconds: 30 + + # Verify that the server is operational + - name: Check OpenSearch Running on Linux + if: ${{ runner.os != 'Windows'}} + run: | + if [ "${{ inputs.security-enabled }}" == "true" ]; then + curl https://localhost:${{ inputs.port || 9200 }}/_cat/plugins -u 'admin:${{ inputs.admin-password }}' -k -v --fail-with-body + else + curl http://localhost:${{ inputs.port || 9200 }}/_cat/plugins -v + fi + shell: bash + working-directory: ${{ inputs.port }} + + - name: Check OpenSearch Running on Windows + if: ${{ runner.os == 'Windows'}} + run: | + if ("${{ inputs.security-enabled }}" -eq "true") { + $credentialBytes = [Text.Encoding]::ASCII.GetBytes("admin:${{ inputs.admin-password }}") + $encodedCredentials = [Convert]::ToBase64String($credentialBytes) + $baseCredentials = "Basic $encodedCredentials" + $Headers = @{ Authorization = $baseCredentials } + $url = 'https://localhost:${{ inputs.port || 9200 }}/_cat/plugins' + } else { + $Headers = @{ } + $url = 'http://localhost:${{ inputs.port || 9200 }}/_cat/plugins' + } + Invoke-WebRequest -SkipCertificateCheck -Uri $url -Headers $Headers; + shell: pwsh + working-directory: ${{ inputs.port }} + + - if: always() + run: cat ./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT${{ inputs.port }}/logs/opensearch.log + shell: bash + working-directory: ${{ inputs.port }} + + - if: always() + run: cat ./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT${{ inputs.port }}/config/opensearch.yml + shell: bash + working-directory: ${{ inputs.port }} \ No newline at end of file diff --git a/.github/workflows/release-e2e-workflow-template-windows.yml b/.github/workflows/release-e2e-workflow-template-windows.yml index f5b9a8873..fd9bc1dc4 100644 --- a/.github/workflows/release-e2e-workflow-template-windows.yml +++ b/.github/workflows/release-e2e-workflow-template-windows.yml @@ -25,6 +25,8 @@ jobs: # 2.12 onwards security demo configuration require a custom admin password OPENSEARCH_INITIAL_ADMIN_PASSWORD: 'myStrongPassword123!' steps: + - name: Checkout Branch + uses: actions/checkout@v3 - name: Set up JDK uses: actions/setup-java@v1 with: @@ -69,6 +71,17 @@ jobs: fi netstat -anP tcp | grep LISTEN | grep 9200 || netstat -ntlp | grep 9200 shell: bash + - uses: ./.github/actions/start-opensearch + with: + opensearch-version: ${{ env.VERSION }} + security-enabled: false + port: 9201 + - uses: ./.github/actions/start-opensearch + with: + opensearch-version: ${{ env.VERSION }} + security-enabled: true + admin-password: admin + port: 9202 - name: Get and run OpenSearch-Dashboards run: | curl -SLO https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/${{ env.VERSION }}/latest/${{ env.PLATFORM }}/x64/zip/dist/opensearch-dashboards/opensearch-dashboards-${{ env.VERSION }}-${{ env.PLATFORM }}-x64.zip diff --git a/.github/workflows/release-e2e-workflow-template.yml b/.github/workflows/release-e2e-workflow-template.yml index bc0788427..2f2b62926 100644 --- a/.github/workflows/release-e2e-workflow-template.yml +++ b/.github/workflows/release-e2e-workflow-template.yml @@ -28,6 +28,8 @@ jobs: # 2.12 onwards security demo configuration require a custom admin password OPENSEARCH_INITIAL_ADMIN_PASSWORD: 'myStrongPassword123!' steps: + - name: Checkout Branch + uses: actions/checkout@v3 - name: Set up JDK uses: actions/setup-java@v1 with: @@ -66,6 +68,17 @@ jobs: timeout 900 bash -c 'while [[ "$(curl -o /dev/null -w ''%{http_code}'' -u admin:${{ env.OPENSEARCH_INITIAL_ADMIN_PASSWORD }} -k https://localhost:9200)" != "200" ]]; do sleep 5; done' curl https://localhost:9200 -u admin:${{ env.OPENSEARCH_INITIAL_ADMIN_PASSWORD }} --insecure fi + - uses: ./.github/actions/start-opensearch + with: + opensearch-version: ${{ env.VERSION }} + security-enabled: false + port: 9201 + - uses: ./.github/actions/start-opensearch + with: + opensearch-version: ${{ env.VERSION }} + security-enabled: true + admin-password: admin + port: 9202 - name: Get OpenSearch-Dashboards run: | wget https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/${{ env.VERSION }}/latest/linux/x64/tar/dist/opensearch-dashboards/opensearch-dashboards-${{ env.VERSION }}-linux-x64.tar.gz diff --git a/.github/workflows/release-signoff-chrome.yml b/.github/workflows/release-signoff-chrome.yml index e6bdb4775..2af455edf 100644 --- a/.github/workflows/release-signoff-chrome.yml +++ b/.github/workflows/release-signoff-chrome.yml @@ -34,6 +34,17 @@ jobs: cd opensearch-${{ env.VERSION }}/ ./opensearch-tar-install.sh & timeout 900 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' -u admin:${{ env.OPENSEARCH_INITIAL_ADMIN_PASSWORD }} -k https://localhost:9200)" != "200" ]]; do sleep 5; done' + - uses: ./.github/actions/start-opensearch + with: + opensearch-version: ${{ env.VERSION }} + security-enabled: false + port: 9201 + - uses: ./.github/actions/start-opensearch + with: + opensearch-version: ${{ env.VERSION }} + security-enabled: true + admin-password: admin + port: 9202 - name: Get OpenSearch-Dashboards run: | wget https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/${{ env.VERSION }}/latest/linux/x64/tar/dist/opensearch-dashboards/opensearch-dashboards-${{ env.VERSION }}-linux-x64.tar.gz diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 61b451ef7..ba36f3755 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -125,6 +125,44 @@ Tests for plugins that are not a part of the [OpenSearch Dashboards](https://git / ``` +### Tests for Multiple Datasources + +Tests surrounding the multiple datasources feature can use the start-opensearch action that lives in this repo. Note that to test these related features, the following OSD config needs to be set: `osd-serve-args: --data_source.enabled=true`. Additionally, if testing with a remote datasource with basic auth enabled using this repo, an additional OSD config needs to be set: `osd-serve-args: --data_source.ssl.verificationMode: none`, so that the self-signed demo certificates can be used. + +Example usage: +``` +- uses: ./.github/actions/start-opensearch + with: + opensearch-version: 3.0.0 + security-enabled: false + port: 9201 +``` + + +This will spin up an OpenSearch backend with version 3.0.0 on port 9201 within the same github runner. This OpenSearch can then be added as an datasource. + +``` +- uses: ./.github/actions/start-opensearch + with: + opensearch-version: 3.0.0 + security-enabled: true + admin-password: admin + port: 9202 +``` +This will spin up an OpenSearch backend with version 3.0.0 on port 9202 with basic auth and admin credentials of "admin:admin" within the same github runner. This OpenSearch can then be added as an datasource. + +To test UI/API compatibility with different versions you may want to spin up a matrix of OpenSearch backends with different versions than the local cluster. The earliest windows distribution supported is 2.4. + + +The DataSourceManagement Plugin exposes a helper function to create a data source on this port: +``` +const [noAuthId, noAuthLabel] = cy.createDataSourceNoAuth(); + +const [basicAuthId, basicAuthLabel] = cy.createDataSourceBasicAuth(); + +# Add tests that make calls using noAuthId and basicAuthId, or test that remote datasource via the UI using the labels noAuthLabel and basicAuthLabel. +``` + ### Experimental Features When writing tests for experimental features, please follow these steps. diff --git a/cypress.json b/cypress.json index 2b6abb7db..b4cce932a 100644 --- a/cypress.json +++ b/cypress.json @@ -11,6 +11,10 @@ "viewportHeight": 1320, "env": { "openSearchUrl": "http://localhost:9200", + "remoteDataSourceNoAuthUrl": "http://localhost:9201", + "remoteDataSourceBasicAuthUrl": "https://localhost:9202", + "remoteDataSourceBasicAuthUsername": "admin", + "remoteDataSourceBasicAuthPassword": "admin", "SECURITY_ENABLED": false, "AGGREGATION_VIEW": false, "username": "admin", diff --git a/cypress/integration/core-opensearch-dashboards/opensearch-dashboards/datasource-management-plugin/1_create_datasource.spec.js b/cypress/integration/core-opensearch-dashboards/opensearch-dashboards/datasource-management-plugin/1_create_datasource.spec.js index ac5e50c03..34982fd0e 100644 --- a/cypress/integration/core-opensearch-dashboards/opensearch-dashboards/datasource-management-plugin/1_create_datasource.spec.js +++ b/cypress/integration/core-opensearch-dashboards/opensearch-dashboards/datasource-management-plugin/1_create_datasource.spec.js @@ -23,6 +23,8 @@ const miscUtils = new MiscUtils(cy); const username = Cypress.env('username'); const password = Cypress.env('password'); +// TODO: create datasource with basic auth and sigv4 + if (Cypress.env('DATASOURCE_MANAGEMENT_ENABLED')) { describe('Create datasources', () => { before(() => { @@ -385,6 +387,10 @@ if (Cypress.env('DATASOURCE_MANAGEMENT_ENABLED')) { 'app/management/opensearch-dashboards/dataSources' ); }); + + it('creates a datasources to a real opensearch instance', () => { + cy.createDataSourceNoAuth(); + }); }); }); } diff --git a/cypress/utils/dashboards/datasource-management-dashboards-plugin/commands.js b/cypress/utils/dashboards/datasource-management-dashboards-plugin/commands.js index 3d34fc494..da4915e70 100644 --- a/cypress/utils/dashboards/datasource-management-dashboards-plugin/commands.js +++ b/cypress/utils/dashboards/datasource-management-dashboards-plugin/commands.js @@ -29,6 +29,56 @@ Cypress.Commands.add('deleteAllDataSources', () => { ); }); +Cypress.Commands.add('createDataSourceNoAuth', () => { + cy.request({ + method: 'POST', + url: `${Cypress.config('baseUrl')}/api/saved_objects/data-source`, + headers: { + 'osd-xsrf': true, + }, + body: { + attributes: { + title: 'RemoteDataSourceNoAuth', + endpoint: Cypress.env('remoteDataSourceNoAuthUrl'), + auth: { + type: 'no_auth', + }, + }, + }, + }).then((resp) => { + if (resp && resp.body && resp.body.id) { + return [resp.body.id, 'RemoteDataSourceNoAuth']; + } + }); +}); + +Cypress.Commands.add('createDataSourceBasicAuth', () => { + cy.request({ + method: 'POST', + url: `${Cypress.config('baseUrl')}/api/saved_objects/data-source`, + headers: { + 'osd-xsrf': true, + }, + body: { + attributes: { + title: 'RemoteDataSourceBasicAuth', + endpoint: Cypress.env('remoteDataSourceBasicAuthUrl'), + auth: { + type: 'username_password', + credentials: { + username: Cypress.env('remoteDataSourceBasicAuthUsername'), + password: Cypress.env('remoteDataSourceBasicAuthPassword'), + }, + }, + }, + }, + }).then((resp) => { + if (resp && resp.body && resp.body.id) { + return [resp.body.id, 'RemoteDataSourceBasicAuth']; + } + }); +}); + Cypress.Commands.add('createDataSource', (dataSourceJSON) => { cy.request({ method: 'POST',