Add Delfin e2e test automation with robot framework (#530)

.github/workflows/delfin_e2e_test.yml

@@ -0,0 +1,71 @@
+name: Delfin E2E Test
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  test:
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      max-parallel: 6
+      matrix:
+        platform:
+          - ubuntu-16.04
+          - ubuntu-18.04
+        python-version: [ 3.6, 3.7, 3.8 ]
+
+    steps:
+      - name: Checkout delfin code
+        uses: actions/checkout@v2
+      - name: Install Python version ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install -r test-requirements.txt
+          pip install tox codecov
+      - name: E2E Testing - Add Test Driver to Delfin
+        run: |
+          str="\ \ \ \ \ \ \ \ \ \ \ \ 'test_vendor test_model = delfin.tests.e2e.testdriver:TestDriver',"
+          sed -i "/FakeStorageDriver',/ a $str" ./setup.py
+        shell: bash
+      - name: E2E Testing - Install RabbitMQ
+        uses: getong/rabbitmq-action@v1.2
+        with:
+          rabbitmq version: '3.8.2-management-alpine'
+          host port: 5672
+          rabbitmq user: 'guest'
+          rabbitmq password: 'guest'
+      - name: E2E Testing - Install Redis
+        uses: supercharge/redis-github-action@1.2.0
+        with:
+          redis-version: 6
+      - name: E2E Testing - Build and Deploy Delfin with Test driver
+        run: |
+          sudo mkdir -p /var/lib/delfin
+          sudo chmod 0777 /var/lib/delfin
+          sudo mkdir -p /etc/delfin
+          sudo chmod 0777 /etc/delfin
+          python3 setup.py install
+          cp ./etc/delfin/api-paste.ini /etc/delfin/
+          python3 ./script/create_db.py --config-file ./etc/delfin/delfin.conf
+          sleep 1
+          python3 ./delfin/cmd/task.py --config-file ./etc/delfin/delfin.conf > /tmp/task.log 2>&1 &
+          python3 ./delfin/cmd/alert.py --config-file ./etc/delfin/delfin.conf > /tmp/alert.log 2>&1 &
+          python3 ./delfin/cmd/api.py --config-file ./etc/delfin/delfin.conf > /tmp/api.log 2>&1 &
+        shell: bash
+      - name: E2E Testing - Run RobotFramework
+        run: |
+          sleep 3
+          pip install robotframework
+          pip install robotframework-requests
+          pip install robotframework-jsonlibrary
+          DELFIN_DIR=`pwd`
+          TOP_DIR="${DELFIN_DIR}/delfin/tests/e2e"
+          ORIG_PATH='"storage.json"'
+          FILE_PATH="${TOP_DIR}/testdriver/storage.json"
+          sed -i "s|${ORIG_PATH}|\"${FILE_PATH}\"|g" $TOP_DIR/test.json
+          sleep 1
+          robot delfin/tests/e2e
+        shell: bash

delfin/tests/e2e/GetResources.robot

@@ -0,0 +1,123 @@
+*** Settings ***
+Documentation    Tests to verify that GET of resources
+
+Library                 RequestsLibrary
+Library                 Collections
+Library                 JSONLibrary
+Library                 OperatingSystem
+
+Suite Setup             Open Application
+Suite Teardown          Close Application
+
+*** Variables ***
+${delfin_url}           http://localhost:8190/v1
+${storage_pools}        storage-pools
+@{res_urls}             storage-pools  volumes  controllers  disks  ports  quotas  qtrees  filesystems  shares
+@{res_indx}             storage_pools  volumes  controllers  disks  ports  quotas  qtrees  filesystems  shares
+
+*** Test Cases ***
+GET Resources when test storage is registered
+    [Tags]    DELFIN
+
+    FOR     ${res_url}   ${res_ind}   IN ZIP    ${res_urls}   ${res_indx}
+            ${ret_json}=            Get All Resource Of     ${res_url}
+            ${res_s}=               Get Value From Json	    ${ret_json}         $..${res_ind}
+            Should Not Be Empty     ${res_s[0]}
+    END
+
+GET Resources with ID
+    [Tags]    DELFIN
+
+    FOR     ${res_url}   ${res_ind}   IN ZIP    ${res_urls}   ${res_indx}
+            ${ret_json}=            Get All Resource Of     ${res_url}
+            ${res_s}=               Get Value From Json	    ${ret_json}         $..${res_ind}
+            Should Not Be Empty     ${res_s[0]}
+            ${resource_ids}         Get Value From Json	    ${ret_json}         $..id
+            ${ret_json}=            Get All Resource with ID    ${res_url}     ${resource_ids[0]}
+            Should Not Be Empty     ${res_s[0]}
+    END
+
+GET Resources with Filter
+    [Tags]    DELFIN
+
+    log to console          \n
+    ${storages}=            Get All Storages
+    ${storages_id}=         Get Value From Json	    ${storages[0]}         $..id
+
+    FOR     ${res_url}   ${res_ind}   IN ZIP    ${res_urls}   ${res_indx}
+            ${ret_json}=            Get All Resource with Filter     ${res_url}   storage_id=${storages_id[0]}
+            ${res_s}=               Get Value From Json	    ${ret_json}         $..${res_ind}
+            Should Not Be Empty     ${res_s[0]}
+            ${ret_json}=            Get All Resource with Filter     ${res_url}   storage_id=123
+            ${res_s}=               Get Value From Json	    ${ret_json}         $..${res_ind}
+            Should Be Empty     ${res_s[0]}
+
+    END
+
+GET Resources when no storages are registered
+    [Tags]    DELFIN
+    Close Application
+    FOR     ${res_url}   ${res_ind}   IN ZIP    ${res_urls}   ${res_indx}
+            ${ret_json}=            Get All Resource Of     ${res_url}
+            ${res_s}=               Get Value From Json	    ${ret_json}         $..${res_ind}
+            Should Be Empty         ${res_s[0]}
+    END
+    Open Application
+
+*** Keywords ***
+Get All Resource Of
+    [Arguments]             ${resource}
+    Create Session          delfin          ${delfin_url}
+    ${resp_get}=            GET On Session  delfin    ${resource}
+    Status Should Be        200    ${resp_get}
+    [Return]                ${resp_get.json()}
+
+Get All Resource with ID
+    [Arguments]             ${resource}     ${resource_id}
+    Create Session          delfin          ${delfin_url}
+    ${resp_get}=            GET On Session  delfin    ${resource}/${resource_id}
+    Status Should Be        200    ${resp_get}
+    [Return]                ${resp_get.json()}
+
+Get All Resource with Filter
+    [Arguments]             ${resource}     ${filter}
+    Create Session          delfin          ${delfin_url}
+    ${resp_get}=            GET On Session  delfin    ${resource}?${filter}
+    Status Should Be        200    ${resp_get}
+    [Return]                ${resp_get.json()}
+
+Delete Storage With ID
+    [Arguments]             ${storage_id}
+    Create Session          delfin      ${delfin_url}
+    ${resp_del}=            DELETE On Session    delfin     storages/${storage_id}
+    Status Should Be        202    ${resp_del}
+
+Register Test Storage
+    ${test}=                 Load Json From File   ${CURDIR}/test.json
+    ${access_info}=          Get Value From Json   ${test}   $.test_register_access_info
+
+    Create Session          delfin      ${delfin_url}
+    ${resp_register}=       POST On Session     delfin     storages    json=${access_info[0]}
+    Status Should Be                            201    ${resp_register}
+    Dictionary Should Contain Key               ${resp_register.json()}     id
+    ${storage_id}=          Get Value From Json	     ${resp_register.json()} 	 $..id
+    [Return]                ${storage_id[0]}
+
+Get All Storages
+    Create Session          delfin      ${delfin_url}
+    ${resp_get}=            GET On Session    delfin    storages
+    Status Should Be        200    ${resp_get}
+    ${resp_get_storage}=    Get Value From Json	        ${resp_get.json()}      $..storages
+    [Return]                ${resp_get_storage[0]}
+
+Open Application
+    ${array_id}=            Register Test Storage
+    Sleep       1s
+
+Close Application
+    @{storages}=            Get All Storages
+    FOR     ${storage}      IN                      @{storages}
+            ${storage_id}=  Get Value From Json	    ${storage} 	        $..id
+            Delete Storage With ID                  ${storage_id[0]}
+    END
+    Sleep       1s

delfin/tests/e2e/GetStorage.robot

@@ -0,0 +1,86 @@
+*** Settings ***
+Documentation    Tests to verify that GET of storages
+
+Library                 RequestsLibrary
+Library                 Collections
+Library                 JSONLibrary
+Library                 OperatingSystem
+
+*** Variables ***
+${delfin_url}           http://localhost:8190/v1
+
+*** Test Cases ***
+GET all Storages when no storages are registered
+    [Tags]    DELFIN
+
+    ${storages}=            Get All Storages
+    Should Be Empty         ${storages}
+
+GET all Storages when two storages are registered
+    [Tags]    DELFIN
+
+    ${storage_id_test}=          Register Test Storage
+    ${storage_id_fake}=          Register Fake Storage
+
+    # GET all storages
+    ${storages}=            Get All Storages
+    ${id_list}=    create list       ${storages[0]['id']}      ${storages[1]['id']}
+    List should contain value  ${id_list}   ${storage_id_test}
+    List should contain value  ${id_list}   ${storage_id_fake}
+
+    Delete Storage With ID      ${storage_id_test}
+    Delete Storage With ID      ${storage_id_fake}
+
+GET Storage with a valid Storage ID
+    [Tags]    DELFIN
+    ${storage_id_test}=          Register Test Storage
+
+    # GET all storages
+    ${storage}=            Get Storage With ID     ${storage_id_test}
+    ${id_list}=    create list       ${storage['id']}
+    List should contain value  ${id_list}   ${storage_id_test}
+
+    Delete Storage With ID      ${storage_id_test}
+
+*** Keywords ***
+Get All Storages
+    Create Session          delfin      ${delfin_url}
+    ${resp_get}=            GET On Session    delfin    storages
+    Status Should Be        200         ${resp_get}
+    ${resp_get_storage}=    Get Value From Json	        ${resp_get.json()}      $..storages
+    [Return]                ${resp_get_storage[0]}
+
+Get Storage With ID
+    [Arguments]             ${storage_id}
+    Create Session          delfin      ${delfin_url}
+    ${resp_get}=            GET On Session    delfin    storages/${storage_id}
+    Status Should Be        200    ${resp_get}
+    [Return]                ${resp_get.json()}
+
+Delete Storage With ID
+    [Arguments]             ${storage_id}
+    Create Session          delfin      ${delfin_url}
+    ${resp_del}=            DELETE On Session    delfin     storages/${storage_id}
+    Status Should Be        202    ${resp_del}
+
+Register Test Storage
+    ${test}=                 Load Json From File   ${CURDIR}/test.json
+    ${access_info}=          Get Value From Json   ${test}   $.test_register_access_info
+
+    Create Session          delfin      ${delfin_url}
+    ${resp_register}=       POST On Session     delfin     storages    json=${access_info[0]}
+    Status Should Be                            201    ${resp_register}
+    Dictionary Should Contain Key               ${resp_register.json()}     id
+    ${storage_id}=          Get Value From Json	     ${resp_register.json()} 	 $..id
+    [Return]                ${storage_id[0]}
+
+Register Fake Storage
+    ${fake_rest}=            Create dictionary  host=10.10.10.100  port=${8080}   username=admin  password=password
+    ${access_info}=          Create dictionary  vendor=fake_storage  model=fake_driver  rest=${fake_rest}
+    ${fake_device}=          Create dictionary  vendor=fake_vendor  model=fake_model
+
+    Create Session           delfin      ${delfin_url}
+    ${resp_register}=        POST On Session     delfin     storages    json=${access_info}
+    ${storage_id}=           Get Value From Json	     ${resp_register.json()} 	 $..id
+    Dictionary Should Contain Sub Dictionary             ${resp_register.json()}     ${fake_device}
+    [Return]                ${storage_id[0]}

delfin/tests/e2e/README.md

@@ -0,0 +1,33 @@
+# Introduction
+This folder contains end to end, automated, testing scripts for Delfin.
+
+These tests are using [Robot Framework](https://robotframework.org/) for automation and report generation.
+
+The end-to-end tests are run against a test driver provided in the path `delfin/tests/e2e/testdriver`.
+This test driver uses, included storage details in file `delfin/tests/e2e/testdriver/storage.json` for storage simulation when testing.
+
+# Supported OS
+Ubuntu 18.04
+
+# Prerequisite
+Prerequisite for [standalone installer](https://github.com/sodafoundation/delfin/blob/master/installer/README.md) is applicable here too.
+
+Install python 3.6+ and pip.
+
+Export PYTHONPATH as below
+
+```bash
+export PYTHONPATH=$(pwd)
+```
+# Run tests
+The end-to-end tests can be run from command prompt as below
+
+```bash
+git clone https://github.com/sodafoundation/delfin.git && cd delfin
+./delfin/tests/e2e/test_e2e.sh
+```
+The above script injects test driver into delfin, builds and installs delfin using delfin standalone installer.
+It runs robot framework scripts against the running delfin application for verifying delfin APIs.
+
+When the script finish execution, robot framework generates the test execution summary and log.
+These are available in the delfin root directory, with names `report.html` and `log.html` respectively.