diff --git a/.github/workflows/fastapi-gunicorn.yaml b/.github/workflows/fastapi-gunicorn.yaml index 4b06d4a..9a4161f 100644 --- a/.github/workflows/fastapi-gunicorn.yaml +++ b/.github/workflows/fastapi-gunicorn.yaml @@ -22,10 +22,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3.10 uses: actions/setup-python@v2 with: - python-version: "3.8" + python-version: "3.10" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -37,17 +37,20 @@ jobs: DOCKER_BUILDKIT: 1 run: ./test.sh release: - runs-on: ubuntu-latest - # Only push the image on new tag event - if: ${{ startsWith(github.ref, 'refs/tags/') }} + runs-on: [self-hosted, docker, integration] + env: + # list of Docker images to use as base name for tags + IMAGE_REPO: quay.io/basisai/express-fastapi + outputs: + image-tags: ${{ env.IMAGE_REPO }}:${{ steps.docker_meta.outputs.version }} steps: - name: Checkout code uses: actions/checkout@v2 - name: Docker meta id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + uses: docker/metadata-action@v3 with: - images: quay.io/basisai/express-fastapi + images: ${{ env.IMAGE_REPO }} - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -67,6 +70,19 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push + if: ${{ github.event_name == 'pull_request' }} + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + # platforms: linux/amd64,linux/arm64,linux/386 + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + outputs: type=docker,dest=/tmp/express-fastapi-${{ github.sha }}.tar + - name: Build and push + if: ${{ github.event_name != 'pull_request' }} id: docker_build uses: docker/build-push-action@v2 with: @@ -75,8 +91,65 @@ jobs: build-args: | APP=fastapi-gunicorn # platforms: linux/amd64,linux/arm64,linux/386 - push: ${{ github.event_name != 'pull_request' }} + push: true tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache + - name: Upload artifact + if: ${{ github.event_name == 'pull_request' }} + uses: actions/upload-artifact@v2 + with: + name: docker-image + path: /tmp/express-fastapi-${{ github.sha }}.tar + retention-days: 1 + scan: + runs-on: [self-hosted, docker, integration] + needs: release + strategy: + matrix: + include: + - severity: "MEDIUM,HIGH" + exit-code: "0" + - severity: "CRITICAL" + exit-code: "1" + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Download artifact + if: ${{ github.event_name == 'pull_request' }} + uses: actions/download-artifact@v2 + with: + name: docker-image + path: tmp + - name: Run Trivy vulnerability scanner for ${{ matrix.severity }} severity + if: ${{ github.event_name == 'pull_request' }} + uses: aquasecurity/trivy-action@master + with: + input: tmp/express-fastapi-${{ github.sha }}.tar + format: 'table' + exit-code: ${{ matrix.exit-code }} + ignore-unfixed: true + vuln-type: 'os,library' + severity: ${{ matrix.severity }} + continue-on-error: true + - name: Run Trivy vulnerability scanner for ${{ matrix.severity }} severity + if: ${{ github.event_name != 'pull_request' }} + uses: aquasecurity/trivy-action@master + with: + image-ref: ${{ needs.release.outputs.image-tags }} + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results-${{ matrix.severity }}.sarif' + exit-code: ${{ matrix.exit-code }} + ignore-unfixed: true + vuln-type: 'os,library' + severity: ${{ matrix.severity }} + continue-on-error: true + - name: Upload Trivy scan results to GitHub Security tab + if: ${{ github.event_name != 'pull_request' }} + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results-${{ matrix.severity }}.sarif' diff --git a/.github/workflows/flask-gunicorn.yaml b/.github/workflows/flask-gunicorn.yaml index 499a19c..64169b7 100644 --- a/.github/workflows/flask-gunicorn.yaml +++ b/.github/workflows/flask-gunicorn.yaml @@ -22,10 +22,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3.10 uses: actions/setup-python@v2 with: - python-version: "3.8" + python-version: "3.10" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -37,17 +37,20 @@ jobs: DOCKER_BUILDKIT: 1 run: ./test.sh release: - runs-on: ubuntu-latest - # Only push the image on new tag event - if: ${{ startsWith(github.ref, 'refs/tags/') }} + runs-on: [self-hosted, docker, integration] + env: + # list of Docker images to use as base name for tags + IMAGE_REPO: quay.io/basisai/express-flask + outputs: + image-tags: ${{ env.IMAGE_REPO }}:${{ steps.docker_meta.outputs.version }} steps: - name: Checkout code uses: actions/checkout@v2 - name: Docker meta id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + uses: docker/metadata-action@v3 with: - images: quay.io/basisai/express-flask + images: ${{ env.IMAGE_REPO }} - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -67,7 +70,19 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push - id: docker_build + if: ${{ github.event_name == 'pull_request' }} + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + # platforms: linux/amd64,linux/arm64,linux/386 + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + outputs: type=docker,dest=/tmp/express-flask-${{ github.sha }}.tar + - name: Build and push + if: ${{ github.event_name != 'pull_request' }} uses: docker/build-push-action@v2 with: context: . @@ -75,9 +90,65 @@ jobs: build-args: | APP=flask-gunicorn # platforms: linux/amd64,linux/arm64,linux/386 - push: ${{ github.event_name != 'pull_request' }} + push: true tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache - + - name: Upload artifact + if: ${{ github.event_name == 'pull_request' }} + uses: actions/upload-artifact@v2 + with: + name: docker-image + path: /tmp/express-flask-${{ github.sha }}.tar + retention-days: 1 + scan: + runs-on: [self-hosted, docker, integration] + needs: release + strategy: + matrix: + include: + - severity: "MEDIUM,HIGH" + exit-code: "0" + - severity: "CRITICAL" + exit-code: "1" + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Download artifact + if: ${{ github.event_name == 'pull_request' }} + uses: actions/download-artifact@v2 + with: + name: docker-image + path: tmp + - name: Run Trivy vulnerability scanner for ${{ matrix.severity }} severity + if: ${{ github.event_name == 'pull_request' }} + uses: aquasecurity/trivy-action@master + with: + input: tmp/express-flask-${{ github.sha }}.tar + format: 'table' + exit-code: ${{ matrix.exit-code }} + ignore-unfixed: true + vuln-type: 'os,library' + severity: ${{ matrix.severity }} + continue-on-error: true + - name: Run Trivy vulnerability scanner for ${{ matrix.severity }} severity + if: ${{ github.event_name != 'pull_request' }} + uses: aquasecurity/trivy-action@master + with: + image-ref: ${{ needs.release.outputs.image-tags }} + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results-${{ matrix.severity }}.sarif' + exit-code: ${{ matrix.exit-code }} + ignore-unfixed: true + vuln-type: 'os,library' + severity: ${{ matrix.severity }} + continue-on-error: true + - name: Upload Trivy scan results to GitHub Security tab + if: ${{ github.event_name != 'pull_request' }} + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results-${{ matrix.severity }}.sarif' diff --git a/Dockerfile b/Dockerfile index bdbbe54..b7b5707 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE=python:3.8.7 +ARG BASE=python:3.9 FROM ${BASE} WORKDIR /app diff --git a/README.md b/README.md index a0a86c5..a9b8d01 100644 --- a/README.md +++ b/README.md @@ -168,12 +168,12 @@ docker build --build-arg APP=fastapi-gunicorn . -t fastapi-gunicorn To run tests for all server and framework flavours, simply run `./test.sh` from the repo directory. -Individual images can be tested using `docker-compose` from the current directory by setting the `APP` and `MODEL` environment variables. For example, +Individual images can be tested using `docker compose` from the current directory by setting the `APP` and `MODEL` environment variables. For example, ```bash # To test the flask-gunicorn stack with lightgbm model -APP=flask-gunicorn MODEL=lightgbm docker-compose -f tests/docker-compose.yml up --build --abort-on-container-exit --always-recreate-deps +APP=flask-gunicorn MODEL=lightgbm docker compose -f tests/docker-compose.yml up --build --abort-on-container-exit --always-recreate-deps # To test the fastapi-gunicorn stack with lightgbm model -APP=fastapi-gunicorn MODEL=lightgbm docker-compose -f tests/docker-compose.yml up --build --abort-on-container-exit --always-recreate-deps +APP=fastapi-gunicorn MODEL=lightgbm docker compose -f tests/docker-compose.yml up --build --abort-on-container-exit --always-recreate-deps ``` diff --git a/fastapi-gunicorn/requirements.txt b/fastapi-gunicorn/requirements.txt index 8b252e9..c29d1d3 100644 --- a/fastapi-gunicorn/requirements.txt +++ b/fastapi-gunicorn/requirements.txt @@ -1,6 +1,6 @@ -bdrk==0.9.1 +bdrk==0.9.2 boxkite==0.0.5 -uvicorn==0.14.0 +uvicorn==0.16.0 gunicorn==20.1.0 -fastapi==0.67.0 +fastapi==0.70.1 python-multipart==0.0.5 diff --git a/flask-gunicorn-gpu/requirements.txt b/flask-gunicorn-gpu/requirements.txt index d720696..ec528f4 100644 --- a/flask-gunicorn-gpu/requirements.txt +++ b/flask-gunicorn-gpu/requirements.txt @@ -1,4 +1,4 @@ -bdrk==0.9.1 +bdrk==0.9.2 boxkite==0.0.5 -flask==2.0.1 +flask==2.0.2 gunicorn==20.1.0 diff --git a/flask-gunicorn/requirements.txt b/flask-gunicorn/requirements.txt index d720696..ec528f4 100644 --- a/flask-gunicorn/requirements.txt +++ b/flask-gunicorn/requirements.txt @@ -1,4 +1,4 @@ -bdrk==0.9.1 +bdrk==0.9.2 boxkite==0.0.5 -flask==2.0.1 +flask==2.0.2 gunicorn==20.1.0 diff --git a/test.sh b/test.sh index d32b083..ef7a895 100755 --- a/test.sh +++ b/test.sh @@ -29,11 +29,11 @@ for app in "${targets[@]}"; do export MODEL="$model" # Save exit code for use later - docker-compose -f tests/docker-compose.yml -p "$app" up \ + docker compose -f tests/docker-compose.yml -p "$app" up \ --build --abort-on-container-exit --always-recreate-deps || exit_code=$? # Clean up containers - docker-compose -f tests/docker-compose.yml -p "$app" down -v --remove-orphans + docker compose -f tests/docker-compose.yml -p "$app" down -v --remove-orphans # Abort early on failure if [[ $exit_code -ne 0 ]]; then diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index d536a12..464073b 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -1,7 +1,7 @@ version: "3" services: test: - image: python:3.8.5 + image: python:3.10 environment: HOST: app MODEL: ${MODEL:-lightgbm} @@ -38,7 +38,7 @@ services: pip install -r requirements.txt # Run unit test python -m unittest -v test_serve.py - /app/entrypoint.sh + exec /app/entrypoint.sh volumes: - ./${MODEL:-lightgbm}/artefact:/artefact - ./${MODEL:-lightgbm}/model-server:/model-server diff --git a/tests/lightgbm/artefact/histogram.prom b/tests/lightgbm/artefact/histogram.prom old mode 100755 new mode 100644 index e05d408..98739d6 --- a/tests/lightgbm/artefact/histogram.prom +++ b/tests/lightgbm/artefact/histogram.prom @@ -1,571 +1,64 @@ -# HELP feature_0_value_baseline_total Baseline values for feature: Intl_Plan -# TYPE feature_0_value_baseline_total counter -feature_0_value_baseline_total{bin="0.0"} 911.0 -feature_0_value_baseline_total{bin="1.0"} 89.0 -# HELP feature_1_value_baseline_total Baseline values for feature: VMail_Plan -# TYPE feature_1_value_baseline_total counter -feature_1_value_baseline_total{bin="0.0"} 719.0 -feature_1_value_baseline_total{bin="1.0"} 281.0 -# HELP feature_2_value_baseline_total Baseline values for feature: VMail_Message -# TYPE feature_2_value_baseline_total counter -feature_2_value_baseline_total{bin="0.0"} 719.0 -feature_2_value_baseline_total{bin="8.0"} 1.0 -feature_2_value_baseline_total{bin="9.0"} 1.0 -feature_2_value_baseline_total{bin="10.0"} 1.0 -feature_2_value_baseline_total{bin="11.0"} 1.0 -feature_2_value_baseline_total{bin="12.0"} 3.0 -feature_2_value_baseline_total{bin="14.0"} 1.0 -feature_2_value_baseline_total{bin="15.0"} 2.0 -feature_2_value_baseline_total{bin="16.0"} 4.0 -feature_2_value_baseline_total{bin="17.0"} 4.0 -feature_2_value_baseline_total{bin="18.0"} 2.0 -feature_2_value_baseline_total{bin="19.0"} 6.0 -feature_2_value_baseline_total{bin="20.0"} 9.0 -feature_2_value_baseline_total{bin="21.0"} 11.0 -feature_2_value_baseline_total{bin="22.0"} 13.0 -feature_2_value_baseline_total{bin="23.0"} 14.0 -feature_2_value_baseline_total{bin="24.0"} 9.0 -feature_2_value_baseline_total{bin="25.0"} 14.0 -feature_2_value_baseline_total{bin="26.0"} 12.0 -feature_2_value_baseline_total{bin="27.0"} 14.0 -feature_2_value_baseline_total{bin="28.0"} 14.0 -feature_2_value_baseline_total{bin="29.0"} 21.0 -feature_2_value_baseline_total{bin="30.0"} 16.0 -feature_2_value_baseline_total{bin="31.0"} 17.0 -feature_2_value_baseline_total{bin="32.0"} 13.0 -feature_2_value_baseline_total{bin="33.0"} 12.0 -feature_2_value_baseline_total{bin="34.0"} 10.0 -feature_2_value_baseline_total{bin="35.0"} 8.0 -feature_2_value_baseline_total{bin="36.0"} 11.0 -feature_2_value_baseline_total{bin="37.0"} 8.0 -feature_2_value_baseline_total{bin="38.0"} 6.0 -feature_2_value_baseline_total{bin="39.0"} 8.0 -feature_2_value_baseline_total{bin="40.0"} 3.0 -feature_2_value_baseline_total{bin="41.0"} 1.0 -feature_2_value_baseline_total{bin="42.0"} 5.0 -feature_2_value_baseline_total{bin="44.0"} 1.0 -feature_2_value_baseline_total{bin="46.0"} 1.0 -feature_2_value_baseline_total{bin="47.0"} 1.0 -feature_2_value_baseline_total{bin="48.0"} 2.0 -feature_2_value_baseline_total{bin="49.0"} 1.0 -# HELP feature_3_value_baseline_total Baseline values for feature: CustServ_Calls -# TYPE feature_3_value_baseline_total counter -feature_3_value_baseline_total{bin="0.0"} 203.0 -feature_3_value_baseline_total{bin="1.0"} 365.0 -feature_3_value_baseline_total{bin="2.0"} 219.0 -feature_3_value_baseline_total{bin="3.0"} 124.0 -feature_3_value_baseline_total{bin="4.0"} 58.0 -feature_3_value_baseline_total{bin="5.0"} 15.0 -feature_3_value_baseline_total{bin="6.0"} 8.0 -feature_3_value_baseline_total{bin="7.0"} 4.0 -feature_3_value_baseline_total{bin="8.0"} 2.0 -feature_3_value_baseline_total{bin="9.0"} 2.0 -# HELP feature_4_value_baseline Baseline values for feature: Day_Mins -# TYPE feature_4_value_baseline histogram -feature_4_value_baseline_bucket{le="0.0"} 1.0 -feature_4_value_baseline_bucket{le="14.61666664481163"} 2.0 -feature_4_value_baseline_bucket{le="29.23333328962326"} 3.0 -feature_4_value_baseline_bucket{le="43.84999993443489"} 5.0 -feature_4_value_baseline_bucket{le="58.46666657924652"} 11.0 -feature_4_value_baseline_bucket{le="73.08333322405815"} 28.0 -feature_4_value_baseline_bucket{le="87.69999986886978"} 47.0 -feature_4_value_baseline_bucket{le="102.31666651368141"} 81.0 -feature_4_value_baseline_bucket{le="116.93333315849304"} 131.0 -feature_4_value_baseline_bucket{le="131.54999980330467"} 189.0 -feature_4_value_baseline_bucket{le="146.1666664481163"} 270.0 -feature_4_value_baseline_bucket{le="160.78333309292793"} 375.0 -feature_4_value_baseline_bucket{le="175.39999973773956"} 469.0 -feature_4_value_baseline_bucket{le="190.0166663825512"} 567.0 -feature_4_value_baseline_bucket{le="204.63333302736282"} 675.0 -feature_4_value_baseline_bucket{le="219.24999967217445"} 764.0 -feature_4_value_baseline_bucket{le="233.86666631698608"} 843.0 -feature_4_value_baseline_bucket{le="248.48333296179771"} 897.0 -feature_4_value_baseline_bucket{le="263.09999960660934"} 936.0 -feature_4_value_baseline_bucket{le="277.716666251421"} 968.0 -feature_4_value_baseline_bucket{le="292.3333328962326"} 983.0 -feature_4_value_baseline_bucket{le="306.94999954104424"} 993.0 -feature_4_value_baseline_bucket{le="321.56666618585587"} 996.0 -feature_4_value_baseline_bucket{le="336.1833328306675"} 998.0 -feature_4_value_baseline_bucket{le="350.7999994754791"} 1000.0 -feature_4_value_baseline_bucket{le="+Inf"} 1000.0 -feature_4_value_baseline_count 1000.0 -feature_4_value_baseline_sum 179179.30002207693 -# HELP feature_5_value_baseline Baseline values for feature: Day_Calls -# TYPE feature_5_value_baseline histogram -feature_5_value_baseline_bucket{le="0.0"} 1.0 -feature_5_value_baseline_bucket{le="5.517241379310345"} 1.0 -feature_5_value_baseline_bucket{le="11.03448275862069"} 1.0 -feature_5_value_baseline_bucket{le="16.551724137931036"} 1.0 -feature_5_value_baseline_bucket{le="22.06896551724138"} 1.0 -feature_5_value_baseline_bucket{le="27.586206896551726"} 1.0 -feature_5_value_baseline_bucket{le="33.10344827586207"} 1.0 -feature_5_value_baseline_bucket{le="38.62068965517241"} 2.0 -feature_5_value_baseline_bucket{le="44.13793103448276"} 3.0 -feature_5_value_baseline_bucket{le="49.65517241379311"} 6.0 -feature_5_value_baseline_bucket{le="55.17241379310345"} 12.0 -feature_5_value_baseline_bucket{le="60.689655172413794"} 22.0 -feature_5_value_baseline_bucket{le="66.20689655172414"} 50.0 -feature_5_value_baseline_bucket{le="71.72413793103449"} 84.0 -feature_5_value_baseline_bucket{le="77.24137931034483"} 130.0 -feature_5_value_baseline_bucket{le="82.75862068965517"} 190.0 -feature_5_value_baseline_bucket{le="88.27586206896552"} 278.0 -feature_5_value_baseline_bucket{le="93.79310344827587"} 363.0 -feature_5_value_baseline_bucket{le="99.31034482758622"} 474.0 -feature_5_value_baseline_bucket{le="104.82758620689656"} 562.0 -feature_5_value_baseline_bucket{le="110.3448275862069"} 692.0 -feature_5_value_baseline_bucket{le="115.86206896551725"} 759.0 -feature_5_value_baseline_bucket{le="121.37931034482759"} 858.0 -feature_5_value_baseline_bucket{le="126.89655172413794"} 903.0 -feature_5_value_baseline_bucket{le="132.41379310344828"} 945.0 -feature_5_value_baseline_bucket{le="137.93103448275863"} 966.0 -feature_5_value_baseline_bucket{le="143.44827586206898"} 987.0 -feature_5_value_baseline_bucket{le="148.96551724137933"} 990.0 -feature_5_value_baseline_bucket{le="154.48275862068965"} 998.0 -feature_5_value_baseline_bucket{le="160.0"} 1000.0 -feature_5_value_baseline_bucket{le="+Inf"} 1000.0 -feature_5_value_baseline_count 1000.0 -feature_5_value_baseline_sum 100467.0 -# HELP feature_6_value_baseline Baseline values for feature: Eve_Mins -# TYPE feature_6_value_baseline histogram -feature_6_value_baseline_bucket{le="42.199999995529645"} 1.0 -feature_6_value_baseline_bucket{le="55.765217378735535"} 3.0 -feature_6_value_baseline_bucket{le="69.33043476194143"} 5.0 -feature_6_value_baseline_bucket{le="82.89565214514732"} 12.0 -feature_6_value_baseline_bucket{le="96.46086952835321"} 18.0 -feature_6_value_baseline_bucket{le="110.0260869115591"} 32.0 -feature_6_value_baseline_bucket{le="123.591304294765"} 67.0 -feature_6_value_baseline_bucket{le="137.1565216779709"} 110.0 -feature_6_value_baseline_bucket{le="150.72173906117678"} 162.0 -feature_6_value_baseline_bucket{le="164.28695644438267"} 246.0 -feature_6_value_baseline_bucket{le="177.8521738275886"} 343.0 -feature_6_value_baseline_bucket{le="191.41739121079448"} 431.0 -feature_6_value_baseline_bucket{le="204.98260859400037"} 526.0 -feature_6_value_baseline_bucket{le="218.54782597720626"} 654.0 -feature_6_value_baseline_bucket{le="232.11304336041215"} 731.0 -feature_6_value_baseline_bucket{le="245.67826074361804"} 826.0 -feature_6_value_baseline_bucket{le="259.2434781268239"} 894.0 -feature_6_value_baseline_bucket{le="272.8086955100298"} 940.0 -feature_6_value_baseline_bucket{le="286.3739128932357"} 965.0 -feature_6_value_baseline_bucket{le="299.93913027644163"} 978.0 -feature_6_value_baseline_bucket{le="313.5043476596475"} 990.0 -feature_6_value_baseline_bucket{le="327.0695650428534"} 994.0 -feature_6_value_baseline_bucket{le="340.6347824260593"} 998.0 -feature_6_value_baseline_bucket{le="354.1999998092652"} 1000.0 -feature_6_value_baseline_bucket{le="+Inf"} 1000.0 -feature_6_value_baseline_count 1000.0 -feature_6_value_baseline_sum 199092.30000826227 -# HELP feature_7_value_baseline Baseline values for feature: Eve_Calls -# TYPE feature_7_value_baseline histogram -feature_7_value_baseline_bucket{le="48.0"} 2.0 -feature_7_value_baseline_bucket{le="53.285714285714285"} 8.0 -feature_7_value_baseline_bucket{le="58.57142857142857"} 19.0 -feature_7_value_baseline_bucket{le="63.85714285714286"} 35.0 -feature_7_value_baseline_bucket{le="69.14285714285714"} 71.0 -feature_7_value_baseline_bucket{le="74.42857142857143"} 100.0 -feature_7_value_baseline_bucket{le="79.71428571428572"} 149.0 -feature_7_value_baseline_bucket{le="85.0"} 225.0 -feature_7_value_baseline_bucket{le="90.28571428571428"} 331.0 -feature_7_value_baseline_bucket{le="95.57142857142857"} 418.0 -feature_7_value_baseline_bucket{le="100.85714285714286"} 510.0 -feature_7_value_baseline_bucket{le="106.14285714285714"} 622.0 -feature_7_value_baseline_bucket{le="111.42857142857143"} 712.0 -feature_7_value_baseline_bucket{le="116.71428571428571"} 806.0 -feature_7_value_baseline_bucket{le="122.0"} 882.0 -feature_7_value_baseline_bucket{le="127.28571428571428"} 927.0 -feature_7_value_baseline_bucket{le="132.57142857142856"} 959.0 -feature_7_value_baseline_bucket{le="137.85714285714286"} 974.0 -feature_7_value_baseline_bucket{le="143.14285714285714"} 986.0 -feature_7_value_baseline_bucket{le="148.42857142857144"} 990.0 -feature_7_value_baseline_bucket{le="153.71428571428572"} 996.0 -feature_7_value_baseline_bucket{le="159.0"} 1000.0 -feature_7_value_baseline_bucket{le="+Inf"} 1000.0 -feature_7_value_baseline_count 1000.0 -feature_7_value_baseline_sum 99815.0 -# HELP feature_8_value_baseline Baseline values for feature: Night_Mins -# TYPE feature_8_value_baseline histogram -feature_8_value_baseline_bucket{le="50.09999991953373"} 1.0 -feature_8_value_baseline_bucket{le="63.37199990391731"} 5.0 -feature_8_value_baseline_bucket{le="76.6439998883009"} 12.0 -feature_8_value_baseline_bucket{le="89.91599987268448"} 19.0 -feature_8_value_baseline_bucket{le="103.18799985706806"} 36.0 -feature_8_value_baseline_bucket{le="116.45999984145165"} 54.0 -feature_8_value_baseline_bucket{le="129.73199982583523"} 89.0 -feature_8_value_baseline_bucket{le="143.00399981021883"} 124.0 -feature_8_value_baseline_bucket{le="156.2759997946024"} 198.0 -feature_8_value_baseline_bucket{le="169.54799977898597"} 272.0 -feature_8_value_baseline_bucket{le="182.81999976336957"} 373.0 -feature_8_value_baseline_bucket{le="196.09199974775316"} 479.0 -feature_8_value_baseline_bucket{le="209.36399973213673"} 565.0 -feature_8_value_baseline_bucket{le="222.6359997165203"} 674.0 -feature_8_value_baseline_bucket{le="235.9079997009039"} 758.0 -feature_8_value_baseline_bucket{le="249.1799996852875"} 842.0 -feature_8_value_baseline_bucket{le="262.45199966967107"} 892.0 -feature_8_value_baseline_bucket{le="275.72399965405464"} 935.0 -feature_8_value_baseline_bucket{le="288.9959996384382"} 970.0 -feature_8_value_baseline_bucket{le="302.26799962282183"} 983.0 -feature_8_value_baseline_bucket{le="315.5399996072054"} 991.0 -feature_8_value_baseline_bucket{le="328.811999591589"} 994.0 -feature_8_value_baseline_bucket{le="342.0839995759726"} 995.0 -feature_8_value_baseline_bucket{le="355.35599956035617"} 997.0 -feature_8_value_baseline_bucket{le="368.62799954473974"} 998.0 -feature_8_value_baseline_bucket{le="381.8999995291233"} 1000.0 -feature_8_value_baseline_bucket{le="+Inf"} 1000.0 -feature_8_value_baseline_count 1000.0 -feature_8_value_baseline_sum 199371.4999850244 -# HELP feature_9_value_baseline Baseline values for feature: Night_Calls -# TYPE feature_9_value_baseline histogram -feature_9_value_baseline_bucket{le="36.0"} 1.0 -feature_9_value_baseline_bucket{le="41.0"} 1.0 -feature_9_value_baseline_bucket{le="46.0"} 1.0 -feature_9_value_baseline_bucket{le="51.0"} 5.0 -feature_9_value_baseline_bucket{le="56.0"} 10.0 -feature_9_value_baseline_bucket{le="61.0"} 23.0 -feature_9_value_baseline_bucket{le="66.0"} 38.0 -feature_9_value_baseline_bucket{le="71.0"} 64.0 -feature_9_value_baseline_bucket{le="76.0"} 119.0 -feature_9_value_baseline_bucket{le="81.0"} 173.0 -feature_9_value_baseline_bucket{le="86.0"} 241.0 -feature_9_value_baseline_bucket{le="91.0"} 347.0 -feature_9_value_baseline_bucket{le="96.0"} 440.0 -feature_9_value_baseline_bucket{le="101.0"} 527.0 -feature_9_value_baseline_bucket{le="106.0"} 640.0 -feature_9_value_baseline_bucket{le="111.0"} 731.0 -feature_9_value_baseline_bucket{le="116.0"} 802.0 -feature_9_value_baseline_bucket{le="121.0"} 865.0 -feature_9_value_baseline_bucket{le="126.0"} 908.0 -feature_9_value_baseline_bucket{le="131.0"} 945.0 -feature_9_value_baseline_bucket{le="136.0"} 970.0 -feature_9_value_baseline_bucket{le="141.0"} 983.0 -feature_9_value_baseline_bucket{le="146.0"} 993.0 -feature_9_value_baseline_bucket{le="151.0"} 997.0 -feature_9_value_baseline_bucket{le="156.0"} 1000.0 -feature_9_value_baseline_bucket{le="+Inf"} 1000.0 -feature_9_value_baseline_count 1000.0 -feature_9_value_baseline_sum 99902.0 -# HELP feature_10_value_baseline Baseline values for feature: Intl_Mins -# TYPE feature_10_value_baseline histogram -feature_10_value_baseline_bucket{le="0.0"} 8.0 -feature_10_value_baseline_bucket{le="0.7142857142857143"} 8.0 -feature_10_value_baseline_bucket{le="1.4285714285714286"} 8.0 -feature_10_value_baseline_bucket{le="2.142857142857143"} 9.0 -feature_10_value_baseline_bucket{le="2.857142857142857"} 10.0 -feature_10_value_baseline_bucket{le="3.5714285714285716"} 14.0 -feature_10_value_baseline_bucket{le="4.285714285714286"} 22.0 -feature_10_value_baseline_bucket{le="5.0"} 34.0 -feature_10_value_baseline_bucket{le="5.714285714285714"} 53.0 -feature_10_value_baseline_bucket{le="6.428571428571429"} 96.0 -feature_10_value_baseline_bucket{le="7.142857142857143"} 142.0 -feature_10_value_baseline_bucket{le="7.857142857142858"} 204.0 -feature_10_value_baseline_bucket{le="8.571428571428571"} 275.0 -feature_10_value_baseline_bucket{le="9.285714285714286"} 342.0 -feature_10_value_baseline_bucket{le="10.0"} 462.0 -feature_10_value_baseline_bucket{le="10.714285714285715"} 570.0 -feature_10_value_baseline_bucket{le="11.428571428571429"} 681.0 -feature_10_value_baseline_bucket{le="12.142857142857142"} 776.0 -feature_10_value_baseline_bucket{le="12.857142857142858"} 843.0 -feature_10_value_baseline_bucket{le="13.571428571428571"} 898.0 -feature_10_value_baseline_bucket{le="14.285714285714286"} 940.0 -feature_10_value_baseline_bucket{le="15.0"} 963.0 -feature_10_value_baseline_bucket{le="15.714285714285715"} 979.0 -feature_10_value_baseline_bucket{le="16.42857142857143"} 988.0 -feature_10_value_baseline_bucket{le="17.142857142857142"} 991.0 -feature_10_value_baseline_bucket{le="17.857142857142858"} 997.0 -feature_10_value_baseline_bucket{le="18.571428571428573"} 998.0 -feature_10_value_baseline_bucket{le="19.285714285714285"} 999.0 -feature_10_value_baseline_bucket{le="20.0"} 1000.0 -feature_10_value_baseline_bucket{le="+Inf"} 1000.0 -feature_10_value_baseline_count 1000.0 -feature_10_value_baseline_sum 10138.699991433852 -# HELP feature_11_value_baseline_total Baseline values for feature: Intl_Calls -# TYPE feature_11_value_baseline_total counter -feature_11_value_baseline_total{bin="0.0"} 8.0 -feature_11_value_baseline_total{bin="1.0"} 53.0 -feature_11_value_baseline_total{bin="2.0"} 156.0 -feature_11_value_baseline_total{bin="3.0"} 198.0 -feature_11_value_baseline_total{bin="4.0"} 171.0 -feature_11_value_baseline_total{bin="5.0"} 141.0 -feature_11_value_baseline_total{bin="6.0"} 109.0 -feature_11_value_baseline_total{bin="7.0"} 59.0 -feature_11_value_baseline_total{bin="8.0"} 34.0 -feature_11_value_baseline_total{bin="9.0"} 27.0 -feature_11_value_baseline_total{bin="10.0"} 17.0 -feature_11_value_baseline_total{bin="11.0"} 13.0 -feature_11_value_baseline_total{bin="12.0"} 6.0 -feature_11_value_baseline_total{bin="13.0"} 2.0 -feature_11_value_baseline_total{bin="14.0"} 2.0 -feature_11_value_baseline_total{bin="15.0"} 1.0 -feature_11_value_baseline_total{bin="16.0"} 1.0 -feature_11_value_baseline_total{bin="17.0"} 1.0 -feature_11_value_baseline_total{bin="18.0"} 1.0 -# HELP feature_12_value_baseline_total Baseline values for feature: Area_Code_408 -# TYPE feature_12_value_baseline_total counter -feature_12_value_baseline_total{bin="0.0"} 752.0 -feature_12_value_baseline_total{bin="1.0"} 248.0 -# HELP feature_13_value_baseline_total Baseline values for feature: Area_Code_415 -# TYPE feature_13_value_baseline_total counter -feature_13_value_baseline_total{bin="0.0"} 516.0 -feature_13_value_baseline_total{bin="1.0"} 484.0 -# HELP feature_14_value_baseline_total Baseline values for feature: Area_Code_510 -# TYPE feature_14_value_baseline_total counter -feature_14_value_baseline_total{bin="0.0"} 732.0 -feature_14_value_baseline_total{bin="1.0"} 268.0 -# HELP feature_15_value_baseline_total Baseline values for feature: State_AK -# TYPE feature_15_value_baseline_total counter -feature_15_value_baseline_total{bin="0.0"} 980.0 -feature_15_value_baseline_total{bin="1.0"} 20.0 -# HELP feature_16_value_baseline_total Baseline values for feature: State_AL -# TYPE feature_16_value_baseline_total counter -feature_16_value_baseline_total{bin="0.0"} 977.0 -feature_16_value_baseline_total{bin="1.0"} 23.0 -# HELP feature_17_value_baseline_total Baseline values for feature: State_AR -# TYPE feature_17_value_baseline_total counter -feature_17_value_baseline_total{bin="0.0"} 984.0 -feature_17_value_baseline_total{bin="1.0"} 16.0 -# HELP feature_18_value_baseline_total Baseline values for feature: State_AZ -# TYPE feature_18_value_baseline_total counter -feature_18_value_baseline_total{bin="0.0"} 982.0 -feature_18_value_baseline_total{bin="1.0"} 18.0 -# HELP feature_19_value_baseline_total Baseline values for feature: State_CA -# TYPE feature_19_value_baseline_total counter -feature_19_value_baseline_total{bin="0.0"} 989.0 -feature_19_value_baseline_total{bin="1.0"} 11.0 -# HELP feature_20_value_baseline_total Baseline values for feature: State_CO -# TYPE feature_20_value_baseline_total counter -feature_20_value_baseline_total{bin="0.0"} 980.0 -feature_20_value_baseline_total{bin="1.0"} 20.0 -# HELP feature_21_value_baseline_total Baseline values for feature: State_CT -# TYPE feature_21_value_baseline_total counter -feature_21_value_baseline_total{bin="0.0"} 977.0 -feature_21_value_baseline_total{bin="1.0"} 23.0 -# HELP feature_22_value_baseline_total Baseline values for feature: State_DC -# TYPE feature_22_value_baseline_total counter -feature_22_value_baseline_total{bin="0.0"} 982.0 -feature_22_value_baseline_total{bin="1.0"} 18.0 -# HELP feature_23_value_baseline_total Baseline values for feature: State_DE -# TYPE feature_23_value_baseline_total counter -feature_23_value_baseline_total{bin="0.0"} 979.0 -feature_23_value_baseline_total{bin="1.0"} 21.0 -# HELP feature_24_value_baseline_total Baseline values for feature: State_FL -# TYPE feature_24_value_baseline_total counter -feature_24_value_baseline_total{bin="0.0"} 983.0 -feature_24_value_baseline_total{bin="1.0"} 17.0 -# HELP feature_25_value_baseline_total Baseline values for feature: State_GA -# TYPE feature_25_value_baseline_total counter -feature_25_value_baseline_total{bin="0.0"} 983.0 -feature_25_value_baseline_total{bin="1.0"} 17.0 -# HELP feature_26_value_baseline_total Baseline values for feature: State_HI -# TYPE feature_26_value_baseline_total counter -feature_26_value_baseline_total{bin="0.0"} 982.0 -feature_26_value_baseline_total{bin="1.0"} 18.0 -# HELP feature_27_value_baseline_total Baseline values for feature: State_IA -# TYPE feature_27_value_baseline_total counter -feature_27_value_baseline_total{bin="0.0"} 989.0 -feature_27_value_baseline_total{bin="1.0"} 11.0 -# HELP feature_28_value_baseline_total Baseline values for feature: State_ID -# TYPE feature_28_value_baseline_total counter -feature_28_value_baseline_total{bin="0.0"} 975.0 -feature_28_value_baseline_total{bin="1.0"} 25.0 -# HELP feature_29_value_baseline_total Baseline values for feature: State_IL -# TYPE feature_29_value_baseline_total counter -feature_29_value_baseline_total{bin="0.0"} 978.0 -feature_29_value_baseline_total{bin="1.0"} 22.0 -# HELP feature_30_value_baseline_total Baseline values for feature: State_IN -# TYPE feature_30_value_baseline_total counter -feature_30_value_baseline_total{bin="0.0"} 980.0 -feature_30_value_baseline_total{bin="1.0"} 20.0 -# HELP feature_31_value_baseline_total Baseline values for feature: State_KS -# TYPE feature_31_value_baseline_total counter -feature_31_value_baseline_total{bin="0.0"} 978.0 -feature_31_value_baseline_total{bin="1.0"} 22.0 -# HELP feature_32_value_baseline_total Baseline values for feature: State_KY -# TYPE feature_32_value_baseline_total counter -feature_32_value_baseline_total{bin="0.0"} 991.0 -feature_32_value_baseline_total{bin="1.0"} 9.0 -# HELP feature_33_value_baseline_total Baseline values for feature: State_LA -# TYPE feature_33_value_baseline_total counter -feature_33_value_baseline_total{bin="0.0"} 983.0 -feature_33_value_baseline_total{bin="1.0"} 17.0 -# HELP feature_34_value_baseline_total Baseline values for feature: State_MA -# TYPE feature_34_value_baseline_total counter -feature_34_value_baseline_total{bin="0.0"} 982.0 -feature_34_value_baseline_total{bin="1.0"} 18.0 -# HELP feature_35_value_baseline_total Baseline values for feature: State_MD -# TYPE feature_35_value_baseline_total counter -feature_35_value_baseline_total{bin="0.0"} 982.0 -feature_35_value_baseline_total{bin="1.0"} 18.0 -# HELP feature_36_value_baseline_total Baseline values for feature: State_ME -# TYPE feature_36_value_baseline_total counter -feature_36_value_baseline_total{bin="0.0"} 990.0 -feature_36_value_baseline_total{bin="1.0"} 10.0 -# HELP feature_37_value_baseline_total Baseline values for feature: State_MI -# TYPE feature_37_value_baseline_total counter -feature_37_value_baseline_total{bin="0.0"} 984.0 -feature_37_value_baseline_total{bin="1.0"} 16.0 -# HELP feature_38_value_baseline_total Baseline values for feature: State_MN -# TYPE feature_38_value_baseline_total counter -feature_38_value_baseline_total{bin="0.0"} 978.0 -feature_38_value_baseline_total{bin="1.0"} 22.0 -# HELP feature_39_value_baseline_total Baseline values for feature: State_MO -# TYPE feature_39_value_baseline_total counter -feature_39_value_baseline_total{bin="0.0"} 982.0 -feature_39_value_baseline_total{bin="1.0"} 18.0 -# HELP feature_40_value_baseline_total Baseline values for feature: State_MS -# TYPE feature_40_value_baseline_total counter -feature_40_value_baseline_total{bin="0.0"} 982.0 -feature_40_value_baseline_total{bin="1.0"} 18.0 -# HELP feature_41_value_baseline_total Baseline values for feature: State_MT -# TYPE feature_41_value_baseline_total counter -feature_41_value_baseline_total{bin="0.0"} 975.0 -feature_41_value_baseline_total{bin="1.0"} 25.0 -# HELP feature_42_value_baseline_total Baseline values for feature: State_NC -# TYPE feature_42_value_baseline_total counter -feature_42_value_baseline_total{bin="0.0"} 975.0 -feature_42_value_baseline_total{bin="1.0"} 25.0 -# HELP feature_43_value_baseline_total Baseline values for feature: State_ND -# TYPE feature_43_value_baseline_total counter -feature_43_value_baseline_total{bin="0.0"} 985.0 -feature_43_value_baseline_total{bin="1.0"} 15.0 -# HELP feature_44_value_baseline_total Baseline values for feature: State_NE -# TYPE feature_44_value_baseline_total counter -feature_44_value_baseline_total{bin="0.0"} 981.0 -feature_44_value_baseline_total{bin="1.0"} 19.0 -# HELP feature_45_value_baseline_total Baseline values for feature: State_NH -# TYPE feature_45_value_baseline_total counter -feature_45_value_baseline_total{bin="0.0"} 987.0 -feature_45_value_baseline_total{bin="1.0"} 13.0 -# HELP feature_46_value_baseline_total Baseline values for feature: State_NJ -# TYPE feature_46_value_baseline_total counter -feature_46_value_baseline_total{bin="0.0"} 983.0 -feature_46_value_baseline_total{bin="1.0"} 17.0 -# HELP feature_47_value_baseline_total Baseline values for feature: State_NM -# TYPE feature_47_value_baseline_total counter -feature_47_value_baseline_total{bin="0.0"} 985.0 -feature_47_value_baseline_total{bin="1.0"} 15.0 -# HELP feature_48_value_baseline_total Baseline values for feature: State_NV -# TYPE feature_48_value_baseline_total counter -feature_48_value_baseline_total{bin="0.0"} 976.0 -feature_48_value_baseline_total{bin="1.0"} 24.0 -# HELP feature_49_value_baseline_total Baseline values for feature: State_NY -# TYPE feature_49_value_baseline_total counter -feature_49_value_baseline_total{bin="0.0"} 975.0 -feature_49_value_baseline_total{bin="1.0"} 25.0 -# HELP feature_50_value_baseline_total Baseline values for feature: State_OH -# TYPE feature_50_value_baseline_total counter -feature_50_value_baseline_total{bin="0.0"} 977.0 -feature_50_value_baseline_total{bin="1.0"} 23.0 -# HELP feature_51_value_baseline_total Baseline values for feature: State_OK -# TYPE feature_51_value_baseline_total counter -feature_51_value_baseline_total{bin="0.0"} 981.0 -feature_51_value_baseline_total{bin="1.0"} 19.0 -# HELP feature_52_value_baseline_total Baseline values for feature: State_OR -# TYPE feature_52_value_baseline_total counter -feature_52_value_baseline_total{bin="0.0"} 971.0 -feature_52_value_baseline_total{bin="1.0"} 29.0 -# HELP feature_53_value_baseline_total Baseline values for feature: State_PA -# TYPE feature_53_value_baseline_total counter -feature_53_value_baseline_total{bin="0.0"} 988.0 -feature_53_value_baseline_total{bin="1.0"} 12.0 -# HELP feature_54_value_baseline_total Baseline values for feature: State_RI -# TYPE feature_54_value_baseline_total counter -feature_54_value_baseline_total{bin="0.0"} 978.0 -feature_54_value_baseline_total{bin="1.0"} 22.0 -# HELP feature_55_value_baseline_total Baseline values for feature: State_SC -# TYPE feature_55_value_baseline_total counter -feature_55_value_baseline_total{bin="0.0"} 986.0 -feature_55_value_baseline_total{bin="1.0"} 14.0 -# HELP feature_56_value_baseline_total Baseline values for feature: State_SD -# TYPE feature_56_value_baseline_total counter -feature_56_value_baseline_total{bin="0.0"} 979.0 -feature_56_value_baseline_total{bin="1.0"} 21.0 -# HELP feature_57_value_baseline_total Baseline values for feature: State_TN -# TYPE feature_57_value_baseline_total counter -feature_57_value_baseline_total{bin="0.0"} 979.0 -feature_57_value_baseline_total{bin="1.0"} 21.0 -# HELP feature_58_value_baseline_total Baseline values for feature: State_TX -# TYPE feature_58_value_baseline_total counter -feature_58_value_baseline_total{bin="0.0"} 978.0 -feature_58_value_baseline_total{bin="1.0"} 22.0 -# HELP feature_59_value_baseline_total Baseline values for feature: State_UT -# TYPE feature_59_value_baseline_total counter -feature_59_value_baseline_total{bin="0.0"} 973.0 -feature_59_value_baseline_total{bin="1.0"} 27.0 -# HELP feature_60_value_baseline_total Baseline values for feature: State_VA -# TYPE feature_60_value_baseline_total counter -feature_60_value_baseline_total{bin="0.0"} 973.0 -feature_60_value_baseline_total{bin="1.0"} 27.0 -# HELP feature_61_value_baseline_total Baseline values for feature: State_VT -# TYPE feature_61_value_baseline_total counter -feature_61_value_baseline_total{bin="0.0"} 975.0 -feature_61_value_baseline_total{bin="1.0"} 25.0 -# HELP feature_62_value_baseline_total Baseline values for feature: State_WA -# TYPE feature_62_value_baseline_total counter -feature_62_value_baseline_total{bin="0.0"} 980.0 -feature_62_value_baseline_total{bin="1.0"} 20.0 -# HELP feature_63_value_baseline_total Baseline values for feature: State_WI -# TYPE feature_63_value_baseline_total counter -feature_63_value_baseline_total{bin="0.0"} 978.0 -feature_63_value_baseline_total{bin="1.0"} 22.0 -# HELP feature_64_value_baseline_total Baseline values for feature: State_WV -# TYPE feature_64_value_baseline_total counter -feature_64_value_baseline_total{bin="0.0"} 976.0 -feature_64_value_baseline_total{bin="1.0"} 24.0 -# HELP feature_65_value_baseline_total Baseline values for feature: State_WY -# TYPE feature_65_value_baseline_total counter -feature_65_value_baseline_total{bin="0.0"} 974.0 -feature_65_value_baseline_total{bin="1.0"} 26.0 -# HELP inference_value_baseline Baseline inference values for test set -# TYPE inference_value_baseline histogram -inference_value_baseline_bucket{le="0.00"} 0.0 -inference_value_baseline_bucket{le="0.02"} 85.0 -inference_value_baseline_bucket{le="0.04"} 632.0 -inference_value_baseline_bucket{le="0.06"} 740.0 -inference_value_baseline_bucket{le="0.08"} 778.0 -inference_value_baseline_bucket{le="0.10"} 799.0 -inference_value_baseline_bucket{le="0.12"} 811.0 -inference_value_baseline_bucket{le="0.14"} 820.0 -inference_value_baseline_bucket{le="0.16"} 828.0 -inference_value_baseline_bucket{le="0.18"} 833.0 -inference_value_baseline_bucket{le="0.20"} 841.0 -inference_value_baseline_bucket{le="0.22"} 847.0 -inference_value_baseline_bucket{le="0.24"} 852.0 -inference_value_baseline_bucket{le="0.26"} 854.0 -inference_value_baseline_bucket{le="0.28"} 856.0 -inference_value_baseline_bucket{le="0.30"} 859.0 -inference_value_baseline_bucket{le="0.32"} 862.0 -inference_value_baseline_bucket{le="0.34"} 866.0 -inference_value_baseline_bucket{le="0.36"} 872.0 -inference_value_baseline_bucket{le="0.38"} 872.0 -inference_value_baseline_bucket{le="0.40"} 874.0 -inference_value_baseline_bucket{le="0.42"} 876.0 -inference_value_baseline_bucket{le="0.44"} 878.0 -inference_value_baseline_bucket{le="0.46"} 880.0 -inference_value_baseline_bucket{le="0.48"} 884.0 -inference_value_baseline_bucket{le="0.50"} 885.0 -inference_value_baseline_bucket{le="0.52"} 885.0 -inference_value_baseline_bucket{le="0.54"} 886.0 -inference_value_baseline_bucket{le="0.56"} 887.0 -inference_value_baseline_bucket{le="0.58"} 891.0 -inference_value_baseline_bucket{le="0.60"} 892.0 -inference_value_baseline_bucket{le="0.62"} 894.0 -inference_value_baseline_bucket{le="0.64"} 895.0 -inference_value_baseline_bucket{le="0.66"} 895.0 -inference_value_baseline_bucket{le="0.68"} 897.0 -inference_value_baseline_bucket{le="0.70"} 899.0 -inference_value_baseline_bucket{le="0.72"} 901.0 -inference_value_baseline_bucket{le="0.74"} 902.0 -inference_value_baseline_bucket{le="0.76"} 909.0 -inference_value_baseline_bucket{le="0.78"} 913.0 -inference_value_baseline_bucket{le="0.80"} 917.0 -inference_value_baseline_bucket{le="0.82"} 923.0 -inference_value_baseline_bucket{le="0.84"} 929.0 -inference_value_baseline_bucket{le="0.86"} 937.0 -inference_value_baseline_bucket{le="0.88"} 943.0 -inference_value_baseline_bucket{le="0.90"} 954.0 -inference_value_baseline_bucket{le="0.92"} 964.0 -inference_value_baseline_bucket{le="0.94"} 967.0 -inference_value_baseline_bucket{le="0.96"} 990.0 -inference_value_baseline_bucket{le="0.98"} 1000.0 -inference_value_baseline_bucket{le="1.00"} 1000.0 -inference_value_baseline_bucket{le="+Inf"} 1000.0 -inference_value_baseline_count 1000.0 -inference_value_baseline_sum 144.2123526859723 +# HELP feature_0_value_baseline Baseline values for feature: sepal length (cm) +# TYPE feature_0_value_baseline histogram +feature_0_value_baseline_bucket{le="4.3"} 1.0 +feature_0_value_baseline_bucket{le="4.75"} 7.0 +feature_0_value_baseline_bucket{le="5.2"} 26.0 +feature_0_value_baseline_bucket{le="5.65"} 35.0 +feature_0_value_baseline_bucket{le="6.1"} 52.0 +feature_0_value_baseline_bucket{le="6.550000000000001"} 66.0 +feature_0_value_baseline_bucket{le="7.0"} 80.0 +feature_0_value_baseline_bucket{le="7.45"} 85.0 +feature_0_value_baseline_bucket{le="7.9"} 89.0 +feature_0_value_baseline_bucket{le="+Inf"} 89.0 +feature_0_value_baseline_count 89.0 +feature_0_value_baseline_sum 526.2 +# HELP feature_1_value_baseline Baseline values for feature: sepal width (cm) +# TYPE feature_1_value_baseline histogram +feature_1_value_baseline_bucket{le="2.2"} 1.0 +feature_1_value_baseline_bucket{le="2.3636363636363638"} 5.0 +feature_1_value_baseline_bucket{le="2.5272727272727273"} 12.0 +feature_1_value_baseline_bucket{le="2.690909090909091"} 16.0 +feature_1_value_baseline_bucket{le="2.8545454545454545"} 28.0 +feature_1_value_baseline_bucket{le="3.018181818181818"} 53.0 +feature_1_value_baseline_bucket{le="3.1818181818181817"} 62.0 +feature_1_value_baseline_bucket{le="3.3454545454545457"} 72.0 +feature_1_value_baseline_bucket{le="3.509090909090909"} 82.0 +feature_1_value_baseline_bucket{le="3.672727272727273"} 84.0 +feature_1_value_baseline_bucket{le="3.836363636363636"} 88.0 +feature_1_value_baseline_bucket{le="4.0"} 89.0 +feature_1_value_baseline_bucket{le="+Inf"} 89.0 +feature_1_value_baseline_count 89.0 +feature_1_value_baseline_sum 267.6 +# HELP feature_2_value_baseline Baseline values for feature: petal length (cm) +# TYPE feature_2_value_baseline histogram +feature_2_value_baseline_bucket{le="1.0"} 1.0 +feature_2_value_baseline_bucket{le="1.7375"} 26.0 +feature_2_value_baseline_bucket{le="2.475"} 28.0 +feature_2_value_baseline_bucket{le="3.2125000000000004"} 29.0 +feature_2_value_baseline_bucket{le="3.95"} 35.0 +feature_2_value_baseline_bucket{le="4.6875"} 50.0 +feature_2_value_baseline_bucket{le="5.425000000000001"} 72.0 +feature_2_value_baseline_bucket{le="6.1625000000000005"} 85.0 +feature_2_value_baseline_bucket{le="6.9"} 89.0 +feature_2_value_baseline_bucket{le="+Inf"} 89.0 +feature_2_value_baseline_count 89.0 +feature_2_value_baseline_sum 341.40000000000003 +# HELP feature_3_value_baseline Baseline values for feature: petal width (cm) +# TYPE feature_3_value_baseline histogram +feature_3_value_baseline_bucket{le="0.1"} 3.0 +feature_3_value_baseline_bucket{le="0.4"} 27.0 +feature_3_value_baseline_bucket{le="0.7"} 28.0 +feature_3_value_baseline_bucket{le="0.9999999999999999"} 28.0 +feature_3_value_baseline_bucket{le="1.3"} 45.0 +feature_3_value_baseline_bucket{le="1.6"} 58.0 +feature_3_value_baseline_bucket{le="1.9"} 71.0 +feature_3_value_baseline_bucket{le="2.2"} 81.0 +feature_3_value_baseline_bucket{le="2.5"} 89.0 +feature_3_value_baseline_bucket{le="+Inf"} 89.0 +feature_3_value_baseline_count 89.0 +feature_3_value_baseline_sum 109.5 +# HELP inference_value_baseline_total Baseline inference values for test set +# TYPE inference_value_baseline_total counter +inference_value_baseline_total{bin="setosa"} 22.0 +inference_value_baseline_total{bin="versicolor"} 23.0 +inference_value_baseline_total{bin="virginica"} 16.0 diff --git a/tests/lightgbm/artefact/model.pkl b/tests/lightgbm/artefact/model.pkl old mode 100755 new mode 100644 index dd8601e..a8d2220 Binary files a/tests/lightgbm/artefact/model.pkl and b/tests/lightgbm/artefact/model.pkl differ diff --git a/tests/lightgbm/model-server/requirements.txt b/tests/lightgbm/model-server/requirements.txt index b9ec155..bdd7e94 100644 --- a/tests/lightgbm/model-server/requirements.txt +++ b/tests/lightgbm/model-server/requirements.txt @@ -1,3 +1,3 @@ -bdrk==0.8.2 -boxkite==0.0.4 -lightgbm==2.3.1 +bdrk==0.9.2 +boxkite==0.0.5 +lightgbm==3.3.1 diff --git a/tests/lightgbm/model-server/serve.py b/tests/lightgbm/model-server/serve.py index e6b492a..b697eea 100644 --- a/tests/lightgbm/model-server/serve.py +++ b/tests/lightgbm/model-server/serve.py @@ -9,5 +9,5 @@ def __init__(self, path: Optional[str] = None): with open(path or "/artefact/model.pkl", "rb") as f: self.model = pickle.load(f) - def predict(self, features: List[List[float]]) -> List[float]: - return self.model.predict_proba(features)[:, 0].tolist() + def predict(self, features: List[List[float]]) -> List[str]: + return self.model.predict(features).tolist() diff --git a/tests/lightgbm/model-server/test_serve.py b/tests/lightgbm/model-server/test_serve.py index 86a274b..0872490 100644 --- a/tests/lightgbm/model-server/test_serve.py +++ b/tests/lightgbm/model-server/test_serve.py @@ -6,10 +6,11 @@ class TestModelServer(TestCase): def test_validate(self): - p = Path(__file__).parent.parent / "artefact" / "model.pkl" - m = Model(path=p) - result = m.validate(http_body=str([[i for i in range(66)]])) + path = Path(__file__).parent.parent / "artefact" / "model.pkl" + m = Model(path=path) + features = list(range(m.model.n_features_)) + result = m.validate(http_body=str([features])) self.assertIn("result", result) self.assertIn("prediction_id", result) self.assertEqual(len(result["prediction_id"].split("/")), 3) - self.assertEqual(result["result"], [0.9793770021409441]) + self.assertEqual(result["result"], ["setosa"]) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 4155baa..f4e1f10 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -1,5 +1,5 @@ from os import getenv -from typing import Optional +from typing import Any, Optional from unittest import TestCase, skipIf from requests import Session @@ -8,13 +8,14 @@ MODELS = { "image": ["torchvision", "tf-vision"], "language": ["transformers", "tf-transformers"], - "churn": ["lightgbm"], + "classic": ["lightgbm"], } -def get_inference_count(body: str, label: Optional[int] = None) -> int: +def get_inference_count(body: str, label: Optional[Any] = None) -> int: if label is not None: - start = body.index(f'inference_value_total{{bin="{label}.0"}}') + bin = f"{label:.1f}" if isinstance(label, int) else str(label) + start = body.index(f'inference_value_total{{bin="{bin}"}}') else: start = body.index("inference_value_count") end = body.index("\n", start) @@ -66,7 +67,7 @@ def test_sentiment(self): resp.raise_for_status() before = get_inference_count(resp.text) - for i in range(4): + for _ in range(4): resp = s.post(self.url, json={"query": "Bedrock is amazing!"}) resp.raise_for_status() result = resp.json() @@ -81,24 +82,26 @@ def test_sentiment(self): after = get_inference_count(resp.text) self.assertEqual(after - before, 4) - @skipIf(getenv("MODEL", None) not in MODELS["churn"], "post body for churn prediction models") + @skipIf(getenv("MODEL", None) not in MODELS["classic"], "post body for classic models") def test_post(self): with Session() as s: resp = s.get(f"{self.url}/metrics") resp.raise_for_status() - before = get_inference_count(resp.text) + before = get_inference_count(resp.text, "setosa") - for i in range(4): - resp = s.post(self.url, json=[[i for i in range(66)]]) + for _ in range(4): + # Updated together with tests/lightgbm/artefact/model.pkl + features = list(range(4)) + resp = s.post(self.url, json=[features]) resp.raise_for_status() result = resp.json() self.assertIn("result", result) self.assertIn("prediction_id", result) self.assertEqual(len(result["prediction_id"].split("/")), 3) - self.assertEqual(result["result"], [0.9793770021409441]) + self.assertEqual(result["result"], ["setosa"]) # Verify that metrics sum up correctly resp = s.get(f"{self.url}/metrics") resp.raise_for_status() - after = get_inference_count(resp.text) + after = get_inference_count(resp.text, "setosa") self.assertEqual(after - before, 4) diff --git a/tests/tf-transformers/model-server/requirements.txt b/tests/tf-transformers/model-server/requirements.txt index 3e4d537..d667fd8 100644 --- a/tests/tf-transformers/model-server/requirements.txt +++ b/tests/tf-transformers/model-server/requirements.txt @@ -1 +1 @@ -transformers[tf]==3.0.2 +transformers[tf]==4.15.0 diff --git a/tests/tf-vision/model-server/requirements.txt b/tests/tf-vision/model-server/requirements.txt index bb6524a..898092a 100644 --- a/tests/tf-vision/model-server/requirements.txt +++ b/tests/tf-vision/model-server/requirements.txt @@ -1,3 +1,3 @@ -tensorflow_hub==0.6.0 -tensorflow==2.5.1 -Pillow==8.3.2 +tensorflow-hub==0.12.0 +tensorflow==2.7.0 +Pillow==9.0.0 diff --git a/tests/torchvision/model-server/requirements.txt b/tests/torchvision/model-server/requirements.txt index a33d1d0..da043cb 100644 --- a/tests/torchvision/model-server/requirements.txt +++ b/tests/torchvision/model-server/requirements.txt @@ -1 +1 @@ -torchvision==0.6.1 +torchvision==0.11.2 diff --git a/tests/transformers/model-server/requirements.txt b/tests/transformers/model-server/requirements.txt index bcf7bd1..dffd564 100644 --- a/tests/transformers/model-server/requirements.txt +++ b/tests/transformers/model-server/requirements.txt @@ -1 +1 @@ -transformers[torch]==3.0.2 +transformers[torch]==4.15.0