From 266123224d4fc19eefdac2b085f6c81dff07432b Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Wed, 12 Oct 2022 15:24:19 -0400 Subject: [PATCH 01/36] initial commit for sql workbench windows and macos Signed-off-by: Derek Ho --- .../sql-workbench-test-and-build-workflow.yml | 101 +++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index 7c0e2549e7..d9322feafa 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -15,7 +15,7 @@ env: OPENSEARCH_PLUGIN_VERSION: 2.4.0.0 jobs: - build: + linux-build: runs-on: ubuntu-latest steps: - name: Checkout Plugin @@ -70,3 +70,102 @@ jobs: with: name: workbench path: ../OpenSearch-Dashboards/plugins/workbench/build + + windows-build: + runs-on: windows-latest + steps: + - name: Checkout Plugin + uses: actions/checkout@v3 + # Enable longer filenames for windows + - name: Enable longer filenames + run: git config --system core.longpaths true + + - name: Checkout OpenSearch Dashboards + uses: actions/checkout@v1 # can't update to v3 because `setup-node` fails + with: + repository: opensearch-project/Opensearch-Dashboards + ref: ${{ env.OPENSEARCH_VERSION }} + path: OpenSearch-Dashboards + + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version-file: "../OpenSearch-Dashboards/.nvmrc" + registry-url: 'https://registry.npmjs.org' + + - name: Move Workbench to Plugins Dir + run: | + mv workbench ../OpenSearch-Dashboards/plugins + + - name: OpenSearch Dashboards Plugin Bootstrap + uses: nick-fields/retry@v2 + with: + timeout_minutes: 60 + max_attempts: 3 + command: cd ../OpenSearch-Dashboards/plugins/workbench; yarn osd bootstrap + + - name: Test + run: | + cd ../OpenSearch-Dashboards/plugins/workbench + yarn test:jest + + - name: Build Artifact + run: | + cd ../OpenSearch-Dashboards/plugins/workbench + yarn build + mv ./build/*.zip ./build/${{ env.PLUGIN_NAME }}-${{ env.OPENSEARCH_PLUGIN_VERSION }}.zip + + - name: Upload Artifact + if: always() + uses: actions/upload-artifact@v1 # can't update to v3 because upload fails + with: + name: workbench + path: ../OpenSearch-Dashboards/plugins/workbench/build + + macos-build: + runs-on: macos-latest + steps: + - name: Checkout Plugin + uses: actions/checkout@v3 + + - name: Checkout OpenSearch Dashboards + uses: actions/checkout@v1 # can't update to v3 because `setup-node` fails + with: + repository: opensearch-project/Opensearch-Dashboards + ref: ${{ env.OPENSEARCH_VERSION }} + path: OpenSearch-Dashboards + + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version-file: "../OpenSearch-Dashboards/.nvmrc" + registry-url: 'https://registry.npmjs.org' + + - name: Move Workbench to Plugins Dir + run: | + mv workbench ../OpenSearch-Dashboards/plugins + + - name: OpenSearch Dashboards Plugin Bootstrap + uses: nick-fields/retry@v2 + with: + timeout_minutes: 60 + max_attempts: 3 + command: cd ../OpenSearch-Dashboards/plugins/workbench; yarn osd bootstrap + + - name: Test + run: | + cd ../OpenSearch-Dashboards/plugins/workbench + yarn test:jest + + - name: Build Artifact + run: | + cd ../OpenSearch-Dashboards/plugins/workbench + yarn build + mv ./build/*.zip ./build/${{ env.PLUGIN_NAME }}-${{ env.OPENSEARCH_PLUGIN_VERSION }}.zip + + - name: Upload Artifact + if: always() + uses: actions/upload-artifact@v1 # can't update to v3 because upload fails + with: + name: workbench + path: ../OpenSearch-Dashboards/plugins/workbench/build \ No newline at end of file From 7a1c7aead9bc8b7922cc00064193ebb6298b1122 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Wed, 12 Oct 2022 15:29:08 -0400 Subject: [PATCH 02/36] name files Signed-off-by: Derek Ho --- .github/workflows/sql-workbench-test-and-build-workflow.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index d9322feafa..0b5ef5d34c 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -68,7 +68,7 @@ jobs: if: always() uses: actions/upload-artifact@v1 # can't update to v3 because upload fails with: - name: workbench + name: workbench-linux path: ../OpenSearch-Dashboards/plugins/workbench/build windows-build: @@ -119,7 +119,7 @@ jobs: if: always() uses: actions/upload-artifact@v1 # can't update to v3 because upload fails with: - name: workbench + name: workbench-windows path: ../OpenSearch-Dashboards/plugins/workbench/build macos-build: @@ -167,5 +167,5 @@ jobs: if: always() uses: actions/upload-artifact@v1 # can't update to v3 because upload fails with: - name: workbench + name: workbench-macos path: ../OpenSearch-Dashboards/plugins/workbench/build \ No newline at end of file From ef535e9db81425baa1aa32e65d617eaf757a5446 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 13 Oct 2022 11:03:32 -0400 Subject: [PATCH 03/36] clean up code with matrix Signed-off-by: Derek Ho --- .../sql-workbench-test-and-build-workflow.yml | 114 ++---------------- 1 file changed, 11 insertions(+), 103 deletions(-) diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index 0b5ef5d34c..a80425a1f2 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -15,9 +15,16 @@ env: OPENSEARCH_PLUGIN_VERSION: 2.4.0.0 jobs: - linux-build: - runs-on: ubuntu-latest + build: + strategy: + matrix: + os: [ubuntu-latest, windows-latest-macos-latest] + runs-on: ${{ matrix.os }} steps: + - name: Enable longer filenames + if: ${{ matrix.os == 'windows-latest' }} + run: git config --system core.longpaths true + - name: Checkout Plugin uses: actions/checkout@v3 @@ -51,7 +58,7 @@ jobs: yarn test:jest --coverage - name: Upload coverage - if: always() + if: ${{ matrix.os == 'ubuntu-latest' }} uses: codecov/codecov-action@v3 with: flags: query-workbench @@ -68,104 +75,5 @@ jobs: if: always() uses: actions/upload-artifact@v1 # can't update to v3 because upload fails with: - name: workbench-linux - path: ../OpenSearch-Dashboards/plugins/workbench/build - - windows-build: - runs-on: windows-latest - steps: - - name: Checkout Plugin - uses: actions/checkout@v3 - # Enable longer filenames for windows - - name: Enable longer filenames - run: git config --system core.longpaths true - - - name: Checkout OpenSearch Dashboards - uses: actions/checkout@v1 # can't update to v3 because `setup-node` fails - with: - repository: opensearch-project/Opensearch-Dashboards - ref: ${{ env.OPENSEARCH_VERSION }} - path: OpenSearch-Dashboards - - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version-file: "../OpenSearch-Dashboards/.nvmrc" - registry-url: 'https://registry.npmjs.org' - - - name: Move Workbench to Plugins Dir - run: | - mv workbench ../OpenSearch-Dashboards/plugins - - - name: OpenSearch Dashboards Plugin Bootstrap - uses: nick-fields/retry@v2 - with: - timeout_minutes: 60 - max_attempts: 3 - command: cd ../OpenSearch-Dashboards/plugins/workbench; yarn osd bootstrap - - - name: Test - run: | - cd ../OpenSearch-Dashboards/plugins/workbench - yarn test:jest - - - name: Build Artifact - run: | - cd ../OpenSearch-Dashboards/plugins/workbench - yarn build - mv ./build/*.zip ./build/${{ env.PLUGIN_NAME }}-${{ env.OPENSEARCH_PLUGIN_VERSION }}.zip - - - name: Upload Artifact - if: always() - uses: actions/upload-artifact@v1 # can't update to v3 because upload fails - with: - name: workbench-windows - path: ../OpenSearch-Dashboards/plugins/workbench/build - - macos-build: - runs-on: macos-latest - steps: - - name: Checkout Plugin - uses: actions/checkout@v3 - - - name: Checkout OpenSearch Dashboards - uses: actions/checkout@v1 # can't update to v3 because `setup-node` fails - with: - repository: opensearch-project/Opensearch-Dashboards - ref: ${{ env.OPENSEARCH_VERSION }} - path: OpenSearch-Dashboards - - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version-file: "../OpenSearch-Dashboards/.nvmrc" - registry-url: 'https://registry.npmjs.org' - - - name: Move Workbench to Plugins Dir - run: | - mv workbench ../OpenSearch-Dashboards/plugins - - - name: OpenSearch Dashboards Plugin Bootstrap - uses: nick-fields/retry@v2 - with: - timeout_minutes: 60 - max_attempts: 3 - command: cd ../OpenSearch-Dashboards/plugins/workbench; yarn osd bootstrap - - - name: Test - run: | - cd ../OpenSearch-Dashboards/plugins/workbench - yarn test:jest - - - name: Build Artifact - run: | - cd ../OpenSearch-Dashboards/plugins/workbench - yarn build - mv ./build/*.zip ./build/${{ env.PLUGIN_NAME }}-${{ env.OPENSEARCH_PLUGIN_VERSION }}.zip - - - name: Upload Artifact - if: always() - uses: actions/upload-artifact@v1 # can't update to v3 because upload fails - with: - name: workbench-macos + name: workbench-${{ matrix.os }} path: ../OpenSearch-Dashboards/plugins/workbench/build \ No newline at end of file From 8a58eb9299d2420bd7319d3af2b4cea80e09e1a7 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 13 Oct 2022 11:06:14 -0400 Subject: [PATCH 04/36] forgot comma Signed-off-by: Derek Ho --- .github/workflows/sql-workbench-test-and-build-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index a80425a1f2..3878872512 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -18,7 +18,7 @@ jobs: build: strategy: matrix: - os: [ubuntu-latest, windows-latest-macos-latest] + os: [ubuntu-latest, windows-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - name: Enable longer filenames From 828101c062f54660797e526e09dd488c97f8e4b3 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 13 Oct 2022 16:22:00 -0400 Subject: [PATCH 05/36] use cross env for windows and enable windows and macos ci in sql Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 12 ++++++------ workbench/package.json | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index b70092713e..3b8fd1b81e 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -21,10 +21,9 @@ jobs: build: strategy: matrix: - java: - - 11 - - 17 - runs-on: ubuntu-latest + java: [11, 17] + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -39,6 +38,7 @@ jobs: run: ./gradlew --continue build assemble - name: Run backward compatibility tests + if: ${{ matrix.os == 'ubuntu-latest' }} run: ./scripts/bwctest.sh - name: Create Artifact Path @@ -48,7 +48,7 @@ jobs: # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action - name: Upload SQL Coverage Report - if: always() + if: ${{ matrix.os == 'ubuntu-latest' }} uses: codecov/codecov-action@v3 with: flags: sql-engine @@ -61,7 +61,7 @@ jobs: path: opensearch-sql-builds - name: Upload test reports - if: always() + if: ${{ matrix.os == 'ubuntu-latest' }} uses: actions/upload-artifact@v2 with: name: test-reports diff --git a/workbench/package.json b/workbench/package.json index aebf11b285..9a9b9706f6 100644 --- a/workbench/package.json +++ b/workbench/package.json @@ -16,7 +16,7 @@ "start": "plugin-helpers start", "test:server": "plugin-helpers test:server", "test:browser": "plugin-helpers test:browser", - "test:jest": "NODE_PATH=../../node_modules ../../node_modules/.bin/jest --config ./test/jest.config.js", + "test:jest": "cross-env NODE_PATH=../../node_modules ../../node_modules/.bin/jest --config ./test/jest.config.js", "build": "yarn plugin_helpers build", "plugin_helpers": "node ../../scripts/plugin_helpers" }, @@ -28,6 +28,7 @@ "@testing-library/user-event": "^13.1.9", "@types/hapi-latest": "npm:@types/hapi@18.0.3", "@types/react-router-dom": "^5.3.2", + "cross-env": "7.0.3", "cypress": "^5.0.0", "eslint": "^6.8.0", "eslint-plugin-no-unsanitized": "^3.0.2", From 53ef534870c8636b0d31caaf951a95dde0d3bf21 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 13 Oct 2022 16:41:07 -0400 Subject: [PATCH 06/36] disable integration and jacoco for windows and mac Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 3b8fd1b81e..014c71a400 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -19,10 +19,17 @@ on: jobs: build: + env: + BUILD_ARGS: ${{ matrix.os_build_args }} strategy: matrix: java: [11, 17] os: [ubuntu-latest, windows-latest, macos-latest] + include: + - os: windows-latest + os_build_args: -x integTest -x jacocoTestReport + - os: macos-latest + os_build_args: -x integTest -x jacocoTestReport runs-on: ${{ matrix.os }} steps: @@ -35,7 +42,7 @@ jobs: java-version: ${{ matrix.java }} - name: Build with Gradle - run: ./gradlew --continue build assemble + run: ./gradlew --continue build assemble ${{ env.BUILD_ARGS }} - name: Run backward compatibility tests if: ${{ matrix.os == 'ubuntu-latest' }} From 7569d6d7cfe03d35660da1aec31e41582c80eb1f Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 13 Oct 2022 16:55:51 -0400 Subject: [PATCH 07/36] add gitattributes file to normalize line endings Signed-off-by: Derek Ho --- .gitattributes | 1 + .github/workflows/sql-test-and-build-workflow.yml | 11 +++-------- 2 files changed, 4 insertions(+), 8 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..176a458f94 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 014c71a400..b5b5a35819 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -19,17 +19,12 @@ on: jobs: build: - env: - BUILD_ARGS: ${{ matrix.os_build_args }} strategy: + # Run all jobs + fail-fast: false matrix: java: [11, 17] os: [ubuntu-latest, windows-latest, macos-latest] - include: - - os: windows-latest - os_build_args: -x integTest -x jacocoTestReport - - os: macos-latest - os_build_args: -x integTest -x jacocoTestReport runs-on: ${{ matrix.os }} steps: @@ -42,7 +37,7 @@ jobs: java-version: ${{ matrix.java }} - name: Build with Gradle - run: ./gradlew --continue build assemble ${{ env.BUILD_ARGS }} + run: ./gradlew --continue build assemble - name: Run backward compatibility tests if: ${{ matrix.os == 'ubuntu-latest' }} From 912731b8edafaaaa6b79400e7213d2565f4a1933 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 13 Oct 2022 17:06:05 -0400 Subject: [PATCH 08/36] move gitattributes into appropriate folder Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 2 +- .gitattributes => sql/.gitattributes | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename .gitattributes => sql/.gitattributes (100%) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index b5b5a35819..f0b9b2705b 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -59,7 +59,7 @@ jobs: - name: Upload Artifacts uses: actions/upload-artifact@v2 with: - name: opensearch-sql + name: opensearch-sql-${{ matrix.os }} path: opensearch-sql-builds - name: Upload test reports diff --git a/.gitattributes b/sql/.gitattributes similarity index 100% rename from .gitattributes rename to sql/.gitattributes From 6a6cfd6602afe2b752fd1e7950c832bac79f9598 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 13 Oct 2022 17:09:16 -0400 Subject: [PATCH 09/36] configure line endings for windows Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index f0b9b2705b..f6c0ed142f 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -29,6 +29,10 @@ jobs: steps: - uses: actions/checkout@v3 + + - name: Change line endings for windows + if: ${{ matrix.os == 'windows-latest' }} + run: git config --global core.autocrlf false - name: Set up JDK ${{ matrix.java }} uses: actions/setup-java@v3 From 0d016efe7b7e40373a89365746596e528c739295 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Fri, 14 Oct 2022 17:15:58 -0400 Subject: [PATCH 10/36] fix one test file and get rid of git attributes Signed-off-by: Derek Ho --- .../format/RawResponseFormatterTest.java | 38 +++++++++---------- sql/.gitattributes | 1 - 2 files changed, 19 insertions(+), 20 deletions(-) delete mode 100644 sql/.gitattributes diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java index 87d2d6f57f..d8e06f81ab 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java @@ -36,7 +36,7 @@ void formatResponse() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("name", "John", "age", 20)), tupleValue(ImmutableMap.of("name", "Smith", "age", 30)))); - String expected = "name|age\nJohn|20\nSmith|30"; + String expected = "name|age%nJohn|20%nSmith|30"; assertEquals(expected, rawFormater.format(response)); } @@ -50,7 +50,7 @@ void sanitizeHeaders() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of( "=firstname", "John", "+lastname", "Smith", "-city", "Seattle", "@age", 20)))); - String expected = "=firstname|+lastname|-city|@age\n" + String expected = "=firstname|+lastname|-city|@age%n" + "John|Smith|Seattle|20"; assertEquals(expected, rawFormater.format(response)); } @@ -66,12 +66,12 @@ void sanitizeData() { tupleValue(ImmutableMap.of("city", "-Seattle")), tupleValue(ImmutableMap.of("city", "@Seattle")), tupleValue(ImmutableMap.of("city", "Seattle=")))); - String expected = "city\n" - + "Seattle\n" - + "=Seattle\n" - + "+Seattle\n" - + "-Seattle\n" - + "@Seattle\n" + String expected = "city%n" + + "Seattle%n" + + "=Seattle%n" + + "+Seattle%n" + + "-Seattle%n" + + "@Seattle%n" + "Seattle="; assertEquals(expected, rawFormater.format(response)); } @@ -83,7 +83,7 @@ void quoteIfRequired() { new ExecutionEngine.Schema.Column("||age", "||age", INTEGER))); QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("na|me", "John|Smith", "||age", "30|||")))); - String expected = "\"na|me\"|\"||age\"\n" + String expected = "\"na|me\"|\"||age\"%n" + "\"John|Smith\"|\"30|||\""; assertEquals(expected, rawFormater.format(response)); } @@ -92,7 +92,7 @@ void quoteIfRequired() { void formatError() { Throwable t = new RuntimeException("This is an exception"); String expected = - "{\n \"type\": \"RuntimeException\",\n \"reason\": \"This is an exception\"\n}"; + "{%n \"type\": \"RuntimeException\",%n \"reason\": \"This is an exception\"%n}"; assertEquals(expected, rawFormater.format(t)); } @@ -104,8 +104,8 @@ void escapeSanitize() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("city", "=Seattle")), tupleValue(ImmutableMap.of("city", "||Seattle")))); - String expected = "city\n" - + "=Seattle\n" + String expected = "city%n" + + "=Seattle%n" + "\"||Seattle\""; assertEquals(expected, escapeFormatter.format(response)); } @@ -117,8 +117,8 @@ void senstiveCharater() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("city", "@Seattle")), tupleValue(ImmutableMap.of("city", "++Seattle")))); - String expected = "city\n" - + "@Seattle\n" + String expected = "city%n" + + "@Seattle%n" + "++Seattle"; assertEquals(expected, rawFormater.format(response)); } @@ -131,8 +131,8 @@ void senstiveCharaterWithSanitize() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("city", "@Seattle")), tupleValue(ImmutableMap.of("city", "++Seattle|||")))); - String expected = "city\n" - + "@Seattle\n" + String expected = "city%n" + + "@Seattle%n" + "\"++Seattle|||\""; assertEquals(expected, testFormater.format(response)); } @@ -148,9 +148,9 @@ void replaceNullValues() { ImmutableMap.of("firstname", LITERAL_NULL, "city", stringValue("Seattle"))), ExprTupleValue.fromExprValueMap( ImmutableMap.of("firstname", stringValue("John"), "city", LITERAL_MISSING)))); - String expected = "name|city\n" - + "John|Seattle\n" - + "|Seattle\n" + String expected = "name|city%n" + + "John|Seattle%n" + + "|Seattle%n" + "John|"; assertEquals(expected, rawFormater.format(response)); } diff --git a/sql/.gitattributes b/sql/.gitattributes deleted file mode 100644 index 176a458f94..0000000000 --- a/sql/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -* text=auto From a8709d2a885354a76f56a8104376ca3e1392ac39 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Fri, 14 Oct 2022 17:17:34 -0400 Subject: [PATCH 11/36] disable doctest and integ test Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index f6c0ed142f..d467741260 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -19,12 +19,17 @@ on: jobs: build: + env: + BUILD_ARGS: ${{ matrix.os_build_args }} strategy: # Run all jobs fail-fast: false matrix: java: [11, 17] os: [ubuntu-latest, windows-latest, macos-latest] + include: + - os: windows-latest + os_build_args: -x doctest -x integTest -x jacocoTestReport runs-on: ${{ matrix.os }} steps: @@ -41,7 +46,7 @@ jobs: java-version: ${{ matrix.java }} - name: Build with Gradle - run: ./gradlew --continue build assemble + run: ./gradlew --continue build assemble ${{ env.BUILD_ARGS }} - name: Run backward compatibility tests if: ${{ matrix.os == 'ubuntu-latest' }} From d4c9f1e0f80a7d2495fbd896d2ac09baece2b3a7 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Mon, 17 Oct 2022 10:09:30 -0400 Subject: [PATCH 12/36] fix up tests Signed-off-by: Derek Ho --- .../workflows/sql-test-and-build-workflow.yml | 10 +- .../AggregationQueryBuilderTest.java | 693 +++++++++--------- .../dsl/MetricAggregationBuilderTest.java | 247 +++---- .../response/format/ErrorFormatter.java | 6 +- .../format/CsvResponseFormatterTest.java | 45 +- .../response/format/ErrorFormatterTest.java | 3 +- .../format/RawResponseFormatterTest.java | 17 +- .../SimpleJsonResponseFormatterTest.java | 61 +- 8 files changed, 544 insertions(+), 538 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index d467741260..057e4ee98b 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -29,15 +29,13 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] include: - os: windows-latest - os_build_args: -x doctest -x integTest -x jacocoTestReport + os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc + - os: macos-latest + os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - - - name: Change line endings for windows - if: ${{ matrix.os == 'windows-latest' }} - run: git config --global core.autocrlf false - name: Set up JDK ${{ matrix.java }} uses: actions/setup-java@v3 @@ -46,7 +44,7 @@ jobs: java-version: ${{ matrix.java }} - name: Build with Gradle - run: ./gradlew --continue build assemble ${{ env.BUILD_ARGS }} + run: ./gradlew --continue build ${{ env.BUILD_ARGS }} - name: Run backward compatibility tests if: ${{ matrix.os == 'ubuntu-latest' }} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java index 04aedc0f01..3614d82e59 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java @@ -12,6 +12,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; +import static org.opensearch.sql.common.utils.StringUtils.format; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; @@ -72,31 +73,31 @@ void set_up() { @Test void should_build_composite_aggregation_for_field_reference() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"name\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"name\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(age)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"name\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"name\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(age)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("avg(age)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER))), @@ -105,31 +106,31 @@ void should_build_composite_aggregation_for_field_reference() { @Test void should_build_composite_aggregation_for_field_reference_with_order() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"name\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"name\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"last\",\n" - + " \"order\" : \"desc\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(age)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"name\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"name\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"last\",%n" + + " \"order\" : \"desc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(age)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("avg(age)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER))), @@ -152,31 +153,31 @@ void should_build_type_mapping_for_field_reference() { @Test void should_build_composite_aggregation_for_field_reference_of_keyword() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"name\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"name.keyword\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(age)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"name\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"name.keyword\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(age)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("avg(age)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER))), @@ -201,37 +202,37 @@ void should_build_composite_aggregation_for_expression() { Expression expr = invocation.getArgument(0); return expr.toString(); }).when(serializer).serialize(any()); - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"age\" : {\n" - + " \"terms\" : {\n" - + " \"script\" : {\n" - + " \"source\" : \"asin(age)\",\n" - + " \"lang\" : \"opensearch_query_expression\"\n" - + " },\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(balance)\" : {\n" - + " \"avg\" : {\n" - + " \"script\" : {\n" - + " \"source\" : \"abs(balance)\",\n" - + " \"lang\" : \"opensearch_query_expression\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"age\" : {%n" + + " \"terms\" : {%n" + + " \"script\" : {%n" + + " \"source\" : \"asin(age)\",%n" + + " \"lang\" : \"opensearch_query_expression\"%n" + + " },%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(balance)\" : {%n" + + " \"avg\" : {%n" + + " \"script\" : {%n" + + " \"source\" : \"abs(balance)\",%n" + + " \"lang\" : \"opensearch_query_expression\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("avg(balance)", new AvgAggregator( @@ -241,40 +242,40 @@ void should_build_composite_aggregation_for_expression() { @Test void should_build_composite_aggregation_follow_with_order_by_position() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"name\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"name\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"last\",\n" - + " \"order\" : \"desc\"\n" - + " }\n" - + " }\n" - + " }, {\n" - + " \"age\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"age\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(balance)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"balance\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"name\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"name\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"last\",%n" + + " \"order\" : \"desc\"%n" + + " }%n" + + " }%n" + + " }, {%n" + + " \"age\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"age\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(balance)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"balance\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( agg(named("avg(balance)", avg(ref("balance", INTEGER), INTEGER))), group(named("age", ref("age", INTEGER)), named("name", ref("name", STRING))), @@ -298,14 +299,14 @@ void should_build_type_mapping_for_expression() { @Test void should_build_aggregation_without_bucket() { - assertEquals( - "{\n" - + " \"avg(balance)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"balance\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"avg(balance)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"balance\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("avg(balance)", new AvgAggregator( @@ -315,29 +316,29 @@ void should_build_aggregation_without_bucket() { @Test void should_build_filter_aggregation() { - assertEquals( - "{\n" - + " \"avg(age) filter(where age > 34)\" : {\n" - + " \"filter\" : {\n" - + " \"range\" : {\n" - + " \"age\" : {\n" - + " \"from\" : 20,\n" - + " \"to\" : null,\n" - + " \"include_lower\" : false,\n" - + " \"include_upper\" : true,\n" - + " \"boost\" : 1.0\n" - + " }\n" - + " }\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(age) filter(where age > 34)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"avg(age) filter(where age > 34)\" : {%n" + + " \"filter\" : {%n" + + " \"range\" : {%n" + + " \"age\" : {%n" + + " \"from\" : 20,%n" + + " \"to\" : null,%n" + + " \"include_lower\" : false,%n" + + " \"include_upper\" : true,%n" + + " \"boost\" : 1.0%n" + + " }%n" + + " }%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(age) filter(where age > 34)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList(named("avg(age) filter(where age > 34)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER) @@ -347,46 +348,46 @@ void should_build_filter_aggregation() { @Test void should_build_filter_aggregation_group_by() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"gender\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"gender\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(age) filter(where age > 34)\" : {\n" - + " \"filter\" : {\n" - + " \"range\" : {\n" - + " \"age\" : {\n" - + " \"from\" : 20,\n" - + " \"to\" : null,\n" - + " \"include_lower\" : false,\n" - + " \"include_upper\" : true,\n" - + " \"boost\" : 1.0\n" - + " }\n" - + " }\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"avg(age) filter(where age > 34)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"gender\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"gender\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(age) filter(where age > 34)\" : {%n" + + " \"filter\" : {%n" + + " \"range\" : {%n" + + " \"age\" : {%n" + + " \"from\" : 20,%n" + + " \"to\" : null,%n" + + " \"include_lower\" : false,%n" + + " \"include_upper\" : true,%n" + + " \"boost\" : 1.0%n" + + " }%n" + + " }%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"avg(age) filter(where age > 34)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList(named("avg(age) filter(where age > 34)", new AvgAggregator(Arrays.asList(ref("age", INTEGER)), INTEGER) @@ -408,32 +409,32 @@ void should_build_type_mapping_without_bucket() { @Test void should_build_histogram() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"SpanExpression(field=age, value=10, unit=NONE)\" : {\n" - + " \"histogram\" : {\n" - + " \"field\" : \"age\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\",\n" - + " \"interval\" : 10.0\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"count(a)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"a\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=age, value=10, unit=NONE)\" : {%n" + + " \"histogram\" : {%n" + + " \"field\" : \"age\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\",%n" + + " \"interval\" : 10.0%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"count(a)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"a\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(a)", new CountAggregator(Arrays.asList(ref("a", INTEGER)), INTEGER))), @@ -442,37 +443,37 @@ void should_build_histogram() { @Test void should_build_histogram_two_metrics() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"SpanExpression(field=age, value=10, unit=NONE)\" : {\n" - + " \"histogram\" : {\n" - + " \"field\" : \"age\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\",\n" - + " \"interval\" : 10.0\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"count(a)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"a\"\n" - + " }\n" - + " },\n" - + " \"avg(b)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"b\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=age, value=10, unit=NONE)\" : {%n" + + " \"histogram\" : {%n" + + " \"field\" : \"age\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\",%n" + + " \"interval\" : 10.0%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"count(a)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"a\"%n" + + " }%n" + + " },%n" + + " \"avg(b)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"b\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(a)", new CountAggregator(Arrays.asList(ref("a", INTEGER)), INTEGER)), @@ -482,32 +483,32 @@ void should_build_histogram_two_metrics() { @Test void fixed_interval_time_span() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"SpanExpression(field=timestamp, value=1, unit=H)\" : {\n" - + " \"date_histogram\" : {\n" - + " \"field\" : \"timestamp\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\",\n" - + " \"fixed_interval\" : \"1h\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"count(a)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"a\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=timestamp, value=1, unit=H)\" : {%n" + + " \"date_histogram\" : {%n" + + " \"field\" : \"timestamp\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\",%n" + + " \"fixed_interval\" : \"1h\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"count(a)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"a\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(a)", new CountAggregator(Arrays.asList(ref("a", INTEGER)), INTEGER))), @@ -516,32 +517,32 @@ void fixed_interval_time_span() { @Test void calendar_interval_time_span() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"SpanExpression(field=date, value=1, unit=W)\" : {\n" - + " \"date_histogram\" : {\n" - + " \"field\" : \"date\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\",\n" - + " \"calendar_interval\" : \"1w\"\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"count(a)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"a\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=date, value=1, unit=W)\" : {%n" + + " \"date_histogram\" : {%n" + + " \"field\" : \"date\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\",%n" + + " \"calendar_interval\" : \"1w\"%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"count(a)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"a\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(a)", new CountAggregator(Arrays.asList(ref("a", INTEGER)), INTEGER))), @@ -550,32 +551,32 @@ void calendar_interval_time_span() { @Test void general_span() { - assertEquals( - "{\n" - + " \"composite_buckets\" : {\n" - + " \"composite\" : {\n" - + " \"size\" : 1000,\n" - + " \"sources\" : [ {\n" - + " \"SpanExpression(field=age, value=1, unit=NONE)\" : {\n" - + " \"histogram\" : {\n" - + " \"field\" : \"age\",\n" - + " \"missing_bucket\" : true,\n" - + " \"missing_order\" : \"first\",\n" - + " \"order\" : \"asc\",\n" - + " \"interval\" : 1.0\n" - + " }\n" - + " }\n" - + " } ]\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"count(a)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"a\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=age, value=1, unit=NONE)\" : {%n" + + " \"histogram\" : {%n" + + " \"field\" : \"age\",%n" + + " \"missing_bucket\" : true,%n" + + " \"missing_order\" : \"first\",%n" + + " \"order\" : \"asc\",%n" + + " \"interval\" : 1.0%n" + + " }%n" + + " }%n" + + " } ]%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"count(a)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"a\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(a)", new CountAggregator(Arrays.asList(ref("a", INTEGER)), INTEGER))), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java index 845e32ba83..5161b35021 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java @@ -9,6 +9,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.when; +import static org.opensearch.sql.common.utils.StringUtils.format; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.DSL.literal; @@ -62,14 +63,14 @@ void set_up() { @Test void should_build_avg_aggregation() { - assertEquals( - "{\n" - + " \"avg(age)\" : {\n" - + " \"avg\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"avg(age)\" : {%n" + + " \"avg\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("avg(age)", @@ -78,14 +79,14 @@ void should_build_avg_aggregation() { @Test void should_build_sum_aggregation() { - assertEquals( - "{\n" - + " \"sum(age)\" : {\n" - + " \"sum\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"sum(age)\" : {%n" + + " \"sum\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("sum(age)", @@ -94,14 +95,14 @@ void should_build_sum_aggregation() { @Test void should_build_count_aggregation() { - assertEquals( - "{\n" - + " \"count(age)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"count(age)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(age)", @@ -110,14 +111,14 @@ void should_build_count_aggregation() { @Test void should_build_count_star_aggregation() { - assertEquals( - "{\n" - + " \"count(*)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"_index\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"count(*)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"_index\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(*)", @@ -126,14 +127,14 @@ void should_build_count_star_aggregation() { @Test void should_build_count_other_literal_aggregation() { - assertEquals( - "{\n" - + " \"count(1)\" : {\n" - + " \"value_count\" : {\n" - + " \"field\" : \"_index\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"count(1)\" : {%n" + + " \"value_count\" : {%n" + + " \"field\" : \"_index\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("count(1)", @@ -142,14 +143,14 @@ void should_build_count_other_literal_aggregation() { @Test void should_build_min_aggregation() { - assertEquals( - "{\n" - + " \"min(age)\" : {\n" - + " \"min\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"min(age)\" : {%n" + + " \"min\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("min(age)", @@ -158,14 +159,14 @@ void should_build_min_aggregation() { @Test void should_build_max_aggregation() { - assertEquals( - "{\n" - + " \"max(age)\" : {\n" - + " \"max\" : {\n" - + " \"field\" : \"age\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"max(age)\" : {%n" + + " \"max\" : {%n" + + " \"field\" : \"age\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("max(age)", @@ -174,15 +175,15 @@ void should_build_max_aggregation() { @Test void should_build_varPop_aggregation() { - assertEquals( - "{\n" - + " \"var_pop(age)\" : {\n" - + " \"extended_stats\" : {\n" - + " \"field\" : \"age\",\n" - + " \"sigma\" : 2.0\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"var_pop(age)\" : {%n" + + " \"extended_stats\" : {%n" + + " \"field\" : \"age\",%n" + + " \"sigma\" : 2.0%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("var_pop(age)", @@ -191,15 +192,15 @@ void should_build_varPop_aggregation() { @Test void should_build_varSamp_aggregation() { - assertEquals( - "{\n" - + " \"var_samp(age)\" : {\n" - + " \"extended_stats\" : {\n" - + " \"field\" : \"age\",\n" - + " \"sigma\" : 2.0\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"var_samp(age)\" : {%n" + + " \"extended_stats\" : {%n" + + " \"field\" : \"age\",%n" + + " \"sigma\" : 2.0%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("var_samp(age)", @@ -208,15 +209,15 @@ void should_build_varSamp_aggregation() { @Test void should_build_stddevPop_aggregation() { - assertEquals( - "{\n" - + " \"stddev_pop(age)\" : {\n" - + " \"extended_stats\" : {\n" - + " \"field\" : \"age\",\n" - + " \"sigma\" : 2.0\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"stddev_pop(age)\" : {%n" + + " \"extended_stats\" : {%n" + + " \"field\" : \"age\",%n" + + " \"sigma\" : 2.0%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("stddev_pop(age)", @@ -225,15 +226,15 @@ void should_build_stddevPop_aggregation() { @Test void should_build_stddevSamp_aggregation() { - assertEquals( - "{\n" - + " \"stddev_samp(age)\" : {\n" - + " \"extended_stats\" : {\n" - + " \"field\" : \"age\",\n" - + " \"sigma\" : 2.0\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"stddev_samp(age)\" : {%n" + + " \"extended_stats\" : {%n" + + " \"field\" : \"age\",%n" + + " \"sigma\" : 2.0%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Arrays.asList( named("stddev_samp(age)", @@ -242,14 +243,14 @@ void should_build_stddevSamp_aggregation() { @Test void should_build_cardinality_aggregation() { - assertEquals( - "{\n" - + " \"count(distinct name)\" : {\n" - + " \"cardinality\" : {\n" - + " \"field\" : \"name\"\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"count(distinct name)\" : {%n" + + " \"cardinality\" : {%n" + + " \"field\" : \"name\"%n" + + " }%n" + + " }%n" + + "}"), buildQuery( Collections.singletonList(named("count(distinct name)", new CountAggregator( Collections.singletonList(ref("name", STRING)), INTEGER).distinct(true))))); @@ -257,29 +258,29 @@ void should_build_cardinality_aggregation() { @Test void should_build_filtered_cardinality_aggregation() { - assertEquals( - "{\n" - + " \"count(distinct name) filter(where age > 30)\" : {\n" - + " \"filter\" : {\n" - + " \"range\" : {\n" - + " \"age\" : {\n" - + " \"from\" : 30,\n" - + " \"to\" : null,\n" - + " \"include_lower\" : false,\n" - + " \"include_upper\" : true,\n" - + " \"boost\" : 1.0\n" - + " }\n" - + " }\n" - + " },\n" - + " \"aggregations\" : {\n" - + " \"count(distinct name) filter(where age > 30)\" : {\n" - + " \"cardinality\" : {\n" - + " \"field\" : \"name\"\n" - + " }\n" - + " }\n" - + " }\n" - + " }\n" - + "}", + assertEquals(format( + "{%n" + + " \"count(distinct name) filter(where age > 30)\" : {%n" + + " \"filter\" : {%n" + + " \"range\" : {%n" + + " \"age\" : {%n" + + " \"from\" : 30,%n" + + " \"to\" : null,%n" + + " \"include_lower\" : false,%n" + + " \"include_upper\" : true,%n" + + " \"boost\" : 1.0%n" + + " }%n" + + " }%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"count(distinct name) filter(where age > 30)\" : {%n" + + " \"cardinality\" : {%n" + + " \"field\" : \"name\"%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), buildQuery(Collections.singletonList(named( "count(distinct name) filter(where age > 30)", new CountAggregator(Collections.singletonList(ref("name", STRING)), INTEGER) diff --git a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java index 40848e959b..e21e731a2d 100644 --- a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java +++ b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java @@ -45,12 +45,14 @@ public static String prettyFormat(Throwable t) { public static String compactJsonify(Object jsonObject) { return AccessController.doPrivileged( - (PrivilegedAction) () -> GSON.toJson(jsonObject)); + (PrivilegedAction) () -> GSON.toJson(jsonObject) + .replaceAll("\\n|\\r\\n", System.getProperty("line.separator"))); } public static String prettyJsonify(Object jsonObject) { return AccessController.doPrivileged( - (PrivilegedAction) () -> PRETTY_PRINT_GSON.toJson(jsonObject)); + (PrivilegedAction) () -> PRETTY_PRINT_GSON.toJson(jsonObject) + .replaceAll("\\n|\\r\\n", System.getProperty("line.separator"))); } @RequiredArgsConstructor diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java index 8998086afc..475cb30946 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java @@ -7,6 +7,7 @@ package org.opensearch.sql.protocol.response.format; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.common.utils.StringUtils.format; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; @@ -37,8 +38,8 @@ void formatResponse() { tupleValue(ImmutableMap.of("name", "John", "age", 20)), tupleValue(ImmutableMap.of("name", "Smith", "age", 30)))); CsvResponseFormatter formatter = new CsvResponseFormatter(); - String expected = "name,age\nJohn,20\nSmith,30"; - assertEquals(expected, formatter.format(response)); + String expected = "name,age%nJohn,20%nSmith,30"; + assertEquals(format(expected), formatter.format(response)); } @Test @@ -51,9 +52,9 @@ void sanitizeHeaders() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of( "=firstname", "John", "+lastname", "Smith", "-city", "Seattle", "@age", 20)))); - String expected = "'=firstname,'+lastname,'-city,'@age\n" + String expected = "'=firstname,'+lastname,'-city,'@age%n" + "John,Smith,Seattle,20"; - assertEquals(expected, formatter.format(response)); + assertEquals(format(expected), formatter.format(response)); } @Test @@ -67,14 +68,14 @@ void sanitizeData() { tupleValue(ImmutableMap.of("city", "-Seattle")), tupleValue(ImmutableMap.of("city", "@Seattle")), tupleValue(ImmutableMap.of("city", "Seattle=")))); - String expected = "city\n" - + "Seattle\n" - + "'=Seattle\n" - + "'+Seattle\n" - + "'-Seattle\n" - + "'@Seattle\n" + String expected = "city%n" + + "Seattle%n" + + "'=Seattle%n" + + "'+Seattle%n" + + "'-Seattle%n" + + "'@Seattle%n" + "Seattle="; - assertEquals(expected, formatter.format(response)); + assertEquals(format(expected), formatter.format(response)); } @Test @@ -84,17 +85,17 @@ void quoteIfRequired() { new ExecutionEngine.Schema.Column(",,age", ",,age", INTEGER))); QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("na,me", "John,Smith", ",,age", "30,,,")))); - String expected = "\"na,me\",\",,age\"\n" + String expected = "\"na,me\",\",,age\"%n" + "\"John,Smith\",\"30,,,\""; - assertEquals(expected, formatter.format(response)); + assertEquals(format(expected), formatter.format(response)); } @Test void formatError() { Throwable t = new RuntimeException("This is an exception"); String expected = - "{\n \"type\": \"RuntimeException\",\n \"reason\": \"This is an exception\"\n}"; - assertEquals(expected, formatter.format(t)); + "{%n \"type\": \"RuntimeException\",%n \"reason\": \"This is an exception\"%n}"; + assertEquals(format(expected), formatter.format(t)); } @Test @@ -105,10 +106,10 @@ void escapeSanitize() { QueryResult response = new QueryResult(schema, Arrays.asList( tupleValue(ImmutableMap.of("city", "=Seattle")), tupleValue(ImmutableMap.of("city", ",,Seattle")))); - String expected = "city\n" - + "=Seattle\n" + String expected = "city%n" + + "=Seattle%n" + "\",,Seattle\""; - assertEquals(expected, escapeFormatter.format(response)); + assertEquals(format(expected), escapeFormatter.format(response)); } @Test @@ -122,11 +123,11 @@ void replaceNullValues() { ImmutableMap.of("firstname", LITERAL_NULL, "city", stringValue("Seattle"))), ExprTupleValue.fromExprValueMap( ImmutableMap.of("firstname", stringValue("John"), "city", LITERAL_MISSING)))); - String expected = "name,city\n" - + "John,Seattle\n" - + ",Seattle\n" + String expected = "name,city%n" + + "John,Seattle%n" + + ",Seattle%n" + "John,"; - assertEquals(expected, formatter.format(response)); + assertEquals(format(expected), formatter.format(response)); } } diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java index f19f6bb2ae..a8e5f6dee5 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java @@ -6,6 +6,7 @@ package org.opensearch.sql.protocol.response.format; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.common.utils.StringUtils.format; import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Test; @@ -16,7 +17,7 @@ class ErrorFormatterTest { @Test void htmlEscaping_should_disabled() { assertEquals( - "{\n" + " \"request\": \"index=test\"\n" + "}", + format("{%n" + " \"request\": \"index=test\"%n" + "}"), ErrorFormatter.prettyJsonify(ImmutableMap.of("request", "index=test"))); assertEquals( "{\"request\":\"index=test\"}", diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java index d8e06f81ab..12cdc4bf96 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java @@ -7,6 +7,7 @@ package org.opensearch.sql.protocol.response.format; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.common.utils.StringUtils.format; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; @@ -37,7 +38,7 @@ void formatResponse() { tupleValue(ImmutableMap.of("name", "John", "age", 20)), tupleValue(ImmutableMap.of("name", "Smith", "age", 30)))); String expected = "name|age%nJohn|20%nSmith|30"; - assertEquals(expected, rawFormater.format(response)); + assertEquals(format(expected), rawFormater.format(response)); } @Test @@ -52,7 +53,7 @@ void sanitizeHeaders() { "=firstname", "John", "+lastname", "Smith", "-city", "Seattle", "@age", 20)))); String expected = "=firstname|+lastname|-city|@age%n" + "John|Smith|Seattle|20"; - assertEquals(expected, rawFormater.format(response)); + assertEquals(format(expected), rawFormater.format(response)); } @Test @@ -73,7 +74,7 @@ void sanitizeData() { + "-Seattle%n" + "@Seattle%n" + "Seattle="; - assertEquals(expected, rawFormater.format(response)); + assertEquals(format(expected), rawFormater.format(response)); } @Test @@ -85,7 +86,7 @@ void quoteIfRequired() { tupleValue(ImmutableMap.of("na|me", "John|Smith", "||age", "30|||")))); String expected = "\"na|me\"|\"||age\"%n" + "\"John|Smith\"|\"30|||\""; - assertEquals(expected, rawFormater.format(response)); + assertEquals(format(expected), rawFormater.format(response)); } @Test @@ -93,7 +94,7 @@ void formatError() { Throwable t = new RuntimeException("This is an exception"); String expected = "{%n \"type\": \"RuntimeException\",%n \"reason\": \"This is an exception\"%n}"; - assertEquals(expected, rawFormater.format(t)); + assertEquals(format(expected), rawFormater.format(t)); } @Test @@ -120,7 +121,7 @@ void senstiveCharater() { String expected = "city%n" + "@Seattle%n" + "++Seattle"; - assertEquals(expected, rawFormater.format(response)); + assertEquals(format(expected), rawFormater.format(response)); } @Test @@ -134,7 +135,7 @@ void senstiveCharaterWithSanitize() { String expected = "city%n" + "@Seattle%n" + "\"++Seattle|||\""; - assertEquals(expected, testFormater.format(response)); + assertEquals(format(expected), testFormater.format(response)); } @Test @@ -152,7 +153,7 @@ void replaceNullValues() { + "John|Seattle%n" + "|Seattle%n" + "John|"; - assertEquals(expected, rawFormater.format(response)); + assertEquals(format(expected), rawFormater.format(response)); } } diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java index 8b4438cf91..eb5d5f5393 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java @@ -7,6 +7,7 @@ package org.opensearch.sql.protocol.response.format; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.common.utils.StringUtils.format; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; import static org.opensearch.sql.data.model.ExprValueUtils.tupleValue; @@ -54,31 +55,31 @@ void formatResponsePretty() { tupleValue(ImmutableMap.of("firstname", "John", "age", 20)), tupleValue(ImmutableMap.of("firstname", "Smith", "age", 30)))); SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(PRETTY); - assertEquals( - "{\n" - + " \"schema\": [\n" - + " {\n" - + " \"name\": \"firstname\",\n" - + " \"type\": \"string\"\n" - + " },\n" - + " {\n" - + " \"name\": \"age\",\n" - + " \"type\": \"integer\"\n" - + " }\n" - + " ],\n" - + " \"datarows\": [\n" - + " [\n" - + " \"John\",\n" - + " 20\n" - + " ],\n" - + " [\n" - + " \"Smith\",\n" - + " 30\n" - + " ]\n" - + " ],\n" - + " \"total\": 2,\n" - + " \"size\": 2\n" - + "}", + assertEquals(format( + "{%n" + + " \"schema\": [%n" + + " {%n" + + " \"name\": \"firstname\",%n" + + " \"type\": \"string\"%n" + + " },%n" + + " {%n" + + " \"name\": \"age\",%n" + + " \"type\": \"integer\"%n" + + " }%n" + + " ],%n" + + " \"datarows\": [%n" + + " [%n" + + " \"John\",%n" + + " 20%n" + + " ],%n" + + " [%n" + + " \"Smith\",%n" + + " 30%n" + + " ]%n" + + " ],%n" + + " \"total\": 2,%n" + + " \"size\": 2%n" + + "}"), formatter.format(response)); } @@ -165,11 +166,11 @@ void formatError() { @Test void formatErrorPretty() { SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(PRETTY); - assertEquals( - "{\n" - + " \"type\": \"RuntimeException\",\n" - + " \"reason\": \"This is an exception\"\n" - + "}", + assertEquals(format( + "{%n" + + " \"type\": \"RuntimeException\",%n" + + " \"reason\": \"This is an exception\"%n" + + "}"), formatter.format(new RuntimeException("This is an exception"))); } } From 9df2acd6e65f4dd0ca59a806cfe8de047f343033 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Mon, 17 Oct 2022 10:22:08 -0400 Subject: [PATCH 13/36] fix tests and add java docs Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 3 +++ .../sql/protocol/response/format/ErrorFormatter.java | 6 ++++++ .../protocol/response/format/RawResponseFormatterTest.java | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 057e4ee98b..320ba7046e 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -35,6 +35,9 @@ jobs: runs-on: ${{ matrix.os }} steps: + - name: Change line endings for windows + if: ${{ matrix.os == 'windows-latest' }} + run: git config --global core.autocrlf input - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.java }} diff --git a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java index e21e731a2d..64a1cd842f 100644 --- a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java +++ b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java @@ -43,12 +43,18 @@ public static String prettyFormat(Throwable t) { return prettyJsonify(error); } + /** + * Util method to format {@link Object} to string in compact printing. + */ public static String compactJsonify(Object jsonObject) { return AccessController.doPrivileged( (PrivilegedAction) () -> GSON.toJson(jsonObject) .replaceAll("\\n|\\r\\n", System.getProperty("line.separator"))); } + /** + * Util method to format {@link Object} to string in pretty printing. + */ public static String prettyJsonify(Object jsonObject) { return AccessController.doPrivileged( (PrivilegedAction) () -> PRETTY_PRINT_GSON.toJson(jsonObject) diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java index 12cdc4bf96..8fe7ee1006 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java @@ -108,7 +108,7 @@ void escapeSanitize() { String expected = "city%n" + "=Seattle%n" + "\"||Seattle\""; - assertEquals(expected, escapeFormatter.format(response)); + assertEquals(format(expected), escapeFormatter.format(response)); } @Test From 8f98fa4cdd24ee0ebb13e7e71d0f08f1d99eb2e7 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Mon, 17 Oct 2022 13:53:30 -0400 Subject: [PATCH 14/36] revert error format Signed-off-by: Derek Ho --- .../sql/protocol/response/format/ErrorFormatter.java | 12 ++---------- .../response/format/CsvResponseFormatterTest.java | 4 ++-- .../protocol/response/format/ErrorFormatterTest.java | 2 +- .../response/format/RawResponseFormatterTest.java | 4 ++-- .../format/SimpleJsonResponseFormatterTest.java | 10 +++++----- 5 files changed, 12 insertions(+), 20 deletions(-) diff --git a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java index 64a1cd842f..40848e959b 100644 --- a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java +++ b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/ErrorFormatter.java @@ -43,22 +43,14 @@ public static String prettyFormat(Throwable t) { return prettyJsonify(error); } - /** - * Util method to format {@link Object} to string in compact printing. - */ public static String compactJsonify(Object jsonObject) { return AccessController.doPrivileged( - (PrivilegedAction) () -> GSON.toJson(jsonObject) - .replaceAll("\\n|\\r\\n", System.getProperty("line.separator"))); + (PrivilegedAction) () -> GSON.toJson(jsonObject)); } - /** - * Util method to format {@link Object} to string in pretty printing. - */ public static String prettyJsonify(Object jsonObject) { return AccessController.doPrivileged( - (PrivilegedAction) () -> PRETTY_PRINT_GSON.toJson(jsonObject) - .replaceAll("\\n|\\r\\n", System.getProperty("line.separator"))); + (PrivilegedAction) () -> PRETTY_PRINT_GSON.toJson(jsonObject)); } @RequiredArgsConstructor diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java index 475cb30946..7008b51fa6 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java @@ -94,8 +94,8 @@ void quoteIfRequired() { void formatError() { Throwable t = new RuntimeException("This is an exception"); String expected = - "{%n \"type\": \"RuntimeException\",%n \"reason\": \"This is an exception\"%n}"; - assertEquals(format(expected), formatter.format(t)); + "{\n \"type\": \"RuntimeException\",\n \"reason\": \"This is an exception\"\n}"; + assertEquals(expected, formatter.format(t)); } @Test diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java index a8e5f6dee5..02714a7ffb 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java @@ -17,7 +17,7 @@ class ErrorFormatterTest { @Test void htmlEscaping_should_disabled() { assertEquals( - format("{%n" + " \"request\": \"index=test\"%n" + "}"), + format("{\n" + " \"request\": \"index=test\"\n" + "}"), ErrorFormatter.prettyJsonify(ImmutableMap.of("request", "index=test"))); assertEquals( "{\"request\":\"index=test\"}", diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java index 8fe7ee1006..24b5a4431d 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java @@ -93,8 +93,8 @@ void quoteIfRequired() { void formatError() { Throwable t = new RuntimeException("This is an exception"); String expected = - "{%n \"type\": \"RuntimeException\",%n \"reason\": \"This is an exception\"%n}"; - assertEquals(format(expected), rawFormater.format(t)); + "{\n \"type\": \"RuntimeException\",\n \"reason\": \"This is an exception\"\n}"; + assertEquals(expected, rawFormater.format(t)); } @Test diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java index eb5d5f5393..8be080a816 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java @@ -166,11 +166,11 @@ void formatError() { @Test void formatErrorPretty() { SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(PRETTY); - assertEquals(format( - "{%n" - + " \"type\": \"RuntimeException\",%n" - + " \"reason\": \"This is an exception\"%n" - + "}"), + assertEquals( + "{\n" + + " \"type\": \"RuntimeException\",\n" + + " \"reason\": \"This is an exception\"\n" + + "}", formatter.format(new RuntimeException("This is an exception"))); } } From 7a39dcb7be2e67dd58f9b81666be13b191119d15 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Mon, 17 Oct 2022 14:09:51 -0400 Subject: [PATCH 15/36] revert pretty format response Signed-off-by: Derek Ho --- .../SimpleJsonResponseFormatterTest.java | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java index 8be080a816..8f47618807 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java @@ -55,31 +55,31 @@ void formatResponsePretty() { tupleValue(ImmutableMap.of("firstname", "John", "age", 20)), tupleValue(ImmutableMap.of("firstname", "Smith", "age", 30)))); SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(PRETTY); - assertEquals(format( - "{%n" - + " \"schema\": [%n" - + " {%n" - + " \"name\": \"firstname\",%n" - + " \"type\": \"string\"%n" - + " },%n" - + " {%n" - + " \"name\": \"age\",%n" - + " \"type\": \"integer\"%n" - + " }%n" - + " ],%n" - + " \"datarows\": [%n" - + " [%n" - + " \"John\",%n" - + " 20%n" - + " ],%n" - + " [%n" - + " \"Smith\",%n" - + " 30%n" - + " ]%n" - + " ],%n" - + " \"total\": 2,%n" - + " \"size\": 2%n" - + "}"), + assertEquals( + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"firstname\",\n" + + " \"type\": \"string\"\n" + + " },\n" + + " {\n" + + " \"name\": \"age\",\n" + + " \"type\": \"integer\"\n" + + " }\n" + + " ],\n" + + " \"datarows\": [\n" + + " [\n" + + " \"John\",\n" + + " 20\n" + + " ],\n" + + " [\n" + + " \"Smith\",\n" + + " 30\n" + + " ]\n" + + " ],\n" + + " \"total\": 2,\n" + + " \"size\": 2\n" + + "}", formatter.format(response)); } From bd8572a7314be83f79e86b4d3eae1a7ae45719a7 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Mon, 17 Oct 2022 14:43:33 -0400 Subject: [PATCH 16/36] revert error formatter test file Signed-off-by: Derek Ho --- .../sql/protocol/response/format/ErrorFormatterTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java index 02714a7ffb..f19f6bb2ae 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/ErrorFormatterTest.java @@ -6,7 +6,6 @@ package org.opensearch.sql.protocol.response.format; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.opensearch.sql.common.utils.StringUtils.format; import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Test; @@ -17,7 +16,7 @@ class ErrorFormatterTest { @Test void htmlEscaping_should_disabled() { assertEquals( - format("{\n" + " \"request\": \"index=test\"\n" + "}"), + "{\n" + " \"request\": \"index=test\"\n" + "}", ErrorFormatter.prettyJsonify(ImmutableMap.of("request", "index=test"))); assertEquals( "{\"request\":\"index=test\"}", From a2b473b27d7d69ee3c687f8bcc90b15412f16a1b Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Mon, 17 Oct 2022 21:02:50 -0400 Subject: [PATCH 17/36] replace carriage return with nothing Signed-off-by: Derek Ho --- .../sql/legacy/unittest/utils/PrettyFormatterTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/PrettyFormatterTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/PrettyFormatterTest.java index fc20d818e6..f876b14110 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/PrettyFormatterTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/utils/PrettyFormatterTest.java @@ -31,11 +31,13 @@ public void assertFormatterWithoutContentInside() throws IOException { public void assertFormatterOutputsPrettyJson() throws IOException { String explainFormattedPrettyFilePath = TestUtils.getResourceFilePath( "/src/test/resources/expectedOutput/explain_format_pretty.json"); - String explainFormattedPretty = Files.toString(new File(explainFormattedPrettyFilePath), StandardCharsets.UTF_8); + String explainFormattedPretty = Files.toString(new File(explainFormattedPrettyFilePath), StandardCharsets.UTF_8) + .replaceAll("\r", ""); String explainFormattedOnelineFilePath = TestUtils.getResourceFilePath( "/src/test/resources/explain_format_oneline.json"); - String explainFormattedOneline = Files.toString(new File(explainFormattedOnelineFilePath), StandardCharsets.UTF_8); + String explainFormattedOneline = Files.toString(new File(explainFormattedOnelineFilePath), StandardCharsets.UTF_8) + .replaceAll("\r", ""); String result = JsonPrettyFormatter.format(explainFormattedOneline); assertThat(result, equalTo(explainFormattedPretty)); From a3747a372343804b72d04ca3d263831cef855203 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Mon, 17 Oct 2022 21:07:49 -0400 Subject: [PATCH 18/36] remove windows git config Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 320ba7046e..057e4ee98b 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -35,9 +35,6 @@ jobs: runs-on: ${{ matrix.os }} steps: - - name: Change line endings for windows - if: ${{ matrix.os == 'windows-latest' }} - run: git config --global core.autocrlf input - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.java }} From e70df11d2d6de9350c4c005ba1c1cc312da610d6 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Wed, 19 Oct 2022 17:08:00 -0400 Subject: [PATCH 19/36] fix PR comments and fail test on purpose to see if upload test reports succeeds Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 13 +++++++++---- .../sql-workbench-test-and-build-workflow.yml | 3 ++- .../format/SimpleJsonResponseFormatterTest.java | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 057e4ee98b..52a30afa44 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -25,8 +25,13 @@ jobs: # Run all jobs fail-fast: false matrix: - java: [11, 17] - os: [ubuntu-latest, windows-latest, macos-latest] + entry: + - { os: ubuntu-latest, java: 11 } + - { os: windows-latest, java: 11 } + - { os: macos-latest, java: 11 } + - { os: ubuntu-latest, java: 17 } + - { os: windows-latest, java: 17 } + - { os: macos-latest, java: 17 } include: - os: windows-latest os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc @@ -57,7 +62,7 @@ jobs: # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action - name: Upload SQL Coverage Report - if: ${{ matrix.os == 'ubuntu-latest' }} + if: ${{ always() && matrix.os == 'ubuntu-latest' }} uses: codecov/codecov-action@v3 with: flags: sql-engine @@ -70,7 +75,7 @@ jobs: path: opensearch-sql-builds - name: Upload test reports - if: ${{ matrix.os == 'ubuntu-latest' }} + if: ${{ always() && matrix.os == 'ubuntu-latest' }} uses: actions/upload-artifact@v2 with: name: test-reports diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index 3878872512..e5f52065b6 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -76,4 +76,5 @@ jobs: uses: actions/upload-artifact@v1 # can't update to v3 because upload fails with: name: workbench-${{ matrix.os }} - path: ../OpenSearch-Dashboards/plugins/workbench/build \ No newline at end of file + path: ../OpenSearch-Dashboards/plugins/workbench/build + \ No newline at end of file diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java index 8f47618807..ce8dd3e92c 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java @@ -40,7 +40,7 @@ void formatResponse() { tupleValue(ImmutableMap.of("firstname", "Smith", "age", 30)))); SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(COMPACT); assertEquals( - "{\"schema\":[{\"name\":\"firstname\",\"type\":\"string\"}," + "{\"schema\":[{\"name\":\"firstname\",\"type\":\"string\"},a;woehgi;whaeg" + "{\"name\":\"age\",\"type\":\"integer\"}],\"datarows\":" + "[[\"John\",20],[\"Smith\",30]],\"total\":2,\"size\":2}", formatter.format(response)); From de65184a74ca54460951de1cfb3f3c9053053cb8 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Wed, 19 Oct 2022 17:13:43 -0400 Subject: [PATCH 20/36] fix matrix entry Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 52a30afa44..46103b0057 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -46,13 +46,13 @@ jobs: uses: actions/setup-java@v3 with: distribution: 'temurin' - java-version: ${{ matrix.java }} + java-version: ${{ matrix.entry.java }} - name: Build with Gradle run: ./gradlew --continue build ${{ env.BUILD_ARGS }} - name: Run backward compatibility tests - if: ${{ matrix.os == 'ubuntu-latest' }} + if: ${{ matrix.entry.os == 'ubuntu-latest' }} run: ./scripts/bwctest.sh - name: Create Artifact Path @@ -62,7 +62,7 @@ jobs: # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action - name: Upload SQL Coverage Report - if: ${{ always() && matrix.os == 'ubuntu-latest' }} + if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} uses: codecov/codecov-action@v3 with: flags: sql-engine @@ -71,11 +71,11 @@ jobs: - name: Upload Artifacts uses: actions/upload-artifact@v2 with: - name: opensearch-sql-${{ matrix.os }} + name: opensearch-sql-${{ matrix.entry.os }} path: opensearch-sql-builds - name: Upload test reports - if: ${{ always() && matrix.os == 'ubuntu-latest' }} + if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} uses: actions/upload-artifact@v2 with: name: test-reports From a6f3ceb80745978778cefcdcf1b5a4d201ff0431 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Wed, 19 Oct 2022 17:23:44 -0400 Subject: [PATCH 21/36] remove test failure Signed-off-by: Derek Ho --- .../response/format/SimpleJsonResponseFormatterTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java index ce8dd3e92c..8f47618807 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java @@ -40,7 +40,7 @@ void formatResponse() { tupleValue(ImmutableMap.of("firstname", "Smith", "age", 30)))); SimpleJsonResponseFormatter formatter = new SimpleJsonResponseFormatter(COMPACT); assertEquals( - "{\"schema\":[{\"name\":\"firstname\",\"type\":\"string\"},a;woehgi;whaeg" + "{\"schema\":[{\"name\":\"firstname\",\"type\":\"string\"}," + "{\"name\":\"age\",\"type\":\"integer\"}],\"datarows\":" + "[[\"John\",20],[\"Smith\",30]],\"total\":2,\"size\":2}", formatter.format(response)); From 3f372bba8689b016690fe2d84616b8b2dcb2688c Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 20 Oct 2022 16:50:27 -0400 Subject: [PATCH 22/36] fix up to run on right os Signed-off-by: Derek Ho --- .../workflows/sql-test-and-build-workflow.yml | 105 +++++++++--------- 1 file changed, 50 insertions(+), 55 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 46103b0057..aaedf7c25e 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -20,72 +20,67 @@ on: jobs: build: env: - BUILD_ARGS: ${{ matrix.os_build_args }} + BUILD_ARGS: ${{ matrix.entry.os_build_args }} strategy: # Run all jobs fail-fast: false matrix: entry: - { os: ubuntu-latest, java: 11 } - - { os: windows-latest, java: 11 } - - { os: macos-latest, java: 11 } + - { os: windows-latest, java: 11, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc} + - { os: macos-latest, java: 11, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc } - { os: ubuntu-latest, java: 17 } - - { os: windows-latest, java: 17 } - - { os: macos-latest, java: 17 } - include: - - os: windows-latest - os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc - - os: macos-latest - os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc - runs-on: ${{ matrix.os }} + - { os: windows-latest, java: 17, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc } + - { os: macos-latest, java: 17, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc } + runs-on: ${{ matrix.entry.os }} steps: - - uses: actions/checkout@v3 - - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: ${{ matrix.entry.java }} - - - name: Build with Gradle - run: ./gradlew --continue build ${{ env.BUILD_ARGS }} + - uses: actions/checkout@v3 - - name: Run backward compatibility tests - if: ${{ matrix.entry.os == 'ubuntu-latest' }} - run: ./scripts/bwctest.sh + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.entry.java }} - - name: Create Artifact Path - run: | - mkdir -p opensearch-sql-builds - cp -r ./plugin/build/distributions/*.zip opensearch-sql-builds/ + - name: Build with Gradle + run: ./gradlew --continue build ${{ env.BUILD_ARGS }} - # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action - - name: Upload SQL Coverage Report - if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} - uses: codecov/codecov-action@v3 - with: - flags: sql-engine - token: ${{ secrets.CODECOV_TOKEN }} + - name: Run backward compatibility tests + if: ${{ matrix.entry.os == 'ubuntu-latest' }} + run: ./scripts/bwctest.sh - - name: Upload Artifacts - uses: actions/upload-artifact@v2 - with: - name: opensearch-sql-${{ matrix.entry.os }} - path: opensearch-sql-builds + - name: Create Artifact Path + run: | + mkdir -p opensearch-sql-builds + cp -r ./plugin/build/distributions/*.zip opensearch-sql-builds/ - - name: Upload test reports - if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} - uses: actions/upload-artifact@v2 - with: - name: test-reports - path: | - sql/build/reports/** - ppl/build/reports/** - core/build/reports/** - common/build/reports/** - opensearch/build/reports/** - integ-test/build/reports/** - protocol/build/reports/** - legacy/build/reports/** - plugin/build/reports/** + # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action + - name: Upload SQL Coverage Report + if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} + uses: codecov/codecov-action@v3 + with: + flags: sql-engine + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Upload Artifacts + uses: actions/upload-artifact@v2 + with: + name: opensearch-sql-${{ matrix.entry.os }} + path: opensearch-sql-builds + + - name: Upload test reports + if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} + uses: actions/upload-artifact@v2 + with: + name: test-reports + path: | + sql/build/reports/** + ppl/build/reports/** + core/build/reports/** + common/build/reports/** + opensearch/build/reports/** + integ-test/build/reports/** + protocol/build/reports/** + legacy/build/reports/** + plugin/build/reports/** From 5e24f3c36b73ddfd5f5bd5708e2ebd008a6742da Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 20 Oct 2022 16:51:36 -0400 Subject: [PATCH 23/36] fix up indentation Signed-off-by: Derek Ho --- .../workflows/sql-test-and-build-workflow.yml | 84 +++++++++---------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index aaedf7c25e..85714ef795 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -35,52 +35,52 @@ jobs: runs-on: ${{ matrix.entry.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: ${{ matrix.entry.java }} + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.entry.java }} - - name: Build with Gradle - run: ./gradlew --continue build ${{ env.BUILD_ARGS }} + - name: Build with Gradle + run: ./gradlew --continue build ${{ env.BUILD_ARGS }} - - name: Run backward compatibility tests - if: ${{ matrix.entry.os == 'ubuntu-latest' }} - run: ./scripts/bwctest.sh + - name: Run backward compatibility tests + if: ${{ matrix.entry.os == 'ubuntu-latest' }} + run: ./scripts/bwctest.sh - - name: Create Artifact Path - run: | - mkdir -p opensearch-sql-builds - cp -r ./plugin/build/distributions/*.zip opensearch-sql-builds/ + - name: Create Artifact Path + run: | + mkdir -p opensearch-sql-builds + cp -r ./plugin/build/distributions/*.zip opensearch-sql-builds/ - # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action - - name: Upload SQL Coverage Report - if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} - uses: codecov/codecov-action@v3 - with: - flags: sql-engine - token: ${{ secrets.CODECOV_TOKEN }} + # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action + - name: Upload SQL Coverage Report + if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} + uses: codecov/codecov-action@v3 + with: + flags: sql-engine + token: ${{ secrets.CODECOV_TOKEN }} - - name: Upload Artifacts - uses: actions/upload-artifact@v2 - with: - name: opensearch-sql-${{ matrix.entry.os }} - path: opensearch-sql-builds + - name: Upload Artifacts + uses: actions/upload-artifact@v2 + with: + name: opensearch-sql-${{ matrix.entry.os }} + path: opensearch-sql-builds - - name: Upload test reports - if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} - uses: actions/upload-artifact@v2 - with: - name: test-reports - path: | - sql/build/reports/** - ppl/build/reports/** - core/build/reports/** - common/build/reports/** - opensearch/build/reports/** - integ-test/build/reports/** - protocol/build/reports/** - legacy/build/reports/** - plugin/build/reports/** + - name: Upload test reports + if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} + uses: actions/upload-artifact@v2 + with: + name: test-reports + path: | + sql/build/reports/** + ppl/build/reports/** + core/build/reports/** + common/build/reports/** + opensearch/build/reports/** + integ-test/build/reports/** + protocol/build/reports/** + legacy/build/reports/** + plugin/build/reports/** From 3fbe3e44abcf44f525065e97ce0991068a19a62b Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 20 Oct 2022 17:34:59 -0400 Subject: [PATCH 24/36] remove env. build_args to make it cleaner Signed-off-by: Derek Ho --- .github/workflows/sql-test-and-build-workflow.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 85714ef795..3d063a2bfc 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -19,8 +19,6 @@ on: jobs: build: - env: - BUILD_ARGS: ${{ matrix.entry.os_build_args }} strategy: # Run all jobs fail-fast: false @@ -44,7 +42,7 @@ jobs: java-version: ${{ matrix.entry.java }} - name: Build with Gradle - run: ./gradlew --continue build ${{ env.BUILD_ARGS }} + run: ./gradlew --continue build ${{ matrix.entry.os_build_args }} - name: Run backward compatibility tests if: ${{ matrix.entry.os == 'ubuntu-latest' }} From 954c072e639ba2dc992c48807190e2c0ba5119df Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 20 Oct 2022 17:48:25 -0400 Subject: [PATCH 25/36] try removing env var Signed-off-by: Derek Ho --- workbench/package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/workbench/package.json b/workbench/package.json index 9a9b9706f6..2360dc4d91 100644 --- a/workbench/package.json +++ b/workbench/package.json @@ -16,7 +16,7 @@ "start": "plugin-helpers start", "test:server": "plugin-helpers test:server", "test:browser": "plugin-helpers test:browser", - "test:jest": "cross-env NODE_PATH=../../node_modules ../../node_modules/.bin/jest --config ./test/jest.config.js", + "test:jest": "../../node_modules/.bin/jest --config ./test/jest.config.js", "build": "yarn plugin_helpers build", "plugin_helpers": "node ../../scripts/plugin_helpers" }, @@ -28,7 +28,6 @@ "@testing-library/user-event": "^13.1.9", "@types/hapi-latest": "npm:@types/hapi@18.0.3", "@types/react-router-dom": "^5.3.2", - "cross-env": "7.0.3", "cypress": "^5.0.0", "eslint": "^6.8.0", "eslint-plugin-no-unsanitized": "^3.0.2", From 79f765c4a49ebcd441885a8d74e3353c757d4d8d Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 20 Oct 2022 17:51:05 -0400 Subject: [PATCH 26/36] remove unecessary import Signed-off-by: Derek Ho --- .../response/format/SimpleJsonResponseFormatterTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java index 8f47618807..8b4438cf91 100644 --- a/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java +++ b/protocol/src/test/java/org/opensearch/sql/protocol/response/format/SimpleJsonResponseFormatterTest.java @@ -7,7 +7,6 @@ package org.opensearch.sql.protocol.response.format; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.opensearch.sql.common.utils.StringUtils.format; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; import static org.opensearch.sql.data.model.ExprValueUtils.tupleValue; From 30a2d27974e3b507aa21f57438c1de22c3bdc558 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Fri, 21 Oct 2022 17:29:18 -0700 Subject: [PATCH 27/36] Fix result order of parse with other run time fields (#934) Signed-off-by: Joshua Li --- .../sql/planner/physical/ProjectOperator.java | 32 +++++---- .../planner/physical/ProjectOperatorTest.java | 49 ++++++++++++- .../opensearch/sql/ppl/ParseCommandIT.java | 69 +++++++++++++++++++ 3 files changed, 133 insertions(+), 17 deletions(-) create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/ParseCommandIT.java diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/ProjectOperator.java b/core/src/main/java/org/opensearch/sql/planner/physical/ProjectOperator.java index 6e21c969db..496e4e6ddb 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/ProjectOperator.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/ProjectOperator.java @@ -10,6 +10,7 @@ import com.google.common.collect.ImmutableMap.Builder; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -55,32 +56,33 @@ public boolean hasNext() { public ExprValue next() { ExprValue inputValue = input.next(); ImmutableMap.Builder mapBuilder = new Builder<>(); + + // ParseExpression will always override NamedExpression when identifier conflicts + // TODO needs a better implementation, see https://github.com/opensearch-project/sql/issues/458 for (NamedExpression expr : projectList) { ExprValue exprValue = expr.valueOf(inputValue.bindingTuples()); - if (namedParseExpressions.stream() - .noneMatch(parsed -> parsed.getNameOrAlias().equals(expr.getNameOrAlias()))) { + Optional optionalParseExpression = namedParseExpressions.stream() + .filter(parseExpr -> parseExpr.getNameOrAlias().equals(expr.getNameOrAlias())) + .findFirst(); + if (optionalParseExpression.isEmpty()) { mapBuilder.put(expr.getNameOrAlias(), exprValue); - } - } - // ParseExpression will always override NamedExpression when identifier conflicts - // TODO needs a better implementation, see https://github.com/opensearch-project/sql/issues/458 - for (NamedExpression expr : namedParseExpressions) { - if (projectList.stream() - .noneMatch(field -> field.getNameOrAlias().equals(expr.getNameOrAlias()))) { continue; } + + NamedExpression parseExpression = optionalParseExpression.get(); ExprValue sourceFieldValue = inputValue.bindingTuples() - .resolve(((ParseExpression) expr.getDelegated()).getSourceField()); + .resolve(((ParseExpression) parseExpression.getDelegated()).getSourceField()); if (sourceFieldValue.isMissing()) { // source field will be missing after stats command, read from inputValue if it exists // otherwise do nothing since it should not appear as a field - ExprValue exprValue = ExprValueUtils.getTupleValue(inputValue).get(expr.getNameOrAlias()); - if (exprValue != null) { - mapBuilder.put(expr.getNameOrAlias(), exprValue); + ExprValue tupleValue = + ExprValueUtils.getTupleValue(inputValue).get(parseExpression.getNameOrAlias()); + if (tupleValue != null) { + mapBuilder.put(parseExpression.getNameOrAlias(), tupleValue); } } else { - ExprValue parsedValue = expr.valueOf(inputValue.bindingTuples()); - mapBuilder.put(expr.getNameOrAlias(), parsedValue); + ExprValue parsedValue = parseExpression.valueOf(inputValue.bindingTuples()); + mapBuilder.put(parseExpression.getNameOrAlias(), parsedValue); } } return ExprTupleValue.fromExprValueMap(mapBuilder.build()); diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/ProjectOperatorTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/ProjectOperatorTest.java index 3fa312d4c2..24be5eb2b8 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/ProjectOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/ProjectOperatorTest.java @@ -118,7 +118,26 @@ public void project_fields_with_parse_expressions() { DSL.literal("action"))), DSL.named("response", DSL.regex(DSL.ref("response", STRING), DSL.literal("(?\\w+) (?\\d+)"), - DSL.literal("response"))), DSL.named("ignored", + DSL.literal("response")))) + ); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(1), + hasItems( + ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", "200"))))); + } + + @Test + public void project_fields_with_unused_parse_expressions() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("response", "GET 200"))); + PhysicalPlan plan = + project(inputPlan, ImmutableList.of(DSL.named("response", DSL.ref("response", STRING))), + ImmutableList.of(DSL.named("ignored", DSL.regex(DSL.ref("response", STRING), DSL.literal("(?\\w+) (?\\d+)"), DSL.literal("ignored")))) @@ -130,7 +149,33 @@ public void project_fields_with_parse_expressions() { allOf( iterableWithSize(1), hasItems( - ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", "200"))))); + ExprValueUtils.tupleValue(ImmutableMap.of("response", "GET 200"))))); + } + + @Test + public void project_fields_with_parse_expressions_and_runtime_fields() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn( + ExprValueUtils.tupleValue(ImmutableMap.of("response", "GET 200", "eval_field", 1))); + PhysicalPlan plan = + project(inputPlan, ImmutableList.of(DSL.named("response", DSL.ref("response", STRING)), + DSL.named("action", DSL.ref("action", STRING)), + DSL.named("eval_field", DSL.ref("eval_field", INTEGER))), + ImmutableList.of(DSL.named("action", + DSL.regex(DSL.ref("response", STRING), + DSL.literal("(?\\w+) (?\\d+)"), + DSL.literal("action")))) + ); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(1), + hasItems( + ExprValueUtils.tupleValue( + ImmutableMap.of("response", "GET 200", "action", "GET", "eval_field", 1))))); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ParseCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ParseCommandIT.java new file mode 100644 index 0000000000..36fcb4bf3b --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ParseCommandIT.java @@ -0,0 +1,69 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.ppl; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyOrder; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Test; + +public class ParseCommandIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(Index.BANK); + } + + @Test + public void testParseCommand() throws IOException { + JSONObject result = executeQuery( + String.format("source=%s | parse email '.+@(?.+)' | fields email, host", + TEST_INDEX_BANK)); + verifyOrder( + result, + rows("amberduke@pyrami.com", "pyrami.com"), + rows("hattiebond@netagy.com", "netagy.com"), + rows("nanettebates@quility.com", "quility.com"), + rows("daleadams@boink.com", "boink.com"), + rows("elinorratliff@scentric.com", "scentric.com"), + rows("virginiaayala@filodyne.com", "filodyne.com"), + rows("dillardmcpherson@quailcom.com", "quailcom.com")); + } + + @Test + public void testParseCommandReplaceOriginalField() throws IOException { + JSONObject result = executeQuery( + String.format("source=%s | parse email '.+@(?.+)' | fields email", TEST_INDEX_BANK)); + verifyOrder( + result, + rows("pyrami.com"), + rows("netagy.com"), + rows("quility.com"), + rows("boink.com"), + rows("scentric.com"), + rows("filodyne.com"), + rows("quailcom.com")); + } + + @Test + public void testParseCommandWithOtherRunTimeFields() throws IOException { + JSONObject result = executeQuery(String.format("source=%s | parse email '.+@(?.+)' | " + + "eval eval_result=1 | fields host, eval_result", TEST_INDEX_BANK)); + verifyOrder( + result, + rows("pyrami.com", 1), + rows("netagy.com", 1), + rows("quility.com", 1), + rows("boink.com", 1), + rows("scentric.com", 1), + rows("filodyne.com", 1), + rows("quailcom.com", 1)); + } +} From bfad32cfe7ac3b8f2d8e8f889917a0e79be6c867 Mon Sep 17 00:00:00 2001 From: vamsi-amazon Date: Tue, 18 Oct 2022 02:10:35 -0700 Subject: [PATCH 28/36] SHOW Catalogs Implementation Signed-off-by: vamsi-amazon --- .../org/opensearch/sql/analysis/Analyzer.java | 28 ++++++-- .../sql/catalog/CatalogService.java | 29 ++++++-- .../opensearch/sql/catalog/model/Catalog.java | 27 +++++++ .../sql/planner/logical/LogicalPlanDSL.java | 2 + .../physical/catalog/CatalogTable.java | 58 +++++++++++++++ .../physical/catalog/CatalogTableScan.java | 70 +++++++++++++++++++ .../physical/catalog/CatalogTableSchema.java | 31 ++++++++ .../sql/utils/SystemIndexUtils.java | 2 + .../opensearch/sql/analysis/AnalyzerTest.java | 12 +++- .../sql/analysis/AnalyzerTestBase.java | 39 ++++++++--- .../catalog/CatalogTableScanTest.java | 70 +++++++++++++++++++ .../physical/catalog/CatalogTableTest.java | 49 +++++++++++++ .../org/opensearch/sql/ppl/StandaloneIT.java | 3 +- .../org/opensearch/sql/plugin/SQLPlugin.java | 4 +- .../plugin/catalog/CatalogServiceImpl.java | 33 +++++---- .../catalog/CatalogServiceImplTest.java | 40 ++++++++--- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 5 ++ .../sql/ppl/config/PPLServiceConfig.java | 6 +- .../opensearch/sql/ppl/parser/AstBuilder.java | 11 +++ .../opensearch/sql/ppl/PPLServiceTest.java | 21 +++--- .../sql/ppl/parser/AstBuilderTest.java | 6 ++ 22 files changed, 481 insertions(+), 67 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/catalog/model/Catalog.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java create mode 100644 core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java create mode 100644 core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 984fbb4d30..5fc642fa06 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -16,6 +16,7 @@ import static org.opensearch.sql.utils.MLCommonsConstants.RCF_SCORE; import static org.opensearch.sql.utils.MLCommonsConstants.RCF_TIMESTAMP; import static org.opensearch.sql.utils.MLCommonsConstants.TIME_FIELD; +import static org.opensearch.sql.utils.SystemIndexUtils.CATALOGS_TABLE_NAME; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; @@ -25,6 +26,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; @@ -60,6 +62,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.catalog.model.Catalog; import org.opensearch.sql.data.model.ExprMissingValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.exception.SemanticCheckException; @@ -89,6 +92,7 @@ import org.opensearch.sql.planner.logical.LogicalRename; import org.opensearch.sql.planner.logical.LogicalSort; import org.opensearch.sql.planner.logical.LogicalValues; +import org.opensearch.sql.planner.physical.catalog.CatalogTable; import org.opensearch.sql.storage.Table; import org.opensearch.sql.utils.ParseUtils; @@ -129,14 +133,24 @@ public LogicalPlan analyze(UnresolvedPlan unresolved, AnalysisContext context) { @Override public LogicalPlan visitRelation(Relation node, AnalysisContext context) { QualifiedName qualifiedName = node.getTableQualifiedName(); + Set allowedCatalogNames = catalogService.getCatalogs() + .stream() + .map(Catalog::getName) + .collect(Collectors.toSet()); CatalogSchemaIdentifierName catalogSchemaIdentifierName - = new CatalogSchemaIdentifierName(qualifiedName.getParts(), catalogService.getCatalogs()); + = new CatalogSchemaIdentifierName(qualifiedName.getParts(), allowedCatalogNames); String tableName = catalogSchemaIdentifierName.getIdentifierName(); context.push(); TypeEnvironment curEnv = context.peek(); - Table table = catalogService - .getStorageEngine(catalogSchemaIdentifierName.getCatalogName()) - .getTable(catalogSchemaIdentifierName.getIdentifierName()); + Table table; + if (CATALOGS_TABLE_NAME.equals(tableName)) { + table = new CatalogTable(catalogService); + } else { + table = catalogService + .getCatalog(catalogSchemaIdentifierName.getCatalogName()) + .getStorageEngine() + .getTable(tableName); + } table.getFieldTypes().forEach((k, v) -> curEnv.define(new Symbol(Namespace.FIELD_NAME, k), v)); // Put index name or its alias in index namespace on type environment so qualifier @@ -163,8 +177,12 @@ public LogicalPlan visitRelationSubquery(RelationSubquery node, AnalysisContext @Override public LogicalPlan visitTableFunction(TableFunction node, AnalysisContext context) { QualifiedName qualifiedName = node.getFunctionName(); + Set allowedCatalogNames = catalogService.getCatalogs() + .stream() + .map(Catalog::getName) + .collect(Collectors.toSet()); CatalogSchemaIdentifierName catalogSchemaIdentifierName - = new CatalogSchemaIdentifierName(qualifiedName.getParts(), catalogService.getCatalogs()); + = new CatalogSchemaIdentifierName(qualifiedName.getParts(), allowedCatalogNames); FunctionName functionName = FunctionName.of(catalogSchemaIdentifierName.getIdentifierName()); List arguments = node.getArguments().stream() diff --git a/core/src/main/java/org/opensearch/sql/catalog/CatalogService.java b/core/src/main/java/org/opensearch/sql/catalog/CatalogService.java index 67512f98d7..4c40920c7b 100644 --- a/core/src/main/java/org/opensearch/sql/catalog/CatalogService.java +++ b/core/src/main/java/org/opensearch/sql/catalog/CatalogService.java @@ -6,20 +6,35 @@ package org.opensearch.sql.catalog; import java.util.Set; +import org.opensearch.sql.catalog.model.Catalog; import org.opensearch.sql.storage.StorageEngine; /** - * Catalog Service defines api for - * providing and managing storage engines and execution engines - * for all the catalogs. - * The storage and execution indirectly make connections to the underlying datastore catalog. + * Catalog Service manages catalogs. */ public interface CatalogService { - StorageEngine getStorageEngine(String catalog); + /** + * Returns all catalog objects. + * + * @return Catalog Catalogs. + */ + Set getCatalogs(); - Set getCatalogs(); + /** + * Returns Catalog with corresponding to the catalog name. + * + * @param catalogName Name of the catalog. + * @return Catalog catalog. + */ + Catalog getCatalog(String catalogName); - void registerOpenSearchStorageEngine(StorageEngine storageEngine); + /** + * Default opensearch engine is not defined in catalog.json. + * So the registration of default catalog happens separately. + * + * @param storageEngine StorageEngine. + */ + void registerDefaultOpenSearchCatalog(StorageEngine storageEngine); } diff --git a/core/src/main/java/org/opensearch/sql/catalog/model/Catalog.java b/core/src/main/java/org/opensearch/sql/catalog/model/Catalog.java new file mode 100644 index 0000000000..5b7eaca523 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/catalog/model/Catalog.java @@ -0,0 +1,27 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.catalog.model; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.storage.StorageEngine; + +@Getter +@RequiredArgsConstructor +@EqualsAndHashCode +public class Catalog { + + private final String name; + + private final ConnectorType connectorType; + + @EqualsAndHashCode.Exclude + private final StorageEngine storageEngine; + +} diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java index 9e07e702de..b18e099afa 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java @@ -11,11 +11,13 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import lombok.experimental.UtilityClass; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.tree.RareTopN.CommandType; import org.opensearch.sql.ast.tree.Sort.SortOption; +import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.NamedExpression; diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java new file mode 100644 index 0000000000..26c9c3d04f --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java @@ -0,0 +1,58 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.planner.physical.catalog; + +import com.google.common.annotations.VisibleForTesting; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.planner.DefaultImplementor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalRelation; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.storage.Table; + + +/** + * Table implementation to handle show catalogs command. + * Since catalog information is not tied to any storage engine, this info + * is handled via Catalog Table. + * + */ +@RequiredArgsConstructor +@EqualsAndHashCode +public class CatalogTable implements Table { + + private final CatalogService catalogService; + + @Override + public Map getFieldTypes() { + return CatalogTableSchema.CATALOG_TABLE_SCHEMA.getMapping(); + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + return plan.accept(new CatalogTableDefaultImplementor(catalogService), null); + } + + @VisibleForTesting + @RequiredArgsConstructor + public static class CatalogTableDefaultImplementor + extends DefaultImplementor { + + private final CatalogService catalogService; + + @Override + public PhysicalPlan visitRelation(LogicalRelation node, Object context) { + return new CatalogTableScan(catalogService); + } + } + +} diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java new file mode 100644 index 0000000000..894ff9f216 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java @@ -0,0 +1,70 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.planner.physical.catalog; + +import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Set; +import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.catalog.model.Catalog; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.storage.TableScanOperator; + +/** + * This class handles table scan of catalog table. + * Right now these are derived from catalogService thorough static fields. + * In future this might scan data from underlying datastore if we start + * persisting catalog info somewhere. + * + */ +public class CatalogTableScan extends TableScanOperator { + + private final CatalogService catalogService; + + private Iterator iterator; + + public CatalogTableScan(CatalogService catalogService) { + this.catalogService = catalogService; + this.iterator = Collections.emptyIterator(); + } + + @Override + public String explain() { + return "GetCatalogRequestRequest{}"; + } + + @Override + public void open() { + List exprValues = new ArrayList<>(); + Set catalogs = catalogService.getCatalogs(); + for (Catalog catalog : catalogs) { + exprValues.add( + new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( + "CATALOG_NAME", ExprValueUtils.stringValue(catalog.getName()), + "CONNECTOR_TYPE", ExprValueUtils.stringValue(catalog.getConnectorType().name()))))); + } + iterator = exprValues.iterator(); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public ExprValue next() { + return iterator.next(); + } + +} diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java new file mode 100644 index 0000000000..35fd63f098 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.physical.catalog; + +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.LinkedHashMap; +import java.util.Map; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.data.type.ExprType; + +/** + * Definition of the system table schema. + */ +@Getter +@RequiredArgsConstructor +public enum CatalogTableSchema { + + CATALOG_TABLE_SCHEMA(new LinkedHashMap<>() { + { + put("CATALOG_NAME", STRING); + put("CONNECTOR_TYPE", STRING); + } + } + ); + private final Map mapping; +} diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index 6c370e53ac..3c4f5cdf39 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -36,6 +36,8 @@ public class SystemIndexUtils { */ public static final String TABLE_INFO = SYS_META_PREFIX + ".ALL"; + public static final String CATALOGS_TABLE_NAME = ".CATALOGS"; + public static Boolean isSystemIndex(String indexName) { return indexName.startsWith(SYS_TABLES_PREFIX); diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index ffc432743a..c1adf0623d 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -6,9 +6,11 @@ package org.opensearch.sql.analysis; +import static java.lang.Boolean.TRUE; import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; import static org.opensearch.sql.ast.dsl.AstDSL.alias; import static org.opensearch.sql.ast.dsl.AstDSL.argument; @@ -29,7 +31,6 @@ import static org.opensearch.sql.ast.tree.Sort.SortOption.DEFAULT_ASC; import static org.opensearch.sql.ast.tree.Sort.SortOrder; import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; -import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.LONG; @@ -67,8 +68,10 @@ import org.opensearch.sql.expression.window.WindowDefinition; import org.opensearch.sql.planner.logical.LogicalAD; import org.opensearch.sql.planner.logical.LogicalMLCommons; +import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.logical.LogicalPlanDSL; import org.opensearch.sql.planner.logical.LogicalRelation; +import org.opensearch.sql.planner.physical.catalog.CatalogTable; import org.springframework.context.annotation.Configuration; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; @@ -953,7 +956,6 @@ public void ad_fitRCF_relation() { ); } - @Test public void table_function() { assertAnalyzeEqual(new LogicalRelation("query_range", table), @@ -998,5 +1000,11 @@ public void table_function_with_wrong_table_function() { assertEquals("unsupported function name: queryrange", exception.getMessage()); } + @Test + public void show_catalogs() { + assertAnalyzeEqual(new LogicalRelation(".CATALOGS", new CatalogTable(catalogService)), + AstDSL.relation(qualifiedName(".CATALOGS"))); + + } } diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java index 9c751ca61d..d66a6f7004 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java @@ -14,14 +14,14 @@ import java.util.List; import java.util.Map; import java.util.Set; -import javax.swing.JTable; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.analysis.symbol.Namespace; import org.opensearch.sql.analysis.symbol.Symbol; import org.opensearch.sql.analysis.symbol.SymbolTable; -import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.catalog.model.Catalog; +import org.opensearch.sql.catalog.model.ConnectorType; import org.opensearch.sql.config.TestConfig; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.ExpressionEvaluationException; @@ -74,6 +74,21 @@ public PhysicalPlan implement(LogicalPlan plan) { }; } + @Bean + protected Table catalogTable() { + return new Table() { + @Override + public Map getFieldTypes() { + return typeMapping(); + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + throw new UnsupportedOperationException(); + } + }; + } + @Bean protected CatalogService catalogService() { return new DefaultCatalogService(); @@ -121,6 +136,9 @@ protected Environment typeEnv() { @Autowired protected Table table; + @Autowired + protected CatalogService catalogService; + @Autowired protected Environment typeEnv; @@ -128,7 +146,7 @@ protected Environment typeEnv() { protected Analyzer analyzer(ExpressionAnalyzer expressionAnalyzer, CatalogService catalogService, StorageEngine storageEngine, BuiltinFunctionRepository functionRepository, Table table) { - catalogService.registerOpenSearchStorageEngine(storageEngine); + catalogService.registerDefaultOpenSearchCatalog(storageEngine); functionRepository.register("prometheus", new FunctionResolver() { @Override @@ -174,20 +192,23 @@ protected LogicalPlan analyze(UnresolvedPlan unresolvedPlan) { private class DefaultCatalogService implements CatalogService { - private StorageEngine storageEngine; + private StorageEngine storageEngine = storageEngine(); + private final Catalog catalog + = new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine); + @Override - public StorageEngine getStorageEngine(String catalog) { - return storageEngine; + public Set getCatalogs() { + return ImmutableSet.of(catalog); } @Override - public Set getCatalogs() { - return ImmutableSet.of("prometheus"); + public Catalog getCatalog(String catalogName) { + return catalog; } @Override - public void registerOpenSearchStorageEngine(StorageEngine storageEngine) { + public void registerDefaultOpenSearchCatalog(StorageEngine storageEngine) { this.storageEngine = storageEngine; } } diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java new file mode 100644 index 0000000000..26374ff042 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java @@ -0,0 +1,70 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.planner.physical.catalog; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Set; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.catalog.model.Catalog; +import org.opensearch.sql.catalog.model.ConnectorType; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.storage.StorageEngine; + +@ExtendWith(MockitoExtension.class) +public class CatalogTableScanTest { + + @Mock + private CatalogService catalogService; + + @Mock + private StorageEngine storageEngine; + + private CatalogTableScan catalogTableScan; + + @BeforeEach + private void setUp() { + catalogTableScan = new CatalogTableScan(catalogService); + } + + @Test + void testExplain() { + assertEquals("GetCatalogRequestRequest{}", catalogTableScan.explain()); + } + + @Test + void testIterator() { + Set catalogSet = new HashSet<>(); + catalogSet.add(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine)); + catalogSet.add(new Catalog("opensearch", ConnectorType.OPENSEARCH, storageEngine)); + when(catalogService.getCatalogs()).thenReturn(catalogSet); + + assertFalse(catalogTableScan.hasNext()); + catalogTableScan.open(); + assertTrue(catalogTableScan.hasNext()); + for (Catalog catalog : catalogSet) { + assertEquals(new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( + "CATALOG_NAME", ExprValueUtils.stringValue(catalog.getName()), + "CONNECTOR_TYPE", ExprValueUtils.stringValue(catalog.getConnectorType().name())))), + catalogTableScan.next()); + } + } + +} diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java new file mode 100644 index 0000000000..59e57a97b3 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java @@ -0,0 +1,49 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.planner.physical.catalog; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.planner.logical.LogicalPlanDSL; +import org.opensearch.sql.planner.physical.PhysicalPlan; + +@ExtendWith(MockitoExtension.class) +public class CatalogTableTest { + + @Mock + private CatalogService catalogService; + + @Test + void testGetFieldTypes() { + CatalogTable catalogTable = new CatalogTable(catalogService); + Map fieldTypes = catalogTable.getFieldTypes(); + Map expectedTypes = new HashMap<>(); + expectedTypes.put("CATALOG_NAME", ExprCoreType.STRING); + expectedTypes.put("CONNECTOR_TYPE", ExprCoreType.STRING); + assertEquals(expectedTypes, fieldTypes); + } + + @Test + void testImplement() { + CatalogTable catalogTable = new CatalogTable(catalogService); + PhysicalPlan physicalPlan + = catalogTable.implement(LogicalPlanDSL.relation(".CATALOGS", catalogTable)); + assertTrue(physicalPlan instanceof CatalogTableScan); + } + +} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java index e6845cb154..94cafef35c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java @@ -34,7 +34,6 @@ import org.opensearch.sql.ppl.domain.PPLQueryRequest; import org.opensearch.sql.protocol.response.QueryResult; import org.opensearch.sql.protocol.response.format.SimpleJsonResponseFormatter; -import org.opensearch.sql.storage.StorageEngine; import org.springframework.context.annotation.AnnotationConfigApplicationContext; /** @@ -59,7 +58,7 @@ public void init() { new OpenSearchExecutionProtector(new AlwaysHealthyMonitor()))); context.register(PPLServiceConfig.class); OpenSearchStorageEngine openSearchStorageEngine = new OpenSearchStorageEngine(client, defaultSettings()); - CatalogServiceImpl.getInstance().registerOpenSearchStorageEngine(openSearchStorageEngine); + CatalogServiceImpl.getInstance().registerDefaultOpenSearchCatalog(openSearchStorageEngine); context.registerBean(CatalogService.class, CatalogServiceImpl::getInstance); context.refresh(); diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index 7deb0c01ab..9234668ea1 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -136,7 +136,7 @@ public Collection createComponents( this.pluginSettings = new OpenSearchSettings(clusterService.getClusterSettings()); this.client = (NodeClient) client; CatalogServiceImpl.getInstance().loadConnectors(clusterService.getSettings()); - CatalogServiceImpl.getInstance().registerOpenSearchStorageEngine(openSearchStorageEngine()); + CatalogServiceImpl.getInstance().registerDefaultOpenSearchCatalog(openSearchStorageEngine()); LocalClusterState.state().setClusterService(clusterService); LocalClusterState.state().setPluginSettings((OpenSearchSettings) pluginSettings); @@ -182,7 +182,7 @@ public ScriptEngine getScriptEngine(Settings settings, Collection storageEngineMap = new HashMap<>(); + private Map catalogMap = new HashMap<>(); public static CatalogServiceImpl getInstance() { return INSTANCE; @@ -85,24 +84,27 @@ public void loadConnectors(Settings settings) { } @Override - public StorageEngine getStorageEngine(String catalog) { - if (!storageEngineMap.containsKey(catalog)) { - return defaultOpenSearchStorageEngine; - } - return storageEngineMap.get(catalog); + public Set getCatalogs() { + return new HashSet<>(catalogMap.values()); } @Override - public Set getCatalogs() { - return Collections.unmodifiableSet(storageEngineMap.keySet()); + public Catalog getCatalog(String catalogName) { + if (!catalogMap.containsKey(catalogName)) { + throw new IllegalArgumentException( + String.format("Catalog with name %s doesn't exist.", catalogName)); + } + return catalogMap.get(catalogName); } + @Override - public void registerOpenSearchStorageEngine(StorageEngine storageEngine) { + public void registerDefaultOpenSearchCatalog(StorageEngine storageEngine) { if (storageEngine == null) { throw new IllegalArgumentException("Default storage engine can't be null"); } - defaultOpenSearchStorageEngine = storageEngine; + catalogMap.put(CatalogName.DEFAULT_CATALOG_NAME, + new Catalog(CatalogName.DEFAULT_CATALOG_NAME, ConnectorType.OPENSEARCH, storageEngine)); } private T doPrivileged(PrivilegedExceptionAction action) { @@ -136,11 +138,12 @@ private StorageEngine createStorageEngine(CatalogMetadata catalog) throws URISyn } private void constructConnectors(List catalogs) throws URISyntaxException { - storageEngineMap = new HashMap<>(); + catalogMap = new HashMap<>(); for (CatalogMetadata catalog : catalogs) { String catalogName = catalog.getName(); StorageEngine storageEngine = createStorageEngine(catalog); - storageEngineMap.put(catalogName, storageEngine); + catalogMap.put(catalogName, + new Catalog(catalog.getName(), catalog.getConnector(), storageEngine)); } } diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java index 805b90e1aa..624467b981 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java @@ -20,6 +20,8 @@ import org.mockito.junit.MockitoJUnitRunner; import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.sql.catalog.model.Catalog; +import org.opensearch.sql.catalog.model.ConnectorType; import org.opensearch.sql.storage.StorageEngine; @RunWith(MockitoJUnitRunner.class) @@ -36,8 +38,8 @@ public class CatalogServiceImplTest { public void testLoadConnectors() { Settings settings = getCatalogSettings("catalogs.json"); CatalogServiceImpl.getInstance().loadConnectors(settings); - Set expected = new HashSet<>() {{ - add("prometheus"); + Set expected = new HashSet<>() {{ + add(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine)); }}; Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); } @@ -48,9 +50,9 @@ public void testLoadConnectors() { public void testLoadConnectorsWithMultipleCatalogs() { Settings settings = getCatalogSettings("multiple_catalogs.json"); CatalogServiceImpl.getInstance().loadConnectors(settings); - Set expected = new HashSet<>() {{ - add("prometheus"); - add("prometheus-1"); + Set expected = new HashSet<>() {{ + add(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine)); + add(new Catalog("prometheus-1", ConnectorType.PROMETHEUS, storageEngine)); }}; Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); } @@ -87,13 +89,33 @@ public void testLoadConnectorsWithMalformedJson() { @Test public void testGetStorageEngineAfterGetCatalogs() { Settings settings = getCatalogSettings("empty_catalog.json"); - CatalogServiceImpl.getInstance().registerOpenSearchStorageEngine(storageEngine); CatalogServiceImpl.getInstance().loadConnectors(settings); - Set expected = new HashSet<>(); + CatalogServiceImpl.getInstance().registerDefaultOpenSearchCatalog(storageEngine); + Set expected = new HashSet<>(); + expected.add(new Catalog(".opensearch", ConnectorType.OPENSEARCH, storageEngine)); Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); - Assert.assertEquals(storageEngine, CatalogServiceImpl.getInstance().getStorageEngine(null)); + Assert.assertEquals(storageEngine, + CatalogServiceImpl.getInstance().getCatalog(".opensearch").getStorageEngine()); + Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); + Assert.assertEquals(storageEngine, + CatalogServiceImpl.getInstance().getCatalog(".opensearch").getStorageEngine()); + IllegalArgumentException illegalArgumentException + = Assert.assertThrows(IllegalArgumentException.class, + () -> CatalogServiceImpl.getInstance().getCatalog("test")); + Assert.assertEquals("Catalog with name test doesn't exist.", + illegalArgumentException.getMessage()); + } + + + @SneakyThrows + @Test + public void testGetStorageEngineAfterLoadingConnectors() { + Settings settings = getCatalogSettings("empty_catalog.json"); + CatalogServiceImpl.getInstance().registerDefaultOpenSearchCatalog(storageEngine); + //Load Connectors will empty the catalogMap.So OpenSearch Storage Engine + CatalogServiceImpl.getInstance().loadConnectors(settings); + Set expected = new HashSet<>(); Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); - Assert.assertEquals(storageEngine, CatalogServiceImpl.getInstance().getStorageEngine(null)); } @SneakyThrows diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index c8f03ff2bc..020d18d76f 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -12,6 +12,7 @@ channels { WHITESPACE, ERRORCHANNEL } // COMMAND KEYWORDS SEARCH: 'SEARCH'; DESCRIBE: 'DESCRIBE'; +SHOW: 'SHOW'; FROM: 'FROM'; WHERE: 'WHERE'; FIELDS: 'FIELDS'; @@ -41,6 +42,7 @@ SOURCE: 'SOURCE'; INDEX: 'INDEX'; D: 'D'; DESC: 'DESC'; +CATALOGS: 'CATALOGS'; // CLAUSE KEYWORDS SORTBY: 'SORTBY'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index cadfc48471..fc87fccfcd 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -21,6 +21,7 @@ pplStatement pplCommands : searchCommand | describeCommand + | showCatalogsCommand ; commands @@ -37,6 +38,10 @@ describeCommand : DESCRIBE tableSourceClause ; +showCatalogsCommand + : SHOW CATALOGS + ; + whereCommand : WHERE logicalExpression ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java b/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java index cfeb0df564..4410f8e33f 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java @@ -40,9 +40,9 @@ public class PPLServiceConfig { @Bean public PPLService pplService() { catalogService.getCatalogs() - .forEach(catalog -> catalogService.getStorageEngine(catalog) - .getFunctions() - .forEach(functionResolver -> functionRepository.register(catalog, functionResolver))); + .forEach(catalog -> catalog.getStorageEngine().getFunctions() + .forEach(functionResolver -> functionRepository + .register(catalog.getName(), functionResolver))); return new PPLService(new PPLSyntaxParser(), executionEngine, functionRepository, catalogService); } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 5898d47658..b0d17940a4 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -24,6 +24,7 @@ import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.TableSourceClauseContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.TopCommandContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.WhereCommandContext; +import static org.opensearch.sql.utils.SystemIndexUtils.CATALOGS_TABLE_NAME; import static org.opensearch.sql.utils.SystemIndexUtils.mappingTable; import com.google.common.collect.ImmutableList; @@ -123,6 +124,16 @@ public UnresolvedPlan visitDescribeCommand(DescribeCommandContext ctx) { return new Relation(qualifiedName(mappingTable(table.getTableName()))); } + /** + * Show command. + */ + @Override + public UnresolvedPlan visitShowCatalogsCommand( + OpenSearchPPLParser.ShowCatalogsCommandContext ctx) { + return new Relation(qualifiedName(CATALOGS_TABLE_NAME)); + } + + /** * Where command. */ diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java index 264bb4dbbd..0ecebf160d 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java @@ -8,22 +8,20 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import java.util.Collections; import java.util.Set; -import org.apache.commons.lang3.tuple.Pair; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import org.opensearch.sql.catalog.CatalogService; +import org.opensearch.sql.catalog.model.Catalog; +import org.opensearch.sql.catalog.model.ConnectorType; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.executor.ExecutionEngine; @@ -32,10 +30,7 @@ import org.opensearch.sql.executor.ExecutionEngine.QueryResponse; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; -import org.opensearch.sql.expression.function.FunctionBuilder; -import org.opensearch.sql.expression.function.FunctionName; import org.opensearch.sql.expression.function.FunctionResolver; -import org.opensearch.sql.expression.function.FunctionSignature; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.ppl.config.PPLServiceConfig; import org.opensearch.sql.ppl.domain.PPLQueryRequest; @@ -84,8 +79,10 @@ public void setUp() { when(table.getFieldTypes()).thenReturn(ImmutableMap.of("a", ExprCoreType.INTEGER)); when(table.implement(any())).thenReturn(plan); when(storageEngine.getTable(any())).thenReturn(table); - when(catalogService.getCatalogs()).thenReturn(Set.of("prometheus")); - when(catalogService.getStorageEngine("prometheus")).thenReturn(storageEngine); + when(catalogService.getCatalogs()) + .thenReturn(Set.of(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine))); + when(catalogService.getCatalog(any())) + .thenReturn(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine)); when(storageEngine.getFunctions()).thenReturn(Collections.singleton(functionResolver)); context.registerBean(StorageEngine.class, () -> storageEngine); @@ -98,7 +95,6 @@ public void setUp() { @Test public void testExecuteShouldPass() { - when(catalogService.getStorageEngine(any())).thenReturn(storageEngine); doAnswer(invocation -> { ResponseListener listener = invocation.getArgument(1); listener.onResponse(new QueryResponse(schema, Collections.emptyList())); @@ -121,7 +117,6 @@ public void onFailure(Exception e) { @Test public void testExecuteCsvFormatShouldPass() { - when(catalogService.getStorageEngine(any())).thenReturn(storageEngine); doAnswer(invocation -> { ResponseListener listener = invocation.getArgument(1); listener.onResponse(new QueryResponse(schema, Collections.emptyList())); @@ -143,7 +138,8 @@ public void onFailure(Exception e) { @Test public void testExplainShouldPass() { - when(catalogService.getStorageEngine(any())).thenReturn(storageEngine); + when(catalogService.getCatalog(any())) + .thenReturn(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine)); doAnswer(invocation -> { ResponseListener listener = invocation.getArgument(1); listener.onResponse(new ExplainResponse(new ExplainResponseNode("test"))); @@ -197,7 +193,6 @@ public void onFailure(Exception e) { @Test public void testPrometheusQuery() { - when(catalogService.getStorageEngine(any())).thenReturn(storageEngine); doAnswer(invocation -> { ResponseListener listener = invocation.getArgument(1); listener.onResponse(new QueryResponse(schema, Collections.emptyList())); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 63326d3424..9bcbe66330 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -773,6 +773,12 @@ public void test_batchRCFADCommand() { new AD(relation("t"), ImmutableMap.of())); } + @Test + public void testShowCatalogsCommand() { + assertEqual("show catalogs", + relation(".CATALOGS")); + } + protected void assertEqual(String query, Node expectedPlan) { Node actualPlan = plan(query); assertEquals(expectedPlan, actualPlan); From 66dc7b7a215ba6c02cbf21be717e42f927be7585 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Tue, 25 Oct 2022 17:20:41 -0400 Subject: [PATCH 29/36] change condition to upload coverage for workbench Signed-off-by: Derek Ho --- .github/workflows/sql-workbench-test-and-build-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index e5f52065b6..06005f3a3a 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -58,7 +58,7 @@ jobs: yarn test:jest --coverage - name: Upload coverage - if: ${{ matrix.os == 'ubuntu-latest' }} + if: ${{ always() && matrix.os == 'ubuntu-latest' }} uses: codecov/codecov-action@v3 with: flags: query-workbench From 2a16227da0cc17a7580ab44aeb6b132d2d053cfb Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Wed, 26 Oct 2022 13:12:16 -0700 Subject: [PATCH 30/36] Add category_field to AD command in PPL (#952) Signed-off-by: Joshua Li --- .../sql/utils/MLCommonsConstants.java | 1 + docs/category.json | 1 + docs/user/ppl/cmd/ad.rst | 69 +- doctest/test_data/nyc_taxi.json | 2919 ++++++----------- doctest/test_mapping/nyc_taxi.json | 5 +- .../planner/physical/ADOperator.java | 118 +- .../physical/MLCommonsOperatorActions.java | 76 +- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 1 + 9 files changed, 1170 insertions(+), 2021 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/MLCommonsConstants.java b/core/src/main/java/org/opensearch/sql/utils/MLCommonsConstants.java index ac560e86bc..883d012d2f 100644 --- a/core/src/main/java/org/opensearch/sql/utils/MLCommonsConstants.java +++ b/core/src/main/java/org/opensearch/sql/utils/MLCommonsConstants.java @@ -9,6 +9,7 @@ public class MLCommonsConstants { public static final String OUTPUT_AFTER = "output_after"; public static final String TIME_DECAY = "time_decay"; public static final String ANOMALY_RATE = "anomaly_rate"; + public static final String CATEGORY_FIELD = "category_field"; public static final String TIME_FIELD = "time_field"; public static final String DATE_FORMAT = "date_format"; public static final String TIME_ZONE = "time_zone"; diff --git a/docs/category.json b/docs/category.json index 3803b7cba9..3f5bdb0a23 100644 --- a/docs/category.json +++ b/docs/category.json @@ -7,6 +7,7 @@ "user/admin/settings.rst" ], "ppl_cli": [ + "user/ppl/cmd/ad.rst", "user/ppl/cmd/dedup.rst", "user/ppl/cmd/describe.rst", "user/ppl/cmd/eval.rst", diff --git a/docs/user/ppl/cmd/ad.rst b/docs/user/ppl/cmd/ad.rst index ed30a2016d..9f61a8dd86 100644 --- a/docs/user/ppl/cmd/ad.rst +++ b/docs/user/ppl/cmd/ad.rst @@ -24,9 +24,10 @@ ad source=nyc_taxi | fields value, timestamp | AD time_field='timestamp' | where value=10844.0' - +----------+---------------+-------+---------------+ - | value | timestamp | score | anomaly_grade | - |----------+---------------+-------+---------------| - | 10844.0 | 1404172800000 | 0.0 | 0.0 | - +----------+---------------+-------+---------------+ + os> source=nyc_taxi | fields value, timestamp | AD time_field='timestamp' | where value=10844.0 + fetched rows / total rows = 1/1 + +---------+---------------------+---------+-----------------+ + | value | timestamp | score | anomaly_grade | + |---------+---------------------+---------+-----------------| + | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | + +---------+---------------------+---------+-----------------+ +Example 2: Detecting events in New York City from taxi ridership data with time-series data independently with each category +============================================================================================================================ -Example2: Detecting events in New York City from taxi ridership data with non-time-series data -============================================================================================== +The example trains an RCF model and uses the model to detect anomalies in the time-series ridership data with multiple category values. + +PPL query:: + + os> source=nyc_taxi | fields category, value, timestamp | AD time_field='timestamp' category_field='category' | where value=10844.0 or value=6526.0 + fetched rows / total rows = 2/2 + +------------+---------+---------------------+---------+-----------------+ + | category | value | timestamp | score | anomaly_grade | + |------------+---------+---------------------+---------+-----------------| + | night | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | + | day | 6526.0 | 2014-07-01 06:00:00 | 0.0 | 0.0 | + +------------+---------+---------------------+---------+-----------------+ + + +Example 3: Detecting events in New York City from taxi ridership data with non-time-series data +=============================================================================================== The example trains an RCF model and uses the model to detect anomalies in the non-time-series ridership data. PPL query:: - os> source=nyc_taxi | fields value | AD | where value=10844.0' - +----------+--------+-----------+ - | value | score | anomalous | - |----------+--------+-----------| - | 10844.0 | 0.0 | false | - +----------+--------+-----------+ + os> source=nyc_taxi | fields value | AD | where value=10844.0 + fetched rows / total rows = 1/1 + +---------+---------+-------------+ + | value | score | anomalous | + |---------+---------+-------------| + | 10844.0 | 0.0 | False | + +---------+---------+-------------+ + +Example 4: Detecting events in New York City from taxi ridership data with non-time-series data independently with each category +================================================================================================================================ + +The example trains an RCF model and uses the model to detect anomalies in the non-time-series ridership data with multiple category values. + +PPL query:: + os> source=nyc_taxi | fields category, value | AD category_field='category' | where value=10844.0 or value=6526.0 + fetched rows / total rows = 2/2 + +------------+---------+---------+-------------+ + | category | value | score | anomalous | + |------------+---------+---------+-------------| + | night | 10844.0 | 0.0 | False | + | day | 6526.0 | 0.0 | False | + +------------+---------+---------+-------------+ diff --git a/doctest/test_data/nyc_taxi.json b/doctest/test_data/nyc_taxi.json index 368dc30076..1b253eefb2 100644 --- a/doctest/test_data/nyc_taxi.json +++ b/doctest/test_data/nyc_taxi.json @@ -1,1946 +1,973 @@ -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 00:00:00","value":10844,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 00:30:00","value":8127,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 01:00:00","value":6210,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 01:30:00","value":4656,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 02:00:00","value":3820,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 02:30:00","value":2873,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 03:00:00","value":2369,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 03:30:00","value":2064,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 04:00:00","value":2221,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 04:30:00","value":2158,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 05:00:00","value":2515,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 05:30:00","value":4364,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 06:00:00","value":6526,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 06:30:00","value":11039,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 07:00:00","value":13857,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 07:30:00","value":15865,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 08:00:00","value":17920,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 08:30:00","value":20346,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 09:00:00","value":19539,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 09:30:00","value":20107,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 10:00:00","value":18984,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 10:30:00","value":17720,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 11:00:00","value":17249,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 11:30:00","value":18463,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 12:00:00","value":18908,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 12:30:00","value":18886,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 13:00:00","value":18178,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 13:30:00","value":19459,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 14:00:00","value":19546,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 14:30:00","value":20591,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 15:00:00","value":19380,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 15:30:00","value":18544,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 16:00:00","value":16228,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 16:30:00","value":15013,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 17:00:00","value":17203,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 17:30:00","value":19525,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 18:00:00","value":22966,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 18:30:00","value":27598,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 19:00:00","value":26827,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 19:30:00","value":24904,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 20:00:00","value":22875,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 20:30:00","value":20394,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 21:00:00","value":23401,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 21:30:00","value":24439,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 22:00:00","value":23318,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 22:30:00","value":21733,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 23:00:00","value":20104,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-01 23:30:00","value":16111,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 00:00:00","value":13370,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 00:30:00","value":9945,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 01:00:00","value":7571,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 01:30:00","value":5917,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 02:00:00","value":4820,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 02:30:00","value":3634,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 03:00:00","value":2993,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 03:30:00","value":2535,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 04:00:00","value":2570,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 04:30:00","value":2485,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 05:00:00","value":2868,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 05:30:00","value":4482,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 06:00:00","value":6788,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 06:30:00","value":11078,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 07:00:00","value":13729,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 07:30:00","value":16700,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 08:00:00","value":19156,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 08:30:00","value":19953,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 09:00:00","value":19502,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 09:30:00","value":18994,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 10:00:00","value":17311,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 10:30:00","value":17904,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 11:00:00","value":17133,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 11:30:00","value":18589,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 12:00:00","value":19134,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 12:30:00","value":19259,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 13:00:00","value":18667,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 13:30:00","value":19078,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 14:00:00","value":18546,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 14:30:00","value":18593,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 15:00:00","value":17967,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 15:30:00","value":16624,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 16:00:00","value":14634,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 16:30:00","value":13888,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 17:00:00","value":17430,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 17:30:00","value":21919,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 18:00:00","value":23633,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 18:30:00","value":24512,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 19:00:00","value":24887,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 19:30:00","value":26872,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 20:00:00","value":22009,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 20:30:00","value":18259,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 21:00:00","value":20844,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 21:30:00","value":22576,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 22:00:00","value":22401,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 22:30:00","value":19056,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 23:00:00","value":17518,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-02 23:30:00","value":15307,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 00:00:00","value":12646,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 00:30:00","value":10562,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 01:00:00","value":8416,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 01:30:00","value":7098,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 02:00:00","value":5826,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 02:30:00","value":4383,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 03:00:00","value":3270,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 03:30:00","value":2948,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 04:00:00","value":3146,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 04:30:00","value":3077,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 05:00:00","value":3000,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 05:30:00","value":4592,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 06:00:00","value":6486,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 06:30:00","value":10113,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 07:00:00","value":12240,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 07:30:00","value":14574,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 08:00:00","value":16778,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 08:30:00","value":18910,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 09:00:00","value":18350,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 09:30:00","value":17218,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 10:00:00","value":16097,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 10:30:00","value":16409,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 11:00:00","value":15893,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 11:30:00","value":16778,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 12:00:00","value":17604,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 12:30:00","value":18665,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 13:00:00","value":19045,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 13:30:00","value":19261,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 14:00:00","value":19363,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 14:30:00","value":19078,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 15:00:00","value":18193,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 15:30:00","value":16635,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 16:00:00","value":14615,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 16:30:00","value":13759,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 17:00:00","value":17008,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 17:30:00","value":19595,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 18:00:00","value":21328,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 18:30:00","value":22661,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 19:00:00","value":29985,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 19:30:00","value":21501,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 20:00:00","value":22684,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 20:30:00","value":22188,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 21:00:00","value":22663,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 21:30:00","value":19573,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 22:00:00","value":17136,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 22:30:00","value":16606,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 23:00:00","value":16166,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-03 23:30:00","value":16020,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 00:00:00","value":15591,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 00:30:00","value":14395,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 01:00:00","value":12535,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 01:30:00","value":11341,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 02:00:00","value":9980,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 02:30:00","value":8404,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 03:00:00","value":7200,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 03:30:00","value":6578,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 04:00:00","value":5657,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 04:30:00","value":4474,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 05:00:00","value":3459,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 05:30:00","value":3276,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 06:00:00","value":3595,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 06:30:00","value":4240,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 07:00:00","value":4828,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 07:30:00","value":4926,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 08:00:00","value":5165,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 08:30:00","value":5776,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 09:00:00","value":7338,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 09:30:00","value":7839,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 10:00:00","value":8623,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 10:30:00","value":9731,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 11:00:00","value":11024,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 11:30:00","value":13231,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 12:00:00","value":13613,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 12:30:00","value":13737,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 13:00:00","value":15574,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 13:30:00","value":14226,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 14:00:00","value":18480,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 14:30:00","value":18265,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 15:00:00","value":16575,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 15:30:00","value":16417,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 16:00:00","value":14703,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 16:30:00","value":13469,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 17:00:00","value":12105,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 17:30:00","value":11676,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 18:00:00","value":15487,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 18:30:00","value":15077,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 19:00:00","value":14999,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 19:30:00","value":14487,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 20:00:00","value":14415,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 20:30:00","value":13796,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 21:00:00","value":14036,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 21:30:00","value":14021,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 22:00:00","value":15593,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 22:30:00","value":16589,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 23:00:00","value":17984,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-04 23:30:00","value":18035,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 00:00:00","value":17576,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 00:30:00","value":16189,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 01:00:00","value":14441,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 01:30:00","value":12535,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 02:00:00","value":11006,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 02:30:00","value":9151,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 03:00:00","value":8010,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 03:30:00","value":7096,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 04:00:00","value":6407,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 04:30:00","value":4421,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 05:00:00","value":3126,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 05:30:00","value":2514,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 06:00:00","value":2550,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 06:30:00","value":3148,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 07:00:00","value":3658,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 07:30:00","value":4345,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 08:00:00","value":4682,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 08:30:00","value":6248,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 09:00:00","value":7454,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 09:30:00","value":9010,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 10:00:00","value":10280,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 10:30:00","value":11488,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 11:00:00","value":11595,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 11:30:00","value":13098,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 12:00:00","value":12623,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 12:30:00","value":13031,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 13:00:00","value":13263,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 13:30:00","value":13349,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 14:00:00","value":13822,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 14:30:00","value":13716,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 15:00:00","value":13919,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 15:30:00","value":14203,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 16:00:00","value":13179,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 16:30:00","value":13708,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 17:00:00","value":13897,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 17:30:00","value":14740,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 18:00:00","value":14575,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 18:30:00","value":16085,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 19:00:00","value":18182,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 19:30:00","value":16861,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 20:00:00","value":14140,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 20:30:00","value":14477,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 21:00:00","value":15293,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 21:30:00","value":15457,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 22:00:00","value":16048,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 22:30:00","value":17477,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 23:00:00","value":16391,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-05 23:30:00","value":17006,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 00:00:00","value":15427,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 00:30:00","value":14615,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 01:00:00","value":13124,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 01:30:00","value":12222,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 02:00:00","value":11134,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 02:30:00","value":9145,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 03:00:00","value":8624,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 03:30:00","value":7885,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 04:00:00","value":7167,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 04:30:00","value":4805,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 05:00:00","value":3103,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 05:30:00","value":2671,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 06:00:00","value":2510,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 06:30:00","value":2917,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 07:00:00","value":3189,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 07:30:00","value":4107,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 08:00:00","value":4122,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 08:30:00","value":5654,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 09:00:00","value":6360,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 09:30:00","value":8406,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 10:00:00","value":9372,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 10:30:00","value":11067,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 11:00:00","value":11595,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 11:30:00","value":12909,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 12:00:00","value":13715,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 12:30:00","value":13648,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 13:00:00","value":14296,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 13:30:00","value":14798,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 14:00:00","value":15473,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 14:30:00","value":16032,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 15:00:00","value":14661,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 15:30:00","value":14836,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 16:00:00","value":13700,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 16:30:00","value":14565,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 17:00:00","value":15392,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 17:30:00","value":16866,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 18:00:00","value":16893,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 18:30:00","value":16877,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 19:00:00","value":17025,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 19:30:00","value":15884,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 20:00:00","value":14487,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 20:30:00","value":14159,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 21:00:00","value":16135,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 21:30:00","value":16165,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 22:00:00","value":14025,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 22:30:00","value":13970,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 23:00:00","value":13198,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-06 23:30:00","value":11355,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 00:00:00","value":8675,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 00:30:00","value":7180,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 01:00:00","value":5178,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 01:30:00","value":3658,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 02:00:00","value":3181,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 02:30:00","value":2402,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 03:00:00","value":1944,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 03:30:00","value":1877,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 04:00:00","value":2257,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 04:30:00","value":2280,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 05:00:00","value":2575,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 05:30:00","value":4174,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 06:00:00","value":6346,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 06:30:00","value":10594,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 07:00:00","value":12632,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 07:30:00","value":14893,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 08:00:00","value":16470,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 08:30:00","value":18998,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 09:00:00","value":17792,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 09:30:00","value":16396,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 10:00:00","value":14128,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 10:30:00","value":14161,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 11:00:00","value":14154,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 11:30:00","value":15074,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 12:00:00","value":15188,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 12:30:00","value":15483,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 13:00:00","value":15338,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 13:30:00","value":16242,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 14:00:00","value":16579,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 14:30:00","value":16885,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 15:00:00","value":16824,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 15:30:00","value":16238,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 16:00:00","value":15702,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 16:30:00","value":15132,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 17:00:00","value":17500,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 17:30:00","value":19167,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 18:00:00","value":21398,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 18:30:00","value":22382,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 19:00:00","value":22270,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 19:30:00","value":20575,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 20:00:00","value":18824,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 20:30:00","value":17909,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 21:00:00","value":19707,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 21:30:00","value":19066,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 22:00:00","value":17755,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 22:30:00","value":16583,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 23:00:00","value":14955,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-07 23:30:00","value":11849,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 00:00:00","value":9292,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 00:30:00","value":8110,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 01:00:00","value":7352,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 01:30:00","value":5049,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 02:00:00","value":3451,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 02:30:00","value":2465,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 03:00:00","value":2125,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 03:30:00","value":1877,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 04:00:00","value":2069,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 04:30:00","value":2080,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 05:00:00","value":2375,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 05:30:00","value":4303,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 06:00:00","value":6537,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 06:30:00","value":11331,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 07:00:00","value":13565,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 07:30:00","value":16455,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 08:00:00","value":18310,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 08:30:00","value":20288,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 09:00:00","value":19564,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 09:30:00","value":19380,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 10:00:00","value":16507,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 10:30:00","value":16939,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 11:00:00","value":16113,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 11:30:00","value":17537,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 12:00:00","value":18120,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 12:30:00","value":18038,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 13:00:00","value":17870,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 13:30:00","value":18427,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 14:00:00","value":18971,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 14:30:00","value":19071,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 15:00:00","value":18646,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 15:30:00","value":18229,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 16:00:00","value":15977,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 16:30:00","value":15026,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 17:00:00","value":17398,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 17:30:00","value":20865,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 18:00:00","value":23875,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 18:30:00","value":25290,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 19:00:00","value":25510,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 19:30:00","value":24535,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 20:00:00","value":21922,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 20:30:00","value":20113,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 21:00:00","value":22079,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 21:30:00","value":23111,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 22:00:00","value":25209,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 22:30:00","value":21978,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 23:00:00","value":18320,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-08 23:30:00","value":14881,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 00:00:00","value":12053,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 00:30:00","value":9409,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 01:00:00","value":7740,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 01:30:00","value":5528,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 02:00:00","value":4667,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 02:30:00","value":3242,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 03:00:00","value":2678,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 03:30:00","value":2370,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 04:00:00","value":2475,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 04:30:00","value":2304,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 05:00:00","value":2491,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 05:30:00","value":4117,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 06:00:00","value":6435,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 06:30:00","value":11067,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 07:00:00","value":13384,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 07:30:00","value":17194,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 08:00:00","value":18510,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 08:30:00","value":20464,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 09:00:00","value":19777,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 09:30:00","value":18928,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 10:00:00","value":17243,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 10:30:00","value":17490,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 11:00:00","value":16558,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 11:30:00","value":17830,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 12:00:00","value":18203,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 12:30:00","value":18126,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 13:00:00","value":18122,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 13:30:00","value":18488,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 14:00:00","value":18487,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 14:30:00","value":18542,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 15:00:00","value":18240,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 15:30:00","value":17393,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 16:00:00","value":15175,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 16:30:00","value":15360,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 17:00:00","value":17103,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 17:30:00","value":19561,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 18:00:00","value":22262,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 18:30:00","value":24725,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 19:00:00","value":25995,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 19:30:00","value":26319,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 20:00:00","value":24995,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 20:30:00","value":20534,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 21:00:00","value":23458,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 21:30:00","value":24681,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 22:00:00","value":23955,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 22:30:00","value":23655,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 23:00:00","value":21896,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-09 23:30:00","value":19338,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 00:00:00","value":15185,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 00:30:00","value":11459,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 01:00:00","value":8847,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 01:30:00","value":6580,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 02:00:00","value":5247,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 02:30:00","value":4127,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 03:00:00","value":3440,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 03:30:00","value":2957,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 04:00:00","value":2779,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 04:30:00","value":2532,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 05:00:00","value":2718,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 05:30:00","value":4449,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 06:00:00","value":6601,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 06:30:00","value":11202,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 07:00:00","value":13934,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 07:30:00","value":17176,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 08:00:00","value":19057,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 08:30:00","value":21112,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 09:00:00","value":19882,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 09:30:00","value":19024,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 10:00:00","value":16989,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 10:30:00","value":16979,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 11:00:00","value":16381,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 11:30:00","value":17815,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 12:00:00","value":18029,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 12:30:00","value":17495,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 13:00:00","value":17075,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 13:30:00","value":18234,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 14:00:00","value":18091,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 14:30:00","value":18495,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 15:00:00","value":17523,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 15:30:00","value":16714,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 16:00:00","value":14735,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 16:30:00","value":13610,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 17:00:00","value":16290,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 17:30:00","value":19152,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 18:00:00","value":21865,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 18:30:00","value":24347,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 19:00:00","value":26186,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 19:30:00","value":25852,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 20:00:00","value":23995,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 20:30:00","value":21664,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 21:00:00","value":25027,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 21:30:00","value":25431,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 22:00:00","value":25643,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 22:30:00","value":24654,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 23:00:00","value":23154,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-10 23:30:00","value":21863,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 00:00:00","value":20051,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 00:30:00","value":16122,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 01:00:00","value":13107,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 01:30:00","value":10506,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 02:00:00","value":8444,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 02:30:00","value":6876,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 03:00:00","value":5375,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 03:30:00","value":4366,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 04:00:00","value":4183,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 04:30:00","value":3249,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 05:00:00","value":3134,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 05:30:00","value":4620,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 06:00:00","value":6725,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 06:30:00","value":10651,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 07:00:00","value":12952,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 07:30:00","value":15808,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 08:00:00","value":17565,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 08:30:00","value":19784,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 09:00:00","value":19699,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 09:30:00","value":18663,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 10:00:00","value":16509,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 10:30:00","value":16600,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 11:00:00","value":15636,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 11:30:00","value":17434,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 12:00:00","value":17668,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 12:30:00","value":17124,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 13:00:00","value":17124,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 13:30:00","value":17489,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 14:00:00","value":18371,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 14:30:00","value":18381,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 15:00:00","value":17898,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 15:30:00","value":16350,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 16:00:00","value":14688,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 16:30:00","value":14227,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 17:00:00","value":16924,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 17:30:00","value":19952,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 18:00:00","value":22665,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 18:30:00","value":23465,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 19:00:00","value":25111,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 19:30:00","value":23984,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 20:00:00","value":21701,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 20:30:00","value":20592,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 21:00:00","value":22630,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 21:30:00","value":22854,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 22:00:00","value":23892,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 22:30:00","value":24959,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 23:00:00","value":26039,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-11 23:30:00","value":26873,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 00:00:00","value":25871,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 00:30:00","value":24874,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 01:00:00","value":23243,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 01:30:00","value":21674,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 02:00:00","value":19221,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 02:30:00","value":16140,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 03:00:00","value":13371,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 03:30:00","value":12041,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 04:00:00","value":10301,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 04:30:00","value":6472,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 05:00:00","value":4507,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 05:30:00","value":3682,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 06:00:00","value":3422,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 06:30:00","value":4554,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 07:00:00","value":5347,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 07:30:00","value":6853,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 08:00:00","value":7107,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 08:30:00","value":9463,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 09:00:00","value":11022,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 09:30:00","value":13393,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 10:00:00","value":13567,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 10:30:00","value":15452,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 11:00:00","value":15525,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 11:30:00","value":17165,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 12:00:00","value":17263,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 12:30:00","value":18418,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 13:00:00","value":18578,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 13:30:00","value":18762,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 14:00:00","value":18076,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 14:30:00","value":18604,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 15:00:00","value":18580,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 15:30:00","value":19306,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 16:00:00","value":18140,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 16:30:00","value":17455,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 17:00:00","value":18980,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 17:30:00","value":21152,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 18:00:00","value":22483,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 18:30:00","value":22534,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 19:00:00","value":22801,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 19:30:00","value":22117,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 20:00:00","value":19864,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 20:30:00","value":19494,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 21:00:00","value":20607,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 21:30:00","value":20627,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 22:00:00","value":21706,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 22:30:00","value":24243,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 23:00:00","value":25204,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-12 23:30:00","value":25752,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 00:00:00","value":25792,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 00:30:00","value":25033,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 01:00:00","value":23935,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 01:30:00","value":21440,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 02:00:00","value":19468,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 02:30:00","value":16622,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 03:00:00","value":14485,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 03:30:00","value":12974,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 04:00:00","value":11191,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 04:30:00","value":6911,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 05:00:00","value":4410,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 05:30:00","value":3467,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 06:00:00","value":3429,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 06:30:00","value":3599,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 07:00:00","value":3575,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 07:30:00","value":4557,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 08:00:00","value":5243,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 08:30:00","value":6588,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 09:00:00","value":8009,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 09:30:00","value":10743,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 10:00:00","value":13524,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 10:30:00","value":16179,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 11:00:00","value":14905,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 11:30:00","value":16916,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 12:00:00","value":17082,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 12:30:00","value":18606,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 13:00:00","value":18935,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 13:30:00","value":20175,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 14:00:00","value":22219,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 14:30:00","value":22868,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 15:00:00","value":20375,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 15:30:00","value":18489,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 16:00:00","value":16187,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 16:30:00","value":14015,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 17:00:00","value":14261,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 17:30:00","value":20081,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 18:00:00","value":21503,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 18:30:00","value":19850,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 19:00:00","value":18383,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 19:30:00","value":17640,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 20:00:00","value":16225,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 20:30:00","value":15566,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 21:00:00","value":17088,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 21:30:00","value":16968,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 22:00:00","value":15271,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 22:30:00","value":14141,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 23:00:00","value":12851,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-13 23:30:00","value":13877,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 00:00:00","value":12484,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 00:30:00","value":9037,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 01:00:00","value":7393,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 01:30:00","value":5176,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 02:00:00","value":3479,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 02:30:00","value":2755,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 03:00:00","value":2027,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 03:30:00","value":1769,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 04:00:00","value":2091,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 04:30:00","value":2553,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 05:00:00","value":2853,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 05:30:00","value":4835,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 06:00:00","value":6603,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 06:30:00","value":11230,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 07:00:00","value":13395,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 07:30:00","value":15650,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 08:00:00","value":17601,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 08:30:00","value":18818,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 09:00:00","value":18515,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 09:30:00","value":16972,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 10:00:00","value":15316,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 10:30:00","value":16003,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 11:00:00","value":14818,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 11:30:00","value":15610,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 12:00:00","value":16536,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 12:30:00","value":16153,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 13:00:00","value":15548,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 13:30:00","value":16500,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 14:00:00","value":16726,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 14:30:00","value":16838,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 15:00:00","value":16550,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 15:30:00","value":16621,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 16:00:00","value":15657,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 16:30:00","value":15334,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 17:00:00","value":17584,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 17:30:00","value":20903,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 18:00:00","value":21968,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 18:30:00","value":26945,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 19:00:00","value":24416,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 19:30:00","value":22401,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 20:00:00","value":23549,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 20:30:00","value":21498,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 21:00:00","value":23114,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 21:30:00","value":23341,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 22:00:00","value":22141,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 22:30:00","value":19110,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 23:00:00","value":16682,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-14 23:30:00","value":12631,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 00:00:00","value":10089,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 00:30:00","value":8553,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 01:00:00","value":6416,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 01:30:00","value":4694,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 02:00:00","value":3933,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 02:30:00","value":2833,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 03:00:00","value":2089,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 03:30:00","value":1896,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 04:00:00","value":2055,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 04:30:00","value":2031,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 05:00:00","value":2449,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 05:30:00","value":4360,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 06:00:00","value":7036,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 06:30:00","value":11730,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 07:00:00","value":14387,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 07:30:00","value":17505,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 08:00:00","value":19091,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 08:30:00","value":21057,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 09:00:00","value":20050,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 09:30:00","value":18637,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 10:00:00","value":17555,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 10:30:00","value":17595,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 11:00:00","value":16312,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 11:30:00","value":18232,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 12:00:00","value":18446,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 12:30:00","value":18204,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 13:00:00","value":17607,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 13:30:00","value":18945,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 14:00:00","value":22208,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 14:30:00","value":21574,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 15:00:00","value":17299,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 15:30:00","value":15515,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 16:00:00","value":13246,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 16:30:00","value":12328,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 17:00:00","value":15342,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 17:30:00","value":18730,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 18:00:00","value":23412,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 18:30:00","value":26340,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 19:00:00","value":27167,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 19:30:00","value":26279,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 20:00:00","value":23392,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 20:30:00","value":21571,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 21:00:00","value":23477,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 21:30:00","value":22612,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 22:00:00","value":21389,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 22:30:00","value":19575,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 23:00:00","value":18165,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-15 23:30:00","value":14923,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 00:00:00","value":11815,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 00:30:00","value":9024,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 01:00:00","value":7363,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 01:30:00","value":5812,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 02:00:00","value":4559,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 02:30:00","value":3673,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 03:00:00","value":2830,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 03:30:00","value":2374,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 04:00:00","value":2556,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 04:30:00","value":2456,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 05:00:00","value":2486,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 05:30:00","value":4451,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 06:00:00","value":6723,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 06:30:00","value":12501,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 07:00:00","value":14763,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 07:30:00","value":18127,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 08:00:00","value":20393,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 08:30:00","value":20753,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 09:00:00","value":20124,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 09:30:00","value":19253,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 10:00:00","value":17981,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 10:30:00","value":17720,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 11:00:00","value":16525,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 11:30:00","value":18153,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 12:00:00","value":18558,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 12:30:00","value":17652,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 13:00:00","value":17292,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 13:30:00","value":17551,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 14:00:00","value":17951,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 14:30:00","value":17909,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 15:00:00","value":17442,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 15:30:00","value":16533,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 16:00:00","value":14776,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 16:30:00","value":13462,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 17:00:00","value":16363,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 17:30:00","value":19310,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 18:00:00","value":22346,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 18:30:00","value":24408,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 19:00:00","value":26225,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 19:30:00","value":25423,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 20:00:00","value":23811,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 20:30:00","value":22028,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 21:00:00","value":24290,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 21:30:00","value":24835,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 22:00:00","value":24269,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 22:30:00","value":23526,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 23:00:00","value":21968,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-16 23:30:00","value":20137,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 00:00:00","value":16928,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 00:30:00","value":12753,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 01:00:00","value":10087,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 01:30:00","value":7881,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 02:00:00","value":6006,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 02:30:00","value":4382,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 03:00:00","value":3676,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 03:30:00","value":3214,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 04:00:00","value":3205,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 04:30:00","value":2849,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 05:00:00","value":2887,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 05:30:00","value":5039,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 06:00:00","value":7132,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 06:30:00","value":12095,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 07:00:00","value":14558,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 07:30:00","value":17298,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 08:00:00","value":19124,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 08:30:00","value":20407,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 09:00:00","value":19379,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 09:30:00","value":18867,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 10:00:00","value":17662,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 10:30:00","value":17447,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 11:00:00","value":16579,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 11:30:00","value":18340,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 12:00:00","value":18760,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 12:30:00","value":18457,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 13:00:00","value":17608,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 13:30:00","value":18913,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 14:00:00","value":19122,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 14:30:00","value":19547,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 15:00:00","value":17267,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 15:30:00","value":15916,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 16:00:00","value":13836,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 16:30:00","value":11985,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 17:00:00","value":14313,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 17:30:00","value":17988,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 18:00:00","value":21181,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 18:30:00","value":23539,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 19:00:00","value":24714,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 19:30:00","value":25079,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 20:00:00","value":23032,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 20:30:00","value":21168,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 21:00:00","value":25514,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 21:30:00","value":26286,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 22:00:00","value":25650,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 22:30:00","value":24850,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 23:00:00","value":23869,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-17 23:30:00","value":22913,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 00:00:00","value":20850,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 00:30:00","value":16734,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 01:00:00","value":14106,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 01:30:00","value":11587,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 02:00:00","value":8951,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 02:30:00","value":7199,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 03:00:00","value":6051,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 03:30:00","value":4693,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 04:00:00","value":4507,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 04:30:00","value":3791,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 05:00:00","value":3586,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 05:30:00","value":4918,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 06:00:00","value":7039,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 06:30:00","value":11262,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 07:00:00","value":13725,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 07:30:00","value":15899,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 08:00:00","value":17329,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 08:30:00","value":19757,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 09:00:00","value":19341,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 09:30:00","value":17660,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 10:00:00","value":16532,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 10:30:00","value":16354,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 11:00:00","value":16054,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 11:30:00","value":17326,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 12:00:00","value":17463,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 12:30:00","value":17091,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 13:00:00","value":16668,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 13:30:00","value":17096,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 14:00:00","value":17811,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 14:30:00","value":17980,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 15:00:00","value":17080,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 15:30:00","value":15185,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 16:00:00","value":13538,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 16:30:00","value":12704,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 17:00:00","value":15019,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 17:30:00","value":18778,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 18:00:00","value":21583,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 18:30:00","value":23834,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 19:00:00","value":25123,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 19:30:00","value":24762,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 20:00:00","value":22761,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 20:30:00","value":22227,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 21:00:00","value":23985,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 21:30:00","value":23788,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 22:00:00","value":23855,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 22:30:00","value":26040,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 23:00:00","value":25863,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-18 23:30:00","value":25851,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 00:00:00","value":26100,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 00:30:00","value":24625,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 01:00:00","value":22657,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 01:30:00","value":20289,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 02:00:00","value":18524,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 02:30:00","value":15943,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 03:00:00","value":13179,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 03:30:00","value":12423,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 04:00:00","value":10478,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 04:30:00","value":6556,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 05:00:00","value":4561,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 05:30:00","value":3513,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 06:00:00","value":3607,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 06:30:00","value":4781,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 07:00:00","value":5423,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 07:30:00","value":6669,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 08:00:00","value":7064,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 08:30:00","value":9363,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 09:00:00","value":10874,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 09:30:00","value":13255,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 10:00:00","value":13164,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 10:30:00","value":15159,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 11:00:00","value":16030,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 11:30:00","value":18256,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 12:00:00","value":17751,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 12:30:00","value":17675,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 13:00:00","value":18557,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 13:30:00","value":18389,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 14:00:00","value":17538,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 14:30:00","value":17506,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 15:00:00","value":17580,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 15:30:00","value":18027,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 16:00:00","value":16959,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 16:30:00","value":17066,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 17:00:00","value":18155,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 17:30:00","value":20610,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 18:00:00","value":20793,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 18:30:00","value":21584,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 19:00:00","value":23493,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 19:30:00","value":22555,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 20:00:00","value":20183,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 20:30:00","value":20441,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 21:00:00","value":21555,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 21:30:00","value":22406,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 22:00:00","value":22512,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 22:30:00","value":24667,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 23:00:00","value":25424,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-19 23:30:00","value":25852,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 00:00:00","value":25137,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 00:30:00","value":24099,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 01:00:00","value":23058,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 01:30:00","value":20786,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 02:00:00","value":19217,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 02:30:00","value":16329,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 03:00:00","value":14293,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 03:30:00","value":13193,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 04:00:00","value":11166,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 04:30:00","value":7518,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 05:00:00","value":4877,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 05:30:00","value":3639,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 06:00:00","value":3412,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 06:30:00","value":3827,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 07:00:00","value":3922,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 07:30:00","value":5241,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 08:00:00","value":5601,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 08:30:00","value":7147,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 09:00:00","value":8425,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 09:30:00","value":10951,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 10:00:00","value":11800,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 10:30:00","value":13936,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 11:00:00","value":14835,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 11:30:00","value":16412,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 12:00:00","value":16763,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 12:30:00","value":17613,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 13:00:00","value":17439,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 13:30:00","value":17921,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 14:00:00","value":18605,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 14:30:00","value":18113,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 15:00:00","value":17579,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 15:30:00","value":16927,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 16:00:00","value":16526,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 16:30:00","value":16956,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 17:00:00","value":17381,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 17:30:00","value":19232,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 18:00:00","value":19127,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 18:30:00","value":19404,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 19:00:00","value":18812,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 19:30:00","value":18253,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 20:00:00","value":16497,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 20:30:00","value":16681,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 21:00:00","value":17334,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 21:30:00","value":17674,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 22:00:00","value":16469,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 22:30:00","value":15128,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 23:00:00","value":13973,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-20 23:30:00","value":12040,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 00:00:00","value":9494,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 00:30:00","value":6963,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 01:00:00","value":5611,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 01:30:00","value":4140,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 02:00:00","value":3370,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 02:30:00","value":2625,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 03:00:00","value":2093,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 03:30:00","value":1854,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 04:00:00","value":2482,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 04:30:00","value":2529,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 05:00:00","value":2968,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 05:30:00","value":4540,"anomaly_type":"EXPECTED"} -{ "index" : { "_index" : "nyc_taxi" } } -{"timestamp":"2014-07-21 06:00:00","value":6868,"anomaly_type":"EXPECTED"} \ No newline at end of file +{"timestamp":"2014-07-01 00:00:00","value":10844,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 00:30:00","value":8127,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 01:00:00","value":6210,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 01:30:00","value":4656,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 02:00:00","value":3820,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 02:30:00","value":2873,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 03:00:00","value":2369,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 03:30:00","value":2064,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 04:00:00","value":2221,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 04:30:00","value":2158,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 05:00:00","value":2515,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 05:30:00","value":4364,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 06:00:00","value":6526,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 06:30:00","value":11039,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 07:00:00","value":13857,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 07:30:00","value":15865,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 08:00:00","value":17920,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 08:30:00","value":20346,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 09:00:00","value":19539,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 09:30:00","value":20107,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 10:00:00","value":18984,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 10:30:00","value":17720,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 11:00:00","value":17249,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 11:30:00","value":18463,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 12:00:00","value":18908,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 12:30:00","value":18886,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 13:00:00","value":18178,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 13:30:00","value":19459,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 14:00:00","value":19546,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 14:30:00","value":20591,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 15:00:00","value":19380,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 15:30:00","value":18544,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 16:00:00","value":16228,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 16:30:00","value":15013,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 17:00:00","value":17203,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 17:30:00","value":19525,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 18:00:00","value":22966,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 18:30:00","value":27598,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 19:00:00","value":26827,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 19:30:00","value":24904,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-01 20:00:00","value":22875,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 20:30:00","value":20394,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 21:00:00","value":23401,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 21:30:00","value":24439,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 22:00:00","value":23318,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 22:30:00","value":21733,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 23:00:00","value":20104,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-01 23:30:00","value":16111,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 00:00:00","value":13370,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 00:30:00","value":9945,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 01:00:00","value":7571,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 01:30:00","value":5917,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 02:00:00","value":4820,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 02:30:00","value":3634,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 03:00:00","value":2993,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 03:30:00","value":2535,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 04:00:00","value":2570,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 04:30:00","value":2485,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 05:00:00","value":2868,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 05:30:00","value":4482,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 06:00:00","value":6788,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 06:30:00","value":11078,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 07:00:00","value":13729,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 07:30:00","value":16700,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 08:00:00","value":19156,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 08:30:00","value":19953,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 09:00:00","value":19502,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 09:30:00","value":18994,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 10:00:00","value":17311,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 10:30:00","value":17904,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 11:00:00","value":17133,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 11:30:00","value":18589,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 12:00:00","value":19134,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 12:30:00","value":19259,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 13:00:00","value":18667,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 13:30:00","value":19078,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 14:00:00","value":18546,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 14:30:00","value":18593,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 15:00:00","value":17967,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 15:30:00","value":16624,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 16:00:00","value":14634,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 16:30:00","value":13888,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 17:00:00","value":17430,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 17:30:00","value":21919,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 18:00:00","value":23633,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 18:30:00","value":24512,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 19:00:00","value":24887,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 19:30:00","value":26872,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-02 20:00:00","value":22009,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 20:30:00","value":18259,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 21:00:00","value":20844,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 21:30:00","value":22576,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 22:00:00","value":22401,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 22:30:00","value":19056,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 23:00:00","value":17518,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-02 23:30:00","value":15307,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 00:00:00","value":12646,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 00:30:00","value":10562,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 01:00:00","value":8416,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 01:30:00","value":7098,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 02:00:00","value":5826,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 02:30:00","value":4383,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 03:00:00","value":3270,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 03:30:00","value":2948,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 04:00:00","value":3146,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 04:30:00","value":3077,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 05:00:00","value":3000,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 05:30:00","value":4592,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 06:00:00","value":6486,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 06:30:00","value":10113,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 07:00:00","value":12240,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 07:30:00","value":14574,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 08:00:00","value":16778,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 08:30:00","value":18910,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 09:00:00","value":18350,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 09:30:00","value":17218,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 10:00:00","value":16097,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 10:30:00","value":16409,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 11:00:00","value":15893,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 11:30:00","value":16778,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 12:00:00","value":17604,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 12:30:00","value":18665,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 13:00:00","value":19045,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 13:30:00","value":19261,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 14:00:00","value":19363,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 14:30:00","value":19078,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 15:00:00","value":18193,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 15:30:00","value":16635,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 16:00:00","value":14615,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 16:30:00","value":13759,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 17:00:00","value":17008,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 17:30:00","value":19595,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 18:00:00","value":21328,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 18:30:00","value":22661,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 19:00:00","value":29985,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 19:30:00","value":21501,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-03 20:00:00","value":22684,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 20:30:00","value":22188,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 21:00:00","value":22663,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 21:30:00","value":19573,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 22:00:00","value":17136,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 22:30:00","value":16606,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 23:00:00","value":16166,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-03 23:30:00","value":16020,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 00:00:00","value":15591,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 00:30:00","value":14395,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 01:00:00","value":12535,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 01:30:00","value":11341,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 02:00:00","value":9980,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 02:30:00","value":8404,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 03:00:00","value":7200,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 03:30:00","value":6578,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 04:00:00","value":5657,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 04:30:00","value":4474,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 05:00:00","value":3459,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 05:30:00","value":3276,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 06:00:00","value":3595,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 06:30:00","value":4240,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 07:00:00","value":4828,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 07:30:00","value":4926,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 08:00:00","value":5165,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 08:30:00","value":5776,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 09:00:00","value":7338,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 09:30:00","value":7839,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 10:00:00","value":8623,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 10:30:00","value":9731,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 11:00:00","value":11024,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 11:30:00","value":13231,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 12:00:00","value":13613,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 12:30:00","value":13737,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 13:00:00","value":15574,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 13:30:00","value":14226,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 14:00:00","value":18480,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 14:30:00","value":18265,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 15:00:00","value":16575,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 15:30:00","value":16417,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 16:00:00","value":14703,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 16:30:00","value":13469,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 17:00:00","value":12105,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 17:30:00","value":11676,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 18:00:00","value":15487,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 18:30:00","value":15077,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 19:00:00","value":14999,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 19:30:00","value":14487,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-04 20:00:00","value":14415,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 20:30:00","value":13796,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 21:00:00","value":14036,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 21:30:00","value":14021,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 22:00:00","value":15593,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 22:30:00","value":16589,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 23:00:00","value":17984,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-04 23:30:00","value":18035,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 00:00:00","value":17576,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 00:30:00","value":16189,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 01:00:00","value":14441,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 01:30:00","value":12535,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 02:00:00","value":11006,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 02:30:00","value":9151,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 03:00:00","value":8010,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 03:30:00","value":7096,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 04:00:00","value":6407,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 04:30:00","value":4421,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 05:00:00","value":3126,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 05:30:00","value":2514,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 06:00:00","value":2550,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 06:30:00","value":3148,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 07:00:00","value":3658,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 07:30:00","value":4345,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 08:00:00","value":4682,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 08:30:00","value":6248,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 09:00:00","value":7454,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 09:30:00","value":9010,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 10:00:00","value":10280,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 10:30:00","value":11488,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 11:00:00","value":11595,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 11:30:00","value":13098,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 12:00:00","value":12623,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 12:30:00","value":13031,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 13:00:00","value":13263,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 13:30:00","value":13349,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 14:00:00","value":13822,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 14:30:00","value":13716,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 15:00:00","value":13919,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 15:30:00","value":14203,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 16:00:00","value":13179,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 16:30:00","value":13708,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 17:00:00","value":13897,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 17:30:00","value":14740,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 18:00:00","value":14575,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 18:30:00","value":16085,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 19:00:00","value":18182,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 19:30:00","value":16861,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-05 20:00:00","value":14140,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 20:30:00","value":14477,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 21:00:00","value":15293,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 21:30:00","value":15457,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 22:00:00","value":16048,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 22:30:00","value":17477,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 23:00:00","value":16391,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-05 23:30:00","value":17006,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 00:00:00","value":15427,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 00:30:00","value":14615,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 01:00:00","value":13124,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 01:30:00","value":12222,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 02:00:00","value":11134,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 02:30:00","value":9145,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 03:00:00","value":8624,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 03:30:00","value":7885,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 04:00:00","value":7167,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 04:30:00","value":4805,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 05:00:00","value":3103,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 05:30:00","value":2671,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 06:00:00","value":2510,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 06:30:00","value":2917,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 07:00:00","value":3189,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 07:30:00","value":4107,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 08:00:00","value":4122,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 08:30:00","value":5654,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 09:00:00","value":6360,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 09:30:00","value":8406,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 10:00:00","value":9372,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 10:30:00","value":11067,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 11:00:00","value":11595,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 11:30:00","value":12909,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 12:00:00","value":13715,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 12:30:00","value":13648,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 13:00:00","value":14296,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 13:30:00","value":14798,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 14:00:00","value":15473,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 14:30:00","value":16032,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 15:00:00","value":14661,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 15:30:00","value":14836,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 16:00:00","value":13700,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 16:30:00","value":14565,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 17:00:00","value":15392,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 17:30:00","value":16866,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 18:00:00","value":16893,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 18:30:00","value":16877,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 19:00:00","value":17025,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 19:30:00","value":15884,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-06 20:00:00","value":14487,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 20:30:00","value":14159,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 21:00:00","value":16135,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 21:30:00","value":16165,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 22:00:00","value":14025,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 22:30:00","value":13970,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 23:00:00","value":13198,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-06 23:30:00","value":11355,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 00:00:00","value":8675,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 00:30:00","value":7180,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 01:00:00","value":5178,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 01:30:00","value":3658,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 02:00:00","value":3181,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 02:30:00","value":2402,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 03:00:00","value":1944,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 03:30:00","value":1877,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 04:00:00","value":2257,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 04:30:00","value":2280,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 05:00:00","value":2575,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 05:30:00","value":4174,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 06:00:00","value":6346,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 06:30:00","value":10594,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 07:00:00","value":12632,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 07:30:00","value":14893,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 08:00:00","value":16470,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 08:30:00","value":18998,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 09:00:00","value":17792,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 09:30:00","value":16396,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 10:00:00","value":14128,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 10:30:00","value":14161,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 11:00:00","value":14154,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 11:30:00","value":15074,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 12:00:00","value":15188,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 12:30:00","value":15483,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 13:00:00","value":15338,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 13:30:00","value":16242,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 14:00:00","value":16579,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 14:30:00","value":16885,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 15:00:00","value":16824,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 15:30:00","value":16238,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 16:00:00","value":15702,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 16:30:00","value":15132,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 17:00:00","value":17500,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 17:30:00","value":19167,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 18:00:00","value":21398,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 18:30:00","value":22382,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 19:00:00","value":22270,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 19:30:00","value":20575,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-07 20:00:00","value":18824,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 20:30:00","value":17909,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 21:00:00","value":19707,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 21:30:00","value":19066,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 22:00:00","value":17755,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 22:30:00","value":16583,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 23:00:00","value":14955,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-07 23:30:00","value":11849,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 00:00:00","value":9292,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 00:30:00","value":8110,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 01:00:00","value":7352,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 01:30:00","value":5049,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 02:00:00","value":3451,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 02:30:00","value":2465,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 03:00:00","value":2125,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 03:30:00","value":1877,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 04:00:00","value":2069,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 04:30:00","value":2080,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 05:00:00","value":2375,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 05:30:00","value":4303,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 06:00:00","value":6537,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 06:30:00","value":11331,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 07:00:00","value":13565,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 07:30:00","value":16455,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 08:00:00","value":18310,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 08:30:00","value":20288,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 09:00:00","value":19564,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 09:30:00","value":19380,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 10:00:00","value":16507,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 10:30:00","value":16939,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 11:00:00","value":16113,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 11:30:00","value":17537,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 12:00:00","value":18120,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 12:30:00","value":18038,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 13:00:00","value":17870,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 13:30:00","value":18427,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 14:00:00","value":18971,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 14:30:00","value":19071,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 15:00:00","value":18646,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 15:30:00","value":18229,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 16:00:00","value":15977,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 16:30:00","value":15026,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 17:00:00","value":17398,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 17:30:00","value":20865,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 18:00:00","value":23875,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 18:30:00","value":25290,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 19:00:00","value":25510,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 19:30:00","value":24535,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-08 20:00:00","value":21922,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 20:30:00","value":20113,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 21:00:00","value":22079,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 21:30:00","value":23111,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 22:00:00","value":25209,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 22:30:00","value":21978,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 23:00:00","value":18320,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-08 23:30:00","value":14881,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 00:00:00","value":12053,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 00:30:00","value":9409,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 01:00:00","value":7740,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 01:30:00","value":5528,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 02:00:00","value":4667,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 02:30:00","value":3242,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 03:00:00","value":2678,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 03:30:00","value":2370,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 04:00:00","value":2475,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 04:30:00","value":2304,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 05:00:00","value":2491,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 05:30:00","value":4117,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 06:00:00","value":6435,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 06:30:00","value":11067,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 07:00:00","value":13384,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 07:30:00","value":17194,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 08:00:00","value":18510,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 08:30:00","value":20464,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 09:00:00","value":19777,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 09:30:00","value":18928,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 10:00:00","value":17243,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 10:30:00","value":17490,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 11:00:00","value":16558,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 11:30:00","value":17830,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 12:00:00","value":18203,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 12:30:00","value":18126,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 13:00:00","value":18122,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 13:30:00","value":18488,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 14:00:00","value":18487,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 14:30:00","value":18542,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 15:00:00","value":18240,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 15:30:00","value":17393,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 16:00:00","value":15175,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 16:30:00","value":15360,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 17:00:00","value":17103,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 17:30:00","value":19561,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 18:00:00","value":22262,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 18:30:00","value":24725,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 19:00:00","value":25995,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 19:30:00","value":26319,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-09 20:00:00","value":24995,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 20:30:00","value":20534,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 21:00:00","value":23458,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 21:30:00","value":24681,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 22:00:00","value":23955,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 22:30:00","value":23655,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 23:00:00","value":21896,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-09 23:30:00","value":19338,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 00:00:00","value":15185,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 00:30:00","value":11459,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 01:00:00","value":8847,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 01:30:00","value":6580,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 02:00:00","value":5247,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 02:30:00","value":4127,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 03:00:00","value":3440,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 03:30:00","value":2957,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 04:00:00","value":2779,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 04:30:00","value":2532,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 05:00:00","value":2718,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 05:30:00","value":4449,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 06:00:00","value":6601,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 06:30:00","value":11202,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 07:00:00","value":13934,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 07:30:00","value":17176,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 08:00:00","value":19057,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 08:30:00","value":21112,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 09:00:00","value":19882,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 09:30:00","value":19024,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 10:00:00","value":16989,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 10:30:00","value":16979,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 11:00:00","value":16381,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 11:30:00","value":17815,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 12:00:00","value":18029,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 12:30:00","value":17495,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 13:00:00","value":17075,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 13:30:00","value":18234,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 14:00:00","value":18091,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 14:30:00","value":18495,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 15:00:00","value":17523,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 15:30:00","value":16714,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 16:00:00","value":14735,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 16:30:00","value":13610,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 17:00:00","value":16290,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 17:30:00","value":19152,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 18:00:00","value":21865,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 18:30:00","value":24347,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 19:00:00","value":26186,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 19:30:00","value":25852,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-10 20:00:00","value":23995,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 20:30:00","value":21664,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 21:00:00","value":25027,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 21:30:00","value":25431,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 22:00:00","value":25643,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 22:30:00","value":24654,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 23:00:00","value":23154,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-10 23:30:00","value":21863,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 00:00:00","value":20051,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 00:30:00","value":16122,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 01:00:00","value":13107,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 01:30:00","value":10506,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 02:00:00","value":8444,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 02:30:00","value":6876,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 03:00:00","value":5375,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 03:30:00","value":4366,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 04:00:00","value":4183,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 04:30:00","value":3249,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 05:00:00","value":3134,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 05:30:00","value":4620,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 06:00:00","value":6725,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 06:30:00","value":10651,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 07:00:00","value":12952,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 07:30:00","value":15808,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 08:00:00","value":17565,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 08:30:00","value":19784,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 09:00:00","value":19699,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 09:30:00","value":18663,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 10:00:00","value":16509,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 10:30:00","value":16600,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 11:00:00","value":15636,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 11:30:00","value":17434,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 12:00:00","value":17668,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 12:30:00","value":17124,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 13:00:00","value":17124,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 13:30:00","value":17489,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 14:00:00","value":18371,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 14:30:00","value":18381,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 15:00:00","value":17898,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 15:30:00","value":16350,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 16:00:00","value":14688,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 16:30:00","value":14227,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 17:00:00","value":16924,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 17:30:00","value":19952,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 18:00:00","value":22665,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 18:30:00","value":23465,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 19:00:00","value":25111,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 19:30:00","value":23984,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-11 20:00:00","value":21701,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 20:30:00","value":20592,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 21:00:00","value":22630,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 21:30:00","value":22854,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 22:00:00","value":23892,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 22:30:00","value":24959,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 23:00:00","value":26039,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-11 23:30:00","value":26873,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 00:00:00","value":25871,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 00:30:00","value":24874,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 01:00:00","value":23243,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 01:30:00","value":21674,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 02:00:00","value":19221,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 02:30:00","value":16140,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 03:00:00","value":13371,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 03:30:00","value":12041,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 04:00:00","value":10301,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 04:30:00","value":6472,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 05:00:00","value":4507,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 05:30:00","value":3682,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 06:00:00","value":3422,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 06:30:00","value":4554,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 07:00:00","value":5347,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 07:30:00","value":6853,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 08:00:00","value":7107,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 08:30:00","value":9463,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 09:00:00","value":11022,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 09:30:00","value":13393,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 10:00:00","value":13567,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 10:30:00","value":15452,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 11:00:00","value":15525,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 11:30:00","value":17165,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 12:00:00","value":17263,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 12:30:00","value":18418,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 13:00:00","value":18578,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 13:30:00","value":18762,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 14:00:00","value":18076,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 14:30:00","value":18604,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 15:00:00","value":18580,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 15:30:00","value":19306,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 16:00:00","value":18140,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 16:30:00","value":17455,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 17:00:00","value":18980,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 17:30:00","value":21152,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 18:00:00","value":22483,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 18:30:00","value":22534,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 19:00:00","value":22801,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 19:30:00","value":22117,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-12 20:00:00","value":19864,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 20:30:00","value":19494,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 21:00:00","value":20607,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 21:30:00","value":20627,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 22:00:00","value":21706,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 22:30:00","value":24243,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 23:00:00","value":25204,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-12 23:30:00","value":25752,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 00:00:00","value":25792,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 00:30:00","value":25033,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 01:00:00","value":23935,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 01:30:00","value":21440,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 02:00:00","value":19468,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 02:30:00","value":16622,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 03:00:00","value":14485,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 03:30:00","value":12974,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 04:00:00","value":11191,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 04:30:00","value":6911,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 05:00:00","value":4410,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 05:30:00","value":3467,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 06:00:00","value":3429,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 06:30:00","value":3599,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 07:00:00","value":3575,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 07:30:00","value":4557,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 08:00:00","value":5243,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 08:30:00","value":6588,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 09:00:00","value":8009,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 09:30:00","value":10743,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 10:00:00","value":13524,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 10:30:00","value":16179,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 11:00:00","value":14905,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 11:30:00","value":16916,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 12:00:00","value":17082,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 12:30:00","value":18606,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 13:00:00","value":18935,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 13:30:00","value":20175,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 14:00:00","value":22219,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 14:30:00","value":22868,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 15:00:00","value":20375,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 15:30:00","value":18489,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 16:00:00","value":16187,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 16:30:00","value":14015,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 17:00:00","value":14261,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 17:30:00","value":20081,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 18:00:00","value":21503,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 18:30:00","value":19850,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 19:00:00","value":18383,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 19:30:00","value":17640,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-13 20:00:00","value":16225,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 20:30:00","value":15566,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 21:00:00","value":17088,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 21:30:00","value":16968,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 22:00:00","value":15271,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 22:30:00","value":14141,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 23:00:00","value":12851,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-13 23:30:00","value":13877,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 00:00:00","value":12484,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 00:30:00","value":9037,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 01:00:00","value":7393,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 01:30:00","value":5176,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 02:00:00","value":3479,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 02:30:00","value":2755,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 03:00:00","value":2027,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 03:30:00","value":1769,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 04:00:00","value":2091,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 04:30:00","value":2553,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 05:00:00","value":2853,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 05:30:00","value":4835,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 06:00:00","value":6603,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 06:30:00","value":11230,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 07:00:00","value":13395,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 07:30:00","value":15650,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 08:00:00","value":17601,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 08:30:00","value":18818,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 09:00:00","value":18515,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 09:30:00","value":16972,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 10:00:00","value":15316,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 10:30:00","value":16003,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 11:00:00","value":14818,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 11:30:00","value":15610,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 12:00:00","value":16536,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 12:30:00","value":16153,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 13:00:00","value":15548,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 13:30:00","value":16500,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 14:00:00","value":16726,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 14:30:00","value":16838,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 15:00:00","value":16550,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 15:30:00","value":16621,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 16:00:00","value":15657,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 16:30:00","value":15334,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 17:00:00","value":17584,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 17:30:00","value":20903,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 18:00:00","value":21968,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 18:30:00","value":26945,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 19:00:00","value":24416,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 19:30:00","value":22401,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-14 20:00:00","value":23549,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 20:30:00","value":21498,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 21:00:00","value":23114,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 21:30:00","value":23341,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 22:00:00","value":22141,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 22:30:00","value":19110,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 23:00:00","value":16682,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-14 23:30:00","value":12631,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 00:00:00","value":10089,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 00:30:00","value":8553,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 01:00:00","value":6416,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 01:30:00","value":4694,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 02:00:00","value":3933,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 02:30:00","value":2833,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 03:00:00","value":2089,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 03:30:00","value":1896,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 04:00:00","value":2055,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 04:30:00","value":2031,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 05:00:00","value":2449,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 05:30:00","value":4360,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 06:00:00","value":7036,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 06:30:00","value":11730,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 07:00:00","value":14387,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 07:30:00","value":17505,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 08:00:00","value":19091,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 08:30:00","value":21057,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 09:00:00","value":20050,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 09:30:00","value":18637,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 10:00:00","value":17555,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 10:30:00","value":17595,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 11:00:00","value":16312,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 11:30:00","value":18232,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 12:00:00","value":18446,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 12:30:00","value":18204,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 13:00:00","value":17607,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 13:30:00","value":18945,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 14:00:00","value":22208,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 14:30:00","value":21574,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 15:00:00","value":17299,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 15:30:00","value":15515,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 16:00:00","value":13246,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 16:30:00","value":12328,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 17:00:00","value":15342,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 17:30:00","value":18730,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 18:00:00","value":23412,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 18:30:00","value":26340,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 19:00:00","value":27167,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 19:30:00","value":26279,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-15 20:00:00","value":23392,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 20:30:00","value":21571,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 21:00:00","value":23477,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 21:30:00","value":22612,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 22:00:00","value":21389,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 22:30:00","value":19575,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 23:00:00","value":18165,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-15 23:30:00","value":14923,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 00:00:00","value":11815,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 00:30:00","value":9024,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 01:00:00","value":7363,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 01:30:00","value":5812,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 02:00:00","value":4559,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 02:30:00","value":3673,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 03:00:00","value":2830,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 03:30:00","value":2374,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 04:00:00","value":2556,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 04:30:00","value":2456,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 05:00:00","value":2486,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 05:30:00","value":4451,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 06:00:00","value":6723,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 06:30:00","value":12501,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 07:00:00","value":14763,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 07:30:00","value":18127,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 08:00:00","value":20393,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 08:30:00","value":20753,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 09:00:00","value":20124,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 09:30:00","value":19253,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 10:00:00","value":17981,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 10:30:00","value":17720,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 11:00:00","value":16525,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 11:30:00","value":18153,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 12:00:00","value":18558,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 12:30:00","value":17652,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 13:00:00","value":17292,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 13:30:00","value":17551,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 14:00:00","value":17951,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 14:30:00","value":17909,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 15:00:00","value":17442,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 15:30:00","value":16533,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 16:00:00","value":14776,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 16:30:00","value":13462,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 17:00:00","value":16363,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 17:30:00","value":19310,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 18:00:00","value":22346,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 18:30:00","value":24408,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 19:00:00","value":26225,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 19:30:00","value":25423,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-16 20:00:00","value":23811,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 20:30:00","value":22028,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 21:00:00","value":24290,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 21:30:00","value":24835,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 22:00:00","value":24269,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 22:30:00","value":23526,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 23:00:00","value":21968,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-16 23:30:00","value":20137,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 00:00:00","value":16928,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 00:30:00","value":12753,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 01:00:00","value":10087,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 01:30:00","value":7881,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 02:00:00","value":6006,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 02:30:00","value":4382,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 03:00:00","value":3676,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 03:30:00","value":3214,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 04:00:00","value":3205,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 04:30:00","value":2849,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 05:00:00","value":2887,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 05:30:00","value":5039,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 06:00:00","value":7132,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 06:30:00","value":12095,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 07:00:00","value":14558,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 07:30:00","value":17298,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 08:00:00","value":19124,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 08:30:00","value":20407,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 09:00:00","value":19379,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 09:30:00","value":18867,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 10:00:00","value":17662,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 10:30:00","value":17447,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 11:00:00","value":16579,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 11:30:00","value":18340,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 12:00:00","value":18760,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 12:30:00","value":18457,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 13:00:00","value":17608,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 13:30:00","value":18913,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 14:00:00","value":19122,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 14:30:00","value":19547,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 15:00:00","value":17267,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 15:30:00","value":15916,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 16:00:00","value":13836,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 16:30:00","value":11985,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 17:00:00","value":14313,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 17:30:00","value":17988,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 18:00:00","value":21181,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 18:30:00","value":23539,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 19:00:00","value":24714,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 19:30:00","value":25079,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-17 20:00:00","value":23032,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 20:30:00","value":21168,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 21:00:00","value":25514,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 21:30:00","value":26286,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 22:00:00","value":25650,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 22:30:00","value":24850,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 23:00:00","value":23869,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-17 23:30:00","value":22913,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 00:00:00","value":20850,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 00:30:00","value":16734,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 01:00:00","value":14106,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 01:30:00","value":11587,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 02:00:00","value":8951,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 02:30:00","value":7199,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 03:00:00","value":6051,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 03:30:00","value":4693,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 04:00:00","value":4507,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 04:30:00","value":3791,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 05:00:00","value":3586,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 05:30:00","value":4918,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 06:00:00","value":7039,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 06:30:00","value":11262,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 07:00:00","value":13725,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 07:30:00","value":15899,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 08:00:00","value":17329,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 08:30:00","value":19757,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 09:00:00","value":19341,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 09:30:00","value":17660,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 10:00:00","value":16532,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 10:30:00","value":16354,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 11:00:00","value":16054,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 11:30:00","value":17326,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 12:00:00","value":17463,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 12:30:00","value":17091,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 13:00:00","value":16668,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 13:30:00","value":17096,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 14:00:00","value":17811,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 14:30:00","value":17980,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 15:00:00","value":17080,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 15:30:00","value":15185,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 16:00:00","value":13538,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 16:30:00","value":12704,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 17:00:00","value":15019,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 17:30:00","value":18778,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 18:00:00","value":21583,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 18:30:00","value":23834,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 19:00:00","value":25123,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 19:30:00","value":24762,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-18 20:00:00","value":22761,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 20:30:00","value":22227,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 21:00:00","value":23985,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 21:30:00","value":23788,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 22:00:00","value":23855,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 22:30:00","value":26040,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 23:00:00","value":25863,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-18 23:30:00","value":25851,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 00:00:00","value":26100,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 00:30:00","value":24625,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 01:00:00","value":22657,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 01:30:00","value":20289,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 02:00:00","value":18524,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 02:30:00","value":15943,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 03:00:00","value":13179,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 03:30:00","value":12423,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 04:00:00","value":10478,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 04:30:00","value":6556,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 05:00:00","value":4561,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 05:30:00","value":3513,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 06:00:00","value":3607,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 06:30:00","value":4781,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 07:00:00","value":5423,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 07:30:00","value":6669,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 08:00:00","value":7064,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 08:30:00","value":9363,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 09:00:00","value":10874,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 09:30:00","value":13255,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 10:00:00","value":13164,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 10:30:00","value":15159,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 11:00:00","value":16030,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 11:30:00","value":18256,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 12:00:00","value":17751,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 12:30:00","value":17675,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 13:00:00","value":18557,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 13:30:00","value":18389,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 14:00:00","value":17538,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 14:30:00","value":17506,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 15:00:00","value":17580,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 15:30:00","value":18027,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 16:00:00","value":16959,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 16:30:00","value":17066,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 17:00:00","value":18155,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 17:30:00","value":20610,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 18:00:00","value":20793,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 18:30:00","value":21584,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 19:00:00","value":23493,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 19:30:00","value":22555,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-19 20:00:00","value":20183,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 20:30:00","value":20441,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 21:00:00","value":21555,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 21:30:00","value":22406,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 22:00:00","value":22512,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 22:30:00","value":24667,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 23:00:00","value":25424,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-19 23:30:00","value":25852,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 00:00:00","value":25137,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 00:30:00","value":24099,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 01:00:00","value":23058,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 01:30:00","value":20786,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 02:00:00","value":19217,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 02:30:00","value":16329,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 03:00:00","value":14293,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 03:30:00","value":13193,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 04:00:00","value":11166,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 04:30:00","value":7518,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 05:00:00","value":4877,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 05:30:00","value":3639,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 06:00:00","value":3412,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 06:30:00","value":3827,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 07:00:00","value":3922,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 07:30:00","value":5241,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 08:00:00","value":5601,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 08:30:00","value":7147,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 09:00:00","value":8425,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 09:30:00","value":10951,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 10:00:00","value":11800,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 10:30:00","value":13936,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 11:00:00","value":14835,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 11:30:00","value":16412,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 12:00:00","value":16763,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 12:30:00","value":17613,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 13:00:00","value":17439,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 13:30:00","value":17921,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 14:00:00","value":18605,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 14:30:00","value":18113,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 15:00:00","value":17579,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 15:30:00","value":16927,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 16:00:00","value":16526,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 16:30:00","value":16956,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 17:00:00","value":17381,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 17:30:00","value":19232,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 18:00:00","value":19127,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 18:30:00","value":19404,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 19:00:00","value":18812,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 19:30:00","value":18253,"anomaly_type":"EXPECTED","category":"day"} +{"timestamp":"2014-07-20 20:00:00","value":16497,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 20:30:00","value":16681,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 21:00:00","value":17334,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 21:30:00","value":17674,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 22:00:00","value":16469,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 22:30:00","value":15128,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 23:00:00","value":13973,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-20 23:30:00","value":12040,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 00:00:00","value":9494,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 00:30:00","value":6963,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 01:00:00","value":5611,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 01:30:00","value":4140,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 02:00:00","value":3370,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 02:30:00","value":2625,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 03:00:00","value":2093,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 03:30:00","value":1854,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 04:00:00","value":2482,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 04:30:00","value":2529,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 05:00:00","value":2968,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 05:30:00","value":4540,"anomaly_type":"EXPECTED","category":"night"} +{"timestamp":"2014-07-21 06:00:00","value":6868,"anomaly_type":"EXPECTED","category":"day"} diff --git a/doctest/test_mapping/nyc_taxi.json b/doctest/test_mapping/nyc_taxi.json index d9932379ad..4d5bb31198 100644 --- a/doctest/test_mapping/nyc_taxi.json +++ b/doctest/test_mapping/nyc_taxi.json @@ -4,6 +4,9 @@ "anomaly_type": { "type": "keyword" }, + "category": { + "type": "keyword" + }, "timestamp": { "type": "date", "format": "yyyy-MM-dd HH:mm:ss" @@ -13,4 +16,4 @@ } } } -} \ No newline at end of file +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/ADOperator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/ADOperator.java index 3730b5725e..7a0ae7c960 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/ADOperator.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/ADOperator.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.utils.MLCommonsConstants.ANOMALY_RATE; import static org.opensearch.sql.utils.MLCommonsConstants.ANOMALY_SCORE_THRESHOLD; +import static org.opensearch.sql.utils.MLCommonsConstants.CATEGORY_FIELD; import static org.opensearch.sql.utils.MLCommonsConstants.DATE_FORMAT; import static org.opensearch.sql.utils.MLCommonsConstants.NUMBER_OF_TREES; import static org.opensearch.sql.utils.MLCommonsConstants.OUTPUT_AFTER; @@ -22,9 +23,11 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.tuple.Pair; import org.opensearch.client.node.NodeClient; import org.opensearch.ml.common.FunctionName; import org.opensearch.ml.common.dataframe.DataFrame; @@ -63,22 +66,37 @@ public class ADOperator extends MLCommonsOperatorActions { @Override public void open() { super.open(); - DataFrame inputDataFrame = generateInputDataset(input); + String categoryField = arguments.containsKey(CATEGORY_FIELD) + ? (String) arguments.get(CATEGORY_FIELD).getValue() : null; + List> + inputDataFrames = generateCategorizedInputDataset(input, categoryField); MLAlgoParams mlAlgoParams = convertArgumentToMLParameter(arguments); - MLPredictionOutput predictionResult = - getMLPredictionResult(rcfType, mlAlgoParams, inputDataFrame, nodeClient); + List predictionResults = inputDataFrames.stream() + .map(pair -> getMLPredictionResult(rcfType, mlAlgoParams, pair.getRight(), nodeClient)) + .collect(Collectors.toList()); - Iterator inputRowIter = inputDataFrame.iterator(); - Iterator resultRowIter = predictionResult.getPredictionResult().iterator(); + Iterator> inputDataFramesIter = inputDataFrames.iterator(); + Iterator predictionResultIter = predictionResults.iterator(); iterator = new Iterator() { + private DataFrame inputDataFrame = null; + private Iterator inputRowIter = null; + private MLPredictionOutput predictionResult = null; + private Iterator resultRowIter = null; + @Override public boolean hasNext() { - return inputRowIter.hasNext(); + return inputRowIter != null && inputRowIter.hasNext() || inputDataFramesIter.hasNext(); } @Override public ExprValue next() { + if (inputRowIter == null || !inputRowIter.hasNext()) { + inputDataFrame = inputDataFramesIter.next().getLeft(); + inputRowIter = inputDataFrame.iterator(); + predictionResult = predictionResultIter.next(); + resultRowIter = predictionResult.getPredictionResult().iterator(); + } return buildResult(inputRowIter, inputDataFrame, predictionResult, resultRowIter); } }; @@ -108,53 +126,53 @@ protected MLAlgoParams convertArgumentToMLParameter(Map argumen if (arguments.get(TIME_FIELD) == null) { rcfType = FunctionName.BATCH_RCF; return BatchRCFParams.builder() - .numberOfTrees(arguments.containsKey(NUMBER_OF_TREES) - ? ((Integer) arguments.get(NUMBER_OF_TREES).getValue()) - : null) - .sampleSize(arguments.containsKey(SAMPLE_SIZE) - ? ((Integer) arguments.get(SAMPLE_SIZE).getValue()) - : null) - .outputAfter(arguments.containsKey(OUTPUT_AFTER) - ? ((Integer) arguments.get(OUTPUT_AFTER).getValue()) - : null) - .trainingDataSize(arguments.containsKey(TRAINING_DATA_SIZE) - ? ((Integer) arguments.get(TRAINING_DATA_SIZE).getValue()) - : null) - .anomalyScoreThreshold(arguments.containsKey(ANOMALY_SCORE_THRESHOLD) - ? ((Double) arguments.get(ANOMALY_SCORE_THRESHOLD).getValue()) - : null) - .build(); + .numberOfTrees(arguments.containsKey(NUMBER_OF_TREES) + ? ((Integer) arguments.get(NUMBER_OF_TREES).getValue()) + : null) + .sampleSize(arguments.containsKey(SAMPLE_SIZE) + ? ((Integer) arguments.get(SAMPLE_SIZE).getValue()) + : null) + .outputAfter(arguments.containsKey(OUTPUT_AFTER) + ? ((Integer) arguments.get(OUTPUT_AFTER).getValue()) + : null) + .trainingDataSize(arguments.containsKey(TRAINING_DATA_SIZE) + ? ((Integer) arguments.get(TRAINING_DATA_SIZE).getValue()) + : null) + .anomalyScoreThreshold(arguments.containsKey(ANOMALY_SCORE_THRESHOLD) + ? ((Double) arguments.get(ANOMALY_SCORE_THRESHOLD).getValue()) + : null) + .build(); } rcfType = FunctionName.FIT_RCF; return FitRCFParams.builder() - .numberOfTrees(arguments.containsKey(NUMBER_OF_TREES) - ? ((Integer) arguments.get(NUMBER_OF_TREES).getValue()) - : null) - .shingleSize(arguments.containsKey(SHINGLE_SIZE) - ? ((Integer) arguments.get(SHINGLE_SIZE).getValue()) - : null) - .sampleSize(arguments.containsKey(SAMPLE_SIZE) - ? ((Integer) arguments.get(SAMPLE_SIZE).getValue()) - : null) - .outputAfter(arguments.containsKey(OUTPUT_AFTER) - ? ((Integer) arguments.get(OUTPUT_AFTER).getValue()) - : null) - .timeDecay(arguments.containsKey(TIME_DECAY) - ? ((Double) arguments.get(TIME_DECAY).getValue()) - : null) - .anomalyRate(arguments.containsKey(ANOMALY_RATE) - ? ((Double) arguments.get(ANOMALY_RATE).getValue()) - : null) - .timeField(arguments.containsKey(TIME_FIELD) - ? ((String) arguments.get(TIME_FIELD).getValue()) - : null) - .dateFormat(arguments.containsKey(DATE_FORMAT) - ? ((String) arguments.get(DATE_FORMAT).getValue()) - : "yyyy-MM-dd HH:mm:ss") - .timeZone(arguments.containsKey(TIME_ZONE) - ? ((String) arguments.get(TIME_ZONE).getValue()) - : null) - .build(); + .numberOfTrees(arguments.containsKey(NUMBER_OF_TREES) + ? ((Integer) arguments.get(NUMBER_OF_TREES).getValue()) + : null) + .shingleSize(arguments.containsKey(SHINGLE_SIZE) + ? ((Integer) arguments.get(SHINGLE_SIZE).getValue()) + : null) + .sampleSize(arguments.containsKey(SAMPLE_SIZE) + ? ((Integer) arguments.get(SAMPLE_SIZE).getValue()) + : null) + .outputAfter(arguments.containsKey(OUTPUT_AFTER) + ? ((Integer) arguments.get(OUTPUT_AFTER).getValue()) + : null) + .timeDecay(arguments.containsKey(TIME_DECAY) + ? ((Double) arguments.get(TIME_DECAY).getValue()) + : null) + .anomalyRate(arguments.containsKey(ANOMALY_RATE) + ? ((Double) arguments.get(ANOMALY_RATE).getValue()) + : null) + .timeField(arguments.containsKey(TIME_FIELD) + ? ((String) arguments.get(TIME_FIELD).getValue()) + : null) + .dateFormat(arguments.containsKey(DATE_FORMAT) + ? ((String) arguments.get(DATE_FORMAT).getValue()) + : "yyyy-MM-dd HH:mm:ss") + .timeZone(arguments.containsKey(TIME_ZONE) + ? ((String) arguments.get(TIME_ZONE).getValue()) + : null) + .build(); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/MLCommonsOperatorActions.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/MLCommonsOperatorActions.java index 61892abe16..9003d2ec47 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/MLCommonsOperatorActions.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/MLCommonsOperatorActions.java @@ -13,6 +13,10 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.opensearch.client.node.NodeClient; import org.opensearch.ml.client.MachineLearningNodeClient; import org.opensearch.ml.common.FunctionName; @@ -48,16 +52,40 @@ public abstract class MLCommonsOperatorActions extends PhysicalPlan { * @return ml-commons dataframe */ protected DataFrame generateInputDataset(PhysicalPlan input) { - List> inputData = new LinkedList<>(); + MLInputRows inputData = new MLInputRows(); while (input.hasNext()) { - inputData.add(new HashMap() { - { - input.next().tupleValue().forEach((key, value) -> put(key, value.value())); - } - }); + inputData.addTupleValue(input.next().tupleValue()); } - return DataFrameBuilder.load(inputData); + return inputData.toDataFrame(); + } + + /** + * Generate ml-commons request input dataset per each category based on a given category field. + * Each category value will be a {@link DataFrame} pair, where the left one contains all fields + * for building response, and the right one contains all fields except the aggregated field for + * ml prediction. This is a temporary solution before ml-commons supports 2 dimensional input. + * + * @param input physical input + * @param categoryField String, the field should be aggregated on + * @return list of ml-commons dataframe pairs + */ + protected List> generateCategorizedInputDataset(PhysicalPlan input, + String categoryField) { + Map inputMap = new HashMap<>(); + while (input.hasNext()) { + Map tupleValue = input.next().tupleValue(); + ExprValue categoryValue = categoryField == null ? null : tupleValue.get(categoryField); + MLInputRows inputData = + inputMap.computeIfAbsent(categoryValue, k -> new MLInputRows()); + inputData.addTupleValue(tupleValue); + } + + // categoryField should be excluded for ml-commons predictions + return inputMap.values().stream().filter(inputData -> inputData.size() > 0).map( + inputData -> new ImmutablePair<>(inputData.toDataFrame(), + inputData.toFilteredDataFrame(e -> !e.getKey().equals(categoryField)))) + .collect(Collectors.toList()); } /** @@ -188,4 +216,38 @@ protected MLPredictionOutput getMLPredictionResult(FunctionName functionName, .actionGet(30, TimeUnit.SECONDS); } + private static class MLInputRows extends LinkedList> { + /** + * Add tuple value to input map, skip if any value is null. + * @param tupleValue a row in input data. + */ + public void addTupleValue(Map tupleValue) { + if (tupleValue.values().stream().anyMatch(e -> e.isNull() || e.isMissing())) { + return; + } + this.add(tupleValue.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().value()))); + } + + /** + * Convert to DataFrame. + * @return DataFrame + */ + public DataFrame toDataFrame() { + return DataFrameBuilder.load(this); + } + + /** + * Filter each row and convert to DataFrame. + * @param filter used to filter fields in each row + * @return DataFrame + */ + public DataFrame toFilteredDataFrame(Predicate> filter) { + return DataFrameBuilder.load(this.stream().map( + row -> row.entrySet().stream().filter(filter) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) + .collect(Collectors.toList())); + } + } + } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 020d18d76f..17e308cbc9 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -69,6 +69,7 @@ SAMPLE_SIZE: 'SAMPLE_SIZE'; OUTPUT_AFTER: 'OUTPUT_AFTER'; TIME_DECAY: 'TIME_DECAY'; ANOMALY_RATE: 'ANOMALY_RATE'; +CATEGORY_FIELD: 'CATEGORY_FIELD'; TIME_FIELD: 'TIME_FIELD'; TIME_ZONE: 'TIME_ZONE'; TRAINING_DATA_SIZE: 'TRAINING_DATA_SIZE'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index fc87fccfcd..a1061b0020 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -141,6 +141,7 @@ adParameter | (OUTPUT_AFTER EQUAL output_after=integerLiteral) | (TIME_DECAY EQUAL time_decay=decimalLiteral) | (ANOMALY_RATE EQUAL anomaly_rate=decimalLiteral) + | (CATEGORY_FIELD EQUAL category_field=stringLiteral) | (TIME_FIELD EQUAL time_field=stringLiteral) | (DATE_FORMAT EQUAL date_format=stringLiteral) | (TIME_ZONE EQUAL time_zone=stringLiteral) From cacdb1b6b5ba179f2a032fda1ee98d7bca236491 Mon Sep 17 00:00:00 2001 From: vamsi-amazon Date: Tue, 25 Oct 2022 01:28:45 -0700 Subject: [PATCH 31/36] Adding docs related to prometheus and catalog functionalities. Signed-off-by: Vamsi Manohar --- .gitignore | 1 + docs/user/ppl/admin/catalog.rst | 86 +++++++++++++++++++++++++++++++++ docs/user/ppl/index.rst | 2 + doctest/build.gradle | 46 +++++++++++++++++- doctest/catalog/catalog.json | 7 +++ 5 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 docs/user/ppl/admin/catalog.rst create mode 100644 doctest/catalog/catalog.json diff --git a/.gitignore b/.gitignore index 7321ead407..851d42e0ee 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,4 @@ gen /artifacts/ /.pid.lock +/.prom.pid.lock diff --git a/docs/user/ppl/admin/catalog.rst b/docs/user/ppl/admin/catalog.rst new file mode 100644 index 0000000000..7b0a08f307 --- /dev/null +++ b/docs/user/ppl/admin/catalog.rst @@ -0,0 +1,86 @@ +.. highlight:: sh + +================= +Catalog Settings +================= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 1 + +Introduction +============ + +The concept of ``catalog`` is introduced to support the federation of SQL/PPL query engine to multiple data sources. +This helps PPL users to leverage data from multiple data sources and derive correlation and insights. +Catalog definition provides the information to connect to a datasource and also gives a name to them to refer in PPL commands. + + +Definitions of catalog and connector +==================================== +* Connector is a component that adapts the query engine to a datasource. For example, Prometheus connector would adapt and help execute the queries to run on Prometheus data source. connector name is enough in the catalog definition json. +* Catalog is a construct to define how to connect to a datasource and which connector to adapt by query engine. + +Example Prometheus Catalog Definition :: + + [{ + "name" : "prometheus", + "connector": "prometheus", + "uri" : "http://localhost:9090", + "authentication" : { + "type" : "basicauth", + "username" : "admin", + "password" : "admin" + } + }] +Catalog configuration Restrictions. + +* ``name``, ``uri``, ``connector`` are required fields in the catalog configuration. +* All the catalog names should be unique. +* Catalog names should match with the regex of an identifier[``[@*A-Za-z]+?[*a-zA-Z_\-0-9]*``]. +* ``prometheus`` is the only connector allowed. + +Configuring catalog in OpenSearch +==================================== + +* Catalogs are configured in opensearch keystore as secure settings under ``plugins.query.federation.catalog.config`` key as they contain credential info. +* A json file containing array of catalog configurations should be injected into keystore with the above mentioned key. sample json file can be seen in the above section. + + +[**To be run on all the nodes in the cluster**] Command to add catalog.json file to OpenSearch Keystore :: + + >> bin/opensearch-keystore add-file plugins.query.federation.catalog.config catalog.json + +Catalogs can be configured during opensearch start up or can be updated while the opensearch is running. +If we update catalog configuration during runtime, the following api should be triggered to update the query engine with the latest changes. + +[**Required only if we update keystore settings during runtime**] Secure Settings refresh api:: + + >> curl --request POST \ + --url http://{{opensearch-domain}}:9200/_nodes/reload_secure_settings \ + --data '{"secure_settings_password":"{{keystore-password}}"}' + + +Using a catalog in PPL command +==================================== +Catalog is referred in source command as show in the code block below. +Based on the abstraction designed by the connector, +one can refer the corresponding entity as table in the source command. +For example in prometheus connector, each metric is abstracted as a table. +so we can refer a metric and apply stats over it in the following way. + +Example source command with prometheus catalog :: + + >> source = prometheus.prometheus_http_requests_total | stats avg(@value) by job; + + +Limitations of catalog +==================================== +* Catalog settings are global and all PPL users are allowed to fetch data from all the defined catalogs. +* In each catalog, PPL users can access all the data available with the credentials provided in the catalog definition. +* With the current release, Basic and AWSSigV4 are the only authentication mechanisms supported with the underlying data sources. + + + diff --git a/docs/user/ppl/index.rst b/docs/user/ppl/index.rst index 4f824df11d..2ec974ea28 100644 --- a/docs/user/ppl/index.rst +++ b/docs/user/ppl/index.rst @@ -34,6 +34,8 @@ The query start with search command and then flowing a set of command delimited - `Monitoring `_ + - `Catalog Settings `_ + * **Commands** - `Syntax `_ diff --git a/doctest/build.gradle b/doctest/build.gradle index eec5a791b8..69fac44d95 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -9,6 +9,7 @@ import org.opensearch.gradle.testclusters.RunTask plugins { id 'base' id 'com.wiredforcode.spawn' + id "de.undercouch.download" version "5.3.0" } apply plugin: 'opensearch.testclusters' @@ -25,6 +26,27 @@ task bootstrap(type: Exec) { commandLine 'sh', "$projectDir/bootstrap.sh" } +task startPrometheus(type: SpawnProcessTask) { + doFirst { + download.run { + src 'https://github.com/prometheus/prometheus/releases/download/v2.39.1/prometheus-2.39.1.linux-amd64.tar.gz' + dest new File("$projectDir/bin", 'prometheus.tar.gz') + } + copy { + from tarTree("$projectDir/bin/prometheus.tar.gz") + into "$projectDir/bin" + } + copy { + from "$projectDir/bin/prometheus.yml" + into "$projectDir/bin/prometheus-2.39.1.linux-amd64/prometheus" + } + } + command "$projectDir/bin/prometheus-2.39.1.linux-amd64/prometheus --storage.tsdb.path=$projectDir/bin/prometheus-2.39.1.linux-amd64/data --config.file=$projectDir/bin/prometheus-2.39.1.linux-amd64/prometheus.yml" + ready 'TSDB started' + pidLockFileName '.prom.pid.lock' +} + + //evaluationDependsOn(':') task startOpenSearch(type: SpawnProcessTask) { command "${path}/gradlew -p ${plugin_path} runRestTestCluster" @@ -40,11 +62,30 @@ task doctest(type: Exec, dependsOn: ['bootstrap']) { } } -task stopOpenSearch(type: KillProcessTask) +task stopOpenSearch(type: KillProcessTask) { + + finalizedBy { + def pidFile = new File(path, ".prom.pid.lock") + if(!pidFile.exists()) { + logger.quiet "No Prometheus server running!" + return + } + + def pid = pidFile.text + def process = "kill $pid".execute() + + try { + process.waitFor() + } finally { + pidFile.delete() + } + println("Killed Prometheus") + } +} doctest.dependsOn startOpenSearch +startOpenSearch.dependsOn startPrometheus doctest.finalizedBy stopOpenSearch - build.dependsOn doctest clean.dependsOn(cleanBootstrap) @@ -60,6 +101,7 @@ String mlCommonsPlugin = 'opensearch-ml' testClusters { docTestCluster { + keystore 'plugins.query.federation.catalog.config', new File("$projectDir/catalog", 'catalog.json') plugin(provider(new Callable(){ @Override RegularFile call() throws Exception { diff --git a/doctest/catalog/catalog.json b/doctest/catalog/catalog.json new file mode 100644 index 0000000000..5d9a6be612 --- /dev/null +++ b/doctest/catalog/catalog.json @@ -0,0 +1,7 @@ +[ + { + "name" : "prometheus", + "connector": "prometheus", + "uri" : "http://localhost:9090" + } +] \ No newline at end of file From eb08f57423eb7e33814d31554ff53c5e0529ee08 Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Wed, 26 Oct 2022 19:20:51 -0700 Subject: [PATCH 32/36] AD timefield name issue (#919) Signed-off-by: Jing Zhang --- .../org/opensearch/sql/analysis/Analyzer.java | 3 +- .../opensearch/sql/analysis/AnalyzerTest.java | 40 +++++++++++++++++++ .../org/opensearch/sql/config/TestConfig.java | 1 + 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 5fc642fa06..e3a8ab1fe4 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -497,7 +497,8 @@ public LogicalPlan visitAD(AD node, AnalysisContext context) { currentEnv.define(new Symbol(Namespace.FIELD_NAME, RCF_ANOMALOUS), ExprCoreType.BOOLEAN); } else { currentEnv.define(new Symbol(Namespace.FIELD_NAME, RCF_ANOMALY_GRADE), ExprCoreType.DOUBLE); - currentEnv.define(new Symbol(Namespace.FIELD_NAME, RCF_TIMESTAMP), ExprCoreType.TIMESTAMP); + currentEnv.define(new Symbol(Namespace.FIELD_NAME, + (String) node.getArguments().get(TIME_FIELD).getValue()), ExprCoreType.TIMESTAMP); } return new LogicalAD(child, options); } diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index c1adf0623d..bd65e011c2 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -31,11 +31,14 @@ import static org.opensearch.sql.ast.tree.Sort.SortOption.DEFAULT_ASC; import static org.opensearch.sql.ast.tree.Sort.SortOrder; import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; +import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; +import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -64,12 +67,14 @@ import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.HighlightExpression; +import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.window.WindowDefinition; import org.opensearch.sql.planner.logical.LogicalAD; import org.opensearch.sql.planner.logical.LogicalMLCommons; import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.logical.LogicalPlanDSL; +import org.opensearch.sql.planner.logical.LogicalProject; import org.opensearch.sql.planner.logical.LogicalRelation; import org.opensearch.sql.planner.physical.catalog.CatalogTable; import org.springframework.context.annotation.Configuration; @@ -956,6 +961,41 @@ public void ad_fitRCF_relation() { ); } + @Test + public void ad_fitRCF_relation_with_time_field() { + Map argumentMap = new HashMap() {{ + put("shingle_size", new Literal(8, DataType.INTEGER)); + put("time_decay", new Literal(0.0001, DataType.DOUBLE)); + put("time_field", new Literal("ts", DataType.STRING)); + }}; + + LogicalPlan actual = analyze(AstDSL.project( + new AD(AstDSL.relation("schema"), argumentMap), AstDSL.allFields())); + assertTrue(((LogicalProject) actual).getProjectList().size() >= 3); + assertTrue(((LogicalProject) actual).getProjectList() + .contains(DSL.named("score", DSL.ref("score", DOUBLE)))); + assertTrue(((LogicalProject) actual).getProjectList() + .contains(DSL.named("anomaly_grade", DSL.ref("anomaly_grade", DOUBLE)))); + assertTrue(((LogicalProject) actual).getProjectList() + .contains(DSL.named("ts", DSL.ref("ts", TIMESTAMP)))); + } + + @Test + public void ad_fitRCF_relation_without_time_field() { + Map argumentMap = new HashMap<>() {{ + put("shingle_size", new Literal(8, DataType.INTEGER)); + put("time_decay", new Literal(0.0001, DataType.DOUBLE)); + }}; + + LogicalPlan actual = analyze(AstDSL.project( + new AD(AstDSL.relation("schema"), argumentMap), AstDSL.allFields())); + assertTrue(((LogicalProject) actual).getProjectList().size() >= 2); + assertTrue(((LogicalProject) actual).getProjectList() + .contains(DSL.named("score", DSL.ref("score", DOUBLE)))); + assertTrue(((LogicalProject) actual).getProjectList() + .contains(DSL.named("anomalous", DSL.ref("anomalous", BOOLEAN)))); + } + @Test public void table_function() { assertAnalyzeEqual(new LogicalRelation("query_range", table), diff --git a/core/src/test/java/org/opensearch/sql/config/TestConfig.java b/core/src/test/java/org/opensearch/sql/config/TestConfig.java index ab78109aa2..3112dc9234 100644 --- a/core/src/test/java/org/opensearch/sql/config/TestConfig.java +++ b/core/src/test/java/org/opensearch/sql/config/TestConfig.java @@ -55,6 +55,7 @@ public class TestConfig { .put(STRING_TYPE_MISSING_VALUE_FIELD, ExprCoreType.STRING) .put("struct_value", ExprCoreType.STRUCT) .put("array_value", ExprCoreType.ARRAY) + .put("timestamp_value", ExprCoreType.TIMESTAMP) .build(); @Bean From f011f0d1014feb7a9fc32aa166dcffc51d6f333b Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Wed, 26 Oct 2022 19:35:57 -0700 Subject: [PATCH 33/36] Bump ansi-regex for workbench (#975) Signed-off-by: Joshua Li --- workbench/package.json | 9 +++--- workbench/yarn.lock | 63 ++++++++++++++++++++---------------------- 2 files changed, 35 insertions(+), 37 deletions(-) diff --git a/workbench/package.json b/workbench/package.json index 2360dc4d91..8a2f5c6c71 100644 --- a/workbench/package.json +++ b/workbench/package.json @@ -43,14 +43,15 @@ "tslint-plugin-prettier": "^2.0.1" }, "resolutions": { - "**/@types/node": "10.12.27", - "@types/react": "16.3.14", - "**/@types/angular": "1.6.50", + "**/@types/node": "^10.12.27", + "@types/react": "^16.3.14", + "**/@types/angular": "^1.6.50", "**/@types/jest": "^24.0.9", "**/@types/react-dom": "^16.0.5", "**/@types/react-router-dom": "^4.3.1", "eslint-utils": "^2.0.0", "json-schema": "^0.4.0", - "**/@types/react": "16.3.14" + "**/@types/react": "^16.3.14", + "ansi-regex": "^5.0.1" } } diff --git a/workbench/yarn.lock b/workbench/yarn.lock index 026ec30743..58182914c4 100644 --- a/workbench/yarn.lock +++ b/workbench/yarn.lock @@ -140,10 +140,10 @@ dependencies: "@babel/runtime" "^7.12.5" -"@types/angular@1.6.50": - version "1.6.50" - resolved "https://registry.yarnpkg.com/@types/angular/-/angular-1.6.50.tgz#8b6599088d80f68ef0cad7d3a2062248ebe72b3d" - integrity sha512-D3KB0PdaxdwtA44yOpK+NtptTscKWgUzXmf8fiLaaVxnX+b7QQ+dUMsyeVDweCQ6VX4PMwkd6x2hJ0X+ISIsoQ== +"@types/angular@^1.6.50": + version "1.8.4" + resolved "https://registry.yarnpkg.com/@types/angular/-/angular-1.8.4.tgz#a2cc163e508389c51d4c4119ebff6b9395cec472" + integrity sha512-wPS/ncJWhyxJsndsW1B6Ta8D4mi97x1yItSu+rkLDytU3oRIh2CFAjMuJceYwFAh9+DIohndWM0QBA9OU2Hv0g== "@types/boom@*": version "7.3.2" @@ -241,10 +241,10 @@ dependencies: "@types/mime-db" "*" -"@types/node@*", "@types/node@10.12.27": - version "10.12.27" - resolved "https://registry.yarnpkg.com/@types/node/-/node-10.12.27.tgz#eb3843f15d0ba0986cc7e4d734d2ee8b50709ef8" - integrity sha512-e9wgeY6gaY21on3ve0xAjgBVjGDWq/xUteK0ujsE53bUoxycMkqfnkUgMt6ffZtykZ5X12Mg3T7Pw4TRCObDKg== +"@types/node@*", "@types/node@^10.12.27": + version "10.17.60" + resolved "https://registry.yarnpkg.com/@types/node/-/node-10.17.60.tgz#35f3d6213daed95da7f0f73e75bcc6980e90597b" + integrity sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw== "@types/parse-json@^4.0.0": version "4.0.0" @@ -256,6 +256,11 @@ resolved "https://registry.yarnpkg.com/@types/podium/-/podium-1.0.1.tgz#f51a419959584f2ffb8f849e5b33f2b99f7ee1f9" integrity sha512-dcg8V5wKgrvNIBk2hzgK3gHobZ0u7pXW36NigfJq+lBA+/fqI6GAI0bdjLAEkZC67EsaVErtePkqipJPsszN7Q== +"@types/prop-types@*": + version "15.7.5" + resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.5.tgz#5f19d2b85a98e9558036f6a3cacc8819420f05cf" + integrity sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w== + "@types/react-dom@^16.0.5": version "16.9.16" resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-16.9.16.tgz#c591f2ed1c6f32e9759dfa6eb4abfd8041f29e39" @@ -289,12 +294,19 @@ "@types/history" "^4.7.11" "@types/react" "*" -"@types/react@*", "@types/react@16.3.14", "@types/react@^16": - version "16.3.14" - resolved "https://registry.yarnpkg.com/@types/react/-/react-16.3.14.tgz#f90ac6834de172e13ecca430dcb6814744225d36" - integrity sha512-wNUGm49fPl7eE2fnYdF0v5vSOrUMdKMQD/4NwtQRnb6mnPwtkhabmuFz37eq90+hhyfz0pWd38jkZHOcaZ6LGw== +"@types/react@*", "@types/react@^16", "@types/react@^16.3.14": + version "16.14.33" + resolved "https://registry.yarnpkg.com/@types/react/-/react-16.14.33.tgz#eec5863d4d7814da37bc43b039a9ff9561e33f1b" + integrity sha512-JO6dkVjLvlAJi7wnGDgqlBdaePNSmUgyw5sS7HLy4B5F1+4Drztbx3SS8klecRvfp5y25YLYv/zq6O27FDh8gQ== dependencies: - csstype "^2.2.0" + "@types/prop-types" "*" + "@types/scheduler" "*" + csstype "^3.0.2" + +"@types/scheduler@*": + version "0.16.2" + resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.2.tgz#1a62f89525723dde24ba1b01b092bf5df8ad4d39" + integrity sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew== "@types/shot@*": version "4.0.1" @@ -377,22 +389,7 @@ ansi-escapes@^4.2.1, ansi-escapes@^4.3.0: dependencies: type-fest "^0.21.3" -ansi-regex@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" - integrity sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA== - -ansi-regex@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-3.0.1.tgz#123d6479e92ad45ad897d4054e3c7ca7db4944e1" - integrity sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw== - -ansi-regex@^4.0.0, ansi-regex@^4.1.0: - version "4.1.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-4.1.1.tgz#164daac87ab2d6f6db3a29875e2d1766582dabed" - integrity sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g== - -ansi-regex@^5.0.1: +ansi-regex@^2.0.0, ansi-regex@^3.0.0, ansi-regex@^4.0.0, ansi-regex@^4.1.0, ansi-regex@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== @@ -785,10 +782,10 @@ cross-spawn@^7.0.0: shebang-command "^2.0.0" which "^2.0.1" -csstype@^2.2.0: - version "2.6.20" - resolved "https://registry.yarnpkg.com/csstype/-/csstype-2.6.20.tgz#9229c65ea0b260cf4d3d997cb06288e36a8d6dda" - integrity sha512-/WwNkdXfckNgw6S5R125rrW8ez139lBHWouiBvX8dfMFtcn6V81REDqnH7+CRpRipfYlyU1CmOnOxrmGcFOjeA== +csstype@^3.0.2: + version "3.1.1" + resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.1.tgz#841b532c45c758ee546a11d5bd7b7b473c8c30b9" + integrity sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw== cypress@^5.0.0: version "5.6.0" From 91baab12260064997191ffefc28f847f8a5522a6 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Thu, 27 Oct 2022 11:34:29 -0700 Subject: [PATCH 34/36] Add time window and window assigner (#950) * Add window and tumbling window assigner Signed-off-by: Chen Dai * Add sliding window assigner Signed-off-by: Chen Dai * Address PR comments Signed-off-by: Chen Dai Signed-off-by: Chen Dai --- .../planner/streaming/windowing/Window.java | 28 +++++++++ .../assigner/SlidingWindowAssigner.java | 57 +++++++++++++++++++ .../assigner/TumblingWindowAssigner.java | 38 +++++++++++++ .../windowing/assigner/WindowAssigner.java | 24 ++++++++ .../opensearch/sql/utils/DateTimeUtils.java | 10 ++++ .../streaming/windowing/WindowTest.java | 21 +++++++ .../assigner/SlidingWindowAssignerTest.java | 52 +++++++++++++++++ .../assigner/TumblingWindowAssignerTest.java | 39 +++++++++++++ 8 files changed, 269 insertions(+) create mode 100644 core/src/main/java/org/opensearch/sql/planner/streaming/windowing/Window.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/SlidingWindowAssigner.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/TumblingWindowAssigner.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/WindowAssigner.java create mode 100644 core/src/test/java/org/opensearch/sql/planner/streaming/windowing/WindowTest.java create mode 100644 core/src/test/java/org/opensearch/sql/planner/streaming/windowing/assigner/SlidingWindowAssignerTest.java create mode 100644 core/src/test/java/org/opensearch/sql/planner/streaming/windowing/assigner/TumblingWindowAssignerTest.java diff --git a/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/Window.java b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/Window.java new file mode 100644 index 0000000000..2a85ea391c --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/Window.java @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.windowing; + +import lombok.Data; + +/** + * A time window is a window of time interval with inclusive start time and exclusive end time. + */ +@Data +public class Window { + + /** Start timestamp (inclusive) of the time window. */ + private final long startTime; + + /** End timestamp (exclusive) of the time window. */ + private final long endTime; + + /** + * Return the maximum timestamp (inclusive) of the window. + */ + public long maxTimestamp() { + return endTime - 1; + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/SlidingWindowAssigner.java b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/SlidingWindowAssigner.java new file mode 100644 index 0000000000..f0f47fd575 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/SlidingWindowAssigner.java @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.windowing.assigner; + +import com.google.common.base.Preconditions; +import java.util.LinkedList; +import java.util.List; +import org.opensearch.sql.planner.streaming.windowing.Window; +import org.opensearch.sql.utils.DateTimeUtils; + +/** + * A sliding window assigner assigns multiple overlapped window per event timestamp. + * The overlap size is determined by the given slide interval. + */ +public class SlidingWindowAssigner implements WindowAssigner { + + /** Window size in millisecond. */ + private final long windowSize; + + /** Slide size in millisecond. */ + private final long slideSize; + + /** + * Create sliding window assigner with the given window and slide size in millisecond. + * + * @param windowSize window size in millisecond + * @param slideSize slide size in millisecond + */ + public SlidingWindowAssigner(long windowSize, long slideSize) { + Preconditions.checkArgument(windowSize > 0, + "Window size [%s] must be positive number", windowSize); + Preconditions.checkArgument(slideSize > 0, + "Slide size [%s] must be positive number", slideSize); + this.windowSize = windowSize; + this.slideSize = slideSize; + } + + @Override + public List assign(long timestamp) { + LinkedList windows = new LinkedList<>(); + + // Assign window from the last start time to the first until timestamp outside current window + long startTime = DateTimeUtils.getWindowStartTime(timestamp, slideSize); + for (Window win = window(startTime); win.maxTimestamp() >= timestamp; win = window(startTime)) { + windows.addFirst(win); + startTime -= slideSize; + } + return windows; + } + + private Window window(long startTime) { + return new Window(startTime, startTime + windowSize); + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/TumblingWindowAssigner.java b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/TumblingWindowAssigner.java new file mode 100644 index 0000000000..192bb6c429 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/TumblingWindowAssigner.java @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.windowing.assigner; + +import com.google.common.base.Preconditions; +import java.util.Collections; +import java.util.List; +import org.opensearch.sql.planner.streaming.windowing.Window; +import org.opensearch.sql.utils.DateTimeUtils; + +/** + * A tumbling window assigner assigns a single window per event timestamp without overlap. + */ +public class TumblingWindowAssigner implements WindowAssigner { + + /** Window size in millisecond. */ + private final long windowSize; + + /** + * Create tumbling window assigner with the given window size. + * + * @param windowSize window size in millisecond + */ + public TumblingWindowAssigner(long windowSize) { + Preconditions.checkArgument(windowSize > 0, + "Window size [%s] must be positive number", windowSize); + this.windowSize = windowSize; + } + + @Override + public List assign(long timestamp) { + long startTime = DateTimeUtils.getWindowStartTime(timestamp, windowSize); + return Collections.singletonList(new Window(startTime, startTime + windowSize)); + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/WindowAssigner.java b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/WindowAssigner.java new file mode 100644 index 0000000000..dac882c5ff --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/WindowAssigner.java @@ -0,0 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.windowing.assigner; + +import java.util.List; +import org.opensearch.sql.planner.streaming.windowing.Window; + +/** + * A window assigner assigns zero or more window to an event timestamp + * based on different windowing approach. + */ +public interface WindowAssigner { + + /** + * Return window(s) assigned to the timestamp. + * @param timestamp given event timestamp + * @return windows assigned + */ + List assign(long timestamp); + +} diff --git a/core/src/main/java/org/opensearch/sql/utils/DateTimeUtils.java b/core/src/main/java/org/opensearch/sql/utils/DateTimeUtils.java index fbcf7deca4..5a99af3f83 100644 --- a/core/src/main/java/org/opensearch/sql/utils/DateTimeUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/DateTimeUtils.java @@ -85,6 +85,16 @@ public static long roundYear(long utcMillis, int interval) { return initDateTime.plusYears(yearToAdd).toInstant().toEpochMilli(); } + /** + * Get window start time which aligns with the given size. + * + * @param timestamp event timestamp + * @param size defines a window's start time to align with + * @return start timestamp of the window + */ + public long getWindowStartTime(long timestamp, long size) { + return timestamp - timestamp % size; + } /** * isValidMySqlTimeZoneId for timezones which match timezone the range set by MySQL. diff --git a/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/WindowTest.java b/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/WindowTest.java new file mode 100644 index 0000000000..9b9aafa933 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/WindowTest.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.windowing; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class WindowTest { + + @Test + void test() { + Window window = new Window(1000, 2000); + assertEquals(1000, window.getStartTime()); + assertEquals(2000, window.getEndTime()); + assertEquals(1999, window.maxTimestamp()); + } +} \ No newline at end of file diff --git a/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/assigner/SlidingWindowAssignerTest.java b/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/assigner/SlidingWindowAssignerTest.java new file mode 100644 index 0000000000..fd69065742 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/assigner/SlidingWindowAssignerTest.java @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.windowing.assigner; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.planner.streaming.windowing.Window; + +class SlidingWindowAssignerTest { + + @Test + void testAssignWindows() { + long windowSize = 1000; + long slideSize = 500; + SlidingWindowAssigner assigner = new SlidingWindowAssigner(windowSize, slideSize); + + assertEquals( + List.of( + new Window(0, 1000), + new Window(500, 1500)), + assigner.assign(500)); + + assertEquals( + List.of( + new Window(0, 1000), + new Window(500, 1500)), + assigner.assign(999)); + + assertEquals( + List.of( + new Window(500, 1500), + new Window(1000, 2000)), + assigner.assign(1000)); + } + + @Test + void testConstructWithIllegalArguments() { + IllegalArgumentException error1 = assertThrows(IllegalArgumentException.class, + () -> new SlidingWindowAssigner(-1, 100)); + assertEquals("Window size [-1] must be positive number", error1.getMessage()); + + IllegalArgumentException error2 = assertThrows(IllegalArgumentException.class, + () -> new SlidingWindowAssigner(1000, 0)); + assertEquals("Slide size [0] must be positive number", error2.getMessage()); + } +} \ No newline at end of file diff --git a/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/assigner/TumblingWindowAssignerTest.java b/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/assigner/TumblingWindowAssignerTest.java new file mode 100644 index 0000000000..4c98c40f7a --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/planner/streaming/windowing/assigner/TumblingWindowAssignerTest.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.streaming.windowing.assigner; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.Collections; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.planner.streaming.windowing.Window; + +class TumblingWindowAssignerTest { + + @Test + void testAssignWindow() { + long windowSize = 1000; + TumblingWindowAssigner assigner = new TumblingWindowAssigner(windowSize); + + assertEquals( + Collections.singletonList(new Window(0, 1000)), + assigner.assign(500)); + assertEquals( + Collections.singletonList(new Window(1000, 2000)), + assigner.assign(1999)); + assertEquals( + Collections.singletonList(new Window(2000, 3000)), + assigner.assign(2000)); + } + + @Test + void testConstructWithIllegalWindowSize() { + IllegalArgumentException error = assertThrows(IllegalArgumentException.class, + () -> new TumblingWindowAssigner(-1)); + assertEquals("Window size [-1] must be positive number", error.getMessage()); + } +} \ No newline at end of file From b286742ab06fdaba3e3ed6d24ad0a935c2fabe04 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Thu, 27 Oct 2022 13:52:15 -0700 Subject: [PATCH 35/36] add table default impl of exist and create Signed-off-by: Peng Huo --- .../java/org/opensearch/sql/storage/Table.java | 10 ++++++++-- .../physical/catalog/CatalogTableTest.java | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/storage/Table.java b/core/src/main/java/org/opensearch/sql/storage/Table.java index 7cdd757cbb..34e6ece30b 100644 --- a/core/src/main/java/org/opensearch/sql/storage/Table.java +++ b/core/src/main/java/org/opensearch/sql/storage/Table.java @@ -18,15 +18,21 @@ public interface Table { /** * Check if current table exists. + * * @return true if exists, otherwise false */ - boolean exists(); + default boolean exists() { + throw new UnsupportedOperationException("Unsupported Operation"); + } /** * Create table given table schema. + * * @param schema table schema */ - void create(Map schema); + default void create(Map schema) { + throw new UnsupportedOperationException("Unsupported Operation"); + } /** * Get the {@link ExprType} for each field in the table. diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java index 59e57a97b3..a35c94a21c 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java @@ -8,6 +8,7 @@ package org.opensearch.sql.planner.physical.catalog; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.HashMap; @@ -46,4 +47,21 @@ void testImplement() { assertTrue(physicalPlan instanceof CatalogTableScan); } + // todo. temporary added for code coverage. remove if required. + @Test + void testExist() { + UnsupportedOperationException exception = + assertThrows(UnsupportedOperationException.class, + () -> new CatalogTable(catalogService).exists()); + assertEquals("Unsupported Operation", exception.getMessage()); + } + + // todo. temporary added for code coverage. remove if required. + @Test + void testCreateTable() { + UnsupportedOperationException exception = + assertThrows(UnsupportedOperationException.class, + () -> new CatalogTable(catalogService).create(new HashMap<>())); + assertEquals("Unsupported Operation", exception.getMessage()); + } } From 63f34494ccfa7e27cf9800c731830c8658128cf1 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Fri, 28 Oct 2022 08:59:48 -0700 Subject: [PATCH 36/36] Add metadatalog interface and default in memory implementation (#974) Signed-off-by: Peng Huo --- .../streaming/DefaultMetadataLog.java | 76 +++++++++++ .../sql/executor/streaming/MetadataLog.java | 61 +++++++++ .../streaming/DefaultMetadataLogTest.java | 126 ++++++++++++++++++ 3 files changed, 263 insertions(+) create mode 100644 core/src/main/java/org/opensearch/sql/executor/streaming/DefaultMetadataLog.java create mode 100644 core/src/main/java/org/opensearch/sql/executor/streaming/MetadataLog.java create mode 100644 core/src/test/java/org/opensearch/sql/executor/streaming/DefaultMetadataLogTest.java diff --git a/core/src/main/java/org/opensearch/sql/executor/streaming/DefaultMetadataLog.java b/core/src/main/java/org/opensearch/sql/executor/streaming/DefaultMetadataLog.java new file mode 100644 index 0000000000..e439d93f6c --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/streaming/DefaultMetadataLog.java @@ -0,0 +1,76 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor.streaming; + +import com.google.common.base.Preconditions; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.SortedMap; +import java.util.TreeMap; +import org.apache.commons.lang3.tuple.Pair; + +/** + * In memory implementation of {@link MetadataLog}. Todo. Current implementation does not guarantee + * thread safe. We will re-evaluate it when adding pipeline execution. + * + * @param type of metadata type. + */ +public class DefaultMetadataLog implements MetadataLog { + + private static final long MIN_ACCEPTABLE_ID = 0L; + + private SortedMap metadataMap = new TreeMap<>(); + + @Override + public boolean add(Long batchId, T metadata) { + Preconditions.checkArgument(batchId >= MIN_ACCEPTABLE_ID, "batch id must large or equal 0"); + + if (metadataMap.containsKey(batchId)) { + return false; + } + metadataMap.put(batchId, metadata); + return true; + } + + @Override + public Optional get(Long batchId) { + if (!metadataMap.containsKey(batchId)) { + return Optional.empty(); + } else { + return Optional.of(metadataMap.get(batchId)); + } + } + + @Override + public List get(Optional startBatchId, Optional endBatchId) { + if (startBatchId.isEmpty() && endBatchId.isEmpty()) { + return new ArrayList<>(metadataMap.values()); + } else { + Long s = startBatchId.orElse(MIN_ACCEPTABLE_ID); + Long e = endBatchId.map(i -> i + 1).orElse(Long.MAX_VALUE); + return new ArrayList<>(metadataMap.subMap(s, e).values()); + } + } + + @Override + public Optional> getLatest() { + if (metadataMap.isEmpty()) { + return Optional.empty(); + } else { + Long latestId = metadataMap.lastKey(); + return Optional.of(Pair.of(latestId, metadataMap.get(latestId))); + } + } + + @Override + public void purge(Long batchId) { + metadataMap.headMap(batchId).clear(); + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/streaming/MetadataLog.java b/core/src/main/java/org/opensearch/sql/executor/streaming/MetadataLog.java new file mode 100644 index 0000000000..d6bb9bacd6 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/streaming/MetadataLog.java @@ -0,0 +1,61 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor.streaming; + +import java.util.List; +import java.util.Optional; +import org.apache.commons.lang3.tuple.Pair; + +/** + * Write-ahead Log (WAL). Which allow client write metadata associate with id. + * + * @param type of metadata type. + */ +public interface MetadataLog { + + /** + * add metadata to WAL. + * + * @param id metadata index in WAL. + * @param metadata metadata. + * @return true if add success, otherwise return false. + */ + boolean add(Long id, T metadata); + + /** + * get metadata from WAL. + * + * @param id metadata index in WAL. + * @return metadata. + */ + Optional get(Long id); + + /** + * Return metadata for id between [startId, endId]. + * + * @param startId If startId is empty, return all metadata before endId (inclusive). + * @param endId If end is empty, return all batches after endId (inclusive). + * @return a list of metadata sorted by id (nature order). + */ + List get(Optional startId, Optional endId); + + /** + * Get latest batchId and metadata. + * + * @return pair of id and metadata if not empty. + */ + Optional> getLatest(); + + /** + * Remove all the metadata less then id (exclusive). + * + * @param id smallest batchId should keep. + */ + void purge(Long id); +} diff --git a/core/src/test/java/org/opensearch/sql/executor/streaming/DefaultMetadataLogTest.java b/core/src/test/java/org/opensearch/sql/executor/streaming/DefaultMetadataLogTest.java new file mode 100644 index 0000000000..4d8c4f3e93 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/executor/streaming/DefaultMetadataLogTest.java @@ -0,0 +1,126 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.sql.executor.streaming; + + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Arrays; +import java.util.Optional; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class DefaultMetadataLogTest { + + private DefaultMetadataLog metadataLog; + + @BeforeEach + void setup() { + metadataLog = new DefaultMetadataLog<>(); + } + + @Test + void addMetadataShouldSuccess() { + assertTrue(metadataLog.add(0L, 0L)); + assertTrue(metadataLog.add(1L, 1L)); + } + + @Test + void addMetadataWithSameBatchIdShouldFail() { + assertTrue(metadataLog.add(0L, 0L)); + assertFalse(metadataLog.add(0L, 1L)); + } + + @Test + void addMetadataWithInvalidIdShouldThrowException() { + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> metadataLog.add(-1L, 0L)); + assertEquals("batch id must large or equal 0", exception.getMessage()); + } + + @Test + void getWithIdReturnMetadata() { + metadataLog.add(0L, 0L); + + assertTrue(metadataLog.get(0L).isPresent()); + assertEquals(0L, metadataLog.get(0L).get()); + } + + @Test + void getWithNotExistShouldReturnEmtpy() { + metadataLog.add(0L, 0L); + + assertTrue(metadataLog.get(1L).isEmpty()); + assertTrue(metadataLog.get(-1L).isEmpty()); + } + + @Test + void getWithIdInRangeShouldReturnMetadataList() { + metadataLog.add(0L, 0L); + metadataLog.add(1L, 1L); + metadataLog.add(2L, 2L); + + assertEquals(Arrays.asList(0L, 1L, 2L), metadataLog.get(Optional.of(0L), Optional.of(2L))); + assertEquals(Arrays.asList(0L, 1L, 2L), metadataLog.get(Optional.of(0L), Optional.of(4L))); + assertEquals(Arrays.asList(0L, 1L, 2L), metadataLog.get(Optional.of(-1L), Optional.of(4L))); + assertEquals(Arrays.asList(0L, 1L), metadataLog.get(Optional.of(0L), Optional.of(1L))); + assertEquals(Arrays.asList(1L, 2L), metadataLog.get(Optional.of(1L), Optional.empty())); + assertEquals(Arrays.asList(0L, 1L), metadataLog.get(Optional.empty(), Optional.of(1L))); + assertEquals(Arrays.asList(0L, 1L, 2L), metadataLog.get(Optional.empty(), Optional.empty())); + } + + @Test + void getWithIdOutOfRangeShouldReturnEmpty() { + metadataLog.add(0L, 0L); + metadataLog.add(1L, 1L); + metadataLog.add(2L, 2L); + + assertTrue(metadataLog.get(Optional.of(3L), Optional.of(5L)).isEmpty()); + } + + @Test + void getLatestShouldReturnMetadata() { + metadataLog.add(0L, 10L); + metadataLog.add(1L, 11L); + + Optional> latest = metadataLog.getLatest(); + assertTrue(latest.isPresent()); + assertEquals(1L, latest.get().getLeft()); + assertEquals(11L, latest.get().getRight()); + } + + @Test + void getLatestFromEmptyWALShouldReturnEmpty() { + Optional> latest = metadataLog.getLatest(); + assertTrue(latest.isEmpty()); + } + + @Test + void purgeLatestShouldOnlyKeepLatest() { + metadataLog.add(0L, 10L); + metadataLog.add(1L, 11L); + metadataLog.add(2L, 12L); + + Optional> latest = metadataLog.getLatest(); + assertTrue(latest.isPresent()); + metadataLog.purge(latest.get().getLeft()); + + latest = metadataLog.getLatest(); + assertTrue(latest.isPresent()); + assertEquals(2L, latest.get().getLeft()); + assertEquals(12L, latest.get().getRight()); + } +}