Removing default integtest.sh (1.x).

.github/workflows/CI.yml

@@ -33,37 +33,37 @@ jobs:
         with:
           repository: 'opensearch-project/OpenSearch'
           path: OpenSearch
-          ref: '1.0'
+          ref: '1.x'
       - name: Build OpenSearch
         working-directory: ./OpenSearch
-        run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=rc1 -Dbuild.snapshot=false
+        run: ./gradlew publishToMavenLocal
 
       # dependencies: common-utils
       - name: Checkout common-utils
         uses: actions/checkout@v2
         with:
-          ref: 'main' # TODO: update to the right branch name once it's ready. e.g. 1.x
+          ref: 'main'
           repository: 'opensearch-project/common-utils'
           path: common-utils
       - name: Build common-utils
         working-directory: ./common-utils
-        run: ./gradlew publishToMavenLocal -Dopensearch.version=1.0.0-rc1
+        run: ./gradlew publishToMavenLocal -Dopensearch.version=1.1.0-SNAPSHOT
 
       # dependencies: job-scheduler
       - name: Checkout job-scheduler
         uses: actions/checkout@v2
         with:
-          ref: 'main' # TODO: update to the right branch name once it's ready. e.g. 1.x
+          ref: 'main'
           repository: 'opensearch-project/job-scheduler'
           path: job-scheduler
 
       - name: Build job-scheduler
         working-directory: ./job-scheduler
-        run: ./gradlew publishToMavenLocal -Dopensearch.version=1.0.0-rc1 -Dbuild.snapshot=false
+        run: ./gradlew publishToMavenLocal -Dopensearch.version=1.1.0-SNAPSHOT
       - name: Assemble job-scheduler
         working-directory: ./job-scheduler
         run: |
-          ./gradlew assemble -Dopensearch.version=1.0.0-rc1 -Dbuild.snapshot=false
+          ./gradlew assemble -Dopensearch.version=1.1.0-SNAPSHOT
           echo "Creating ../src/test/resources/job-scheduler ..."
           mkdir -p ../src/test/resources/job-scheduler
           pwd
@@ -72,28 +72,43 @@ jobs:
           cp ./build/distributions/*.zip ../src/test/resources/job-scheduler
           echo "Copied ./build/distributions/*.zip to ../src/test/resources/job-scheduler ..."
           ls ../src/test/resources/job-scheduler
+          echo "Creating ../src/test/resources/org/opensearch/ad/bwc/job-scheduler/1.1.0.0-SNAPSHOT ..."
+          mkdir -p ../src/test/resources/org/opensearch/ad/bwc/job-scheduler/1.1.0.0-SNAPSHOT
+          echo "Copying ./build/distributions/*.zip to ../src/test/resources/org/opensearch/ad/bwc/job-scheduler/1.1.0.0-SNAPSHOT ..."
+          ls ./build/distributions/
+          cp ./build/distributions/*.zip ../src/test/resources/org/opensearch/ad/bwc/job-scheduler/1.1.0.0-SNAPSHOT
+          echo "Copied ./build/distributions/*.zip to ../src/test/resources/org/opensearch/ad/bwc/job-scheduler/1.1.0.0-SNAPSHOT ..."
+          ls ../src/test/resources/org/opensearch/ad/bwc/job-scheduler/1.1.0.0-SNAPSHOT
+
+      - name: Assemble anomaly-detection
+        run: |
+          ./gradlew assemble -Dopensearch.version=1.1.0-SNAPSHOT
+          echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.1.0.0-SNAPSHOT ..."
+          mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.1.0.0-SNAPSHOT
+          echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.1.0.0-SNAPSHOT ..."
+          ls ./build/distributions/
+          cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.1.0.0-SNAPSHOT
+          echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.1.0.0-SNAPSHOT ..."
+          ls ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.1.0.0-SNAPSHOT    
 
       - name: Build and Run Tests
         run: |
-          ./gradlew build -Dopensearch.version=1.0.0-rc1
+          ./gradlew build -Dopensearch.version=1.1.0-SNAPSHOT
 
       - name: Publish to Maven Local
         run: |
-          ./gradlew publishToMavenLocal -Dopensearch.version=1.0.0-rc1
+          ./gradlew publishToMavenLocal -Dopensearch.version=1.1.0-SNAPSHOT
 
       - name: Multi Nodes Integration Testing
         run: |
-          ./gradlew integTest  -PnumNodes=3
+          ./gradlew integTest -PnumNodes=3
 
       - name: Pull and Run Docker
         run: |
-          ## plugin=`ls build/distributions/*.zip`
-          ## version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3`
-          ## plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4`
-          ## TODO: remove these two hard code versions below after GA release
-          version=1.0.0-rc1
-          plugin_version=1.0.0.0-rc1
-          echo $version
+          plugin=`ls build/distributions/*.zip`
+          version=1.1.0-SNAPSHOT
+          plugin_version=1.1.0.0-SNAPSHOT
+          echo Using OpenSearch $version with AD $plugin_version
           cd ..
           if docker pull opensearchstaging/opensearch:$version
           then
@@ -127,6 +142,11 @@ jobs:
             ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster"
           fi
 
+      - name: Run AD Backwards Compatibility Tests
+        run: |
+          echo "Running backwards compatibility tests ..."
+          ./gradlew bwcTestSuite -Dtests.security.manager=false
+
       - name: Upload Coverage Report
         uses: codecov/codecov-action@v1
         with:

CONTRIBUTING.md

@@ -1,105 +1,4 @@
-# Contributing Guidelines
-
-Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional
-documentation, we greatly value feedback and contributions from our community.
-
-Please read through this document before submitting any issues or pull requests to ensure we have all the necessary
-information to effectively respond to your bug report or contribution.
-
-
-## Reporting Bugs/Feature Requests
-
-We welcome you to use the GitHub issue tracker to report bugs or suggest features.
-
-When filing an issue, please check [existing open](../../issues), or [recently closed](../../issues?q=is%3Aissue+is%3Aclosed), issues to make sure somebody else hasn't already
-reported the issue. Please try to include as much information as you can. Details like these are incredibly useful:
-
-* A reproducible test case or series of steps
-* The version of our code being used
-* Any modifications you've made relevant to the bug
-* Anything unusual about your environment or deployment
-
-
-## Contributing via Pull Requests
-Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that:
-
-1. You are working against the latest source on the *master* branch.
-2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already.
-3. You open an issue to discuss any significant work - we would hate for your time to be wasted.
-
-To send us a pull request, please:
-
-1. Fork the repository.
-2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change.
-3. Ensure local tests pass.
-4. Commit to your fork using clear commit messages.
-5. Send us a pull request, answering any default questions in the pull request interface.
-6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation.
-
-GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and
-[creating a pull request](https://help.github.com/articles/creating-a-pull-request/).
-
-
-## Finding contributions to work on
-Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](../../labels/help%20wanted) issues is a great place to start.
-
-
-## Code of Conduct
-
-Please refer to [Code of Conduct](./CODE_OF_CONDUCT.md) on this repository.
-
-
-## Developer Certificate of Origin
-
-OpenSearch is an open source product released under the Apache 2.0 license (see either [the Apache site](https://www.apache.org/licenses/LICENSE-2.0) or the [LICENSE.txt file](./LICENSE.txt)).  The Apache 2.0 license allows you to freely use, modify, distribute, and sell your own products that include Apache 2.0 licensed software.
-
-We respect intellectual property rights of others and we want to make sure all incoming contributions are correctly attributed and licensed. A Developer Certificate of Origin (DCO) is a lightweight mechanism to do that.
-
-The DCO is a declaration attached to every contribution made by every developer. In the commit message of the contribution, the developer simply adds a `Signed-off-by` statement and thereby agrees to the DCO, which you can find below or at [DeveloperCertificate.org](http://developercertificate.org/).
-
-```
-Developer's Certificate of Origin 1.1
-
-By making a contribution to this project, I certify that:
-
-(a) The contribution was created in whole or in part by me and I
-    have the right to submit it under the open source license
-    indicated in the file; or
-
-(b) The contribution is based upon previous work that, to the
-    best of my knowledge, is covered under an appropriate open
-    source license and I have the right under that license to
-    submit that work with modifications, whether created in whole
-    or in part by me, under the same open source license (unless
-    I am permitted to submit under a different license), as
-    Indicated in the file; or
-
-(c) The contribution was provided directly to me by some other
-    person who certified (a), (b) or (c) and I have not modified
-    it.
-
-(d) I understand and agree that this project and the contribution
-    are public and that a record of the contribution (including
-    all personal information I submit with it, including my
-    sign-off) is maintained indefinitely and may be redistributed
-    consistent with this project or the open source license(s)
-    involved.
- ```
-We require that every contribution to OpenSearch is signed with a Developer Certificate of Origin.  Additionally, please use your real name.  We do not accept anonymous contributors nor those utilizing pseudonyms.
-
-Each commit must include a DCO which looks like this
-
-```
-Signed-off-by: Jane Smith <jane.smith@email.com>
-```
-You may type this line on your own when writing your commit messages.  However, if your user.name and user.email are set in your git configs, you can use `-s` or `– – signoff` to add the `Signed-off-by` line to the end of the commit message.
-
-## Security issue notifications
-
-If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public GitHub issue.
-
-
-## Licensing
-
-See the [LICENSE](./LICENSE.txt) file for our project's licensing. We will ask you to confirm the licensing of your contribution.
+## Contributing to this Project
 
+OpenSearch is a community project that is built and maintained by people just like **you**.
+[This document](https://github.com/opensearch-project/.github/blob/main/CONTRIBUTING.md) explains how you can contribute to this and related projects.

DEVELOPER_GUIDE.md

@@ -1,41 +1,58 @@
-# Developer Guide
+- [Developer Guide](#developer-guide)
+  - [Forking and Cloning](#forking-and-cloning)
+  - [Install Prerequisites](#install-prerequisites)
+    - [JDK 14](#jdk-14)
+  - [Setup](#setup)
+  - [Build](#build)
+    - [Building from the command line](#building-from-the-command-line)
+    - [Building from the IDE](#building-from-the-ide)
+    - [Debugging](#debugging)
+    - [Advanced: Launching multi node clusters locally](#advanced-launching-multi-node-clusters-locally)
 
-## Setup
+## Developer Guide
 
-1. Checkout source code of this package from the Github repo.
-1. Launch Intellij IDEA, Choose Import Project and select the settings.gradle file in the root of this package.
-1. To build from command line set JAVA_HOME to point to a JDK 14 before running `./gradlew`.
+### Forking and Cloning
 
-  * Unix System
-    * export JAVA_HOME=jdk-install-dir: Replace jdk-install-dir by the JAVA_HOME directory of your system.
-    * export PATH=$JAVA_HOME/bin:$PATH
-  * Windows System
-    * Find **My Computers** from file directory, right click and select **properties**.
-    * Select the **Advanced** tab, select **Environment variables**.
-    * Edit **JAVA_HOME** to path of where JDK software is installed.
+Fork this repository on GitHub, and clone locally with `git clone`.
 
-## Build
+### Install Prerequisites
+
+#### JDK 14
+
+OpenSearch components build using Java 14 at a minimum. This means you must have a JDK 14 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 14 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-14`.
+
+### Setup
+
+1. Clone the repository (see [Forking and Cloning](#forking-and-cloning))
+2. Make sure `JAVA_HOME` is pointing to a Java 14 JDK (see [Install Prerequisites](#install-prerequisites))
+3. Launch Intellij IDEA, Choose Import Project and select the settings.gradle file in the root of this package.
+
+### Build
 
 This package uses the [Gradle](https://docs.gradle.org/current/userguide/userguide.html) build system. Gradle comes with excellent documentation that should be your first stop when trying to figure out how to operate or modify the build. we also use the OpenSearch build tools for Gradle. These tools are idiosyncratic and don't always follow the conventions and instructions for building regular Java code using Gradle. Not everything in this package will work the way it's described in the Gradle documentation. If you encounter such a situation, the OpenSearch build tools [source code](https://github.com/opensearch-project/OpenSearch/tree/main/buildSrc/src/main/groovy/org/opensearch/gradle) is your best bet for figuring out what's going on.
 
 Currently we just put RCF jar in lib as dependency. Plan to publish to Maven and we can import it later. Before publishing to Maven, you can still build this package directly and find source code in RCF Github package.
 
-### Building from the command line
+#### Building from the command line
 
 1. `./gradlew build` builds and tests
-1. `./gradlew :run` launches a single node cluster with anomaly-detection (and job-scheduler) plugin installed
-1. `./gradlew :integTest` launches a single node cluster with anomaly-detection (and job-scheduler) plugin installed and runs all integration tests except security
-1. ` ./gradlew :integTest --tests="**.test execute foo"` runs a single integration test class or method
-1. `./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin` launches integration tests against a local cluster and run tests with security
-1. `./gradlew spotlessApply` formats code. And/or import formatting rules in `.eclipseformat.xml` with IDE.
+2. `./gradlew :run` launches a single node cluster with anomaly-detection (and job-scheduler) plugin installed
+3. `./gradlew :integTest` launches a single node cluster with anomaly-detection (and job-scheduler) plugin installed and runs all integration tests except security
+4. ` ./gradlew :integTest --tests="**.test execute foo"` runs a single integration test class or method
+5. `./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin` launches integration tests against a local cluster and run tests with security
+6. `./gradlew spotlessApply` formats code. And/or import formatting rules in `.eclipseformat.xml` with IDE.
+7. `./gradlew adBwcCluster#mixedClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by upgrading one of the nodes with the current version of OpenSearch with anomaly-detection and job-scheduler creating a mixed cluster.
+8. `./gradlew adBwcCluster#rollingUpgradeClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by performing rolling upgrade of each node with the current version of OpenSearch with anomaly-detection and job-scheduler.
+9. `./gradlew adBwcCluster#fullRestartClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by performing a full restart on the cluster upgrading all the nodes with the current version of OpenSearch with anomaly-detection and job-scheduler.
+10. `./gradlew bwcTestSuite -Dtests.security.manager=false` runs all the above bwc tests combined.
 
 When launching a cluster using one of the above commands logs are placed in `/build/cluster/run node0/opensearch-<version>/logs`. Though the logs are teed to the console, in practices it's best to check the actual log file.
 
-### Building from the IDE
+#### Building from the IDE
 
 Currently, the only IDE we support is IntelliJ IDEA.  It's free, it's open source, it works. The gradle tasks above can also be launched from IntelliJ's Gradle toolbar and the extra parameters can be passed in via the Launch Configurations VM arguments. 
 
-### Debugging
+#### Debugging
 
 Sometimes it's useful to attach a debugger to either the OpenSearch cluster or the integ tests to see what's going on. When running unit tests you can just hit 'Debug' from the IDE's gutter to debug the tests.  To debug code running in an actual server run:
 
@@ -55,7 +72,7 @@ To debug code running in an integ test (which exercises the server from a separa
 
 The test runner JVM will start suspended and wait for a debugger to attach to `localhost:5005` before running the tests.
 
-### Advanced: Launching multi node clusters locally
+#### Advanced: Launching multi node clusters locally
 
 Sometimes you need to launch a cluster with more than one OpenSearch server process.