Distribution bundle bwc tests (#579)

* Distribution bundle bwc tests

Signed-off-by: Varun Jain <varunudr@amazon.com>

CHANGELOG.md

@@ -28,6 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - BWC tests for Neural Search ([#515](https://github.com/opensearch-project/neural-search/pull/515))
 - Github action to run integ tests in secure opensearch cluster ([#535](https://github.com/opensearch-project/neural-search/pull/535))
 - BWC tests for Multimodal search, Hybrid Search and Neural Sparse Search ([#533](https://github.com/opensearch-project/neural-search/pull/533))
+- Distribution bundle bwc tests ([#579])(https://github.com/opensearch-project/neural-search/pull/579)
 ### Documentation
 ### Maintenance
 ### Refactoring

build.gradle

@@ -58,10 +58,26 @@ apply plugin: "com.diffplug.spotless"
 apply plugin: 'io.freefair.lombok'
 apply from: 'gradle/formatting.gradle'
 
-def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
+ext.opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
 opensearch_tmp_dir.mkdirs()
 
 ext {
+    default_bwc_version = System.getProperty("bwc.version")
+    default_bwc_bundle_version= System.getProperty("bwc.bundle.version")
+    bwcBundleTest = (project.findProperty('customDistributionDownloadType') != null && project.properties['customDistributionDownloadType'] == "bundle")
+    neural_search_bwc_version = bwcBundleTest ? System.getProperty("tests.bwc.bundle.version",rootProject.ext.default_bwc_bundle_version): System.getProperty("tests.bwc.version", rootProject.ext.default_bwc_version)
+    currentBundleVersion = opensearch_version.replace("-SNAPSHOT","")
+
+    // Config below including files are copied from security demo configuration
+    ['esnode.pem', 'esnode-key.pem', 'root-ca.pem','kirk.pem','kirk-key.pem'].forEach { file ->
+        File local = Paths.get(opensearch_tmp_dir.absolutePath, file).toFile()
+        download.run {
+            src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file
+            dest local
+            overwrite false
+        }
+    }
+
     isSnapshot = "true" == System.getProperty("build.snapshot", "true")
     projectSubstitutions = [:]
 
@@ -91,16 +107,10 @@ ext {
             }
         }
 
-        // Config below including files are copied from security demo configuration
-        ['esnode.pem', 'esnode-key.pem', 'root-ca.pem'].forEach { file ->
-            File local = Paths.get(opensearch_tmp_dir.absolutePath, file).toFile()
-            download.run {
-                src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file
-                dest local
-                overwrite false
-            }
-            cluster.extraConfigFile(file, local)
-        }
+        // // Config below including files are copied from security demo configuration
+        cluster.extraConfigFile("esnode.pem", file("$opensearch_tmp_dir/esnode.pem"))
+        cluster.extraConfigFile("esnode-key.pem", file("$opensearch_tmp_dir/esnode-key.pem"))
+        cluster.extraConfigFile("root-ca.pem", file("$opensearch_tmp_dir/root-ca.pem"))
 
         // This configuration is copied from the security plugins demo install:
         // https://github.com/opensearch-project/security/blob/2.11.1.0/tools/install_demo_configuration.sh#L365-L388

gradle.properties

@@ -8,6 +8,7 @@
 # Wired compatibility of OpenSearch works like 3.x version is compatible with 2.(latest-major) version.
 # Therefore, to run rolling-upgrade BWC Test on local machine the BWC version here should be set 2.(latest-major).
 systemProp.bwc.version=2.12.0-SNAPSHOT
+systemProp.bwc.bundle.version=2.12.0
 
 # For fixing Spotless check with Java 17
 org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \

qa/restart-upgrade/build.gradle

@@ -7,36 +7,66 @@ import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 apply from : "$rootDir/qa/build.gradle"
 
-String default_bwc_version = System.getProperty("bwc.version")
-String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version)
+def ext=rootProject.ext
 String baseName = "neuralSearchBwcCluster-restart"
 
 // Creates a test cluster of previous version and loads k-NN plugin of bwcVersion
 testClusters {
     "${baseName}" {
         testDistribution = "ARCHIVE"
-        versions = [neural_search_bwc_version, opensearch_version]
         numberOfNodes = 3
         jvmArgs("-Xms1g", "-Xmx4g")
-        plugin(project.tasks.unZipBwcMlCommonsPlugin.archiveFile)
-        plugin(project.tasks.unZipBwcKnnPlugin.archiveFile)
-        plugin(project.tasks.unZipBwcPlugin.archiveFile)
+        if(ext.bwcBundleTest){
+            versions = [ext.neural_search_bwc_version, ext.currentBundleVersion]
+            def path=ext.opensearch_tmp_dir
+            nodes.each { node ->
+                node.extraConfigFile("kirk.pem", file("$path/kirk.pem"))
+                node.extraConfigFile("kirk-key.pem", file("$path/kirk-key.pem"))
+                node.extraConfigFile("esnode.pem", file("$path/esnode.pem"))
+                node.extraConfigFile("esnode-key.pem", file("$path/esnode-key.pem"))
+                node.extraConfigFile("root-ca.pem", file("$path/root-ca.pem"))
+                node.setting("plugins.security.disabled", "true")
+                node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
+                node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
+                node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
+                node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
+                node.setting("plugins.security.ssl.http.enabled", "true")
+                node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
+                node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
+                node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
+                node.setting("plugins.security.allow_unsafe_democertificates", "true")
+                node.setting("plugins.security.allow_default_init_securityindex", "true")
+                node.setting("plugins.security.authcz.admin_dn", "CN=kirk,OU=client,O=client,L=test,C=de")
+                node.setting("plugins.security.audit.type", "internal_elasticsearch")
+                node.setting("plugins.security.enable_snapshot_restore_privilege", "true")
+                node.setting("plugins.security.check_snapshot_restore_write_privileges", "true")
+                node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]")
+                node.setting("plugins.security.system_indices.enabled", "true")
+            }
+        }else{
+            versions = [ext.neural_search_bwc_version, opensearch_version]
+            plugin(project.tasks.unZipBwcMlCommonsPlugin.archiveFile)
+            plugin(project.tasks.unZipBwcKnnPlugin.archiveFile)
+            plugin(project.tasks.unZipBwcPlugin.archiveFile)
+        }
         setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
         setting 'http.content_type.required', 'true'
     }
 }
 
 // Task to run BWC tests against the old cluster
 task testAgainstOldCluster(type: StandaloneRestIntegTestTask) {
-    dependsOn "unZipBwcPlugin"
+    if(!ext.bwcBundleTest){
+        dependsOn "unZipBwcPlugin"
+    }
     useCluster testClusters."${baseName}"
     systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster'
     systemProperty 'tests.is_old_cluster', 'true'
     systemProperty 'tests.skip_delete_model_index', 'true'
-    systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version
+    systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version
 
     //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version.
-    if (neural_search_bwc_version.startsWith("2.9") || neural_search_bwc_version.startsWith("2.10")){
+    if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){
         filter {
             excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*"
             excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*"
@@ -54,16 +84,22 @@ task testAgainstNewCluster(type: StandaloneRestIntegTestTask) {
     dependsOn "testAgainstOldCluster"
     dependsOn rootProject.tasks.assemble
     useCluster testClusters."${baseName}"
-    doFirst {
-        testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion(project.ext.plugins)
+    if(ext.bwcBundleTest){
+        doFirst {
+            testClusters."${baseName}".goToNextVersion()
+        }
+    }else{
+        doFirst {
+            testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion(project.ext.plugins)
+        }
     }
     systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster'
     systemProperty 'tests.skip_delete_model_index', 'true'
     systemProperty 'tests.is_old_cluster', 'false'
-    systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version
+    systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version
 
     //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version.
-    if (neural_search_bwc_version.startsWith("2.9") || neural_search_bwc_version.startsWith("2.10")){
+    if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){
         filter {
             excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*"
             excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*"

qa/rolling-upgrade/build.gradle

@@ -7,20 +7,48 @@ import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 apply from : "$rootDir/qa/build.gradle"
 
-String default_bwc_version = System.getProperty("bwc.version")
-String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version)
+def ext=rootProject.ext
 String baseName = "neuralSearchBwcCluster-rolling"
 
 // Creates a test cluster of previous version and loads k-NN plugin of bwcVersion
 testClusters {
     "${baseName}" {
         testDistribution = "ARCHIVE"
-        versions = [neural_search_bwc_version, opensearch_version]
-        numberOfNodes = 3
-        plugin(project.tasks.unZipBwcMlCommonsPlugin.archiveFile)
-        plugin(project.tasks.unZipBwcKnnPlugin.archiveFile)
-        plugin(project.tasks.unZipBwcPlugin.archiveFile)
         jvmArgs("-Xms1g", "-Xmx4g")
+        numberOfNodes = 3
+        if(ext.bwcBundleTest){
+            versions = [ext.neural_search_bwc_version, ext.currentBundleVersion]
+            def path=ext.opensearch_tmp_dir
+            nodes.each { node ->
+                node.extraConfigFile("kirk.pem", file("$path/kirk.pem"))
+                node.extraConfigFile("kirk-key.pem", file("$path/kirk-key.pem"))
+                node.extraConfigFile("esnode.pem", file("$path/esnode.pem"))
+                node.extraConfigFile("esnode-key.pem", file("$path/esnode-key.pem"))
+                node.extraConfigFile("root-ca.pem", file("$path/root-ca.pem"))
+                node.setting("plugins.security.disabled", "true")
+                node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
+                node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
+                node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
+                node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
+                node.setting("plugins.security.ssl.http.enabled", "true")
+                node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
+                node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
+                node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
+                node.setting("plugins.security.allow_unsafe_democertificates", "true")
+                node.setting("plugins.security.allow_default_init_securityindex", "true")
+                node.setting("plugins.security.authcz.admin_dn", "CN=kirk,OU=client,O=client,L=test,C=de")
+                node.setting("plugins.security.audit.type", "internal_elasticsearch")
+                node.setting("plugins.security.enable_snapshot_restore_privilege", "true")
+                node.setting("plugins.security.check_snapshot_restore_write_privileges", "true")
+                node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]")
+                node.setting("plugins.security.system_indices.enabled", "true")
+            }
+        }else{
+            versions = [ext.neural_search_bwc_version, opensearch_version]
+            plugin(project.tasks.unZipBwcMlCommonsPlugin.archiveFile)
+            plugin(project.tasks.unZipBwcKnnPlugin.archiveFile)
+            plugin(project.tasks.unZipBwcPlugin.archiveFile)
+        }
         setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
         setting 'http.content_type.required', 'true'
     }
@@ -29,14 +57,16 @@ testClusters {
 
 // Task to run BWC tests against the old cluster
 task testAgainstOldCluster(type: StandaloneRestIntegTestTask) {
-    dependsOn "unZipBwcPlugin"
+    if(!ext.bwcBundleTest){
+        dependsOn "unZipBwcPlugin"
+    }
     useCluster testClusters."${baseName}"
     systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster'
-    systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version
+    systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version
     systemProperty 'tests.skip_delete_model_index', 'true'
 
     //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version.
-    if (neural_search_bwc_version.startsWith("2.9") || neural_search_bwc_version.startsWith("2.10")){
+    if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){
         filter {
             excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*"
             excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*"
@@ -55,16 +85,22 @@ task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) {
     useCluster testClusters."${baseName}"
     dependsOn rootProject.tasks.assemble
     dependsOn "testAgainstOldCluster"
-    doFirst {
-        testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(project.ext.plugins)
+    if (ext.bwcBundleTest){
+        doFirst {
+            testClusters."${baseName}".nextNodeToNextVersion()
+        }
+    }else{
+        doFirst {
+            testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(project.ext.plugins)
+        }
     }
     systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster'
     systemProperty 'tests.rest.first_round', 'true'
     systemProperty 'tests.skip_delete_model_index', 'true'
-    systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version
+    systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version
 
     //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version.
-    if (neural_search_bwc_version.startsWith("2.9") || neural_search_bwc_version.startsWith("2.10")){
+    if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){
         filter {
             excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*"
             excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*"
@@ -82,16 +118,22 @@ task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) {
 task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) {
     dependsOn "testAgainstOneThirdUpgradedCluster"
     useCluster testClusters."${baseName}"
-    doFirst {
-        testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(project.ext.plugins)
+    if (ext.bwcBundleTest){
+        doFirst {
+            testClusters."${baseName}".nextNodeToNextVersion()
+        }
+    }else{
+        doFirst {
+            testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(project.ext.plugins)
+        }
     }
     systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster'
     systemProperty 'tests.rest.first_round', 'false'
     systemProperty 'tests.skip_delete_model_index', 'true'
-    systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version
+    systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version
 
     //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version.
-    if (neural_search_bwc_version.startsWith("2.9") || neural_search_bwc_version.startsWith("2.10")){
+    if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){
         filter {
             excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*"
             excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*"
@@ -109,16 +151,22 @@ task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) {
 task testRollingUpgrade(type: StandaloneRestIntegTestTask) {
     dependsOn "testAgainstTwoThirdsUpgradedCluster"
     useCluster testClusters."${baseName}"
-    doFirst {
-        testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(project.ext.plugins)
+    if (ext.bwcBundleTest){
+        doFirst {
+            testClusters."${baseName}".nextNodeToNextVersion()
+        }
+    }else{
+        doFirst {
+            testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(project.ext.plugins)
+        }
     }
     mustRunAfter "testAgainstOneThirdUpgradedCluster"
     systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster'
     systemProperty 'tests.skip_delete_model_index', 'true'
-    systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version
+    systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version
 
     //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version.
-    if (neural_search_bwc_version.startsWith("2.9") || neural_search_bwc_version.startsWith("2.10")){
+    if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){
         filter {
             excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*"
             excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*"