Refactor Build Pipeline to Support Multiple Java Versions

.github/workflows/codeql-analysis.yml

@@ -40,11 +40,29 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v2
 
+    - name: Set up JDK 21
+      uses: actions/setup-java@v2
+      with:
+        distribution: 'temurin'
+        java-version: '21'
+
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
+        config: |
+          query-filters:
+            - exclude:
+                id: java/URL-forward-from-remote-source
+            - exclude:
+                id: java/Disabled-Spring-CSRF-protection
+            - exclude:
+                id: java/unvalidated-url-forward
+            - exclude:
+                id: java/spring-disabled-csrf-protection
+
+
         # If you wish to specify custom queries, you can do so here or in a config file.
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
@@ -53,7 +71,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -67,4 +85,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/manual.yml

@@ -34,11 +34,11 @@ jobs: # This workflow contains a single job called "greet"
     steps: # Runs a single command using the runners shell
       - name: Checkout Code
         uses: actions/checkout@v3
-      - name: Set up JDK 11
+      - name: Set up JDK 21
         uses: actions/setup-java@v3
         with:
-          java-version: '11'
-          distribution: 'adopt'
+          java-version: '21'
+          distribution: 'temurin'
           cache: 'maven'
       - name: Setup Golang
         uses: actions/setup-go@v3

.github/workflows/maven.yml

@@ -10,45 +10,109 @@ on:
     branches: [ main ]
 
 jobs:
-  build:
+  build-java-21:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
+    - name: Set up JDK 21
+      uses: actions/setup-java@v4
+      with:
+        java-version: '21'
+        distribution: 'temurin'
+        cache: 'maven'
+
+    - name: Set up properties
+      run: sh configure-dependencies.sh
+
+    - name: Check disk space before freeing up space
+      run: df -h
+
+    - name: Free up space
+      run: |
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf /usr/local/lib/android
+        sudo rm -rf /usr/local/.ghcup/ghc/9.6.4
+
+    - name: Check disk space before build
+      run: df -h
+
+    - name: Install Docker Compose
+      run: |
+        curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+        chmod +x /usr/local/bin/docker-compose
+        docker-compose --version
+
+    - name: Build
+      run: make build-main
+      env:
+        JAVA_HOME: ${{ steps.setup-java.outputs.java-home }}
+
+    - name: Check disk space after build
+      run: df -h
+
+  build-java-11:
+    runs-on: ubuntu-latest
+    needs: build-java-21  # This ensures that Java 21 build runs first
+    steps:
+    - uses: actions/checkout@v4
     - name: Set up JDK 11
       uses: actions/setup-java@v4
       with:
         java-version: '11'
         distribution: 'adopt'
         cache: 'maven'
+
     - name: Set up properties
       run: sh configure-dependencies.sh
-    - name: Check space before free up the space
+
+    - name: Check disk space before freeing up space
       run: df -h
+
     - name: Free up space
       run: |
         sudo rm -rf /usr/share/dotnet
         sudo rm -rf /usr/local/lib/android
         sudo rm -rf /usr/local/.ghcup/ghc/9.6.4
-    - name: Check space before tests
+
+    - name: Check disk space before build
       run: df -h
+
     - name: Install Docker Compose
       run: |
         curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
         chmod +x /usr/local/bin/docker-compose
         docker-compose --version
-      # debug step
-#    - name: Setup upterm session
-#      uses: lhotari/action-upterm@v1
-#      with:
-#        ## limits ssh access and adds the ssh public key for the user which triggered the workflow ie holashchand
-#        limit-access-to-actor: true
-    - name: Build and test
-      run: make test
-    - name: Check space after tests
+
+    - name: Build
+      run: make build-legacy-services
+      env:
+        JAVA_HOME: ${{ steps.setup-java.outputs.java-home }}
+
+    - name: Check disk space after build
       run: df -h
-#  test:
-#    runs-on: ubuntu-latest
-#    steps:
-#      - uses: actions/checkout@v2
-#      - name: Run all the containers and test
-#        run : make test
+
+  test:
+    needs: [build-java-11]
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up JDK 21
+      uses: actions/setup-java@v4
+      with:
+        java-version: '21'
+        distribution: 'temurin'
+        cache: 'maven'
+    - name: Install Docker Compose
+      run: |
+        curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+        chmod +x /usr/local/bin/docker-compose
+        docker-compose --version
+    ## debug step
+    # - name: Setup upterm session
+    #   uses: lhotari/action-upterm@v1
+    #   with:
+    #     ## limits ssh access and adds the ssh public key for the user which triggered the workflow
+    #     limit-access-to-actor: true
+
+    - name: Run tests
+      run: make test

.gitignore

@@ -90,4 +90,5 @@ out
 db-data*
 vault-data
 es-data*
-keycloak-mobile*.jar
+keycloak-mobile*.jar
+.lh

.mvn/wrapper/MavenWrapperDownloader.java

@@ -0,0 +1,117 @@
+/*
+ * Copyright 2007-present the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.net.*;
+import java.io.*;
+import java.nio.channels.*;
+import java.util.Properties;
+
+public class MavenWrapperDownloader {
+
+    private static final String WRAPPER_VERSION = "0.5.6";
+    /**
+     * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
+     */
+    private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
+        + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
+
+    /**
+     * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
+     * use instead of the default one.
+     */
+    private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
+            ".mvn/wrapper/maven-wrapper.properties";
+
+    /**
+     * Path where the maven-wrapper.jar will be saved to.
+     */
+    private static final String MAVEN_WRAPPER_JAR_PATH =
+            ".mvn/wrapper/maven-wrapper.jar";
+
+    /**
+     * Name of the property which should be used to override the default download url for the wrapper.
+     */
+    private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
+
+    public static void main(String args[]) {
+        System.out.println("- Downloader started");
+        File baseDirectory = new File(args[0]);
+        System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
+
+        // If the maven-wrapper.properties exists, read it and check if it contains a custom
+        // wrapperUrl parameter.
+        File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
+        String url = DEFAULT_DOWNLOAD_URL;
+        if(mavenWrapperPropertyFile.exists()) {
+            FileInputStream mavenWrapperPropertyFileInputStream = null;
+            try {
+                mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
+                Properties mavenWrapperProperties = new Properties();
+                mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
+                url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
+            } catch (IOException e) {
+                System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
+            } finally {
+                try {
+                    if(mavenWrapperPropertyFileInputStream != null) {
+                        mavenWrapperPropertyFileInputStream.close();
+                    }
+                } catch (IOException e) {
+                    // Ignore ...
+                }
+            }
+        }
+        System.out.println("- Downloading from: " + url);
+
+        File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
+        if(!outputFile.getParentFile().exists()) {
+            if(!outputFile.getParentFile().mkdirs()) {
+                System.out.println(
+                        "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'");
+            }
+        }
+        System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
+        try {
+            downloadFileFromURL(url, outputFile);
+            System.out.println("Done");
+            System.exit(0);
+        } catch (Throwable e) {
+            System.out.println("- Error downloading");
+            e.printStackTrace();
+            System.exit(1);
+        }
+    }
+
+    private static void downloadFileFromURL(String urlString, File destination) throws Exception {
+        if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) {
+            String username = System.getenv("MVNW_USERNAME");
+            char[] password = System.getenv("MVNW_PASSWORD").toCharArray();
+            Authenticator.setDefault(new Authenticator() {
+                @Override
+                protected PasswordAuthentication getPasswordAuthentication() {
+                    return new PasswordAuthentication(username, password);
+                }
+            });
+        }
+        URL website = new URL(urlString);
+        ReadableByteChannel rbc;
+        rbc = Channels.newChannel(website.openStream());
+        FileOutputStream fos = new FileOutputStream(destination);
+        fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+        fos.close();
+        rbc.close();
+    }
+
+}

.mvn/wrapper/maven-wrapper.properties

@@ -0,0 +1,2 @@
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar

Makefile

@@ -1,32 +1,45 @@
+.PHONY: show-dir
+show-dir:
+	@echo "Current directory: $$(pwd)"
+	@echo "Files in current directory:"
+	@ls -l
+
 #SOURCES = $(wildcard java/**/*.java)
 rwildcard=$(wildcard $1$2) $(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2))
 SOURCES := $(call rwildcard,java/,*.java)
-RELEASE_VERSION = v2.0.1
+RELEASE_VERSION = 2.0.4-SNAPSHOT
 IMAGES := ghcr.io/sunbird-rc/sunbird-rc-core ghcr.io/sunbird-rc/sunbird-rc-claim-ms \
 			ghcr.io/sunbird-rc/sunbird-rc-notification-service ghcr.io/sunbird-rc/sunbird-rc-metrics \
 			ghcr.io/sunbird-rc/id-gen-service ghcr.io/sunbird-rc/encryption-service \
 			ghcr.io/sunbird-rc/sunbird-rc-identity-service ghcr.io/sunbird-rc/sunbird-rc-credential-schema \
 			ghcr.io/sunbird-rc/sunbird-rc-credentials-service
-build: java/registry/target/registry.jar
-	echo ${SOURCES}
+
+build: build-main build-legacy-services
+
+build-legacy-services:
+	make -C services/id-gen-service docker
+	make -C services/encryption-service docker
+
+build-main: java/registry/target/registry.jar
+	#echo ${SOURCES}
 	rm -rf java/claim/target/*.jar
-	cd target && rm -rf * && jar xvf ../java/registry/target/registry.jar && cp ../java/Dockerfile ./ && docker build -t ghcr.io/sunbird-rc/sunbird-rc-core .
+	@$(MAKE) show-dir
+	mkdir -p target
+	cd target && rm -rf * &&cp ../java/registry/target/registry-$(RELEASE_VERSION).jar ./registry.jar && cp ../java/Dockerfile ./ && docker build -t ghcr.io/sunbird-rc/sunbird-rc-core .
 	make -C java/claim
 	make -C services/notification-service docker
 	make -C services/metrics docker
-	make -C services/id-gen-service docker
-	make -C services/encryption-service docker
 	make -C services/identity-service/ docker
 	make -C services/credential-schema docker
 	make -C services/credentials-service/ docker
 
 
 java/registry/target/registry.jar: $(SOURCES)
-	echo $(SOURCES)
+	#echo $(SOURCES)
 	sh configure-dependencies.sh
 	cd java && ./mvnw clean install
 
-test: build
+test:
 	@docker-compose -f docker-compose-v1.yml down
 	@sudo rm -rf db-data* es-data* || echo "no permission to delete"
 	# test with distributed definition manager and native search

java/.gitignore

@@ -24,4 +24,5 @@ hs_err_pid*
 target
 mvn*
 .mvn*
-deps/keycloak/providers/keycloak-mobile-number-login-spi-1.0-SNAPSHOT.jar
+deps/keycloak/providers/keycloak-mobile-number-login-spi-1.0-SNAPSHOT.jar
+.lh

java/.mvn/wrapper/MavenWrapperDownloader.java

@@ -89,6 +89,7 @@ public static void main(String args[]) {
             System.exit(0);
         } catch (Throwable e) {
             System.out.println("- Error downloading");
+            e.printStackTrace();
             System.exit(1);
         }
     }