Deploy native artifacts in GHA runners

tensorflow · Feb 13, 2024 · 959bb16 · 959bb16
1 parent aec5ab7
commit 959bb16
Show file tree

Hide file tree

Showing 5 changed files with 76 additions and 29 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -12,6 +12,8 @@ on:
     types: [opened, reopened, synchronize, labeled, unlabeled]
 env:
   STAGING_PROFILE_ID: 46f80d0729c92d
+  DEPLOY_SNAPSHOT: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/staging') }}
+  DEPLOY_RELEASE: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/heads/r') }}
 jobs:
   check-format:
     if: github.event_name == 'pull_request'
@@ -35,19 +37,30 @@ jobs:
   prepare:
     runs-on: ubuntu-20.04
     outputs:
-      stagingRepositoryId: ${{ steps.staging.outputs.stagingRepositoryId }}
+      repositoryUrl: ${{ steps.repository.outputs.repositoryUrl }}
     steps:
       - name: Create staging repository
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/heads/r')
+        if: env.DEPLOY_RELEASE == 'true'
         id: staging
         run: |
           echo "Creating staging repository with profile $STAGING_PROFILE_ID"
           echo "<promoteRequest><data><description>Releasing TF Java - created by CI build</description></data></promoteRequest>" > request.xml
           curl -X POST -d @request.xml -s -o response.xml -u ${{ secrets.CI_DEPLOY_USERNAME }}:${{ secrets.CI_DEPLOY_PASSWORD }} -H "Content-Type:application/xml" \
               https://oss.sonatype.org/service/local/staging/profiles/$STAGING_PROFILE_ID/start
-          STAGING_REPOSITORY_ID=`awk -F'[<>]' '/stagedRepositoryId/{print $3}' response.xml`
+          export STAGING_REPOSITORY_ID=`awk -F'[<>]' '/stagedRepositoryId/{print $3}' response.xml`
           echo "Staging repository created: $STAGING_REPOSITORY_ID"
-          echo "::set-output name=stagingRepositoryId::$STAGING_REPOSITORY_ID"
+      - name: Checkout repository
+        uses: actions/checkout@v1
+      - name: Extract distribution repository URL
+        id: repository
+        run: |
+          if [[ "${{ env.DEPLOY_RELEASE }}" = "true" ]]; then
+            export REPOSITORY_URL=`mvn exec:exec -q -N -Dexec.executable='echo' -Dexec.args="\\${project.distributionManagement.repository.url}" -DstagingRepositoryId=$STAGING_REPOSITORY_ID`
+          else
+            export REPOSITORY_URL=`mvn exec:exec -q -N -Dexec.executable='echo' -Dexec.args="\\${project.distributionManagement.snapshotRepository.url}"`
+          fi
+          echo "Repository URL: $REPOSITORY_URL"
+          echo "::set-output name=repositoryUrl::$REPOSITORY_URL"
   linux-x86_64:
     runs-on: ubuntu-20.04
     needs: prepare
@@ -67,7 +80,10 @@ jobs:
           gcc --version
           mvn -version
           echo "<settings><servers><server><id>ossrh</id><username>${{ secrets.CI_DEPLOY_USERNAME }}</username><password>${{ secrets.CI_DEPLOY_PASSWORD }}</password></server></servers></settings>" > $HOME/.m2/settings.xml
-          mvn clean install -pl '!tensorflow-framework' -B -U -e -Djavacpp.platform=linux-x86_64 -Djavacpp.platform.extension=${{ matrix.ext }} -am -DstagingRepositoryId=${{ needs.prepare.outputs.stagingRepositoryId }}
+          mvn clean install -pl '!tensorflow-framework' -B -U -e -Djavacpp.platform.extension=${{ matrix.ext }} -am
+      - name: Deploy native artifact
+        if: env.DEPLOY_RELEASE == 'true' || env.DEPLOY_SNAPSHOT == 'true'
+        run: mvn -f tensorflow-core/tensorflow-core-native/pom.xml deploy:deploy-file@native-only -B -U -e -Durl=${{ needs.prepare.outputs.repositoryUrl }} -Djavacpp.platform.extension=${{ matrix.ext }}
   macosx-arm64:
     runs-on: macos-14
     needs: prepare
@@ -88,7 +104,10 @@ jobs:
           clang --version
           mvn -version
           echo "<settings><servers><server><id>ossrh</id><username>${{ secrets.CI_DEPLOY_USERNAME }}</username><password>${{ secrets.CI_DEPLOY_PASSWORD }}</password></server></servers></settings>" > $HOME/.m2/settings.xml
-          mvn clean install -pl '!tensorflow-framework' -B -U -e -Djavacpp.platform=macosx-arm64 -Djavacpp.platform.extension=${{ matrix.ext }} -am -DstagingRepositoryId=${{ needs.prepare.outputs.stagingRepositoryId }}
+          mvn clean install -pl '!tensorflow-framework' -B -U -e -Djavacpp.platform.extension=${{ matrix.ext }} -am
+      - name: Deploy native artifact
+        if: env.DEPLOY_RELEASE == 'true' || env.DEPLOY_SNAPSHOT == 'true'
+        run: mvn -f tensorflow-core/tensorflow-core-native/pom.xml deploy:deploy-file@native-only -B -U -e -Durl=${{ needs.prepare.outputs.repositoryUrl }} -Djavacpp.platform.extension=${{ matrix.ext }}
   macosx-x86_64:
     runs-on: macos-11
     needs: prepare
@@ -108,9 +127,11 @@ jobs:
           clang --version
           mvn -version
           echo "<settings><servers><server><id>ossrh</id><username>${{ secrets.CI_DEPLOY_USERNAME }}</username><password>${{ secrets.CI_DEPLOY_PASSWORD }}</password></server></servers></settings>" > $HOME/.m2/settings.xml
-          mvn clean install -pl '!tensorflow-framework' -B -U -e -Djavacpp.platform=macosx-x86_64 -Djavacpp.platform.extension=${{ matrix.ext }} -am -DstagingRepositoryId=${{ needs.prepare.outputs.stagingRepositoryId }}
+          mvn clean install -pl '!tensorflow-framework' -B -U -e -Djavacpp.platform.extension=${{ matrix.ext }} -am
+      - name: Deploy native artifact
+        if: env.DEPLOY_RELEASE == 'true' || env.DEPLOY_SNAPSHOT == 'true'
+        run: mvn -f tensorflow-core/tensorflow-core-native/pom.xml deploy:deploy-file@native-only -B -U -e -Durl=${{ needs.prepare.outputs.repositoryUrl }} -Djavacpp.platform.extension=${{ matrix.ext }}
   windows-x86_64:
-    if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'CI build')
     runs-on: windows-2019
     needs: prepare
     strategy:
@@ -148,10 +169,13 @@ jobs:
           call mvn -version
           echo ^<settings^>^<servers^>^<server^>^<id^>ossrh^</id^>^<username^>${{ secrets.CI_DEPLOY_USERNAME }}^</username^>^<password^>${{ secrets.CI_DEPLOY_PASSWORD }}^</password^>^</server^>^</servers^>^</settings^> > %USERPROFILE%\.m2\settings.xml
           set "SKIP_EXPORT=true"
-          call mvn clean install -pl "!tensorflow-framework" -B -U -e -Djavacpp.platform=windows-x86_64 -Djavacpp.platform.extension=${{ matrix.ext }} -am -DstagingRepositoryId=${{ needs.prepare.outputs.stagingRepositoryId }}
+          call mvn clean install -pl "!tensorflow-framework" -B -U -e -Djavacpp.platform.extension=${{ matrix.ext }} -am
           if ERRORLEVEL 1 exit /b
-  deploy:
-    if: github.event_name == 'push' && (contains(github.ref, 'master') || contains(github.ref, 'staging'))
+      - name: Deploy native artifact
+        if: env.DEPLOY_RELEASE == 'true' || env.DEPLOY_SNAPSHOT == 'true'
+        run: mvn -f tensorflow-core/tensorflow-core-native/pom.xml deploy:deploy-file@native-only -B -U -e -Durl=${{ needs.prepare.outputs.repositoryUrl }} -Djavacpp.platform.extension=${{ matrix.ext }}
+  deploy-snapshot:
+    if: env.DEPLOY_SNAPSHOT == 'true'
     needs: [linux-x86_64, macosx-x86_64, windows-x86_64, macosx-arm64]
     runs-on: ubuntu-20.04
     steps:

diff --git a/pom.xml b/pom.xml
@@ -148,6 +148,7 @@
         <maven.source.skip>false</maven.source.skip>
       </properties>
     </profile>
+
     <!--
     Releasing profile
         Activate GPG signing on all built artifacts and fetch native artifacts from staging repository

diff --git a/tensorflow-core/pom.xml b/tensorflow-core/pom.xml
@@ -68,7 +68,7 @@
     <profile>
       <!--
         When deploying, the build packages and distributes native binaries for all supported platforms. These native artifacts
-        must have already been built, tested and published priorly on their respective platform.
+        must have already been built, tested and deployed priorly from their respective platform.
       -->
       <id>deploying</id>
       <properties>

diff --git a/tensorflow-core/tensorflow-core-api/pom.xml b/tensorflow-core/tensorflow-core-api/pom.xml
@@ -177,18 +177,6 @@
         </plugins>
       </build>
     </profile>
-
-    <profile>
-      <!--
-        Deploying Profile
-          Avoid deploying snapshots since all artifacts must have the same timestamp to avoid
-          issues when using Gradle.
-      -->
-      <id>deploying</id>
-      <properties>
-        <maven.deploy.skip>true</maven.deploy.skip>
-      </properties>
-    </profile>
   </profiles>
 
   <build>

diff --git a/tensorflow-core/tensorflow-core-native/pom.xml b/tensorflow-core/tensorflow-core-native/pom.xml
@@ -58,6 +58,7 @@
   </dependencies>
 
   <profiles>
+    <!-- Do a full native build using Bazel -->
     <profile>
       <id>native-build</id>
       <activation>
@@ -73,13 +74,18 @@
       </properties>
     </profile>
 
+    <!-- Deploying mode: don't create native artifacts, use ones published by CI build -->
+    <profile>
+      <id>deploying</id>
+      <properties>
+        <javacpp.build.skip>true</javacpp.build.skip>
+        <dist.download.skip>true</dist.download.skip>
+      </properties>
+    </profile>
+
+    <!-- Generating mode: produce Java files from native resources -->
     <profile>
       <id>generating</id>
-      <activation>
-        <property>
-          <name>native.generate</name>
-        </property>
-      </activation>
       <properties>
         <bazel.generate.skip>false</bazel.generate.skip>
         <bazel.clean.skip>false</bazel.clean.skip>
@@ -467,6 +473,34 @@
         </executions>
       </plugin>
 
+      <plugin>
+        <artifactId>maven-deploy-plugin</artifactId>
+        <version>3.1.1</version>
+        <executions>
+          <execution>
+            <!--
+              Deploy only native artifact
+                This is used by the CI build to only deploy the native JAR artifact in the platform-specific runners.
+                Other artifacts will be deployed in a second pass, when activating the `deploying` profile
+            -->
+            <id>native-only</id>
+            <goals>
+              <goal>deploy-file</goal>
+            </goals>
+            <configuration>
+              <!-- 'url' property must be provided in command line -->
+              <file>${project.build.directory}/${project.artifactId}-${project.version}-${native.classifier}.jar</file>
+              <repositoryId>ossrh</repositoryId>
+              <groupId>${project.groupId}</groupId>
+              <artifactId>${project.artifactId}</artifactId>
+              <classifier>${native.classifier}</classifier>
+              <pomFile>pom.xml</pomFile>
+              <generatePom>false</generatePom>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+
       <plugin>
         <artifactId>maven-surefire-plugin</artifactId>
         <executions>