Backport Sonatype publishing and datatypes module for v0.3.x (#407)

* Use Nexus staging plugin for deployment (#394) * Use Nexus staging pluging for deployment * Fix Javadoc error * Hard coded parent version as variable substitution is not supported * Introduce datatypes/java module for proto generation Rather than the Maven protobuf plugin running on the same symlinked definitions in several Java modules, localize this process into one module that the others depend on. This provides a single module that can be depended on by third-party extensions with the bare minimum of dependencies. Also removes proto files that are no longer used. * Java SDK release script (#406) * Use back revision variable in pom.xml So user or CI system can easily override revision from external sources such as Git tag name * Add flatten maven plugin This plugin is useful during deployment so the final pom is resolved without parent dependency, i.e. we do not necessarily need to upload parent library * Increase versions for maven source,javadoc,spotless plugins So it has newer features and more fixes * Add gpg-plugin needed to sign releases * Use oss configure for flatten plugin, add developers info in pom.xml (required for releasing library * Add publish-java-sdk script * Add more logs to publish-java-sdk.sh * Add ProwJob publish-java-sdk * Use GPG_KEY_IMPORT_DIR variable * Update revision in pom.xml to 0.4.2-SNAPSHOT * Publish datatypes/java along with sdk/java Co-authored-by: Khor Shu Heng <32997938+khorshuheng@users.noreply.github.com> Co-authored-by: David Heryanto <david.heryanto@hotmail.com>
feast-dev · Jan 8, 2020 · 801408a · 801408a
1 parent 505bdc6
commit 801408a
Show file tree

Hide file tree

Showing 30 changed files with 379 additions and 150 deletions.
diff --git a/.gitignore b/.gitignore
@@ -173,3 +173,9 @@ dmypy.json
 # Pyre type checker
 .pyre/
 .vscode
+
+# .flattened-pom.xml is generated by flatten-maven-plugin.
+# This pom should not be committed because it is only used during release / deployment.
+.flattened-pom.xml
+
+sdk/python/docs/html
diff --git a/.prow/config.yaml b/.prow/config.yaml
@@ -169,6 +169,34 @@ postsubmits:
     # https://github.com/semver/semver/issues/232
     - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
 
+  - name: publish-java-sdk
+    decorate: true
+    spec:
+      containers:
+      - image: maven:3.6-jdk-8
+        command:
+        - bash
+        - -c
+        - .prow/scripts/publish-java-sdk.sh --revision ${PULL_BASE_REF:1}
+        volumeMounts:
+        - name: gpg-keys
+          mountPath: /etc/gpg
+          readOnly: true
+        - name: maven-settings
+          mountPath: /root/.m2/settings.xml
+          subPath: settings.xml
+          readOnly: true
+      volumes:
+      - name: gpg-keys
+        secret:
+          secretName: gpg-keys
+      - name: maven-settings
+        secret:
+          secretName: maven-settings
+    branches:
+    # Filter on tags with semantic versioning, prefixed with "v"
+    - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
+
   - name: publish-docker-images
     decorate: true
     spec:
@@ -278,4 +306,5 @@ postsubmits:
         secret:
           secretName: feast-service-account
     branches:
+    # Filter on tags with semantic versioning, prefixed with "v"
     - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
diff --git a/.prow/scripts/publish-java-sdk.sh b/.prow/scripts/publish-java-sdk.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+set -e 
+set -o pipefail
+
+GPG_KEY_IMPORT_DIR=/etc/gpg
+
+usage()
+{
+    echo "usage: publish-java-sdk.sh
+
+  --revision            Value for the revision e.g. '0.2.3'
+  --gpg-key-import-dir  Directory containing existing GPG keys to import.
+                        The directory should contain these 2 files:
+                        - public-key
+                        - private-key
+                        The default value is '/etc/gpg'
+  
+  This script assumes the GPG private key is protected by a passphrase.
+  The passphrase can be specified in \$HOME/.m2/settings.xml. In the same xml
+  file, credentials to upload releases to Sonatype must also be provided.
+
+  # Example settings: ~/.m2/settings.xml
+  <settings>
+    <servers>
+      <server>
+        <id>ossrh</id>
+        <username>SONATYPE_USER</username>
+        <password>SONATYPE_PASSWORD</password>
+      </server>
+    </servers>
+    <profiles>
+      <profile>
+        <id>ossrh</id>
+        <properties>
+          <gpg.passphrase>GPG_PASSPHRASE</gpg.passphrase>
+        </properties>
+      </profile>
+    </profiles>
+  </settings>
+"
+}
+
+while [ "$1" != "" ]; do
+  case "$1" in
+      --revision )             REVISION="$2";            shift;;
+      --gpg-key-import-dir )   GPG_KEY_IMPORT_DIR="$2";  shift;;
+      -h | --help )            usage;                    exit;; 
+      * )                      usage;                    exit 1
+  esac
+  shift
+done
+
+if [ -z $REVISION ]; then usage; exit 1; fi
+
+echo "============================================================"
+echo "Checking Maven and GPG versions"
+echo "============================================================"
+mvn --version 
+echo ""
+gpg --version
+
+echo "============================================================"
+echo "Importing GPG keys"
+echo "============================================================"
+gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/public-key
+gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/private-key
+
+echo "============================================================"
+echo "Deploying Java SDK with revision: $REVISION"
+echo "============================================================"
+mvn --projects datatypes/java,sdk/java -Drevision=$REVISION --batch-mode clean deploy
diff --git a/.prow/scripts/test-end-to-end-batch.sh b/.prow/scripts/test-end-to-end-batch.sh
@@ -138,7 +138,7 @@ management:
         enabled: false
 EOF
 
-nohup java -jar core/target/feast-core-0.3.2-SNAPSHOT.jar \
+nohup java -jar core/target/feast-core-*-SNAPSHOT.jar \
   --spring.config.location=file:///tmp/core.application.yml \
   &> /var/log/feast-core.log &
 sleep 30
@@ -191,7 +191,7 @@ spring:
     web-environment: false
 EOF
 
-nohup java -jar serving/target/feast-serving-0.3.2-SNAPSHOT.jar \
+nohup java -jar serving/target/feast-serving-*-SNAPSHOT.jar \
   --spring.config.location=file:///tmp/serving.warehouse.application.yml \
   &> /var/log/feast-serving-warehouse.log &
 sleep 15

diff --git a/.prow/scripts/test-end-to-end.sh b/.prow/scripts/test-end-to-end.sh
@@ -75,6 +75,9 @@ Building jars for Feast
 # Build jars for Feast
 mvn --quiet --batch-mode --define skipTests=true clean package
 
+ls -lh core/target/*jar
+ls -lh serving/target/*jar
+
 echo "
 ============================================================
 Starting Feast Core
@@ -121,7 +124,7 @@ management:
         enabled: false
 EOF
 
-nohup java -jar core/target/feast-core-0.3.2-SNAPSHOT.jar \
+nohup java -jar core/target/feast-core-*-SNAPSHOT.jar \
   --spring.config.location=file:///tmp/core.application.yml \
   &> /var/log/feast-core.log &
 sleep 30
@@ -172,7 +175,7 @@ spring:
     web-environment: false
 EOF
 
-nohup java -jar serving/target/feast-serving-0.3.2-SNAPSHOT.jar \
+nohup java -jar serving/target/feast-serving-*-SNAPSHOT.jar \
   --spring.config.location=file:///tmp/serving.online.application.yml \
   &> /var/log/feast-serving-online.log &
 sleep 15

diff --git a/Makefile b/Makefile
@@ -32,7 +32,7 @@ build-cli:
 	$(MAKE) -C cli build-all
 
 build-java:
-	mvn clean verify -Drevision=$(VERSION)
+	mvn clean verify
 
 build-docker:
 	docker build -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile .

diff --git a/core/pom.xml b/core/pom.xml
@@ -21,7 +21,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <parent>
-        <groupId>feast</groupId>
+        <groupId>dev.feast</groupId>
         <artifactId>feast-parent</artifactId>
         <version>${revision}</version>
     </parent>
@@ -39,16 +39,12 @@
                     <skip>false</skip>
                 </configuration>
             </plugin>
-            <plugin>
-                <groupId>org.xolstice.maven.plugins</groupId>
-                <artifactId>protobuf-maven-plugin</artifactId>
-            </plugin>
         </plugins>
     </build>
 
     <dependencies>
         <dependency>
-            <groupId>feast</groupId>
+            <groupId>dev.feast</groupId>
             <artifactId>feast-ingestion</artifactId>
             <version>${project.version}</version>
         </dependency>

diff --git a/core/src/main/java/feast/core/config/MonitoringConfig.java b/core/src/main/java/feast/core/config/MonitoringConfig.java
@@ -66,7 +66,7 @@ public FeastResourceCollector feastResourceCollector(
   /**
    * Register custom Prometheus collector that exports metrics about JVM resource usage.
    *
-   * @return @{link {@link JVMResourceCollector}}
+   * @return {@link JVMResourceCollector}
    */
   @Bean
   public JVMResourceCollector jvmResourceCollector() {

diff --git a/core/src/main/java/feast/core/service/SpecService.java b/core/src/main/java/feast/core/service/SpecService.java
@@ -82,7 +82,7 @@ public SpecService(
    * required. If the version is provided then it will be used for the lookup. If the version is
    * omitted then the latest version will be returned.
    *
-   * @param GetFeatureSetRequest containing the name and version of the feature set
+   * @param request containing the name and version of the feature set
    * @return GetFeatureSetResponse containing a single feature set
    */
   public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request)
@@ -141,7 +141,7 @@ public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request)
    *
    * <p>The version filter is optional; If not provided, this method will return all featureSet
    * versions of the featureSet name provided. Valid version filters should optionally contain a
-   * comparator (<, <=, >, etc) and a version number, e.g. 10, <10, >=1
+   * comparator (&lt;, &lt;=, &gt;, etc) and a version number, e.g. 10, &lt;10, &gt;=1
    *
    * @param filter filter containing the desired featureSet name and version filter
    * @return ListFeatureSetsResponse with list of featureSets found matching the filter

diff --git a/core/src/main/java/feast/core/util/PackageUtil.java b/core/src/main/java/feast/core/util/PackageUtil.java
@@ -49,8 +49,9 @@ public class PackageUtil {
    * handled by default in Apache Beam.
    *
    * <pre>
-   * @code
+   * <code>
    * URL url = new URL("jar:file:/tmp/springexample/target/spring-example-1.0-SNAPSHOT.jar!/BOOT-INF/lib/beam-sdks-java-core-2.16.0.jar!/");
+   * </code>
    * String resolvedPath = resolveSpringBootPackageClasspath(url);
    * // resolvedPath should point to "/tmp/springexample/target/spring-example-1.0-SNAPSHOT/BOOT-INF/lib/beam-sdks-java-core-2.16.0.jar"
    * // Note that spring-example-1.0-SNAPSHOT.jar is extracted in the process.

diff --git a/core/src/main/proto/feast b/core/src/main/proto/feast
diff --git a/core/src/main/proto/third_party b/core/src/main/proto/third_party
diff --git a/datatypes/java/README.md b/datatypes/java/README.md
@@ -0,0 +1,43 @@
+Feast Data Types for Java
+=========================
+
+This module produces Java class files for Feast's data type and gRPC service
+definitions, from Protobuf IDL. These are used across Feast components for wire
+interchange, contracts, etc.
+
+End users of Feast will be best served by our Java SDK which adds higher-level
+conveniences, but the data types are published independently for custom needs,
+without any additional dependencies the SDK may add.
+
+Dependency Coordinates
+----------------------
+
+```xml
+<dependency>
+  <groupId>dev.feast</groupId>
+  <artifactId>datatypes-java</artifactId>
+  <version>0.3.6-SNAPSHOT</version>
+</dependency>
+```
+
+Using the `.proto` Definitions
+------------------------------
+
+The `.proto` definitions are packaged as resources within the Maven artifact,
+which may be useful to `include` them in dependent Protobuf definitions in a
+downstream project, or for other JVM languages to consume from their builds to
+generate more idiomatic bindings.
+
+Google's Gradle plugin, for instance, [can use protos in dependencies][Gradle]
+either for `include` or to compile with a different `protoc` plugin than Java.
+
+[sbt-protoc] offers similar functionality for sbt/Scala.
+
+[Gradle]: https://github.com/google/protobuf-gradle-plugin#protos-in-dependencies
+[sbt-protoc]: https://github.com/thesamet/sbt-protoc
+
+Publishing
+----------
+
+TODO: this module should be published to Maven Central upon Feast releases—this
+needs to be set up in POM configuration and release automation.
diff --git a/datatypes/java/pom.xml b/datatypes/java/pom.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright 2019 The Feast Authors
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     https://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <name>Feast Data Types for Java</name>
+    <description>
+        Data types and service contracts used throughout Feast components and
+        their interchanges. These are generated from Protocol Buffers and gRPC
+        definitions included in the package.
+    </description>
+    <artifactId>datatypes-java</artifactId>
+
+    <parent>
+      <groupId>dev.feast</groupId>
+      <artifactId>feast-parent</artifactId>
+      <version>${revision}</version>
+      <relativePath>../..</relativePath>
+    </parent>
+
+    <build>
+      <plugins>
+        <plugin>
+          <groupId>org.xolstice.maven.plugins</groupId>
+          <artifactId>protobuf-maven-plugin</artifactId>
+          <configuration>
+            <checkStaleness>true</checkStaleness>
+            <protocArtifact>
+                com.google.protobuf:protoc:${protocVersion}:exe:${os.detected.classifier}
+            </protocArtifact>
+            <pluginId>grpc-java</pluginId>
+            <pluginArtifact>
+                io.grpc:protoc-gen-grpc-java:${grpcVersion}:exe:${os.detected.classifier}
+            </pluginArtifact>
+          </configuration>
+          <executions>
+            <execution>
+              <goals>
+                <goal>compile</goal>
+                <goal>compile-custom</goal>
+                <goal>test-compile</goal>
+              </goals>
+            </execution>
+          </executions>
+        </plugin>
+      </plugins>
+    </build>
+
+    <dependencies>
+      <dependency>
+        <groupId>io.grpc</groupId>
+        <artifactId>grpc-services</artifactId>
+      </dependency>
+    </dependencies>
+</project>
diff --git a/sdk/java/src/main/proto/feast → datatypes/java/src/main/proto/feast b/sdk/java/src/main/proto/feast → datatypes/java/src/main/proto/feast
diff --git a/datatypes/java/src/main/proto/third_party b/datatypes/java/src/main/proto/third_party
@@ -0,0 +1 @@
+../../../../../protos/third_party
diff --git a/infra/docker/core/Dockerfile b/infra/docker/core/Dockerfile
@@ -12,7 +12,7 @@ WORKDIR /build
 # the existing .m2 directory to $FEAST_REPO_ROOT/.m2
 #
 ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false"
-RUN mvn --also-make --projects core,ingestion -Drevision=$REVISION \
+RUN mvn --also-make --projects core,ingestion \
   -DskipTests=true --batch-mode package
 #
 # Unpack the jar and copy the files into production Docker image

diff --git a/infra/docker/serving/Dockerfile b/infra/docker/serving/Dockerfile
@@ -12,7 +12,7 @@ WORKDIR /build
 # the existing .m2 directory to $FEAST_REPO_ROOT/.m2
 #
 ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false"
-RUN mvn --also-make --projects serving -Drevision=$REVISION \
+RUN mvn --also-make --projects serving \
   -DskipTests=true --batch-mode package
 
 # ============================================================