[Playground] Use current Go SDK by default

.github/workflows/playground_examples_ci_reusable.yml

@@ -123,14 +123,16 @@ jobs:
         with:
             cache-read-only: false
 
-      - name: Build Python SDK runner base image
-        if: env.SDK == 'python'
+      - name: Build an SDK runner base image and set SDK_TAG if needed
         run: |
              set -uex
-             ./gradlew -i :sdks:python:container:py37:docker -Pdocker-tag="$DOCKERTAG"
-             echo "SDK_TAG=${DOCKERTAG}" >> $GITHUB_ENV
-
-      # TODO build from the current ./sdks/... for Java and Go
+             if [ "$SDK" == "python" ]; then
+               # builds apache/beam_python3.7_sdk:$DOCKERTAG image
+               # and sets SDK_TAG to DOCKERTAG so that the next step would found it
+               ./gradlew -i :sdks:python:container:py37:docker -Pdocker-tag="$DOCKERTAG"
+               echo "SDK_TAG=${DOCKERTAG}" >> $GITHUB_ENV
+             fi
+
       - name: Build SDK Backend Docker image
         run: |
              set -ex
@@ -139,11 +141,16 @@ jobs:
                 opts="$opts -Psdk-tag=$SDK_TAG"
              fi
 
+             # by default (w/o -Psdk-tag) runner uses BEAM from local ./sdks
+             # TODO Java SDK doesn't, it uses 2.42.0, fix this
              ./gradlew -i playground:backend:containers:$SDK:docker $opts
 
       - name: Set docker image
         run: echo "IMAGE_TAG=apache/beam_playground-backend-$SDK:$DOCKERTAG" >> $GITHUB_ENV
 
+      - name: Set docker image
+        run: echo "IMAGE_TAG=apache/beam_playground-backend-$SDK:$DOCKERTAG" >> $GITHUB_ENV
+
       - name: Start SDK runner in background
         run: |
           set -uex

playground/backend/containers/go/Dockerfile

@@ -16,35 +16,34 @@
 # limitations under the License.
 ###############################################################################
 #Dokerfile to set up the Beam Go SDK
-ARG BASE_IMAGE=golang:1.18-bullseye
+ARG BASE_IMAGE
 FROM $BASE_IMAGE
+ARG SDK_TAG
+ARG SDK_TAG_LOCAL
 
 # Setup Go Environment
 ENV GOPATH /go
 ENV PATH $GOPATH/bin:$PATH
-RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
+RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" &&\
+    chmod -R 777 "$GOPATH/bin"
+
+
 RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1 &&\
     go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0
 
-# Prepare Application
-COPY src /go/src/playground/backend
+# Setup Go SDK dependencies
+COPY beam /opt/playground/beam
+COPY setup_sdk.sh /
 
-ARG BEAM_VERSION=2.42.0
-ENV PREPARED_MOD_DIR=/opt/playground/prepared_folder/
-ENV PIPELINES_FOLDER_NAME="executable_files"
-RUN mkdir -p /opt/playground/ $PREPARED_MOD_DIR
-WORKDIR $PREPARED_MOD_DIR
+ENV BEAM_SRC /opt/playground/beam
+ENV PREPARED_MOD_DIR /opt/playground/prepared_folder
+RUN /setup_sdk.sh /opt/playground/prepared_folder
 
-RUN go mod init $PIPELINES_FOLDER_NAME &&\
-    go get github.com/apache/beam/sdks/v2/go/pkg/beam@v$BEAM_VERSION &&\
-    go get github.com/apache/beam/sdks/v2/go/pkg/beam/io/filesystem/gcs@v$BEAM_VERSION &&\
-    go mod download all
+# Prepare Application
+COPY src /go/src/playground/backend
 
 # Build Application
-
-#RUN
 WORKDIR /go/src/playground/backend
-RUN ls
 # Build Application
 RUN go mod download &&\
     go mod tidy &&\
@@ -62,7 +61,7 @@ COPY src/configs /opt/playground/backend/configs/
 # Install mitmpoxy
 RUN mkdir /opt/mitmproxy &&\
     cd /opt/mitmproxy &&\
-    wget https://snapshots.mitmproxy.org/7.0.4/mitmproxy-7.0.4-linux.tar.gz &&\
+    wget -q https://snapshots.mitmproxy.org/7.0.4/mitmproxy-7.0.4-linux.tar.gz &&\
     tar -zxvf mitmproxy-7.0.4-linux.tar.gz &&\
     mkdir /usr/local/share/ca-certificates/extra
 COPY allow_list_proxy.py /opt/mitmproxy/

playground/backend/containers/go/build.gradle

@@ -43,6 +43,10 @@ task copyDockerfileDependencies(type: Copy) {
       from 'entrypoint.sh'
       into 'build/'
    }
+   copy {
+      from 'setup_sdk.sh'
+      into 'build/'
+   }
    copy {
       from '../../../infrastructure/proxy/allow_list.py'
       into 'build/'
@@ -55,6 +59,18 @@ task copyDockerfileDependencies(type: Copy) {
       from '../../../playground'
       into 'build/playground'
    }
+   copy {
+      from '../../../../sdks/go'
+      into 'build/beam/go'
+   }
+   copy {
+      from '../../../../sdks/go.mod'
+      into 'build/beam'
+   }
+   copy {
+      from '../../../../sdks/go.sum'
+      into 'build/beam'
+   }
 }
 
 docker {
@@ -67,7 +83,10 @@ docker {
   tags containerImageTags()
   buildArgs(['BASE_IMAGE': project.rootProject.hasProperty(["base-image"]) ?
                            project.rootProject["base-image"] :
-                           "golang:1.18-bullseye" ])
+                           "golang:1.18-bullseye",
+             'SDK_TAG': project.rootProject.hasProperty(["sdk-tag"]) ?
+                  project.rootProject["sdk-tag"] : project.rootProject.sdk_version,
+             'SDK_TAG_LOCAL': project.rootProject.sdk_version])
 }
 
 // Ensure that we build the required resources and copy and file dependencies from related projects

playground/backend/containers/go/setup_sdk.sh

@@ -0,0 +1,47 @@
+#!/bin/bash -uex
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Usage: ./setup_sdk.sh /opt/playground/prepared_folder
+# Setup Go Beam SDK dev environment, so that examples' dependencies were met.
+# Consumed envs:
+# - PREPARED_MOD_DIR: result module path
+# - SDK_TAG: desired SDK version, for ex. "2.42.0", "2.44.0dev"
+# - SDK_TAG_LOCAL: special value for SDK_TAG for the local Go SDK (defined by build system)
+# - BEAM_SRC: Go SDK location
+
+PIPELINES_MODULE="executable_files"
+
+# Project convention for local versions 2.44.0.dev is incorrect in Go
+# Replace this with 2.44.0-dev
+SDK_TAG=${SDK_TAG/%.dev/-dev}
+SDK_TAG_LOCAL=${SDK_TAG_LOCAL/%.dev/-dev}
+
+mkdir -p $PREPARED_MOD_DIR
+cd $PREPARED_MOD_DIR
+
+BEAM_PKG=github.com/apache/beam/sdks/v2
+
+go mod init $PIPELINES_MODULE
+go mod edit -require=$BEAM_PKG@v$SDK_TAG
+
+if [ "$SDK_TAG" == "$SDK_TAG_LOCAL" ]; then
+  go mod edit -replace=$BEAM_PKG@v$SDK_TAG=$BEAM_SRC
+fi
+
+go get -x $BEAM_PKG/go/pkg/beam@v$SDK_TAG
+go get -x $BEAM_PKG/go/test/integration@v$SDK_TAG
+
+go mod download -x all

playground/backend/containers/python/Dockerfile

@@ -16,7 +16,7 @@
 # limitations under the License.
 ###############################################################################
 ARG GO_BASE_IMAGE=golang:1.18-bullseye
-ARG SDK_TAG
+ARG SDK_TAG=2.42.0
 ARG BASE_IMAGE=apache/beam_python3.7_sdk:$SDK_TAG
 FROM $GO_BASE_IMAGE AS build
 

playground/backend/containers/python/build.gradle

@@ -69,14 +69,12 @@ docker {
                   project.docker_image_default_repo_root)
   files "./build/"
   tags containerImageTags()
-  buildArgs(['BASE_IMAGE': project.rootProject.hasProperty(["base-image"]) ?
-                           project.rootProject["base-image"] :
-                           "apache/beam_python3.7_sdk" ])
   buildArgs(['GO_BASE_IMAGE': project.rootProject.hasProperty(["go-base-image"]) ?
                              project.rootProject["go-base-image"] :
-                             "golang:1.18-bullseye" ])
-  buildArgs(['SDK_TAG': project.rootProject.hasProperty(["sdk-tag"]) ?
-                  project.rootProject["sdk-tag"] : project.rootProject.sdk_version])
+                             "golang:1.18-bullseye",
+             'SDK_TAG': project.rootProject.hasProperty(["sdk-tag"]) ?
+                             project.rootProject["sdk-tag"] : project.rootProject.sdk_version
+   ])
 }
 
 // Ensure that we build the required resources and copy and file dependencies from related projects