[CI] Build and publish integration test images (apache#70)

* [CI] Build and publish integration test images

*Motivation*

Build and publish integration test images to avoid building images in integration tests.

.github/workflows/pr-build-test-images.yml

@@ -0,0 +1,19 @@
+name: kop build test images test
+
+on:
+  pull_request:
+    branches: 
+      - master
+    paths:
+      - 'integrations/**'
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: build test images
+      run: |
+        ./integrations/build.sh

.github/workflows/push-build-test-images.yml

@@ -0,0 +1,20 @@
+name: kop build test images
+
+on:
+  push:
+    branches: 
+      - master
+    paths:
+      - 'integrations/**'
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: build test images
+      run: |
+        ./integrations/build.sh
+        ./integrations/publish.sh

dev/get-project-version.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import xml.etree.ElementTree as ET
+from os.path import dirname, realpath, join
+
+# Derive the POM path from the current script location
+TOP_LEVEL_PATH = dirname(dirname(realpath(__file__)))
+POM_PATH = join(TOP_LEVEL_PATH, 'pom.xml')
+
+root = ET.XML(open(POM_PATH).read())
+print(root.find('{http://maven.apache.org/POM/4.0.0}version').text)

integrations/README.md

@@ -0,0 +1,27 @@
+# Integration tests for KoP
+
+# Produce and Consume support
+
+## Golang
+
+* [https://github.com/Shopify/sarama](https://github.com/Shopify/sarama)
+* [https://github.com/confluentinc/confluent-kafka-go](https://github.com/confluentinc/confluent-kafka-go)
+
+## Rust
+
+* [https://github.com/fede1024/rust-rdkafka](https://github.com/fede1024/rust-rdkafka)
+
+## NodeJS
+
+* [https://github.com/Blizzard/node-rdkafka](https://github.com/Blizzard/node-rdkafka)
+
+# Partial support
+
+### [kafka-node](https://www.npmjs.com/package/kafka-node)
+
+Producing is working, but consuming is failing as the library is sending FETCH v0 regardless of API_VERSIONS responses:
+
+```
+DEBUG io.streamnative.kop.KafkaCommandDecoder - Write kafka cmd response back to client. request: RequestHeader(apiKey=FETCH, apiVersion=0, clientId=kafka-node-client, correlationId=4)
+INFO  io.streamnative.kop.KafkaIntegrationTest - STDOUT: Error: Not a message set. Magic byte is 2
+```

integrations/build.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BINDIR=$(dirname "$0")
+INTR_HOME=`cd ${BINDIR}/..;pwd`
+
+VERSION=$(${INTR_HOME}/dev/get-project-version.py)
+TAG=${VERSION%"-SNAPSHOT"}
+IMAGE_NAME_PREFIX="kop-test-"
+
+for img_dir in `ls -d ${INTR_HOME}/integrations/*/`; do
+    BASE_NAME=$(basename ${img_dir})
+    cd ${img_dir}
+    IMAGE="streamnative/${IMAGE_NAME_PREFIX}${BASE_NAME}:${TAG}"
+    echo "Building test image : ${IMAGE}"
+    docker build . -t ${IMAGE}
+    echo "Successfully built test image : ${IMAGE}"
+done

integrations/golang-confluent-kafka/Dockerfile

@@ -0,0 +1,39 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+FROM golang:1.13.4-alpine
+
+RUN apk -U add ca-certificates git bash build-base sudo  pkgconf build-base
+RUN git clone https://github.com/edenhill/librdkafka.git && cd librdkafka && ./configure --prefix /usr && make && make install
+
+WORKDIR /go/src/app
+COPY . .
+
+RUN go get -d -v ./...
+RUN go install -v ./...
+
+CMD sh -c '/go/bin/golang-confluent-kafka; echo "ExitCode=$?"'

integrations/golang-confluent-kafka/go.mod

@@ -0,0 +1,5 @@
+module github.com/apache/pulsar/kop/integration/golang-confluent-kafka
+
+go 1.13
+
+require gopkg.in/confluentinc/confluent-kafka-go.v1 v1.1.0

integrations/golang-confluent-kafka/go.sum

@@ -0,0 +1,2 @@
+gopkg.in/confluentinc/confluent-kafka-go.v1 v1.1.0 h1:roy97m/3wj9/o8OuU3sZ5wildk30ep38k2x8nhNbKrI=
+gopkg.in/confluentinc/confluent-kafka-go.v1 v1.1.0/go.mod h1:ZdI3yfYmdNSLQPNCpO1y00EHyWaHG5EnQEyL/ntAegY=

integrations/golang-confluent-kafka/main.go

@@ -0,0 +1,141 @@
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"gopkg.in/confluentinc/confluent-kafka-go.v1/kafka"
+)
+
+func main() {
+	limit, err := strconv.Atoi(getEnv("KOP_LIMIT", "10"))
+	if err != nil {
+		panic(err)
+	}
+
+	shouldProduce, err := strconv.ParseBool(getEnv("KOP_PRODUCE", "false"))
+	if err != nil {
+		panic(err)
+	}
+
+	shouldConsume, err := strconv.ParseBool(getEnv("KOP_CONSUME", "false"))
+	if err != nil {
+		panic(err)
+	}
+
+	brokers := []string{getEnv("KOP_BROKER", "localhost:9092")}
+	topic := getEnv("KOP_TOPIC", "my-confluent-go-topic")
+	topics := []string{topic}
+
+	if shouldProduce {
+
+		fmt.Println("starting to produce")
+
+		p, err := kafka.NewProducer(&kafka.ConfigMap{"bootstrap.servers": getEnv("KOP_BROKER", "localhost:9092")})
+		if err != nil {
+			panic(err)
+		}
+		defer p.Close()
+
+		// Delivery report handler for produced messages
+		go func() {
+			for e := range p.Events() {
+				switch ev := e.(type) {
+				case *kafka.Message:
+					if ev.TopicPartition.Error != nil {
+						fmt.Printf("Delivery failed: %v\n", ev.TopicPartition)
+					} else {
+						fmt.Printf("Delivered message to %v\n", ev.TopicPartition)
+					}
+				}
+			}
+		}()
+
+		for i := 0; i < limit; i++ {
+			p.Produce(&kafka.Message{
+				TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
+				Value:          []byte("hello from confluent go"),
+			}, nil)
+			fmt.Println("send a message")
+
+		}
+		fmt.Printf("produced all messages successfully (%d) \n", limit)
+		// Wait for message deliveries before shutting down
+		p.Flush(15 * 1000)
+
+	}
+
+	if shouldConsume {
+		fmt.Println("starting to consume")
+
+		c, err := kafka.NewConsumer(&kafka.ConfigMap{
+			"bootstrap.servers":       strings.Join(brokers, ","),
+			"group.id":                "myGroup",
+			"auto.offset.reset":       "earliest",
+			"broker.version.fallback": "2.0.0",
+			"debug":                   "all",
+		})
+		if err != nil {
+			panic(err)
+		}
+
+		c.SubscribeTopics(topics, nil)
+
+		counter := 0
+
+		for counter < limit {
+			msg, err := c.ReadMessage(-1)
+			if err == nil {
+				fmt.Println("received msg")
+				fmt.Printf("Message on %s: %s\n", msg.TopicPartition, string(msg.Value))
+				counter++
+			} else {
+				// The client will automatically try to recover from all errors.
+				fmt.Printf("Consumer error: %v (%v)\n", err, msg)
+				panic(err)
+			}
+		}
+		fmt.Println("consumed all messages successfully")
+	}
+
+	fmt.Println("exiting normally")
+}
+
+func getEnv(key, fallback string) string {
+	value, exists := os.LookupEnv(key)
+	if !exists {
+		value = fallback
+	}
+	return value
+}

integrations/golang-sarama/Dockerfile

@@ -0,0 +1,36 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+FROM golang:1.13.4-stretch
+
+WORKDIR /go/src/app
+COPY . .
+
+RUN go get -d -v ./...
+RUN go install -v ./...
+
+CMD sh -c '/go/bin/sarama-golang; echo "ExitCode=$?"'

integrations/golang-sarama/go.mod

@@ -0,0 +1,5 @@
+module github.com/apache/pulsar/kop/integration/sarama-golang
+
+go 1.13
+
+require github.com/Shopify/sarama v1.24.1