From 6b95290d04f51b19e9afe51b0999af505fcb039f Mon Sep 17 00:00:00 2001 From: Vaibhav Date: Sun, 3 Mar 2019 16:04:14 -0500 Subject: [PATCH] Trello gateway (#193) * feature(): adding trello gateway * feat(): update trello gateway * feat(): update trello gateway --- DEPENDENCIES.md | 3 + Gopkg.lock | 13 +- Gopkg.toml | 4 + Makefile | 10 ++ .../gateways/trello-gateway-configmap.yaml | 16 +++ examples/gateways/trello.yaml | 35 +++++ examples/sensors/trello.yaml | 52 +++++++ gateways/community/trello/Dockerfile | 3 + gateways/community/trello/cmd/main.go | 44 ++++++ gateways/community/trello/config.go | 57 ++++++++ gateways/community/trello/start.go | 135 ++++++++++++++++++ gateways/community/trello/validate.go | 47 ++++++ gateways/community/trello/validate_test.go | 72 ++++++++++ 13 files changed, 489 insertions(+), 2 deletions(-) create mode 100644 examples/gateways/trello-gateway-configmap.yaml create mode 100644 examples/gateways/trello.yaml create mode 100644 examples/sensors/trello.yaml create mode 100644 gateways/community/trello/Dockerfile create mode 100644 gateways/community/trello/cmd/main.go create mode 100644 gateways/community/trello/config.go create mode 100644 gateways/community/trello/start.go create mode 100644 gateways/community/trello/validate.go create mode 100644 gateways/community/trello/validate_test.go diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index d7e4e1f1d8..300a1b97a1 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -23,3 +23,6 @@ | cloud.google.com/go | Apache-2.0 | | colinmarc/hdfs | MIT | | jcmturner/gokrb5 | Apache-2.0 | +| adlio/trello | MIT | +| aws/aws-sdk-go | Apache-2.0 | + diff --git a/Gopkg.lock b/Gopkg.lock index fe2582cf8c..aee4eba493 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -57,6 +57,14 @@ revision = "4602b5a8c6e826f9e0737865818dd43b2339a092" version = "v1.21.0" +[[projects]] + branch = "master" + digest = "1:797da30d559e1fbc24acfe026226cfb49f129216fb65db1fa6b786f2ace0cc2c" + name = "github.com/adlio/trello" + packages = ["."] + pruneopts = "UT" + revision = "b0faddb69566d3b80c1b70aa6daf2677b3f7a551" + [[projects]] digest = "1:4b9792d4bd04b67ce23f3d8a896df2f3ae49e1574ea2722501959f2b86f35002" name = "github.com/argoproj/argo" @@ -880,7 +888,7 @@ [[projects]] branch = "master" - digest = "1:3e05d78c0f71922115e3f0bc4740e8e46e0919c27551be4b51d05a3ba998a948" + digest = "1:8c41805cf4d0776e4e4aaedd60cb7b184113a265054433bc39644f289f7c86cc" name = "golang.org/x/sys" packages = [ "cpu", @@ -888,7 +896,7 @@ "windows", ] pruneopts = "UT" - revision = "d455e41777fca6e8a5a79e34a14b8368bc11d9ba" + revision = "c2f5717e611cf8e89a852a41081355194d80c943" [[projects]] digest = "1:0c56024909189aee3364b7f21a95a27459f718aa7c199a5c111c36cfffd9eaef" @@ -1456,6 +1464,7 @@ "cloud.google.com/go/pubsub", "github.com/Knetic/govaluate", "github.com/Shopify/sarama", + "github.com/adlio/trello", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1", "github.com/aws/aws-sdk-go/aws", "github.com/aws/aws-sdk-go/aws/credentials", diff --git a/Gopkg.toml b/Gopkg.toml index 861529c3d2..a67f12bf15 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -13,6 +13,10 @@ required = [ "gopkg.in/src-d/go-git.v4" ] +[[constraint]] + name = "github.com/adlio/trello" + branch = "master" + [[constraint]] name = "k8s.io/code-generator" branch = "release-1.10" diff --git a/Makefile b/Makefile index 302f12f26e..42b6f0c04f 100644 --- a/Makefile +++ b/Makefile @@ -255,6 +255,16 @@ sqs-image: sqs-linux docker build -t $(IMAGE_PREFIX)aws-sqs-gateway:$(IMAGE_TAG) -f ./gateways/community/aws-sqs/Dockerfile . @if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)aws-sqs-gateway:$(IMAGE_TAG) ; fi +trello: + go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/trello-gateway ./gateways/community/trello/cmd + +trello-linux: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 make trello + +trello-image: trello-linux + docker build -t $(IMAGE_PREFIX)trello-gateway:$(IMAGE_TAG) -f ./gateways/community/trello/Dockerfile . + @if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)trello-gateway:$(IMAGE_TAG) ; fi + test: go test $(shell go list ./... | grep -v /vendor/) -race -short -v diff --git a/examples/gateways/trello-gateway-configmap.yaml b/examples/gateways/trello-gateway-configmap.yaml new file mode 100644 index 0000000000..eb5c265380 --- /dev/null +++ b/examples/gateways/trello-gateway-configmap.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: trello-gateway-configmap +data: + notification_1: |- + endpoint: "/" + port: "9600" + apiKey: + name: trello-secret + key: apikey + token: + name trello-secret + key: tokenkey + url: "URL_TO_REGISTER" + description: "test webhook" diff --git a/examples/gateways/trello.yaml b/examples/gateways/trello.yaml new file mode 100644 index 0000000000..58d4af87d1 --- /dev/null +++ b/examples/gateways/trello.yaml @@ -0,0 +1,35 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Gateway +metadata: + name: trello-gateway + labels: + gateways.argoproj.io/gateway-controller-instanceid: argo-events + gateway-name: "trello-gateway" +spec: + processorPort: "9330" + eventProtocol: + type: "HTTP" + http: + port: "9300" + deploySpec: + metadata: + name: "trello-gateway" + labels: + gateway-name: "trello-gateway" + spec: + containers: + - name: "gateway-client" + image: "argoproj/gateway-client" + imagePullPolicy: "Always" + command: ["/bin/gateway-client"] + - name: "trello-events" + image: "argoproj/trello-gateway" + imagePullPolicy: "Always" + command: ["/bin/trello-gateway"] + serviceAccountName: "argo-events-sa" + configMap: "trello-gateway-configmap" + type: "trello" + eventVersion: "1.0" + watchers: + sensors: + - name: "trello-sensor" diff --git a/examples/sensors/trello.yaml b/examples/sensors/trello.yaml new file mode 100644 index 0000000000..1efd3dd7c2 --- /dev/null +++ b/examples/sensors/trello.yaml @@ -0,0 +1,52 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Sensor +metadata: + name: trello-sensor + labels: + sensors.argoproj.io/sensor-controller-instanceid: argo-events +spec: + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "HTTP" + http: + port: "9300" + dependencies: + - name: "trello-gateway:notification_1" + triggers: + - name: trello-workflow + resource: + namespace: argo-events + group: argoproj.io + version: v1alpha1 + kind: Workflow + parameters: + - src: + event: "trello-gateway:notification_1" + dest: spec.arguments.parameters.0.value + source: + inline: | + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: trello- + spec: + entrypoint: whalesay + arguments: + parameters: + - name: message + # this is the value that should be overridden + value: hello world + templates: + - name: whalesay + inputs: + parameters: + - name: message + container: + image: docker/whalesay:latest + command: [cowsay] + args: ["{{inputs.parameters.message}}"] diff --git a/gateways/community/trello/Dockerfile b/gateways/community/trello/Dockerfile new file mode 100644 index 0000000000..2df46e146a --- /dev/null +++ b/gateways/community/trello/Dockerfile @@ -0,0 +1,3 @@ +FROM scratch +COPY dist/trello-gateway /bin/ +ENTRYPOINT [ "/bin/trello-gateway" ] diff --git a/gateways/community/trello/cmd/main.go b/gateways/community/trello/cmd/main.go new file mode 100644 index 0000000000..913dac80b6 --- /dev/null +++ b/gateways/community/trello/cmd/main.go @@ -0,0 +1,44 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + + "github.com/argoproj/argo-events/common" + "github.com/argoproj/argo-events/gateways" + "github.com/argoproj/argo-events/gateways/community/trello" + "k8s.io/client-go/kubernetes" +) + +func main() { + kubeConfig, _ := os.LookupEnv(common.EnvVarKubeConfig) + restConfig, err := common.GetClientConfig(kubeConfig) + if err != nil { + panic(err) + } + clientset := kubernetes.NewForConfigOrDie(restConfig) + namespace, ok := os.LookupEnv(common.EnvVarGatewayNamespace) + if !ok { + panic("namespace is not provided") + } + gateways.StartGateway(&trello.TrelloEventSourceExecutor{ + Log: common.GetLoggerContext(common.LoggerConf()).Logger(), + Clientset: clientset, + Namespace: namespace, + }) +} diff --git a/gateways/community/trello/config.go b/gateways/community/trello/config.go new file mode 100644 index 0000000000..5a44aeda02 --- /dev/null +++ b/gateways/community/trello/config.go @@ -0,0 +1,57 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package trello + +import ( + "github.com/ghodss/yaml" + "github.com/rs/zerolog" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" +) + +type TrelloEventSourceExecutor struct { + Log zerolog.Logger + // Clientset is kubernetes client + Clientset kubernetes.Interface + // Namespace where gateway is deployed + Namespace string +} + +type trello struct { + // ApiKey for client + ApiKey *corev1.SecretKeySelector `json:"apiKey"` + // Token for client + Token *corev1.SecretKeySelector `json:"token"` + // Endpoint is REST API endpoint + Endpoint string `json:"endpoint"` + // Port to run the http server on + Port string `json:"port"` + // URL to register at trello + URL string `json:"url"` + // Description for webhook + // +optional + Description string `json:"description,omitempty"` +} + +func parseEventSource(es string) (interface{}, error) { + var n *trello + err := yaml.Unmarshal([]byte(es), &n) + if err != nil { + return nil, err + } + return n, nil +} diff --git a/gateways/community/trello/start.go b/gateways/community/trello/start.go new file mode 100644 index 0000000000..4cb205f2ed --- /dev/null +++ b/gateways/community/trello/start.go @@ -0,0 +1,135 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package trello + +import ( + "fmt" + "io/ioutil" + "net/http" + + trellolib "github.com/adlio/trello" + "github.com/argoproj/argo-events/common" + "github.com/argoproj/argo-events/gateways" + gwcommon "github.com/argoproj/argo-events/gateways/common" + "github.com/argoproj/argo-events/store" +) + +const ( + LabelTrelloConfig = "trelloConfig" + LabelTrelloClient = "trelloClient" +) + +var ( + helper = gwcommon.NewWebhookHelper() +) + +func init() { + go gwcommon.InitRouteChannels(helper) +} + +// routeActiveHandler handles new route +func RouteActiveHandler(writer http.ResponseWriter, request *http.Request, rc *gwcommon.RouteConfig) { + var response string + + logger := rc.Log.With().Str("event-source", rc.EventSource.Name).Str("endpoint", rc.Webhook.Endpoint). + Str("port", rc.Webhook.Port). + Str("http-method", request.Method).Logger() + logger.Info().Msg("request received") + + if !helper.ActiveEndpoints[rc.Webhook.Endpoint].Active { + response = fmt.Sprintf("the route: endpoint %s and method %s is deactived", rc.Webhook.Endpoint, rc.Webhook.Method) + logger.Info().Msg("endpoint is not active") + common.SendErrorResponse(writer, response) + return + } + + body, err := ioutil.ReadAll(request.Body) + if err != nil { + logger.Error().Err(err).Msg("failed to parse request body") + return + } + + helper.ActiveEndpoints[rc.Webhook.Endpoint].DataCh <- body + + response = "request successfully processed" + logger.Info().Msg(response) + common.SendSuccessResponse(writer, response) +} + +func (ese *TrelloEventSourceExecutor) PostActivate(rc *gwcommon.RouteConfig) error { + logger := rc.Log.With().Str("event-source", rc.EventSource.Name).Str("endpoint", rc.Webhook.Endpoint). + Str("port", rc.Webhook.Port).Logger() + + tl := rc.Configs[LabelTrelloConfig].(*trello) + + logger.Info().Msg("retrieving apikey and token") + apiKey, err := store.GetSecrets(ese.Clientset, ese.Namespace, tl.ApiKey.Name, tl.ApiKey.Key) + if err != nil { + logger.Error().Err(err).Msg("failed to retrieve api key") + return err + } + token, err := store.GetSecrets(ese.Clientset, ese.Namespace, tl.Token.Name, tl.Token.Key) + if err != nil { + logger.Error().Err(err).Msg("failed to retrieve token") + return err + } + + client := trellolib.NewClient(apiKey, token) + + if err = client.CreateWebhook(&trellolib.Webhook{ + Active: true, + Description: tl.Description, + CallbackURL: tl.URL, + }); err != nil { + logger.Error().Err(err).Msg("failed to create webhook") + return err + } + + rc.Configs[LabelTrelloClient] = client + + logger.Info().Msg("webhook created") + return nil +} + +// StartConfig runs a configuration +func (ese *TrelloEventSourceExecutor) StartEventSource(eventSource *gateways.EventSource, eventStream gateways.Eventing_StartEventSourceServer) error { + defer gateways.Recover(eventSource.Name) + + ese.Log.Info().Str("event-source-name", eventSource.Name).Msg("operating on event source") + config, err := parseEventSource(eventSource.Data) + if err != nil { + ese.Log.Error().Err(err).Str("event-source-name", eventSource.Name).Msg("failed to parse event source") + return err + } + tl := config.(*trello) + + return gwcommon.ProcessRoute(&gwcommon.RouteConfig{ + Webhook: &gwcommon.Webhook{ + Endpoint: tl.Endpoint, + Port: tl.Port, + }, + Configs: map[string]interface{}{ + LabelTrelloConfig: tl, + }, + Log: ese.Log, + EventSource: eventSource, + PostActivate: ese.PostActivate, + PostStop: gwcommon.DefaultPostStop, + RouteActiveHandler: RouteActiveHandler, + StartCh: make(chan struct{}), + }, helper, eventStream) +} diff --git a/gateways/community/trello/validate.go b/gateways/community/trello/validate.go new file mode 100644 index 0000000000..60c94535cd --- /dev/null +++ b/gateways/community/trello/validate.go @@ -0,0 +1,47 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package trello + +import ( + "context" + "fmt" + "github.com/argoproj/argo-events/gateways" + gwcommon "github.com/argoproj/argo-events/gateways/common" +) + +// ValidateEventSource validates a s3 event source +func (ese *TrelloEventSourceExecutor) ValidateEventSource(ctx context.Context, eventSource *gateways.EventSource) (*gateways.ValidEventSource, error) { + return gwcommon.ValidateGatewayEventSource(eventSource.Data, parseEventSource, validateTrello) +} + +// validates trello config +func validateTrello(config interface{}) error { + tl := config.(*trello) + if tl == nil { + return gwcommon.ErrNilEventSource + } + if tl.URL == "" { + return fmt.Errorf("url is not specified") + } + if tl.Token == nil { + return fmt.Errorf("token can't be empty") + } + if tl.ApiKey == nil { + return fmt.Errorf("api key can't be empty") + } + return gwcommon.ValidateWebhook(tl.Endpoint, tl.Port) +} diff --git a/gateways/community/trello/validate_test.go b/gateways/community/trello/validate_test.go new file mode 100644 index 0000000000..daaaf0f2ce --- /dev/null +++ b/gateways/community/trello/validate_test.go @@ -0,0 +1,72 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package trello + +import ( + "context" + "testing" + + "github.com/argoproj/argo-events/gateways" + "github.com/smartystreets/goconvey/convey" +) + +var ( + configKey = "testConfig" + configId = "1234" + validConfig = ` +endpoint: "/" +port: "8080" +apiKey: + key: api + name: trello +token: + key: token + name: trello +url: "URL to register on trello" +description: "test hook" +` + + invalidConfig = ` +endpoint: "/" +port: "8080" +` +) + +func TestTrelloEventSourceExecutor_ValidateEventSource(t *testing.T) { + convey.Convey("Given a valid trello event source spec, parse it and make sure no error occurs", t, func() { + ese := &TrelloEventSourceExecutor{} + valid, _ := ese.ValidateEventSource(context.Background(), &gateways.EventSource{ + Name: configKey, + Id: configId, + Data: validConfig, + }) + convey.So(valid, convey.ShouldNotBeNil) + convey.So(valid.IsValid, convey.ShouldBeTrue) + }) + + convey.Convey("Given an invalid trello event source spec, parse it and make sure error occurs", t, func() { + ese := &TrelloEventSourceExecutor{} + valid, _ := ese.ValidateEventSource(context.Background(), &gateways.EventSource{ + Data: invalidConfig, + Id: configId, + Name: configKey, + }) + convey.So(valid, convey.ShouldNotBeNil) + convey.So(valid.IsValid, convey.ShouldBeFalse) + convey.So(valid.Reason, convey.ShouldNotBeEmpty) + }) +}