Remove redundant script from Travis build and improve log information

.gitignore

@@ -24,7 +24,7 @@ _testmain.go
 *.prof
 
 
-#OS X
+# OS X
 **/.DS_Store
 
 #binaries
@@ -43,6 +43,7 @@ Packages/
 .gradle
 .gogradle
 build/
+release/
 
 #emacs
 *~

.travis.yml

@@ -36,33 +36,64 @@ notifications:
       # travis2slack webhook to enable DMs on openwhisk-team.slack.com to PR authors with TravisCI results
       secure: "NaDCtHwbFUEy0h0u/QrAt395o0/9FstJ5/Tv4uBSLCBdaUNU+0q6/RXVH76dKbujoxrLded2gyeJSAbI4kl4SM7FyilSHzKTikF0VsgM3G5veh65tVY5ztsvl40hDmK2E/PmWI3HkZNeWJO32WjraqPmhSWL0BZOWhA/4gJSCIsLnNRQXmpZoDqC7V8IG1BWYGwd5qcNkEjItLjYCUy1s2K8bj3QmhshYRVLTNbXxh0yJVKjM04I+bWhMXqvL8GS55qeicJ3fUwm5g1PDVCtzxLwAHCPo2jusjtNSnV+BaZgMBSahv3MD7ApxfF58e1buejggH3qZAhNGccC4bYJCahqVv/KKoA10kO6exH5iGwlHPWQTjMuF++PHmAk+FeQ1jh+JoUSBnIHExPnaD4CQIArHvUuQom+WJBnOz+L1H755VyPGzeDQ+ZUDlhOiQ6CDP/sqaRH3Wmo4IzhsqTsLaQs0dS1YeEWOS8gcawwNTVf/WtqiedYu24rQN9RiUXelRoEaXJy/pldb2KpyQRo2hbKoStDDB5fEij1xyGsj4kSXZ8uk5G10oDGnZYd+okduzJazN8wr0flrLvH32DRgDZDcG/D90I19JmqqfqbcG/SEmhxTikHZeVQd5DfWcqssLvWNrtQa/EJgAQkmZ2gaSsZ73AvPk/KyMV30+m8xTU="
 
+# The complete sequence of phases of the lifecycle:
+# ---------------------------------------------------------
+# before_install
+# install
+# before_script
+# script
+# before_cache (OPTIONAL) (if and only if caching is effective)
+# after_success (or after_failure)
+# before_deploy (OPTIONAL) (if and only if deployment is active)
+# deploy (OPTIONAL)
+# after_deploy (OPTIONAL) (if and only if deployment is active)
+# after_script
+
 before_install:
   - "./tools/travis/cloneutils.sh"
+  - sudo apt-get install tree
+
+# Install tools we use for linting and unit/integration testing
+# TODO: build.gradle (gradlew) already installs 'golint' and runs linting
+#       in the `goLint` task; see if we can remove here.
+# TODO: The "gogradle" plugin in build.gradle (gradlew) already provides a
+#       task called "goTest" which runs all `unit` tests.  See if we can use that
+#       instead and only run `integration` tests here.
 install:
   - export DEPLOY_BUILD_READY=false
   - go get -u golang.org/x/lint/golint
   - go get -u github.com/stretchr/testify
+
+# Identify Golang files that are not properly formatted
+# TODO: See if we can create custom tasks to perform this in build.gradle
+#       as a custom `goFmt` task
 before_script:
   - GO_FILES=$(find . -iname '*.go' -type f -not -path "./wski18n/i18n_resources.go")
   - export BAD_GO=$(gofmt -s -l $(echo $GO_FILES))
   - echo "["$BAD_GO"]"
 #- test -z "$BAD_GO"
 #- test -z "$(gofmt -s -l $(echo $GO_FILES))"
 
+# Clone and build the OpenWhisk platform (i.e., apache/openwhisk)
+# and then use Ansible to deploy it and prepare it to run our
+# integration tests.
+# TODO: Do not use Makefile for running integration tests (in script.sh);
+#       change to using gradle wrapper
 script:
-  - echo $TRAVIS
-  - echo $TRAVIS_PULL_REQUEST
-  - echo $TRAVIS_SECURE_ENV_VARS
-  - printenv
+  - echo "Installing and building OpenWhisk platform using Ansible..."
+  - "./tools/travis/showenv.sh"
   - "./tools/travis/script.sh"
+
 after_success:
   - DEPLOY_BUILD_READY=true
   - if [ "$TRAVIS_EVENT_TYPE" == "cron" ] ; then
     export DEPLOY_BUILD_READY=false;
     fi
 
-after_script:
-  - make clean
+# If this is a "tagged" GitHub build, then use Gradle Wrapper
+# to build binaries for all supported OS platforms and architectures.
+# The result would be a "/build" directory that has a subdirectory named
+# for each supported GOOS/GOARCH
 before_deploy:
   - export build_file_name=wskdeploy
   - go get github.com/inconshreveable/mousetrap
@@ -72,10 +103,14 @@ before_deploy:
     export GIT_TAG=$TRAVIS_TAG;
     export TAG=true;
     fi
+  - echo "Calling Gradle Wrapper task to build and 'releaseBinaries'..."
+  - "./tools/travis/showenv.sh"
   - ./gradlew --console=plain releaseBinaries -PpackageVersion=$GIT_TAG -PgitCommit=$(git rev-parse HEAD) -PbuildDate=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
-  - "./tools/travis/build_tag_releases.sh $build_file_name $GIT_TAG"
+  - echo "Build binaries and release archives created:"
+  - tree /build
+  - tree /release
   - export RELEASE_PKG_FILE="$(cd "$TRAVIS_BUILD_DIR/release" && ls ${zip_file_name}-*.tgz ${zip_file_name}-*.zip)"
-  - echo "Deploying $RELEASE_PKG_FILE to GitHub releases."
+  - echo "Deploying "$RELEASE_PKG_FILE" to GitHub releases..."
   - if [ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_EVENT_TYPE" == "push" ] && [ "$TRAVIS_OS_NAME" == "linux" ] ; then
     git config --global user.email "builds@travis-ci.com";
     git config --global user.name "Travis CI";
@@ -84,11 +119,11 @@ before_deploy:
     GIT_COMMITTER_DATE="$(git show --format=%aD | head -1)" git tag $GIT_TAG -a -m "Generated tag from Travis CI build $TRAVIS_BUILD_NUMBER";
     git push -f -q https://$API_KEY@github.com/apache/openwhisk-wskdeploy $GIT_TAG;
     fi
-  - echo "TRAVIS_EVENT_TYPE is "
-  - echo $TRAVIS_EVENT_TYPE
-  - echo "GIT_TAG is "
-  - echo $GIT_TAG
+  - echo "Completing before_deploy stage"
 
+# Utilize two providers to create a github release and publish a docker image.
+# For information on the "releases" provider,
+# see https://docs.travis-ci.com/user/deployment/releases/
 deploy:
   - provider: releases
     api_key:
@@ -112,6 +147,18 @@ deploy:
       repo: apache/openwhisk-wskdeploy
       tags: true
       condition: "$DEPLOY_BUILD_READY = true"
+
+# Clean up wskdeploy binary (single)
+# Fundamentally, remove both /build and /release directory trees and contents
+# TODO: See if this is really needed in Travis environments as the
+# Makefile is not used elsewhere here (especially if we do not use th
+# build_tag_release.sh script) as it is for Windows developers primarily
+# and we are building using Linux.
+after_script:
+  - make clean
+  - tree /build
+  - tree /release
+
 env:
   global:
     - BLUEMIX_APIHOST=openwhisk.ng.bluemix.net

build.gradle

@@ -73,7 +73,7 @@ rat {
     storage is in "GO" format:
 
         OS: linux, darwin, windows
-        Arch: 386, amd64, s390x, ppc64le, arm
+        Arch: 386, amd64, s390x, ppc64le, arm, arm64
 
     TODO - It may be appropriate to refactor into a general purpose Platform
            class for all builds, then to extend with specifics needed for

cmd/export.go

@@ -29,8 +29,10 @@ import (
 	"github.com/apache/openwhisk-wskdeploy/parsers"
 	"github.com/apache/openwhisk-wskdeploy/runtimes"
 	"github.com/apache/openwhisk-wskdeploy/utils"
+	"github.com/apache/openwhisk-wskdeploy/wskderrors"
 	"github.com/apache/openwhisk-wskdeploy/wski18n"
 	"github.com/apache/openwhisk-wskdeploy/wskprint"
+	"github.com/davecgh/go-spew/spew"
 	"github.com/spf13/cobra"
 )
 
@@ -117,9 +119,6 @@ func ExportAction(actionName string, packageName string, maniyaml *parsers.YAML,
 
 func exportProject(projectName string, targetManifest string) error {
 
-	whisk.SetVerbose(utils.Flags.Verbose)
-	whisk.SetDebug(utils.Flags.Trace)
-
 	maniyaml := &parsers.YAML{}
 	maniyaml.Project.Name = projectName
 
@@ -129,13 +128,19 @@ func exportProject(projectName string, targetManifest string) error {
 		return err
 	}
 
+	// Emit additional trace data (primarily in Travis)
+	if utils.Flags.Trace {
+		spew.Dump(packages)
+	}
+
 	var bindings = make(map[string]whisk.Binding)
 
 	// iterate over each package to find managed annotations
 	// check if "managed" annotation is attached to a package
 	// add to export when managed project name matches with the
 	// specified project name
 	for _, pkg := range packages {
+
 		if a := pkg.Annotations.GetValue(utils.MANAGED); a != nil {
 			// decode the JSON blob and retrieve __OW_PROJECT_NAME
 			pa := a.(map[string]interface{})
@@ -199,59 +204,66 @@ func exportProject(projectName string, targetManifest string) error {
 
 	// iterate over the list of triggers to determine whether any of them part of specified managed project
 	for _, trg := range triggers {
-		// trigger has attached managed annotation
-		if a := trg.Annotations.GetValue(utils.MANAGED); a != nil {
-			// decode the JSON blob and retrieve __OW_PROJECT_NAME
-			ta := a.(map[string]interface{})
-			if ta[utils.OW_PROJECT_NAME] == projectName {
-
-				for pkgName := range maniyaml.Packages {
-					if maniyaml.Packages[pkgName].Namespace == trg.Namespace {
-						if maniyaml.Packages[pkgName].Triggers == nil {
-							pkg := maniyaml.Packages[pkgName]
-							pkg.Triggers = make(map[string]parsers.Trigger)
-							maniyaml.Packages[pkgName] = pkg
-						}
 
-						// export trigger to manifest
-
-						if feedname, isFeed := utils.IsFeedAction(&trg); isFeed {
-							// check if feed name starts with namespace and workaround it
-							// the current problem is that client has user namespace and when feed specified with different namespace it will fail to invoke the feed action
-							// we need to transform the path from e.g.
-							// /api/v1/namespaces/kpavel@il.ibm.com_uspace/actions//whisk.system/alarms/interval?blocking=true
-							// in to
-							// /api/v1/namespaces/kpavel@il.ibm.com_uspace/actions/../../whisk.system/actions/alarms/interval?blocking=true
-							if strings.HasPrefix(feedname, "/") {
-								//  /whisk.system/alarms/interval  ->  ../../whisk.system/actions/alarms/interval
-								prts := strings.SplitN(feedname, "/", 3)
-								feedname = "../../" + prts[1] + "/actions/" + prts[2]
+		// trigger has attached managed annotation
+		if trg.Annotations != nil {
+			if a := trg.Annotations.GetValue(utils.MANAGED); a != nil {
+				// decode the JSON blob and retrieve __OW_PROJECT_NAME
+				ta := a.(map[string]interface{})
+				if ta[utils.OW_PROJECT_NAME] == projectName {
+
+					for pkgName := range maniyaml.Packages {
+						if maniyaml.Packages[pkgName].Namespace == trg.Namespace {
+							if maniyaml.Packages[pkgName].Triggers == nil {
+								pkg := maniyaml.Packages[pkgName]
+								pkg.Triggers = make(map[string]parsers.Trigger)
+								maniyaml.Packages[pkgName] = pkg
 							}
 
-							// export feed input parameters
-							params := make(map[string]interface{})
-							params["authKey"] = client.Config.AuthToken
-							params["lifecycleEvent"] = "READ"
-							params["triggerName"] = "/" + client.Namespace + "/" + trg.Name
-							res, _, err := client.Actions.Invoke(feedname, params, true, true)
-							if err != nil {
-								return err
-							}
-							feedConfig := res["config"]
+							// export trigger to manifest
+
+							if feedname, isFeed := utils.IsFeedAction(&trg); isFeed {
+								// check if feed name starts with namespace and workaround it
+								// the current problem is that client has user namespace and when feed specified with different namespace it will fail to invoke the feed action
+								// we need to transform the path from e.g.
+								// /api/v1/namespaces/kpavel@il.ibm.com_uspace/actions//whisk.system/alarms/interval?blocking=true
+								// in to
+								// /api/v1/namespaces/kpavel@il.ibm.com_uspace/actions/../../whisk.system/actions/alarms/interval?blocking=true
+								if strings.HasPrefix(feedname, "/") {
+									//  /whisk.system/alarms/interval  ->  ../../whisk.system/actions/alarms/interval
+									prts := strings.SplitN(feedname, "/", 3)
+									feedname = "../../" + prts[1] + "/actions/" + prts[2]
+								}
+
+								// export feed input parameters
+								params := make(map[string]interface{})
+								params["authKey"] = client.Config.AuthToken
+								params["lifecycleEvent"] = "READ"
+								params["triggerName"] = "/" + client.Namespace + "/" + trg.Name
+								res, _, err := client.Actions.Invoke(feedname, params, true, true)
+								if err != nil {
+									return err
+								}
+								feedConfig := res["config"]
 
-							if feedConfig != nil {
-								for key, val := range feedConfig.(map[string]interface{}) {
-									if key != "startDate" {
-										trg.Parameters = trg.Parameters.AddOrReplace(&whisk.KeyValue{Key: key, Value: val})
+								if feedConfig != nil {
+									for key, val := range feedConfig.(map[string]interface{}) {
+										if key != "startDate" {
+											trg.Parameters = trg.Parameters.AddOrReplace(&whisk.KeyValue{Key: key, Value: val})
+										}
 									}
 								}
 							}
-						}
 
-						maniyaml.Packages[pkgName].Triggers[trg.Name] = *maniyaml.ComposeParsersTrigger(trg)
+							maniyaml.Packages[pkgName].Triggers[trg.Name] = *maniyaml.ComposeParsersTrigger(trg)
+						}
 					}
 				}
 			}
+		} else {
+			// Emit additional trace data on common Travis failures
+			spew.Dump(trg)
+			return wskderrors.NewCommandError("Export", "Trigger missing annotations.")
 		}
 	}
 
@@ -263,29 +275,36 @@ func exportProject(projectName string, targetManifest string) error {
 
 	// iterate over the list of rules to determine whether any of them is part of the manage dproject
 	for _, rule := range rules {
+
 		// get rule from OW
 		wskRule, _, _ := client.Rules.Get(rule.Name)
-		// rule has attached managed annotation
-		if a := wskRule.Annotations.GetValue(utils.MANAGED); a != nil {
-			// decode the JSON blob and retrieve __OW_PROJECT_NAME
-			ta := a.(map[string]interface{})
-			if ta[utils.OW_PROJECT_NAME] == projectName {
-
-				for pkgName := range maniyaml.Packages {
-					if maniyaml.Packages[pkgName].Namespace == wskRule.Namespace {
-						if maniyaml.Packages[pkgName].Rules == nil {
-							pkg := maniyaml.Packages[pkgName]
-							pkg.Rules = make(map[string]parsers.Rule)
-							maniyaml.Packages[pkgName] = pkg
-						}
 
-						// export rule to manifest
-						maniyaml.Packages[pkgName].Rules[wskRule.Name] = *maniyaml.ComposeParsersRule(*wskRule)
+		if wskRule.Annotations != nil {
+			// rule has attached managed annotation
+			if a := wskRule.Annotations.GetValue(utils.MANAGED); a != nil {
+				// decode the JSON blob and retrieve __OW_PROJECT_NAME
+				ta := a.(map[string]interface{})
+				if ta[utils.OW_PROJECT_NAME] == projectName {
+
+					for pkgName := range maniyaml.Packages {
+						if maniyaml.Packages[pkgName].Namespace == wskRule.Namespace {
+							if maniyaml.Packages[pkgName].Rules == nil {
+								pkg := maniyaml.Packages[pkgName]
+								pkg.Rules = make(map[string]parsers.Rule)
+								maniyaml.Packages[pkgName] = pkg
+							}
+
+							// export rule to manifest
+							maniyaml.Packages[pkgName].Rules[wskRule.Name] = *maniyaml.ComposeParsersRule(*wskRule)
+						}
 					}
 				}
 			}
+		} else {
+			// Emit additional trace data on common Travis failures
+			spew.Dump(wskRule)
+			return wskderrors.NewCommandError("Export", "Rule missing annotations.")
 		}
-
 	}
 
 	// API Gateway is an optional component. Export APIs only when ApigwAccessToken is configured
@@ -386,7 +405,14 @@ func exportProject(projectName string, targetManifest string) error {
 
 	// find exported manifest parent directory
 	manifestDir := filepath.Dir(utils.Flags.ManifestPath)
-	os.MkdirAll(manifestDir, os.ModePerm)
+	errMkDir := os.MkdirAll(manifestDir, os.ModePerm)
+
+	// Exit if unable to create export dir. structure
+	// TODO: This sometimes fails in Travis, perhaps retry?
+	if errMkDir != nil {
+		wskprint.PrintOpenWhiskError(errMkDir.Error())
+		return errMkDir
+	}
 
 	// export manifest to file
 	parsers.Write(maniyaml, targetManifest)

go.mod

@@ -5,16 +5,15 @@ go 1.15
 require (
 	github.com/apache/openwhisk-client-go v0.0.0-20210311185314-87edc2364717
 	github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/fatih/color v1.10.0
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
-	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519
 	github.com/magiconair/properties v1.8.4 // indirect
 	github.com/mattn/go-colorable v0.1.8
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
 	github.com/nicksnyder/go-i18n v1.10.1
 	github.com/nxadm/tail v1.4.8 // indirect
-	github.com/onsi/ginkgo v1.15.0 // indirect
 	github.com/pelletier/go-toml v1.8.1 // indirect
 	github.com/spf13/afero v1.5.1 // indirect
 	github.com/spf13/cast v1.3.1 // indirect
@@ -26,5 +25,4 @@ require (
 	golang.org/x/text v0.3.5 // indirect
 	gopkg.in/ini.v1 v1.62.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )