gcbmgr

#!/bin/bash
#
# Copyright 2017 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Set PROGram name
PROG=${0##*/}
########################################################################
#+
#+ NAME
#+     $PROG - Manage Kubernetes Container Release Builds
#+
#+ SYNOPSIS
#+   Reporting:
#+     $PROG  list [STATUS]
#+     $PROG  staged
#+     $PROG  tail|stream [BUILD_ID]
#+   Staging:
#+     $PROG  stage <branch> [--buildversion=<known version>]
#+             [--build-at-head] [--nomock] [--official] [--rc]
#+   Releasing:
#+     $PROG  release <branch> --buildversion=<staged build>
#+             [--nomock] [--official] [--rc]
#+     $PROG  [--helpshort|--usage|-?]
#+     $PROG  [--help|-man]
#+
#+ DESCRIPTION
#+     Manage Kubernetes/Kubernetes release builds created on Container Builder
#+     on the Google Cloud Platform.
#+
#+     Releases can only occur from staged builds.  The --buildversion arg
#+     works for both staging and releasing.  The format of a build version is
#+     vX.Y.Z{-alpha.n|-beta.n}.NN+commit-hash
#+     (Ex. v1.8.8-beta.0.72+cce11c6a185279)
#+
#+     'list' shows a columnized output of job associated tags, the job id and
#+     the start time of the job.  The tags provide lots of useful info:
#+
#+     * Who - $USER or GCP USER
#+     * Source commit - HEAD, (result of) find_green_build or --buildversion
#+     * Branch
#+     * Type - STAGE or RELEASE
#+     * Flags - --official --nomock and --rc
#+
#+     Typical usage (in mock mode):
#+
#+     # List running jobs
#+     $ $PROG
#+
#+     # List up to last 5 succeeding jobs
#+     $ $PROG list success
#+
#+     # "tail"/stream most recent running build
#+     $ $PROG tail
#+
#+     # Check Staged builds
#+     $ $PROG staged
#+
#+     # Stage a new master branch release (on passing blocking tests)
#+     $ $PROG stage master
#+
#+     # Stage a new master branch release (at head)
#+     $ $PROG stage master --build-at-head
#+
#+     # Stage a new release-1.9 branch release (on passing blocking tests)
#+     $ $PROG stage release-1.9 --official
#+
#+     # Stage a new release-1.9 release candidate (on passing blocking tests)
#+     $ $PROG stage release-1.9 --rc
#+
#+     # Stage a new release-1.9 release candidate (at head)
#+     $ $PROG stage release-1.9 --rc --build-at-head
#+
#+     # Stage a new release-1.9 branch release (at head)
#+     $ $PROG stage release-1.9 --official --build-at-head
#+
#+     # Release a previously staged build
#+     $ $PROG release release-1.8 --buildversion=v1.8.8-beta.0.72+cce11c6a185279
#+
#+ OPTIONS
#+     tail|stream               - tail/stream the latest running job to your
#+                                 console window.  You can also specify a
#+                                 BUILD_ID to stream.
#+     list                      - List last (up to) 5 jobs.
#+                                 Default is to list jobs of any status
#+                                 Or specified by one of WORKING, FAILURE
#+                                 SUCCESS or CANCELLED (case not important)
#+     stage|release             - Top level arg for staging or releasing.
#+                                 See examples.
#+     staged                    - Show all staged builds.  With --nomock
#+                                 non-mock staged builds are shown.  Without
#+                                 --nomock, both stage/*-$GCP_USER
#+                                 & stage/*-gcb builds are shown and
#+                                 available for release.
#+     --build-at-head           - Build directly from head. Do not analyze
#+                                 test data to pick a candidate.
#+     --nomock                  - Stage a non-mocked release build to prepare
#+                                 for an actual release.
#+     --official                - For release-* branches, stage an official
#+                                 X.Y.Z release.  Without --official, only the
#+                                 -beta on the branch is incremented and built.
#+     --rc                      - For release-* branches, stage a release
#+                                 candidate by building and incrementing -rc on
#+                                 the branch.
#+     --buildversion            - Release from a previously staged build or
#+                                 stage a known version.
#+     [--help | -man]           - display man page for this script
#+     [--usage | -?]            - display in-line usage
#+
#+ EXAMPLES
#+     $PROG                     - Default 'list' behavior
#+     $PROG stream              - Show the most recent active log
#+     $PROG stage master --build-at-head
#+                               - Stage a build at head on the master branch
#+     $PROG stage release-1.8 --official
#+                               - Analyze test data to find a candidate and
#+                                 stage an official build for release in mock
#+                                 mode.
#+     $PROG stage release-1.8 --official --nomock
#+                               - Same as above but stage in non-mock bucket.
#+
#+ FILES
#+     anago                    - Main release tool
#+     find_green_build         - Test analysis tool
#+
#+ SEE ALSO
#+     lib/common.sh            - base function definitions
#+     lib/github.sh            - github specific functions
#+     lib/releaselib.sh        - release specific functions
#+
#+ BUGS/TODO
#+
########################################################################
# Deal with OSX limitations out the gate for anyone that tries this there
BASE_ROOT="$(dirname "$(readlink -e "${BASH_SOURCE[0]}" 2>&1)")" \
 || BASE_ROOT="${BASH_SOURCE[0]}"
# shellcheck source=./lib/common.sh
source "$BASE_ROOT/lib/common.sh"
# shellcheck source=./lib/gitlib.sh
source "$TOOL_LIB_PATH/gitlib.sh"
# shellcheck source=./lib/releaselib.sh
source "$TOOL_LIB_PATH/releaselib.sh"

readonly RELEASE_TOOL_REPO="${RELEASE_TOOL_REPO:-https://github.com/kubernetes/release}"
readonly RELEASE_TOOL_BRANCH="${RELEASE_TOOL_BRANCH:-master}"

###############################################################################
# FUNCTIONS
###############################################################################
###############################################################################
# common::cleanexit prog-specific override function
# @param exit code
#
common::cleanexit () {
  [[ -t 1 ]] && tput cnorm

  common::timestamp end
  exit "${1:-0}"
}


list_staged_builds () {
  local bucket
  local i
  local n
  local staging_buckets

  # Gather last releases
  gitlib::last_releases

  logecho
  logecho "${TPUT[BOLD]}Last published releases by branch:${TPUT[OFF]}"
  for i in $(for n in ${!LAST_RELEASE[*]}; do echo "$n"; done |sort -h); do
    logecho "$i": "${LAST_RELEASE[$i]}"
  done

  for bucket in $BUCKET $USER_BUCKET; do
    # Store existing buckets in an array for later use
    staging_buckets+=("gs://$bucket/stage")
    logecho
    logecho "${TPUT[BOLD]}Staged builds on $bucket:${TPUT[OFF]}"
    gsutil ls -h "gs://$bucket/stage" 2>&-
  done

  # TODO: Experiment to look at all current versions and then compare
  # the related builds in staging to them to only show valid/usable
  # staged builds.  The below experiment works except that it excludes
  # master branch Z+1s until the first Z+1 is published.  Still looking
  # for a clever, uncomplicated way of displaying this without sending
  # people through the gauntlet of trying to figure this out themselves.
  # On the plus side, anago does a pretty good job of cleaning up older
  # staged builds so the effect here is minimal
  #for i in $(for n in ${!LAST_RELEASE[*]}; do echo $n; done |sort -h); do
  #  [[ ${LAST_RELEASE[$i]} =~ (v[0-9]+\.[0-9]+\.) ]]
  #  minor=${BASH_REMATCH[1]}
  #  n=$(echo ${LAST_RELEASE[$i]} | tr -d '[a-z-.]')
  #    for build in $(gsutil ls -h ${staging_buckets[*]} |fgrep $minor); do
  #      [[ $build =~ ${VER_REGEX[release]} ]]
  #      n2=$(echo ${BASH_REMATCH[0]} | tr -d '[a-z-.]')
  #      ((n2<n)) || logecho $build
  #    done
  #done

  logecho
  logecho
  logecho "You may use one of these staged builds to release by passing it" \
          "as --buildversion to $PROG or anago."
  logecho
  logecho "For example, to submit a release job to GCB:"
  logecho "$ $PROG release release-1.8 --official --buildversion=v1.8.4-beta.0.52+d56fafdc081d5f"
  logecho "To run a release job on the desktop:"
  logecho "$ anago release-1.8 --official --buildversion=v1.8.4-beta.0.52+d56fafdc081d5f"
  logecho
  logecho "NOTE: The staged version (X.Y.Z-{alpha|beta}.n) should be Z+1" \
          "or n+1 as compared to the last published releases above" \
          "to indicate a candidate for the next release.  Any earlier staged" \
          "builds listed are no longer valid for use."
}

###############################################################################
# Stream the latest working job log to the console
# @optparam build_id - A specific build id to stream
stream_job_log () {
  local build_id=${1:-$($GCLOUD builds list |\
                        awk '/WORKING/{print $1;exit}')}
                        #highlighting fix'

  # Validate build_id
  if ! [[ $build_id =~ [a-f0-9-]{32} ]]; then
    logecho "Invalid/Null build id $build_id or no running jobs found."
    return 1
  fi

  logecho "Waiting for GCP to initiate stream..."
  logecho
  $GCLOUD builds log --stream "$build_id"
}

###############################################################################
# List Last N GCB jobs exposing more interesting/useful info than
# gcloud 'list'
# @optparam $state - One of WORKING SUCCESS FAILURE CANCELLED
list_jobs () {
  local state=$1
  # getting confused about scope of count
  # shellcheck disable=SC2178
  local count=3
  local counter=0
  local id
  local time
  local elapsed
  local status
  local tags

  # force upper-case as a convenience
  state=${state^^}

  # getting confused about type of count
  # shellcheck disable=SC2128
  logecho "Last $count $state jobs:"
  logecho
  {
  echo "TAGS S ID START($(date +%Z))"
  echo "=============== = ===================================== ==============="
  $GCLOUD builds list | sed -n '2,$p' | \
  while read -r id time elapsed status; do
    [[ $status != "${state:-$status}" ]] && continue
    ((counter++))

    tags=$($GCLOUD builds describe "$id" --format=json|\
           jq -r '.tags[]' 2>&-)

    # Convert time to local time
    time=$(date -d "$time" +%Y-%b-%d+%R:%S)
    echo "$tags" "${status:0:1}" "$id" "$time"
    echo
    ((counter>=count)) && break
  done
  } |column -te -c80
}

submit_it () {
  local substitutions

  # Additional TAG substitutions
  substitutions="_OFFICIAL_TAG=$OFFICIAL_TAG,_NOMOCK_TAG=$NOMOCK_TAG,_RC_TAG=$RC_TAG"
  substitutions+=",_BUILD_POINT=$BUILD_POINT,_GCP_USER_TAG=$GCP_USER_TAG"

  [[ $COMMAND == stage ]] && substitutions+=",_BUILD_AT_HEAD=$BUILD_AT_HEAD" && substitutions+=",_KUBE_CROSS_VERSION=$KUBE_CROSS_VERSION"

  # The usual suspects
  substitutions+=",_RELEASE_BRANCH=$RELEASE_BRANCH,_OFFICIAL=$OFFICIAL,_RC=$RC"
  substitutions+=",_NOMOCK=$NOMOCK,_BUILDVERSION=$BUILDVERSION"
  substitutions+=",_RELEASE_TOOL_REPO=$RELEASE_TOOL_REPO"
  substitutions+=",_RELEASE_TOOL_BRANCH=$RELEASE_TOOL_BRANCH"

  if ! JOB_DATA=$($GCLOUD builds submit --no-source \
                          --config="$YAML_FILE" --async --disk-size="$DISK_SIZE" \
                          --substitutions "$substitutions" 2>&1); then
    logecho "$FAILED: Job was not submitted.  Details:"
    logecho "$JOB_DATA"
    exit 1
  fi

  BUILD_ID=$(echo "$JOB_DATA" |awk 'END{print $1}')
  [[ "$JOB_DATA" =~ Logs\ are\ available\ at\ \[(https.*)\]\. ]] \
   && BUILD_URL=${BASH_REMATCH[1]}
  logecho "$HR"
  logecho "${COMMAND^^} $BUILD_ID submitted successfully."
  logecho "$HR"

  logecho
  logecho "To view last build:"
  logecho "$ $PROG tail"
  logecho
  logecho "To view this specific build:"
  logecho "$ $PROG tail $BUILD_ID"
  logecho "-OR-"
  logecho "$ $GCLOUD builds log --stream $BUILD_ID"
  logecho "-OR-"
  logecho "$BUILD_URL"
}

release_it () {
  if [[ -z $RELEASE_BRANCH ]]; then
    logecho "Branch not set.  Can't continue"
    return 1
  fi

  # Check for mandatory buildversion
  # Disable shellcheck for dynamically defined variable
  # shellcheck disable=SC2154
  if [[ -z $FLAGS_buildversion ]]; then
    logecho "--buildversion=<staged build> required for 'release' jobs."
    return 1
  fi

  submit_it
}

stage_it () {
  if [[ -z $RELEASE_BRANCH ]]; then
    logecho "Branch not set.  Can't continue"
    return 1
  fi

  # Get kube_cross version
  KUBE_CROSS_VERSION=$(curl -s "https://raw.githubusercontent.com/kubernetes/kubernetes/$RELEASE_BRANCH/build/build-image/cross/VERSION")

  [[ -z $KUBE_CROSS_VERSION ]] \
    && common::exit 1 "Unable to set KUBE_CROSS_VERSION.  Exiting..."

  # Submit it
  submit_it
}


###############################################################################
# MAIN
###############################################################################
##############################################################################
# Initialize logs
##############################################################################
# Initialize and save up to 10 (rotated logs)
MYLOG=$TMPDIR/$PROG.log
common::logfileinit "$MYLOG" 10
# BEGIN script
common::timestamp begin

common::check_binaries jq git column || common::exit 1 "Exiting..."
gitlib::repo_state || common::exit 1 "Exiting..."

# Ensure some prerequisites
if ! common::set_cloud_binaries; then
  logecho "Releasing Kubernetes requires gsutil and gcloud. Please download,"
  logecho "install and authorize through the Google Cloud SDK:"
  logecho
  logecho "https://developers.google.com/cloud/sdk/"
  common::exit 1 "Exiting..."
fi

logecho

# Defaults to "list" operation
COMMAND=${POSITIONAL_ARGV[0]:-"list"}
ARG2=${POSITIONAL_ARGV[1]}
PROJECT="kubernetes-release-test"
# Append $PROJECT to GCLOUD
GCLOUD+=" --project $PROJECT"
BUCKET="kubernetes-release"
# disk size must take base image into consideration.
# The amount specified is not what is "free"
DISK_SIZE="300"
GCP_USER=$($GCLOUD auth list --filter=status:ACTIVE --format="value(account)")
# This is used as a tag so convert @ to -at- (for yaml tag field
# where @ is invalid)
# First, for @google.com users strip off the @domain.tld
# If non-google.com, convert @ to -at-
GCP_USER_TAG=${GCP_USER%%@google.com}
GCP_USER_TAG=${GCP_USER_TAG/@/-at-}
# GCP also doesn't like anything even remotely looking like a domain name
# in the bucket name so convert . to -
GCP_USER_TAG=${GCP_USER_TAG/\./-}

# A build cannot be a release candidate and official at the same time
# Disable shellcheck for dynamically defined variable
# shellcheck disable=SC2154
if ((FLAGS_official)) && ((FLAGS_rc)); then
    logecho "A release cannot be an RC and official"
    common::exit 1 "Exiting..."
fi

if ((FLAGS_official)); then
  OFFICIAL_TAG="official"
  OFFICIAL="--$OFFICIAL_TAG"
elif ((FLAGS_rc)); then
  RC_TAG="rc"
  RC="--$RC_TAG"
fi

# Disable shellcheck for dynamically defined variable
# shellcheck disable=SC2154
if ((FLAGS_build_at_head)); then
  BUILD_POINT="HEAD"
  BUILD_AT_HEAD="--build-at-head"
fi

# Disable shellcheck for dynamically defined variable
# shellcheck disable=SC2154
if ((FLAGS_nomock)); then
  NOMOCK_TAG="nomock"
  NOMOCK="--$NOMOCK_TAG"
else
  USER_BUCKET=$BUCKET-$GCP_USER_TAG
  BUCKET+="-gcb"
fi

# BUILD_POINT used in yaml tags field and is one of:
# * HEAD (from --build-at-head)
# * Value of FLAGS_buildversion
# * Defaulting to searching for a build using find_green_build
#
if [[ -n $FLAGS_buildversion ]]; then
  BUILD_POINT="${FLAGS_buildversion/+/-}"
  BUILDVERSION="--buildversion=$FLAGS_buildversion"
else
  BUILD_POINT=${BUILD_POINT:-"find_green_build"}
fi

# TODO: Add a check here that user is allowed to submit jobs to $PROJECT
#       Respond with guidance on how to get on board

case $COMMAND in
  list) list_jobs "$ARG2"
        common::exit 0
        ;;
  staged) list_staged_builds
        common::exit 0
        ;;
  stream|tail) stream_job_log "$ARG2"
        common::exit 0
        ;;
  stage) YAML_FILE="$TOOL_ROOT/gcb/stage.yaml"
         RELEASE_BRANCH="$ARG2"
         stage_it
        ;;
  release) DISK_SIZE="100"
           YAML_FILE="$TOOL_ROOT/gcb/release.yaml"
           RELEASE_BRANCH="$ARG2"

           if ((FLAGS_nomock)); then
             logecho
             logecho "$ATTENTION!!"
             if ! common::askyorn "Really submit a --nomock release job against the $RELEASE_BRANCH branch"; then
               common::exit 1 "Exiting..."
             fi
           fi

           release_it
        ;;
       *) common::exit 1 "Unknown option $COMMAND.  Exiting..."
        ;;
esac || common::exit 1 "Exiting..."

# END script
common::timestamp end