From 7273618dd8603aba8bc5bb45c60fdd78edf32a84 Mon Sep 17 00:00:00 2001 From: John Stairs Date: Wed, 2 Oct 2024 20:51:36 +0000 Subject: [PATCH] Publish container images to MCR --- .github/workflows/pr-ci.yml | 27 +++---------------- Makefile | 20 ++++++++------ cli/.goreleaser.yaml | 3 ++- cli/internal/install/cloudinstall/config.tpl | 2 +- cli/internal/install/cloudinstall/helm.go | 18 ++++--------- .../install/dockerinstall/validation.go | 2 +- cli/internal/install/installer.go | 22 +++++++++++---- deploy/config/microsoft/devconfig.yml | 10 ++++--- .../installation/cloud-installation.md | 6 ++--- scripts/build-images.sh | 25 ++++++++++++++--- 10 files changed, 72 insertions(+), 63 deletions(-) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 889b37b8..2234f59a 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -282,27 +282,6 @@ jobs: set -euo pipefail make -s integration-test-no-up - publish-official-images: - needs: - - test-azure-needs-hosted-runner - runs-on: ${{ fromJson(needs.test-azure-needs-hosted-runner.outputs.AZURE_RUNS_ON_JSON)}} - defaults: - run: - shell: bash - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Log in to Azure - uses: ./.github/actions/login-azure - - - name: Publish official images - run: | - set -euo pipefail - make publish-official-images - build-windows-binaries: runs-on: ubuntu-latest defaults: @@ -539,8 +518,10 @@ jobs: - name: get container registry run: | set -euo pipefail - official_container_registry=$(echo "$DEVELOPER_CONFIG_BASE64" | base64 -d | jq -r '.officialContainerRegistry.fqdn') - echo "OFFICIAL_CONTAINER_REGISTRY=$(echo $official_container_registry)" >> $GITHUB_ENV + official_pull_container_registry=$(echo "$DEVELOPER_CONFIG_BASE64" | base64 -d | jq -r '.officialPullContainerRegistry.fqdn') + echo "OFFICIAL_PULL_CONTAINER_REGISTRY=$(echo $official_pull_container_registry)" >> $GITHUB_ENV + official_pull_container_registry_directory=$(echo "$DEVELOPER_CONFIG_BASE64" | base64 -d | jq -r '.officialPullContainerRegistry.directory // ""') + echo "OFFICIAL_PULL_CONTAINER_REGISTRY_DIRECTORY=$(echo $official_pull_container_registry_directory)" >> $GITHUB_ENV - name: Run GoReleaser uses: goreleaser/goreleaser-action@v5 diff --git a/Makefile b/Makefile index 45bc342d..0b30faf3 100644 --- a/Makefile +++ b/Makefile @@ -67,9 +67,11 @@ _docker-build: target_arg="--${DOCKER_BUILD_TARGET}" tag=$${EXPLICIT_IMAGE_TAG:-dev} + dev_config=$$(scripts/get-config.sh --dev -o json) - registry=$$(scripts/get-config.sh --dev -e .wipContainerRegistry.fqdn) - scripts/build-images.sh $$target_arg ${DOCKER_BUILD_ARCH_FLAGS} ${DOCKER_BUILD_PUSH_FLAGS} --tag "$$tag" --registry "$${registry}" + registry=$$(echo "$${dev_config}" | jq -r '.wipContainerRegistry.fqdn') + directory=$$(echo "$${dev_config}" | jq -r '.wipContainerRegistry.directory // ""') + scripts/build-images.sh $$target_arg ${DOCKER_BUILD_ARCH_FLAGS} ${DOCKER_BUILD_PUSH_FLAGS} --tag "$$tag" --registry "$${registry}" --registry-directory "$${directory}" docker-build-test: login-wip-acr $(MAKE) _docker-build DOCKER_BUILD_TARGET=test-connectivity @@ -86,11 +88,13 @@ docker-build-worker-waiter: login-wip-acr docker-build: docker-build-test docker-build-tyger-server docker-build-buffer-sidecar docker-build-worker-waiter publish-official-images: - registry=$$(scripts/get-config.sh --dev -e .officialContainerRegistry.fqdn) + dev_config=$$(scripts/get-config.sh --dev -o json) + registry=$$(echo "$${dev_config}" | jq -r '.officialPushContainerRegistry.fqdn') registry_name=$$(echo "$$registry" | cut -d'.' -f1) az acr login --name "$${registry_name}" + registry_directory=$$(echo "$${dev_config}" | jq -r '.officialPushContainerRegistry.directory // ""') tag=$$(git describe --tags) - scripts/build-images.sh --push --push-force --arch amd64 --arch arm64 --tyger-server --worker-waiter --buffer-sidecar --helm --tag "$${tag}" --registry "$${registry}" + scripts/build-images.sh --push --push-force --arch amd64 --arch arm64 --tyger-server --worker-waiter --buffer-sidecar --helm --tag "$${tag}" --registry "$${registry}" --registry-directory "$${registry_directory}" integration-test-no-up-prereqs: @@ -119,13 +123,13 @@ get-tyger-uri: echo ${TYGER_URI} install-cli: - official_container_registry=$$(scripts/get-config.sh --dev -e .officialContainerRegistry.fqdn) + dev_config=$$(scripts/get-config.sh --dev -o json) + container_registry=$$(echo "$${dev_config}" | jq -r '.wipContainerRegistry.fqdn') + container_registry_directory=$$(echo "$${dev_config}" | jq -r '.wipContainerRegistry.directory // ""') cd cli tag=$$(git describe --tags 2> /dev/null || echo "0.0.0") CGO_ENABLED=0 go install -buildvcs=false -ldflags="-s -w \ - -X main.version=$${tag} \ - -X github.com/microsoft/tyger/cli/internal/install.ContainerRegistry=$${official_container_registry} \ - -X github.com/microsoft/tyger/cli/internal/install.ContainerImageTag=$${tag}" \ + -X main.version=$${tag}" \ ./cmd/tyger ./cmd/buffer-sidecar ./cmd/tyger-proxy cli-ready: install-cli diff --git a/cli/.goreleaser.yaml b/cli/.goreleaser.yaml index af16e0c7..f8af9a40 100644 --- a/cli/.goreleaser.yaml +++ b/cli/.goreleaser.yaml @@ -11,7 +11,8 @@ builds: ldflags: - -s -w - -X main.version={{ .Tag }} - - -X github.com/microsoft/tyger/cli/internal/install.ContainerRegistry={{ .Env.OFFICIAL_CONTAINER_REGISTRY }} + - -X github.com/microsoft/tyger/cli/internal/install.ContainerRegistry={{ .Env.OFFICIAL_PULL_CONTAINER_REGISTRY }} + - -X github.com/microsoft/tyger/cli/internal/install.ContainerRegistryDirectory={{ .Env.OFFICIAL_PULL_CONTAINER_REGISTRY_DIRECTORY }} - -X github.com/microsoft/tyger/cli/internal/install.ContainerImageTag={{ .Tag }} goos: - linux diff --git a/cli/internal/install/cloudinstall/config.tpl b/cli/internal/install/cloudinstall/config.tpl index 0ef6a697..4d944041 100644 --- a/cli/internal/install/cloudinstall/config.tpl +++ b/cli/internal/install/cloudinstall/config.tpl @@ -111,7 +111,7 @@ api: # tyger: # repoName: # repoUrl: # not set if using `chartRef` - # chartRef: # e.g. oci://tyger.azurecr.io/helm/tyger + # chartRef: # e.g. oci://mcr.microsoft.com/tyger/helm/tyger # namespace: # version: # values: diff --git a/cli/internal/install/cloudinstall/helm.go b/cli/internal/install/cloudinstall/helm.go index 41fe9894..d309ff61 100644 --- a/cli/internal/install/cloudinstall/helm.go +++ b/cli/internal/install/cloudinstall/helm.go @@ -321,14 +321,6 @@ func (inst *Installer) InstallTyger(ctx context.Context) error { } func (inst *Installer) InstallTygerHelmChart(ctx context.Context, restConfig *rest.Config, dryRun bool) (manifest string, valuesYaml string, err error) { - if install.ContainerRegistry == "" { - panic("officialContainerRegistry not set during build") - } - - if install.ContainerImageTag == "" { - panic("officialContainerImageTag not set during build") - } - clustersConfigJson, err := json.Marshal(inst.Config.Cloud.Compute.Clusters) if err != nil { return "", "", fmt.Errorf("failed to marshal cluster configuration: %w", err) @@ -412,13 +404,13 @@ func (inst *Installer) InstallTygerHelmChart(ctx context.Context, restConfig *re helmConfig := HelmChartConfig{ Namespace: TygerNamespace, ReleaseName: DefaultTygerReleaseName, - ChartRef: fmt.Sprintf("oci://%s/helm/tyger", install.ContainerRegistry), + ChartRef: fmt.Sprintf("oci://%s%shelm/tyger", install.ContainerRegistry, install.GetNormalizedContainerRegistryDirectory()), Version: install.ContainerImageTag, Values: map[string]any{ - "image": fmt.Sprintf("%s/tyger-server:%s", install.ContainerRegistry, install.ContainerImageTag), - "bufferSidecarImage": fmt.Sprintf("%s/buffer-sidecar:%s", install.ContainerRegistry, install.ContainerImageTag), - "bufferCopierImage": fmt.Sprintf("%s/buffer-copier:%s", install.ContainerRegistry, install.ContainerImageTag), - "workerWaiterImage": fmt.Sprintf("%s/worker-waiter:%s", install.ContainerRegistry, install.ContainerImageTag), + "image": fmt.Sprintf("%s%styger-server:%s", install.ContainerRegistry, install.GetNormalizedContainerRegistryDirectory(), install.ContainerImageTag), + "bufferSidecarImage": fmt.Sprintf("%s%sbuffer-sidecar:%s", install.ContainerRegistry, install.GetNormalizedContainerRegistryDirectory(), install.ContainerImageTag), + "bufferCopierImage": fmt.Sprintf("%s%sbuffer-copier:%s", install.ContainerRegistry, install.GetNormalizedContainerRegistryDirectory(), install.ContainerImageTag), + "workerWaiterImage": fmt.Sprintf("%s%sworker-waiter:%s", install.ContainerRegistry, install.GetNormalizedContainerRegistryDirectory(), install.ContainerImageTag), "hostname": inst.Config.Api.DomainName, "identity": map[string]any{ "tygerServer": map[string]any{ diff --git a/cli/internal/install/dockerinstall/validation.go b/cli/internal/install/dockerinstall/validation.go index fe716f0c..3e333a11 100644 --- a/cli/internal/install/dockerinstall/validation.go +++ b/cli/internal/install/dockerinstall/validation.go @@ -154,7 +154,7 @@ func (inst *Installer) QuickValidateConfig() bool { } func defaultImage(repo string) string { - return fmt.Sprintf("%s/%s:%s", install.ContainerRegistry, repo, install.ContainerImageTag) + return fmt.Sprintf("%s%s%s:%s", install.ContainerRegistry, install.GetNormalizedContainerRegistryDirectory(), repo, install.ContainerImageTag) } func validationError(success *bool, format string, args ...any) { diff --git a/cli/internal/install/installer.go b/cli/internal/install/installer.go index c7b7883c..75f5813e 100644 --- a/cli/internal/install/installer.go +++ b/cli/internal/install/installer.go @@ -7,6 +7,8 @@ import ( "context" "errors" "io" + "strings" + "sync" ) var ( @@ -15,11 +17,21 @@ var ( ) var ( - // Set during build but we provide defaults so that there is some value when debugging. - // We will need to update these from time to time. Alternatively, you can set the registry - // values using the --set command-line argument. - ContainerRegistry string = "tyger.azurecr.io" - ContainerImageTag string = "v0.4.0-112-g428a5e8" + // Set during build + ContainerRegistry string = "" + ContainerRegistryDirectory string = "" + ContainerImageTag string = "" + + GetNormalizedContainerRegistryDirectory = sync.OnceValue(func() string { + normalized := ContainerRegistryDirectory + if !strings.HasPrefix(normalized, "/") { + normalized = "/" + normalized + } + if !strings.HasSuffix(normalized, "/") { + normalized = normalized + "/" + } + return normalized + }) ) type Installer interface { diff --git a/deploy/config/microsoft/devconfig.yml b/deploy/config/microsoft/devconfig.yml index f7d0e915..b77b8a2d 100644 --- a/deploy/config/microsoft/devconfig.yml +++ b/deploy/config/microsoft/devconfig.yml @@ -1,10 +1,12 @@ subscriptionId: 87d8acb3-5176-4651-b457-6ab9cefd8e3d wipContainerRegistry: - name: eminence fqdn: eminence.azurecr.io -officialContainerRegistry: - name: tyger - fqdn: tyger.azurecr.io +officialPushContainerRegistry: + fqdn: tygerprod.azurecr.io + directory: /public/tyger +officialPullContainerRegistry: + fqdn: mcr.microsoft.com + directory: /tyger keyVault: eminence testAppUri: api://tyger-test-client pemCertSecret: diff --git a/docs/introduction/installation/cloud-installation.md b/docs/introduction/installation/cloud-installation.md index 7779ec2f..ce17cdb2 100644 --- a/docs/introduction/installation/cloud-installation.md +++ b/docs/introduction/installation/cloud-installation.md @@ -157,7 +157,7 @@ api: # tyger: # repoName: # repoUrl: # not set if using `chartRef` - # chartRef: # e.g. oci://tyger.azurecr.io/helm/tyger + # chartRef: # e.g. oci://mcr.microsoft.com/tyger/helm/tyger # namespace: # version: # values: # Helm values overrides @@ -175,7 +175,7 @@ command-line with `--set`. For example: ```bash tyger api install -f config.yml \ - --set api.helm.tyger.chartRef=oci://tyger.azurecr.io/helm/tyger \ + --set api.helm.tyger.chartRef=oci://mcr.microsoft.com/tyger/helm/tyger \ --set api.helm.tyger.version=v0.4.0 ``` @@ -216,7 +216,7 @@ tyger api install -f config.yml ``` This command installs the Tyger API from the Helm chart in the -`tyger.azurecr.io/helm/tyger` registry, using a version baked into the `tyger` +`mcr.microsoft.com/tyger/helm/tyger` registry, using a version baked into the `tyger` CLI. Upgrade the server by updating the CLI and rerunning `tyger api install`. The API's TLS certificate is automatically created using [Let's diff --git a/scripts/build-images.sh b/scripts/build-images.sh index 2171a094..83733273 100755 --- a/scripts/build-images.sh +++ b/scripts/build-images.sh @@ -19,6 +19,7 @@ Options: --worker-waiter Build (and optionally push) the worker-waiter image --buffer-sidecar Build (and optionally push) the buffer-sidecar image --helm Package and push the Tyger Helm chart + --registry-directory The parent directory of the repositories. e.g. // --arch amd64|arm64 The architecture to build for. Can be specified multiple times. --push Push runtime images (requires --tag or --use-git-hash-as-tag) --push-force Force runtime images, will overwrite images with same tag (requires --tag or --use-git-hash-as-tag) @@ -28,6 +29,8 @@ Options: EOF } +registry_dir="/" + while [[ $# -gt 0 ]]; do key="$1" @@ -71,6 +74,10 @@ while [[ $# -gt 0 ]]; do push=1 shift ;; + --registry-directory) + registry_dir="$2" + shift 2 + ;; --push-force) push=1 force=1 @@ -96,6 +103,16 @@ while [[ $# -gt 0 ]]; do esac done +# Ensure registry_dir starts with a / +if [[ ! "$registry_dir" =~ ^/ ]]; then + registry_dir="/$registry_dir" +fi + +# Ensure registry_dir ends with a / +if [[ ! "$registry_dir" =~ /$ ]]; then + registry_dir="$registry_dir/" +fi + # if nether amd64 nor arm64 is specified, build for both if [[ -z "${amd64:-}" && -z "${arm64:-}" ]]; then amd64=true @@ -107,7 +124,7 @@ export DOCKER_BUILDKIT=1 repo_root_dir="$(dirname "$0")/.." function build_and_push_platform() { - full_image="${container_registry_fqdn}/${repo}:${image_tag_with_platform}" + full_image="${container_registry_fqdn}${registry_dir}${repo}:${image_tag_with_platform}" echo "Building image ${full_image}..." set +e @@ -156,8 +173,8 @@ function build_and_push() { return 0 fi - manifest_name="${container_registry_fqdn}/${repo}:${image_tag}" - docker manifest create --amend "${manifest_name}" "${container_registry_fqdn}/${repo}:${image_tag}-amd64" "${container_registry_fqdn}/${repo}:${image_tag}-arm64" >/dev/null + manifest_name="${container_registry_fqdn}${registry_dir}${repo}:${image_tag}" + docker manifest create --amend "${manifest_name}" "${container_registry_fqdn}${registry_dir}${repo}:${image_tag}-amd64" "${container_registry_fqdn}${registry_dir}${repo}:${image_tag}-arm64" >/dev/null # Push manigest if [[ -z "${force:-}" ]]; then @@ -232,7 +249,7 @@ if [[ -n "${helm:-}" ]]; then username="00000000-0000-0000-0000-000000000000" echo "${token}" | docker login "${container_registry_fqdn}" -u "${username}" --password-stdin - helm_repo_namespace="oci://${container_registry_fqdn}/helm" + helm_repo_namespace="oci://${container_registry_fqdn}${registry_dir}helm" chart_dir=${repo_root_dir}/deploy/helm/tyger package_dir=$(mktemp -d)