Merge branch 'master' into pcheck-fix-push

openSUSE · Oct 30, 2024 · 6ebe282 · 6ebe282
2 parents f87a440 + fca81c8
commit 6ebe282
Show file tree

Hide file tree

Showing 7 changed files with 303 additions and 28 deletions.
diff --git a/docker_publisher.py b/docker_publisher.py
@@ -126,14 +126,20 @@ def _getManifestlist(self):
 
         return self.cached_manifestlist
 
+    def _manifestIsForArch(self, manifest, docker_arch, docker_variant):
+        if 'variant' in manifest['platform'] and manifest['platform']['variant'] != docker_variant:
+            return False
+
+        return manifest['platform']['architecture'] == docker_arch
+
     def releasedDockerImageVersion(self, arch):
         docker_arch, docker_variant = self.getDockerArch(arch)
 
         manifestlist = self._getManifestlist()
 
         if manifestlist is None:
-            # No manifest -> force outdated version
-            return "0"
+            # No manifest
+            return None
 
         for manifest in manifestlist['manifests']:
             if docker_variant is not None:
@@ -144,8 +150,7 @@ def releasedDockerImageVersion(self, arch):
                 if 'vnd-opensuse-version' in manifest:
                     return manifest['vnd-opensuse-version']
 
-        # Arch not in the manifest -> force outdated version
-        return "0"
+        return None
 
     def prepareReleasing(self):
         if self.new_manifestlist is not None:
@@ -192,6 +197,12 @@ def convertV1ToV2Manifest(self, path, manifest_v1):
                                                   "application/vnd.docker.container.image.v1+json"),
                 'layers': layers}
 
+    def removeImage(self, arch):
+        docker_arch, docker_variant = self.getDockerArch(arch)
+
+        self.new_manifestlist['manifests'] = [m for m in self.new_manifestlist['manifests']
+                                              if not self._manifestIsForArch(m, docker_arch, docker_variant)]
+
     def addImage(self, version, arch, image_path):
         docker_arch, docker_variant = self.getDockerArch(arch)
 
@@ -221,18 +232,17 @@ def addImage(self, version, arch, image_path):
         # Register the manifest in the list
         replaced = False
         for manifest in self.new_manifestlist['manifests']:
-            if 'variant' in manifest['platform'] and manifest['platform']['variant'] != docker_variant:
+            if not self._manifestIsForArch(manifest, docker_arch, docker_variant):
                 continue
 
-            if manifest['platform']['architecture'] == docker_arch:
-                manifest['mediaType'] = manifest_v2['mediaType']
-                manifest['size'] = len(manifest_content)
-                manifest['digest'] = manifest_digest
-                manifest['vnd-opensuse-version'] = version
-                if docker_variant is not None:
-                    manifest['platform']['variant'] = docker_variant
+            manifest['mediaType'] = manifest_v2['mediaType']
+            manifest['size'] = len(manifest_content)
+            manifest['digest'] = manifest_digest
+            manifest['vnd-opensuse-version'] = version
+            if docker_variant is not None:
+                manifest['platform']['variant'] = docker_variant
 
-                replaced = True
+            replaced = True
 
         if not replaced:
             # Add it instead
@@ -393,11 +403,22 @@ def run():
             'fetchers': {
                 'x86_64': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.6/containers/x86_64/opensuse-leap-image:docker", maintenance_release=True),  # noqa: E501
                 'aarch64': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.6/containers/aarch64/opensuse-leap-image:docker", maintenance_release=True),  # noqa: E501
-                'armv7l': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.6/containers_armv7/armv7l/opensuse-leap-image:docker", maintenance_release=True),  # noqa: E501
+                'armv7l': None,
                 'ppc64le': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.6/containers/ppc64le/opensuse-leap-image:docker", maintenance_release=True),  # noqa: E501
                 's390x': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.6/containers/s390x/opensuse-leap-image:docker", maintenance_release=True),  # noqa: E501
             },
-            'publisher': DockerImagePublisherRegistry(drc_leap, "latest", ["15.6", "15"]),
+            'publisher': DockerImagePublisherRegistry(drc_leap, "15.6"),
+        },
+        # Like Leap 15.6, but using the 15.5 image for armv7l
+        'leap-15': {
+            'fetchers': {
+                'x86_64': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.6/containers/x86_64/opensuse-leap-image:docker", maintenance_release=True),  # noqa: E501
+                'aarch64': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.6/containers/aarch64/opensuse-leap-image:docker", maintenance_release=True),  # noqa: E501
+                'armv7l': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.5/containers_armv7/armv7l/opensuse-leap-image:docker", maintenance_release=True),  # noqa: E501
+                'ppc64le': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.6/containers/ppc64le/opensuse-leap-image:docker", maintenance_release=True),  # noqa: E501
+                's390x': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.6/containers/s390x/opensuse-leap-image:docker", maintenance_release=True),  # noqa: E501
+            },
+            'publisher': DockerImagePublisherRegistry(drc_leap, "latest", ["15"]),
         },
     }
 
@@ -423,7 +444,7 @@ def run():
         for arch in fetchers:
             print(f"\tArchitecture {arch}")
             try:
-                current = fetchers[arch].currentVersion()
+                current = fetchers[arch].currentVersion() if fetchers[arch] else None
                 print(f"\t\tAvailable version: {current}")
 
                 released = publisher.releasedDockerImageVersion(arch)
@@ -446,6 +467,12 @@ def run():
         need_to_upload = False
 
         for arch, version in archs_to_update.items():
+            if fetchers[arch] is None:
+                print(f"\tRemoving {arch} image")
+                publisher.removeImage(arch)
+                need_to_upload = True
+                continue
+
             print(f"\tUpdating {arch} image to version {version}")
             try:
                 fetchers[arch].getDockerImage(lambda image_path: publisher.addImage(version=version,

diff --git a/gocd/bci.gocd.yaml b/gocd/bci.gocd.yaml
@@ -103,7 +103,7 @@ pipelines:
     - Release.Images.to.Product:
         approval: manual
         roles:
-        - SLE
+        - BCI
         environment_variables:
           OSC_CONFIG: /home/go/config/oscrc-totest-manager
           BCI_TOKEN: '{{SECRET:[opensuse.secrets][BCI_TOKEN]}}'
@@ -211,7 +211,7 @@ pipelines:
     - Release.Images.to.Product:
         approval: manual
         roles:
-        - SLE
+        - BCI
         environment_variables:
           OSC_CONFIG: /home/go/config/oscrc-totest-manager
           BCI_TOKEN: '{{SECRET:[opensuse.secrets][BCI_TOKEN]}}'
@@ -258,3 +258,111 @@ pipelines:
             export PYTHONPATH=scripts
             ./scripts/gocd/bci_repo_publish.py -A https://api.suse.de --verbose --debug run "--token=$BCI_TOKEN" 15-SP6
 
+  SLE_BCI_15SP7.RelPkgs:
+    group: BCI
+    lock_behavior: unlockWhenFinished
+    timer:
+      spec: 0 10 * ? * *
+      only_on_changes: false
+    materials:
+      git:
+        git: https://github.com/openSUSE/openSUSE-release-tools.git
+    environment_variables:
+      OSC_CONFIG: /home/go/config/oscrc-staging-bot
+    stages:
+    - Create.Release.Packages:
+        approval: manual
+        resources:
+        - repo-checker
+        tasks:
+        - script: ./pkglistgen.py -A https://api.suse.de update_and_solve -p SUSE:SLE-15-SP7:Update:BCI -s target --only-release-packages --force
+
+  Pkglistgen.SLE_BCI_15SP7:
+    group: BCI
+    lock_behavior: unlockWhenFinished
+    timer:
+      spec: 0 10 * ? * *
+      only_on_changes: false
+    materials:
+      repos:
+        git: git://botmaster.suse.de/suse-repos.git
+        auto_update: true
+        destination: repos
+        whitelist:
+          - SUSE:SLE-15-SP7:Update_-_standard.yaml
+      scripts:
+        auto_update: true
+        git: https://github.com/openSUSE/openSUSE-release-tools.git
+        whitelist:
+          - DO_NOT_TRIGGER
+        destination: scripts
+    environment_variables:
+      OSC_CONFIG: /home/go/config/oscrc-staging-bot
+    stages:
+    - pkglistgen:
+        approval:
+          type: manual
+        jobs:
+          BCI_target:
+            resources:
+            - repo-checker
+            tasks:
+            - script: ./scripts/pkglistgen.py -d -A https://api.suse.de update_and_solve -p SUSE:SLE-15-SP7:Update:BCI -s target
+    - Expect.Images.To.Finish:
+        resources:
+        - staging-bot
+        tasks:
+        - script: |
+            export PYTHONPATH=scripts
+            ./scripts/gocd/verify-repo-built-successful.py -A https://api.suse.de -p  SUSE:SLE-15-SP7:Update:BCI -r images
+
+    - Release.Images.to.Product:
+        approval: manual
+        roles:
+        - BCI
+        environment_variables:
+          OSC_CONFIG: /home/go/config/oscrc-totest-manager
+          BCI_TOKEN: '{{SECRET:[opensuse.secrets][BCI_TOKEN]}}'
+        resources:
+        - staging-bot
+        tasks:
+        # can't use osc command due to https://github.com/openSUSE/osc/issues/1194
+        - script: |-
+            SPRJ=SUSE:SLE-15-SP7:Update:BCI
+            for arch in aarch64 ppc64le s390x x86_64 ; do
+              PKG="000product:SLE_BCI-ftp-POOL-$arch"
+              PRJ="SUSE:Products:SLE-BCI:15-SP7:$arch"
+              curl -X POST -H "Authorization: Token $BCI_TOKEN" "https://api.suse.de/trigger/release?project=${SPRJ}&package=${PKG}&targetproject=${PRJ}&targetrepository=images&filter_source_repository=images"
+            done
+            for arch in aarch64 ppc64le s390x x86_64 ; do
+              sleep 600
+              while (osc -A https://api.suse.de/ api "/build/$PRJ/_result?view=summary&repository=images" | grep "result project" | grep -v 'code="published" state="published">'); do
+                  echo PENDING
+                  sleep 600
+              done
+              osc -A https://api.suse.de/ api "/build/$PRJ/_result?view=summary&repository=images" | grep "result project" | grep 'code="published" state="published">' && echo PUBLISHED
+            done
+
+  SLE_BCI_15SP7.RepoPublisher:
+    group: BCI
+    lock_behavior: unlockWhenFinished
+    timer:
+      spec: 0 23 * ? * *
+      only_on_changes: false
+    materials:
+      git:
+        git: https://github.com/openSUSE/openSUSE-release-tools.git
+        destination: scripts
+    environment_variables:
+        OSC_CONFIG: /home/go/config/oscrc-totest-manager
+        BCI_TOKEN: '{{SECRET:[opensuse.secrets][BCI_TOKEN]}}'
+    stages:
+    - Run:
+        approval: manual
+        resources:
+        - staging-bot
+        tasks:
+        - script: |
+            export PYTHONPATH=scripts
+            ./scripts/gocd/bci_repo_publish.py -A https://api.suse.de --verbose --debug run "--token=$BCI_TOKEN" 15-SP7
+
diff --git a/gocd/bci.gocd.yaml.erb b/gocd/bci.gocd.yaml.erb
@@ -41,7 +41,7 @@ pipelines:
             tasks:
             - script: python3 ./pkglistgen.py -d -A https://api.opensuse.org update_and_solve -p devel:BCI -s target
 
-<% ['SP5', 'SP6'].each do |sp| %>
+<% ['SP5', 'SP6', 'SP7'].each do |sp| %>
   SLE_BCI_15<%= sp %>.RelPkgs:
     group: BCI
     lock_behavior: unlockWhenFinished
@@ -103,7 +103,7 @@ pipelines:
     - Release.Images.to.Product:
         approval: manual
         roles:
-        - SLE
+        - BCI
         environment_variables:
           OSC_CONFIG: /home/go/config/oscrc-totest-manager
           BCI_TOKEN: '{{SECRET:[opensuse.secrets][BCI_TOKEN]}}'

diff --git a/gocd/microos.target.gocd.yaml b/gocd/microos.target.gocd.yaml
@@ -153,6 +153,93 @@ pipelines:
               done
               osc -A https://api.suse.de/ api "/build/SUSE:SLFO:Products:SL-Micro:6.1:PUBLISH/_result?view=summary&repository=${repo}" | grep "result project" | grep 'code="published" state="published">' && echo PUBLISHED
             done
+  SL-Micro6.2.Images.To.Test:
+    group: MicroOS
+    lock_behavior: unlockWhenFinished
+    materials:
+      repos:
+        git: git://botmaster.suse.de/suse-repos.git
+        auto_update: true
+        whitelist:
+          - SUSE:SLFO:Products:SL-Micro:6.2_-_images.yaml
+        destination: repos
+      scripts:
+        auto_update: true
+        git: https://github.com/openSUSE/openSUSE-release-tools.git
+        whitelist:
+          - DO_NOT_TRIGGER
+        destination: scripts
+    environment_variables:
+      OSC_CONFIG: /home/go/config/oscrc-staging-bot
+    stages:
+    - Expect.Images.To.Finish:
+        resources:
+        - staging-bot
+        tasks:
+        - script: |
+            export PYTHONPATH=scripts
+            ./scripts/gocd/verify-repo-built-successful.py -A https://api.suse.de -p SUSE:SLFO:Products:SL-Micro:6.2 -r images
+
+    - Release.Images:
+        approval: manual
+        roles:
+        - SLE
+        environment_variables:
+          OSC_CONFIG: /home/go/config/oscrc-totest-manager
+        resources:
+        - staging-bot
+        tasks:
+        - script: |-
+            set -e
+            for product in 000productcompose SL-Micro; do
+              osc -A https://api.suse.de release SUSE:SLFO:Products:SL-Micro:6.2 $product
+            done
+            sleep 600
+            for repo in product images; do
+              while (osc -A https://api.suse.de/ api "/build/SUSE:SLFO:Products:SL-Micro:6.2:ToTest/_result?view=summary&repository=${repo}" | grep "result project" | grep -v 'code="published" state="published">'); do
+                  echo PENDING
+                  sleep 600
+              done
+              osc -A https://api.suse.de/ api "/build/SUSE:SLFO:Products:SL-Micro:6.2:ToTest/_result?view=summary&repository=${repo}" | grep "result project" | grep 'code="published" state="published">' && echo PUBLISHED
+            done
+  SL-Micro6.2.Images.To.Publish:
+    group: MicroOS
+    materials:
+      repos:
+        git: git://botmaster.suse.de/suse-repos.git
+        auto_update: true
+        whitelist:
+          - SUSE:SLFO:Products:SL-Micro:6.2_-_images.yaml
+        destination: repos
+      scripts:
+        auto_update: true
+        git: https://github.com/openSUSE/openSUSE-release-tools.git
+        whitelist:
+          - DO_NOT_TRIGGER
+        destination: scripts
+    environment_variables:
+      OSC_CONFIG: /home/go/config/oscrc-staging-bot
+    stages:
+    - Release.Images:
+        approval: manual
+        roles:
+        - SLE
+        environment_variables:
+          OSC_CONFIG: /home/go/config/oscrc-totest-manager
+        resources:
+        - staging-bot
+        tasks:
+        - script: |-
+            set -e
+            osc -A https://api.suse.de release SUSE:SLFO:Products:SL-Micro:6.2:ToTest
+            sleep 600
+            for repo in product images; do
+              while (osc -A https://api.suse.de/ api "/build/SUSE:SLFO:Products:SL-Micro:6.2:PUBLISH/_result?view=summary&repository=${repo}" | grep "result project" | grep -v 'code="published" state="published">'); do
+                  echo PENDING
+                  sleep 600
+              done
+              osc -A https://api.suse.de/ api "/build/SUSE:SLFO:Products:SL-Micro:6.2:PUBLISH/_result?view=summary&repository=${repo}" | grep "result project" | grep 'code="published" state="published">' && echo PUBLISHED
+            done
   SLCS.Images:
     group: MicroOS
     lock_behavior: unlockWhenFinished