From a983a4ef6a5205573f202da81d13255cd4b22038 Mon Sep 17 00:00:00 2001 From: Fabian Vogt Date: Thu, 20 Jul 2023 16:17:58 +0200 Subject: [PATCH] Add BCI repo publishing bot Unfortunately TTM itself doesn't quite fit here due to the reasons outlined in the code and the additional checks are not a good fit for adding it into the go pipeline description, so write yet another OBS publishing bot! For the initial test runs this is not triggered by timer runs yet. --- gocd/bci.gocd.yaml | 57 +++++++++++ gocd/bci.gocd.yaml.erb | 19 ++++ gocd/bci_repo_publish.py | 199 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 275 insertions(+) create mode 100755 gocd/bci_repo_publish.py diff --git a/gocd/bci.gocd.yaml b/gocd/bci.gocd.yaml index 6aab15805..09d7dcf0b 100644 --- a/gocd/bci.gocd.yaml +++ b/gocd/bci.gocd.yaml @@ -127,6 +127,25 @@ pipelines: osc -A https://api.suse.de/ api "/build/$PRJ/_result?view=summary&repository=images" | grep "result project" | grep 'code="published" state="published">' && echo PUBLISHED done + SLE_BCI_15SP3.RepoPublisher: + group: BCI + lock_behavior: unlockWhenFinished + materials: + git: + git: https://github.com/openSUSE/openSUSE-release-tools.git + environment_variables: + OSC_CONFIG: /home/go/config/oscrc-totest-manager + BCI_TOKEN: '{{SECRET:[opensuse.secrets][BCI_TOKEN]}}' + stages: + - Run: + approval: manual + resources: + - staging-bot + tasks: + - script: | + export PYTHONPATH=scripts + ./scripts/gocd/bci_repo_publish.py -A https://api.suse.de --verbose --debug run "--token=$BCI_TOKEN" 15-SP3 + SLE_BCI_15SP4.RelPkgs: group: BCI lock_behavior: unlockWhenFinished @@ -212,6 +231,25 @@ pipelines: osc -A https://api.suse.de/ api "/build/$PRJ/_result?view=summary&repository=images" | grep "result project" | grep 'code="published" state="published">' && echo PUBLISHED done + SLE_BCI_15SP4.RepoPublisher: + group: BCI + lock_behavior: unlockWhenFinished + materials: + git: + git: https://github.com/openSUSE/openSUSE-release-tools.git + environment_variables: + OSC_CONFIG: /home/go/config/oscrc-totest-manager + BCI_TOKEN: '{{SECRET:[opensuse.secrets][BCI_TOKEN]}}' + stages: + - Run: + approval: manual + resources: + - staging-bot + tasks: + - script: | + export PYTHONPATH=scripts + ./scripts/gocd/bci_repo_publish.py -A https://api.suse.de --verbose --debug run "--token=$BCI_TOKEN" 15-SP4 + SLE_BCI_15SP5.RelPkgs: group: BCI lock_behavior: unlockWhenFinished @@ -297,3 +335,22 @@ pipelines: osc -A https://api.suse.de/ api "/build/$PRJ/_result?view=summary&repository=images" | grep "result project" | grep 'code="published" state="published">' && echo PUBLISHED done + SLE_BCI_15SP5.RepoPublisher: + group: BCI + lock_behavior: unlockWhenFinished + materials: + git: + git: https://github.com/openSUSE/openSUSE-release-tools.git + environment_variables: + OSC_CONFIG: /home/go/config/oscrc-totest-manager + BCI_TOKEN: '{{SECRET:[opensuse.secrets][BCI_TOKEN]}}' + stages: + - Run: + approval: manual + resources: + - staging-bot + tasks: + - script: | + export PYTHONPATH=scripts + ./scripts/gocd/bci_repo_publish.py -A https://api.suse.de --verbose --debug run "--token=$BCI_TOKEN" 15-SP5 + diff --git a/gocd/bci.gocd.yaml.erb b/gocd/bci.gocd.yaml.erb index 715bdbe65..a26d6f604 100644 --- a/gocd/bci.gocd.yaml.erb +++ b/gocd/bci.gocd.yaml.erb @@ -126,4 +126,23 @@ pipelines: done osc -A https://api.suse.de/ api "/build/$PRJ/_result?view=summary&repository=images" | grep "result project" | grep 'code="published" state="published">' && echo PUBLISHED done + + SLE_BCI_15<%= sp %>.RepoPublisher: + group: BCI + lock_behavior: unlockWhenFinished + materials: + git: + git: https://github.com/openSUSE/openSUSE-release-tools.git + environment_variables: + OSC_CONFIG: /home/go/config/oscrc-totest-manager + BCI_TOKEN: '{{SECRET:[opensuse.secrets][BCI_TOKEN]}}' + stages: + - Run: + approval: manual + resources: + - staging-bot + tasks: + - script: | + export PYTHONPATH=scripts + ./scripts/gocd/bci_repo_publish.py -A https://api.suse.de --verbose --debug run "--token=$BCI_TOKEN" 15-<%= sp %> <% end %> diff --git a/gocd/bci_repo_publish.py b/gocd/bci_repo_publish.py new file mode 100755 index 000000000..89efeb1bd --- /dev/null +++ b/gocd/bci_repo_publish.py @@ -0,0 +1,199 @@ +#!/usr/bin/python3 +# (c) 2023 fvogt@suse.de +# GPL-2.0-or-later + +# This is a "mini ttm" for the BCI repo. Differences: +# * No :ToTest staging area +# * Only the repo is built, so it needs BCI specific openQA queries +# * The publishing location is arch specific +# * Uses a token for releasing to the publishing project + + +import cmdln +import logging +import ToolBase +import requests +import sys +import time +import re +from lxml import etree as ET +from openqa_client.client import OpenQA_Client +from osc.core import http_GET, makeurl + + +class BCIRepoPublisher(ToolBase.ToolBase): + def __init__(self): + ToolBase.ToolBase.__init__(self) + self.logger = logging.getLogger(__name__) + self.openqa = OpenQA_Client(server='https://openqa.suse.de') + + def version_of_product(self, project, package, repo, arch): + """Get the build version of the given product build, based on the binary name.""" + url = makeurl(self.apiurl, ['build', project, repo, arch, package]) + root = ET.parse(http_GET(url)).getroot() + for binary in root.findall('binary'): + result = re.match(r'.*-Build(.*)-Media1.report', binary.get('filename')) + if result: + return result.group(1) + + raise RuntimeError(f"Failed to get version of {project}/{package}") + + def mtime_of_product(self, project, package, repo, arch): + """Get the build time stamp of the given product, based on _buildenv.""" + url = makeurl(self.apiurl, ['build', project, repo, arch, package]) + root = ET.parse(http_GET(url)).getroot() + mtime = root.xpath('/binarylist/binary[@filename = "_buildenv"]/@mtime') + return mtime[0] + + def openqa_jobs_for_product(self, arch, version, build): + """Query openQA for all relevant jobs""" + values = { + 'group': 'BCI repo', + 'flavor': 'BCI-Repo-Updates', + 'arch': arch, + 'version': version, + 'build': build, + 'scope': 'current', + 'latest': '1', + } + return self.openqa.openqa_request('GET', 'jobs', values)['jobs'] + + def is_repo_published(self, project, repo): + """Validates that the given prj/repo is fully published and all builds + have succeeded.""" + url = makeurl(self.apiurl, ['build', project, '_result'], + {'view': 'summary', 'repository': repo}) + root = ET.parse(http_GET(url)).getroot() + for result in root.findall('result'): + if result.get('dirty', 'false') != 'false': + return False + if result.get('code') != 'published' or result.get('state') != 'published': + return False + + for statuscount in root.findall('statuscount'): + if statuscount.get('code') not in ('succeeded', 'disabled', 'excluded'): + return False + + return True + + def run(self, version, token=None): + build_prj = f'SUSE:SLE-{version}:Update:BCI' + + if not self.is_repo_published(build_prj, 'images'): + self.logger.info(f'{build_prj}/images not successfully built') + return + + # Build the list of packages with metainfo + packages = [] + # As long as it's the same everywhere, hardcoding this list here + # is easier and safer than trying to derive it from the package list. + for arch in ('aarch64', 'ppc64le', 's390x', 'x86_64'): + packages.append({ + 'arch': arch, + 'name': f'000product:SLE_BCI-ftp-POOL-{arch}', + 'build_prj': build_prj, + 'publish_prj': f'SUSE:Products:SLE-BCI:{version}:{arch}' + }) + + # Fetch the build numbers of built products. + # After release, the BuildXXX part vanishes, so the mtime has to be + # used instead for comparing built and published binaries. + for pkg in packages: + pkg['built_version'] = self.version_of_product(pkg['build_prj'], pkg['name'], + 'images', 'local') + pkg['built_mtime'] = self.mtime_of_product(pkg['build_prj'], pkg['name'], + 'images', 'local') + pkg['published_mtime'] = self.mtime_of_product(pkg['publish_prj'], pkg['name'], + 'images', 'local') + + # Verify that the builds for all archs are in sync + built_versions = {pkg['built_version'] for pkg in packages} + if len(built_versions) != 1: + # This should not be the case if everything is built and idle + self.logger.warning(f'Different builds found - not releasing: {packages}') + return + + # Compare versions + newer_version_available = [int(pkg['built_mtime']) > int(pkg['published_mtime']) + for pkg in packages] + if not any(newer_version_available): + self.logger.info('Current build already published, nothing to do.') + return + + # Check openQA results + openqa_passed = True + for pkg in packages: + passed = 0 + pending = 0 + failed = 0 + for job in self.openqa_jobs_for_product(arch=pkg['arch'], version=version, + build=pkg['built_version']): + if job['result'] in ('passed', 'softfailed'): + passed += 1 + elif job['result'] == 'none': + self.logger.info(f'https://openqa.suse.de/tests/{job["id"]} pending') + pending += 1 + else: + self.logger.warning(f'https://openqa.suse.de/tests/{job["id"]} failed') + failed += 1 + + if passed == 0 or pending > 0 or failed > 0: + openqa_passed = False + + if not openqa_passed: + self.logger.info('No positive result from openQA (yet)') + return + + # Trigger publishing + if token is None: + self.logger.warning('Would publish now, but no token specified') + return + + for pkg in packages: + self.logger.info(f'Releasing {pkg["name"]}...') + params = { + 'project': pkg['build_prj'], 'package': pkg['name'], + 'filter_source_repository': 'images', + 'targetproject': pkg['publish_prj'], 'targetrepository': 'images' + } + url = makeurl(self.apiurl, ['trigger', 'release'], params) + # No bindings for using tokens yet, so do the request manually + req = requests.post(url, headers={'Authorization': f'Token {token}'}) + if req.status_code != 200: + raise RuntimeError(f'Releasing failed: {req.text}') + + self.logger.info('Waiting for publishing to finish') + for pkg in packages: + while not self.is_repo_published(pkg['publish_prj'], 'images'): + self.logger.debug(f'Waiting for {pkg["publish_prj"]}') + time.sleep(20) + + +class CommandLineInterface(ToolBase.CommandLineInterface): + def __init__(self, *args, **kwargs): + ToolBase.CommandLineInterface.__init__(self, args, kwargs) + + def setup_tool(self): + tool = BCIRepoPublisher() + if self.options.debug: + logging.basicConfig(level=logging.DEBUG) + elif self.options.verbose: + logging.basicConfig(level=logging.INFO) + + return tool + + @cmdln.option('--token', help='The token for publishing. Does a dry run if not given.') + def do_run(self, subcmd, opts, project): + """${cmd_name}: run the BCI repo publisher for the specified project, + e.g. 15-SP3 + + ${cmd_usage} + ${cmd_option_list} + """ + + self.tool.run(project, token=opts.token) + + +if __name__ == "__main__": + cli = CommandLineInterface() + sys.exit(cli.main())