From 56ddb71df01e61604c4823fe811ab17bef2f5976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 4 Dec 2024 15:50:57 +0200 Subject: [PATCH] Move master-protected-branches to common function --- master-protected-branches/master.cfg | 163 ++++++++++++--------------- 1 file changed, 73 insertions(+), 90 deletions(-) diff --git a/master-protected-branches/master.cfg b/master-protected-branches/master.cfg index 228a478c..af066fd9 100644 --- a/master-protected-branches/master.cfg +++ b/master-protected-branches/master.cfg @@ -2,38 +2,44 @@ # ex: set filetype=python: import os -import sys from collections import defaultdict -from datetime import timedelta -import docker -from buildbot.plugins import * -from buildbot.process.properties import Properties, Property -from buildbot.process.remotecommand import RemoteCommand -from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver -from buildbot.steps.shell import Compile, SetPropertyFromCommand, ShellCommand, Test -from buildbot.steps.source.github import GitHub -from twisted.internet import defer - -sys.setrecursionlimit(10000) - -sys.path.insert(0, "/srv/buildbot/master") - -from common_factories import * -from constants import * -from locks import * -from schedulers_definition import SCHEDULERS -from utils import * - -# This is the dictionary that the buildmaster pays attention to. We also use -# a shorter alias to save typing. -c = BuildmasterConfig = {} +from buildbot.plugins import steps, util, worker +from buildbot.process.properties import Property +from common_factories import getLastNFailedBuildsFactory, getQuickBuildFactory +from locks import getLocks +from master_common import base_master_config +from utils import ( + canStartBuild, + createWorker, + dockerfile, + isJepsenBranch, + isStagingBranch, + ls2list, + nextBuild, + waitIfStaging, +) # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. config = {"private": {}} exec(open("master-private.cfg").read(), config, {}) +c = BuildmasterConfig = base_master_config( + title=os.getenv("TITLE", default="MariaDB CI"), + title_url=os.getenv("TITLE_URL", default="https://github.com/MariaDB/server"), + buildbot_url=os.getenv("BUILDMASTER_URL", default="https://buildbot.mariadb.org/"), + github_access_token=config["private"]["gh_mdbci"]["access_token"], + # TODO(cvicentiu) undo this hardcoding + secrets_provider_file=os.getenv( + "MASTER_CREDENTIALS_DIR", + default="/srv/buildbot/master/master-credential-provider", + ), + master_port=int(os.getenv("PORT", default=9994)), + db_url=config["private"]["db_url"], + mq_router_url=os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"), +) + ####### BUILDBOT SERVICES # 'services' is a list of BuildbotService items like reporter targets. The @@ -41,46 +47,6 @@ exec(open("master-private.cfg").read(), config, {}) # has a variety to choose from, like IRC bots. -c["services"] = [] -context = util.Interpolate("buildbot/%(prop:buildername)s") -gs = reporters.GitHubStatusPush( - token=config["private"]["gh_mdbci"]["access_token"], - context=context, - startDescription="Build started.", - endDescription="Build done.", - verbose=True, - builders=GITHUB_STATUS_BUILDERS, -) -c["services"].append(gs) -c['secretsProviders'] = [secrets.SecretInAFile(dirname=os.getenv("MASTER_CREDENTIALS_DIR", default="/srv/buildbot/master/master-credential-provider"))] -####### PROJECT IDENTITY - -# the 'title' string will appear at the top of this buildbot installation's -# home pages (linked to the 'titleURL'). -c["title"] = os.getenv("TITLE", default="MariaDB CI") -c["titleURL"] = os.getenv("TITLE_URL", default="https://github.com/MariaDB/server") - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server is visible. This typically uses the port number set in -# the 'www' entry below, but with an externally-visible host name which the -# buildbot cannot figure out without some help. -c["buildbotURL"] = os.getenv("BUILDMASTER_URL", default="https://buildbot.mariadb.org/") - -# 'protocols' contains information about protocols which master will use for -# communicating with workers. You must define at least 'port' option that workers -# could connect to your master with this protocol. -# 'port' must match the value configured into the workers (with their -# --master option) -port = int(os.getenv("PORT", default="9994")) -c["protocols"] = {"pb": {"port": port}} - -####### DB URL - -c["db"] = { - # This specifies what database buildbot uses to store its state. - "db_url": config["private"]["db_url"] -} - mtrDbPool = util.EqConnectionPool( "MySQLdb", config["private"]["db_host"], @@ -92,11 +58,6 @@ mtrDbPool = util.EqConnectionPool( ####### Disable net usage reports from being sent to buildbot.net c["buildbotNetUsageData"] = None -####### SCHEDULERS - -# Configure the Schedulers, which decide how to react to incoming changes. -c["schedulers"] = SCHEDULERS - ####### WORKERS # The 'workers' list defines the set of recognized workers. Each element is @@ -116,7 +77,10 @@ c["workers"].append( "hz-bbw1-docker-tarball-debian-10", None, docker_host=config["private"]["docker_workers"]["hz-bbw1-docker"], - image=os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian10", + image=os.getenv( + "CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:" + ) + + "debian10", followStartupLogs=False, autopull=True, alwaysPull=True, @@ -134,7 +98,10 @@ c["workers"].append( "hz-bbw4-docker-tarball-debian-10", None, docker_host=config["private"]["docker_workers"]["hz-bbw4-docker"], - image=os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian10", + image=os.getenv( + "CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:" + ) + + "debian10", followStartupLogs=False, autopull=True, alwaysPull=True, @@ -180,7 +147,11 @@ for w_name in ["hz-bbw"]: w_name, i, "debian-11-debug-ps-embed", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian11", + os.getenv( + "CONTAINER_REGISTRY_URL", + default="quay.io/mariadb-foundation/bb-worker:", + ) + + "debian11", jobs=14, save_packages=False, ) @@ -188,7 +159,11 @@ for w_name in ["hz-bbw"]: w_name, i, "debian-12", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian12", + os.getenv( + "CONTAINER_REGISTRY_URL", + default="quay.io/mariadb-foundation/bb-worker:", + ) + + "debian12", jobs=jobs, save_packages=True, ) @@ -196,7 +171,11 @@ for w_name in ["hz-bbw"]: w_name, i, "debian-12-debug-embed", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "debian12", + os.getenv( + "CONTAINER_REGISTRY_URL", + default="quay.io/mariadb-foundation/bb-worker:", + ) + + "debian12", jobs=14, save_packages=False, ) @@ -204,7 +183,11 @@ for w_name in ["hz-bbw"]: w_name, i, "fedora-40", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "fedora40", + os.getenv( + "CONTAINER_REGISTRY_URL", + default="quay.io/mariadb-foundation/bb-worker:", + ) + + "fedora40", jobs=jobs, save_packages=True, ) @@ -212,7 +195,11 @@ for w_name in ["hz-bbw"]: w_name, i, "last-N-failed", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "rhel9", + os.getenv( + "CONTAINER_REGISTRY_URL", + default="quay.io/mariadb-foundation/bb-worker:", + ) + + "rhel9", jobs=jobs, save_packages=True, ) @@ -228,7 +215,11 @@ for w_name in ["hz-bbw"]: w_name, i, "ubuntu-2004-debug", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "ubuntu20.04", + os.getenv( + "CONTAINER_REGISTRY_URL", + default="quay.io/mariadb-foundation/bb-worker:", + ) + + "ubuntu20.04", jobs=14, save_packages=True, ) @@ -236,7 +227,11 @@ for w_name in ["hz-bbw"]: w_name, i, "ubuntu-2204-debug-ps", - os.getenv("CONTAINER_REGISTRY_URL", default="quay.io/mariadb-foundation/bb-worker:") + "ubuntu22.04", + os.getenv( + "CONTAINER_REGISTRY_URL", + default="quay.io/mariadb-foundation/bb-worker:", + ) + + "ubuntu22.04", jobs=14, save_packages=False, ) @@ -560,15 +555,3 @@ c["builders"].append( factory=getLastNFailedBuildsFactory("debug", mtrDbPool), ) ) - -c["logEncoding"] = "utf-8" - -c["multiMaster"] = True - -c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now. - "type": "wamp", - "router_url": os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"), - "realm": "realm1", - # valid are: none, critical, error, warn, info, debug, trace - "wamp_debug_level": "info", -}