From ab8d0ae928109a13723865e0dbbc57814e9a7471 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Fri, 27 Sep 2024 01:08:49 -0600 Subject: [PATCH 1/4] Use :atc-postgres-version: role in documentation --- .env | 1 + docs/source/_ext/atc.py | 28 +++++++++++++++++++++++++ docs/source/admin/traffic_ops.rst | 2 +- docs/source/development/traffic_ops.rst | 5 ++++- 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/.env b/.env index e631acedac..cbd6788d3e 100644 --- a/.env +++ b/.env @@ -1,3 +1,4 @@ ATS_VERSION=9.2.4 GO_VERSION=1.23.1 +POSTGRES_VERSION=13.2 TOMCAT_VERSION=9.0.91 diff --git a/docs/source/_ext/atc.py b/docs/source/_ext/atc.py index f566ec9a32..39b92fa479 100644 --- a/docs/source/_ext/atc.py +++ b/docs/source/_ext/atc.py @@ -128,6 +128,33 @@ def atc_go_version( strong_node = nodes.strong(major_minor_version, major_minor_version) return [strong_node], [] +def atc_postgres_version( + unused_typ: None, + unused_rawtext: None, + unused_text: None, + lineno: int, + unused_inliner: None, + unused_options: None=None, + unused_content: None=None +) -> Tuple[List[nodes.Node], List[nodes.Node]]: + """ + A role that inserts the Postgres version used/required by this version of ATC. + + Example: + + :atc-postgres-version:_ + """ + env_file = os.path.join(os.path.dirname(__file__), "../../../.env") + with open(file=env_file, encoding="utf-8") as env_file: + env_file_contents = env_file.read() + + matches = re.search(pattern=r"^POSTGRES_VERSION=(\d+\.\d+)$", string=env_file_contents, flags=re.MULTILINE) + if matches is None: + raise ValueError(f"Postgres version found that could not be parsed: '{env_file_contents}' (from line {lineno})") + major_minor_version = matches.group(1) + strong_node = nodes.strong(major_minor_version, major_minor_version) + return [strong_node], [] + # -- Issue role -------------------------------------------------------------- ISSUE_URI: Final = REPO_URI + "issues/%s" @@ -330,6 +357,7 @@ def setup(app: Sphinx) -> Dict[str, Any]: app.add_directive("impl-detail", ImplementationDetail) app.add_directive("versionremoved", VersionRemoved) app.add_role("atc-go-version", atc_go_version) + app.add_role("atc-postgres-version", atc_postgres_version) app.add_role("issue", issue_role) app.add_role("pr", pr_role) app.add_role("pull-request", pr_role) diff --git a/docs/source/admin/traffic_ops.rst b/docs/source/admin/traffic_ops.rst index ce6fdc9a25..0b9021851e 100644 --- a/docs/source/admin/traffic_ops.rst +++ b/docs/source/admin/traffic_ops.rst @@ -87,7 +87,7 @@ Guide to-# psql -h pg -U postgres Password for user postgres: - psql (13.2) + psql (13.16) Type "help" for help. postgres=# diff --git a/docs/source/development/traffic_ops.rst b/docs/source/development/traffic_ops.rst index 2494cdea90..e88aa88491 100644 --- a/docs/source/development/traffic_ops.rst +++ b/docs/source/development/traffic_ops.rst @@ -28,7 +28,10 @@ Software Requirements ===================== Traffic Ops is only supported on CentOS 7+ systems (although many developers do use Mac OS with some success). Here are the requirements: -- `PostgreSQL 13.2 `_ - the machine where Traffic Ops is running must have the client tool set (e.g. :manpage:`psql(1)`), but the actual database can be run anywhere so long as it is accessible. +- |install-postgres-link|_ - the machine where Traffic Ops is running must have the client tool set (e.g. :manpage:`psql(1)`), but the actual database can be run anywhere so long as it is accessible. + +.. |install-postgres-link| replace:: PostgreSQL :atc-postgres-version:`_` +.. _install-postgres-link: https://www.postgresql.org/download/ .. note:: Prior to version 13.2, Traffic Ops used version 9.6. For upgrading an existing Mac OS Homebrew-based PostgreSQL instance, you can use `Homebrew `_ to easily upgrade from 9.6 to 13.2: From 7ff96e453949e58810e115c6cc5bc1e7eb698b3e Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Fri, 27 Sep 2024 01:09:44 -0600 Subject: [PATCH 2/4] Get postgres version from .env in Traffic Ops spec file --- traffic_ops/build/build_rpm.sh | 3 ++- traffic_ops/build/traffic_ops.spec | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/traffic_ops/build/build_rpm.sh b/traffic_ops/build/build_rpm.sh index 96043dedaf..126cb2a6fe 100755 --- a/traffic_ops/build/build_rpm.sh +++ b/traffic_ops/build/build_rpm.sh @@ -111,8 +111,9 @@ initBuildArea() { cp "$TO_DIR"/build/traffic_ops.spec "$RPMBUILD"/SPECS/. || \ { echo "Could not copy spec files: $?"; return 1; } + source "${TC_DIR}/.env" PLUGINS="$(grep -l 'AddPlugin(' "${TO_DIR}/traffic_ops_golang/plugin/"*.go | grep -v 'func AddPlugin(' | xargs -I '{}' basename {} '.go')" - export PLUGINS + export POSTGRES_VERSION PLUGINS echo "The build area has been initialized." } diff --git a/traffic_ops/build/traffic_ops.spec b/traffic_ops/build/traffic_ops.spec index 536e95074e..53fb0d94f6 100644 --- a/traffic_ops/build/traffic_ops.spec +++ b/traffic_ops/build/traffic_ops.spec @@ -21,6 +21,7 @@ %define TRAFFIC_OPS_LOG_DIR /var/log/traffic_ops %define TRAFFIC_OPS_ROOT_CERTIFICATES_DIR /var/log/traffic_ops %define debug_package %{nil} +%define postgres_version %{getenv:POSTGRES_VERSION} Summary: Traffic Ops Name: traffic_ops @@ -36,7 +37,7 @@ AutoReqProv: no Requires: cpanminus, expat-devel, libcurl, libpcap-devel, mkisofs, tar Requires: openssl-devel, perl, perl-core, perl-DBD-Pg, perl-DBI, perl-Digest-SHA1 Requires: libidn-devel, libcurl-devel, libcap -Requires: postgresql13 >= 13.2 +Requires: postgresql13 >= %{postgres_version} Requires: perl-JSON, perl-libwww-perl, perl-Test-CPAN-Meta, perl-WWW-Curl, perl-TermReadKey, perl-Crypt-ScryptKDF Requires: python3 Requires(pre): /usr/sbin/useradd, /usr/bin/getent From b3e3e011439648ca5a8c3be9e8251743466aefb6 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Fri, 27 Sep 2024 01:11:06 -0600 Subject: [PATCH 3/4] Get POSTGRES_VERSION from .env in Dockerfiles and docker-compose files --- cache-config/testing/docker/.env | 1 + cache-config/testing/docker/docker-compose.yml | 3 ++- cache-config/testing/docker/traffic_ops/Dockerfile | 2 +- docker-compose.yml | 2 +- infrastructure/cdn-in-a-box/docker-compose.yml | 2 ++ infrastructure/cdn-in-a-box/traffic_ops/Dockerfile-db | 3 ++- tc-health-client/testing/docker/.env | 1 + tc-health-client/testing/docker/docker-compose.yml | 2 +- tc-health-client/testing/docker/traffic_ops/Dockerfile | 2 +- traffic_ops/INSTALL.md | 2 +- traffic_ops/app/db/trafficvault/test/.env | 1 + traffic_ops/app/db/trafficvault/test/Dockerfile-db-admin | 2 +- traffic_ops/app/db/trafficvault/test/Dockerfile-tvdb | 4 ++-- traffic_ops/app/db/trafficvault/test/docker-compose.yml | 4 ++-- traffic_ops/traffic_ops_golang/README.md | 2 +- traffic_ops_db/docker/.env | 1 + traffic_ops_db/docker/Dockerfile | 3 ++- traffic_ops_db/docker/docker-compose.yml | 2 ++ traffic_ops_db/test/docker/.env | 1 + traffic_ops_db/test/docker/Dockerfile-db | 4 ++-- traffic_ops_db/test/docker/Dockerfile-db-admin | 2 +- traffic_ops_db/test/docker/docker-compose.yml | 4 ++-- 22 files changed, 31 insertions(+), 19 deletions(-) create mode 120000 cache-config/testing/docker/.env create mode 120000 tc-health-client/testing/docker/.env create mode 120000 traffic_ops/app/db/trafficvault/test/.env create mode 120000 traffic_ops_db/docker/.env create mode 120000 traffic_ops_db/test/docker/.env diff --git a/cache-config/testing/docker/.env b/cache-config/testing/docker/.env new file mode 120000 index 0000000000..e72b3932f4 --- /dev/null +++ b/cache-config/testing/docker/.env @@ -0,0 +1 @@ +../../../.env \ No newline at end of file diff --git a/cache-config/testing/docker/docker-compose.yml b/cache-config/testing/docker/docker-compose.yml index f5e9156567..08712fb8a0 100644 --- a/cache-config/testing/docker/docker-compose.yml +++ b/cache-config/testing/docker/docker-compose.yml @@ -35,7 +35,7 @@ volumes: services: db: - image: postgres:13.2 + image: postgres:${POSTGRES_VERSION} env_file: - variables.env ports: @@ -50,6 +50,7 @@ services: args: - OS_DISTRO=${OS_DISTRO:-rockylinux} - OS_VERSION=${OS_VERSION:-8} + - POSTGRES_VERSION=${POSTGRES_VERSION} context: ../../.. dockerfile: cache-config/testing/docker/traffic_ops/Dockerfile volumes: diff --git a/cache-config/testing/docker/traffic_ops/Dockerfile b/cache-config/testing/docker/traffic_ops/Dockerfile index c5f9d8c89c..83d1c38b22 100644 --- a/cache-config/testing/docker/traffic_ops/Dockerfile +++ b/cache-config/testing/docker/traffic_ops/Dockerfile @@ -39,7 +39,7 @@ RUN set -o nounset -o errexit && \ if [[ "${OS_VERSION%%.*}" -eq 7 ]]; then \ use_repo=''; \ enable_repo=''; \ - # needed for llvm-toolset-7-clang, which is needed for postgresql13-devel-13.2-1PGDG, required by TO rpm + # needed for llvm-toolset-7-clang, which is needed for postgresql13-devel-13.X-1PGDG, required by TO rpm dnf -y install gcc centos-release-scl-rh; \ else \ use_repo='--repo=pgdg13'; \ diff --git a/docker-compose.yml b/docker-compose.yml index ca8cc4f24b..87a1a6e017 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,7 +38,7 @@ services: - .:/go/src/github.com/apache/trafficcontrol db: - image: postgres:13.2-alpine + image: postgres:${POSTGRES_VERSION}-alpine ports: - 5432:5432 environment: diff --git a/infrastructure/cdn-in-a-box/docker-compose.yml b/infrastructure/cdn-in-a-box/docker-compose.yml index 22872ad847..60894e3f0f 100644 --- a/infrastructure/cdn-in-a-box/docker-compose.yml +++ b/infrastructure/cdn-in-a-box/docker-compose.yml @@ -37,6 +37,8 @@ services: # All other components access the database thru the Traffic Ops API db: build: + args: + POSTGRES_VERSION: ${POSTGRES_VERSION} context: . dockerfile: traffic_ops/Dockerfile-db hostname: db diff --git a/infrastructure/cdn-in-a-box/traffic_ops/Dockerfile-db b/infrastructure/cdn-in-a-box/traffic_ops/Dockerfile-db index 593db61c2c..f824a0b86f 100644 --- a/infrastructure/cdn-in-a-box/traffic_ops/Dockerfile-db +++ b/infrastructure/cdn-in-a-box/traffic_ops/Dockerfile-db @@ -19,7 +19,8 @@ # Dockerfile for trafficops db ############################################################ -FROM postgres:13.2-alpine +ARG POSTGRES_VERSION +FROM postgres:${POSTGRES_VERSION}-alpine ENV POSTGRES_PASSWORD=$POSTGRES_PASSWORD RUN apk add bind-tools \ diff --git a/tc-health-client/testing/docker/.env b/tc-health-client/testing/docker/.env new file mode 120000 index 0000000000..e72b3932f4 --- /dev/null +++ b/tc-health-client/testing/docker/.env @@ -0,0 +1 @@ +../../../.env \ No newline at end of file diff --git a/tc-health-client/testing/docker/docker-compose.yml b/tc-health-client/testing/docker/docker-compose.yml index 8d142aab4e..c01d759012 100644 --- a/tc-health-client/testing/docker/docker-compose.yml +++ b/tc-health-client/testing/docker/docker-compose.yml @@ -35,7 +35,7 @@ volumes: services: db: - image: postgres:13.2 + image: postgres:${POSTGRES_VERSION} env_file: - variables.env ports: diff --git a/tc-health-client/testing/docker/traffic_ops/Dockerfile b/tc-health-client/testing/docker/traffic_ops/Dockerfile index 1e999e80f7..f4a33f97c5 100644 --- a/tc-health-client/testing/docker/traffic_ops/Dockerfile +++ b/tc-health-client/testing/docker/traffic_ops/Dockerfile @@ -37,7 +37,7 @@ RUN set -o nounset -o errexit && \ if [[ "${OS_VERSION%%.*}" -eq 7 ]]; then \ use_repo=''; \ enable_repo=''; \ - # needed for llvm-toolset-7-clang, which is needed for postgresql13-devel-13.2-1PGDG, required by TO rpm + # needed for llvm-toolset-7-clang, which is needed for postgresql13-devel-13.x-1PGDG, required by TO rpm dnf -y install gcc centos-release-scl-rh; \ else \ use_repo='--repo=pgdg13'; \ diff --git a/traffic_ops/INSTALL.md b/traffic_ops/INSTALL.md index 7a3fbcc76b..5edd18575c 100644 --- a/traffic_ops/INSTALL.md +++ b/traffic_ops/INSTALL.md @@ -30,7 +30,7 @@ Instructions are here: https://yum.postgresql.org/ `$ sudo yum install https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm` -## Install Postgres 13.2 server (in a container or on the host) +## Install Postgres 13.16 server (in a container or on the host) ### on the host: diff --git a/traffic_ops/app/db/trafficvault/test/.env b/traffic_ops/app/db/trafficvault/test/.env new file mode 120000 index 0000000000..7e66cb76ec --- /dev/null +++ b/traffic_ops/app/db/trafficvault/test/.env @@ -0,0 +1 @@ +../../../../../.env \ No newline at end of file diff --git a/traffic_ops/app/db/trafficvault/test/Dockerfile-db-admin b/traffic_ops/app/db/trafficvault/test/Dockerfile-db-admin index e039037cf0..680a4a8a32 100644 --- a/traffic_ops/app/db/trafficvault/test/Dockerfile-db-admin +++ b/traffic_ops/app/db/trafficvault/test/Dockerfile-db-admin @@ -20,7 +20,7 @@ ############################################################ FROM rockylinux:8 -ARG POSTGRES_VERSION=13.2 +ARG POSTGRES_VERSION ENV POSTGRES_VERSION=$POSTGRES_VERSION # NOTE: temporary workaround for removal of golang packages from CentOS 7 base repo diff --git a/traffic_ops/app/db/trafficvault/test/Dockerfile-tvdb b/traffic_ops/app/db/trafficvault/test/Dockerfile-tvdb index 5fdeeb5a08..9c054d1bfe 100644 --- a/traffic_ops/app/db/trafficvault/test/Dockerfile-tvdb +++ b/traffic_ops/app/db/trafficvault/test/Dockerfile-tvdb @@ -19,9 +19,9 @@ # Dockerfile for traffic vault db ############################################################ -ARG POSTGRES_VERSION=13.2 +ARG POSTGRES_VERSION FROM postgres:${POSTGRES_VERSION} -ARG POSTGRES_VERSION=13.2 +ARG POSTGRES_VERSION ENV POSTGRES_VERSION=$POSTGRES_VERSION ENV POSTGRES_PASSWORD=twelve diff --git a/traffic_ops/app/db/trafficvault/test/docker-compose.yml b/traffic_ops/app/db/trafficvault/test/docker-compose.yml index a833db84cf..b0a4b66d44 100644 --- a/traffic_ops/app/db/trafficvault/test/docker-compose.yml +++ b/traffic_ops/app/db/trafficvault/test/docker-compose.yml @@ -32,7 +32,7 @@ services: context: . dockerfile: Dockerfile-tvdb args: - POSTGRES_VERSION: ${POSTGRES_VERSION:-13.2} + POSTGRES_VERSION: ${POSTGRES_VERSION} ports: - 5432 @@ -41,7 +41,7 @@ services: context: . dockerfile: Dockerfile-db-admin args: - POSTGRES_VERSION: ${POSTGRES_VERSION:-13.2} + POSTGRES_VERSION: ${POSTGRES_VERSION} TRAFFIC_OPS_RPM: traffic_ops.rpm depends_on: - tvdb diff --git a/traffic_ops/traffic_ops_golang/README.md b/traffic_ops/traffic_ops_golang/README.md index a28db5e81a..a80c05e29a 100644 --- a/traffic_ops/traffic_ops_golang/README.md +++ b/traffic_ops/traffic_ops_golang/README.md @@ -24,7 +24,7 @@ To run `traffic_ops_golang` proxy locally the following prerequisites are needed: * Golang version greater or equal to the Go version found in the `GO_VERSION` file at the base of this repository. See: [https://golang.org/doc/install](https://golang.org/doc/install) -* Postgres 13.2 or greater +* Postgres 13 or greater ## Vendoring and Building diff --git a/traffic_ops_db/docker/.env b/traffic_ops_db/docker/.env new file mode 120000 index 0000000000..c7360fb82d --- /dev/null +++ b/traffic_ops_db/docker/.env @@ -0,0 +1 @@ +../../.env \ No newline at end of file diff --git a/traffic_ops_db/docker/Dockerfile b/traffic_ops_db/docker/Dockerfile index ce3f53ab2a..fbb343a397 100644 --- a/traffic_ops_db/docker/Dockerfile +++ b/traffic_ops_db/docker/Dockerfile @@ -11,7 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -FROM postgres:13.2 +ARG POSTGRES_VERSION +FROM postgres:${POSTGRES_VERSION} MAINTAINER dev@trafficcontrol.apache.org diff --git a/traffic_ops_db/docker/docker-compose.yml b/traffic_ops_db/docker/docker-compose.yml index 4d88ac10ea..68729f927d 100644 --- a/traffic_ops_db/docker/docker-compose.yml +++ b/traffic_ops_db/docker/docker-compose.yml @@ -17,6 +17,8 @@ services: build: context: .. dockerfile: /opt/traffic_ops_db/docker/Dockerfile + args: + POSTGRES_VERSION: ${POSTGRES_VERSION} env_file: - /opt/traffic_ops_db/docker/docker-compose.env ports: diff --git a/traffic_ops_db/test/docker/.env b/traffic_ops_db/test/docker/.env new file mode 120000 index 0000000000..e72b3932f4 --- /dev/null +++ b/traffic_ops_db/test/docker/.env @@ -0,0 +1 @@ +../../../.env \ No newline at end of file diff --git a/traffic_ops_db/test/docker/Dockerfile-db b/traffic_ops_db/test/docker/Dockerfile-db index e2dc6c32b3..aaccabb4ef 100644 --- a/traffic_ops_db/test/docker/Dockerfile-db +++ b/traffic_ops_db/test/docker/Dockerfile-db @@ -19,9 +19,9 @@ # Dockerfile for trafficops db ############################################################ -ARG POSTGRES_VERSION=13.2 +ARG POSTGRES_VERSION FROM postgres:${POSTGRES_VERSION} -ARG POSTGRES_VERSION=13.2 +ARG POSTGRES_VERSION ENV POSTGRES_VERSION=$POSTGRES_VERSION ENV POSTGRES_PASSWORD=twelve diff --git a/traffic_ops_db/test/docker/Dockerfile-db-admin b/traffic_ops_db/test/docker/Dockerfile-db-admin index b2add5929a..fa24860d67 100644 --- a/traffic_ops_db/test/docker/Dockerfile-db-admin +++ b/traffic_ops_db/test/docker/Dockerfile-db-admin @@ -20,7 +20,7 @@ ############################################################ FROM rockylinux:8 -ARG POSTGRES_VERSION=13.2 +ARG POSTGRES_VERSION ENV POSTGRES_VERSION=$POSTGRES_VERSION RUN dnf install -y \ diff --git a/traffic_ops_db/test/docker/docker-compose.yml b/traffic_ops_db/test/docker/docker-compose.yml index 8764e775d1..2bb2e65e33 100644 --- a/traffic_ops_db/test/docker/docker-compose.yml +++ b/traffic_ops_db/test/docker/docker-compose.yml @@ -32,7 +32,7 @@ services: context: . dockerfile: Dockerfile-db args: - POSTGRES_VERSION: ${POSTGRES_VERSION:-13.2} + POSTGRES_VERSION: ${POSTGRES_VERSION} ports: - 5432 @@ -41,7 +41,7 @@ services: context: . dockerfile: Dockerfile-db-admin args: - POSTGRES_VERSION: ${POSTGRES_VERSION:-13.2} + POSTGRES_VERSION: ${POSTGRES_VERSION} TRAFFIC_OPS_RPM: traffic_ops.rpm depends_on: - db From 01eb347ee479a6e2fcbe9bac1982de7a9d131d4e Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Fri, 27 Sep 2024 01:11:55 -0600 Subject: [PATCH 4/4] Update Postgres version to 13.16 --- .env | 2 +- CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.env b/.env index cbd6788d3e..2fd755e41b 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ ATS_VERSION=9.2.4 GO_VERSION=1.23.1 -POSTGRES_VERSION=13.2 +POSTGRES_VERSION=13.16 TOMCAT_VERSION=9.0.91 diff --git a/CHANGELOG.md b/CHANGELOG.md index 777a42c36c..abdf411652 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - [#7846](https://github.com/apache/trafficcontrol/pull/7846) *Traffic Portal* Increase State character limit - [#8010](https://github.com/apache/trafficcontrol/pull/8010) *Traffic Stats* Omit NPM dev dependencies from Traffic Stats RPM - [#8071](https://github.com/apache/trafficcontrol/pull/8071) *Traffic Ops* Improve validation for the `id` field of the `PUT /deliveryservice_request_comments` endpoint. +- [#8078](https://github.com/apache/trafficcontrol/pull/8078) *Traffic Ops DB* Update Postgres version to 13.16. ### Removed - [#7832](https://github.com/apache/trafficcontrol/pull/7832) *t3c* Removed Perl dependency