Passer sur une distribution Debian comme base d'image pour la base de données

.github/workflows/cicd.yml

@@ -14,8 +14,10 @@ on:
       - "**.tsx"
       - "**.yml"
       - ".github/workflows/cicd.yml"
+      - ".github/workflows/database.yml"
       - "infra/configurations/**.properties"
       - "infra/docker/app/**"
+      - "infra/docker/database/**"
       - "**.sql"
       - "backend/pom.xml"
       - "backend/build.gradle.kts"

Makefile

@@ -80,7 +80,7 @@ upgrade-postgres-11-to-13:
 		-v $(PG_13_DATA_VOLUME_NAME):/var/lib/postgresql/13/data \
 		ghcr.io/mtes-mct/monitorfish/monitorfish-database-upgrade:pg11_to_pg13-ts2.3.1-postgis3.3.4 -O "-c timescaledb.restoring='on'" -O "-c shared_preload_libraries=timescaledb";
 
-upgrade-postgres-13-to-16-dev:
+upgrade-postgres-13-to-16:
 	docker run --rm \
 		-v $(PG_13_DATA_VOLUME_NAME):/var/lib/postgresql/13/data \
 		-v $(PG_16_DATA_VOLUME_NAME):/var/lib/postgresql/16/data \

docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3"
 services:
   db:
     image: ghcr.io/mtes-mct/monitorfish/monitorfish-database:pg16-ts2.14.2-postgis3.4.2

infra/docker/database/docker-entrypoint-initdb.d/000_install_timescaledb.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+create_sql=`mktemp`
+
+# Checks to support bitnami image with same scripts so they stay in sync
+if [ ! -z "${BITNAMI_APP_NAME:-}" ]; then
+	if [ -z "${POSTGRES_USER:-}" ]; then
+		POSTGRES_USER=postgres
+	fi
+
+	if [ -z "${POSTGRESQL_PASSWORD:-}" ]; then
+		POSTGRESQL_PASSWORD=${POSTGRESQL_POSTGRES_PASSWORD}
+	fi
+
+	if [ -z "${POSTGRES_DB:-}" ]; then
+		POSTGRES_DB=${POSTGRESQL_DATABASE}
+	fi
+
+	if [ -z "${PGDATA:-}" ]; then
+		PGDATA=${POSTGRESQL_DATA_DIR}
+	fi
+fi
+
+if [ -z "${POSTGRESQL_CONF_DIR:-}" ]; then
+	POSTGRESQL_CONF_DIR=${PGDATA}
+fi
+
+cat <<EOF >${create_sql}
+CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
+EOF
+
+TS_TELEMETRY='basic'
+if [ "${TIMESCALEDB_TELEMETRY:-}" == "off" ]; then
+	TS_TELEMETRY='off'
+
+	# We delete the job as well to ensure that we do not spam the
+	# log with other messages related to the Telemetry job.
+	cat <<EOF >>${create_sql}
+SELECT alter_job(1,scheduled:=false);
+EOF
+fi
+
+echo "timescaledb.telemetry_level=${TS_TELEMETRY}" >> ${POSTGRESQL_CONF_DIR}/postgresql.conf
+
+if [ -z "${POSTGRESQL_PASSWORD:-}" ]; then
+	POSTGRESQL_PASSWORD=${POSTGRES_PASSWORD:-}
+fi
+export PGPASSWORD="$POSTGRESQL_PASSWORD"
+
+# create extension timescaledb in initial databases
+psql -U "${POSTGRES_USER}" postgres -f ${create_sql}
+psql -U "${POSTGRES_USER}" template1 -f ${create_sql}
+
+if [ "${POSTGRES_DB:-postgres}" != 'postgres' ]; then
+    psql -U "${POSTGRES_USER}" "${POSTGRES_DB}" -f ${create_sql}
+fi

infra/docker/database/docker-entrypoint-initdb.d/001_timescaledb_tune.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+NO_TS_TUNE=${NO_TS_TUNE:-""}
+TS_TUNE_MEMORY=${TS_TUNE_MEMORY:-""}
+TS_TUNE_NUM_CPUS=${TS_TUNE_NUM_CPUS:-""}
+TS_TUNE_MAX_CONNS=${TS_TUNE_MAX_CONNS:-""}
+TS_TUNE_MAX_BG_WORKERS=${TS_TUNE_MAX_BG_WORKERS:-""}
+
+if [ ! -z "${NO_TS_TUNE:-}" ]; then
+    # The user has explicitly requested not to run timescaledb-tune; exit this script
+    exit 0
+fi
+
+
+if [ -z "${POSTGRESQL_CONF_DIR:-}" ]; then
+        POSTGRESQL_CONF_DIR=${PGDATA}
+fi
+
+if [ -z "${TS_TUNE_MEMORY:-}" ]; then
+    # See if we can get the container's total allocated memory from the cgroups metadata
+    if [ -f /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then
+        TS_TUNE_MEMORY=$(cat /sys/fs/cgroup/memory/memory.limit_in_bytes)
+
+        if [ "${TS_TUNE_MEMORY}" = "18446744073709551615" ]; then
+            # Bash seems to error out for numbers greater than signed 64-bit,
+            # so if the value of limit_in_bytes is the 64-bit UNSIGNED max value
+            # we should just bail out and hope timescaledb-tune can figure this
+            # out. If we don't, the next comparison is likely going to fail
+            # or it might store a negative value which will crash later.
+            TS_TUNE_MEMORY=""
+        fi
+
+        FREE_KB=$(grep MemTotal: /proc/meminfo | awk '{print $2}')
+        FREE_BYTES=$(( ${FREE_KB} * 1024 ))
+        if [ ${TS_TUNE_MEMORY} -gt ${FREE_BYTES} ]; then
+            # Something weird is going on if the cgroups memory limit exceeds the total available
+            # amount of system memory reported by "free", which is the total amount of memory available on the host.
+            # Most likely, it is this issue: https://github.com/moby/moby/issues/18087 (if no limit is
+            # set, the max limit is set to the max 64 bit integer). In this case, we just leave
+            # TS_TUNE_MEMORY blank and let timescaledb-tune derive the memory itself using syscalls.
+            TS_TUNE_MEMORY=""
+        else
+            # Convert the bytes to MB so it plays nicely with timescaledb-tune
+            TS_TUNE_MEMORY="$(echo ${TS_TUNE_MEMORY} | awk '{print int($1 / 1024 / 1024)}')MB"
+        fi
+    fi
+fi
+
+if [ -z "${TS_TUNE_NUM_CPUS:-}" ]; then
+    # See if we can get the container's available CPUs from the cgroups metadata
+    if [ -f /sys/fs/cgroup/cpuset/cpuset.cpus ]; then
+        TS_TUNE_NUM_CPUS=$(cat /sys/fs/cgroup/cpuset/cpuset.cpus)
+        if [[ ${TS_TUNE_NUM_CPUS} == *-* ]]; then
+            # The CPU limits have been defined as a range (e.g., 0-3 for 4 CPUs). Subtract them and add 1
+            # to convert the range to the number of CPUs.
+            TS_TUNE_NUM_CPUS=$(echo ${TS_TUNE_NUM_CPUS} | tr "-" " " | awk '{print ($2 - $1) + 1}')
+        elif [[ ${TS_TUNE_NUM_CPUS} == *,* ]]; then
+            # The CPU limits have been defined as a comma separated list (e.g., 0,1,2,3 for 4 CPUs). Count each CPU
+            TS_TUNE_NUM_CPUS=$(echo ${TS_TUNE_NUM_CPUS} | tr "," "\n" | wc -l)
+        elif [ $(echo -n ${TS_TUNE_NUM_CPUS} | wc -c) -eq 1 ]; then
+            # The CPU limit has been defined as a single numbered CPU. In this case the CPU limit is 1
+            # regardless of what that number is
+            TS_TUNE_NUM_CPUS=1
+        fi
+    fi
+fi
+
+if [ ! -z "${TS_TUNE_MEMORY:-}" ]; then
+    TS_TUNE_MEMORY_FLAGS=--memory="${TS_TUNE_MEMORY}"
+fi
+
+if [ ! -z "${TS_TUNE_NUM_CPUS:-}" ]; then
+    TS_TUNE_NUM_CPUS_FLAGS=--cpus=${TS_TUNE_NUM_CPUS}
+fi
+
+if [ ! -z "${TS_TUNE_MAX_CONNS:-}" ]; then
+    TS_TUNE_MAX_CONNS_FLAGS=--max-conns=${TS_TUNE_MAX_CONNS}
+fi
+
+if [ ! -z "${TS_TUNE_MAX_BG_WORKERS:-}" ]; then
+    TS_TUNE_MAX_BG_WORKERS_FLAGS=--max-bg-workers=${TS_TUNE_MAX_BG_WORKERS}
+fi
+
+if [ ! -z "${PG_MAJOR}" ]; then
+    TS_TUNE_PG_VERSION=--pg-version=${PG_MAJOR}
+fi
+
+/usr/local/bin/timescaledb-tune --quiet --yes --conf-path="${POSTGRESQL_CONF_DIR}/postgresql.conf" ${TS_TUNE_MEMORY_FLAGS} ${TS_TUNE_NUM_CPUS_FLAGS} ${TS_TUNE_MAX_CONNS_FLAGS} ${TS_TUNE_MAX_BG_WORKERS_FLAGS} ${TS_TUNE_PG_VERSION}

infra/docker/database/pg13_16.Dockerfile

@@ -2,79 +2,59 @@ ARG PG_MAJOR
 ARG TIMESCALEDB_VERSION
 ARG POSTGIS_VERSION
 
-FROM timescale/timescaledb:${TIMESCALEDB_VERSION}-pg${PG_MAJOR}
+############################
+# Build tools binaries in separate image
+############################
+ARG GO_VERSION=1.18.7
+FROM golang:${GO_VERSION}-alpine AS tools
+
+ENV TOOLS_VERSION 0.8.1
+
+RUN apk update && apk add --no-cache git gcc musl-dev \
+    && go install github.com/timescale/timescaledb-tune/cmd/timescaledb-tune@latest \
+    && go install github.com/timescale/timescaledb-parallel-copy/cmd/timescaledb-parallel-copy@latest
+
+############################
+# Now build image and copy in tools
+############################
+FROM postgres:"$PG_MAJOR"-bookworm
+ARG PG_MAJOR
+ARG TIMESCALEDB_VERSION
 ARG POSTGIS_VERSION
 
-RUN set -eux \
-    && apk add --no-cache --virtual .fetch-deps \
-        ca-certificates \
-        openssl \
-        tar \
-    \
-    && wget -O postgis.tar.gz "https://github.com/postgis/postgis/archive/${POSTGIS_VERSION}.tar.gz" \
-    && mkdir -p /usr/src/postgis \
-    && tar \
-        --extract \
-        --file postgis.tar.gz \
-        --directory /usr/src/postgis \
-        --strip-components 1 \
-    && rm postgis.tar.gz \
-    \
-    && apk add --no-cache --virtual .build-deps \
-        \
-        gdal-dev \
-        geos-dev \
-        proj-dev \
-        proj-util \
-        sfcgal-dev \
-        \
-        # The upstream variable, '$DOCKER_PG_LLVM_DEPS' contains
-        #  the correct versions of 'llvm-dev' and 'clang' for the current version of PostgreSQL.
-        # This improvement has been discussed in https://github.com/docker-library/postgres/pull/1077
-        $DOCKER_PG_LLVM_DEPS \
-        \
-        autoconf \
-        automake \
-        cunit-dev \
-        file \
-        g++ \
-        gcc \
-        gettext-dev \
-        git \
-        json-c-dev \
-        libtool \
-        libxml2-dev \
-        make \
-        pcre2-dev \
-        perl \
-        protobuf-c-dev \
-    \
-# build PostGIS - with Link Time Optimization (LTO) enabled
-    && cd /usr/src/postgis \
-    && gettextize \
-    && ./autogen.sh \
-    && ./configure \
-        --enable-lto \
-    && make -j$(nproc) \
-    && make install \
-    \
-# add .postgis-rundeps
-    && apk add --no-cache --virtual .postgis-rundeps \
-        \
-        gdal \
-        geos \
-        proj \
-        sfcgal \
-        \
-        json-c \
-        libstdc++ \
-        pcre2 \
-        protobuf-c \
-        \
-        # ca-certificates: for accessing remote raster files
-        #   fix https://github.com/postgis/docker-postgis/issues/307
-        ca-certificates \
-# clean
-    && cd / \
-    && rm -rf /usr/src/postgis \
-    && apk del .fetch-deps .build-deps \
+COPY infra/docker/database/docker-entrypoint-initdb.d/* /docker-entrypoint-initdb.d/
+COPY --from=tools /go/bin/* /usr/local/bin/
+
+RUN \
+    apt-get update && \
+    apt-get install -y \
+        wget \
+        lsb-release && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install TimescaleDB extension
+RUN echo "deb https://packagecloud.io/timescale/timescaledb/debian/ $(lsb_release -c -s) main"
+RUN echo "deb https://packagecloud.io/timescale/timescaledb/debian/ $(lsb_release -c -s) main" > /etc/apt/sources.list.d/timescaledb.list
+RUN wget --quiet -O - https://packagecloud.io/timescale/timescaledb/gpgkey | apt-key add -
+
+RUN \
+    TIMESCALEDB_MAJOR=$(echo "$TIMESCALEDB_VERSION" | cut -c1) && \
+    apt-get update && \
+    apt-get install -y \
+        timescaledb-"$TIMESCALEDB_MAJOR"-postgresql-"$PG_MAJOR"="$TIMESCALEDB_VERSION"* \
+        timescaledb-"$TIMESCALEDB_MAJOR"-loader-postgresql-"$PG_MAJOR"="$TIMESCALEDB_VERSION"* && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN for file in $(find /usr/share/postgresql -name 'postgresql.conf.sample'); do \
+        # We want timescaledb to be loaded in this image by every created cluster
+        sed -r -i "s/[#]*\s*(shared_preload_libraries)\s*=\s*'(.*)'/\1 = 'timescaledb,\2'/;s/,'/'/" "$file" \
+    done
+
+# Install PostGIS extension
+RUN \
+    POSTGIS_MAJOR=$(echo "$POSTGIS_VERSION" | cut -c1) && \
+    apt update && \
+    apt install -y --no-install-recommends \
+        postgresql-"$PG_MAJOR"-postgis-"$POSTGIS_MAJOR"="$POSTGIS_VERSION"* \
+        postgresql-"$PG_MAJOR"-postgis-"$POSTGIS_MAJOR"-scripts  && \
+    rm -rf /var/lib/apt/lists/*

infra/docker/database/upgrade_pg.Dockerfile

@@ -52,6 +52,11 @@ RUN \
     apt-get install -y timescaledb-$TIMESCALEDB_MAJOR-postgresql-$FROM_PG_MAJOR=$TIMESCALEDB_VERSION* timescaledb-$TIMESCALEDB_MAJOR-loader-postgresql-$FROM_PG_MAJOR=$TIMESCALEDB_VERSION* && \
     apt-get install -y timescaledb-$TIMESCALEDB_MAJOR-postgresql-$TO_PG_MAJOR=$TIMESCALEDB_VERSION* timescaledb-$TIMESCALEDB_MAJOR-loader-postgresql-$TO_PG_MAJOR=$TIMESCALEDB_VERSION*
 
+RUN for file in $(find /usr/share/postgresql -name 'postgresql.conf.sample'); do \
+    # We want timescaledb to be loaded in this image by every created cluster
+    sed -r -i "s/[#]*\s*(shared_preload_libraries)\s*=\s*'(.*)'/\1 = 'timescaledb,\2'/;s/,'/'/" "$file" \
+done
+
 # Install PostGIS extension in both versions of Postgres
 RUN apt update
 RUN \

infra/remote/docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3"
 services:
   db:
     image: ghcr.io/mtes-mct/monitorfish/monitorfish-database:$DATABASE_VERSION