POC: HTTP reverse proxy cache

.docker-hub/varnish/Dockerfile

@@ -0,0 +1,13 @@
+ARG VERSION=7.5.0
+
+FROM varnish:${VERSION}
+
+USER root
+
+RUN set -e; \
+    apt-get update; \
+    apt-get -y install prometheus-varnish-exporter;
+
+RUN rm -rf /var/lib/apt/lists/*;
+
+USER varnish

.github/workflows/reusable-build-and-push.yml

@@ -95,6 +95,18 @@ jobs:
           cache-from: type=gha,scope=print
           cache-to: type=gha,scope=print,mode=max
 
+      - name: Build and push varnish docker image
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          file: .docker-hub/varnish/Dockerfile
+          tags: |
+            ${{ ((inputs.tag != '') && format('{0}/ecamp3-varnish:{1}', vars.DOCKER_HUB_USERNAME, inputs.tag) || '') }}
+            ${{ vars.DOCKER_HUB_USERNAME }}/ecamp3-varnish:${{ inputs.sha }}
+          context: .
+          cache-from: type=gha,scope=print
+          cache-to: type=gha,scope=print,mode=max
+
       - name: Build and push db-backup-restore docker image
         uses: docker/build-push-action@v5
         with:

.github/workflows/reusable-dev-deployment.yml

@@ -93,6 +93,7 @@ jobs:
             --set ingress.basicAuth.enabled=${{ vars.BASIC_AUTH_ENABLED || false  }} \
             --set ingress.basicAuth.username=${{ secrets.BASIC_AUTH_USERNAME }} \
             --set ingress.basicAuth.password='${{ secrets.BASIC_AUTH_PASSWORD }}' \
+            --set apiCache.enabled=${{ vars.API_CACHE_ENABLED || false }} \
             --set mail.dummyEnabled=true \
             --set postgresql.url='${{ secrets.POSTGRES_URL }}/ecamp3${{ inputs.name }}?sslmode=require' \
             --set postgresql.adminUrl='${{ secrets.POSTGRES_ADMIN_URL }}/ecamp3${{ inputs.name }}?sslmode=require' \

.github/workflows/reusable-e2e-tests-run.yml

@@ -49,7 +49,7 @@ jobs:
             docker-compose-
 
       # start necessary containers
-      - run: docker compose up -d php caddy frontend pdf print browserless database docker-host
+      - run: docker compose up -d php caddy frontend pdf print browserless database docker-host http-cache mail
 
       - uses: cypress-io/github-action@v5
         with:

.github/workflows/reusable-stage-prod-deployment.yml

@@ -52,6 +52,7 @@ jobs:
             --set ingress.basicAuth.enabled=${{ vars.BASIC_AUTH_ENABLED || false  }} \
             --set ingress.basicAuth.username=${{ secrets.BASIC_AUTH_USERNAME }} \
             --set ingress.basicAuth.password='${{ secrets.BASIC_AUTH_PASSWORD }}' \
+            --set apiCache.enabled=${{ vars.API_CACHE_ENABLED || false }} \
             --set mail.dsn=${{ secrets.MAILER_DSN }} \
             --set postgresql.url='${{ secrets.POSTGRES_URL }}/${{ secrets.DB_NAME }}?sslmode=require' \
             --set postgresql.dropDBOnUninstall=false \

.helm/.env-example

@@ -5,6 +5,8 @@ domain=ecamp3.ch
 POSTGRES_URL=
 POSTGRES_ADMIN_URL=
 
+API_CACHE_ENABLED=false
+
 BASIC_AUTH_ENABLED=false
 BASIC_AUTH_USERNAME=test
 BASIC_AUTH_PASSWORD=test

.helm/build-images.sh

@@ -39,6 +39,10 @@ print_image_tag="${docker_hub_account}/ecamp3-print:${version}"
 docker build "$REPO_DIR" -f "$REPO_DIR"/.docker-hub/print/Dockerfile $print_sentry_build_args -t "$print_image_tag"
 docker push "$print_image_tag"
 
+varnish_image_tag="${docker_hub_account}/ecamp3-varnish:${version}"
+docker build "$REPO_DIR" -f "$REPO_DIR"/.docker-hub/varnish/Dockerfile -t "$varnish_image_tag"
+docker push "$varnish_image_tag"
+
 export REPO_OWNER=${docker_hub_account}
 export VERSION=${version}
 db_backup_restore_docker_compose_path="$REPO_DIR"/.helm/ecamp3/files/db-backup-restore-image/docker-compose.yml

.helm/deploy-to-cluster.sh

@@ -42,6 +42,7 @@ for i in 1; do
   values="$values --set ingress.basicAuth.enabled=$BASIC_AUTH_ENABLED"
   values="$values --set ingress.basicAuth.username=$BASIC_AUTH_USERNAME"
   values="$values --set ingress.basicAuth.password=$BASIC_AUTH_PASSWORD"
+  values="$values --set apiCache.enabled=$API_CACHE_ENABLED"
   values="$values --set postgresql.enabled=false"
   values="$values --set postgresql.url=$POSTGRES_URL/ecamp3$instance_name-"$i"?sslmode=require"
   values="$values --set postgresql.adminUrl=$POSTGRES_ADMIN_URL/ecamp3$instance_name-"$i"?sslmode=require"
@@ -98,6 +99,8 @@ for i in 1; do
     values="$values --set $imagespec.image.repository=docker.io/${docker_hub_account}/ecamp3-api-$imagespec"
   done
 
+  values="$values --set apiCache.image.repository=docker.io/${docker_hub_account}/ecamp3-varnish"
+
   values="$values --set postgresql.dbBackupRestoreImage.pullPolicy=$pull_policy"
   values="$values --set postgresql.dbBackupRestoreImage.repository=docker.io/${docker_hub_account}/ecamp3-db-backup-restore"
 

.helm/ecamp3/files/vcl

@@ -0,0 +1 @@
+../../../api/docker/varnish/vcl

.helm/ecamp3/templates/_helpers.tpl

@@ -82,6 +82,19 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- end }}
 {{- end }}
 
+{{/*
+Name for all HTTP cache-related resources.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "apiCache.name" -}}
+{{- $name := default .Chart.Name .Values.chartNameOverride }}
+{{- if contains $name (include "app.name" .) }}
+{{- printf "%s-api-cache" (include "app.name" .) | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s-api-cache" (include "app.name" .) $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+
 {{/*
 Name for all db_backup_job releated resources.
 We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
@@ -227,6 +240,14 @@ app.kubernetes.io/name: {{ include "chart.name" . }}-browserless
 {{ include "app.commonSelectorLabels" . }}
 {{- end }}
 
+{{/*
+Selector labels for HTTP Cache
+*/}}
+{{- define "apiCache.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "chart.name" . }}-api-cache
+{{ include "app.commonSelectorLabels" . }}
+{{- end }}
+
 {{/*
 Selector labels for db-backup-job
 */}}

.helm/ecamp3/templates/api_cache_deployment.yaml

@@ -0,0 +1,116 @@
+{{- if .Values.apiCache.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "apiCache.name" . }}
+  labels:
+    {{- include "apiCache.selectorLabels" . | nindent 4 }}
+    {{- include "app.commonLabels" . | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "apiCache.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "apiCache.selectorLabels" . | nindent 8 }}
+      annotations:
+        checksum/vclConfigmap: {{ include (print $.Template.BasePath "/api_cache_vcl_configmap.yaml") . | sha256sum }}
+        rollme: {{ .Values.imageTag | quote }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "app.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      enableServiceLinks: false
+      containers:
+        - name: {{ .Chart.Name }}-api-cache-varnishd
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.apiCache.image.repository }}:{{ .Values.apiCache.image.tag | default .Values.imageTag }}"
+          imagePullPolicy: {{ .Values.apiCache.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.apiCache.varnishHttpPort }}
+              protocol: TCP
+            - name: purge
+              containerPort: {{ .Values.apiCache.varnishPurgePort }}
+              protocol: TCP
+          env:
+            - name: VARNISH_SIZE
+              value: "{{ .Values.apiCache.varnishSize }}"
+            - name: VARNISH_HTTP_PORT
+              value: "{{ .Values.apiCache.varnishHttpPort }}"
+            - name: COOKIE_PREFIX
+              value: {{ include "api.cookiePrefix" . | quote }}
+          args:
+            - -a
+            - {{ printf ":%d,HTTP" (.Values.apiCache.varnishPurgePort | int) }}
+            - -p
+            - http_max_hdr=96
+          resources:
+            {{- toYaml .Values.apiCache.resources | nindent 12 }}
+          volumeMounts:
+            - name: vcl-configmap
+              mountPath: /etc/varnish
+            - name: vsm
+              mountPath: /var/lib/varnish
+        {{- if .Values.apiCache.logging.enabled }}
+        - name: {{ .Chart.Name }}-api-cache-varnishncsa
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.apiCache.image.repository }}:{{ .Values.apiCache.image.tag | default .Values.imageTag }}"
+          imagePullPolicy: {{ .Values.apiCache.image.pullPolicy }}
+          command:
+          - varnishncsa
+          - -b
+          - -c
+          {{- if .Values.apiCache.logging.customOutputJsonFormat }}
+          - -j
+          {{- end }}
+          {{- if .Values.apiCache.logging.customOutput }}
+          - -F
+          - {{ .Values.apiCache.logging.customOutput | squote }}
+          {{- end }}
+          - -t
+          - {{ .Values.apiCache.logging.timeout | quote }}
+          resources:
+            {{- toYaml .Values.apiCache.logging.resources | nindent 12 }}
+          volumeMounts:
+          - name: vsm
+            mountPath: /var/lib/varnish
+        {{- end }}
+        {{- if .Values.apiCache.prometheus.enabled }}
+        - name: {{ .Chart.Name }}-api-cache-prometheus-exporter
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.apiCache.image.repository }}:{{ .Values.apiCache.image.tag | default .Values.imageTag }}"
+          imagePullPolicy: {{ .Values.apiCache.image.pullPolicy }}
+          ports:
+          - name: metrics
+            containerPort: {{ .Values.apiCache.prometheus.port }}
+            protocol: TCP
+          resources:
+            {{- toYaml .Values.apiCache.prometheus.resources | nindent 12 }}
+          command:
+          - prometheus-varnish-exporter
+          - -web.telemetry-path
+          - "{{ .Values.apiCache.prometheus.path }}"
+          - -web.listen-address
+          - ":{{ .Values.apiCache.prometheus.port }}"
+          volumeMounts:
+          - name: vsm
+            mountPath: /var/lib/varnish
+        {{- end }}
+      volumes:
+      - name: vcl-configmap
+        configMap:
+          name: {{ include "apiCache.name" . }}-vcl-configmap
+      - name: vsm
+        emptyDir:
+          medium: Memory
+{{- end }}

.helm/ecamp3/templates/api_cache_service.yaml

@@ -0,0 +1,26 @@
+{{- if .Values.apiCache.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "apiCache.name" . }}
+  labels:
+    {{- include "apiCache.selectorLabels" . | nindent 4 }}
+    {{- include "app.commonLabels" . | nindent 4 }}
+spec:
+  type: {{ .Values.apiCache.service.type }}
+  ports:
+    - port: {{ .Values.apiCache.service.ports.http }}
+      targetPort: http
+      protocol: TCP
+      name: http
+    - port: {{ .Values.apiCache.service.ports.purge }}
+      targetPort: purge
+      protocol: TCP
+      name: purge
+    - port: {{ .Values.apiCache.prometheus.port }}
+      targetPort: metrics
+      protocol: TCP
+      name: metrics
+  selector:
+    {{- include "apiCache.selectorLabels" . | nindent 4 }}
+{{- end }}

.helm/ecamp3/templates/api_cache_vcl_configmap.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.apiCache.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "apiCache.name" . }}-vcl-configmap
+  labels:
+    {{- include "apiCache.selectorLabels" . | nindent 4 }}
+    {{- include "app.commonLabels" . | nindent 4 }}
+data:
+# includes all files except the ones starting with _
+{{ (.Files.Glob "files/vcl/[!_]*").AsConfig | indent 2 }}
+  # override backend config
+  _config.vcl: |-
+    backend default {
+      .host = "{{ include "api.name" .}}";
+      .port = "{{ .Values.api.service.port }}";
+    }
+{{- end }}

.helm/ecamp3/templates/api_configmap.yaml

@@ -22,3 +22,9 @@ data:
   SENTRY_API_DSN: {{ "" | quote }}
   {{- end }}
   FRONTEND_BASE_URL: {{ include "frontend.url" . | quote }}
+  API_CACHE_ENABLED: {{ .Values.apiCache.enabled | quote }}
+  {{- if .Values.apiCache.enabled }}
+  VARNISH_API_URL: {{  printf "%s:%d" (include "apiCache.name" .) (.Values.apiCache.service.ports.purge | int)  | quote }}
+  {{- else }}
+  VARNISH_API_URL: {{ "" | quote }}
+  {{- end}}

.helm/ecamp3/templates/api_ingress.yaml

@@ -29,7 +29,13 @@ spec:
             pathType: Prefix
             backend:
               service:
+                {{- if .Values.apiCache.enabled }}
+                name: {{ include "apiCache.name" . }}
+                port:
+                  number: {{ .Values.apiCache.service.ports.http }}
+                {{- else }}
                 name: {{ include "api.name" . }}
                 port:
                   number: {{ .Values.api.service.port }}
+                {{- end }}
 {{- end }}

.helm/ecamp3/templates/print_ingress.yaml

@@ -12,6 +12,7 @@ metadata:
     {{- end }}
     {{- include "ingress.basicAuth.annotations" . | nindent 4 }}
     {{- if not (.Values.print.ingress.readTimeoutSeconds | empty) }}
+    nginx.ingress.kubernetes.io/use-regex: "true"
     nginx.ingress.kubernetes.io/proxy-read-timeout: {{ .Values.print.ingress.readTimeoutSeconds | quote }}
     {{- end }}
 spec:

.helm/ecamp3/values.yaml

@@ -222,7 +222,48 @@ ingress:
   className: nginx
   tls:
 
-
+apiCache:
+  enabled: false
+  image:
+    repository: "docker.io/ecamp/ecamp3-varnish"
+    pullPolicy: IfNotPresent
+    # Overrides the image tag whose shared default is .Values.imageTag
+    tag:
+  service:
+    type: ClusterIP
+    ports:
+      http: 3000
+      purge: 3001
+  varnishSize: 50M
+  varnishHttpPort: 8080
+  varnishPurgePort: 8081
+  resources:
+    requests:
+      cpu: 10m
+      memory: 100Mi
+  logging:
+    enabled: true
+    customOutput: '{ "received_at": "%t", "varnish_side": "%{Varnish:side}x", "method": "%m", "url": "%U", "query": "%q", "response_bytes": %b, "time_taken": %D, "status": %s, "handling": "%{Varnish:handling}x", "response_reason": "%{VSL:RespReason}x", "fetch_error": "%{VSL:FetchError}x" }'
+    customOutputJsonFormat: true
+    # Timeout before returning error on initial VSM connection.
+    # If set the VSM connection is retried every 0.5 seconds for this many seconds.
+    # If zero the connection is attempted only once and will fail immediately if unsuccessful.
+    # If set to "off", the connection will not fail, allowing the utility to start and wait indefinetely for the Varnish instance to appear.
+    # Defaults to "off" in this case.
+    timeout: "off"
+    resources:
+      requests:
+        cpu: 10m
+        memory: 20Mi
+  prometheus:
+    enabled: true
+    path: "/metrics"
+    port: 9131
+    resources:
+      requests:
+        cpu: 10m
+        memory: 20Mi
+
 autoscaling:
   enabled: false
   minReplicas: 1

.ops/observability/prometheus-values-dev.yml

@@ -31,6 +31,16 @@ prometheus:
           - default
       endpoints:
         - port: "api-metrics"
+    - name: "varnish"
+      selector:
+        matchLabels:
+          app.kubernetes.io/instance: ecamp3-dev
+          app.kubernetes.io/name: ecamp3-api-cache
+      namespaceSelector:
+        matchNames:
+          - default
+      endpoints:
+        - port: "api-cache-metrics"
   prometheusSpec:
     storageSpec:
       volumeClaimTemplate: