Issue gravitystorm#2913: Refactor docker structure [docker]

docker-compose.yml → docker/docker-compose.yml

@@ -1,34 +1,41 @@
-version: '2'
+version: '3.2'
+
 services:
+
   kosmtik:
     image: kosmtik:v1
     build:
-      context: .
-      dockerfile: Dockerfile
+      context: images/kosmtik
+      args:
+        - APP_UID=1000
     volumes:
-      - .:/openstreetmap-carto
+      - ../:/openstreetmap-carto
     depends_on:
       - db
     ports:
       - "6789:6789"
     environment:
       - PGHOST=db
       - PGUSER=postgres
+
   db:
     image: db:v1
     build:
-      context: .
-      dockerfile: Dockerfile.db
+      context: images/db
+    volumes:
+      - db:/openstreetmap-carto
     environment:
       - PG_WORK_MEM
       - PG_MAINTENANCE_WORK_MEM
+
   import:
     image: import:v1
     build:
-      context: .
-      dockerfile: Dockerfile.import
+      context: images/import
+      args:
+        - APP_UID=1000
     volumes:
-      - .:/openstreetmap-carto
+      - ../:/openstreetmap-carto
     depends_on:
       - db
     environment:
@@ -37,5 +44,8 @@ services:
       - PG_WORK_MEM
       - PG_MAINTENANCE_WORK_MEM
       - OSM2PGSQL_CACHE
-      - OSM2PGSQL_NUMPROC
+      - OSM2PGSQL_NUMPROC=8
       - OSM2PGSQL_DATAFILE
+
+volumes:
+  db:

docker/images/db/Dockerfile

@@ -0,0 +1,3 @@
+FROM mdillon/postgis:9.6
+
+COPY tune-postgis.sh /docker-entrypoint-initdb.d

Dockerfile.import → docker/images/import/Dockerfile

@@ -1,15 +1,26 @@
 FROM debian:stretch
 
 # Install tools, utilities and osm2pgsql
+
 RUN set -x && \
     DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
     apt-get install -y --no-install-recommends osm2pgsql postgresql-client && \
     rm -rf /var/lib/apt/lists/*
 
-COPY openstreetmap-carto.style /
+COPY entrypoint.sh /
 
 RUN mkdir -p /openstreetmap-carto
 WORKDIR /openstreetmap-carto
 
-CMD sh scripts/docker-startup.sh import
+# Create auxiliary user for running commands
+
+ARG APP_UID=1000
+ARG APP_USER="user"
+
+RUN set -x && \
+    useradd -u ${APP_UID} -m -s /bin/bash ${APP_USER}
+
+USER ${APP_USER}
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/images/import/entrypoint.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Testing if database is ready
+i=1
+MAXCOUNT=60
+echo "Waiting for PostgreSQL to be running"
+while [ $i -le $MAXCOUNT ]
+do
+  pg_isready -q && echo "PostgreSQL running" && break
+  sleep 2
+  i=$((i+1))
+done
+test $i -gt $MAXCOUNT && echo "Timeout while waiting for PostgreSQL to be running"  && exit 1
+
+# Creating default database
+psql -c "SELECT 1 FROM pg_database WHERE datname = 'gis';" | grep -q 1 || createdb gis && \
+psql -d gis -c 'CREATE EXTENSION IF NOT EXISTS postgis;' && \
+psql -d gis -c 'CREATE EXTENSION IF NOT EXISTS hstore;' && \
+
+# Creating default import settings file editable by user and passing values for osm2pgsql
+if [ ! -e "docker/.env" ]; then
+  cat > docker/.env <<EOF
+# Environment settings for importing to a Docker container database
+PG_WORK_MEM=${PG_WORK_MEM:-16MB}
+PG_MAINTENANCE_WORK_MEM=${PG_MAINTENANCE_WORK_MEM:-256MB}
+OSM2PGSQL_CACHE=${OSM2PGSQL_CACHE:-512}
+OSM2PGSQL_NUMPROC=${OSM2PGSQL_NUMPROC:-1}
+OSM2PGSQL_DATAFILE=${OSM2PGSQL_DATAFILE:-data.osm.pbf}
+EOF
+  chmod a+rw docker/.env
+  export OSM2PGSQL_CACHE=${OSM2PGSQL_CACHE:-512}
+  export OSM2PGSQL_NUMPROC=${OSM2PGSQL_NUMPROC:-1}
+  export OSM2PGSQL_DATAFILE=${OSM2PGSQL_DATAFILE:-data.osm.pbf}
+fi
+
+# Importing data to a database
+osm2pgsql \
+  --cache $OSM2PGSQL_CACHE \
+  --number-processes $OSM2PGSQL_NUMPROC \
+  --hstore \
+  --multi-geometry \
+  --database gis \
+  --style openstreetmap-carto.style \
+  --tag-transform-script openstreetmap-carto.lua \
+  $OSM2PGSQL_DATAFILE

docker/images/kosmtik/Dockerfile

@@ -0,0 +1,62 @@
+FROM debian:stretch
+
+# Install main services, tools and utilities
+ARG BUILD_DEPS="build-essential libmapnik-dev"
+RUN set -x && \
+    DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install --no-install-recommends -y \
+            unzip curl ca-certificates gnupg python postgresql-client gnupg mapnik-utils jq \
+            fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted ttf-unifont fonts-hanazono \
+            ${BUILD_DEPS} \
+            && \
+    curl -sL https://deb.nodesource.com/setup_8.x -o /tmp/nodejs-8-setup && \
+    bash /tmp/nodejs-8-setup && \
+    rm /tmp/nodejs-8-setup && \
+    apt-get install --no-install-recommends -y \
+            nodejs && \
+    npm install -g npm@latest
+
+# Install Kosmtik and plugins
+RUN set -x && \
+    npm install --unsafe-perm -g kosmtik && \
+    cd /usr/lib/node_modules/kosmtik/ && \
+    kosmtik plugins --install kosmtik-overpass-layer \
+                    --install kosmtik-fetch-remote \
+                    --install kosmtik-overlay \
+                    --install kosmtik-open-in-josm \
+                    --install kosmtik-map-compare \
+                    --install kosmtik-osm-data-overlay \
+                    --install kosmtik-mapnik-reference \
+                    --install kosmtik-geojson-overlay \
+                    && \
+    cp /root/.config/kosmtik.yml /tmp/.kosmtik-config.yml
+
+# Install additional fonts
+COPY noto-fonts.json /usr/local/etc/
+COPY get-noto-fonts.sh /usr/local/bin/
+RUN set -x && \
+    get-noto-fonts.sh
+
+# Clean up the system
+RUN set -x && \
+    DEBIAN_FRONTEND=noninteractive && \
+    apt-get autoremove --purge -y ${BUILD_DEPS} && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY entrypoint.sh /
+
+RUN mkdir -p /openstreetmap-carto
+WORKDIR /openstreetmap-carto
+
+# Create auxiliary user for running commands
+
+ARG APP_UID=1000
+ARG APP_USER="user"
+
+RUN set -x && \
+    useradd -u ${APP_UID} -m -s /bin/bash ${APP_USER}
+
+USER ${APP_USER}
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/images/kosmtik/entrypoint.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -x
+
+# Testing if database is ready
+i=1
+MAXCOUNT=60
+echo "Waiting for PostgreSQL to be running"
+while [ $i -le $MAXCOUNT ]
+do
+  pg_isready -q && echo "PostgreSQL running" && break
+  sleep 2
+  i=$((i+1))
+done
+test $i -gt $MAXCOUNT && echo "Timeout while waiting for PostgreSQL to be running"  && exit 1
+
+# Downloading needed shapefiles
+python scripts/get-shapefiles.py -n
+
+# Creating default Kosmtik settings file
+if [ ! -e ".kosmtik-config.yml" ]; then
+  cp /tmp/.kosmtik-config.yml .kosmtik-config.yml
+fi
+export KOSMTIK_CONFIGPATH=".kosmtik-config.yml"
+
+# Starting Kosmtik
+kosmtik serve project.mml --host 0.0.0.0
+# It needs Ctrl+C to be interrupted

docker/images/kosmtik/get-noto-fonts.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# Fetches additional fonts
+if output=$(jq --stream 'select(length == 2) | [(.[0][] | select(strings)), .[1]] | join("/")' < /usr/local/etc/noto-fonts.json); then
+  while read -r font; do
+    font=${font:1:-1}
+    curl -L https://github.com/${font}?raw=true -o /usr/share/fonts/truetype/noto/$(basename ${font})
+  done <<< "$output";
+fi

docker/images/kosmtik/noto-fonts.json

@@ -0,0 +1,10 @@
+{
+  "googlei18n/noto-emoji/blob/master/fonts": [
+    "NotoColorEmoji.ttf",
+    "NotoEmoji-Regular.ttf"
+  ],
+  "googlei18n/noto-fonts/blob/master/hinted": [
+    "NotoSansArabicUI-Regular.ttf",
+    "NotoSansArabicUI-Bold.ttf"
+  ]
+}