From ad237cdd726385f6d4b64ccd6a42d9643ed1b381 Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Thu, 20 Jun 2019 03:51:40 +0600 Subject: [PATCH 01/23] Add Dockerfile for rails app added Dockerfile for the ruby on rails app inside a separate docker directory. this will build a image from the official ruby (specifically ruby:2.5-slim). --- docker/Dockerfile | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 docker/Dockerfile diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000000..bea079573c --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,25 @@ +FROM ruby:2.5-slim + +#install packages +#fixes dpkg man page softlink error while installing postgresql-client [source: https://github.com/debuerreotype/debuerreotype/issues/10#issuecomment-438342078] +RUN mkdir -p /usr/share/man/man1 && mkdir -p /usr/share/man/man7 +RUN apt-get update && apt-get install curl -y +RUN curl -sL https://deb.nodesource.com/setup_10.x -o nodesource_setup.sh && bash nodesource_setup.sh && rm nodesource_setup.sh +RUN apt-get install -y --no-install-recommends ruby-dev libarchive-dev libmagickwand-dev libxml2-dev libxslt1-dev build-essential libpq-dev libsasl2-dev imagemagick libffi-dev locales postgresql-client-9.6 nodejs && apt-get clean && rm -rf /var/lib/apt/lists/* +RUN npm install yarn -g + +# Setup app location +RUN mkdir -p /app +WORKDIR /app + +# Install gems +ADD Gemfile* /app/ +RUN bundle install --jobs=2 + +# Setup local +RUN sed -i -e 's/# en_GB.UTF-8 UTF-8/en_GB.UTF-8 UTF-8/' /etc/locale.gen && \ + echo 'LANG="en_GB.UTF-8"'>/etc/default/locale && \ + dpkg-reconfigure --frontend=noninteractive locales && \ + update-locale LANG=en_GB.UTF-8 + +ENV LANG en_GB.UTF-8 From 3fe73f04b483c80ef0f5273fb79cc85982abb4f2 Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Thu, 20 Jun 2019 04:08:27 +0600 Subject: [PATCH 02/23] Modify Dockerfile to fix comment for postgresql-client installation error postgresql-client throws a dpkg error when it can't create a softlink in for man page and aborts installation. found a solve at stack overflow and modified Dockerfile to add a comment to mention the problem with slove source. --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index bea079573c..044d014628 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ FROM ruby:2.5-slim #install packages -#fixes dpkg man page softlink error while installing postgresql-client [source: https://github.com/debuerreotype/debuerreotype/issues/10#issuecomment-438342078] +#fixes dpkg man page softlink error while installing postgresql-client [source: https://stackoverflow.com/a/52655008/5350059] RUN mkdir -p /usr/share/man/man1 && mkdir -p /usr/share/man/man7 RUN apt-get update && apt-get install curl -y RUN curl -sL https://deb.nodesource.com/setup_10.x -o nodesource_setup.sh && bash nodesource_setup.sh && rm nodesource_setup.sh From 4eeb36ac38533e1b3e08cb75e47d01b489148725 Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Thu, 20 Jun 2019 17:37:54 +0600 Subject: [PATCH 03/23] Modify Dockerfile to separate apt cache clean, npm comment, and removed jobs from bundle install modified Dockerfile to separate apt-get cache cleaning command, added a comment explaining why and how npm is added and removed --jobs flag from bundle install command --- docker/Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 044d014628..95401b6e39 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,8 +4,11 @@ FROM ruby:2.5-slim #fixes dpkg man page softlink error while installing postgresql-client [source: https://stackoverflow.com/a/52655008/5350059] RUN mkdir -p /usr/share/man/man1 && mkdir -p /usr/share/man/man7 RUN apt-get update && apt-get install curl -y +RUN apt-get clean && rm -rf /var/lib/apt/lists/* +#npm is not available in Debian repo so following official instruction [source: https://github.com/nodesource/distributions/blob/master/README.md#debinstall] RUN curl -sL https://deb.nodesource.com/setup_10.x -o nodesource_setup.sh && bash nodesource_setup.sh && rm nodesource_setup.sh -RUN apt-get install -y --no-install-recommends ruby-dev libarchive-dev libmagickwand-dev libxml2-dev libxslt1-dev build-essential libpq-dev libsasl2-dev imagemagick libffi-dev locales postgresql-client-9.6 nodejs && apt-get clean && rm -rf /var/lib/apt/lists/* +RUN apt-get install -y --no-install-recommends ruby-dev libarchive-dev libmagickwand-dev libxml2-dev libxslt1-dev build-essential libpq-dev libsasl2-dev imagemagick libffi-dev locales postgresql-client-9.6 nodejs +RUN apt-get clean && rm -rf /var/lib/apt/lists/* RUN npm install yarn -g # Setup app location @@ -14,7 +17,7 @@ WORKDIR /app # Install gems ADD Gemfile* /app/ -RUN bundle install --jobs=2 +RUN bundle install # Setup local RUN sed -i -e 's/# en_GB.UTF-8 UTF-8/en_GB.UTF-8 UTF-8/' /etc/locale.gen && \ From caaa3507b910ef40fe854ce30d272a4ca8e2c01a Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Thu, 20 Jun 2019 17:41:14 +0600 Subject: [PATCH 04/23] Add docker_postgres.sh for PostgreSQL extension and functions added docker_postgres.sh for PostgreSQL extension and functions that installs PostgreSQL Btree-gist Extension, compiles libpgosm.so and installs maptile_for_point, tile_for_point and xid_to_int4 functions --- db/docker_postgres.sh | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 db/docker_postgres.sh diff --git a/db/docker_postgres.sh b/db/docker_postgres.sh new file mode 100644 index 0000000000..9da4956ec4 --- /dev/null +++ b/db/docker_postgres.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e +psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE EXTENSION btree_gist" openstreetmap +make -C db/functions libpgosm.so +psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE FUNCTION maptile_for_point(int8, int8, int4) RETURNS int4 AS '/db/functions/libpgosm', 'maptile_for_point' LANGUAGE C STRICT" openstreetmap +psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '/db/functions/libpgosm', 'tile_for_point' LANGUAGE C STRICT" openstreetmap +psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE FUNCTION xid_to_int4(xid) RETURNS int4 AS '/db/functions/libpgosm', 'xid_to_int4' LANGUAGE C STRICT" openstreetmap From b6d373faf011e5bc20b60e6616bf6039d50f81f5 Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Thu, 20 Jun 2019 17:49:21 +0600 Subject: [PATCH 05/23] Add Dockerfile.postgres for postgres app added Dockerfile.postgres for the postgres database app. this will build a image from the official postgres (specifically postgres:9.4) image. --- docker/Dockerfile.postgres | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 docker/Dockerfile.postgres diff --git a/docker/Dockerfile.postgres b/docker/Dockerfile.postgres new file mode 100644 index 0000000000..080fdff8a3 --- /dev/null +++ b/docker/Dockerfile.postgres @@ -0,0 +1,8 @@ +FROM postgres:9.4 + +ADD /db/docker_postgres.sh docker-entrypoint-initdb.d/docker_postgres.sh +ADD db/functions/ db/functions/ + +RUN apt-get update +RUN apt-get install -y make build-essential postgresql-server-dev-all +RUN apt-get clean && rm -rf /var/lib/apt/lists/* From 30ad01e1b40d7a73bb95391609b0ada30cd57100 Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Thu, 20 Jun 2019 17:51:53 +0600 Subject: [PATCH 06/23] Add docker-compose.yml to build the complete app --- docker/docker-compose.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 docker/docker-compose.yml diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000000..23d81e1a07 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,23 @@ +version: '2' +services: + web: + image: openstreetmap-website:v1 + build: + context: .. + dockerfile: docker/Dockerfile + volumes: + - .:/app + ports: + - "3000:3000" + command: bundle exec rails s -p 3000 -b '0.0.0.0' + depends_on: + - db + db: + image: openstreetmap-db:v1 + build: + context: .. + dockerfile: docker/Dockerfile.postgres + ports: + - "5432:5432" + environment: + POSTGRES_DB: openstreetmap From 447c55e9a97def4e176508928973acda35803a6b Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Mon, 22 Jul 2019 03:03:07 +0600 Subject: [PATCH 07/23] modify Dockerfile.postgres to fix script path --- docker/Dockerfile.postgres | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.postgres b/docker/Dockerfile.postgres index 080fdff8a3..f09cc29356 100644 --- a/docker/Dockerfile.postgres +++ b/docker/Dockerfile.postgres @@ -1,6 +1,6 @@ FROM postgres:9.4 -ADD /db/docker_postgres.sh docker-entrypoint-initdb.d/docker_postgres.sh +ADD db/docker_postgres.sh docker-entrypoint-initdb.d/docker_postgres.sh ADD db/functions/ db/functions/ RUN apt-get update From 0cf0d0cf2fedcb75d2d26d2e9b1afab775f370c8 Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Mon, 22 Jul 2019 03:03:34 +0600 Subject: [PATCH 08/23] modify Dockerfile.postgres to upgrade postgres image to 11 --- docker/Dockerfile.postgres | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.postgres b/docker/Dockerfile.postgres index f09cc29356..13066abde0 100644 --- a/docker/Dockerfile.postgres +++ b/docker/Dockerfile.postgres @@ -1,4 +1,4 @@ -FROM postgres:9.4 +FROM postgres:11 ADD db/docker_postgres.sh docker-entrypoint-initdb.d/docker_postgres.sh ADD db/functions/ db/functions/ From 18ef6b6db13350dba8cca9eae81e4a8a97a1eb89 Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Mon, 22 Jul 2019 03:04:48 +0600 Subject: [PATCH 09/23] modify Dockerfile to change postgresql-client package and install phantomjs via npm modified Dockerfile to changes postgresql-client package from postgresql-client-10 to postgresql-client, and added phantomjs installation via npm --- docker/Dockerfile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 95401b6e39..6fdbb25603 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,15 +1,18 @@ FROM ruby:2.5-slim -#install packages -#fixes dpkg man page softlink error while installing postgresql-client [source: https://stackoverflow.com/a/52655008/5350059] +# install packages +# fixes dpkg man page softlink error while installing postgresql-client [source: https://stackoverflow.com/a/52655008/5350059] RUN mkdir -p /usr/share/man/man1 && mkdir -p /usr/share/man/man7 RUN apt-get update && apt-get install curl -y RUN apt-get clean && rm -rf /var/lib/apt/lists/* + + #npm is not available in Debian repo so following official instruction [source: https://github.com/nodesource/distributions/blob/master/README.md#debinstall] RUN curl -sL https://deb.nodesource.com/setup_10.x -o nodesource_setup.sh && bash nodesource_setup.sh && rm nodesource_setup.sh -RUN apt-get install -y --no-install-recommends ruby-dev libarchive-dev libmagickwand-dev libxml2-dev libxslt1-dev build-essential libpq-dev libsasl2-dev imagemagick libffi-dev locales postgresql-client-9.6 nodejs +RUN apt-get install -y --no-install-recommends ruby-dev libarchive-dev libmagickwand-dev libxml2-dev libxslt1-dev build-essential libpq-dev libsasl2-dev imagemagick libffi-dev locales postgresql-client nodejs RUN apt-get clean && rm -rf /var/lib/apt/lists/* -RUN npm install yarn -g +RUN npm install yarn phantomjs-prebuilt -g --unsafe-perm + # Setup app location RUN mkdir -p /app @@ -18,7 +21,6 @@ WORKDIR /app # Install gems ADD Gemfile* /app/ RUN bundle install - # Setup local RUN sed -i -e 's/# en_GB.UTF-8 UTF-8/en_GB.UTF-8 UTF-8/' /etc/locale.gen && \ echo 'LANG="en_GB.UTF-8"'>/etc/default/locale && \ From f75eea4b16f27149d4de4dd83b6784cd745f70f8 Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Mon, 22 Jul 2019 03:05:22 +0600 Subject: [PATCH 10/23] modify docker-compose.yml to fix context path --- docker/docker-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 23d81e1a07..39a4166e89 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -6,12 +6,14 @@ services: context: .. dockerfile: docker/Dockerfile volumes: - - .:/app + - ..:/app ports: - "3000:3000" command: bundle exec rails s -p 3000 -b '0.0.0.0' depends_on: - db + environment: + DATABASE_URL: postgres://postgres@db:5432 db: image: openstreetmap-db:v1 build: From 0d15f1c3553e01bb3be3c2dccff6efa00e2e2dc6 Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Mon, 22 Jul 2019 03:07:33 +0600 Subject: [PATCH 11/23] add DOCKER.md file with docker setup instructions --- docker/DOCKER.md | 83 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 docker/DOCKER.md diff --git a/docker/DOCKER.md b/docker/DOCKER.md new file mode 100644 index 0000000000..328a13eba9 --- /dev/null +++ b/docker/DOCKER.md @@ -0,0 +1,83 @@ +# Using Docker to run OpenStreetMap + +Using [Docker](https://www.docker.com/) will allow you to install the OpenStreetMap application and all its' dependencies in a container, almost with a single command. + +These instructions gloss over the precise details of the dependencies and their configuration but these can be found in full detail at [INSTALL.md](INSTALL.md). + +The first step is to fork/clone the repo to your local machine. The repository is reasonably large (~150MB) and it's unlikely that you need the full history. If you are happy to wait for it all to download, run: +``` +git clone https://github.com/openstreetmap/openstreetmap-website.git +``` + +To clone only the most recent version (~23MB), instead use a 'shallow clone': + +``` +git clone --depth=1 https://github.com/openstreetmap/openstreetmap-website.git +``` + +Now change working directory to the `openstreetmap-website`: + +``` +cd openstreetmap-website +``` + +### App configuration + +``` +cp config/settings.yml config/settings.local.yml +``` + +### Database + +``` +cp config/example.database.yml config/database.yml +``` + +Set `username` to postgres and `host` to db leave the `password` blank + +### Installation + +In the root directory run: + +``` +docker-compose -f docker/docker-compose.yml up +``` +Now if this is your first time running or you have removed cache this will take some time to complete. So grab tea/coffee and seat tight. Upon successfull build it should show + +### Migrations +While `docker-compose up` is running, open another terminal windows and run: + +``` +docker-compose -f docker/docker-compose.yml exec web bundle exec rake db:migrate +``` + +### Node.js modules +We use Yarn to manage the Node.js modules required for the project.: + +``` +docker-compose -f docker/docker-compose.yml exec web bundle exec rake yarn:install +``` + +Once these are complete you should be able to visit the app at http://localhost:3000 + +If localhost does not work, you can use the IP address of the docker-machine. + +### Tests + +``` +docker-compose -f docker/docker-compose.yml exec web bundle exec rake test:db +``` + +### Bash + +If you want to get onto the web container and run specific commands you can fire up bash via: + +``` +docker-compose -f docker/docker-compose.yml exec web /bin/bash +``` + +Similarly, if you want to get onto the db container use: + +``` +docker-compose -f docker/docker-compose.yml exec db /bin/bash +``` From c4c450f57a5d1fa9bf062debf2b3ce5db600d18a Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Mon, 22 Jul 2019 03:08:01 +0600 Subject: [PATCH 12/23] add .dockerignore file to list ignorable file for docker --- docker/.dockerignore | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docker/.dockerignore diff --git a/docker/.dockerignore b/docker/.dockerignore new file mode 100644 index 0000000000..79ba099601 --- /dev/null +++ b/docker/.dockerignore @@ -0,0 +1,5 @@ +* +!Gemfile +!Gemfile.lock +!db/functions +!db/docker_postgres.sh From 076073bd091b9b3ed5b69e2040310bcbb9d99b41 Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Tue, 23 Jul 2019 18:33:44 +0600 Subject: [PATCH 13/23] modify DOCKER.md to add database population --- docker/DOCKER.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docker/DOCKER.md b/docker/DOCKER.md index 328a13eba9..f88071947b 100644 --- a/docker/DOCKER.md +++ b/docker/DOCKER.md @@ -81,3 +81,12 @@ Similarly, if you want to get onto the db container use: ``` docker-compose -f docker/docker-compose.yml exec db /bin/bash ``` + +### Populating the database +This installation comes with no geographic data loaded. You can either create new data using one of the editors (Potlatch 2, iD, JOSM etc) or by loading an OSM extract. + +After installing but before creating any users or data, import an extract with [Osmosis](https://wiki.openstreetmap.org/wiki/Osmosis) and the `--write-apidb` task. + +``` +docker-compose -f docker/docker-compose.yml exec web osmosis --read-pbf /path/to/file.osm.pbf --write-apidb host="db" database="openstreetmap" user="postgres" validateSchemaVersion="no" +``` From fc139b05c2dae140644f15a0f621a063f4404fc8 Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Fri, 6 Sep 2019 20:56:33 +0600 Subject: [PATCH 14/23] modify DOCKER.md to add app config --- docker/DOCKER.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docker/DOCKER.md b/docker/DOCKER.md index f88071947b..cee59ba94b 100644 --- a/docker/DOCKER.md +++ b/docker/DOCKER.md @@ -21,10 +21,10 @@ Now change working directory to the `openstreetmap-website`: cd openstreetmap-website ``` -### App configuration +### Storage setup ``` -cp config/settings.yml config/settings.local.yml +cp config/example.storage.yml config/storage.yml ``` ### Database @@ -35,6 +35,13 @@ cp config/example.database.yml config/database.yml Set `username` to postgres and `host` to db leave the `password` blank + +### App configuration + +``` +cp config/settings.yml config/settings.local.yml +``` + ### Installation In the root directory run: From ce38a10a9e10e17ec82d27ee436567cda7a347d6 Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Mon, 9 Sep 2019 12:40:30 +0600 Subject: [PATCH 15/23] moved docker_postgres.sh to docker directory --- {db => docker}/docker_postgres.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {db => docker}/docker_postgres.sh (100%) diff --git a/db/docker_postgres.sh b/docker/docker_postgres.sh similarity index 100% rename from db/docker_postgres.sh rename to docker/docker_postgres.sh From b03bfd110c3729886fe71f22b3592cc0e826fc96 Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Mon, 9 Sep 2019 19:05:29 +0600 Subject: [PATCH 16/23] modify Dockerfile.postgres file to change pgsql script location modified Dockerfile.postgres file to changes docker_postgres.sh location from db to docker --- docker/Dockerfile.postgres | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.postgres b/docker/Dockerfile.postgres index 13066abde0..9a42f4dd97 100644 --- a/docker/Dockerfile.postgres +++ b/docker/Dockerfile.postgres @@ -1,6 +1,6 @@ FROM postgres:11 -ADD db/docker_postgres.sh docker-entrypoint-initdb.d/docker_postgres.sh +ADD docker/docker_postgres.sh docker-entrypoint-initdb.d/docker_postgres.sh ADD db/functions/ db/functions/ RUN apt-get update From c6edb2924eb96c6d70de5021d54c631188005c7d Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Mon, 9 Sep 2019 21:32:19 +0600 Subject: [PATCH 17/23] modify Dockerfile.postgres file to add app location and gem install modified Dockerfile.postgres file to add app location setup and gem install via bundle --- docker/Dockerfile.postgres | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.postgres b/docker/Dockerfile.postgres index 9a42f4dd97..6566496322 100644 --- a/docker/Dockerfile.postgres +++ b/docker/Dockerfile.postgres @@ -4,5 +4,17 @@ ADD docker/docker_postgres.sh docker-entrypoint-initdb.d/docker_postgres.sh ADD db/functions/ db/functions/ RUN apt-get update -RUN apt-get install -y make build-essential postgresql-server-dev-all +RUN apt-get install -y software-properties-common +RUN apt-add-repository ppa:brightbox/ruby-ng +RUN apt-get update +RUN apt-get install -y make build-essential postgresql-server-dev-all ruby2.4 ruby2.4-dev zlib1g-dev RUN apt-get clean && rm -rf /var/lib/apt/lists/* + +# Setup app location +RUN mkdir -p /app +WORKDIR /app + +# Install gems +ADD Gemfile* /app/ +RUN gem install bundle +RUN bundle install From 8383453fc9f8f6de1d94431084a0d188860b00ee Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Mon, 9 Sep 2019 21:35:13 +0600 Subject: [PATCH 18/23] modify Dockerfile.postgres file to move db function directory modified Dockerfile.postgres file to move database function directory from /db to /app/db --- docker/Dockerfile.postgres | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.postgres b/docker/Dockerfile.postgres index 6566496322..7c735b3310 100644 --- a/docker/Dockerfile.postgres +++ b/docker/Dockerfile.postgres @@ -1,7 +1,6 @@ FROM postgres:11 ADD docker/docker_postgres.sh docker-entrypoint-initdb.d/docker_postgres.sh -ADD db/functions/ db/functions/ RUN apt-get update RUN apt-get install -y software-properties-common @@ -14,6 +13,9 @@ RUN apt-get clean && rm -rf /var/lib/apt/lists/* RUN mkdir -p /app WORKDIR /app +# add database functions directory +ADD db/functions/ /app/db/functions/ + # Install gems ADD Gemfile* /app/ RUN gem install bundle From cb96ddd3865654544bc5e975a267eaa8529a98fe Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Mon, 9 Sep 2019 21:36:56 +0600 Subject: [PATCH 19/23] modify docker_postgres.sh to fix libpgosm location modified docker_postgres.sh to fix the location of libpgosm as it moved from /db/functions to /app/db/functions --- docker/docker_postgres.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/docker_postgres.sh b/docker/docker_postgres.sh index 9da4956ec4..8f0fa295fa 100644 --- a/docker/docker_postgres.sh +++ b/docker/docker_postgres.sh @@ -2,6 +2,6 @@ set -e psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE EXTENSION btree_gist" openstreetmap make -C db/functions libpgosm.so -psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE FUNCTION maptile_for_point(int8, int8, int4) RETURNS int4 AS '/db/functions/libpgosm', 'maptile_for_point' LANGUAGE C STRICT" openstreetmap -psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '/db/functions/libpgosm', 'tile_for_point' LANGUAGE C STRICT" openstreetmap -psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE FUNCTION xid_to_int4(xid) RETURNS int4 AS '/db/functions/libpgosm', 'xid_to_int4' LANGUAGE C STRICT" openstreetmap +psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE FUNCTION maptile_for_point(int8, int8, int4) RETURNS int4 AS '/app/db/functions/libpgosm', 'maptile_for_point' LANGUAGE C STRICT" openstreetmap +psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '/app/db/functions/libpgosm', 'tile_for_point' LANGUAGE C STRICT" openstreetmap +psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE FUNCTION xid_to_int4(xid) RETURNS int4 AS '/app/db/functions/libpgosm', 'xid_to_int4' LANGUAGE C STRICT" openstreetmap From a1413afe4724c2d711f8d54f61d5d9759c3594cc Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Tue, 10 Sep 2019 13:22:07 +0600 Subject: [PATCH 20/23] modify Dockerfile to add osmosis modified Dockerfile to add osmosis package needed for populating database with osm extract --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 6fdbb25603..d176cbb2cf 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get clean && rm -rf /var/lib/apt/lists/* #npm is not available in Debian repo so following official instruction [source: https://github.com/nodesource/distributions/blob/master/README.md#debinstall] RUN curl -sL https://deb.nodesource.com/setup_10.x -o nodesource_setup.sh && bash nodesource_setup.sh && rm nodesource_setup.sh -RUN apt-get install -y --no-install-recommends ruby-dev libarchive-dev libmagickwand-dev libxml2-dev libxslt1-dev build-essential libpq-dev libsasl2-dev imagemagick libffi-dev locales postgresql-client nodejs +RUN apt-get install -y --no-install-recommends ruby-dev libarchive-dev libmagickwand-dev libxml2-dev libxslt1-dev build-essential libpq-dev libsasl2-dev imagemagick libffi-dev locales postgresql-client nodejs osmosis RUN apt-get clean && rm -rf /var/lib/apt/lists/* RUN npm install yarn phantomjs-prebuilt -g --unsafe-perm From 02851d26ca15fc5bdc96ca577773451c681e7b20 Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Tue, 10 Sep 2019 13:23:51 +0600 Subject: [PATCH 21/23] modify DOCKER.md to add osmosis instructions modified DOCKER.md to add osmosis instructions needed to use osmosis and populate database --- docker/DOCKER.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/DOCKER.md b/docker/DOCKER.md index cee59ba94b..5e8be10540 100644 --- a/docker/DOCKER.md +++ b/docker/DOCKER.md @@ -92,7 +92,7 @@ docker-compose -f docker/docker-compose.yml exec db /bin/bash ### Populating the database This installation comes with no geographic data loaded. You can either create new data using one of the editors (Potlatch 2, iD, JOSM etc) or by loading an OSM extract. -After installing but before creating any users or data, import an extract with [Osmosis](https://wiki.openstreetmap.org/wiki/Osmosis) and the `--write-apidb` task. +After installing but before creating any users or data, import an extract with [Osmosis](https://wiki.openstreetmap.org/wiki/Osmosis) and the `--write-apidb` task. The `web` container comes with `osmosis` pre-installed. So to populate data with a `.osm.pbf` use the following command: ``` docker-compose -f docker/docker-compose.yml exec web osmosis --read-pbf /path/to/file.osm.pbf --write-apidb host="db" database="openstreetmap" user="postgres" validateSchemaVersion="no" From 6ce9e776e08110e48a4d7a0328863aeebe545164 Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Tue, 10 Sep 2019 13:28:31 +0600 Subject: [PATCH 22/23] modify Dockerfile.postgres to add ruby compilation modified Dockerfile.postgres to add ruby compilation from source as we need ruby=>2.4 and also removed bundle installation via gem install command because bundle is already compiled installed while ruby source is being compiled and installed. --- docker/Dockerfile.postgres | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile.postgres b/docker/Dockerfile.postgres index 7c735b3310..35c000efe2 100644 --- a/docker/Dockerfile.postgres +++ b/docker/Dockerfile.postgres @@ -2,13 +2,15 @@ FROM postgres:11 ADD docker/docker_postgres.sh docker-entrypoint-initdb.d/docker_postgres.sh +# compiling ruby from source as official debian has ruby 2.3 and we need => 2.4 RUN apt-get update -RUN apt-get install -y software-properties-common -RUN apt-add-repository ppa:brightbox/ruby-ng -RUN apt-get update -RUN apt-get install -y make build-essential postgresql-server-dev-all ruby2.4 ruby2.4-dev zlib1g-dev +RUN apt-get install -y curl build-essential libreadline-dev libssl-dev libcurl4-openssl-dev postgresql-server-dev-all zlib1g-dev libxml2-dev +RUN curl -L https://cache.ruby-lang.org/pub/ruby/2.6/ruby-2.6.4.tar.gz | tar zx +RUN cd ruby-2.6.4 && ./configure && make && make install +RUN cd .. && rm -r ruby-2.6.4 RUN apt-get clean && rm -rf /var/lib/apt/lists/* + # Setup app location RUN mkdir -p /app WORKDIR /app @@ -18,5 +20,4 @@ ADD db/functions/ /app/db/functions/ # Install gems ADD Gemfile* /app/ -RUN gem install bundle RUN bundle install From 004227e1c283cfa5ff9d79976c83a416834ec8cf Mon Sep 17 00:00:00 2001 From: Fazle Rabbi Date: Tue, 10 Sep 2019 15:25:27 +0600 Subject: [PATCH 23/23] modify Dockerfile.postgres to change owner to postgres --- docker/Dockerfile.postgres | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/Dockerfile.postgres b/docker/Dockerfile.postgres index 35c000efe2..f1c75c15c4 100644 --- a/docker/Dockerfile.postgres +++ b/docker/Dockerfile.postgres @@ -21,3 +21,6 @@ ADD db/functions/ /app/db/functions/ # Install gems ADD Gemfile* /app/ RUN bundle install + +# change ownership to postgres as while running docker_postgres.sh postgres will need write access +RUN chown -R postgres /app/db