From b0ef085ebeb0189e2a5dcf5c95b4214468b93b1a Mon Sep 17 00:00:00 2001 From: Sebastian Silva Date: Wed, 3 Apr 2019 23:14:27 -0500 Subject: [PATCH 01/16] Switch back to Debian 9 Stretch --- Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5fbfd32b1..5f650f4ff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # https://github.com/publiclab/mapknitter/ # This image deploys Mapknitter! -FROM debian:buster +FROM debian:stretch # Set correct environment variables. ENV HOME /root @@ -34,9 +34,6 @@ ADD Gemfile /tmp/Gemfile ADD Gemfile.lock /tmp/Gemfile.lock RUN bundle install -# HOTFIX Workaround for mysql2 gem incompatibility with libmariadb-dev -RUN sed -i "s/ LONG_PASSWORD |//g" /usr/local/rvm/gems/ruby-*/gems/mysql2-*/lib/mysql2/client.rb - # Add the Rails app WORKDIR /app COPY Gemfile /app/Gemfile From 5963c2ee0129e762972656048d6ea006b344bdb7 Mon Sep 17 00:00:00 2001 From: Sebastian Silva Date: Wed, 3 Apr 2019 23:24:10 -0500 Subject: [PATCH 02/16] Simplify docker image --- Dockerfile | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5f650f4ff..76e178bd7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,31 +2,18 @@ # https://github.com/publiclab/mapknitter/ # This image deploys Mapknitter! -FROM debian:stretch +FROM ruby:2.4-stretch # Set correct environment variables. ENV HOME /root # Install dependencies RUN apt-get update -qq && apt-get install -y \ - bundler ruby-rmagick libfreeimage3 \ - libfreeimage-dev zip nodejs gdal-bin \ - curl g++ gcc autoconf automake bison \ - libc6-dev libffi-dev libgdbm-dev \ - libncurses5-dev libsqlite3-dev libtool \ - libyaml-dev make pkg-config sqlite3 \ - zlib1g-dev libgmp-dev libreadline-dev libssl-dev \ - procps libmariadb-dev-compat libmariadb-dev git python-gdal \ - imagemagick - -# Ruby -RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB && curl -sSL https://get.rvm.io | bash -s stable && usermod -a -G rvm root -RUN /bin/bash -l -c ". /etc/profile.d/rvm.sh && rvm install 2.4.4 && rvm use 2.4.4 --default" + nodejs gdal-bin curl procps git imagemagick RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - && apt-get install -y npm RUN npm install -g bower - # Install bundle of gems SHELL [ "/bin/bash", "-l", "-c" ] WORKDIR /tmp From 9afbb8d57b0792b0b136d01dcf5411530ff653ff Mon Sep 17 00:00:00 2001 From: Sebastian Silva Date: Wed, 3 Apr 2019 23:29:08 -0500 Subject: [PATCH 03/16] Bump Ruby to 2.4.6 --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 6c0d4876b..8e3c5dd63 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,6 @@ source "https://rubygems.org" -ruby "2.4.4" +ruby "2.4.6" gem "rails", "~>3.2" gem 'rake', '~> 12.3.2' From 044da106fe29f59de58e483db21838d70592820c Mon Sep 17 00:00:00 2001 From: Sebastian Silva Date: Wed, 3 Apr 2019 23:35:10 -0500 Subject: [PATCH 04/16] Re-add dependency --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 76e178bd7..0dbf301e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ ENV HOME /root # Install dependencies RUN apt-get update -qq && apt-get install -y \ - nodejs gdal-bin curl procps git imagemagick + nodejs gdal-bin curl procps git imagemagick python-gdal RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - && apt-get install -y npm RUN npm install -g bower From a587b95ceb537a0d93d4d904c751f6b6290ef011 Mon Sep 17 00:00:00 2001 From: Sebastian Silva Date: Wed, 3 Apr 2019 23:47:36 -0500 Subject: [PATCH 05/16] Add dependency (zip) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0dbf301e4..adcf829c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ ENV HOME /root # Install dependencies RUN apt-get update -qq && apt-get install -y \ - nodejs gdal-bin curl procps git imagemagick python-gdal + nodejs gdal-bin curl procps git imagemagick python-gdal zip RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - && apt-get install -y npm RUN npm install -g bower From 7505a2e4f1528e52ef70368d1be86b1d7b2ab02b Mon Sep 17 00:00:00 2001 From: Sebastian Silva Date: Thu, 4 Apr 2019 00:07:33 -0500 Subject: [PATCH 06/16] Try pip install gdal --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index adcf829c3..96675a3f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,10 +9,11 @@ ENV HOME /root # Install dependencies RUN apt-get update -qq && apt-get install -y \ - nodejs gdal-bin curl procps git imagemagick python-gdal zip + nodejs gdal-bin curl procps git imagemagick python-gdal zip python-pip RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - && apt-get install -y npm RUN npm install -g bower +RUN pip install --upgrade GDAL # Install bundle of gems SHELL [ "/bin/bash", "-l", "-c" ] From f658abef5cd47e70fb8f9c7d8ca058a17b522cd4 Mon Sep 17 00:00:00 2001 From: Sebastian Silva Date: Thu, 4 Apr 2019 00:23:59 -0500 Subject: [PATCH 07/16] Install libgdal-dev --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 96675a3f9..c725110fa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ ENV HOME /root # Install dependencies RUN apt-get update -qq && apt-get install -y \ - nodejs gdal-bin curl procps git imagemagick python-gdal zip python-pip + nodejs gdal-bin curl procps git imagemagick python-gdal zip python-pip libgdal-dev RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - && apt-get install -y npm RUN npm install -g bower From 6f44ca67ad1a9ba2f458446c663bddd3cd45316d Mon Sep 17 00:00:00 2001 From: Sebastian Silva Date: Thu, 4 Apr 2019 01:24:45 -0500 Subject: [PATCH 08/16] Revert attept to use pip --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c725110fa..adcf829c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,11 +9,10 @@ ENV HOME /root # Install dependencies RUN apt-get update -qq && apt-get install -y \ - nodejs gdal-bin curl procps git imagemagick python-gdal zip python-pip libgdal-dev + nodejs gdal-bin curl procps git imagemagick python-gdal zip RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - && apt-get install -y npm RUN npm install -g bower -RUN pip install --upgrade GDAL # Install bundle of gems SHELL [ "/bin/bash", "-l", "-c" ] From 0a6add833f33e9eea37777b5f173be8e494b13d1 Mon Sep 17 00:00:00 2001 From: Sebastian Silva Date: Thu, 4 Apr 2019 02:02:16 -0500 Subject: [PATCH 09/16] Bump ruby --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index adcf829c3..ac7c122c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # https://github.com/publiclab/mapknitter/ # This image deploys Mapknitter! -FROM ruby:2.4-stretch +FROM ruby:2.4.6-stretch # Set correct environment variables. ENV HOME /root From 9ef4b6b683e2ae12ceffc1d61087e1da0f6402e2 Mon Sep 17 00:00:00 2001 From: Sebastian Silva Date: Thu, 4 Apr 2019 02:10:14 -0500 Subject: [PATCH 10/16] Avoid naming containers in compose file --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5f3d5987e..85b2c89d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,14 +1,12 @@ version: '3.3' services: db: - container_name: db image: mysql:5.7 env_file: - mapknitter.env volumes: - ../mysql:/var/lib/mysql web: - container_name: web build: . env_file: - mapknitter.env From 9f2a32b7a7fb05b8ee300bfbfbd8a8298e29a57b Mon Sep 17 00:00:00 2001 From: Sebastian Silva Date: Thu, 4 Apr 2019 02:19:15 -0500 Subject: [PATCH 11/16] Avoid overwriting database on redeploy-container --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 176e64d8d..b63547b4b 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,10 @@ build: cp config/database.yml.example config/database.yml cp db/schema.rb.example db/schema.rb docker-compose build - docker-compose run --rm web bash -l -c "sleep 10 && bower install --allow-root && rake db:setup && rake db:migrate && rake assets:precompile" deploy-container: docker-compose up -d + docker-compose exec -T web bash -l -c "sleep 10 && rake db:setup && rake db:migrate && rake assets:precompile" redeploy-container: docker-compose up --force-recreate -d From bfef89c5318efa4f31be2a10b1e7a3cb6c5b94cb Mon Sep 17 00:00:00 2001 From: Sebastian Silva Date: Thu, 4 Apr 2019 02:33:58 -0500 Subject: [PATCH 12/16] Allow to load mysql dump --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 85b2c89d1..e048537a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ services: - mapknitter.env volumes: - ../mysql:/var/lib/mysql + - ../dump:/docker-entrypoint-initdb.d web: build: . env_file: From f76da9b0fb35d5150f6ce0402cb5806b43837a00 Mon Sep 17 00:00:00 2001 From: Sebastian Silva Date: Fri, 5 Apr 2019 00:16:54 -0500 Subject: [PATCH 13/16] Include own GDAL packages --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index ac7c122c6..0033739e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,9 @@ FROM ruby:2.4.6-stretch # Set correct environment variables. ENV HOME /root +RUN echo "deb http://packages.laboratoriopublico.org/publiclab/ stretch main" > /etc/apt/sources.list.d/publiclab.list +RUN apt-key adv --keyserver hkps.pool.sks-keyservers.net --recv-keys BF26EE05EA6A68F0 + # Install dependencies RUN apt-get update -qq && apt-get install -y \ nodejs gdal-bin curl procps git imagemagick python-gdal zip From 09c63781529c3d66ae1fd0c7249d3f605cbe0d9d Mon Sep 17 00:00:00 2001 From: Sebastian Silva Date: Fri, 5 Apr 2019 00:29:23 -0500 Subject: [PATCH 14/16] Disable ipv6 to prevent error --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index 0033739e7..967d32054 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,12 @@ FROM ruby:2.4.6-stretch # Set correct environment variables. ENV HOME /root +# Backported GDAL RUN echo "deb http://packages.laboratoriopublico.org/publiclab/ stretch main" > /etc/apt/sources.list.d/publiclab.list + +# Obtain key +RUN mkdir ~/.gnupg +RUN echo "disable-ipv6" >> ~/.gnupg/dirmngr.conf RUN apt-key adv --keyserver hkps.pool.sks-keyservers.net --recv-keys BF26EE05EA6A68F0 # Install dependencies From 4a3e89f3679ee988d88585591af7cc7ecfb81d03 Mon Sep 17 00:00:00 2001 From: Sebastian Silva Date: Fri, 5 Apr 2019 01:11:08 -0500 Subject: [PATCH 15/16] Add missing Amazon S3 yml to Makefile --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index b63547b4b..f8728c5a8 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ export COMPOSE_HTTP_TIMEOUT=360 build: cp config/database.yml.example config/database.yml + cp config/amazon_s3.yml.example config/amazon_s3.yml cp db/schema.rb.example db/schema.rb docker-compose build From eb1e4ada52c2e7f1c45eef1b31dd4508b5b12318 Mon Sep 17 00:00:00 2001 From: Sebastian Silva Date: Fri, 5 Apr 2019 01:50:04 -0500 Subject: [PATCH 16/16] Document unstable instance --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 560b85281..b200e0db6 100644 --- a/README.md +++ b/README.md @@ -116,3 +116,6 @@ Help improve Public Lab software! * Review contributor guidelines at http://publiclab.org/wiki/contributing-to-public-lab-software * Some devs hang out in http://publiclab.org/chat (irc webchat) +## Staging infrastructure and testing + +In addition automatic testing with Travis CI - we have a branch (`unstable`) is set to auto-build and deploy to [a staging instance](http://mapknitter-unstable.laboratoriopublico.org/). This instance includes a copy of the production database and is intended for experimenting or debugging purposes with a production-like environment.