From dc15c8bfd47f6d83e9346135fa52fc4565a6343c Mon Sep 17 00:00:00 2001 From: Matt Hampel Date: Sun, 19 Aug 2018 10:55:19 -0400 Subject: [PATCH 1/9] Ignore byebug logs --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d1fdbe7..3887a0c 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ # Ignore all logfiles and tempfiles. /log/*.log /tmp +.byebug_history # Ignore foreman environment .env From f54f2abb32f4787da1307582fdd6e498edb256b1 Mon Sep 17 00:00:00 2001 From: Matt Hampel Date: Sun, 19 Aug 2018 10:56:01 -0400 Subject: [PATCH 2/9] Store lat-lngs from service requests for #75 --- Gemfile | 1 + Gemfile.lock | 10 +++++++++- README.md | 11 +++++++++-- app/models/service_request.rb | 3 +++ config/database.yml | 2 +- ...20180819143420_add_geometry_to_service_requests.rb | 6 ++++++ db/schema.rb | 5 ++++- spec/factories/service_requests.rb | 4 ++++ spec/models/service_request_spec.rb | 5 +++++ 9 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 db/migrate/20180819143420_add_geometry_to_service_requests.rb diff --git a/Gemfile b/Gemfile index 35d984a..f00c652 100644 --- a/Gemfile +++ b/Gemfile @@ -2,6 +2,7 @@ source 'https://rubygems.org' ruby File.read(File.join(File.dirname(__FILE__), '.ruby-version')).strip git_source(:github) { |repo| "https://github.com/#{repo}.git" } +gem 'activerecord-postgis-adapter' gem 'autoprefixer-rails' gem 'aws-sdk-s3' gem 'babosa' diff --git a/Gemfile.lock b/Gemfile.lock index 54db4c5..b00c877 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -49,6 +49,9 @@ GEM activemodel (= 5.2.1) activesupport (= 5.2.1) arel (>= 9.0) + activerecord-postgis-adapter (5.2.1) + activerecord (~> 5.1) + rgeo-activerecord (~> 6.0) activestorage (5.2.1) actionpack (= 5.2.1) activerecord (= 5.2.1) @@ -256,6 +259,10 @@ GEM redcarpet (3.4.0) request_store (1.4.1) rack (>= 1.4) + rgeo (1.1.1) + rgeo-activerecord (6.0.0) + activerecord (~> 5.0) + rgeo (~> 1.0) rspec-core (3.8.0) rspec-support (~> 3.8.0) rspec-expectations (3.8.1) @@ -351,6 +358,7 @@ PLATFORMS ruby DEPENDENCIES + activerecord-postgis-adapter annotate autoprefixer-rails aws-sdk-s3 @@ -405,4 +413,4 @@ RUBY VERSION ruby 2.5.1p57 BUNDLED WITH - 1.16.2 + 1.16.3 diff --git a/README.md b/README.md index 3cbca97..f9a7928 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Open311 Status **Website**: http://status.open311.org -**Project Backlog**: https://github.com/codeforamerica/open311status/projects/1 +**Project Backlog**: https://github.com/codeforamerica/open311status/projects/1 Open311 Status monitors and aggregates the status of dozens of Open311 API endpoints, providing benchmarks and comparative insights into: @@ -39,7 +39,8 @@ bruhl: 2. Check the ruby version in `.ruby-version` and ensure you have it installed locally e.g. `rbenv install 2.5.1` 3. Install [bundler](https://bundler.io/) (the latest Heroku-compatible version): `gem install bundler` 4. [Install Postgres](https://github.com/codeforamerica/howto/blob/master/PostgreSQL.md). If setting up Postgres.app, you will also need to add the binary to your path. e.g. Add to your `~/.bashrc`: -`export PATH="$PATH:/Applications/Postgres.app/Contents/Versions/latest/bin"` +`export PATH="$PATH:/Applications/Postgres.app/Contents/Versions/latest/bin"`. +5. [Install PostGIS](https://postgis.net/install/), the Postgres geospatial extension, if it's not included in your distribution. Postgres.app comes with postgis. ### Application Setup @@ -47,3 +48,9 @@ bruhl: 2. Create the databases and load schema and seeds: `bin/rails db:setup` 3. Run the tests: `bin/rspec` 4. Run the server: `bin/rails server`, and visit the web-browser: [`http://localhost:3000`](http://localhost:3000) +5. Load cities: `rake cities:load` +6. Load service requests: `rake cities:service_requests` + +#### Migration guide + +You may need to run `rake db:gis:setup` to enable PostGIS on your database. diff --git a/app/models/service_request.rb b/app/models/service_request.rb index 9c2f060..f11ddb0 100644 --- a/app/models/service_request.rb +++ b/app/models/service_request.rb @@ -3,6 +3,7 @@ # Table name: service_requests # # id :bigint(8) not null, primary key +# geometry :geography({:srid geometry, 4326 # raw_data :json # requested_datetime :datetime # status :string @@ -16,6 +17,7 @@ # # index_service_requests_on_city_id (city_id) # index_service_requests_on_city_id_and_service_request_id (city_id,service_request_id) UNIQUE +# index_service_requests_on_geometry (geometry) USING gist # index_service_requests_on_status (status) # @@ -34,6 +36,7 @@ def raw_data=(json) self[:status] = json['status'] self[:requested_datetime] = DateTime.iso8601(json['requested_datetime']) if json['requested_datetime'].present? self[:updated_datetime] = DateTime.iso8601(json['updated_datetime']) if json['updated_datetime'].present? + self[:geometry] = "POINT(#{json['long']} #{json['lat']})" if json['lat'].present? end rescue => e Raven.capture_exception e, diff --git a/config/database.yml b/config/database.yml index dedb90e..6ced4c8 100644 --- a/config/database.yml +++ b/config/database.yml @@ -1,5 +1,5 @@ defaults: &defaults - adapter: postgresql + adapter: postgis encoding: unicode pool: 20 username: <%= ENV['POSTGRES_USERNAME'] || ENV['USER'] %> diff --git a/db/migrate/20180819143420_add_geometry_to_service_requests.rb b/db/migrate/20180819143420_add_geometry_to_service_requests.rb new file mode 100644 index 0000000..73c922d --- /dev/null +++ b/db/migrate/20180819143420_add_geometry_to_service_requests.rb @@ -0,0 +1,6 @@ +class AddGeometryToServiceRequests < ActiveRecord::Migration[5.2] + def change + add_column :service_requests, :geometry, :geometry, geographic: true + add_index :service_requests, :geometry, using: :gist + end +end diff --git a/db/schema.rb b/db/schema.rb index cab4e9b..467419f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,10 +10,11 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2018_08_07_141658) do +ActiveRecord::Schema.define(version: 2018_08_19_143420) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" + enable_extension "postgis" create_table "cities", force: :cascade do |t| t.string "slug" @@ -42,8 +43,10 @@ t.datetime "created_at" t.datetime "updated_at" t.integer "city_id" + t.geography "geometry", limit: {:srid=>4326, :type=>"geometry", :geographic=>true} t.index ["city_id", "service_request_id"], name: "index_service_requests_on_city_id_and_service_request_id", unique: true t.index ["city_id"], name: "index_service_requests_on_city_id" + t.index ["geometry"], name: "index_service_requests_on_geometry", using: :gist t.index ["status"], name: "index_service_requests_on_status" end diff --git a/spec/factories/service_requests.rb b/spec/factories/service_requests.rb index 794a258..662e1ce 100644 --- a/spec/factories/service_requests.rb +++ b/spec/factories/service_requests.rb @@ -10,6 +10,9 @@ service_name { Faker::Commerce.department(2, true) } description { Faker::Lorem.paragraph(1, false, 5) } requested_datetime { 10.minutes.ago } + lat { Faker::Address.latitude } + long { Faker::Address.longitude } + geometry { "POINT(#{long} #{lat})"} end raw_data do @@ -19,6 +22,7 @@ 'description' => description, 'status' => status, 'requested_datetime' => requested_datetime.iso8601, + 'geometry' => geometry, } end end diff --git a/spec/models/service_request_spec.rb b/spec/models/service_request_spec.rb index 9dddb83..6887365 100644 --- a/spec/models/service_request_spec.rb +++ b/spec/models/service_request_spec.rb @@ -24,6 +24,11 @@ it 'extracts :updated_datetime' do expect(sr.updated_datetime).to eq DateTime.iso8601 json['updated_datetime'] end + + it 'extracts a point' do + expect(sr.geometry.lat).to eq json['lat'] + expect(sr.geometry.lon).to eq json['long'] + end end describe '#slug' do From f26230dc2c4e7c68f12c1a6cdca6848b5796b990 Mon Sep 17 00:00:00 2001 From: Matt Hampel Date: Sun, 19 Aug 2018 11:35:18 -0400 Subject: [PATCH 3/9] Move instructions for adding real data --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f9a7928..7101e70 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,12 @@ bruhl: jurisdiction: 'bruehl.de' ``` +### Loading real data + +By default, running `db:setup` will load cities and generate fake service +requests. To load cities, run `rake cities:load`. And to load service requests, +`rake cities:service_requests` + ### Application Dependencies 1. Install Ruby with your ruby version manager of choice, like [rbenv](https://github.com/rbenv/rbenv) or [RVM](https://github.com/codeforamerica/howto/blob/master/Ruby.md) 2. Check the ruby version in `.ruby-version` and ensure you have it installed locally e.g. `rbenv install 2.5.1` @@ -48,8 +54,6 @@ bruhl: 2. Create the databases and load schema and seeds: `bin/rails db:setup` 3. Run the tests: `bin/rspec` 4. Run the server: `bin/rails server`, and visit the web-browser: [`http://localhost:3000`](http://localhost:3000) -5. Load cities: `rake cities:load` -6. Load service requests: `rake cities:service_requests` #### Migration guide From 26c96089db8332134da609fa3c0bab7141a58c76 Mon Sep 17 00:00:00 2001 From: Matt Hampel Date: Sun, 19 Aug 2018 11:35:48 -0400 Subject: [PATCH 4/9] Rely on the raw_data setter to set geometry --- spec/factories/service_requests.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/factories/service_requests.rb b/spec/factories/service_requests.rb index 662e1ce..b5866fd 100644 --- a/spec/factories/service_requests.rb +++ b/spec/factories/service_requests.rb @@ -12,7 +12,6 @@ requested_datetime { 10.minutes.ago } lat { Faker::Address.latitude } long { Faker::Address.longitude } - geometry { "POINT(#{long} #{lat})"} end raw_data do From e215e447e0f5f1cc595b1abfc8e4960c36e4091b Mon Sep 17 00:00:00 2001 From: Matt Hampel Date: Mon, 20 Aug 2018 09:55:38 -0400 Subject: [PATCH 5/9] Match raw request --- spec/factories/service_requests.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/factories/service_requests.rb b/spec/factories/service_requests.rb index b5866fd..aac6234 100644 --- a/spec/factories/service_requests.rb +++ b/spec/factories/service_requests.rb @@ -21,7 +21,8 @@ 'description' => description, 'status' => status, 'requested_datetime' => requested_datetime.iso8601, - 'geometry' => geometry, + 'lat' => lat, + 'long' => long, } end end From 3d8e41cdd63b34ad311870bb13b343cdfd6465ee Mon Sep 17 00:00:00 2001 From: Ben Sheldon Date: Mon, 20 Aug 2018 09:19:40 -0700 Subject: [PATCH 6/9] Update README.md --- README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/README.md b/README.md index cc2b7a9..7101e70 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,6 @@ requests. To load cities, run `rake cities:load`. And to load service requests, ### Application Setup 1. Install ruby gem dependencies: `bundle install` -<<<<<<< HEAD 2. Create the databases and load schema and seeds: `bin/rails db:setup` 3. Run the tests: `bin/rspec` 4. Run the server: `bin/rails server`, and visit the web-browser: [`http://localhost:3000`](http://localhost:3000) @@ -59,9 +58,3 @@ requests. To load cities, run `rake cities:load`. And to load service requests, #### Migration guide You may need to run `rake db:gis:setup` to enable PostGIS on your database. -======= -2. Install node dependencies: `yarn install` -3. Create the databases and load schema and seeds: `bin/rails db:setup` -4. Run the tests: `bin/rspec` -5. Run the server: `bin/rails server`, and visit the web-browser: [`http://localhost:3000`](http://localhost:3000) ->>>>>>> master From 9ff22546537d7127fcab2c683d27d752eb4d51ee Mon Sep 17 00:00:00 2001 From: Ben Sheldon Date: Mon, 20 Aug 2018 09:24:59 -0700 Subject: [PATCH 7/9] Use CircleCI postgis image --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 08415ea..c0b1336 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ jobs: PGHOST: localhost PGUSER: open311status RAILS_ENV: test - - image: postgres:10 + - image: circleci/postgres:10-postgis environment: POSTGRES_USER: open311status POSTGRES_DB: open311status_test From 37cfd23925d3f26a8fbb9f1a62f14940dd8920ef Mon Sep 17 00:00:00 2001 From: Ben Sheldon Date: Mon, 20 Aug 2018 09:41:03 -0700 Subject: [PATCH 8/9] Ensure postgis extension is enabled in migration --- db/migrate/20180819143420_add_geometry_to_service_requests.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db/migrate/20180819143420_add_geometry_to_service_requests.rb b/db/migrate/20180819143420_add_geometry_to_service_requests.rb index 73c922d..30a890a 100644 --- a/db/migrate/20180819143420_add_geometry_to_service_requests.rb +++ b/db/migrate/20180819143420_add_geometry_to_service_requests.rb @@ -1,5 +1,7 @@ class AddGeometryToServiceRequests < ActiveRecord::Migration[5.2] def change + enable_extension "postgis" + add_column :service_requests, :geometry, :geometry, geographic: true add_index :service_requests, :geometry, using: :gist end From 9d2eb87bc2f76fb74382195aa4475c6c50147574 Mon Sep 17 00:00:00 2001 From: Ben Sheldon Date: Mon, 20 Aug 2018 09:54:30 -0700 Subject: [PATCH 9/9] Replace postgres with postgis in production db connection string --- config/database.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/database.yml b/config/database.yml index 6ced4c8..ae44592 100644 --- a/config/database.yml +++ b/config/database.yml @@ -20,3 +20,4 @@ staging: production: database: open311status_production <<: *defaults + url: <%= ENV.fetch("DATABASE_URL", "").gsub(/postgres/, 'postgis') %>