RES-1858 Switch to use mapbox for geocoding

.circleci/config.yml

@@ -98,6 +98,7 @@ jobs:
           - POSTGRESQL_CLIENT_POSTGRES_USER=postgres
           - POSTGRESQL_CLIENT_CREATE_DATABASE_NAME=bitnami_discourse
           - POSTGRESQL_CLIENT_CREATE_DATABASE_EXTENSIONS=hstore,pg_trgm
+          - DISCOURSE_EXTRA_CONF_CONTENT=personal_message_enabled_groups \= 10
 #    networks:
 #      - app-network
 
@@ -139,8 +140,7 @@ jobs:
       # Need access to timezones.
       - run: mysql --host="127.0.0.1" -u root -ps3cr3t -e "GRANT SELECT ON mysql.time_zone_name TO 'restarters'@'%';"
 
-      # We have Discourse on CircleCI.  Create Discourse API key.  We do this hackily in the DB because the Bitnami image
-      # doesn't expose an easy way to do it.
+      # We have Discourse on CircleCI.  The API key is inserted using psql below.
       - run: sed -i 's/FEATURE__DISCOURSE_INTEGRATION=.*$/FEATURE__DISCOURSE_INTEGRATION=true/g' .env
       - run: sed -i 's/DISCOURSE_URL=.*$/DISCOURSE_URL=http:\/\/restarters_discourse/g' .env
       - run: sed -i 's/DISCOURSE_APIKEY=.*$/DISCOURSE_APIKEY=fb71f38ca2b8b7cd6a041e57fd8202c9937088f0ecae7db40722bd758dda92fc/g' .env
@@ -183,7 +183,10 @@ jobs:
 
       # Wait for Discourse to finish initialising.
       - run: while ! nc -z restarters_discourse 80; do sleep 1 ; done
+
+      # Add the config we need.
       - run: psql -h postgresql -U postgres -c "INSERT INTO api_keys (id, user_id, created_by_id, created_at, updated_at, allowed_ips, hidden, last_used_at, revoked_at, description, key_hash, truncated_key) VALUES (1, NULL, 1, '2021-10-25 13:56:20.033338', '2021-10-25 13:56:20.033338', NULL, false, NULL, NULL, 'Restarters', 'd89e9dfacfb611fbaf004807648187ce7ed474df44dcb0ada230fab5c8dd6a5b', '9fd7');" bitnami_discourse
+      - run: php artisan discourse:setting personal_message_enabled_groups 10
 
       # Run phpunit.  Discourse makes things slow, so up the timeout.
       - run:

Dockerfile

@@ -7,7 +7,7 @@ WORKDIR /var/www
 # Install dependencies
 RUN sudo apt-get update && \
     sudo apt install dnsutils openssl zip unzip git libxml2-dev libzip-dev zlib1g-dev libcurl4-openssl-dev iputils-ping default-mysql-client vim libpng-dev libgmp-dev libjpeg-turbo8-dev && \
-    sudo apt install php8.1-xmlrpc php8.1-intl php8.1-xdebug php8.1-xmlrpc php8.1-mbstring php8.1-simplexml php8.1-curl php8.1-zip python postgresql-client
+    sudo apt install php8.1-xmlrpc php8.1-intl php8.1-xdebug php8.1-xmlrpc php8.1-mbstring php8.1-simplexml php8.1-curl php8.1-zip python2 postgresql-client procps telnet vim openssh-server php-xmlrpc
 
 # Clear cache - reduces image size.
 RUN sudo apt-get clean && sudo rm -rf /var/lib/apt/lists/*
@@ -28,8 +28,6 @@ RUN wget https://getcomposer.org/composer-1.phar
 EXPOSE 9000
 
 # Install sshd
-RUN sudo apt-get update
-RUN sudo apt-get install vim openssh-server
 RUN sudo sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
 RUN sudo mkdir /var/run/sshd
 RUN sudo bash -c 'install -m755 <(printf "#!/bin/sh\nexit 0") /usr/sbin/policy-rc.d'
@@ -39,9 +37,8 @@ RUN sudo RUNLEVEL=1 dpkg-reconfigure openssh-server
 RUN sudo ssh-keygen -A -v
 RUN sudo update-rc.d ssh defaults
 
-RUN sudo apt-get install python2
+RUN sudo pecl install channel://pecl.php.net/xmlrpc-1.0.0RC3  xmlrpc
 
 EXPOSE 22
-CMD ["/usr/sbin/sshd", "-D"]
 
 CMD ["bash", "docker_run.sh"]

app/Console/Commands/CheckGroupLocations.php

@@ -19,7 +19,7 @@ class CheckGroupLocations extends Command
      *
      * @var string
      */
-    protected $description = 'Check that all gtroup locations are geocodeable';
+    protected $description = 'Check that all group locations are geocodeable';
 
     /**
      * Create a new command instance.

app/Console/Commands/CheckTranslations.php

@@ -49,6 +49,8 @@ public function handle()
                 // This is probably a special case used for form validation.  It throws up many errors, and these
                 // are at best not visible and at worst not too bad given that they only occur in error cases.  So
                 // ignore it.
+            } else if ($file == 'countries.php') {
+                // This is a special case used for country names.
             } else if ($file == 'pagination.php') {
                 // This is probably a special case used for paging through data.
             } else if (strpos($file, '-audits')) {

app/Console/Commands/DiscourseChangeSetting.php

@@ -0,0 +1,69 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Group;
+use App\Helpers\Geocoder;
+use App\Services\DiscourseService;
+use Illuminate\Console\Command;
+
+class DiscourseChangeSetting extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'discourse:setting {setting} {value}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Change Discourse setting';
+
+    private $discourseApiKey;
+    private $discourseApiUser;
+    private $discourseService;
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct(DiscourseService $discourseService)
+    {
+        parent::__construct();
+        $this->discourseService = $discourseService;
+
+        $discourseApiKey = env('DISCOURSE_APIKEY');
+        $discourseApiUser = env('DISCOURSE_APIUSER');
+
+        if (is_null($discourseApiKey) || empty($discourseApiKey)) {
+            $this->error('DISCOURSE_APIKEY is not set');
+            exit();
+        }
+
+        if (is_null($discourseApiUser) || empty($discourseApiUser)) {
+            $this->error('DISCOURSE_APIUSER is not set');
+            exit();
+        }
+
+        $this->discourseApiKey = $discourseApiKey;
+        $this->discourseApiUser = $discourseApiUser;
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return mixed
+     */
+    public function handle(DiscourseService $discourseService)
+    {
+        $setting = $this->argument('setting');
+        $value = $this->argument('value');
+
+        $this->discourseService->setSetting($setting, $value);
+    }
+}

app/Console/Commands/GroupCountryField.php

@@ -0,0 +1,39 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Group;
+use App\Helpers\Fixometer;
+use Illuminate\Console\Command;
+
+class GroupCountryField extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'groups:country';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Set the group country field from the country_code field';
+
+    /**
+     * Execute the console command.
+     *
+     * @return mixed
+     */
+    public function handle()
+    {
+        $groups = Group::all();
+
+        foreach ($groups as $group) {
+            $group->country = Fixometer::getCountryFromCountryCode($group->country_code);
+            $group->save();
+        }
+    }
+}

app/Console/Commands/ImportGroups.php

@@ -61,7 +61,7 @@ public function handle()
                     return iconv( "iso-8859-15", "UTF-8", $str );
                 }, $fields);
 
-                // Format is  'Name', 'Location', 'Postcode', 'Area', 'Country', 'Latitude', 'Longitude', 'Website', 'Phone', 'Networks', 'Description'.
+                // Format is  'Name', 'Location', 'Postcode', 'Area', 'CountryCode', 'Latitude', 'Longitude', 'Website', 'Phone', 'Networks', 'Description'.
 
                 $groupname = $fields[0];
                 $location = $fields[1];
@@ -122,7 +122,7 @@ public function handle()
                     $group->area = $area;
                     $group->latitude = $lat;
                     $group->longitude = $lng;
-                    $group->country = $country;
+                    $group->country_code = $country;
                     $group->website = $website;
                     $group->phone = $phone;
                     $group->free_text = $description;

app/Console/Commands/SyncDiscourseUsernames.php

@@ -69,8 +69,8 @@ public function handle()
         foreach ($usersFromDiscourse as $discourseUser) {
             if (property_exists($discourseUser, 'single_sign_on_record') &&
                 $discourseUser->single_sign_on_record &&
-                property_exists($discourseUser->single_sign_on_record, 'external_email')) {
-                $user = User::where('email', $discourseUser->single_sign_on_record->external_email)->first();
+                property_exists($discourseUser->single_sign_on_record, 'external_id')) {
+                $user = User::find($discourseUser->single_sign_on_record->external_id);
 
                 if (! is_null($user)) {
                     $usersFoundInRestarters++;

app/Console/Commands/SyncEvents.php

@@ -3,6 +3,7 @@
 namespace App\Console\Commands;
 
 use App\Group;
+use App\Helpers\Fixometer;
 use App\Party;
 use DateTime;
 use Illuminate\Console\Command;
@@ -77,7 +78,7 @@ public function handle()
 
                 $custom_fields = [
                     ['key' => 'party_grouphash', 'value' => $event->group],
-                    ['key' => 'party_groupcountry', 'value' => $group->country],
+                    ['key' => 'party_groupcountry', 'value' => Fixometer::getCountryFromCountryCode($group->country_code)],
                     ['key' => 'party_groupcity', 'value' => $group->area],
                     ['key' => 'party_venue', 'value' => $event->venue],
                     ['key' => 'party_location', 'value' => $event->location],

app/Console/Commands/SyncGroups.php

@@ -3,6 +3,7 @@
 namespace App\Console\Commands;
 
 use App\Group;
+use App\Helpers\Fixometer;
 use Illuminate\Console\Command;
 
 class SyncGroups extends Command
@@ -54,7 +55,7 @@ public function handle()
             try {
                 $custom_fields = [
                     ['key' => 'group_city', 'value' => $group->area],
-                    ['key' => 'group_country', 'value' => $group->country],
+                    ['key' => 'group_country', 'value' => Fixometer::getCountryFromCountryCode($group->country_code)],
                     ['key' => 'group_website', 'value' => $group->website],
                     ['key' => 'group_hash', 'value' => $group->idgroups],
                     ['key' => 'group_latitude', 'value' => $group->latitude],

app/Console/Commands/UserCreate.php

@@ -92,6 +92,7 @@ public function handle(DiscourseService $discourseService)
                 $this->info("User created #" . $user->id);
 
                 if (config('restarters.features.discourse_integration')) {
+                    $user->generateAndSetUsername();
                     $discourseService->syncSso($user);
                 }
             } else

app/Console/Kernel.php

@@ -41,6 +41,8 @@ protected function schedule(Schedule $schedule)
             ->daily()
             ->sendOutputTo(storage_path().'/logs/discourse_usernames.log')
             ->emailOutputTo(env('SEND_COMMAND_LOGS_TO'), 'tech@therestartproject.org');
+
+        $schedule->command('groups:country')->hourly();
     }
 
     /**

app/Device.php

@@ -57,7 +57,6 @@ public static function boot()
 
         static::deleting(function ($device) {
             $device->barriers()->detach();
-            $device->urls()->delete();
         });
     }
 
@@ -216,11 +215,6 @@ public function deviceEvent()
         return $this->hasOne(\App\Party::class, 'idevents', 'event');
     }
 
-    public function urls()
-    {
-        return $this->hasMany(\App\DeviceUrl::class, 'device_id', 'iddevices');
-    }
-
     public function barriers()
     {
         return $this->belongsToMany(\App\Barrier::class, 'devices_barriers', 'device_id', 'barrier_id');
@@ -234,10 +228,12 @@ public function eCo2Diverted($emissionRatio, $displacementFactor)
         $footprint = 0;
 
         if ($this->isFixed()) {
-            if ($this->deviceCategory->isMiscPowered() && $this->estimate > 0) {
+            if ($this->category == env('MISC_CATEGORY_ID_POWERED') && $this->estimate > 0) {
                 $footprint = $this->estimate * $emissionRatio;
             } else {
-                $footprint = $this->deviceCategory->footprint;
+                $footprint = \Cache::remember('category-' . $this->category, 15, function() {
+                    return $this->deviceCategory;
+                })->footprint;
             }
         }
 
@@ -271,11 +267,19 @@ public function eWasteDiverted()
     {
         $ewasteDiverted = 0;
 
-        if ($this->isFixed() && $this->deviceCategory->isPowered()) {
-            if ($this->deviceCategory->isMiscPowered() && $this->estimate > 0) {
+        $powered = \Cache::remember('category-powered-' . $this->category, 15, function() {
+            return $this->deviceCategory->powered;
+        });
+
+        if ($this->isFixed() && $powered) {
+            if ($this->category == env('MISC_CATEGORY_ID_POWERED') && $this->estimate > 0) {
                 $ewasteDiverted = $this->estimate;
             } else {
-                $ewasteDiverted = $this->deviceCategory->weight;
+                $category = \Cache::remember('category-' . $this->category, 15, function() {
+                    return $this->deviceCategory;
+                });
+
+                $ewasteDiverted = $category->weight;
             }
         }
 
@@ -407,7 +411,18 @@ public function poweredCount()
     public static function getItemTypes()
     {
         // List the item types
-        $types = DB::table('devices')->whereNotNull('item_type')->select('item_type', DB::raw('COUNT(*) as count'))->groupBy('item_type')->orderBy('count', 'desc')->get()->toArray();
+        $types = DB::select(DB::raw("SELECT s.* FROM 
+(SELECT item_type, powered, idcategories, categories.name as categoryname, COUNT(*) AS count 
+FROM devices INNER JOIN categories ON devices.category = categories.idcategories 
+WHERE item_type IS NOT NULL GROUP BY item_type, categoryname
+) s 
+JOIN 
+(SELECT item_type, MAX(count) AS maxcount FROM 
+(SELECT item_type, powered, idcategories, categories.name as categoryname, COUNT(*) AS count 
+FROM devices INNER JOIN categories ON devices.category = categories.idcategories 
+WHERE item_type IS NOT NULL GROUP BY item_type, categoryname) s
+GROUP BY s.item_type) AS m
+ON s.item_type = m.item_type AND s.count = m.maxcount;"));
 
         return $types;
     }