Merge pull request #324 from e-picsa/content/zm-climate-data

chore(climate): update mw data

apps/picsa-apps/dashboard/src/app/modules/climate/pages/station-details/components/rainfall-summary/rainfall-summary.ts

@@ -114,7 +114,7 @@ export class RainfallSummaryComponent implements AfterViewInit {
       // HACK - use either end_rains or end_season depending on which has data populated
       // TODO - push for single value to be populated at api level
       End: el.end_rains_doy || el.end_season_doy,
-      Extreme_events: 0,
+      Extreme_events: null as any,
       Length: el.season_length,
       // HACK - replace 0mm with null value
       Rainfall: el.annual_rain || undefined,

apps/picsa-server/scripts/climate-data-parser/src/index.ts

@@ -25,7 +25,7 @@ function main() {
         definitions: null,
       });
     }
-    // TODO - QC checks (e.g. 0mm rainfall)
+    // TODO - QC checks (e.g. 0mm rainfall, null string)
     stationData[id].push({ Year: s_year, Rainfall: sum_Rain, Start: start_rain, End: end_rains, Length: length });
   }
 

apps/picsa-tools/climate-tool/src/app/data/stations/index.ts

@@ -1,7 +1,11 @@
 import type { IStationMeta } from '@picsa/models';
 
 import MW_STATIONS from './mw';
-import TJ_STATIONS from './tj';
+// import TJ_STATIONS from './tj';
 import ZM_STATIONS from './zm';
 
-export const HARDCODED_STATIONS = ([] as IStationMeta[]).concat(MW_STATIONS, TJ_STATIONS, ZM_STATIONS);
+export const HARDCODED_STATIONS = ([] as IStationMeta[]).concat(
+  MW_STATIONS,
+  // TJ_STATIONS,
+  ZM_STATIONS
+);

apps/picsa-tools/climate-tool/src/app/data/stations/mw.ts

@@ -1,5 +1,16 @@
 import { CLIMATE_CHART_DEFINTIONS } from '@picsa/data/climate/chart_definitions';
 import type { IStationMeta } from '@picsa/models';
+/** Draft Stations (pending data validation)
+
+Chiradzulu: remove the high value (2015-16) and keep
+Luwazi: I would suggest we keep and start at 1968 as all the 0s are before that date
+Kamuona: last year is very low and this is often as the data are incomplete. I think remove the last year please
+Kasiya: there is a very high value 2015-16 that we should remove if possible please
+Mtakataka: same as above but start from 1947-48
+Mzandu: I would remove the last year
+Nalunga: I would remove the last year
+
+ */
 
 const stations: IStationMeta[] = [
   // 2021-2023 legacy station data
@@ -115,6 +126,7 @@ const stations: IStationMeta[] = [
     longitude: 35.18,
     countryCode: 'mw',
     definitions: CLIMATE_CHART_DEFINTIONS.mw,
+    draft: true,
   },
   {
     id: 'dowa_agr',
@@ -139,6 +151,7 @@ const stations: IStationMeta[] = [
     longitude: 33.53,
     countryCode: 'mw',
     definitions: CLIMATE_CHART_DEFINTIONS.mw,
+    draft: true,
   },
   {
     id: 'kia',
@@ -187,6 +200,7 @@ const stations: IStationMeta[] = [
     longitude: 34.2,
     countryCode: 'mw',
     definitions: CLIMATE_CHART_DEFINTIONS.mw,
+    draft: true,
   },
   {
     id: 'mtakataka',
@@ -195,6 +209,7 @@ const stations: IStationMeta[] = [
     longitude: 34.52,
     countryCode: 'mw',
     definitions: CLIMATE_CHART_DEFINTIONS.mw,
+    draft: true,
   },
   {
     id: 'ndakwera',
@@ -227,6 +242,7 @@ const stations: IStationMeta[] = [
     longitude: 34.3,
     countryCode: 'mw',
     definitions: CLIMATE_CHART_DEFINTIONS.mw,
+    draft: true,
   },
   {
     id: 'mzandu',
@@ -235,6 +251,7 @@ const stations: IStationMeta[] = [
     longitude: 34.5,
     countryCode: 'mw',
     definitions: CLIMATE_CHART_DEFINTIONS.mw,
+    draft: true,
   },
   {
     id: 'nalunga',
@@ -243,6 +260,7 @@ const stations: IStationMeta[] = [
     longitude: 34.066,
     countryCode: 'mw',
     definitions: CLIMATE_CHART_DEFINTIONS.mw,
+    draft: true,
   },
   {
     id: 'bunda',

apps/picsa-tools/climate-tool/src/app/pages/site-select/site-select.page.html

@@ -4,7 +4,7 @@
       (onMarkerClick)="onMarkerClick($event)"
       [mapOptions]="mapOptions"
       [basemapOptions]="basemapOptions"
-      [markers]="mapMarkers"
+      [markers]="mapMarkers()"
       #picsaMap
     ></picsa-map>
   </div>

apps/picsa-tools/climate-tool/src/app/pages/site-select/site-select.page.ts

@@ -1,10 +1,9 @@
-import { Component, NgZone, OnInit, ViewChild } from '@angular/core';
+import { Component, computed, NgZone, OnInit, ViewChild } from '@angular/core';
 import { ActivatedRoute, Router } from '@angular/router';
-import { ConfigurationService } from '@picsa/configuration';
 import { IStationMeta } from '@picsa/models';
 import { IBasemapOptions, IMapMarker, IMapOptions, PicsaMapComponent } from '@picsa/shared/features/map/map';
 
-import { HARDCODED_STATIONS } from '../../data';
+import { ClimateDataService } from '../../services/climate-data.service';
 
 @Component({
   selector: 'climate-site-select',
@@ -21,17 +20,25 @@ export class SiteSelectPage implements OnInit {
     src: 'assets/mapTiles/raw/{z}/{x}/{y}.webp',
     maxNativeZoom: 8,
   };
-  mapMarkers: IMapMarker[] = [];
+  public mapMarkers = computed(() => {
+    const stations = this.dataService.stations();
+    const markers: IMapMarker[] = stations.map((station, _index) => ({
+      latlng: [station.latitude, station.longitude],
+      data: station,
+      number: _index + 1,
+      _index,
+    }));
+    return markers;
+  });
 
   constructor(
     private ngZone: NgZone,
     private router: Router,
     private route: ActivatedRoute,
-    private configurationService: ConfigurationService
+    private dataService: ClimateDataService
   ) {}
 
   ngOnInit() {
-    this.populateSites();
     this.getUserLocationAndSelectClosestStation();
   }
 
@@ -56,27 +63,6 @@ export class SiteSelectPage implements OnInit {
     });
   }
 
-  populateSites() {
-    let stations = HARDCODED_STATIONS;
-    const { climateTool, country_code } = this.configurationService.deploymentSettings();
-    const filterFn = climateTool?.station_filter;
-    if (filterFn) {
-      stations = stations.filter((station) => filterFn(station));
-    } else {
-      stations = stations.filter((station) => station.countryCode === country_code);
-    }
-    const markers: IMapMarker[] = stations.map((station, _index) => {
-      return {
-        latlng: [station.latitude, station.longitude],
-        data: station,
-        number: _index + 1,
-        _index,
-      };
-    });
-    this.mapMarkers = markers;
-    return { stations, markers };
-  }
-
   private getUserLocationAndSelectClosestStation() {
     if (navigator.geolocation) {
       navigator.geolocation.getCurrentPosition(
@@ -97,7 +83,7 @@ export class SiteSelectPage implements OnInit {
 
   private selectClosestStation(userLat: number, userLng: number) {
     let minDistance = Number.MAX_VALUE;
-    const nearest = this.mapMarkers.reduce((previous, current) => {
+    const nearest = this.mapMarkers().reduce((previous, current) => {
       const stationLat = current.latlng[0];
       const stationLng = current.latlng[1];
       const distance = this.calculateDistance(userLat, userLng, stationLat, stationLng);

apps/picsa-tools/climate-tool/src/app/pages/site-view/site-view.page.html

@@ -14,8 +14,8 @@
       (selectionChange)="handleStationSelect(select.value)"
       [value]="chartService.station?.id"
     >
-      @for(option of stationSelectOptions; track option.value){
-      <mat-option [value]="option.value">{{option.label}}</mat-option>
+      @for(option of stationSelectOptions(); track option.value){
+      <mat-option [value]="option.value" [attr.data-draft]="option.draft">{{option.label}}</mat-option>
       }
     </mat-select>
   </div>

apps/picsa-tools/climate-tool/src/app/pages/site-view/site-view.page.scss

@@ -46,3 +46,6 @@ mat-sidenav-content.page-content {
 mat-sidenav-container {
   margin-top: 0 !important;
 }
+mat-option[data-draft='true'] {
+  opacity: 0.5;
+}

apps/picsa-tools/climate-tool/src/app/pages/site-view/site-view.page.ts

@@ -4,6 +4,7 @@ import {
   ChangeDetectionStrategy,
   ChangeDetectorRef,
   Component,
+  computed,
   effect,
   OnDestroy,
   TemplateRef,
@@ -39,12 +40,12 @@ interface ISiteViewParams {
 export class ClimateSiteViewComponent implements OnDestroy, AfterViewInit {
   public showRotateAnimation = false;
 
-  public stationSelectOptions = Object.entries(this.dataService.dataByStation)
-    .map(([id, data]) => ({
-      value: id,
-      label: data.name,
-    }))
-    .sort((a, b) => (a.label > b.label ? 1 : -1));
+  public stationSelectOptions = computed(() => {
+    const stations = this.dataService.stations();
+    return stations
+      .map(({ id, name, draft }) => ({ value: id, label: name, draft }))
+      .sort((a, b) => (a.label > b.label ? 1 : -1));
+  });
 
   private viewId = toSignal(this.route.queryParams.pipe(map(({ view }: ISiteViewQueryParams) => view)));
   private siteId = toSignal(this.route.params.pipe(map(({ siteId }: ISiteViewParams) => siteId)));

apps/picsa-tools/climate-tool/src/app/services/climate-data.service.ts

@@ -1,4 +1,5 @@
-import { Injectable } from '@angular/core';
+import { computed, Injectable } from '@angular/core';
+import { ConfigurationService } from '@picsa/configuration';
 import { IChartMeta, IStationData, IStationMeta } from '@picsa/models';
 import { arrayToHashmap, loadCSV } from '@picsa/utils';
 
@@ -9,32 +10,70 @@ export class ClimateDataService {
   public activeChart: IChartMeta;
   public yValues: number[];
 
-  public dataByStation: { [name: string]: IStationMeta } = {};
+  /** Store populated station data to in-memory cache for fast future retrieval*/
+  private stationDataCache: { [name: string]: IStationMeta } = {};
 
-  constructor() {
-    this.dataByStation = arrayToHashmap(DATA.HARDCODED_STATIONS, 'id');
+  /** List of all stations for current  */
+  public stations = computed(() => {
+    const { climateTool, country_code } = this.configurationService.deploymentSettings();
+    const filterFn = climateTool?.station_filter;
+    if (filterFn) {
+      return DATA.HARDCODED_STATIONS.filter((station) => filterFn(station));
+    } else {
+      return DATA.HARDCODED_STATIONS.filter((station) => station.countryCode === country_code && !station.draft);
+    }
+  });
+
+  constructor(private configurationService: ConfigurationService) {
+    this.stationDataCache = arrayToHashmap(DATA.HARDCODED_STATIONS, 'id');
   }
 
   public async getStationMeta(stationID: string): Promise<IStationMeta> {
-    const station = this.dataByStation[stationID];
+    const station = this.stationDataCache[stationID];
     if (!station) {
       console.error('No data for station');
       return { data: [] as any[], name: 'Data not found' } as IStationMeta;
     }
     if (!station.data) {
       console.log('[Climate] Load Data', stationID);
       const summaries = await this.loadStationSummaries(stationID);
-      this.dataByStation[stationID].data = summaries;
+      const cleaned = this.hackCleanMissingValues(summaries);
+      this.stationDataCache[stationID].data = cleaned;
     }
     // HACK - ensure chart definitions don't persist across sites
-    return JSON.parse(JSON.stringify(this.dataByStation[stationID]));
+    return JSON.parse(JSON.stringify(this.stationDataCache[stationID]));
+  }
+
+  /**
+   * Remove data entries where 1 of start, end, length or rainfall not defined
+   * This may be a null value, or 0 entry
+   */
+  private hackCleanMissingValues(data: IStationData[]) {
+    return data.map((el) => {
+      const { Start, End, Rainfall, Length } = el;
+      const isMissingData = !Start || !End || !Rainfall || !Length;
+      if (isMissingData) {
+        el.Start = null as any;
+        el.End = null as any;
+        el.Length = null as any;
+        el.Rainfall = null as any;
+      }
+      return el;
+    });
   }
+
   private async loadStationSummaries(stationID: string) {
-    // TODO - ensure still working
     return loadCSV<IStationData>(`assets/summaries/${stationID}.csv`, {
       download: true,
       dynamicTyping: true,
       header: true,
+      transform: (v) => {
+        // Ensure null string parsed
+        if (v === 'null') return null;
+        // HACK - replace 0 with null value
+        if (v === '0') return null;
+        return v;
+      },
     });
   }
 }

apps/picsa-tools/climate-tool/src/assets/summaries/bunda.csv

@@ -1,5 +1,5 @@
 Year,Rainfall,Start,End,Length
-1966,516.5,null,287,null
+1966,516.5,,287,
 1967,649.5,133,285,152
 1968,1224.7,135,300,165
 1969,619.5,156,253,97
@@ -21,7 +21,7 @@ Year,Rainfall,Start,End,Length
 1985,1060.3,139,303,164
 1986,764.9,181,276,95
 1987,171.3,193,193,0
-1988,768.7,null,291,null
+1988,768.7,,291,
 1989,1030.4,134,304,170
 1990,759.9,148,281,133
 1991,805.1,149,271,122
@@ -31,8 +31,8 @@ Year,Rainfall,Start,End,Length
 1995,750.4,164,285,121
 1996,1099.2,156,280,124
 1997,934,146,273,127
-1998,499.8,176,null,null
-1999,601,null,285,null
+1998,499.8,176,,
+1999,601,,285,
 2000,559.3,195,278,83
 2001,482,172,238,66
 2002,1002.6,164,275,111
@@ -43,10 +43,10 @@ Year,Rainfall,Start,End,Length
 2007,937.7,173,285,112
 2008,999.9,133,278,145
 2009,766.1,183,259,76
-2010,933.6,159,null,null
-2011,468.9,null,null,null
-2012,970.9,null,294,null
-2013,0,null,null,null
-2014,651.3,null,290,null
-2015,77.4,null,null,null
-2016,333.7,null,275,null
+2010,933.6,159,,
+2011,468.9,,,
+2012,970.9,,294,
+2013,0,,,
+2014,651.3,,290,
+2015,77.4,,,
+2016,333.7,,275,

apps/picsa-tools/climate-tool/src/assets/summaries/chapananga.csv

@@ -1,5 +1,5 @@
 Year,Rainfall,Start,End,Length
-1952,565.7,null,298,null
+1952,565.7,,298,
 1953,626.4,152,274,122
 1954,913.6,161,303,142
 1955,774.5,144,297,153
@@ -62,8 +62,8 @@ Year,Rainfall,Start,End,Length
 2012,2313.3,162,285,123
 2013,3684.9,166,288,122
 2014,3213.5,163,263,100
-2015,449.2,null,266,null
+2015,449.2,,266,
 2016,836.6,149,268,119
 2017,543.8,134,271,137
 2018,951.4,146,259,113
-2019,180.1,null,184,null
+2019,180.1,,184,