Some bug fixes and adding date to key path

fetcher/lib/providers.js

@@ -10,6 +10,7 @@ const {
     fetchSecret,
     getObject,
     putObject,
+    putFile,
     prettyPrintStation
 } = require('./utils');
 
@@ -149,6 +150,7 @@ class Providers {
 
         if (DRYRUN) {
             console.log(`Would have saved ${measures.length} measurements to '${Bucket}/${Key}'`);
+            putFile(compressedString, Key);
             return new Promise((y) => y(true));
         }
         if (VERBOSE) console.debug(`Saving measurements to ${Bucket}/${Key}`);
@@ -170,13 +172,13 @@ class Providers {
         const Bucket = process.env.BUCKET;
         const today = dayjs().format('YYYY-MM-DD');
         const filename = id || `${Math.floor(Date.now() / 1000)}-${Math.random().toString(36).substring(8)}`;
-        //const Key = `${process.env.STACK}/measures/${provider}/${filename}.json.gz`;
-        const Key = `local-testing/measures/${provider}/${today}/${filename}.json.gz`;
+        const Key = `${process.env.STACK}/measures/${provider}/${today}/${filename}.json.gz`;
         const compressedString = await gzip(JSON.stringify(data));
 
 
         if (DRYRUN) {
             console.log(`Would have saved ${data.measures.length} measurements and ${data.locations.length} stations to '${Bucket}/${Key}'`);
+            putFile(compressedString, Key);
             return new Promise((y) => y(true));
         }
         if (VERBOSE) console.debug(`Saving measurements to ${Bucket}/${Key}`);

fetcher/lib/utils.js

@@ -1,6 +1,8 @@
 const zlib = require('zlib');
 const { promisify } = require('util');
 const request = promisify(require('request'));
+const fs = require('node:fs');
+const path = require('path');
 
 const { SecretsManagerClient, GetSecretValueCommand } = require('@aws-sdk/client-secrets-manager');
 const { S3Client, GetObjectCommand, PutObjectCommand } = require('@aws-sdk/client-s3');
@@ -63,6 +65,20 @@ async function putObject(text, Bucket, Key, gzip = true, ContentType = 'applicat
     return await s3.send(cmd);
 }
 
+
+/**
+ *
+ * @param {} text
+ * @param {} key
+ */
+async function putFile(text, key) {
+    let fpath = `/home/christian/Downloads/${key}`;
+    await fs.mkdirSync(path.dirname(fpath), { recursive: true });
+    await fs.writeFileSync(fpath, text);
+}
+
+
+
 /**
  * Retrieve secret from AWS Secrets Manager
  * @param {string} source The source object for which we are fetching a secret.
@@ -199,6 +215,7 @@ module.exports = {
     DRYRUN,
     getObject,
     putObject,
+    putFile,
     prettyPrintStation,
     checkResponseData
 };

fetcher/providers/clarity.js

@@ -100,15 +100,10 @@ class ClarityApi {
      */
     getSensorId(meas) {
         const measurand = this.measurands[meas.metric];
-        const datasource = this.datasources[meas.datasourceId];
         if(!measurand) {
             throw new Error(`Could not find measurand for ${meas.metric}`);
         }
-        if(!datasource) {
-            this.addToMissingDatasources(meas);
-            throw new Error(`Could not find datasource for ${meas.datasourceId}/${meas.metric}`);
-        }
-        return `clarity-${datasource.datasourceId}-${measurand.parameter}`;
+        return `clarity-${meas.datasourceId}-${measurand.parameter}`;
     }
 
     getLocationId(loc) {
@@ -167,6 +162,7 @@ class ClarityApi {
                 this.locations.push({
                     location: this.getLocationId(d),
                     label: this.getLabel(d),
+                    ismobile: false,
                     lon: d.lon,
                     lat: d.lat,
                 });
@@ -207,7 +203,7 @@ class ClarityApi {
             meta: {
                 schema: 'v0.1',
                 source: 'clarity',
-                matching_method: 'source-spatial'
+                matching_method: 'ingest-id'
             },
             measures: this.measures.measures,
             locations: this.locations,