Add support for Grid 3

__tests__/grid.ts

@@ -0,0 +1,20 @@
+import path from "path";
+import Grid from "../main/server/apps/grid";
+
+test.only("reads data grid data from a grid file", async () => {
+  // Arrange
+  const pathToTestFile = path.join(__dirname, "./grid-test-data.sqlite");
+  const gridDataSource = new Grid({
+    name: "Grid",
+    staticLocations: [pathToTestFile],
+    gridRootDirectories: [],
+  });
+
+  // Act
+  const phrases = await gridDataSource.getText();
+
+  // Assert
+  expect(phrases).toBe(
+    `hey guys and a bit like this.\ni'm sorry that you have any queries.\ni'll get the best price you are looking to get.\ni'm sorry that this would like a plan.`
+  );
+});

main/server/apps/grid.ts

@@ -0,0 +1,206 @@
+import fs from "fs";
+import sqlite3 from "sqlite3";
+import path from "path";
+import { getHashFromFile, hashString } from "../lib/hash";
+import { AAppDataGetters, AppName } from "./types";
+
+interface DatabaseRow {
+  Text: string;
+}
+
+class Grid extends AAppDataGetters {
+  private name: AppName = "Grid";
+  private staticLocations: string[];
+  private gridRootDirectories: string[];
+  private validLocations?: string[];
+
+  constructor({
+    name,
+    staticLocations,
+    gridRootDirectories,
+  }: {
+    name?: AppName;
+    staticLocations: string[];
+    gridRootDirectories: string[];
+  }) {
+    super();
+
+    if (name) {
+      this.name = name;
+    }
+
+    this.staticLocations = staticLocations;
+    this.gridRootDirectories = gridRootDirectories;
+  }
+
+  private async addValidStaticLocations(): Promise<string[]> {
+    const validStaticLocations = [];
+
+    for await (const location of this.staticLocations) {
+      try {
+        await fs.promises.access(location);
+
+        validStaticLocations.push(location);
+      } catch {}
+    }
+
+    return validStaticLocations;
+  }
+
+  /**
+   * Goes through the potential grid roots and looks for sqlite files.
+   * Each Grid 3 root can have any number of users, we return all the users that we find
+   *
+   * A Grid 3 folder has this structure:
+   * /Grid 3
+   *  /Users
+   *    /first
+   *      /en-GB
+   *        /Phrases
+   *          history.sqlite
+   *    /second
+   *      /en-GB
+   *        /Phrases
+   *          history.sqlite
+   */
+  private async addValidDynamicLocations(): Promise<string[]> {
+    const validDynamicLocations: string[] = [];
+
+    for await (const currentRoot of this.gridRootDirectories) {
+      try {
+        const userDirectory = path.join(currentRoot, "./Users");
+
+        await fs.promises.access(userDirectory);
+
+        const allFilesInUserDir = await fs.promises.readdir(userDirectory, {
+          withFileTypes: true,
+        });
+        const users = allFilesInUserDir.filter((source) =>
+          source.isDirectory()
+        );
+
+        for await (const currentUser of users) {
+          const userHistory = path.join(
+            userDirectory,
+            currentUser.name,
+            "./en-GB/Phrases/history.sqlite"
+          );
+
+          await fs.promises.access(userHistory);
+
+          validDynamicLocations.push(userHistory);
+        }
+      } catch {}
+    }
+
+    return validDynamicLocations;
+  }
+
+  private async getLocations(): Promise<string[]> {
+    if (this.validLocations) {
+      return this.validLocations;
+    }
+
+    const validDynamicLocations = await this.addValidDynamicLocations();
+    const validStaticLocations = await this.addValidStaticLocations();
+
+    this.validLocations = [...validDynamicLocations, ...validStaticLocations];
+
+    return this.validLocations;
+  }
+
+  async doesExist() {
+    try {
+      const locations = await this.getLocations();
+      return locations.length > 0;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * We get the raw contents of all the files and hash them.
+   *
+   * We combine the hashes so that if any file changes we
+   * re-fetch the text
+   */
+  async getHash() {
+    const locations = await this.getLocations();
+    const hashes = await Promise.all(
+      locations.map(async (location) => await getHashFromFile(location))
+    );
+    const allHashes = hashes.join("");
+
+    return hashString(allHashes);
+  }
+
+  async getText() {
+    const locations = await this.getLocations();
+
+    const phrases = await Promise.all(
+      locations.map(async (location) => await this.getTextForLocation(location))
+    );
+
+    // Every phrase needs to end in a full stop
+    const sentences = phrases.flat().map((phrase) => {
+      if (phrase.endsWith(".")) {
+        return phrase;
+      } else {
+        return `${phrase}.`;
+      }
+    });
+
+    const rawPhrases = sentences.join("\n");
+
+    return rawPhrases;
+  }
+
+  /**
+   * Queries the sqlite database given.
+   *
+   * Selects all the phrases in the PhraseHistory table.
+   *
+   * Only takes phrases with a real timestamp. For some reason
+   * every phrase exists in history once without a timestamp even
+   * when its never been said
+   */
+  private async getTextForLocation(location: string): Promise<string[]> {
+    const database = new sqlite3.Database(location);
+
+    const databaseResult = (await new Promise((resolve, reject) => {
+      database.all(
+        `
+        SELECT Text
+        FROM PhraseHistory ph
+        INNER JOIN Phrases p ON p.Id = ph.PhraseId 
+        WHERE "Timestamp" <> 0
+      `,
+        (err, result) => {
+          if (err) return reject(err);
+
+          resolve(result);
+        }
+      );
+    })) as Array<DatabaseRow>;
+
+    database.close();
+
+    const phrases = databaseResult.map((result) => result.Text);
+
+    return phrases;
+  }
+
+  getName() {
+    return this.name;
+  }
+
+  /**
+   * Join all the paths together as there is multiple paths
+   */
+  async getPath() {
+    const locations = await this.getLocations();
+    return locations.join(";");
+  }
+}
+
+export default Grid;

main/server/apps/index.ts

@@ -1,6 +1,8 @@
 import bufferReplace from "buffer-replace";
 import path from "path";
+import { app } from "electron";
 import PlainText from "./plain-text";
+import Grid from "./grid";
 import { AAppDataGetters, AppName } from "./types";
 
 const DASHER_PATHS = [
@@ -27,6 +29,19 @@ const dasher = new PlainText({
   },
 });
 
+const GRID_PATHS = [path.join(__dirname, "../../../test-data/grid.sqlite")];
+const GRID_ROOTS: Array<string> = [
+  path.join(__dirname, "../../../test-data/Grid 3"),
+  path.join(app.getPath("home"), "../Public/Documents/Smartbox/Grid 3"),
+  path.join(app.getPath("home"), "./Documents/Smartbox/Grid 3"),
+];
+
+const grid = new Grid({
+  name: "Grid",
+  staticLocations: GRID_PATHS,
+  gridRootDirectories: GRID_ROOTS,
+});
+
 export const appFactory = ({
   name,
   path,
@@ -37,9 +52,11 @@ export const appFactory = ({
   switch (name) {
     case "Dasher":
       return dasher;
+    case "Grid":
+      return grid;
     case "Plain Text":
       return new PlainText({ locations: [path] });
   }
 };
 
-export default { dasher };
+export default { dasher, grid };

main/server/apps/types.ts

@@ -1,4 +1,4 @@
-export type AppName = "Dasher" | "Plain Text";
+export type AppName = "Dasher" | "Grid" | "Plain Text";
 
 export abstract class AAppDataGetters {
   abstract doesExist(): Promise<boolean>;

main/server/index.ts

@@ -18,6 +18,10 @@ const getInstalledApps = async () => {
     installedApps.push(apps.dasher);
   }
 
+  if (await apps.grid.doesExist()) {
+    installedApps.push(apps.grid);
+  }
+
   return installedApps;
 };
 
@@ -365,6 +369,13 @@ export const registerIPCHandlers = async (): Promise<void> => {
       }
     }
 
+    if (!apps.find((a) => a.name === "Grid")) {
+      const grid = appFactory({ name: "Grid", path: "" });
+      if (await grid.doesExist()) {
+        possible.push("Grid");
+      }
+    }
+
     return possible;
   });
 

main/server/lib/hash.ts

@@ -1,12 +1,18 @@
 import fs from "fs";
 import crypto from "crypto";
 
+const HASH_TYPE = "sha256";
+
 export const getHashFromFile = (path: string): Promise<string> => {
   return new Promise((resolve, reject) => {
-    const hash = crypto.createHash("sha256");
+    const hash = crypto.createHash(HASH_TYPE);
     const stream = fs.createReadStream(path);
     stream.on("error", (err) => reject(err));
     stream.on("data", (chunk) => hash.update(chunk));
     stream.on("end", () => resolve(hash.digest("hex")));
   });
 };
+
+export const hashString = (input: string): string => {
+  return crypto.createHash(HASH_TYPE).update(input).digest("hex");
+};

package.json

@@ -41,7 +41,7 @@
     "react-virtuoso": "^1.0.0",
     "reflect-metadata": "^0.1.13",
     "reselect": "^4.0.0",
-    "sqlite3": "5.0.0",
+    "sqlite3": "^5.0.2",
     "typeorm": "^0.2.29",
     "uuid": "^8.3.2",
     "wink-eng-lite-model": "https://github.com/winkjs/wink-eng-lite-model/releases/download/1.1.0/wink-eng-lite-model-1.1.0.tgz",
@@ -58,6 +58,7 @@
     "@types/node": "^14.14.13",
     "@types/react": "^17.0.0",
     "@types/react-dom": "^17.0.0",
+    "@types/sqlite3": "^3.1.7",
     "@types/uuid": "^8.3.0",
     "@types/yargs": "^16.0.0",
     "@typescript-eslint/eslint-plugin": "^4.9.1",

renderer/pages/settings/add-source.tsx

@@ -65,6 +65,14 @@ const AddSource = () => {
         </Select>
       </Grid>
 
+      {selectedSource === "Grid" && (
+        <Grid item xs={12}>
+          <Typography variant="body1" color="error">
+            This will add phrases from every Grid user on this computer.
+          </Typography>
+        </Grid>
+      )}
+
       {selectedSource === "Plain Text" && (
         <Grid item container xs={12} alignItems="center" spacing={1}>
           <Grid item>

renderer/pages/setup/4.tsx

@@ -62,6 +62,14 @@ const ThirdStep = () => {
         </Typography>
       </Grid>
 
+      {appNames.includes("Grid") && (
+        <Grid item xs={8}>
+          <Typography variant="body1" color="error">
+            This will add phrases from every Grid user on this computer.
+          </Typography>
+        </Grid>
+      )}
+
       <Grid item xs={12} />
 
       <Grid item>