MySQL doesn't support FULL JOIN

UNNEST Support for pipelines
malloydata · Oct 6, 2024 · 0d08b88 · 0d08b88
1 parent 50b092a
commit 0d08b88
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 28 deletions.
diff --git a/packages/malloy/src/dialect/dialect.ts b/packages/malloy/src/dialect/dialect.ts
@@ -155,6 +155,9 @@ export abstract class Dialect {
 
   supportsCountApprox = false;
 
+  // MYSQL doesn't have full join, maybe others.
+  supportsFullJoin = true;
+
   nativeBoolean = true;
 
   abstract getDialectFunctionOverrides(): {

diff --git a/packages/malloy/src/dialect/mysql/mysql.ts b/packages/malloy/src/dialect/mysql/mysql.ts
@@ -109,6 +109,7 @@ export class MySQLDialect extends Dialect {
   supportsNesting = true;
   experimental = true;
   nativeBoolean = false;
+  supportsFullJoin = false;
 
   malloyTypeToSQLType(malloyType: FieldAtomicTypeDef): string {
     if (malloyType.type === 'number') {
@@ -192,23 +193,39 @@ export class MySQLDialect extends Dialect {
     return `COALESCE(MAX(CASE WHEN group_set=${groupSet} THEN JSON_OBJECT(${fields}) END),JSON_OBJECT(${nullValues}))`;
   }
 
-  // TODO: investigate if its possible to make it work when source is table.field.
+  malloyToSQL(t: string) {
+    if (t === 'number') {
+      return 'DOUBLE';
+    } else if (t === 'string') {
+      return 'TEXT';
+    } else return t;
+  }
+  unnestColumns(fieldList: DialectFieldList) {
+    const fields: string[] = [];
+    for (const f of fieldList) {
+      fields.push(
+        `${f.sqlOutputName} ${this.malloyToSQL(f.type)} PATH "$.${f.rawName}"`
+      );
+    }
+    return fields.join(',\n');
+  }
+
+  // LTNOTE: We'll make this work with Arrays once MToy's changes land.
   sqlUnnestAlias(
-    _source: string,
-    _alias: string,
-    _fieldList: DialectFieldList,
+    source: string,
+    alias: string,
+    fieldList: DialectFieldList,
     _needDistinctKey: boolean,
     _isArray: boolean,
     _isInNestedPipeline: boolean
   ): string {
-    throw new Error('MySQL dialect does not support unnest.');
-    /* if (isArray) {
-      throw new Error('MySQL dialect does not support unnest.');
-    } else if (needDistinctKey) {
-      return `LEFT JOIN JSON_TABLE(cast(concat("[1",repeat(",1",JSON_LENGTH(${source})),"]") as JSON),"$[*]" COLUMNS(__row_id FOR ORDINALITY)) as ${alias} ON ${alias}.\`__row_id\` <= JSON_LENGTH(${source})`;
-    } else {
-      return `LEFT JOIN (SELECT json_unquote(json_extract(${source}, CONCAT('$[', __row_id - 1, ']'))) as ${alias}  FROM (SELECT json_unquote(json_extract(${source}, CONCAT('$[', __row_id, ']'))) as d) as b LEFT JOIN JSON_TABLE(cast(concat("[1",repeat(",1",JSON_LENGTH(${source}) - 1),"]") as JSON),"$[*]" COLUMNS(__row_id FOR ORDINALITY)) as e on TRUE) as __tbl ON true`;
-    } */
+    return `
+      LEFT JOIN JSON_TABLE(${source}, '$[*]'
+        COLUMNS (
+          __row_id FOR ORDINALITY,
+          ${this.unnestColumns(fieldList)}
+        )
+      ) as ${alias} ON 1=1`;
   }
 
   sqlSumDistinctHashedKey(_sqlDistinctKey: string): string {

diff --git a/packages/malloy/src/model/malloy_query.ts b/packages/malloy/src/model/malloy_query.ts
@@ -2787,6 +2787,9 @@ class QueryQuery extends QueryField {
     let structSQL = qs.structSourceSQL(stageWriter);
     if (isJoinOn(structRelationship)) {
       const matrixOperation = structRelationship.matrixOperation.toUpperCase();
+      if (!this.parent.dialect.supportsFullJoin && matrixOperation === 'FULL') {
+        throw new Error('FULL JOIN not supported');
+      }
       if (ji.makeUniqueKey) {
         const passKeys = this.generateSQLPassthroughKeys(qs);
         structSQL = `(SELECT ${qs.dialect.sqlGenerateUUID()} as ${qs.dialect.sqlMaybeQuoteIdentifier(

diff --git a/test/src/databases/all/nomodel.spec.ts b/test/src/databases/all/nomodel.spec.ts
@@ -25,11 +25,7 @@
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 
 import {RuntimeList, allDatabases} from '../../runtimes';
-import {
-  booleanCode,
-  booleanResult,
-  databasesFromEnvironmentOr,
-} from '../../util';
+import {databasesFromEnvironmentOr} from '../../util';
 import '../../util/db-jest-matchers';
 
 const runtimes = new RuntimeList(databasesFromEnvironmentOr(allDatabases));
@@ -422,11 +418,13 @@ runtimes.runtimeMap.forEach((runtime, databaseName) => {
     });
   });
 
-  it(`join full - ${databaseName}`, async () => {
-    // a cross join produces a Many to Many result.
-    // symmetric aggregate are needed on both sides of the join
-    // Check the row count and that sums on each side work properly.
-    await expect(`
+  it.when(runtime.dialect.supportsFullJoin)(
+    `join full - ${databaseName}`,
+    async () => {
+      // a cross join produces a Many to Many result.
+      // symmetric aggregate are needed on both sides of the join
+      // Check the row count and that sums on each side work properly.
+      await expect(`
       ${matrixModel}
       run: ac_states -> {
         extend: {
@@ -440,12 +438,13 @@ runtimes.runtimeMap.forEach((runtime, databaseName) => {
 
       }
       `).malloyResultMatches(runtime, {
-      ac_count: 49,
-      ac_sum: 21336,
-      am_count: 12,
-      am_sum: 4139,
-    });
-  });
+        ac_count: 49,
+        ac_sum: 21336,
+        am_count: 12,
+        am_sum: 4139,
+      });
+    }
+  );
 
   it(`leafy count - ${databaseName}`, async () => {
     // in a joined table when the joined is leafiest