refactor TypeDef for arrays and records (#2032)

* refactor TypeDef for arrays and records

* fix trino test

* fix join parsing

* fix join dscriminators

* fixed array of array

* fix composite source

* ubreak postgres array?

packages/malloy-db-bigquery/src/bigquery_connection.ts

@@ -35,7 +35,7 @@ import {ResourceStream} from '@google-cloud/paginator';
 import * as googleCommon from '@google-cloud/common';
 import {GaxiosError} from 'gaxios';
 import {
-  arrayEachFields,
+  mkArrayDef,
   Connection,
   ConnectionConfig,
   Malloy,
@@ -520,14 +520,7 @@ export class BigQueryConnection
         // Malloy treats repeated values as an array of scalars.
         const malloyType = this.dialect.sqlTypeToMalloyType(type);
         if (malloyType) {
-          const arrayField: StructDef = {
-            ...structShared,
-            type: 'array',
-            elementTypeDef: malloyType,
-            join: 'many',
-            fields: arrayEachFields(malloyType),
-          };
-          structDef.fields.push(arrayField);
+          structDef.fields.push(mkArrayDef(malloyType, name, this.dialectName));
         }
       } else if (isRecord) {
         const ifRepeatedRecord: StructDef = {

packages/malloy-db-duckdb/src/duckdb.spec.ts

@@ -23,7 +23,7 @@
 
 import {DuckDBCommon} from './duckdb_common';
 import {DuckDBConnection} from './duckdb_connection';
-import {arrayEachFields, SQLSourceDef, StructDef} from '@malloydata/malloy';
+import {SQLSourceDef, StructDef, mkArrayDef} from '@malloydata/malloy';
 import {describeIfDatabaseAvailable} from '@malloydata/malloy/test';
 
 const [describe] = describeIfDatabaseAvailable(['duckdb']);
@@ -132,14 +132,9 @@ describe('DuckDBConnection', () => {
     it('parses arrays', () => {
       const structDef = makeStructDef();
       connection.fillStructDefFromTypeMap(structDef, {test: ARRAY_SCHEMA});
-      expect(structDef.fields[0]).toEqual({
-        name: 'test',
-        type: 'array',
-        elementTypeDef: intTyp,
-        join: 'many',
-        dialect: 'duckdb',
-        fields: arrayEachFields({type: 'number', numberType: 'integer'}),
-      });
+      expect(structDef.fields[0]).toEqual(
+        mkArrayDef({type: 'number', numberType: 'integer'}, 'test', 'duckdb')
+      );
     });
 
     it('parses inline', () => {

packages/malloy-db-duckdb/src/duckdb_common.ts

@@ -35,7 +35,7 @@ import {
   DuckDBDialect,
   SQLSourceDef,
   TableSourceDef,
-  arrayEachFields,
+  mkArrayDef,
 } from '@malloydata/malloy';
 import {BaseConnection} from '@malloydata/malloy/connection';
 
@@ -240,15 +240,7 @@ export abstract class DuckDBCommon
       } else {
         if (arrayMatch) {
           malloyType = this.dialect.sqlTypeToMalloyType(duckDBType);
-          const innerStructDef: StructDef = {
-            type: 'array',
-            elementTypeDef: malloyType,
-            name,
-            dialect: this.dialectName,
-            join: 'many',
-            fields: arrayEachFields(malloyType),
-          };
-          structDef.fields.push(innerStructDef);
+          structDef.fields.push(mkArrayDef(malloyType, name, this.dialectName));
         } else {
           structDef.fields.push({...malloyType, name});
         }

packages/malloy-db-postgres/src/postgres_connection.ts

@@ -42,9 +42,9 @@ import {
   RunSQLOptions,
   SQLSourceDef,
   TableSourceDef,
-  arrayEachFields,
   StreamingConnection,
   StructDef,
+  mkArrayDef,
 } from '@malloydata/malloy';
 import {BaseConnection} from '@malloydata/malloy/connection';
 
@@ -237,14 +237,7 @@ export class PostgresConnection
         const elementType = this.dialect.sqlTypeToMalloyType(
           row['element_type'] as string
         );
-        structDef.fields.push({
-          type: 'array',
-          elementTypeDef: elementType,
-          name,
-          dialect: this.dialectName,
-          join: 'many',
-          fields: arrayEachFields(elementType),
-        });
+        structDef.fields.push(mkArrayDef(elementType, name, this.dialectName));
       } else {
         const malloyType = this.dialect.sqlTypeToMalloyType(postgresDataType);
         structDef.fields.push({...malloyType, name});

packages/malloy-db-snowflake/src/snowflake_connection.ts

@@ -36,12 +36,12 @@ import {
   QueryDataRow,
   SnowflakeDialect,
   TestableConnection,
-  arrayEachFields,
   TinyParser,
   Dialect,
-  FieldDef,
   RecordDef,
-  ArrayTypeDef,
+  mkArrayDef,
+  AtomicFieldDef,
+  ArrayDef,
 } from '@malloydata/malloy';
 import {BaseConnection} from '@malloydata/malloy/connection';
 
@@ -75,7 +75,7 @@ class SnowField {
     readonly type: string,
     readonly dialect: Dialect
   ) {}
-  fieldDef(): FieldDef {
+  fieldDef(): AtomicFieldDef {
     return {
       ...this.dialect.sqlTypeToMalloyType(this.type),
       name: this.name,
@@ -102,15 +102,15 @@ class SnowObject extends SnowField {
     super(name, 'object', d);
   }
 
-  get fields(): FieldDef[] {
-    const fields: FieldDef[] = [];
+  get fields(): AtomicFieldDef[] {
+    const fields: AtomicFieldDef[] = [];
     for (const [_, fieldObj] of this.fieldMap) {
       fields.push(fieldObj.fieldDef());
     }
     return fields;
   }
 
-  fieldDef() {
+  fieldDef(): RecordDef {
     const rec: RecordDef = {
       type: 'record',
       name: this.name,
@@ -171,26 +171,27 @@ class SnowArray extends SnowField {
     }
   }
 
-  fieldDef(): ArrayTypeDef {
-    const arr: ArrayTypeDef = {
-      type: 'array',
-      name: this.name,
-      join: 'many',
-      dialect: this.dialect.name,
-      elementTypeDef: {type: 'string'},
-      fields: [],
-    };
+  fieldDef(): ArrayDef {
     if (this.objectChild) {
-      arr.fields = this.objectChild.fieldDef().fields;
-      arr.elementTypeDef = {type: 'record_element'};
-    } else if (this.arrayChild) {
-      arr.elementTypeDef = this.arrayChild.fieldDef();
-      arr.fields = arrayEachFields(arr.elementTypeDef);
-    } else {
-      arr.elementTypeDef = this.dialect.sqlTypeToMalloyType(this.arrayOf);
-      arr.fields = arrayEachFields(arr.elementTypeDef);
+      const t = mkArrayDef(
+        {type: 'record', fields: this.objectChild.fields},
+        this.name,
+        this.dialect.name
+      );
+      return t;
+    }
+    if (this.arrayChild) {
+      return mkArrayDef(
+        this.arrayChild.fieldDef(),
+        this.name,
+        this.dialect.name
+      );
     }
-    return arr;
+    return mkArrayDef(
+      this.dialect.sqlTypeToMalloyType(this.arrayOf),
+      this.name,
+      this.dialect.name
+    );
   }
 
   walk(path: PathChain, fieldType: string) {

packages/malloy-db-trino/src/trino_connection.spec.ts

@@ -21,7 +21,7 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-import {arrayEachFields, AtomicTypeDef, FieldDef} from '@malloydata/malloy';
+import {AtomicTypeDef, FieldDef} from '@malloydata/malloy';
 import {TrinoConnection, TrinoExecutor} from '.';
 
 // array(varchar) is array
@@ -64,22 +64,15 @@ describe('Trino connection', () => {
   describe('schema parser', () => {
     it('parses arrays', () => {
       expect(connection.malloyTypeFromTrinoType('test', ARRAY_SCHEMA)).toEqual({
-        'name': 'test',
-        'type': 'array',
-        'dialect': 'trino',
-        'elementTypeDef': intType,
-        'join': 'many',
-        'fields': arrayEachFields(intType),
+        type: 'array',
+        elementTypeDef: intType,
       });
     });
 
     it('parses inline', () => {
       expect(connection.malloyTypeFromTrinoType('test', INLINE_SCHEMA)).toEqual(
         {
-          'name': 'test',
           'type': 'record',
-          'dialect': 'trino',
-          'join': 'one',
           'fields': recordSchema,
         }
       );
@@ -88,11 +81,8 @@ describe('Trino connection', () => {
     it('parses nested', () => {
       expect(connection.malloyTypeFromTrinoType('test', NESTED_SCHEMA)).toEqual(
         {
-          'name': 'test',
           'type': 'array',
           'elementTypeDef': {type: 'record_element'},
-          'dialect': 'trino',
-          'join': 'many',
           'fields': recordSchema,
         }
       );
@@ -106,11 +96,8 @@ describe('Trino connection', () => {
 
     it('parses deep nesting', () => {
       expect(connection.malloyTypeFromTrinoType('test', DEEP_SCHEMA)).toEqual({
-        'name': 'test',
         'type': 'array',
-        'dialect': 'trino',
         'elementTypeDef': {type: 'record_element'},
-        'join': 'many',
         'fields': [
           {'name': 'a', ...doubleType},
           {