🐛 BigQuery source: Fix nested arrays

airbyte-db/src/main/java/io/airbyte/db/bigquery/BigQueryUtils.java

@@ -90,10 +90,26 @@ private static void setJsonField(Field field, FieldValue fieldValue, ObjectNode
     } else if (fieldValue.getAttribute().equals(Attribute.REPEATED)) {
       ArrayNode arrayNode = node.putArray(fieldName);
       StandardSQLTypeName fieldType = field.getType().getStandardType();
-      fieldValue.getRepeatedValue().forEach(arrayFieldValue -> fillObjectNode(fieldName, fieldType, arrayFieldValue, arrayNode.addObject()));
+      FieldList subFields = field.getSubFields();
+      // Array of primitive
+      if (subFields == null || subFields.isEmpty()) {
+        fieldValue.getRepeatedValue().forEach(arrayFieldValue -> fillObjectNode(fieldName, fieldType, arrayFieldValue, arrayNode.addObject()));
+        // Array of records
+      } else {
+        for (FieldValue arrayFieldValue : fieldValue.getRepeatedValue()) {
+          int count = 0; // named get doesn't work here for some reasons.
+          ObjectNode newNode = arrayNode.addObject();
+          for (Field repeatedField : subFields) {
+            setJsonField(repeatedField, arrayFieldValue.getRecordValue().get(count++),
+                newNode);
+          }
+        }
+      }
     } else if (fieldValue.getAttribute().equals(Attribute.RECORD)) {
       ObjectNode newNode = node.putObject(fieldName);
-      field.getSubFields().forEach(recordField -> setJsonField(recordField, fieldValue.getRecordValue().get(recordField.getName()), newNode));
+      field.getSubFields().forEach(recordField -> {
+        setJsonField(recordField, fieldValue.getRecordValue().get(recordField.getName()), newNode);
+      });
     }
   }
 
@@ -113,6 +129,8 @@ public static JsonSchemaPrimitive getType(StandardSQLTypeName bigQueryType) {
       case BOOL -> JsonSchemaPrimitive.BOOLEAN;
       case INT64, FLOAT64, NUMERIC, BIGNUMERIC -> JsonSchemaPrimitive.NUMBER;
       case STRING, BYTES, TIMESTAMP, DATE, TIME, DATETIME -> JsonSchemaPrimitive.STRING;
+      case ARRAY -> JsonSchemaPrimitive.ARRAY;
+      case STRUCT -> JsonSchemaPrimitive.OBJECT;
       default -> JsonSchemaPrimitive.STRING;
     };
   }

airbyte-integrations/connectors/source-bigquery/Dockerfile

@@ -9,5 +9,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
 RUN tar xf ${APPLICATION}.tar --strip-components=1
 
 # Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile.
-LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.version=0.1.1
 LABEL io.airbyte.name=airbyte/source-bigquery

airbyte-integrations/connectors/source-bigquery/build.gradle

@@ -17,8 +17,6 @@ dependencies {
     implementation project(':airbyte-integrations:connectors:source-jdbc')
     implementation project(':airbyte-integrations:connectors:source-relational-db')
 
-    //TODO Add jdbc driver import here. Ex: implementation 'com.microsoft.sqlserver:mssql-jdbc:8.4.1.jre14'
-
     testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc'))
 
     testImplementation 'org.apache.commons:commons-lang3:3.11'

airbyte-integrations/connectors/source-bigquery/src/test-integration/java/io/airbyte/integrations/source/bigquery/BigQuerySourceComprehensiveTest.java

@@ -316,6 +316,26 @@ protected void initTests() {
             .addInsertValues("STRUCT('s' as frst, 1 as sec, STRUCT(555 as id_col, STRUCT(TIME(15, 30, 00) as time) as mega_obbj) as obbj)")
             .addExpectedValues("{\"frst\":\"s\",\"sec\":1,\"obbj\":{\"id_col\":555,\"mega_obbj\":{\"last_col\":\"15:30:00\"}}}")
             .build());
+
+    addDataTypeTestData(
+        TestDataHolder.builder()
+            .sourceType("array")
+            .fullSourceDataType("array<STRUCT<fff String, ggg int64>>")
+            .airbyteType(JsonSchemaPrimitive.STRING)
+            .createTablePatternSql(CREATE_SQL_PATTERN)
+            .addInsertValues("[STRUCT('qqq' as fff, 1 as ggg), STRUCT('kkk' as fff, 2 as ggg)]")
+            .addExpectedValues("[{\"fff\":\"qqq\",\"ggg\":1},{\"fff\":\"kkk\",\"ggg\":2}]")
+            .build());
+
+    addDataTypeTestData(
+        TestDataHolder.builder()
+            .sourceType("array")
+            .fullSourceDataType("array<STRUCT<fff String, ggg array<STRUCT<ooo String, kkk int64>>>>")
+            .airbyteType(JsonSchemaPrimitive.STRING)
+            .createTablePatternSql(CREATE_SQL_PATTERN)
+            .addInsertValues("[STRUCT('qqq' as fff, [STRUCT('fff' as ooo, 1 as kkk), STRUCT('hhh' as ooo, 2 as kkk)] as ggg)]")
+            .addExpectedValues("[{\"fff\":\"qqq\",\"ggg\":[{\"ooo\":\"fff\",\"kkk\":1},{\"ooo\":\"hhh\",\"kkk\":2}]}]")
+            .build());
   }
 
   @Override