Skip to content

Commit

Permalink
Fix bug where bigquery table couldn't add columns (#4606) (#8803)
Browse files Browse the repository at this point in the history 
Signed-off-by: Modular Magician <magic-modules@google.com>
  • Loading branch information
@modular-magician
modular-magician authored Mar 30, 2021
1 parent 870286e commit 662007c
Show file tree
Hide file tree
Showing 3 changed files with 128 additions and 8 deletions.
3 changes: 3 additions & 0 deletions .changelog/4606.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:bug
bigquery: fixed issue where you couldn't extend an existing `schema` with additional columns in `google_bigquery_table`
```
23 changes: 17 additions & 6 deletions google/resource_bigquery_table.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,15 @@ func bigQueryTableSortArrayByName(array []interface{}) {
})
}

func bigQueryArrayToMapIndexedByName(array []interface{}) map[string]interface{} {
out := map[string]interface{}{}
for _, v := range array {
name := v.(map[string]interface{})["name"].(string)
out[name] = v
}
return out
}

func bigQueryTablecheckNameExists(jsonList []interface{}) error {
for _, m := range jsonList {
if _, ok := m.(map[string]interface{})["name"]; !ok {
Expand Down Expand Up @@ -197,14 +206,18 @@ func resourceBigQueryTableSchemaIsChangeable(old, new interface{}) (bool, error)
if err := bigQueryTablecheckNameExists(arrayOld); err != nil {
return false, err
}
bigQueryTableSortArrayByName(arrayOld)
mapOld := bigQueryArrayToMapIndexedByName(arrayOld)
if err := bigQueryTablecheckNameExists(arrayNew); err != nil {
return false, err
}
bigQueryTableSortArrayByName(arrayNew)
for i := range arrayOld {
mapNew := bigQueryArrayToMapIndexedByName(arrayNew)
for key := range mapOld {
// all old keys should be represented in the new config
if _, ok := mapNew[key]; !ok {
return false, nil
}
if isChangable, err :=
resourceBigQueryTableSchemaIsChangeable(arrayOld[i], arrayNew[i]); err != nil || !isChangable {
resourceBigQueryTableSchemaIsChangeable(mapOld[key], mapNew[key]); err != nil || !isChangable {
return false, err
}
}
Expand All @@ -216,15 +229,13 @@ func resourceBigQueryTableSchemaIsChangeable(old, new interface{}) (bool, error)
// if both aren't objects
return false, nil
}

var unionOfKeys map[string]bool = make(map[string]bool)
for key := range objectOld {
unionOfKeys[key] = true
}
for key := range objectNew {
unionOfKeys[key] = true
}

for key := range unionOfKeys {
valOld := objectOld[key]
valNew := objectNew[key]
Expand Down
110 changes: 108 additions & 2 deletions google/resource_bigquery_table_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -509,6 +509,39 @@ func TestAccBigQueryDataTable_canReorderParameters(t *testing.T) {
})
}

func TestAccBigQueryDataTable_expandArray(t *testing.T) {
t.Parallel()

datasetID := fmt.Sprintf("tf_test_%s", randString(t, 10))
tableID := fmt.Sprintf("tf_test_%s", randString(t, 10))

vcrTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckBigQueryTableDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccBigQueryTable_arrayInitial(datasetID, tableID),
},
{
ResourceName: "google_bigquery_table.test",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"etag", "last_modified_time", "deletion_protection"},
},
{
Config: testAccBigQueryTable_arrayExpanded(datasetID, tableID),
},
{
ResourceName: "google_bigquery_table.test",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"etag", "last_modified_time", "deletion_protection"},
},
},
})
}

func TestUnitBigQueryDataTable_jsonEquivalency(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -632,12 +665,12 @@ var testUnitBigQueryDataTableIsChangableTestCases = []testUnitBigQueryDataTableJ
{
name: "arraySizeIncreases",
jsonOld: "[{\"name\": \"someValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }]",
jsonNew: "[{\"name\": \"someValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }, {\"name\": \"someValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }]",
jsonNew: "[{\"name\": \"someValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }, {\"name\": \"asomeValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }]",
changeable: true,
},
{
name: "arraySizeDecreases",
jsonOld: "[{\"name\": \"someValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }, {\"name\": \"someValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }]",
jsonOld: "[{\"name\": \"someValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }, {\"name\": \"asomeValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }]",
jsonNew: "[{\"name\": \"someValue\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }]",
changeable: false,
},
Expand Down Expand Up @@ -1630,6 +1663,79 @@ resource "google_bigquery_table" "test" {
`, datasetID, tableID)
}

func testAccBigQueryTable_arrayInitial(datasetID, tableID string) string {
return fmt.Sprintf(`
resource "google_bigquery_dataset" "test" {
dataset_id = "%s"
}
resource "google_bigquery_table" "test" {
deletion_protection = false
table_id = "%s"
dataset_id = google_bigquery_dataset.test.dataset_id
friendly_name = "bigquerytest"
labels = {
"terrafrom_managed" = "true"
}
schema = jsonencode(
[
{
description = "Time snapshot was taken, in Epoch milliseconds. Same across all rows and all tables in the snapshot, and uniquely defines a particular snapshot."
name = "snapshot_timestamp"
mode = "NULLABLE"
type = "INTEGER"
},
{
description = "Timestamp of dataset creation"
name = "creation_time"
type = "TIMESTAMP"
},
])
}
`, datasetID, tableID)
}

func testAccBigQueryTable_arrayExpanded(datasetID, tableID string) string {
return fmt.Sprintf(`
resource "google_bigquery_dataset" "test" {
dataset_id = "%s"
}
resource "google_bigquery_table" "test" {
deletion_protection = false
table_id = "%s"
dataset_id = google_bigquery_dataset.test.dataset_id
friendly_name = "bigquerytest"
labels = {
"terrafrom_managed" = "true"
}
schema = jsonencode(
[
{
description = "Time snapshot was taken, in Epoch milliseconds. Same across all rows and all tables in the snapshot, and uniquely defines a particular snapshot."
name = "snapshot_timestamp"
mode = "NULLABLE"
type = "INTEGER"
},
{
description = "Timestamp of dataset creation"
name = "creation_time"
type = "TIMESTAMP"
},
{
description = "some new value"
name = "a_new_value"
type = "TIMESTAMP"
},
])
}
`, datasetID, tableID)
}

var TEST_CSV = `lifelock,LifeLock,,web,Tempe,AZ,1-May-07,6850000,USD,b
lifelock,LifeLock,,web,Tempe,AZ,1-Oct-06,6000000,USD,a
lifelock,LifeLock,,web,Tempe,AZ,1-Jan-08,25000000,USD,c
Expand Down

0 comments on commit 662007c

Please sign in to comment.