From 33047930e5670c0fffeaebbbffca6671d9ec35d5 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Thu, 20 Jul 2017 17:41:08 -0400 Subject: [PATCH 1/9] Support views in Terraform.BigQuery --- google/resource_bigquery_dataset.go | 21 +++++++++++++++ google/resource_bigquery_table.go | 41 ++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/google/resource_bigquery_dataset.go b/google/resource_bigquery_dataset.go index 8080b8dbc9a..45fad71be43 100644 --- a/google/resource_bigquery_dataset.go +++ b/google/resource_bigquery_dataset.go @@ -238,6 +238,12 @@ func resourceBigQueryDatasetRead(d *schema.ResourceData, meta interface{}) error d.Set("dataset_id", res.DatasetReference.DatasetId) d.Set("default_table_expiration_ms", res.DefaultTableExpirationMs) + // Older Tables in BigQuery have no Location set in the API response. We can safely assume that these tables + // are in the US. + if res.Location == "" { + d.Set("location", "US") + } + return nil } @@ -274,3 +280,18 @@ func resourceBigQueryDatasetDelete(d *schema.ResourceData, meta interface{}) err d.SetId("") return nil } + +// +//func resourceBigQueryDatasetImport(d *schema.ResourceData, m interface{}) ([]*ResourceData, error) { +// x := schema.ImportStatePassthrough(d, m) +// +// // +// // +// // // HACK(jmcgill) +// // +// // return []*ResourceData{d}, nil +// //} +// // test := &schema.ResourceImporter{ +// // State: schema.ImportStatePassthrough, +// // }, +//} diff --git a/google/resource_bigquery_table.go b/google/resource_bigquery_table.go index 298152a86c8..de8b0be63c1 100644 --- a/google/resource_bigquery_table.go +++ b/google/resource_bigquery_table.go @@ -92,6 +92,23 @@ func resourceBigQueryTable() *schema.Resource { }, }, + // View: [Experimental] If specified, configures this table as a view. + "view": &schema.Schema{ + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + // Type: [Required] The only type supported is DAY, which will generate + // one partition per day based on data loading time. + "query": { + Type: schema.TypeString, + Required: true, + }, + }, + }, + }, + // TimePartitioning: [Experimental] If specified, configures time-based // partitioning for this table. "time_partitioning": &schema.Schema{ @@ -202,12 +219,17 @@ func resourceTable(d *schema.ResourceData, meta interface{}) (*bigquery.Table, e }, } + // TODO(jmcgill): How do I do nested objects in a terraform configuration. + if v, ok := d.GetOk("view"); ok { + table.View = expandView(v) + } + if v, ok := d.GetOk("description"); ok { table.Description = v.(string) } if v, ok := d.GetOk("expiration_time"); ok { - table.ExpirationTime = v.(int64) + table.ExpirationTime = int64(v.(int)) } if v, ok := d.GetOk("friendly_name"); ok { @@ -317,6 +339,11 @@ func resourceBigQueryTableRead(d *schema.ResourceData, meta interface{}) error { d.Set("schema", schema) } + if res.View != nil { + view := flattenView(res.View) + d.Set("view", view) + } + return nil } @@ -394,3 +421,15 @@ func flattenTimePartitioning(tp *bigquery.TimePartitioning) []map[string]interfa return []map[string]interface{}{result} } + +func expandView(configured interface{}) *bigquery.ViewDefinition { + raw := configured.([]interface{})[0].(map[string]interface{}) + vd := &bigquery.ViewDefinition{Query: raw["query"].(string)} + + return vd +} + +func flattenView(v *bigquery.ViewDefinition) []map[string]interface{} { + result := map[string]interface{}{"query": v.Query} + return []map[string]interface{}{result} +} From d161eca7bb386099a01b88df1ec450259c968f1d Mon Sep 17 00:00:00 2001 From: Jimmy Date: Thu, 20 Jul 2017 22:58:29 -0400 Subject: [PATCH 2/9] Add tests for Table with view, and fix existing Table test --- google/resource_bigquery_table_test.go | 72 +++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/google/resource_bigquery_table_test.go b/google/resource_bigquery_table_test.go index f01b7e0f473..72f73113d82 100644 --- a/google/resource_bigquery_table_test.go +++ b/google/resource_bigquery_table_test.go @@ -2,6 +2,7 @@ package google import ( "fmt" + "strings" "testing" "github.com/hashicorp/terraform/helper/acctest" @@ -37,6 +38,26 @@ func TestAccBigQueryTable_Basic(t *testing.T) { }) } +func TestAccBigQueryTable_View(t *testing.T) { + datasetID := fmt.Sprintf("tf_test_%s", acctest.RandString(10)) + tableID := fmt.Sprintf("tf_test_%s", acctest.RandString(10)) + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckBigQueryTableDestroy, + Steps: []resource.TestStep{ + { + Config: testAccBigQueryTableWithView(datasetID, tableID), + Check: resource.ComposeTestCheckFunc( + testAccBigQueryTableExistsWithView( + "google_bigquery_table.test"), + ), + }, + }, + }) +} + func testAccCheckBigQueryTableDestroy(s *terraform.State) error { for _, rs := range s.RootModule().Resources { if rs.Type != "google_bigquery_table" { @@ -64,11 +85,40 @@ func testAccBigQueryTableExists(n string) resource.TestCheckFunc { return fmt.Errorf("No ID is set") } config := testAccProvider.Meta().(*Config) - _, err := config.clientBigQuery.Tables.Get(config.Project, rs.Primary.Attributes["dataset_id"], rs.Primary.Attributes["name"]).Do() + table, err := config.clientBigQuery.Tables.Get(config.Project, rs.Primary.Attributes["dataset_id"], rs.Primary.Attributes["table_id"]).Do() + if err != nil { + return fmt.Errorf("BigQuery Table not present") + } + + if !strings.HasSuffix(table.Id, rs.Primary.Attributes["table_id"]) { + return fmt.Errorf("BigQuery Table ID does not match expected value") + } + + return nil + } +} + +func testAccBigQueryTableExistsWithView(n string) resource.TestCheckFunc { + return func(s *terraform.State) error { + rs, ok := s.RootModule().Resources[n] + if !ok { + return fmt.Errorf("Not found: %s", n) + } + + if rs.Primary.ID == "" { + return fmt.Errorf("No ID is set") + } + config := testAccProvider.Meta().(*Config) + + table, err := config.clientBigQuery.Tables.Get(config.Project, rs.Primary.Attributes["dataset_id"], rs.Primary.Attributes["table_id"]).Do() if err != nil { return fmt.Errorf("BigQuery Table not present") } + if table.View == nil { + return fmt.Errorf("View object missing on table") + } + return nil } } @@ -114,6 +164,26 @@ EOH }`, datasetID, tableID) } +func testAccBigQueryTableWithView(datasetID, tableID string) string { + return fmt.Sprintf(` +resource "google_bigquery_dataset" "test" { + dataset_id = "%s" +} + +resource "google_bigquery_table" "test" { + table_id = "%s" + dataset_id = "${google_bigquery_dataset.test.dataset_id}" + + time_partitioning { + type = "DAY" + } + + view { + query = "SELECT state FROM [lookerdata:cdc.project_tycho_reports]" + } +}`, datasetID, tableID) +} + func testAccBigQueryTableUpdated(datasetID, tableID string) string { return fmt.Sprintf(` resource "google_bigquery_dataset" "test" { From d8aa4f4d6e2ca40a46522bff8c99e37534f34646 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Thu, 20 Jul 2017 23:02:52 -0400 Subject: [PATCH 3/9] Remove dead code --- google/resource_bigquery_dataset.go | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/google/resource_bigquery_dataset.go b/google/resource_bigquery_dataset.go index 45fad71be43..fbb5581277f 100644 --- a/google/resource_bigquery_dataset.go +++ b/google/resource_bigquery_dataset.go @@ -279,19 +279,4 @@ func resourceBigQueryDatasetDelete(d *schema.ResourceData, meta interface{}) err d.SetId("") return nil -} - -// -//func resourceBigQueryDatasetImport(d *schema.ResourceData, m interface{}) ([]*ResourceData, error) { -// x := schema.ImportStatePassthrough(d, m) -// -// // -// // -// // // HACK(jmcgill) -// // -// // return []*ResourceData{d}, nil -// //} -// // test := &schema.ResourceImporter{ -// // State: schema.ImportStatePassthrough, -// // }, -//} +} \ No newline at end of file From efac6313f64cfb520faf127df3871fb47a1e5a37 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Thu, 20 Jul 2017 23:08:09 -0400 Subject: [PATCH 4/9] run gofmt --- google/resource_bigquery_dataset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/resource_bigquery_dataset.go b/google/resource_bigquery_dataset.go index fbb5581277f..da9935bcd78 100644 --- a/google/resource_bigquery_dataset.go +++ b/google/resource_bigquery_dataset.go @@ -279,4 +279,4 @@ func resourceBigQueryDatasetDelete(d *schema.ResourceData, meta interface{}) err d.SetId("") return nil -} \ No newline at end of file +} From fbc53912d6a391590a0b964e35e19ba03b99b565 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Thu, 17 Aug 2017 17:04:58 +1200 Subject: [PATCH 5/9] Address comments --- google/resource_bigquery_dataset.go | 8 +++++--- google/resource_bigquery_table.go | 28 +++++++++++++++++++++----- google/resource_bigquery_table_test.go | 2 +- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/google/resource_bigquery_dataset.go b/google/resource_bigquery_dataset.go index da9935bcd78..2f2b2bfd386 100644 --- a/google/resource_bigquery_dataset.go +++ b/google/resource_bigquery_dataset.go @@ -229,7 +229,6 @@ func resourceBigQueryDatasetRead(d *schema.ResourceData, meta interface{}) error d.Set("etag", res.Etag) d.Set("labels", res.Labels) - d.Set("location", res.Location) d.Set("self_link", res.SelfLink) d.Set("description", res.Description) d.Set("friendly_name", res.FriendlyName) @@ -238,10 +237,13 @@ func resourceBigQueryDatasetRead(d *schema.ResourceData, meta interface{}) error d.Set("dataset_id", res.DatasetReference.DatasetId) d.Set("default_table_expiration_ms", res.DefaultTableExpirationMs) - // Older Tables in BigQuery have no Location set in the API response. We can safely assume that these tables - // are in the US. + // Older Tables in BigQuery have no Location set in the API response. This may be an issue when importing + // tables created before BigQuery was available in multiple zones. We can safely assume that these tables + // are in the US, as this was the default at the time. if res.Location == "" { d.Set("location", "US") + } else { + d.Set("location", res.Location) } return nil diff --git a/google/resource_bigquery_table.go b/google/resource_bigquery_table.go index de8b0be63c1..8decf429a73 100644 --- a/google/resource_bigquery_table.go +++ b/google/resource_bigquery_table.go @@ -92,19 +92,29 @@ func resourceBigQueryTable() *schema.Resource { }, }, - // View: [Experimental] If specified, configures this table as a view. + // View: [Optional] If specified, configures this table as a view. "view": &schema.Schema{ Type: schema.TypeList, Optional: true, MaxItems: 1, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ - // Type: [Required] The only type supported is DAY, which will generate - // one partition per day based on data loading time. + // Query: [Required] A query that BigQuery executes when the view is + // referenced. "query": { Type: schema.TypeString, Required: true, }, + + // UseLegacySQL: [Optional] Specifies whether to use BigQuery's + // legacy SQL for this view. The default value is true. If set to + // false, the view will use BigQuery's standard SQL: + "use_legacy_sql": { + Type: schema.TypeBool, + Required: false, + }, + + }, }, }, @@ -219,7 +229,6 @@ func resourceTable(d *schema.ResourceData, meta interface{}) (*bigquery.Table, e }, } - // TODO(jmcgill): How do I do nested objects in a terraform configuration. if v, ok := d.GetOk("view"); ok { table.View = expandView(v) } @@ -426,10 +435,19 @@ func expandView(configured interface{}) *bigquery.ViewDefinition { raw := configured.([]interface{})[0].(map[string]interface{}) vd := &bigquery.ViewDefinition{Query: raw["query"].(string)} + if v, ok := raw["use_legacy_sql"]; ok { + vd.UseLegacySql = v.(bool) + } + return vd } -func flattenView(v *bigquery.ViewDefinition) []map[string]interface{} { +func flattenView(vd *bigquery.ViewDefinition) []map[string]interface{} { result := map[string]interface{}{"query": v.Query} + + if vd.UseLegacySql { + result["use_legacy_sql"] = vd.UseLegacySql + } + return []map[string]interface{}{result} } diff --git a/google/resource_bigquery_table_test.go b/google/resource_bigquery_table_test.go index 72f73113d82..b127030ea8b 100644 --- a/google/resource_bigquery_table_test.go +++ b/google/resource_bigquery_table_test.go @@ -65,7 +65,7 @@ func testAccCheckBigQueryTableDestroy(s *terraform.State) error { } config := testAccProvider.Meta().(*Config) - _, err := config.clientBigQuery.Tables.Get(config.Project, rs.Primary.Attributes["dataset_id"], rs.Primary.Attributes["name"]).Do() + _, err := config.clientBigQuery.Tables.Get(config.Project, rs.Primary.Attributes["dataset_id"], rs.Primary.Attributes["table_id"]).Do() if err == nil { return fmt.Errorf("Table still present") } From 61f67c2defd9fdb8627e623dc670f606cc9e3ffa Mon Sep 17 00:00:00 2001 From: Jimmy Date: Thu, 17 Aug 2017 17:27:45 +1200 Subject: [PATCH 6/9] Address review comments and add support for use_legacy_sql --- google/resource_bigquery_table.go | 6 ++---- google/resource_bigquery_table_test.go | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/google/resource_bigquery_table.go b/google/resource_bigquery_table.go index 8decf429a73..6ee2560de22 100644 --- a/google/resource_bigquery_table.go +++ b/google/resource_bigquery_table.go @@ -111,10 +111,8 @@ func resourceBigQueryTable() *schema.Resource { // false, the view will use BigQuery's standard SQL: "use_legacy_sql": { Type: schema.TypeBool, - Required: false, + Optional: true, }, - - }, }, }, @@ -443,7 +441,7 @@ func expandView(configured interface{}) *bigquery.ViewDefinition { } func flattenView(vd *bigquery.ViewDefinition) []map[string]interface{} { - result := map[string]interface{}{"query": v.Query} + result := map[string]interface{}{"query": vd.Query} if vd.UseLegacySql { result["use_legacy_sql"] = vd.UseLegacySql diff --git a/google/resource_bigquery_table_test.go b/google/resource_bigquery_table_test.go index b127030ea8b..6e6ab26dcbf 100644 --- a/google/resource_bigquery_table_test.go +++ b/google/resource_bigquery_table_test.go @@ -180,6 +180,7 @@ resource "google_bigquery_table" "test" { view { query = "SELECT state FROM [lookerdata:cdc.project_tycho_reports]" + use_legacy_sql = true } }`, datasetID, tableID) } From aefd68a968c4cc853b4297b8b110ba814ec8590b Mon Sep 17 00:00:00 2001 From: Jimmy Date: Tue, 22 Aug 2017 12:57:35 -0400 Subject: [PATCH 7/9] Force transmission/storage of UseLegacySQL --- google/resource_bigquery_table.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/google/resource_bigquery_table.go b/google/resource_bigquery_table.go index 6ee2560de22..7ca5a707f5d 100644 --- a/google/resource_bigquery_table.go +++ b/google/resource_bigquery_table.go @@ -435,6 +435,7 @@ func expandView(configured interface{}) *bigquery.ViewDefinition { if v, ok := raw["use_legacy_sql"]; ok { vd.UseLegacySql = v.(bool) + vd.ForceSendFields = append(vd.ForceSendFields, "UseLegacySql") } return vd @@ -442,10 +443,7 @@ func expandView(configured interface{}) *bigquery.ViewDefinition { func flattenView(vd *bigquery.ViewDefinition) []map[string]interface{} { result := map[string]interface{}{"query": vd.Query} - - if vd.UseLegacySql { - result["use_legacy_sql"] = vd.UseLegacySql - } + result["use_legacy_sql"] = vd.UseLegacySql return []map[string]interface{}{result} } From aa67b7b060bdde6b046bffd87df7928ffffb2075 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Thu, 24 Aug 2017 15:57:45 -0400 Subject: [PATCH 8/9] Trying to fix tests --- google/resource_bigquery_table_test.go | 64 ++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/google/resource_bigquery_table_test.go b/google/resource_bigquery_table_test.go index 6e6ab26dcbf..36d79731cd6 100644 --- a/google/resource_bigquery_table_test.go +++ b/google/resource_bigquery_table_test.go @@ -2,6 +2,7 @@ package google import ( "fmt" + "strconv" "strings" "testing" @@ -48,7 +49,7 @@ func TestAccBigQueryTable_View(t *testing.T) { CheckDestroy: testAccCheckBigQueryTableDestroy, Steps: []resource.TestStep{ { - Config: testAccBigQueryTableWithView(datasetID, tableID), + Config: testAccBigQueryTableWithView(datasetID, tableID, false), Check: resource.ComposeTestCheckFunc( testAccBigQueryTableExistsWithView( "google_bigquery_table.test"), @@ -58,6 +59,33 @@ func TestAccBigQueryTable_View(t *testing.T) { }) } +func TestAccBigQueryTable_ViewWithLegacySQL(t *testing.T) { + datasetID := fmt.Sprintf("tf_test_%s", acctest.RandString(10)) + tableID := fmt.Sprintf("tf_test_%s", acctest.RandString(10)) + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckBigQueryTableDestroy, + Steps: []resource.TestStep{ + { + Config: testAccBigQueryTableWithView(datasetID, tableID, true), + Check: resource.ComposeTestCheckFunc( + testAccBigQueryTableExistsWithLegacySql( + "google_bigquery_table.test", true), + ), + }, + { + Config: testAccBigQueryTableWithView(datasetID, tableID, false), + Check: resource.ComposeTestCheckFunc( + testAccBigQueryTableExistsWithLegacySql( + "google_bigquery_table.test", false), + ), + }, + }, + }) +} + func testAccCheckBigQueryTableDestroy(s *terraform.State) error { for _, rs := range s.RootModule().Resources { if rs.Type != "google_bigquery_table" { @@ -123,6 +151,35 @@ func testAccBigQueryTableExistsWithView(n string) resource.TestCheckFunc { } } +func testAccBigQueryTableExistsWithLegacySql(n string, useLegacySql bool) resource.TestCheckFunc { + return func(s *terraform.State) error { + rs, ok := s.RootModule().Resources[n] + if !ok { + return fmt.Errorf("Not found: %s", n) + } + + if rs.Primary.ID == "" { + return fmt.Errorf("No ID is set") + } + config := testAccProvider.Meta().(*Config) + + table, err := config.clientBigQuery.Tables.Get(config.Project, rs.Primary.Attributes["dataset_id"], rs.Primary.Attributes["table_id"]).Do() + if err != nil { + return fmt.Errorf("BigQuery Table not present") + } + + if table.View == nil { + return fmt.Errorf("View object missing on table") + } + + if table.View.UseLegacySql != useLegacySql { + return fmt.Errorf("Value of UseLegacySQL does not match expected value") + } + + return nil + } +} + func testAccBigQueryTable(datasetID, tableID string) string { return fmt.Sprintf(` resource "google_bigquery_dataset" "test" { @@ -164,7 +221,7 @@ EOH }`, datasetID, tableID) } -func testAccBigQueryTableWithView(datasetID, tableID string) string { +func testAccBigQueryTableWithView(datasetID, tableID string, useLegacySql bool) string { return fmt.Sprintf(` resource "google_bigquery_dataset" "test" { dataset_id = "%s" @@ -180,9 +237,8 @@ resource "google_bigquery_table" "test" { view { query = "SELECT state FROM [lookerdata:cdc.project_tycho_reports]" - use_legacy_sql = true } -}`, datasetID, tableID) +}`, datasetID, tableID, strconv.FormatBool(useLegacySql)) } func testAccBigQueryTableUpdated(datasetID, tableID string) string { From f5b30369b6ccdc8345945cb1f430875e3785935d Mon Sep 17 00:00:00 2001 From: Jimmy Date: Thu, 24 Aug 2017 17:09:26 -0400 Subject: [PATCH 9/9] add tests for useLegacySQL --- google/resource_bigquery_table_test.go | 33 +++++++++++++++++++++----- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/google/resource_bigquery_table_test.go b/google/resource_bigquery_table_test.go index 36d79731cd6..1e681795166 100644 --- a/google/resource_bigquery_table_test.go +++ b/google/resource_bigquery_table_test.go @@ -2,7 +2,6 @@ package google import ( "fmt" - "strconv" "strings" "testing" @@ -49,7 +48,7 @@ func TestAccBigQueryTable_View(t *testing.T) { CheckDestroy: testAccCheckBigQueryTableDestroy, Steps: []resource.TestStep{ { - Config: testAccBigQueryTableWithView(datasetID, tableID, false), + Config: testAccBigQueryTableWithView(datasetID, tableID), Check: resource.ComposeTestCheckFunc( testAccBigQueryTableExistsWithView( "google_bigquery_table.test"), @@ -69,14 +68,14 @@ func TestAccBigQueryTable_ViewWithLegacySQL(t *testing.T) { CheckDestroy: testAccCheckBigQueryTableDestroy, Steps: []resource.TestStep{ { - Config: testAccBigQueryTableWithView(datasetID, tableID, true), + Config: testAccBigQueryTableWithView(datasetID, tableID), Check: resource.ComposeTestCheckFunc( testAccBigQueryTableExistsWithLegacySql( "google_bigquery_table.test", true), ), }, { - Config: testAccBigQueryTableWithView(datasetID, tableID, false), + Config: testAccBigQueryTableWithNewSqlView(datasetID, tableID), Check: resource.ComposeTestCheckFunc( testAccBigQueryTableExistsWithLegacySql( "google_bigquery_table.test", false), @@ -221,7 +220,7 @@ EOH }`, datasetID, tableID) } -func testAccBigQueryTableWithView(datasetID, tableID string, useLegacySql bool) string { +func testAccBigQueryTableWithView(datasetID, tableID string) string { return fmt.Sprintf(` resource "google_bigquery_dataset" "test" { dataset_id = "%s" @@ -237,8 +236,30 @@ resource "google_bigquery_table" "test" { view { query = "SELECT state FROM [lookerdata:cdc.project_tycho_reports]" + use_legacy_sql = true } -}`, datasetID, tableID, strconv.FormatBool(useLegacySql)) +}`, datasetID, tableID) +} + +func testAccBigQueryTableWithNewSqlView(datasetID, tableID string) string { + return fmt.Sprintf(` +resource "google_bigquery_dataset" "test" { + dataset_id = "%s" +} + +resource "google_bigquery_table" "test" { + table_id = "%s" + dataset_id = "${google_bigquery_dataset.test.dataset_id}" + + time_partitioning { + type = "DAY" + } + + view { + query = "%s" + use_legacy_sql = false + } +}`, datasetID, tableID, "SELECT state FROM `lookerdata:cdc.project_tycho_reports`") } func testAccBigQueryTableUpdated(datasetID, tableID string) string {