-
Notifications
You must be signed in to change notification settings - Fork 1.8k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Support Bigquery Views #230
Changes from all commits
3304793
d161eca
d8aa4f4
efac631
fbc5391
61f67c2
aefd68a
aa67b7b
f5b3036
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -92,6 +92,31 @@ func resourceBigQueryTable() *schema.Resource { | |
}, | ||
}, | ||
|
||
// View: [Optional] If specified, configures this table as a view. | ||
"view": &schema.Schema{ | ||
Type: schema.TypeList, | ||
Optional: true, | ||
MaxItems: 1, | ||
Elem: &schema.Resource{ | ||
Schema: map[string]*schema.Schema{ | ||
// Query: [Required] A query that BigQuery executes when the view is | ||
// referenced. | ||
"query": { | ||
Type: schema.TypeString, | ||
Required: true, | ||
}, | ||
|
||
// UseLegacySQL: [Optional] Specifies whether to use BigQuery's | ||
// legacy SQL for this view. The default value is true. If set to | ||
// false, the view will use BigQuery's standard SQL: | ||
"use_legacy_sql": { | ||
Type: schema.TypeBool, | ||
Optional: true, | ||
}, | ||
}, | ||
}, | ||
}, | ||
|
||
// TimePartitioning: [Experimental] If specified, configures time-based | ||
// partitioning for this table. | ||
"time_partitioning": &schema.Schema{ | ||
|
@@ -202,12 +227,16 @@ func resourceTable(d *schema.ResourceData, meta interface{}) (*bigquery.Table, e | |
}, | ||
} | ||
|
||
if v, ok := d.GetOk("view"); ok { | ||
table.View = expandView(v) | ||
} | ||
|
||
if v, ok := d.GetOk("description"); ok { | ||
table.Description = v.(string) | ||
} | ||
|
||
if v, ok := d.GetOk("expiration_time"); ok { | ||
table.ExpirationTime = v.(int64) | ||
table.ExpirationTime = int64(v.(int)) | ||
} | ||
|
||
if v, ok := d.GetOk("friendly_name"); ok { | ||
|
@@ -317,6 +346,11 @@ func resourceBigQueryTableRead(d *schema.ResourceData, meta interface{}) error { | |
d.Set("schema", schema) | ||
} | ||
|
||
if res.View != nil { | ||
view := flattenView(res.View) | ||
d.Set("view", view) | ||
} | ||
|
||
return nil | ||
} | ||
|
||
|
@@ -394,3 +428,22 @@ func flattenTimePartitioning(tp *bigquery.TimePartitioning) []map[string]interfa | |
|
||
return []map[string]interface{}{result} | ||
} | ||
|
||
func expandView(configured interface{}) *bigquery.ViewDefinition { | ||
raw := configured.([]interface{})[0].(map[string]interface{}) | ||
vd := &bigquery.ViewDefinition{Query: raw["query"].(string)} | ||
|
||
if v, ok := raw["use_legacy_sql"]; ok { | ||
vd.UseLegacySql = v.(bool) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You'll also need to set There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good catch - thank you! Would you like me to add an end to end test for this particular case (UseLegaclySQL = false)? Running these AF tests costs $$$ so I wasn't sure on how to balance coverage vs cost. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Sure! You'd want to do a test specifically for updating it, since the default is false so if you set it to false explicitly on create it won't make a difference whether you forced it to send. If cost is an issue for you let me know and I'm happy to run it and post the results. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Perfect! Tests added for converting useLegacySQL from true to false - The API is kind enough to reject views with incorrect syntax when using new-SQL so I am now extra confident that this is working as expected! |
||
vd.ForceSendFields = append(vd.ForceSendFields, "UseLegacySql") | ||
} | ||
|
||
return vd | ||
} | ||
|
||
func flattenView(vd *bigquery.ViewDefinition) []map[string]interface{} { | ||
result := map[string]interface{}{"query": vd.Query} | ||
result["use_legacy_sql"] = vd.UseLegacySql | ||
|
||
return []map[string]interface{}{result} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,6 +2,7 @@ package google | |
|
||
import ( | ||
"fmt" | ||
"strings" | ||
"testing" | ||
|
||
"github.com/hashicorp/terraform/helper/acctest" | ||
|
@@ -37,14 +38,61 @@ func TestAccBigQueryTable_Basic(t *testing.T) { | |
}) | ||
} | ||
|
||
func TestAccBigQueryTable_View(t *testing.T) { | ||
datasetID := fmt.Sprintf("tf_test_%s", acctest.RandString(10)) | ||
tableID := fmt.Sprintf("tf_test_%s", acctest.RandString(10)) | ||
|
||
resource.Test(t, resource.TestCase{ | ||
PreCheck: func() { testAccPreCheck(t) }, | ||
Providers: testAccProviders, | ||
CheckDestroy: testAccCheckBigQueryTableDestroy, | ||
Steps: []resource.TestStep{ | ||
{ | ||
Config: testAccBigQueryTableWithView(datasetID, tableID), | ||
Check: resource.ComposeTestCheckFunc( | ||
testAccBigQueryTableExistsWithView( | ||
"google_bigquery_table.test"), | ||
), | ||
}, | ||
}, | ||
}) | ||
} | ||
|
||
func TestAccBigQueryTable_ViewWithLegacySQL(t *testing.T) { | ||
datasetID := fmt.Sprintf("tf_test_%s", acctest.RandString(10)) | ||
tableID := fmt.Sprintf("tf_test_%s", acctest.RandString(10)) | ||
|
||
resource.Test(t, resource.TestCase{ | ||
PreCheck: func() { testAccPreCheck(t) }, | ||
Providers: testAccProviders, | ||
CheckDestroy: testAccCheckBigQueryTableDestroy, | ||
Steps: []resource.TestStep{ | ||
{ | ||
Config: testAccBigQueryTableWithView(datasetID, tableID), | ||
Check: resource.ComposeTestCheckFunc( | ||
testAccBigQueryTableExistsWithLegacySql( | ||
"google_bigquery_table.test", true), | ||
), | ||
}, | ||
{ | ||
Config: testAccBigQueryTableWithNewSqlView(datasetID, tableID), | ||
Check: resource.ComposeTestCheckFunc( | ||
testAccBigQueryTableExistsWithLegacySql( | ||
"google_bigquery_table.test", false), | ||
), | ||
}, | ||
}, | ||
}) | ||
} | ||
|
||
func testAccCheckBigQueryTableDestroy(s *terraform.State) error { | ||
for _, rs := range s.RootModule().Resources { | ||
if rs.Type != "google_bigquery_table" { | ||
continue | ||
} | ||
|
||
config := testAccProvider.Meta().(*Config) | ||
_, err := config.clientBigQuery.Tables.Get(config.Project, rs.Primary.Attributes["dataset_id"], rs.Primary.Attributes["name"]).Do() | ||
_, err := config.clientBigQuery.Tables.Get(config.Project, rs.Primary.Attributes["dataset_id"], rs.Primary.Attributes["table_id"]).Do() | ||
if err == nil { | ||
return fmt.Errorf("Table still present") | ||
} | ||
|
@@ -64,11 +112,69 @@ func testAccBigQueryTableExists(n string) resource.TestCheckFunc { | |
return fmt.Errorf("No ID is set") | ||
} | ||
config := testAccProvider.Meta().(*Config) | ||
_, err := config.clientBigQuery.Tables.Get(config.Project, rs.Primary.Attributes["dataset_id"], rs.Primary.Attributes["name"]).Do() | ||
table, err := config.clientBigQuery.Tables.Get(config.Project, rs.Primary.Attributes["dataset_id"], rs.Primary.Attributes["table_id"]).Do() | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good catch, thanks! Want to fix that up in There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done! |
||
if err != nil { | ||
return fmt.Errorf("BigQuery Table not present") | ||
} | ||
|
||
if !strings.HasSuffix(table.Id, rs.Primary.Attributes["table_id"]) { | ||
return fmt.Errorf("BigQuery Table ID does not match expected value") | ||
} | ||
|
||
return nil | ||
} | ||
} | ||
|
||
func testAccBigQueryTableExistsWithView(n string) resource.TestCheckFunc { | ||
return func(s *terraform.State) error { | ||
rs, ok := s.RootModule().Resources[n] | ||
if !ok { | ||
return fmt.Errorf("Not found: %s", n) | ||
} | ||
|
||
if rs.Primary.ID == "" { | ||
return fmt.Errorf("No ID is set") | ||
} | ||
config := testAccProvider.Meta().(*Config) | ||
|
||
table, err := config.clientBigQuery.Tables.Get(config.Project, rs.Primary.Attributes["dataset_id"], rs.Primary.Attributes["table_id"]).Do() | ||
if err != nil { | ||
return fmt.Errorf("BigQuery Table not present") | ||
} | ||
|
||
if table.View == nil { | ||
return fmt.Errorf("View object missing on table") | ||
} | ||
|
||
return nil | ||
} | ||
} | ||
|
||
func testAccBigQueryTableExistsWithLegacySql(n string, useLegacySql bool) resource.TestCheckFunc { | ||
return func(s *terraform.State) error { | ||
rs, ok := s.RootModule().Resources[n] | ||
if !ok { | ||
return fmt.Errorf("Not found: %s", n) | ||
} | ||
|
||
if rs.Primary.ID == "" { | ||
return fmt.Errorf("No ID is set") | ||
} | ||
config := testAccProvider.Meta().(*Config) | ||
|
||
table, err := config.clientBigQuery.Tables.Get(config.Project, rs.Primary.Attributes["dataset_id"], rs.Primary.Attributes["table_id"]).Do() | ||
if err != nil { | ||
return fmt.Errorf("BigQuery Table not present") | ||
} | ||
|
||
if table.View == nil { | ||
return fmt.Errorf("View object missing on table") | ||
} | ||
|
||
if table.View.UseLegacySql != useLegacySql { | ||
return fmt.Errorf("Value of UseLegacySQL does not match expected value") | ||
} | ||
|
||
return nil | ||
} | ||
} | ||
|
@@ -114,6 +220,48 @@ EOH | |
}`, datasetID, tableID) | ||
} | ||
|
||
func testAccBigQueryTableWithView(datasetID, tableID string) string { | ||
return fmt.Sprintf(` | ||
resource "google_bigquery_dataset" "test" { | ||
dataset_id = "%s" | ||
} | ||
|
||
resource "google_bigquery_table" "test" { | ||
table_id = "%s" | ||
dataset_id = "${google_bigquery_dataset.test.dataset_id}" | ||
|
||
time_partitioning { | ||
type = "DAY" | ||
} | ||
|
||
view { | ||
query = "SELECT state FROM [lookerdata:cdc.project_tycho_reports]" | ||
use_legacy_sql = true | ||
} | ||
}`, datasetID, tableID) | ||
} | ||
|
||
func testAccBigQueryTableWithNewSqlView(datasetID, tableID string) string { | ||
return fmt.Sprintf(` | ||
resource "google_bigquery_dataset" "test" { | ||
dataset_id = "%s" | ||
} | ||
|
||
resource "google_bigquery_table" "test" { | ||
table_id = "%s" | ||
dataset_id = "${google_bigquery_dataset.test.dataset_id}" | ||
|
||
time_partitioning { | ||
type = "DAY" | ||
} | ||
|
||
view { | ||
query = "%s" | ||
use_legacy_sql = false | ||
} | ||
}`, datasetID, tableID, "SELECT state FROM `lookerdata:cdc.project_tycho_reports`") | ||
} | ||
|
||
func testAccBigQueryTableUpdated(datasetID, tableID string) string { | ||
return fmt.Sprintf(` | ||
resource "google_bigquery_dataset" "test" { | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Do you think it makes sense to add the other
view
attributes, or do they not make sense in terraform? (useLegacySql, userDefinedFunctionResources)There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I've added support for useLegacySQL - I think that makes a lot of sense.
It makes sense to capture userDefinedFunctionResources in terraform, but I'm not particularly familiar with them. I'll try and carve out some time to ramp up and get them added, but I'd recommend doing that in a second CL/pass.