Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support Bigquery Views #230

Merged
merged 9 commits into from
Aug 26, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion google/resource_bigquery_dataset.go
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,6 @@ func resourceBigQueryDatasetRead(d *schema.ResourceData, meta interface{}) error

d.Set("etag", res.Etag)
d.Set("labels", res.Labels)
d.Set("location", res.Location)
d.Set("self_link", res.SelfLink)
d.Set("description", res.Description)
d.Set("friendly_name", res.FriendlyName)
Expand All @@ -238,6 +237,15 @@ func resourceBigQueryDatasetRead(d *schema.ResourceData, meta interface{}) error
d.Set("dataset_id", res.DatasetReference.DatasetId)
d.Set("default_table_expiration_ms", res.DefaultTableExpirationMs)

// Older Tables in BigQuery have no Location set in the API response. This may be an issue when importing
// tables created before BigQuery was available in multiple zones. We can safely assume that these tables
// are in the US, as this was the default at the time.
if res.Location == "" {
d.Set("location", "US")
} else {
d.Set("location", res.Location)
}

return nil
}

Expand Down
55 changes: 54 additions & 1 deletion google/resource_bigquery_table.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,31 @@ func resourceBigQueryTable() *schema.Resource {
},
},

// View: [Optional] If specified, configures this table as a view.
"view": &schema.Schema{
Type: schema.TypeList,
Optional: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you think it makes sense to add the other view attributes, or do they not make sense in terraform? (useLegacySql, userDefinedFunctionResources)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've added support for useLegacySQL - I think that makes a lot of sense.

It makes sense to capture userDefinedFunctionResources in terraform, but I'm not particularly familiar with them. I'll try and carve out some time to ramp up and get them added, but I'd recommend doing that in a second CL/pass.

// Query: [Required] A query that BigQuery executes when the view is
// referenced.
"query": {
Type: schema.TypeString,
Required: true,
},

// UseLegacySQL: [Optional] Specifies whether to use BigQuery's
// legacy SQL for this view. The default value is true. If set to
// false, the view will use BigQuery's standard SQL:
"use_legacy_sql": {
Type: schema.TypeBool,
Optional: true,
},
},
},
},

// TimePartitioning: [Experimental] If specified, configures time-based
// partitioning for this table.
"time_partitioning": &schema.Schema{
Expand Down Expand Up @@ -202,12 +227,16 @@ func resourceTable(d *schema.ResourceData, meta interface{}) (*bigquery.Table, e
},
}

if v, ok := d.GetOk("view"); ok {
table.View = expandView(v)
}

if v, ok := d.GetOk("description"); ok {
table.Description = v.(string)
}

if v, ok := d.GetOk("expiration_time"); ok {
table.ExpirationTime = v.(int64)
table.ExpirationTime = int64(v.(int))
}

if v, ok := d.GetOk("friendly_name"); ok {
Expand Down Expand Up @@ -317,6 +346,11 @@ func resourceBigQueryTableRead(d *schema.ResourceData, meta interface{}) error {
d.Set("schema", schema)
}

if res.View != nil {
view := flattenView(res.View)
d.Set("view", view)
}

return nil
}

Expand Down Expand Up @@ -394,3 +428,22 @@ func flattenTimePartitioning(tp *bigquery.TimePartitioning) []map[string]interfa

return []map[string]interface{}{result}
}

func expandView(configured interface{}) *bigquery.ViewDefinition {
raw := configured.([]interface{})[0].(map[string]interface{})
vd := &bigquery.ViewDefinition{Query: raw["query"].(string)}

if v, ok := raw["use_legacy_sql"]; ok {
vd.UseLegacySql = v.(bool)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You'll also need to set vd.ForceSendFields = append(vd.ForceSendFields, "UseLegacySql" so if it gets updated to false, it still gets sent along the wire (since false values end up defaulting to empty)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch - thank you!

Would you like me to add an end to end test for this particular case (UseLegaclySQL = false)? Running these AF tests costs $$$ so I wasn't sure on how to balance coverage vs cost.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure! You'd want to do a test specifically for updating it, since the default is false so if you set it to false explicitly on create it won't make a difference whether you forced it to send. If cost is an issue for you let me know and I'm happy to run it and post the results.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Perfect! Tests added for converting useLegacySQL from true to false - The API is kind enough to reject views with incorrect syntax when using new-SQL so I am now extra confident that this is working as expected!

vd.ForceSendFields = append(vd.ForceSendFields, "UseLegacySql")
}

return vd
}

func flattenView(vd *bigquery.ViewDefinition) []map[string]interface{} {
result := map[string]interface{}{"query": vd.Query}
result["use_legacy_sql"] = vd.UseLegacySql

return []map[string]interface{}{result}
}
152 changes: 150 additions & 2 deletions google/resource_bigquery_table_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package google

import (
"fmt"
"strings"
"testing"

"github.com/hashicorp/terraform/helper/acctest"
Expand Down Expand Up @@ -37,14 +38,61 @@ func TestAccBigQueryTable_Basic(t *testing.T) {
})
}

func TestAccBigQueryTable_View(t *testing.T) {
datasetID := fmt.Sprintf("tf_test_%s", acctest.RandString(10))
tableID := fmt.Sprintf("tf_test_%s", acctest.RandString(10))

resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckBigQueryTableDestroy,
Steps: []resource.TestStep{
{
Config: testAccBigQueryTableWithView(datasetID, tableID),
Check: resource.ComposeTestCheckFunc(
testAccBigQueryTableExistsWithView(
"google_bigquery_table.test"),
),
},
},
})
}

func TestAccBigQueryTable_ViewWithLegacySQL(t *testing.T) {
datasetID := fmt.Sprintf("tf_test_%s", acctest.RandString(10))
tableID := fmt.Sprintf("tf_test_%s", acctest.RandString(10))

resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckBigQueryTableDestroy,
Steps: []resource.TestStep{
{
Config: testAccBigQueryTableWithView(datasetID, tableID),
Check: resource.ComposeTestCheckFunc(
testAccBigQueryTableExistsWithLegacySql(
"google_bigquery_table.test", true),
),
},
{
Config: testAccBigQueryTableWithNewSqlView(datasetID, tableID),
Check: resource.ComposeTestCheckFunc(
testAccBigQueryTableExistsWithLegacySql(
"google_bigquery_table.test", false),
),
},
},
})
}

func testAccCheckBigQueryTableDestroy(s *terraform.State) error {
for _, rs := range s.RootModule().Resources {
if rs.Type != "google_bigquery_table" {
continue
}

config := testAccProvider.Meta().(*Config)
_, err := config.clientBigQuery.Tables.Get(config.Project, rs.Primary.Attributes["dataset_id"], rs.Primary.Attributes["name"]).Do()
_, err := config.clientBigQuery.Tables.Get(config.Project, rs.Primary.Attributes["dataset_id"], rs.Primary.Attributes["table_id"]).Do()
if err == nil {
return fmt.Errorf("Table still present")
}
Expand All @@ -64,11 +112,69 @@ func testAccBigQueryTableExists(n string) resource.TestCheckFunc {
return fmt.Errorf("No ID is set")
}
config := testAccProvider.Meta().(*Config)
_, err := config.clientBigQuery.Tables.Get(config.Project, rs.Primary.Attributes["dataset_id"], rs.Primary.Attributes["name"]).Do()
table, err := config.clientBigQuery.Tables.Get(config.Project, rs.Primary.Attributes["dataset_id"], rs.Primary.Attributes["table_id"]).Do()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch, thanks! Want to fix that up in testAccCheckBigQueryTableDestroy too?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done!

if err != nil {
return fmt.Errorf("BigQuery Table not present")
}

if !strings.HasSuffix(table.Id, rs.Primary.Attributes["table_id"]) {
return fmt.Errorf("BigQuery Table ID does not match expected value")
}

return nil
}
}

func testAccBigQueryTableExistsWithView(n string) resource.TestCheckFunc {
return func(s *terraform.State) error {
rs, ok := s.RootModule().Resources[n]
if !ok {
return fmt.Errorf("Not found: %s", n)
}

if rs.Primary.ID == "" {
return fmt.Errorf("No ID is set")
}
config := testAccProvider.Meta().(*Config)

table, err := config.clientBigQuery.Tables.Get(config.Project, rs.Primary.Attributes["dataset_id"], rs.Primary.Attributes["table_id"]).Do()
if err != nil {
return fmt.Errorf("BigQuery Table not present")
}

if table.View == nil {
return fmt.Errorf("View object missing on table")
}

return nil
}
}

func testAccBigQueryTableExistsWithLegacySql(n string, useLegacySql bool) resource.TestCheckFunc {
return func(s *terraform.State) error {
rs, ok := s.RootModule().Resources[n]
if !ok {
return fmt.Errorf("Not found: %s", n)
}

if rs.Primary.ID == "" {
return fmt.Errorf("No ID is set")
}
config := testAccProvider.Meta().(*Config)

table, err := config.clientBigQuery.Tables.Get(config.Project, rs.Primary.Attributes["dataset_id"], rs.Primary.Attributes["table_id"]).Do()
if err != nil {
return fmt.Errorf("BigQuery Table not present")
}

if table.View == nil {
return fmt.Errorf("View object missing on table")
}

if table.View.UseLegacySql != useLegacySql {
return fmt.Errorf("Value of UseLegacySQL does not match expected value")
}

return nil
}
}
Expand Down Expand Up @@ -114,6 +220,48 @@ EOH
}`, datasetID, tableID)
}

func testAccBigQueryTableWithView(datasetID, tableID string) string {
return fmt.Sprintf(`
resource "google_bigquery_dataset" "test" {
dataset_id = "%s"
}

resource "google_bigquery_table" "test" {
table_id = "%s"
dataset_id = "${google_bigquery_dataset.test.dataset_id}"

time_partitioning {
type = "DAY"
}

view {
query = "SELECT state FROM [lookerdata:cdc.project_tycho_reports]"
use_legacy_sql = true
}
}`, datasetID, tableID)
}

func testAccBigQueryTableWithNewSqlView(datasetID, tableID string) string {
return fmt.Sprintf(`
resource "google_bigquery_dataset" "test" {
dataset_id = "%s"
}

resource "google_bigquery_table" "test" {
table_id = "%s"
dataset_id = "${google_bigquery_dataset.test.dataset_id}"

time_partitioning {
type = "DAY"
}

view {
query = "%s"
use_legacy_sql = false
}
}`, datasetID, tableID, "SELECT state FROM `lookerdata:cdc.project_tycho_reports`")
}

func testAccBigQueryTableUpdated(datasetID, tableID string) string {
return fmt.Sprintf(`
resource "google_bigquery_dataset" "test" {
Expand Down