Skip to content

Commit

Permalink
bootstrap network for dataproc tests (#9304) (#6563)
Browse files Browse the repository at this point in the history 
[upstream:fb15c6938133f8b429746d8e0d50e8a560043710]

Signed-off-by: Modular Magician <magic-modules@google.com>
  • Loading branch information
@modular-magician
modular-magician committed Oct 24, 2023
1 parent 9fd403f commit ecc1617
Show file tree
Hide file tree
Showing 7 changed files with 528 additions and 199 deletions.
3 changes: 3 additions & 0 deletions .changelog/9304.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:none

```
73 changes: 73 additions & 0 deletions google-beta/acctest/bootstrap_test_utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1060,6 +1060,79 @@ func BootstrapNetworkAttachment(t *testing.T, networkAttachmentName string, subn
return networkAttachment.Name
}

// The default network within GCP already comes pre configured with
// certain firewall rules open to allow internal communication. As we
// are boostrapping a network for dataproc tests, we need to additionally
// open up similar rules to allow the nodes to talk to each other
// internally as part of their configuration or this will just hang.
const SharedTestFirewallPrefix = "tf-bootstrap-firewall-"

func BootstrapFirewallForDataprocSharedNetwork(t *testing.T, firewallName string, networkName string) string {
project := envvar.GetTestProjectFromEnv()
firewallName = SharedTestFirewallPrefix + firewallName

config := BootstrapConfig(t)
if config == nil {
return ""
}

log.Printf("[DEBUG] Getting Firewall %q for Network %q", firewallName, networkName)
_, err := config.NewComputeClient(config.UserAgent).Firewalls.Get(project, firewallName).Do()
if err != nil && transport_tpg.IsGoogleApiErrorWithCode(err, 404) {
log.Printf("[DEBUG] firewallName %q not found, bootstrapping", firewallName)
url := fmt.Sprintf("%sprojects/%s/global/firewalls", config.ComputeBasePath, project)

networkId := fmt.Sprintf("projects/%s/global/networks/%s", project, networkName)
allowObj := []interface{}{
map[string]interface{}{
"IPProtocol": "icmp",
},
map[string]interface{}{
"IPProtocol": "tcp",
"ports": []string{"0-65535"},
},
map[string]interface{}{
"IPProtocol": "udp",
"ports": []string{"0-65535"},
},
}

firewallObj := map[string]interface{}{
"name": firewallName,
"network": networkId,
"allowed": allowObj,
}

res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{
Config: config,
Method: "POST",
Project: project,
RawURL: url,
UserAgent: config.UserAgent,
Body: firewallObj,
Timeout: 4 * time.Minute,
})
if err != nil {
t.Fatalf("Error bootstrapping Firewall %q for Network %q: %s", firewallName, networkName, err)
}

log.Printf("[DEBUG] Waiting for Firewall creation to finish")
err = tpgcompute.ComputeOperationWaitTime(config, res, project, "Error bootstrapping Firewall", config.UserAgent, 4*time.Minute)
if err != nil {
t.Fatalf("Error bootstrapping Firewall %q: %s", firewallName, err)
}
}

firewall, err := config.NewComputeClient(config.UserAgent).Firewalls.Get(project, firewallName).Do()
if err != nil {
t.Errorf("Error getting Firewall %q: %s", firewallName, err)
}
if firewall == nil {
t.Fatalf("Error getting Firewall %q: is nil", firewallName)
}
return firewall.Name
}

func SetupProjectsAndGetAccessToken(org, billing, pid, service string, config *transport_tpg.Config) (string, error) {
// Create project-1 and project-2
rmService := config.NewResourceManagerClient(config.UserAgent)
Expand Down
40 changes: 28 additions & 12 deletions google-beta/services/dataproc/iam_dataproc_cluster_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ func TestAccDataprocClusterIamBinding(t *testing.T) {
account := "tf-dataproc-iam-" + acctest.RandString(t, 10)
role := "roles/editor"

networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster")
subnetworkName := acctest.BootstrapSubnet(t, "dataproc-cluster", networkName)
acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName)

importId := fmt.Sprintf("projects/%s/regions/%s/clusters/%s %s",
envvar.GetTestProjectFromEnv(), "us-central1", cluster, role)

Expand All @@ -27,7 +31,7 @@ func TestAccDataprocClusterIamBinding(t *testing.T) {
Steps: []resource.TestStep{
{
// Test IAM Binding creation
Config: testAccDataprocClusterIamBinding_basic(cluster, account, role),
Config: testAccDataprocClusterIamBinding_basic(cluster, subnetworkName, account, role),
Check: resource.ComposeTestCheckFunc(
resource.TestCheckResourceAttr(
"google_dataproc_cluster_iam_binding.binding", "role", role),
Expand All @@ -41,7 +45,7 @@ func TestAccDataprocClusterIamBinding(t *testing.T) {
},
{
// Test IAM Binding update
Config: testAccDataprocClusterIamBinding_update(cluster, account, role),
Config: testAccDataprocClusterIamBinding_update(cluster, subnetworkName, account, role),
},
{
ResourceName: "google_dataproc_cluster_iam_binding.binding",
Expand All @@ -60,6 +64,10 @@ func TestAccDataprocClusterIamMember(t *testing.T) {
account := "tf-dataproc-iam-" + acctest.RandString(t, 10)
role := "roles/editor"

networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster")
subnetworkName := acctest.BootstrapSubnet(t, "dataproc-cluster", networkName)
acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName)

importId := fmt.Sprintf("projects/%s/regions/%s/clusters/%s %s serviceAccount:%s",
envvar.GetTestProjectFromEnv(),
"us-central1",
Expand All @@ -73,7 +81,7 @@ func TestAccDataprocClusterIamMember(t *testing.T) {
Steps: []resource.TestStep{
{
// Test IAM Binding creation
Config: testAccDataprocClusterIamMember(cluster, account, role),
Config: testAccDataprocClusterIamMember(cluster, subnetworkName, account, role),
Check: resource.ComposeTestCheckFunc(
resource.TestCheckResourceAttr(
"google_dataproc_cluster_iam_member.member", "role", role),
Expand All @@ -98,6 +106,10 @@ func TestAccDataprocClusterIamPolicy(t *testing.T) {
account := "tf-dataproc-iam-" + acctest.RandString(t, 10)
role := "roles/editor"

networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster")
subnetworkName := acctest.BootstrapSubnet(t, "dataproc-cluster", networkName)
acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName)

importId := fmt.Sprintf("projects/%s/regions/%s/clusters/%s",
envvar.GetTestProjectFromEnv(), "us-central1", cluster)

Expand All @@ -107,7 +119,7 @@ func TestAccDataprocClusterIamPolicy(t *testing.T) {
Steps: []resource.TestStep{
{
// Test IAM Binding creation
Config: testAccDataprocClusterIamPolicy(cluster, account, role),
Config: testAccDataprocClusterIamPolicy(cluster, subnetworkName, account, role),
Check: resource.TestCheckResourceAttrSet("data.google_dataproc_cluster_iam_policy.policy", "policy_data"),
},
{
Expand All @@ -120,7 +132,7 @@ func TestAccDataprocClusterIamPolicy(t *testing.T) {
})
}

func testAccDataprocClusterIamBinding_basic(cluster, account, role string) string {
func testAccDataprocClusterIamBinding_basic(cluster, subnetworkName, account, role string) string {
return fmt.Sprintf(testDataprocIamSingleNodeCluster+`
resource "google_service_account" "test-account1" {
account_id = "%s-1"
Expand All @@ -140,10 +152,10 @@ resource "google_dataproc_cluster_iam_binding" "binding" {
"serviceAccount:${google_service_account.test-account1.email}",
]
}
`, cluster, account, account, role)
`, cluster, subnetworkName, account, account, role)
}

func testAccDataprocClusterIamBinding_update(cluster, account, role string) string {
func testAccDataprocClusterIamBinding_update(cluster, subnetworkName, account, role string) string {
return fmt.Sprintf(testDataprocIamSingleNodeCluster+`
resource "google_service_account" "test-account1" {
account_id = "%s-1"
Expand All @@ -164,10 +176,10 @@ resource "google_dataproc_cluster_iam_binding" "binding" {
"serviceAccount:${google_service_account.test-account2.email}",
]
}
`, cluster, account, account, role)
`, cluster, subnetworkName, account, account, role)
}

func testAccDataprocClusterIamMember(cluster, account, role string) string {
func testAccDataprocClusterIamMember(cluster, subnetworkName, account, role string) string {
return fmt.Sprintf(testDataprocIamSingleNodeCluster+`
resource "google_service_account" "test-account" {
account_id = "%s"
Expand All @@ -179,10 +191,10 @@ resource "google_dataproc_cluster_iam_member" "member" {
role = "%s"
member = "serviceAccount:${google_service_account.test-account.email}"
}
`, cluster, account, role)
`, cluster, subnetworkName, account, role)
}

func testAccDataprocClusterIamPolicy(cluster, account, role string) string {
func testAccDataprocClusterIamPolicy(cluster, subnetworkName, account, role string) string {
return fmt.Sprintf(testDataprocIamSingleNodeCluster+`
resource "google_service_account" "test-account" {
account_id = "%s"
Expand All @@ -207,7 +219,7 @@ data "google_dataproc_cluster_iam_policy" "policy" {
region = "us-central1"
}
`, cluster, account, role)
`, cluster, subnetworkName, account, role)
}

// Smallest cluster possible for testing
Expand All @@ -217,6 +229,10 @@ resource "google_dataproc_cluster" "cluster" {
region = "us-central1"
cluster_config {
gce_cluster_config {
subnetwork = "%s"
}
# Keep the costs down with smallest config we can get away with
software_config {
override_properties = {
Expand Down
36 changes: 24 additions & 12 deletions google-beta/services/dataproc/iam_dataproc_job_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ func TestAccDataprocJobIamBinding(t *testing.T) {
account := "tf-dataproc-iam-" + acctest.RandString(t, 10)
role := "roles/editor"

networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster")
subnetworkName := acctest.BootstrapSubnet(t, "dataproc-cluster", networkName)
acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName)

importId := fmt.Sprintf("projects/%s/regions/%s/jobs/%s %s",
envvar.GetTestProjectFromEnv(), "us-central1", job, role)

Expand All @@ -28,7 +32,7 @@ func TestAccDataprocJobIamBinding(t *testing.T) {
Steps: []resource.TestStep{
{
// Test IAM Binding creation
Config: testAccDataprocJobIamBinding_basic(cluster, job, account, role),
Config: testAccDataprocJobIamBinding_basic(cluster, subnetworkName, job, account, role),
},
{
ResourceName: "google_dataproc_job_iam_binding.binding",
Expand All @@ -38,7 +42,7 @@ func TestAccDataprocJobIamBinding(t *testing.T) {
},
{
// Test IAM Binding update
Config: testAccDataprocJobIamBinding_update(cluster, job, account, role),
Config: testAccDataprocJobIamBinding_update(cluster, subnetworkName, job, account, role),
},
{
ResourceName: "google_dataproc_job_iam_binding.binding",
Expand All @@ -58,6 +62,10 @@ func TestAccDataprocJobIamMember(t *testing.T) {
account := "tf-dataproc-iam-" + acctest.RandString(t, 10)
role := "roles/editor"

networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster")
subnetworkName := acctest.BootstrapSubnet(t, "dataproc-cluster", networkName)
acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName)

importId := fmt.Sprintf("projects/%s/regions/%s/jobs/%s %s serviceAccount:%s",
envvar.GetTestProjectFromEnv(),
"us-central1",
Expand All @@ -71,7 +79,7 @@ func TestAccDataprocJobIamMember(t *testing.T) {
Steps: []resource.TestStep{
{
// Test IAM Binding creation
Config: testAccDataprocJobIamMember(cluster, job, account, role),
Config: testAccDataprocJobIamMember(cluster, subnetworkName, job, account, role),
},
{
ResourceName: "google_dataproc_job_iam_member.member",
Expand All @@ -91,6 +99,10 @@ func TestAccDataprocJobIamPolicy(t *testing.T) {
account := "tf-dataproc-iam-" + acctest.RandString(t, 10)
role := "roles/editor"

networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster")
subnetworkName := acctest.BootstrapSubnet(t, "dataproc-cluster", networkName)
acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName)

importId := fmt.Sprintf("projects/%s/regions/%s/jobs/%s",
envvar.GetTestProjectFromEnv(), "us-central1", job)

Expand All @@ -100,7 +112,7 @@ func TestAccDataprocJobIamPolicy(t *testing.T) {
Steps: []resource.TestStep{
{
// Test IAM Binding creation
Config: testAccDataprocJobIamPolicy(cluster, job, account, role),
Config: testAccDataprocJobIamPolicy(cluster, subnetworkName, job, account, role),
Check: resource.TestCheckResourceAttrSet("data.google_dataproc_job_iam_policy.policy", "policy_data"),
},
{
Expand Down Expand Up @@ -141,7 +153,7 @@ resource "google_dataproc_job" "pyspark" {
}
`

func testAccDataprocJobIamBinding_basic(cluster, job, account, role string) string {
func testAccDataprocJobIamBinding_basic(cluster, subnetworkName, job, account, role string) string {
return fmt.Sprintf(testDataprocIamJobConfig+`
resource "google_service_account" "test-account1" {
account_id = "%s-1"
Expand All @@ -161,10 +173,10 @@ resource "google_dataproc_job_iam_binding" "binding" {
"serviceAccount:${google_service_account.test-account1.email}",
]
}
`, cluster, job, account, account, role)
`, cluster, subnetworkName, job, account, account, role)
}

func testAccDataprocJobIamBinding_update(cluster, job, account, role string) string {
func testAccDataprocJobIamBinding_update(cluster, subnetworkName, job, account, role string) string {
return fmt.Sprintf(testDataprocIamJobConfig+`
resource "google_service_account" "test-account1" {
account_id = "%s-1"
Expand All @@ -185,10 +197,10 @@ resource "google_dataproc_job_iam_binding" "binding" {
"serviceAccount:${google_service_account.test-account2.email}",
]
}
`, cluster, job, account, account, role)
`, cluster, subnetworkName, job, account, account, role)
}

func testAccDataprocJobIamMember(cluster, job, account, role string) string {
func testAccDataprocJobIamMember(cluster, subnetworkName, job, account, role string) string {
return fmt.Sprintf(testDataprocIamJobConfig+`
resource "google_service_account" "test-account" {
account_id = "%s"
Expand All @@ -200,10 +212,10 @@ resource "google_dataproc_job_iam_member" "member" {
role = "%s"
member = "serviceAccount:${google_service_account.test-account.email}"
}
`, cluster, job, account, role)
`, cluster, subnetworkName, job, account, role)
}

func testAccDataprocJobIamPolicy(cluster, job, account, role string) string {
func testAccDataprocJobIamPolicy(cluster, subnetworkName, job, account, role string) string {
return fmt.Sprintf(testDataprocIamJobConfig+`
resource "google_service_account" "test-account" {
account_id = "%s"
Expand All @@ -228,5 +240,5 @@ data "google_dataproc_job_iam_policy" "policy" {
region = "us-central1"
}
`, cluster, job, account, role)
`, cluster, subnetworkName, job, account, role)
}
Loading

0 comments on commit ecc1617

Please sign in to comment.