From 48e2d848ecfbd2514516a26f78dc6c50c0539343 Mon Sep 17 00:00:00 2001 From: rajeshbal65 Date: Thu, 1 Jul 2021 14:14:46 +0530 Subject: [PATCH 1/3] Add table azure_data_factory. Closes #156 --- .../tests/azure_data_factory/dependencies.txt | 0 .../azure_data_factory/test-get-expected.json | 13 ++ .../azure_data_factory/test-get-query.sql | 3 + .../test-list-expected.json | 7 + .../azure_data_factory/test-list-query.sql | 3 + .../test-not-found-expected.json | 1 + .../test-not-found-query.sql | 3 + .../test-turbot-expected.json | 13 ++ .../azure_data_factory/test-turbot-query.sql | 3 + .../tests/azure_data_factory/variables.json | 1 + .../tests/azure_data_factory/variables.tf | 67 ++++++ azure/plugin.go | 1 + azure/table_azure_data_factory.go | 195 ++++++++++++++++++ docs/tables/azure_data_factory.md | 48 +++++ 14 files changed, 358 insertions(+) create mode 100644 azure-test/tests/azure_data_factory/dependencies.txt create mode 100644 azure-test/tests/azure_data_factory/test-get-expected.json create mode 100644 azure-test/tests/azure_data_factory/test-get-query.sql create mode 100644 azure-test/tests/azure_data_factory/test-list-expected.json create mode 100644 azure-test/tests/azure_data_factory/test-list-query.sql create mode 100644 azure-test/tests/azure_data_factory/test-not-found-expected.json create mode 100644 azure-test/tests/azure_data_factory/test-not-found-query.sql create mode 100644 azure-test/tests/azure_data_factory/test-turbot-expected.json create mode 100644 azure-test/tests/azure_data_factory/test-turbot-query.sql create mode 100644 azure-test/tests/azure_data_factory/variables.json create mode 100644 azure-test/tests/azure_data_factory/variables.tf create mode 100644 azure/table_azure_data_factory.go create mode 100644 docs/tables/azure_data_factory.md diff --git a/azure-test/tests/azure_data_factory/dependencies.txt b/azure-test/tests/azure_data_factory/dependencies.txt new file mode 100644 index 00000000..e69de29b diff --git a/azure-test/tests/azure_data_factory/test-get-expected.json b/azure-test/tests/azure_data_factory/test-get-expected.json new file mode 100644 index 00000000..619e2748 --- /dev/null +++ b/azure-test/tests/azure_data_factory/test-get-expected.json @@ -0,0 +1,13 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{resourceName}}", + "region": "eastus", + "resource_group": "{{resourceName}}", + "subscription_id": "{{ output.subscription_id.value }}", + "tags": { + "name": "{{resourceName}}" + }, + "type": "Microsoft.DataFactory/factories" + } +] \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-get-query.sql b/azure-test/tests/azure_data_factory/test-get-query.sql new file mode 100644 index 00000000..83808052 --- /dev/null +++ b/azure-test/tests/azure_data_factory/test-get-query.sql @@ -0,0 +1,3 @@ +select name, id, type, tags, region, resource_group, subscription_id +from azure.azure_data_factory +where name = '{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-list-expected.json b/azure-test/tests/azure_data_factory/test-list-expected.json new file mode 100644 index 00000000..f15afcd8 --- /dev/null +++ b/azure-test/tests/azure_data_factory/test-list-expected.json @@ -0,0 +1,7 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{resourceName}}", + "type": "Microsoft.DataFactory/factories" + } +] \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-list-query.sql b/azure-test/tests/azure_data_factory/test-list-query.sql new file mode 100644 index 00000000..f900f679 --- /dev/null +++ b/azure-test/tests/azure_data_factory/test-list-query.sql @@ -0,0 +1,3 @@ +select name, id, type +from azure.azure_data_factory +where name = '{{resourceName}}' \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-not-found-expected.json b/azure-test/tests/azure_data_factory/test-not-found-expected.json new file mode 100644 index 00000000..ec747fa4 --- /dev/null +++ b/azure-test/tests/azure_data_factory/test-not-found-expected.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-not-found-query.sql b/azure-test/tests/azure_data_factory/test-not-found-query.sql new file mode 100644 index 00000000..04ecb573 --- /dev/null +++ b/azure-test/tests/azure_data_factory/test-not-found-query.sql @@ -0,0 +1,3 @@ +select name, tags, title, akas +from azure.azure_data_factory +where name = 'dummy-{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-turbot-expected.json b/azure-test/tests/azure_data_factory/test-turbot-expected.json new file mode 100644 index 00000000..3ceae2f7 --- /dev/null +++ b/azure-test/tests/azure_data_factory/test-turbot-expected.json @@ -0,0 +1,13 @@ +[ + { + "akas": [ + "{{ output.resource_aka.value }}", + "{{ output.resource_aka_lower.value }}" + ], + "name": "{{resourceName}}", + "tags": { + "name": "{{resourceName}}" + }, + "title": "{{resourceName}}" + } +] \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-turbot-query.sql b/azure-test/tests/azure_data_factory/test-turbot-query.sql new file mode 100644 index 00000000..434fc5ae --- /dev/null +++ b/azure-test/tests/azure_data_factory/test-turbot-query.sql @@ -0,0 +1,3 @@ +select name, tags, title, akas +from azure.azure_data_factory +where name = '{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/variables.json b/azure-test/tests/azure_data_factory/variables.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/azure-test/tests/azure_data_factory/variables.json @@ -0,0 +1 @@ +{} diff --git a/azure-test/tests/azure_data_factory/variables.tf b/azure-test/tests/azure_data_factory/variables.tf new file mode 100644 index 00000000..20dd0733 --- /dev/null +++ b/azure-test/tests/azure_data_factory/variables.tf @@ -0,0 +1,67 @@ + +variable "resource_name" { + type = string + default = "steampipe-test" + description = "Name of the resource used throughout the test." +} + +variable "azure_environment" { + type = string + default = "public" + description = "Azure environment used for the test." +} + +variable "azure_subscription" { + type = string + default = "3510ae4d-530b-497d-8f30-53b9616fc6c1" + description = "Azure subscription used for the test." +} + +provider "azurerm" { + # Cannot be passed as a variable + version = "=1.36.0" + environment = var.azure_environment + subscription_id = var.azure_subscription +} + +data "azurerm_client_config" "current" {} + +data "null_data_source" "resource" { + inputs = { + scope = "azure:///subscriptions/${data.azurerm_client_config.current.subscription_id}" + } +} + +resource "azurerm_resource_group" "named_test_resource" { + name = var.resource_name + location = "East US" +} + +resource "azurerm_data_factory" "named_test_resource" { + name = var.resource_name + location = "East US" + resource_group_name = azurerm_resource_group.named_test_resource.name + tags = { + name = var.resource_name + } +} + +output "resource_aka" { + value = "azure://${azurerm_data_factory.named_test_resource.id}" +} + +output "resource_aka_lower" { + value = "azure://${lower(azurerm_data_factory.named_test_resource.id)}" +} + +output "resource_name" { + value = var.resource_name +} + +output "resource_id" { + value = azurerm_data_factory.named_test_resource.id +} + +output "subscription_id" { + value = var.azure_subscription +} diff --git a/azure/plugin.go b/azure/plugin.go index 63feab55..c165e4c8 100644 --- a/azure/plugin.go +++ b/azure/plugin.go @@ -41,6 +41,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "azure_cosmosdb_account": tableAzureCosmosDBAccount(ctx), "azure_cosmosdb_mongo_database": tableAzureCosmosDBMongoDatabase(ctx), "azure_cosmosdb_sql_database": tableAzureCosmosDBSQLDatabase(ctx), + "azure_data_factory": tableAzureDataFactory(ctx), "azure_diagnostic_setting": tableAzureDiagnosticSetting(ctx), "azure_firewall": tableAzureFirewall(ctx), "azure_key_vault": tableAzureKeyVault(ctx), diff --git a/azure/table_azure_data_factory.go b/azure/table_azure_data_factory.go new file mode 100644 index 00000000..ae62b6f2 --- /dev/null +++ b/azure/table_azure_data_factory.go @@ -0,0 +1,195 @@ +package azure + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/services/datafactory/mgmt/2018-06-01/datafactory" + "github.com/turbot/steampipe-plugin-sdk/grpc/proto" + "github.com/turbot/steampipe-plugin-sdk/plugin/transform" + + "github.com/turbot/steampipe-plugin-sdk/plugin" +) + +//// TABLE DEFINITION //// + +func tableAzureDataFactory(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "azure_data_factory", + Description: "Azure Data Factory", + Get: &plugin.GetConfig{ + KeyColumns: plugin.AllColumns([]string{"name", "resource_group"}), + Hydrate: getFactory, + ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound", "404"}), + }, + List: &plugin.ListConfig{ + Hydrate: listFactories, + }, + Columns: []*plugin.Column{ + { + Name: "name", + Description: "The resource name.", + Type: proto.ColumnType_STRING, + }, + { + Name: "id", + Description: "The resource identifier.", + Type: proto.ColumnType_STRING, + Transform: transform.FromGo(), + }, + { + Name: "etag", + Description: "Etag identifies change in the resource.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ETag"), + }, + { + Name: "type", + Description: "The resource type.", + Type: proto.ColumnType_STRING, + }, + { + Name: "provisioning_state", + Description: "Factory provisioning state, example Succeeded.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("FactoryProperties.ProvisioningState"), + }, + { + Name: "create_time", + Description: "Time the factory was created in ISO8601 format.", + Type: proto.ColumnType_TIMESTAMP, + Transform: transform.FromField("FactoryProperties.CreateTime").Transform(convertDateToTime), + }, + { + Name: "version", + Description: "Version of the factory.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("FactoryProperties.Version"), + }, + { + Name: "public_network_access", + Description: "Whether or not public network access is allowed for the data factory.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("FactoryProperties.PublicNetworkAccess"), + }, + { + Name: "identity", + Description: "Managed service identity of the factory.", + Type: proto.ColumnType_JSON, + }, + { + Name: "encryption", + Description: "Properties to enable Customer Managed Key for the factory.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("FactoryProperties.EncryptionConfiguration"), + }, + { + Name: "repo_configuration", + Description: "Git repo information of the factory.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("FactoryProperties.RepoConfiguration"), + }, + { + Name: "global_parameters", + Description: "List of parameters for factory.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("FactoryProperties.GlobalParameters"), + }, + + // Standard columns + { + Name: "title", + Description: ColumnDescriptionTitle, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Name"), + }, + { + Name: "tags", + Description: ColumnDescriptionTags, + Type: proto.ColumnType_JSON, + }, + { + Name: "akas", + Description: ColumnDescriptionAkas, + Type: proto.ColumnType_JSON, + Transform: transform.FromField("ID").Transform(idToAkas), + }, + { + Name: "region", + Description: ColumnDescriptionRegion, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Location").Transform(toLower), + }, + { + Name: "resource_group", + Description: ColumnDescriptionResourceGroup, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(extractResourceGroupFromID), + }, + { + Name: "subscription_id", + Description: ColumnDescriptionSubscription, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(idToSubscriptionID), + }, + }, + } +} + +//// LIST FUNCTIONS //// + +func listFactories(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + factoryClient := datafactory.NewFactoriesClient(subscriptionID) + factoryClient.Authorizer = session.Authorizer + pagesLeft := true + + for pagesLeft { + result, err := factoryClient.List(context.Background()) + if err != nil { + return nil, err + } + + for _, factory := range result.Values() { + d.StreamListItem(ctx, factory) + } + result.NextWithContext(context.Background()) + pagesLeft = result.NotDone() + } + return nil, err +} + +//// HYDRATE FUNCTIONS //// + +func getFactory(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("getFactory") + + name := d.KeyColumnQuals["name"].GetStringValue() + resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue() + + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + factoryClient := datafactory.NewFactoriesClient(subscriptionID) + factoryClient.Authorizer = session.Authorizer + + op, err := factoryClient.Get(ctx, resourceGroup, name, "*") + if err != nil { + return nil, err + } + + // In some cases resource does not give any notFound error + // instead of notFound error, it returns empty data + if op.ID != nil { + return op, nil + } + + return nil, nil +} diff --git a/docs/tables/azure_data_factory.md b/docs/tables/azure_data_factory.md new file mode 100644 index 00000000..9d2aef7e --- /dev/null +++ b/docs/tables/azure_data_factory.md @@ -0,0 +1,48 @@ +# Table: azure_data_factory + +Azure Data Factory is the platform that solves such data scenarios. It is the cloud-based ETL and data integration service that allows to create data-driven workflows for orchestrating data movement and transforming data at scale. + +## Examples + +### Basic info + +```sql +select + name, + id, + type, + provisioning_state, + etag +from + azure_data_factory; +``` + + +### List system assigned identity type Factories + +```sql +select + name, + id, + type, + identity ->> 'type' as identity_type +from + azure_data_factory +where + identity ->> 'type' = 'SystemAssigned'; +``` + + +### List factories with plubic network access is allowed + +```sql +select + name, + id, + type, + public_network_access +from + azure_data_factory +where + public_network_access = 'Enabled'; +``` \ No newline at end of file From 2395565f394af1d5200147c87aeafbeeefd91315 Mon Sep 17 00:00:00 2001 From: rajeshbal65 Date: Wed, 7 Jul 2021 10:17:10 +0530 Subject: [PATCH 2/3] update --- azure/table_azure_data_factory.go | 16 +++++++++++----- docs/tables/azure_data_factory.md | 4 ++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/azure/table_azure_data_factory.go b/azure/table_azure_data_factory.go index ae62b6f2..2dc01329 100644 --- a/azure/table_azure_data_factory.go +++ b/azure/table_azure_data_factory.go @@ -10,7 +10,7 @@ import ( "github.com/turbot/steampipe-plugin-sdk/plugin" ) -//// TABLE DEFINITION //// +//// TABLE DEFINITION func tableAzureDataFactory(_ context.Context) *plugin.Table { return &plugin.Table{ @@ -55,7 +55,7 @@ func tableAzureDataFactory(_ context.Context) *plugin.Table { }, { Name: "create_time", - Description: "Time the factory was created in ISO8601 format.", + Description: "Specifies the time, the factory was created.", Type: proto.ColumnType_TIMESTAMP, Transform: transform.FromField("FactoryProperties.CreateTime").Transform(convertDateToTime), }, @@ -95,7 +95,7 @@ func tableAzureDataFactory(_ context.Context) *plugin.Table { Transform: transform.FromField("FactoryProperties.GlobalParameters"), }, - // Standard columns + // Steampipe standard columns { Name: "title", Description: ColumnDescriptionTitle, @@ -113,6 +113,8 @@ func tableAzureDataFactory(_ context.Context) *plugin.Table { Type: proto.ColumnType_JSON, Transform: transform.FromField("ID").Transform(idToAkas), }, + + // Azure standard column { Name: "region", Description: ColumnDescriptionRegion, @@ -135,7 +137,7 @@ func tableAzureDataFactory(_ context.Context) *plugin.Table { } } -//// LIST FUNCTIONS //// +//// LIST FUNCTION func listFactories(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { session, err := GetNewSession(ctx, d, "MANAGEMENT") @@ -163,7 +165,7 @@ func listFactories(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateDa return nil, err } -//// HYDRATE FUNCTIONS //// +//// HYDRATE FUNCTIONS func getFactory(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { plugin.Logger(ctx).Trace("getFactory") @@ -171,6 +173,10 @@ func getFactory(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) name := d.KeyColumnQuals["name"].GetStringValue() resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue() + if name == "" || resourceGroup == "" { + return nil, nil + } + session, err := GetNewSession(ctx, d, "MANAGEMENT") if err != nil { return nil, err diff --git a/docs/tables/azure_data_factory.md b/docs/tables/azure_data_factory.md index 9d2aef7e..5a38d8f4 100644 --- a/docs/tables/azure_data_factory.md +++ b/docs/tables/azure_data_factory.md @@ -18,7 +18,7 @@ from ``` -### List system assigned identity type Factories +### List system assigned identity type factories ```sql select @@ -33,7 +33,7 @@ where ``` -### List factories with plubic network access is allowed +### List factories with publicbic network access enabled ```sql select From 006369bd89dfc0694f39d2a38db0fd67bdf9b6d2 Mon Sep 17 00:00:00 2001 From: Subhajit Kumar Mondal Date: Thu, 8 Jul 2021 17:49:38 +0530 Subject: [PATCH 3/3] Fixed column reference and updated tests --- .../azure_data_factory/test-get-expected.json | 10 +-- .../azure_data_factory/test-get-query.sql | 2 +- .../test-list-expected.json | 4 +- .../azure_data_factory/test-list-query.sql | 2 +- .../test-not-found-expected.json | 2 +- .../test-not-found-query.sql | 2 +- .../test-turbot-expected.json | 8 +-- .../azure_data_factory/test-turbot-query.sql | 2 +- .../tests/azure_data_factory/variables.tf | 6 +- azure/table_azure_data_factory.go | 70 +++++++++---------- docs/tables/azure_data_factory.md | 4 +- 11 files changed, 58 insertions(+), 54 deletions(-) diff --git a/azure-test/tests/azure_data_factory/test-get-expected.json b/azure-test/tests/azure_data_factory/test-get-expected.json index 619e2748..754b90a6 100644 --- a/azure-test/tests/azure_data_factory/test-get-expected.json +++ b/azure-test/tests/azure_data_factory/test-get-expected.json @@ -1,13 +1,13 @@ [ { "id": "{{ output.resource_id.value }}", - "name": "{{resourceName}}", - "region": "eastus", - "resource_group": "{{resourceName}}", + "name": "{{ resourceName }}", + "region": "{{ output.location.value }}", + "resource_group": "{{ resourceName }}", "subscription_id": "{{ output.subscription_id.value }}", "tags": { - "name": "{{resourceName}}" + "name": "{{ resourceName }}" }, "type": "Microsoft.DataFactory/factories" } -] \ No newline at end of file +] diff --git a/azure-test/tests/azure_data_factory/test-get-query.sql b/azure-test/tests/azure_data_factory/test-get-query.sql index 83808052..cea97285 100644 --- a/azure-test/tests/azure_data_factory/test-get-query.sql +++ b/azure-test/tests/azure_data_factory/test-get-query.sql @@ -1,3 +1,3 @@ select name, id, type, tags, region, resource_group, subscription_id from azure.azure_data_factory -where name = '{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-list-expected.json b/azure-test/tests/azure_data_factory/test-list-expected.json index f15afcd8..602b9f72 100644 --- a/azure-test/tests/azure_data_factory/test-list-expected.json +++ b/azure-test/tests/azure_data_factory/test-list-expected.json @@ -1,7 +1,7 @@ [ { "id": "{{ output.resource_id.value }}", - "name": "{{resourceName}}", + "name": "{{ resourceName }}", "type": "Microsoft.DataFactory/factories" } -] \ No newline at end of file +] diff --git a/azure-test/tests/azure_data_factory/test-list-query.sql b/azure-test/tests/azure_data_factory/test-list-query.sql index f900f679..b484ff06 100644 --- a/azure-test/tests/azure_data_factory/test-list-query.sql +++ b/azure-test/tests/azure_data_factory/test-list-query.sql @@ -1,3 +1,3 @@ select name, id, type from azure.azure_data_factory -where name = '{{resourceName}}' \ No newline at end of file +where name = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-not-found-expected.json b/azure-test/tests/azure_data_factory/test-not-found-expected.json index ec747fa4..19765bd5 100644 --- a/azure-test/tests/azure_data_factory/test-not-found-expected.json +++ b/azure-test/tests/azure_data_factory/test-not-found-expected.json @@ -1 +1 @@ -null \ No newline at end of file +null diff --git a/azure-test/tests/azure_data_factory/test-not-found-query.sql b/azure-test/tests/azure_data_factory/test-not-found-query.sql index 04ecb573..ecc6ad16 100644 --- a/azure-test/tests/azure_data_factory/test-not-found-query.sql +++ b/azure-test/tests/azure_data_factory/test-not-found-query.sql @@ -1,3 +1,3 @@ select name, tags, title, akas from azure.azure_data_factory -where name = 'dummy-{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file +where name = 'dummy-{{ resourceName }}' and resource_group = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-turbot-expected.json b/azure-test/tests/azure_data_factory/test-turbot-expected.json index 3ceae2f7..43ffbc2c 100644 --- a/azure-test/tests/azure_data_factory/test-turbot-expected.json +++ b/azure-test/tests/azure_data_factory/test-turbot-expected.json @@ -4,10 +4,10 @@ "{{ output.resource_aka.value }}", "{{ output.resource_aka_lower.value }}" ], - "name": "{{resourceName}}", + "name": "{{ resourceName }}", "tags": { - "name": "{{resourceName}}" + "name": "{{ resourceName }}" }, - "title": "{{resourceName}}" + "title": "{{ resourceName }}" } -] \ No newline at end of file +] diff --git a/azure-test/tests/azure_data_factory/test-turbot-query.sql b/azure-test/tests/azure_data_factory/test-turbot-query.sql index 434fc5ae..911c3463 100644 --- a/azure-test/tests/azure_data_factory/test-turbot-query.sql +++ b/azure-test/tests/azure_data_factory/test-turbot-query.sql @@ -1,3 +1,3 @@ select name, tags, title, akas from azure.azure_data_factory -where name = '{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/variables.tf b/azure-test/tests/azure_data_factory/variables.tf index 20dd0733..7608c275 100644 --- a/azure-test/tests/azure_data_factory/variables.tf +++ b/azure-test/tests/azure_data_factory/variables.tf @@ -39,7 +39,7 @@ resource "azurerm_resource_group" "named_test_resource" { resource "azurerm_data_factory" "named_test_resource" { name = var.resource_name - location = "East US" + location = azurerm_resource_group.named_test_resource.location resource_group_name = azurerm_resource_group.named_test_resource.name tags = { name = var.resource_name @@ -62,6 +62,10 @@ output "resource_id" { value = azurerm_data_factory.named_test_resource.id } +output "location" { + value = azurerm_resource_group.named_test_resource.location +} + output "subscription_id" { value = var.azure_subscription } diff --git a/azure/table_azure_data_factory.go b/azure/table_azure_data_factory.go index 2dc01329..49e901e3 100644 --- a/azure/table_azure_data_factory.go +++ b/azure/table_azure_data_factory.go @@ -18,11 +18,11 @@ func tableAzureDataFactory(_ context.Context) *plugin.Table { Description: "Azure Data Factory", Get: &plugin.GetConfig{ KeyColumns: plugin.AllColumns([]string{"name", "resource_group"}), - Hydrate: getFactory, - ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound", "404"}), + Hydrate: getDataFactory, + ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound", "Invalid input"}), }, List: &plugin.ListConfig{ - Hydrate: listFactories, + Hydrate: listDataFactories, }, Columns: []*plugin.Column{ { @@ -36,22 +36,16 @@ func tableAzureDataFactory(_ context.Context) *plugin.Table { Type: proto.ColumnType_STRING, Transform: transform.FromGo(), }, - { - Name: "etag", - Description: "Etag identifies change in the resource.", - Type: proto.ColumnType_STRING, - Transform: transform.FromField("ETag"), - }, { Name: "type", Description: "The resource type.", Type: proto.ColumnType_STRING, }, { - Name: "provisioning_state", - Description: "Factory provisioning state, example Succeeded.", + Name: "version", + Description: "Version of the factory.", Type: proto.ColumnType_STRING, - Transform: transform.FromField("FactoryProperties.ProvisioningState"), + Transform: transform.FromField("FactoryProperties.Version"), }, { Name: "create_time", @@ -60,16 +54,27 @@ func tableAzureDataFactory(_ context.Context) *plugin.Table { Transform: transform.FromField("FactoryProperties.CreateTime").Transform(convertDateToTime), }, { - Name: "version", - Description: "Version of the factory.", + Name: "etag", + Description: "An unique read-only string that changes whenever the resource is updated.", Type: proto.ColumnType_STRING, - Transform: transform.FromField("FactoryProperties.Version"), + Transform: transform.FromField("ETag"), + }, + { + Name: "provisioning_state", + Description: "Factory provisioning state, example Succeeded.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("FactoryProperties.ProvisioningState"), }, { Name: "public_network_access", Description: "Whether or not public network access is allowed for the data factory.", Type: proto.ColumnType_STRING, - Transform: transform.FromField("FactoryProperties.PublicNetworkAccess"), + Transform: transform.FromField("FactoryProperties.PublicNetworkAccess").Transform(transform.ToString), + }, + { + Name: "additional_properties", + Description: "Unmatched properties from the message are deserialized this collection.", + Type: proto.ColumnType_JSON, }, { Name: "identity", @@ -139,7 +144,7 @@ func tableAzureDataFactory(_ context.Context) *plugin.Table { //// LIST FUNCTION -func listFactories(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { +func listDataFactories(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { session, err := GetNewSession(ctx, d, "MANAGEMENT") if err != nil { return nil, err @@ -148,14 +153,13 @@ func listFactories(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateDa factoryClient := datafactory.NewFactoriesClient(subscriptionID) factoryClient.Authorizer = session.Authorizer - pagesLeft := true + pagesLeft := true for pagesLeft { result, err := factoryClient.List(context.Background()) if err != nil { return nil, err } - for _, factory := range result.Values() { d.StreamListItem(ctx, factory) } @@ -167,16 +171,10 @@ func listFactories(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateDa //// HYDRATE FUNCTIONS -func getFactory(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { - plugin.Logger(ctx).Trace("getFactory") - - name := d.KeyColumnQuals["name"].GetStringValue() - resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue() - - if name == "" || resourceGroup == "" { - return nil, nil - } +func getDataFactory(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("getDataFactory") + // Create session session, err := GetNewSession(ctx, d, "MANAGEMENT") if err != nil { return nil, err @@ -186,16 +184,18 @@ func getFactory(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) factoryClient := datafactory.NewFactoriesClient(subscriptionID) factoryClient.Authorizer = session.Authorizer + name := d.KeyColumnQuals["name"].GetStringValue() + resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue() + + // Return nil, if no input provide + if name == "" || resourceGroup == "" { + return nil, nil + } + op, err := factoryClient.Get(ctx, resourceGroup, name, "*") if err != nil { return nil, err } - // In some cases resource does not give any notFound error - // instead of notFound error, it returns empty data - if op.ID != nil { - return op, nil - } - - return nil, nil + return op, nil } diff --git a/docs/tables/azure_data_factory.md b/docs/tables/azure_data_factory.md index 5a38d8f4..238ad115 100644 --- a/docs/tables/azure_data_factory.md +++ b/docs/tables/azure_data_factory.md @@ -33,7 +33,7 @@ where ``` -### List factories with publicbic network access enabled +### List factories with public network access enabled ```sql select @@ -45,4 +45,4 @@ from azure_data_factory where public_network_access = 'Enabled'; -``` \ No newline at end of file +```