From dee9e72df953165a49117240d2c6429c8559fd4b Mon Sep 17 00:00:00 2001 From: ParthaI Date: Fri, 1 Apr 2022 20:57:57 +0530 Subject: [PATCH 1/4] Add table azure_storage_share_file closes #455 --- .../azure_storage_share_file/dependencies.txt | 0 .../test-get-expected.json | 7 + .../test-get-query.sql | 3 + .../test-list-expected.json | 6 + .../test-list-query.sql | 3 + .../test-not-found-expected.json | 1 + .../test-not-found-query.sql | 3 + .../test-turbot-expected.json | 9 + .../test-turbot-query.sql | 3 + .../azure_storage_share_file/variables.json | 1 + .../azure_storage_share_file/variables.tf | 80 +++++ azure/plugin.go | 1 + azure/table_azure_storage_share_file.go | 277 ++++++++++++++++++ docs/tables/azure_storage_share_file.md | 68 +++++ 14 files changed, 462 insertions(+) create mode 100644 azure-test/tests/azure_storage_share_file/dependencies.txt create mode 100644 azure-test/tests/azure_storage_share_file/test-get-expected.json create mode 100644 azure-test/tests/azure_storage_share_file/test-get-query.sql create mode 100644 azure-test/tests/azure_storage_share_file/test-list-expected.json create mode 100644 azure-test/tests/azure_storage_share_file/test-list-query.sql create mode 100644 azure-test/tests/azure_storage_share_file/test-not-found-expected.json create mode 100644 azure-test/tests/azure_storage_share_file/test-not-found-query.sql create mode 100644 azure-test/tests/azure_storage_share_file/test-turbot-expected.json create mode 100644 azure-test/tests/azure_storage_share_file/test-turbot-query.sql create mode 100644 azure-test/tests/azure_storage_share_file/variables.json create mode 100644 azure-test/tests/azure_storage_share_file/variables.tf create mode 100644 azure/table_azure_storage_share_file.go create mode 100644 docs/tables/azure_storage_share_file.md diff --git a/azure-test/tests/azure_storage_share_file/dependencies.txt b/azure-test/tests/azure_storage_share_file/dependencies.txt new file mode 100644 index 00000000..e69de29b diff --git a/azure-test/tests/azure_storage_share_file/test-get-expected.json b/azure-test/tests/azure_storage_share_file/test-get-expected.json new file mode 100644 index 00000000..cd6793e2 --- /dev/null +++ b/azure-test/tests/azure_storage_share_file/test-get-expected.json @@ -0,0 +1,7 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{ resourceName }}", + "storage_account_name": "{{ resourceName }}" + } + ] \ No newline at end of file diff --git a/azure-test/tests/azure_storage_share_file/test-get-query.sql b/azure-test/tests/azure_storage_share_file/test-get-query.sql new file mode 100644 index 00000000..74fc3b91 --- /dev/null +++ b/azure-test/tests/azure_storage_share_file/test-get-query.sql @@ -0,0 +1,3 @@ +select name, id, storage_account_name +from azure_storage_share_file +where name = '{{resourceName}}' and resource_group = '{{resourceName}}' and storage_account_name = '{{resourceName}}'; \ No newline at end of file diff --git a/azure-test/tests/azure_storage_share_file/test-list-expected.json b/azure-test/tests/azure_storage_share_file/test-list-expected.json new file mode 100644 index 00000000..db8cea80 --- /dev/null +++ b/azure-test/tests/azure_storage_share_file/test-list-expected.json @@ -0,0 +1,6 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{resourceName}}" + } +] \ No newline at end of file diff --git a/azure-test/tests/azure_storage_share_file/test-list-query.sql b/azure-test/tests/azure_storage_share_file/test-list-query.sql new file mode 100644 index 00000000..b4ed114b --- /dev/null +++ b/azure-test/tests/azure_storage_share_file/test-list-query.sql @@ -0,0 +1,3 @@ +select name, id +from azure_storage_share_file +where name = '{{resourceName}}'; \ No newline at end of file diff --git a/azure-test/tests/azure_storage_share_file/test-not-found-expected.json b/azure-test/tests/azure_storage_share_file/test-not-found-expected.json new file mode 100644 index 00000000..ec747fa4 --- /dev/null +++ b/azure-test/tests/azure_storage_share_file/test-not-found-expected.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/azure-test/tests/azure_storage_share_file/test-not-found-query.sql b/azure-test/tests/azure_storage_share_file/test-not-found-query.sql new file mode 100644 index 00000000..757b46ab --- /dev/null +++ b/azure-test/tests/azure_storage_share_file/test-not-found-query.sql @@ -0,0 +1,3 @@ +select name, id +from azure_storage_share_file +where name = 'dummy{{resourceName}}' and resource_group = '{{resourceName}}'; \ No newline at end of file diff --git a/azure-test/tests/azure_storage_share_file/test-turbot-expected.json b/azure-test/tests/azure_storage_share_file/test-turbot-expected.json new file mode 100644 index 00000000..8c46e3f9 --- /dev/null +++ b/azure-test/tests/azure_storage_share_file/test-turbot-expected.json @@ -0,0 +1,9 @@ +[ + { + "akas": [ + "{{ output.resource_aka.value }}", + "{{ output.resource_aka_lower.value }}" + ], + "title": "{{ resourceName }}" + } + ] \ No newline at end of file diff --git a/azure-test/tests/azure_storage_share_file/test-turbot-query.sql b/azure-test/tests/azure_storage_share_file/test-turbot-query.sql new file mode 100644 index 00000000..ed750fca --- /dev/null +++ b/azure-test/tests/azure_storage_share_file/test-turbot-query.sql @@ -0,0 +1,3 @@ +select title, akas +from azure_storage_share_file +where name = '{{resourceName}}' and resource_group = '{{resourceName}}'; \ No newline at end of file diff --git a/azure-test/tests/azure_storage_share_file/variables.json b/azure-test/tests/azure_storage_share_file/variables.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/azure-test/tests/azure_storage_share_file/variables.json @@ -0,0 +1 @@ +{} diff --git a/azure-test/tests/azure_storage_share_file/variables.tf b/azure-test/tests/azure_storage_share_file/variables.tf new file mode 100644 index 00000000..85ea11dd --- /dev/null +++ b/azure-test/tests/azure_storage_share_file/variables.tf @@ -0,0 +1,80 @@ + +variable "resource_name" { + type = string + default = "turbot-test-20200125-create-update" + description = "Name of the resource used throughout the test." +} + +variable "azure_environment" { + type = string + default = "public" + description = "Azure environment used for the test." +} + +variable "azure_subscription" { + type = string + default = "3510ae4d-530b-497d-8f30-53b9616fc6c1" + description = "Azure subscription used for the test." +} + +provider "azurerm" { + # Cannot be passed as a variable + version = "=3.0.0" + environment = var.azure_environment + subscription_id = var.azure_subscription + features{} +} + +data "azurerm_client_config" "current" {} + +data "null_data_source" "resource" { + inputs = { + scope = "azure:///subscriptions/${data.azurerm_client_config.current.subscription_id}" + } +} + +resource "azurerm_resource_group" "named_test_resource" { + name = var.resource_name + location = "East US" +} + +resource "azurerm_storage_account" "named_test_resource" { + name = var.resource_name + resource_group_name = azurerm_resource_group.named_test_resource.name + location = azurerm_resource_group.named_test_resource.location + account_tier = "Standard" + account_kind = "StorageV2" + access_tier = "Cool" + account_replication_type = "LRS" + + tags = { + name = var.resource_name + } +} + +resource "azurerm_storage_share" "named_test_resource" { + name = var.resource_name + storage_account_name = azurerm_storage_account.named_test_resource.name + quota = 50 +} + +resource "azurerm_storage_share_file" "named_test_resource" { + name = var.resource_name + storage_share_id = azurerm_storage_share.named_test_resource.id +} + +output "resource_aka" { + value = "azure:///subscriptions/${var.azure_subscription}/resourceGroups/${var.resource_name}/providers/Microsoft.Storage/storageAccounts/${var.resource_name}/fileServices/default/shares/${var.resource_name}" +} + +output "resource_aka_lower" { + value = "azure:///subscriptions/${lower(var.azure_subscription)}/resourcegroups/${lower(var.resource_name)}/providers/microsoft.storage/storageaccounts/${lower(var.resource_name)}/fileservices/default/shares/${lower(var.resource_name)}" +} + +output "resource_name" { + value = var.resource_name +} + +output "resource_id" { + value = "/subscriptions/${var.azure_subscription}/resourceGroups/${var.resource_name}/providers/Microsoft.Storage/storageAccounts/${var.resource_name}/fileServices/default/shares/${var.resource_name}" +} diff --git a/azure/plugin.go b/azure/plugin.go index 8f9f585f..36e87025 100644 --- a/azure/plugin.go +++ b/azure/plugin.go @@ -131,6 +131,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "azure_sql_database": tableAzureSqlDatabase(ctx), "azure_sql_server": tableAzureSQLServer(ctx), "azure_storage_account": tableAzureStorageAccount(ctx), + "azure_storage_share_file": tableAzureStorageShareFile(ctx), "azure_storage_blob": tableAzureStorageBlob(ctx), "azure_storage_blob_service": tableAzureStorageBlobService(ctx), "azure_storage_container": tableAzureStorageContainer(ctx), diff --git a/azure/table_azure_storage_share_file.go b/azure/table_azure_storage_share_file.go new file mode 100644 index 00000000..09859b8f --- /dev/null +++ b/azure/table_azure_storage_share_file.go @@ -0,0 +1,277 @@ +package azure + +import ( + "context" + "strings" + + "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage" + "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe-plugin-sdk/v2/grpc/proto" + "github.com/turbot/steampipe-plugin-sdk/v2/plugin/transform" + + "github.com/turbot/steampipe-plugin-sdk/v2/plugin" +) + +//// TABLE DEFINITION + +func tableAzureStorageShareFile(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "azure_storage_share_file", + Description: "Azure Storage Share File", + Get: &plugin.GetConfig{ + KeyColumns: plugin.AllColumns([]string{"name", "resource_group", "storage_account_name"}), + Hydrate: StorageAccountsFileShare, + ShouldIgnoreError: isNotFoundError([]string{"ResourceGroupNotFound", "ResourceNotFound", "404"}), + }, + List: &plugin.ListConfig{ + ParentHydrate: listStorageAccounts, + Hydrate: listStorageAccountsFileShares, + }, + Columns: azureColumns([]*plugin.Column{ + { + Name: "name", + Type: proto.ColumnType_STRING, + Description: "The name of the resource.", + }, + { + Name: "storage_account_name", + Type: proto.ColumnType_STRING, + Description: "The name of the storage account.", + }, + { + Name: "id", + Type: proto.ColumnType_STRING, + Description: "Fully qualified resource ID for the resource.", + Transform: transform.FromField("ID"), + }, + { + Name: "type", + Type: proto.ColumnType_STRING, + Description: "The type of the resource.", + }, + { + Name: "access_tier", + Type: proto.ColumnType_STRING, + Description: "Access tier for specific share. GpV2 account can choose between TransactionOptimized (default), Hot, and Cool.", + Transform: transform.FromField("FileShareProperties.AccessTier"), + }, + { + Name: "access_tier_change_time", + Type: proto.ColumnType_TIMESTAMP, + Description: "Indicates the last modification time for share access tier.", + Transform: transform.FromField("FileShareProperties.AccessTierChangeTime").Transform(convertDateToTime), + }, + { + Name: "access_tier_status", + Type: proto.ColumnType_STRING, + Description: "Indicates if there is a pending transition for access tier.", + Transform: transform.FromField("FileShareProperties.AccessTierStatus"), + }, + { + Name: "last_modified_time", + Type: proto.ColumnType_TIMESTAMP, + Description: "Returns the date and time the share was last modified.", + Transform: transform.FromField("FileShareProperties.LastModifiedTime").Transform(convertDateToTime), + }, + { + Name: "deleted", + Type: proto.ColumnType_BOOL, + Description: "Indicates whether the share was deleted.", + Transform: transform.FromField("FileShareProperties.Deleted"), + }, + { + Name: "deleted_time", + Type: proto.ColumnType_TIMESTAMP, + Description: "The deleted time if the share was deleted.", + Transform: transform.FromField("FileShareProperties.DeletedTime").Transform(convertDateToTime), + }, + { + Name: "enabled_protocols", + Type: proto.ColumnType_STRING, + Description: "The authentication protocol that is used for the file share. Can only be specified when creating a share. Possible values include: 'SMB', 'NFS'.", + Transform: transform.FromField("FileShareProperties.EnabledProtocols"), + }, + { + Name: "remaining_retention_days", + Type: proto.ColumnType_INT, + Description: "Remaining retention days for share that was soft deleted.", + Transform: transform.FromField("FileShareProperties.RemainingRetentionDays"), + }, + { + Name: "root_squash", + Type: proto.ColumnType_STRING, + Description: "The property is for NFS share only. The default is NoRootSquash. Possible values include: 'NoRootSquash', 'RootSquash', 'AllSquash'.", + Transform: transform.FromField("FileShareProperties.RootSquash"), + }, + { + Name: "share_quota", + Type: proto.ColumnType_INT, + Description: "The maximum size of the share, in gigabytes. Must be greater than 0, and less than or equal to 5TB (5120). For Large File Shares, the maximum size is 102400.", + Transform: transform.FromField("FileShareProperties.ShareQuota"), + }, + { + Name: "share_usage_bytes", + Type: proto.ColumnType_INT, + Description: "The approximate size of the data stored on the share. Note that this value may not include all recently created or recently resized files.", + Transform: transform.FromField("FileShareProperties.ShareUsageBytes"), + }, + { + Name: "version", + Type: proto.ColumnType_STRING, + Description: "The version of the share.", + Transform: transform.FromField("FileShareProperties.Version"), + }, + { + Name: "metadata", + Type: proto.ColumnType_JSON, + Description: "A name-value pair to associate with the share as metadata.", + Transform: transform.FromField("FileShareProperties.Metadata"), + }, + + // Steampipe standard columns + { + Name: "title", + Description: ColumnDescriptionTitle, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Name"), + }, + { + Name: "akas", + Description: ColumnDescriptionAkas, + Type: proto.ColumnType_JSON, + Transform: transform.FromField("ID").Transform(idToAkas), + }, + + // Azure standard columns + { + Name: "resource_group", + Description: ColumnDescriptionResourceGroup, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ResourceGroup").Transform(toLower), + }, + }), + } +} + +type FileShareInfo struct { + storage.FileShareProperties + Name string + ID string + Type string + StorageAccountName string + ResourceGroup string +} + +//// LIST FUNCTION + +func listStorageAccountsFileShares(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + storageAccount := h.Item.(*storageAccountInfo) + + session, err := GetNewSession(ctx, d, "MANAGEMENT") + logger := plugin.Logger(ctx) + if err != nil { + logger.Error("listStorageAccountsFileShare", "get session error", err) + return nil, err + } + subscriptionID := session.SubscriptionID + fileShareCLient := storage.NewFileSharesClientWithBaseURI(session.ResourceManagerEndpoint, subscriptionID) + fileShareCLient.Authorizer = session.Authorizer + + // Limiting the results + limit := d.QueryContext.Limit + maxResult := "100" + if d.QueryContext.Limit != nil { + if *limit < 100 { + maxResult = types.IntToString(*limit) + } + } + + result, err := fileShareCLient.List(ctx, *storageAccount.ResourceGroup, *storageAccount.Name, maxResult, "", "") + if err != nil { + logger.Error("listStorageAccountsFileShare", "api error", err) + return nil, err + } + + for _, fileShare := range result.Values() { + d.StreamListItem(ctx, &FileShareInfo{ + FileShareProperties: *fileShare.FileShareProperties, + Name: *fileShare.Name, + ID: *fileShare.ID, + Type: *fileShare.Type, + StorageAccountName: *storageAccount.Name, + ResourceGroup: *storageAccount.ResourceGroup, + }) + + // Check if context has been cancelled or if the limit has been hit (if specified) + // if there is a limit, it will return the number of rows required to reach this limit + if d.QueryStatus.RowsRemaining(ctx) == 0 { + return nil, nil + } + } + + for result.NotDone() { + err = result.NextWithContext(ctx) + if err != nil { + return nil, err + } + + for _, fileShare := range result.Values() { + d.StreamListItem(ctx, &FileShareInfo{ + FileShareProperties: *fileShare.FileShareProperties, + Name: *fileShare.Name, + ID: *fileShare.ID, + Type: *fileShare.Type, + StorageAccountName: *storageAccount.Name, + ResourceGroup: *storageAccount.ResourceGroup, + }) + + // Check if context has been cancelled or if the limit has been hit (if specified) + // if there is a limit, it will return the number of rows required to reach this limit + if d.QueryStatus.RowsRemaining(ctx) == 0 { + return nil, nil + } + } + } + + return nil, err +} + +//// HYDRATE FUNCTIONS + +func StorageAccountsFileShare(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("StorageAccountsFileShare") + + session, err := GetNewSession(ctx, d, "MANAGEMENT") + logger := plugin.Logger(ctx) + if err != nil { + logger.Error("listStorageAccountsFileShare", "get session error", err) + return nil, err + } + + resourceGroup := d.KeyColumnQualString("resource_group") + storageAccountName := d.KeyColumnQualString("storage_account_name") + name := d.KeyColumnQualString("name") + + if strings.Trim(name, " ") != "" || strings.Trim(resourceGroup, " ") != "" || strings.Trim(storageAccountName, " ") != "" { + return nil, nil + } + + subscriptionID := session.SubscriptionID + fileShareCLient := storage.NewFileSharesClientWithBaseURI(session.ResourceManagerEndpoint, subscriptionID) + fileShareCLient.Authorizer = session.Authorizer + + result, err := fileShareCLient.Get(ctx, resourceGroup, storageAccountName, name, "") + + if err != nil { + return nil, err + } + + return &FileShareInfo{ + FileShareProperties: *result.FileShareProperties, + Name: *result.Name, + ID: *result.ID, + Type: *result.Type, + StorageAccountName: storageAccountName, + ResourceGroup: resourceGroup, + }, nil +} diff --git a/docs/tables/azure_storage_share_file.md b/docs/tables/azure_storage_share_file.md new file mode 100644 index 00000000..8473719d --- /dev/null +++ b/docs/tables/azure_storage_share_file.md @@ -0,0 +1,68 @@ +# Table: azure_storage_blob + +Azure Files is Microsoft's easy-to-use cloud file system. Azure file shares can be mounted in Windows, Linux, and macOS. + +## Examples + +### Basic info + +```sql +select + name, + storage_account_name, + type, + access_tier, + share_quota, + enabled_protocols +from + azure_storage_share_file; +``` + +### List file shares for storage accounts + +```sql +select + name, + storage_account_name, + type, + access_tier, + share_quota, + enabled_protocols +from + azure_storage_share_file +where + type = 'Microsoft.Storage/storageAccounts/fileServices/shares'; +``` + +### List file shares with default access tier + +```sql +select + name, + storage_account_name, + type, + access_tier, + access_tier_change_time, + share_quota, + enabled_protocols +from + azure_storage_share_file +where + access_tier = 'TransactionOptimized'; +``` + +### Get file share with maximum share quota + +```sql +select + name, + storage_account_name, + type, + access_tier, + access_tier_change_time, + share_quota, + enabled_protocols +from + azure_storage_share_file +order by share_quota desc limit 1; +``` \ No newline at end of file From 8069c7d60c92c6159e93fa5bb79a369eb9d40e1e Mon Sep 17 00:00:00 2001 From: ParthaI Date: Fri, 1 Apr 2022 21:06:31 +0530 Subject: [PATCH 2/4] Updated the function name --- azure/table_azure_storage_share_file.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure/table_azure_storage_share_file.go b/azure/table_azure_storage_share_file.go index 09859b8f..abc99354 100644 --- a/azure/table_azure_storage_share_file.go +++ b/azure/table_azure_storage_share_file.go @@ -20,7 +20,7 @@ func tableAzureStorageShareFile(_ context.Context) *plugin.Table { Description: "Azure Storage Share File", Get: &plugin.GetConfig{ KeyColumns: plugin.AllColumns([]string{"name", "resource_group", "storage_account_name"}), - Hydrate: StorageAccountsFileShare, + Hydrate: getStorageAccountsFileShare, ShouldIgnoreError: isNotFoundError([]string{"ResourceGroupNotFound", "ResourceNotFound", "404"}), }, List: &plugin.ListConfig{ @@ -238,13 +238,13 @@ func listStorageAccountsFileShares(ctx context.Context, d *plugin.QueryData, h * //// HYDRATE FUNCTIONS -func StorageAccountsFileShare(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { - plugin.Logger(ctx).Trace("StorageAccountsFileShare") +func getStorageAccountsFileShare(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("getStorageAccountsFileShare") session, err := GetNewSession(ctx, d, "MANAGEMENT") logger := plugin.Logger(ctx) if err != nil { - logger.Error("listStorageAccountsFileShare", "get session error", err) + logger.Error("getStorageAccountsFileShare", "get session error", err) return nil, err } From 9d637b255e73a05bf8d90533adbdec59a43fef84 Mon Sep 17 00:00:00 2001 From: ParthaI <47887552+ParthaI@users.noreply.github.com> Date: Mon, 4 Apr 2022 09:50:58 +0530 Subject: [PATCH 3/4] Update docs/tables/azure_storage_share_file.md Co-authored-by: souravTurbot <78197905+bigdatasourav@users.noreply.github.com> --- docs/tables/azure_storage_share_file.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tables/azure_storage_share_file.md b/docs/tables/azure_storage_share_file.md index 8473719d..03cc20ba 100644 --- a/docs/tables/azure_storage_share_file.md +++ b/docs/tables/azure_storage_share_file.md @@ -1,4 +1,4 @@ -# Table: azure_storage_blob +# Table: azure_storage_share_file Azure Files is Microsoft's easy-to-use cloud file system. Azure file shares can be mounted in Windows, Linux, and macOS. From 6be05165d20d5b16213a1d605416ff12b074e3ed Mon Sep 17 00:00:00 2001 From: ParthaI <47887552+ParthaI@users.noreply.github.com> Date: Tue, 5 Apr 2022 11:15:29 +0530 Subject: [PATCH 4/4] Updated the doc --- docs/tables/azure_storage_share_file.md | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/docs/tables/azure_storage_share_file.md b/docs/tables/azure_storage_share_file.md index 03cc20ba..10780318 100644 --- a/docs/tables/azure_storage_share_file.md +++ b/docs/tables/azure_storage_share_file.md @@ -18,22 +18,6 @@ from azure_storage_share_file; ``` -### List file shares for storage accounts - -```sql -select - name, - storage_account_name, - type, - access_tier, - share_quota, - enabled_protocols -from - azure_storage_share_file -where - type = 'Microsoft.Storage/storageAccounts/fileServices/shares'; -``` - ### List file shares with default access tier ```sql @@ -65,4 +49,4 @@ select from azure_storage_share_file order by share_quota desc limit 1; -``` \ No newline at end of file +```