Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix issue where schema extraction is broken for remote services. Closes #1497 #1502

Merged
merged 5 commits into from
Mar 7, 2022
Merged

Fix issue where schema extraction is broken for remote services. Closes #1497 #1502

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ead4dce
fix schema extraction issue for remote
binaek Feb 23, 2022
ca7e2b9
optimising local schema fetch
binaek Feb 28, 2022
f3b9506
review corrections
binaek Mar 2, 2022
00e328c
rename
binaek Mar 2, 2022
23cad58
review corrections
binaek Mar 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions db/db_client/db_client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,8 @@ func (c *DbClient) RefreshConnectionAndSearchPaths(ctx context.Context) *steampi
return res
}

// GetSchemaFromDB requests for all columns of tables backed by steampipe plugins
// and creates golang struct representations from the result
func (c *DbClient) GetSchemaFromDB(ctx context.Context) (*schema.Metadata, error) {
utils.LogTime("db_client.GetSchemaFromDB start")
defer utils.LogTime("db_client.GetSchemaFromDB end")
Expand Down
12 changes: 8 additions & 4 deletions db/db_local/local_db_client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,8 @@ func (c *LocalDbClient) ContructSearchPath(ctx context.Context, requiredSearchPa
return c.client.ContructSearchPath(ctx, requiredSearchPath, searchPathPrefix, currentSearchPath)
}

// GetSchemaFromDB for LocalDBClient optimises the schema extraction by extracting schema
// information for connections backed by distinct plugins and then fanning back out.
func (c *LocalDbClient) GetSchemaFromDB(ctx context.Context) (*schema.Metadata, error) {
kaidaguerre marked this conversation as resolved.
Show resolved Hide resolved
// build a ConnectionSchemaMap object to identify the schemas to load
// (pass nil for connection state - this forces NewConnectionSchemaMap to load it)
Expand Down Expand Up @@ -194,6 +196,8 @@ func (c *LocalDbClient) GetSchemaFromDB(ctx context.Context) (*schema.Metadata,
return metadata, nil
}

// update schemaMetadata to add in all other schemas which have the same schemas as those we have loaded
// NOTE: this mutates schemaMetadata
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

feel free to remove the extra spaces from my suggested comment

func (c *LocalDbClient) populateSchemaMetadata(schemaMetadata *schema.Metadata, connectionSchemaMap steampipeconfig.ConnectionSchemaMap) {
kaidaguerre marked this conversation as resolved.
Show resolved Hide resolved
// we now need to add in all other schemas which have the same schemas as those we have loaded
for loadedSchema, otherSchemas := range connectionSchemaMap {
Expand All @@ -210,11 +214,11 @@ func (c *LocalDbClient) populateSchemaMetadata(schemaMetadata *schema.Metadata,
}
}

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

function comment to explain why this is different from the base version

func (c *LocalDbClient) buildSchemasQuery(hintSchemas []string) string {
for idx, s := range hintSchemas {
hintSchemas[idx] = fmt.Sprintf("'%s'", s)
func (c *LocalDbClient) buildSchemasQuery(schemas []string) string {
for idx, s := range schemas {
schemas[idx] = fmt.Sprintf("'%s'", s)
}
schemaClause := strings.Join(hintSchemas, ",")
schemaClause := strings.Join(schemas, ",")
query := fmt.Sprintf(`
SELECT
table_name,
Expand Down