From f7fb31a5f2e9ead4c43e90dbc759341a2227c314 Mon Sep 17 00:00:00 2001 From: Xin Hao Zhang Date: Fri, 19 Aug 2022 16:57:15 -0400 Subject: [PATCH] ui/cluster-ui: create statements insights api This commit adds a function to the `cluster-ui` pkg that queries `crdb_internal.cluster_execution_insights` to surface slow running queries. The information retrieved is intended to be used in the insights statement overview and details pages. A new field `statementInsights` is added to the `cachedData` object in the db-console redux store, and the corresponding function to issue the data fetch is also added in `apiReducers`. This commit does not add any reducers or sagas to CC to fetch and store this data. Release justification: non-production code change Release note: None --- .../cluster-ui/src/api/insightsApi.ts | 218 +++++++++++++----- .../cluster-ui/src/insights/types.ts | 22 +- .../cluster-ui/src/insights/utils.ts | 6 +- .../transactionInsights.fixture.ts | 12 +- .../transactionInsightsTable.tsx | 10 +- .../db-console/src/redux/apiReducers.ts | 11 + 6 files changed, 202 insertions(+), 77 deletions(-) diff --git a/pkg/ui/workspaces/cluster-ui/src/api/insightsApi.ts b/pkg/ui/workspaces/cluster-ui/src/api/insightsApi.ts index 857510245f4f..ced67d02bd04 100644 --- a/pkg/ui/workspaces/cluster-ui/src/api/insightsApi.ts +++ b/pkg/ui/workspaces/cluster-ui/src/api/insightsApi.ts @@ -18,8 +18,10 @@ import { InsightEventDetails, InsightExecEnum, InsightNameEnum, + StatementInsightEvent, } from "src/insights"; import moment from "moment"; +import { HexStringToInt64String } from "src/util"; export type InsightEventState = Omit & { insightName: string; @@ -30,10 +32,7 @@ export type InsightEventsResponse = InsightEventState[]; type InsightQuery = { name: InsightNameEnum; query: string; - toState: ( - response: SqlExecutionResponse, - results: Record, - ) => State[]; + toState: (response: SqlExecutionResponse) => State; }; // The only insight we currently report is "High Wait Time", which is the insight @@ -53,29 +52,28 @@ type TransactionContentionResponseColumns = { function transactionContentionResultsToEventState( response: SqlExecutionResponse, - results: Record, ): InsightEventState[] { - response.execution.txn_results[0].rows.forEach(row => { - const key = row.blocking_txn_id; - if (!results[key]) { - results[key] = { - executionID: row.blocking_txn_id, - queries: row.blocking_queries, - startTime: moment(row.collection_ts), - elapsedTime: moment.duration(row.contention_duration).asMilliseconds(), - application: row.app_name, - insightName: highWaitTimeQuery.name, - execType: InsightExecEnum.TRANSACTION, - }; - } - }); + if (!response.execution.txn_results[0].rows) { + // No data. + return []; + } - return Object.values(results); + return response.execution.txn_results[0].rows.map(row => ({ + transactionID: row.blocking_txn_id, + queries: row.blocking_queries, + startTime: moment(row.collection_ts), + elapsedTimeMillis: moment + .duration(row.contention_duration) + .asMilliseconds(), + application: row.app_name, + insightName: highWaitTimeQuery.name, + execType: InsightExecEnum.TRANSACTION, + })); } const highWaitTimeQuery: InsightQuery< TransactionContentionResponseColumns, - InsightEventState + InsightEventsResponse > = { name: InsightNameEnum.highWaitTime, query: `SELECT @@ -116,13 +114,7 @@ export function getInsightEventState(): Promise { }; return executeSql(request).then( result => { - if (!result.execution.txn_results[0].rows) { - // No data. - return []; - } - - const results: Record = {}; - return highWaitTimeQuery.toState(result, results); + return highWaitTimeQuery.toState(result); }, ); } @@ -154,40 +146,36 @@ type TransactionContentionDetailsResponseColumns = { function transactionContentionDetailsResultsToEventState( response: SqlExecutionResponse, - results: Record, ): InsightEventDetailsState[] { - response.execution.txn_results[0].rows.forEach(row => { - const key = row.blocking_txn_id; - if (!results[key]) { - results[key] = { - executionID: row.blocking_txn_id, - queries: row.blocking_queries, - startTime: moment(row.collection_ts), - elapsedTime: moment.duration(row.contention_duration).asMilliseconds(), - application: row.app_name, - fingerprintID: row.blocking_txn_fingerprint_id, - waitingExecutionID: row.waiting_txn_id, - waitingFingerprintID: row.waiting_txn_fingerprint_id, - waitingQueries: row.waiting_queries, - schemaName: row.schema_name, - databaseName: row.database_name, - tableName: row.table_name, - indexName: row.index_name, - contendedKey: row.key, - insightName: highWaitTimeQuery.name, - execType: InsightExecEnum.TRANSACTION, - }; - } - }); - - return Object.values(results); + if (!response.execution.txn_results[0].rows) { + // No data. + return []; + } + return response.execution.txn_results[0].rows.map(row => ({ + executionID: row.blocking_txn_id, + queries: row.blocking_queries, + startTime: moment(row.collection_ts), + elapsedTime: moment.duration(row.contention_duration).asMilliseconds(), + application: row.app_name, + fingerprintID: row.blocking_txn_fingerprint_id, + waitingExecutionID: row.waiting_txn_id, + waitingFingerprintID: row.waiting_txn_fingerprint_id, + waitingQueries: row.waiting_queries, + schemaName: row.schema_name, + databaseName: row.database_name, + tableName: row.table_name, + indexName: row.index_name, + contendedKey: row.key, + insightName: highWaitTimeQuery.name, + execType: InsightExecEnum.TRANSACTION, + })); } const highWaitTimeDetailsQuery = ( id: string, ): InsightQuery< TransactionContentionDetailsResponseColumns, - InsightEventDetailsState + InsightEventDetailsResponse > => { return { name: InsightNameEnum.highWaitTime, @@ -251,12 +239,120 @@ export function getInsightEventDetailsState( }; return executeSql(request).then( result => { - if (!result.execution.txn_results[0].rows) { - // No data. - return []; - } - const results: Record = {}; - return detailsQuery.toState(result, results); + return detailsQuery.toState(result); }, ); } + +type ExecutionInsightsResponseRow = { + session_id: string; + txn_id: string; + txn_fingerprint_id: string; // hex string + stmt_id: string; + stmt_fingerprint_id: string; // hex string + query: string; + start_time: string; // Timestamp + end_time: string; // Timestamp + full_scan: boolean; + user_name: string; + app_name: string; + database_name: string; + rows_read: number; + rows_written: number; + priority: string; + retries: number; + exec_node_ids: number[]; + contention: string; // interval + last_retry_reason?: string; + problems: string[]; +}; + +export type StatementInsights = Omit[]; + +function getStatementInsightsFromClusterExecutionInsightsResponse( + response: SqlExecutionResponse, +): StatementInsights { + if (!response.execution.txn_results[0].rows) { + // No data. + return []; + } + + return response.execution.txn_results[0].rows.map(row => { + const start = moment.utc(row.start_time); + const end = moment.utc(row.end_time); + return { + transactionID: row.txn_id, + transactionFingerprintID: HexStringToInt64String(row.txn_fingerprint_id), + queries: [row.query], + startTime: start, + endTime: end, + databaseName: row.database_name, + elapsedTimeMillis: start.diff(end, "milliseconds"), + application: row.app_name, + execType: InsightExecEnum.STATEMENT, + statementID: row.stmt_id, + statementFingerprintID: HexStringToInt64String(row.stmt_fingerprint_id), + sessionID: row.session_id, + isFullScan: row.full_scan, + rowsRead: row.rows_read, + rowsWritten: row.rows_written, + priority: row.priority, + retries: row.retries, + lastRetryReason: row.last_retry_reason, + timeSpentWaiting: row.contention ? moment.duration(row.contention) : null, + problems: row.problems, + }; + }); +} + +const statemenInsightsQuery: InsightQuery< + ExecutionInsightsResponseRow, + StatementInsights +> = { + name: InsightNameEnum.highWaitTime, + // We only surface the most recently observed problem for a given statement. + query: `SELECT * from ( + SELECT + session_id, + txn_id, + encode(txn_fingerprint_id, 'hex') AS txn_fingerprint_id, + stmt_id, + encode(stmt_fingerprint_id, 'hex') AS stmt_fingerprint_id, + query, + start_time, + end_time, + full_scan, + app_name, + database_name, + rows_read, + rows_written, + priority, + retries, + contention, + last_retry_reason, + problems, + row_number() OVER ( + PARTITION BY txn_fingerprint_id + ORDER BY end_time DESC + ) AS rank + FROM crdb_internal.cluster_execution_insights + WHERE array_length(problems, 1) > 0 + ) WHERE rank = 1 + `, + toState: getStatementInsightsFromClusterExecutionInsightsResponse, +}; + +export function getStatementInsightsApi(): Promise { + const request: SqlExecutionRequest = { + statements: [ + { + sql: `${statemenInsightsQuery.query}`, + }, + ], + execute: true, + max_result_size: 50000, // 50 kib + }; + return executeSql(request).then(result => { + return statemenInsightsQuery.toState(result); + }); +} diff --git a/pkg/ui/workspaces/cluster-ui/src/insights/types.ts b/pkg/ui/workspaces/cluster-ui/src/insights/types.ts index c44fbedc8eea..3c660155278a 100644 --- a/pkg/ui/workspaces/cluster-ui/src/insights/types.ts +++ b/pkg/ui/workspaces/cluster-ui/src/insights/types.ts @@ -22,11 +22,11 @@ export enum InsightExecEnum { } export type InsightEvent = { - executionID: string; + transactionID: string; queries: string[]; insights: Insight[]; startTime: Moment; - elapsedTime: number; + elapsedTimeMillis: number; application: string; execType: InsightExecEnum; }; @@ -50,6 +50,24 @@ export type InsightEventDetails = { execType: InsightExecEnum; }; +export type StatementInsightEvent = InsightEvent & { + // Some of these can probably be moved to InsightEvent type once txn query is updated. + statementID: string; + statementFingerprintID: string; + transactionFingerprintID: string; + sessionID: string; + timeSpentWaiting?: moment.Duration; + isFullScan: boolean; + endTime: Moment; + databaseName: string; + rowsRead: number; + rowsWritten: number; + lastRetryReason?: string; + priority: string; + retries: number; + problems: string[]; +}; + export type Insight = { name: InsightNameEnum; label: string; diff --git a/pkg/ui/workspaces/cluster-ui/src/insights/utils.ts b/pkg/ui/workspaces/cluster-ui/src/insights/utils.ts index e7c53d889ae6..0ca686c203e8 100644 --- a/pkg/ui/workspaces/cluster-ui/src/insights/utils.ts +++ b/pkg/ui/workspaces/cluster-ui/src/insights/utils.ts @@ -50,11 +50,11 @@ export function getInsightsFromState( return; } else { insightEvents.push({ - executionID: e.executionID, + transactionID: e.transactionID, queries: e.queries, insights: insightsForEvent, startTime: e.startTime, - elapsedTime: e.elapsedTime, + elapsedTimeMillis: e.elapsedTimeMillis, application: e.application, execType: InsightExecEnum.TRANSACTION, }); @@ -114,7 +114,7 @@ export const filterTransactionInsights = ( filteredTransactions = filteredTransactions.filter( txn => !search || - txn.executionID.toLowerCase()?.includes(search) || + txn.transactionID.toLowerCase()?.includes(search) || txn.queries?.find(query => query.toLowerCase().includes(search)), ); } diff --git a/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsights/transactionInsights/transactionInsights.fixture.ts b/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsights/transactionInsights/transactionInsights.fixture.ts index 8761c945c4e2..dc715dd5a8dd 100644 --- a/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsights/transactionInsights/transactionInsights.fixture.ts +++ b/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsights/transactionInsights/transactionInsights.fixture.ts @@ -15,36 +15,36 @@ import { InsightExecEnum } from "../../types"; export const transactionInsightsPropsFixture: TransactionInsightsViewProps = { transactions: [ { - executionID: "f72f37ea-b3a0-451f-80b8-dfb27d0bc2a5", + transactionID: "f72f37ea-b3a0-451f-80b8-dfb27d0bc2a5", queries: [ "SELECT IFNULL(a, b) FROM (SELECT (SELECT code FROM promo_codes WHERE code > $1 ORDER BY code LIMIT _) AS a, (SELECT code FROM promo_codes ORDER BY code LIMIT _) AS b)", ], insightName: "HIGH_WAIT_TIME", startTime: moment.utc("2022.08.10"), - elapsedTime: moment.duration("00:00:00.25").asMilliseconds(), + elapsedTimeMillis: moment.duration("00:00:00.25").asMilliseconds(), application: "demo", execType: InsightExecEnum.TRANSACTION, }, { - executionID: "e72f37ea-b3a0-451f-80b8-dfb27d0bc2a5", + transactionID: "e72f37ea-b3a0-451f-80b8-dfb27d0bc2a5", queries: [ "INSERT INTO vehicles VALUES ($1, $2, __more6__)", "INSERT INTO vehicles VALUES ($1, $2, __more6__)", ], insightName: "HIGH_WAIT_TIME", startTime: moment.utc("2022.08.10"), - elapsedTime: moment.duration("00:00:00.25").asMilliseconds(), + elapsedTimeMillis: moment.duration("00:00:00.25").asMilliseconds(), application: "demo", execType: InsightExecEnum.TRANSACTION, }, { - executionID: "f72f37ea-b3a0-451f-80b8-dfb27d0bc2a0", + transactionID: "f72f37ea-b3a0-451f-80b8-dfb27d0bc2a0", queries: [ "UPSERT INTO vehicle_location_histories VALUES ($1, $2, now(), $3, $4)", ], insightName: "HIGH_WAIT_TIME", startTime: moment.utc("2022.08.10"), - elapsedTime: moment.duration("00:00:00.25").asMilliseconds(), + elapsedTimeMillis: moment.duration("00:00:00.25").asMilliseconds(), application: "demo", execType: InsightExecEnum.TRANSACTION, }, diff --git a/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsights/transactionInsights/transactionInsightsTable.tsx b/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsights/transactionInsights/transactionInsightsTable.tsx index 347279e46a29..820fac5eac95 100644 --- a/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsights/transactionInsights/transactionInsightsTable.tsx +++ b/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsights/transactionInsights/transactionInsightsTable.tsx @@ -35,11 +35,11 @@ export function makeTransactionInsightsColumns(): ColumnDescriptor name: "executionID", title: insightsTableTitles.executionID(execType), cell: (item: InsightEvent) => ( - - {String(item.executionID)} + + {String(item.transactionID)} ), - sort: (item: InsightEvent) => item.executionID, + sort: (item: InsightEvent) => item.transactionID, }, { name: "query", @@ -67,8 +67,8 @@ export function makeTransactionInsightsColumns(): ColumnDescriptor { name: "elapsedTime", title: insightsTableTitles.elapsedTime(execType), - cell: (item: InsightEvent) => Duration(item.elapsedTime * 1e6), - sort: (item: InsightEvent) => item.elapsedTime, + cell: (item: InsightEvent) => Duration(item.elapsedTimeMillis * 1e6), + sort: (item: InsightEvent) => item.elapsedTimeMillis, }, { name: "applicationName", diff --git a/pkg/ui/workspaces/db-console/src/redux/apiReducers.ts b/pkg/ui/workspaces/db-console/src/redux/apiReducers.ts index 05df89112e1a..3b6771978376 100644 --- a/pkg/ui/workspaces/db-console/src/redux/apiReducers.ts +++ b/pkg/ui/workspaces/db-console/src/redux/apiReducers.ts @@ -403,6 +403,14 @@ const insightsReducerObj = new CachedDataReducer( ); export const refreshInsights = insightsReducerObj.refresh; +const statementInsightsReducerObj = new CachedDataReducer( + clusterUiApi.getStatementInsightsApi, + "statementInsights", + null, + moment.duration(30, "s"), // Timeout +); +export const refreshStatementInsights = statementInsightsReducerObj.refresh; + export const insightRequestKey = ( req: clusterUiApi.InsightEventDetailsRequest, ): string => `${req.id}`; @@ -451,6 +459,7 @@ export interface APIReducersState { clusterLocks: CachedDataReducerState; insights: CachedDataReducerState; insightDetails: KeyedCachedDataReducerState; + statementInsights: CachedDataReducerState; } export const apiReducersReducer = combineReducers({ @@ -494,6 +503,8 @@ export const apiReducersReducer = combineReducers({ [clusterLocksReducerObj.actionNamespace]: clusterLocksReducerObj.reducer, [insightsReducerObj.actionNamespace]: insightsReducerObj.reducer, [insightDetailsReducerObj.actionNamespace]: insightDetailsReducerObj.reducer, + [statementInsightsReducerObj.actionNamespace]: + statementInsightsReducerObj.reducer, }); export { CachedDataReducerState, KeyedCachedDataReducerState };