Skip to content

Commit

Permalink
Modify the import for timeline visualization to includes data source …
Browse files Browse the repository at this point in the history 
…name in MDS scenario (opensearch-project#6954)

Signed-off-by: Yuanqi(Ella) Zhu <zhyuanqi@amazon.com>
  • Loading branch information
@zhyuanqi
zhyuanqi authored Jun 7, 2024
1 parent 75e6087 commit 7eaab64
Show file tree
Hide file tree
Showing 5 changed files with 263 additions and 4 deletions.
114 changes: 114 additions & 0 deletions src/core/server/saved_objects/import/check_conflict_for_data_source.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,120 @@ describe('#checkConflictsForDataSource', () => {
);
});

/*
* Timeline test cases
*/
it('will not change timeline expression when importing from datasource to different datasource', async () => {
const timelineSavedObject = createObject('visualization', 'old-datasource-id_some-object-id');
// @ts-expect-error
timelineSavedObject.attributes.visState =
'{"title":"(Timeline) Avg bytes over time","type":"timelion","aggs":[],"params":{"expression":".opensearch(opensearch_dashboards_sample_data_logs, metric=avg:bytes, timefield=@timestamp, data_source_name=newDataSource).lines(show=true).points(show=true).yaxis(label=\\"Average bytes\\")","interval":"auto"}}';
const params = setupParams({
objects: [timelineSavedObject],
ignoreRegularConflicts: true,
dataSourceId: 'some-datasource-id',
savedObjectsClient: getSavedObjectClient(),
});
const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);

expect(checkConflictsForDataSourceResult).toEqual(
expect.objectContaining({
filteredObjects: [
{
...timelineSavedObject,
attributes: {
title: 'some-title',
visState:
'{"title":"(Timeline) Avg bytes over time","type":"timelion","aggs":[],"params":{"expression":".opensearch(opensearch_dashboards_sample_data_logs, metric=avg:bytes, timefield=@timestamp, data_source_name=newDataSource).lines(show=true).points(show=true).yaxis(label=\\"Average bytes\\")","interval":"auto"}}',
},
id: 'some-datasource-id_some-object-id',
},
],
errors: [],
importIdMap: new Map([
[
`visualization:old-datasource-id_some-object-id`,
{ id: 'some-datasource-id_some-object-id', omitOriginId: true },
],
]),
})
);
});

it('will change timeline expression when importing expression does not have a datasource name', async () => {
const timelineSavedObject = createObject('visualization', 'old-datasource-id_some-object-id');
// @ts-expect-error
timelineSavedObject.attributes.visState =
'{"title":"(Timeline) Avg bytes over time","type":"timelion","aggs":[],"params":{"expression":".opensearch(opensearch_dashboards_sample_data_logs, metric=avg:bytes, timefield=@timestamp).lines(show=true).points(show=true).yaxis(label=\\"Average bytes\\")","interval":"auto"}}';
const params = setupParams({
objects: [timelineSavedObject],
ignoreRegularConflicts: true,
dataSourceId: 'some-datasource-id',
savedObjectsClient: getSavedObjectClient(),
});
const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);

expect(checkConflictsForDataSourceResult).toEqual(
expect.objectContaining({
filteredObjects: [
{
...timelineSavedObject,
attributes: {
title: 'some-title',
visState:
'{"title":"(Timeline) Avg bytes over time","type":"timelion","aggs":[],"params":{"expression":".opensearch(opensearch_dashboards_sample_data_logs, metric=avg:bytes, timefield=@timestamp, data_source_name=\\"some-datasource-title\\").lines(show=true).points(show=true).yaxis(label=\\"Average bytes\\")","interval":"auto"}}',
},
id: 'some-datasource-id_some-object-id',
},
],
errors: [],
importIdMap: new Map([
[
`visualization:old-datasource-id_some-object-id`,
{ id: 'some-datasource-id_some-object-id', omitOriginId: true },
],
]),
})
);
});

it('When there are multiple opensearch queries in the expression, it would go through each query and add data source name if it does not have any.', async () => {
const timelineSavedObject = createObject('visualization', 'old-datasource-id_some-object-id');
// @ts-expect-error
timelineSavedObject.attributes.visState =
'{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"aos 211\\"), .elasticsearch(index=old-datasource-title, timefield=@timestamp)"},"aggs":[]}';
const params = setupParams({
objects: [timelineSavedObject],
ignoreRegularConflicts: true,
dataSourceId: 'some-datasource-id',
savedObjectsClient: getSavedObjectClient(),
});
const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);

expect(checkConflictsForDataSourceResult).toEqual(
expect.objectContaining({
filteredObjects: [
{
...timelineSavedObject,
attributes: {
title: 'some-title',
visState:
'{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"aos 211\\"), .elasticsearch(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"some-datasource-title\\")"},"aggs":[]}',
},
id: 'some-datasource-id_some-object-id',
},
],
errors: [],
importIdMap: new Map([
[
`visualization:old-datasource-id_some-object-id`,
{ id: 'some-datasource-id_some-object-id', omitOriginId: true },
],
]),
})
);
});

/**
* TSVB test cases
*/
Expand Down
18 changes: 18 additions & 0 deletions src/core/server/saved_objects/import/check_conflict_for_data_source.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ import {
getDataSourceTitleFromId,
getUpdatedTSVBVisState,
updateDataSourceNameInVegaSpec,
extractTimelineExpression,
updateDataSourceNameInTimeline,
} from './utils';

export interface ConflictsForDataSourceParams {
Expand Down Expand Up @@ -120,6 +122,22 @@ export async function checkConflictsForDataSource({
}
}

// For timeline visualizations, update the data source name in the timeline expression
const timelineExpression = extractTimelineExpression(object);
if (!!timelineExpression && !!dataSourceTitle) {
// Get the timeline expression with the updated data source name
const modifiedExpression = updateDataSourceNameInTimeline(
timelineExpression,
dataSourceTitle
);

// @ts-expect-error
const timelineStateObject = JSON.parse(object.attributes?.visState);
timelineStateObject.params.expression = modifiedExpression;
// @ts-expect-error
object.attributes.visState = JSON.stringify(timelineStateObject);
}

if (!!dataSourceId) {
const visualizationObject = object as VisualizationObject;
const { visState, references } = getUpdatedTSVBVisState(
Expand Down
114 changes: 111 additions & 3 deletions src/core/server/saved_objects/import/create_saved_objects.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,39 @@ const getVegaMDSVisualizationObj = (id: string, dataSourceId: string) => ({
},
],
});

const getTimelineVisualizationObj = (id: string, dataSourceId: string) => ({
type: 'visualization',
id: dataSourceId ? `${dataSourceId}_${id}` : id,
attributes: {
title: 'some-other-title',
visState:
'{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp)"},"aggs":[]}',
},
references: [],
});

const getTimelineVisualizationObjWithMultipleQueries = (id: string, dataSourceId: string) => ({
type: 'visualization',
id: dataSourceId ? `${dataSourceId}_${id}` : id,
attributes: {
title: 'some-other-title',
visState:
'{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"aos 211\\"), .elasticsearch(index=old-datasource-title, timefield=@timestamp)"},"aggs":[]}',
},
references: [],
});

const getTimelineVisualizationObjWithDataSourceName = (id: string, dataSourceId: string) => ({
type: 'visualization',
id: dataSourceId ? `${dataSourceId}_${id}` : id,
attributes: {
title: 'some-other-title',
visState:
'{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=ds1)"},"aggs":[]}',
},
references: [],
});
// non-multi-namespace types shouldn't have origin IDs, but we include test cases to ensure it's handled gracefully
// non-multi-namespace types by definition cannot result in an unresolvable conflict, so we don't include test cases for those
const importId3 = 'id-foo';
Expand Down Expand Up @@ -571,7 +604,7 @@ describe('#createSavedObjects', () => {
expect(results).toEqual(expectedResultsWithDataSource);
};

const testVegaVisualizationsWithDataSources = async (params: {
const testVegaTimelineVisualizationsWithDataSources = async (params: {
objects: SavedObject[];
expectedFilteredObjects: Array<Record<string, unknown>>;
dataSourceId?: string;
Expand Down Expand Up @@ -673,7 +706,7 @@ describe('#createSavedObjects', () => {
],
},
];
await testVegaVisualizationsWithDataSources({
await testVegaTimelineVisualizationsWithDataSources({
objects,
expectedFilteredObjects,
dataSourceId: 'some-datasource-id',
Expand All @@ -699,7 +732,82 @@ describe('#createSavedObjects', () => {
},
},
];
await testVegaVisualizationsWithDataSources({
await testVegaTimelineVisualizationsWithDataSources({
objects,
expectedFilteredObjects,
dataSourceId: 'some-datasource-id',
dataSourceTitle: 'dataSourceName',
});
});
});

describe('with a data source for timeline saved objects', () => {
test('can attach a data source name to the timeline expression', async () => {
const objects = [getTimelineVisualizationObj('some-timeline-id', 'some-datasource-id')];
const expectedObject = getTimelineVisualizationObj('some-timeline-id', 'some-datasource-id');
const expectedFilteredObjects = [
{
...expectedObject,
attributes: {
title: 'some-other-title_dataSourceName',
visState:
'{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"dataSourceName\\")"},"aggs":[]}',
},
},
];
await testVegaTimelineVisualizationsWithDataSources({
objects,
expectedFilteredObjects,
dataSourceId: 'some-datasource-id',
dataSourceTitle: 'dataSourceName',
});
});

test('will not update the data source name in the timeline expression if no local cluster queries', async () => {
const objects = [
getTimelineVisualizationObjWithDataSourceName('some-timeline-id', 'old-datasource-id'),
];
const expectedObject = getTimelineVisualizationObjWithDataSourceName(
'some-timeline-id',
'old-datasource-id'
);
const expectedFilteredObjects = [
{
...expectedObject,
attributes: {
title: 'some-other-title_dataSourceName',
visState:
'{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=ds1)"},"aggs":[]}',
},
},
];
await testVegaTimelineVisualizationsWithDataSources({
objects,
expectedFilteredObjects,
dataSourceId: 'some-datasource-id',
dataSourceTitle: 'dataSourceName',
});
});

test('When muliple opensearch query exists in expression, we can add data source name to the queries that missing data source name.', async () => {
const objects = [
getTimelineVisualizationObjWithMultipleQueries('some-timeline-id', 'some-datasource-id'),
];
const expectedObject = getTimelineVisualizationObjWithMultipleQueries(
'some-timeline-id',
'some-datasource-id'
);
const expectedFilteredObjects = [
{
...expectedObject,
attributes: {
title: 'some-other-title_dataSourceName',
visState:
'{"title":"some-other-title","type":"timelion","params":{"expression":".es(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"aos 211\\"), .elasticsearch(index=old-datasource-title, timefield=@timestamp, data_source_name=\\"dataSourceName\\")"},"aggs":[]}',
},
},
];
await testVegaTimelineVisualizationsWithDataSources({
objects,
expectedFilteredObjects,
dataSourceId: 'some-datasource-id',
Expand Down
18 changes: 18 additions & 0 deletions src/core/server/saved_objects/import/create_saved_objects.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ import {
extractVegaSpecFromSavedObject,
getUpdatedTSVBVisState,
updateDataSourceNameInVegaSpec,
extractTimelineExpression,
updateDataSourceNameInTimeline,
} from './utils';

interface CreateSavedObjectsParams<T> {
Expand Down Expand Up @@ -130,6 +132,22 @@ export const createSavedObjects = async <T>({
});
}

// Some visualization types will need special modifications, like TSVB visualizations
const timelineExpression = extractTimelineExpression(object);
if (!!timelineExpression && !!dataSourceTitle) {
// Get the timeline expression with the updated data source name
const modifiedExpression = updateDataSourceNameInTimeline(
timelineExpression,
dataSourceTitle
);

// @ts-expect-error
const timelineStateObject = JSON.parse(object.attributes?.visState);
timelineStateObject.params.expression = modifiedExpression;
// @ts-expect-error
object.attributes.visState = JSON.stringify(timelineStateObject);
}

const visualizationObject = object as VisualizationObject;
const { visState, references } = getUpdatedTSVBVisState(
visualizationObject,
Expand Down
3 changes: 2 additions & 1 deletion src/core/server/saved_objects/import/import_saved_objects.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,8 @@ export async function importSavedObjectsFromStream({
supportedTypes,
dataSourceId,
});
// if not enable data_source, throw error early
// if dataSource is not enabled, but object type is data-source, or saved object id contains datasource id
// return unsupported type error
if (!dataSourceEnabled) {
const notSupportedErrors: SavedObjectsImportError[] = collectSavedObjectsResult.collectedObjects.reduce(
(errors: SavedObjectsImportError[], obj) => {
Expand Down

0 comments on commit 7eaab64

Please sign in to comment.