Skip to content

Commit

Permalink
[8.16] [Entity Analytics] API changes for right placement of deleting…
Browse files Browse the repository at this point in the history
… the old component template (#199734) (#200591)

# Backport

This will backport the following commits from `main` to `8.16`:
- [[Entity Analytics] API changes for right placement of deleting the
old component template
(#199734)](#199734)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Abhishek
Bhatia","email":"117628830+abhishekbhatia1710@users.noreply.github.com"},"sourceCommit":{"committedDate":"2024-11-18T15:07:13Z","message":"[Entity
Analytics] API changes for right placement of deleting the old component
template (#199734)\n\n## Summary\r\n\r\n- Delete the old component
template after the index template has\r\nreferenced to the new component
template\r\n- Test cases for the same flow\r\n\r\n\r\n```JSON\r\n# Let's
assume this is 8.15.3\r\n# Create the component template when Risk Score
engine is initialised\r\n# Create the index template which references
the created component template\r\n\r\nPUT
/_component_template/.risk-score-mappings\r\n{\r\n \"template\": {\r\n
\"settings\": {\r\n \"number_of_shards\": 1\r\n },\r\n \"mappings\":
{\r\n \"properties\": {\r\n \"timestamp\": {\r\n \"type\": \"date\"\r\n
},\r\n \"user\": {\r\n \"properties\": {\r\n \"id\": {\r\n \"type\":
\"keyword\"\r\n },\r\n \"name\": {\r\n \"type\": \"text\"\r\n }\r\n
}\r\n }\r\n }\r\n }\r\n },\r\n \"version\": 1\r\n}\r\n\r\nPUT
/_index_template/.risk-score.risk-score-default-index-template\r\n{\r\n
\"index_patterns\":
[\".risk-score.risk-score-default-index-template\"],\r\n \"template\":
{\r\n \"settings\": {\r\n \"number_of_replicas\": 1\r\n }\r\n },\r\n
\"composed_of\": [\".risk-score-mappings\"],\r\n \"priority\": 100,\r\n
\"version\": 1,\r\n \"_meta\": {\r\n \"description\": \"Index template
for indices with the pattern my_index-*\"\r\n }\r\n}\r\n\r\n# The
deployment is updated to 8.16\r\n# User tries to enable the Entity store
which init's the Risk Score engine (again!!) \r\n# Fails, but creates
the component template and cannot update the index template to reference
the new component template due to the error\r\n\r\nPUT
/_component_template/.risk-score-mappings-default\r\n{\r\n \"template\":
{\r\n \"settings\": {\r\n \"number_of_shards\": 1\r\n },\r\n
\"mappings\": {\r\n \"properties\": {\r\n \"timestamp\": {\r\n \"type\":
\"date\"\r\n },\r\n \"user\": {\r\n \"properties\": {\r\n \"id\": {\r\n
\"type\": \"keyword\"\r\n },\r\n \"name\": {\r\n \"type\": \"text\"\r\n
}\r\n }\r\n }\r\n }\r\n }\r\n },\r\n \"version\": 1\r\n}\r\n\r\nGET
/_component_template?filter_path=component_templates.name&name=.risk-score-mappings*\r\n\r\n\r\nDELETE
/_component_template/.risk-score-mappings\r\n\r\n# Fails \r\n\r\n#
changed flow\r\nPUT
/_index_template/.risk-score.risk-score-default-index-template\r\n{\r\n
\"index_patterns\":
[\".risk-score.risk-score-default-index-template\"],\r\n \"template\":
{\r\n \"settings\": {\r\n \"number_of_replicas\": 1\r\n }\r\n },\r\n
\"composed_of\": [\".risk-score-mappings-default\"],\r\n \"priority\":
100,\r\n \"version\": 1,\r\n \"_meta\": {\r\n \"description\": \"Index
template for indices with the pattern my_index-*\"\r\n
}\r\n}\r\n\r\nDELETE /_component_template/.risk-score-mappings\r\n\r\n#
Succeeds\r\n\r\n#########\r\n```\r\n\r\n\r\n### Checklist\r\n\r\nDelete
any items that are not applicable to this PR.\r\n\r\n- [ ] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common scenarios\r\n- [ ] [Flaky
Test\r\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\r\nused on any tests changed\r\n\r\n\r\n\r\n### For
maintainers\r\n\r\n- [ ] This was checked for breaking API changes and
was
[labeled\r\nappropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#_add_your_labels)\r\n-
[ ] This will appear in the **Release Notes** and follow
the\r\n[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)","sha":"771c139269ac0eac27274e0858be2fbcf87b31e3","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["bug","release_note:fix","v9.0.0","Team:Entity
Analytics","backport:version","v8.17.0","v8.16.1"],"number":199734,"url":"https://github.com/elastic/kibana/pull/199734","mergeCommit":{"message":"[Entity
Analytics] API changes for right placement of deleting the old component
template (#199734)\n\n## Summary\r\n\r\n- Delete the old component
template after the index template has\r\nreferenced to the new component
template\r\n- Test cases for the same flow\r\n\r\n\r\n```JSON\r\n# Let's
assume this is 8.15.3\r\n# Create the component template when Risk Score
engine is initialised\r\n# Create the index template which references
the created component template\r\n\r\nPUT
/_component_template/.risk-score-mappings\r\n{\r\n \"template\": {\r\n
\"settings\": {\r\n \"number_of_shards\": 1\r\n },\r\n \"mappings\":
{\r\n \"properties\": {\r\n \"timestamp\": {\r\n \"type\": \"date\"\r\n
},\r\n \"user\": {\r\n \"properties\": {\r\n \"id\": {\r\n \"type\":
\"keyword\"\r\n },\r\n \"name\": {\r\n \"type\": \"text\"\r\n }\r\n
}\r\n }\r\n }\r\n }\r\n },\r\n \"version\": 1\r\n}\r\n\r\nPUT
/_index_template/.risk-score.risk-score-default-index-template\r\n{\r\n
\"index_patterns\":
[\".risk-score.risk-score-default-index-template\"],\r\n \"template\":
{\r\n \"settings\": {\r\n \"number_of_replicas\": 1\r\n }\r\n },\r\n
\"composed_of\": [\".risk-score-mappings\"],\r\n \"priority\": 100,\r\n
\"version\": 1,\r\n \"_meta\": {\r\n \"description\": \"Index template
for indices with the pattern my_index-*\"\r\n }\r\n}\r\n\r\n# The
deployment is updated to 8.16\r\n# User tries to enable the Entity store
which init's the Risk Score engine (again!!) \r\n# Fails, but creates
the component template and cannot update the index template to reference
the new component template due to the error\r\n\r\nPUT
/_component_template/.risk-score-mappings-default\r\n{\r\n \"template\":
{\r\n \"settings\": {\r\n \"number_of_shards\": 1\r\n },\r\n
\"mappings\": {\r\n \"properties\": {\r\n \"timestamp\": {\r\n \"type\":
\"date\"\r\n },\r\n \"user\": {\r\n \"properties\": {\r\n \"id\": {\r\n
\"type\": \"keyword\"\r\n },\r\n \"name\": {\r\n \"type\": \"text\"\r\n
}\r\n }\r\n }\r\n }\r\n }\r\n },\r\n \"version\": 1\r\n}\r\n\r\nGET
/_component_template?filter_path=component_templates.name&name=.risk-score-mappings*\r\n\r\n\r\nDELETE
/_component_template/.risk-score-mappings\r\n\r\n# Fails \r\n\r\n#
changed flow\r\nPUT
/_index_template/.risk-score.risk-score-default-index-template\r\n{\r\n
\"index_patterns\":
[\".risk-score.risk-score-default-index-template\"],\r\n \"template\":
{\r\n \"settings\": {\r\n \"number_of_replicas\": 1\r\n }\r\n },\r\n
\"composed_of\": [\".risk-score-mappings-default\"],\r\n \"priority\":
100,\r\n \"version\": 1,\r\n \"_meta\": {\r\n \"description\": \"Index
template for indices with the pattern my_index-*\"\r\n
}\r\n}\r\n\r\nDELETE /_component_template/.risk-score-mappings\r\n\r\n#
Succeeds\r\n\r\n#########\r\n```\r\n\r\n\r\n### Checklist\r\n\r\nDelete
any items that are not applicable to this PR.\r\n\r\n- [ ] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common scenarios\r\n- [ ] [Flaky
Test\r\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\r\nused on any tests changed\r\n\r\n\r\n\r\n### For
maintainers\r\n\r\n- [ ] This was checked for breaking API changes and
was
[labeled\r\nappropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#_add_your_labels)\r\n-
[ ] This will appear in the **Release Notes** and follow
the\r\n[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)","sha":"771c139269ac0eac27274e0858be2fbcf87b31e3"}},"sourceBranch":"main","suggestedTargetBranches":["8.x","8.16"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","labelRegex":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/199734","number":199734,"mergeCommit":{"message":"[Entity
Analytics] API changes for right placement of deleting the old component
template (#199734)\n\n## Summary\r\n\r\n- Delete the old component
template after the index template has\r\nreferenced to the new component
template\r\n- Test cases for the same flow\r\n\r\n\r\n```JSON\r\n# Let's
assume this is 8.15.3\r\n# Create the component template when Risk Score
engine is initialised\r\n# Create the index template which references
the created component template\r\n\r\nPUT
/_component_template/.risk-score-mappings\r\n{\r\n \"template\": {\r\n
\"settings\": {\r\n \"number_of_shards\": 1\r\n },\r\n \"mappings\":
{\r\n \"properties\": {\r\n \"timestamp\": {\r\n \"type\": \"date\"\r\n
},\r\n \"user\": {\r\n \"properties\": {\r\n \"id\": {\r\n \"type\":
\"keyword\"\r\n },\r\n \"name\": {\r\n \"type\": \"text\"\r\n }\r\n
}\r\n }\r\n }\r\n }\r\n },\r\n \"version\": 1\r\n}\r\n\r\nPUT
/_index_template/.risk-score.risk-score-default-index-template\r\n{\r\n
\"index_patterns\":
[\".risk-score.risk-score-default-index-template\"],\r\n \"template\":
{\r\n \"settings\": {\r\n \"number_of_replicas\": 1\r\n }\r\n },\r\n
\"composed_of\": [\".risk-score-mappings\"],\r\n \"priority\": 100,\r\n
\"version\": 1,\r\n \"_meta\": {\r\n \"description\": \"Index template
for indices with the pattern my_index-*\"\r\n }\r\n}\r\n\r\n# The
deployment is updated to 8.16\r\n# User tries to enable the Entity store
which init's the Risk Score engine (again!!) \r\n# Fails, but creates
the component template and cannot update the index template to reference
the new component template due to the error\r\n\r\nPUT
/_component_template/.risk-score-mappings-default\r\n{\r\n \"template\":
{\r\n \"settings\": {\r\n \"number_of_shards\": 1\r\n },\r\n
\"mappings\": {\r\n \"properties\": {\r\n \"timestamp\": {\r\n \"type\":
\"date\"\r\n },\r\n \"user\": {\r\n \"properties\": {\r\n \"id\": {\r\n
\"type\": \"keyword\"\r\n },\r\n \"name\": {\r\n \"type\": \"text\"\r\n
}\r\n }\r\n }\r\n }\r\n }\r\n },\r\n \"version\": 1\r\n}\r\n\r\nGET
/_component_template?filter_path=component_templates.name&name=.risk-score-mappings*\r\n\r\n\r\nDELETE
/_component_template/.risk-score-mappings\r\n\r\n# Fails \r\n\r\n#
changed flow\r\nPUT
/_index_template/.risk-score.risk-score-default-index-template\r\n{\r\n
\"index_patterns\":
[\".risk-score.risk-score-default-index-template\"],\r\n \"template\":
{\r\n \"settings\": {\r\n \"number_of_replicas\": 1\r\n }\r\n },\r\n
\"composed_of\": [\".risk-score-mappings-default\"],\r\n \"priority\":
100,\r\n \"version\": 1,\r\n \"_meta\": {\r\n \"description\": \"Index
template for indices with the pattern my_index-*\"\r\n
}\r\n}\r\n\r\nDELETE /_component_template/.risk-score-mappings\r\n\r\n#
Succeeds\r\n\r\n#########\r\n```\r\n\r\n\r\n### Checklist\r\n\r\nDelete
any items that are not applicable to this PR.\r\n\r\n- [ ] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common scenarios\r\n- [ ] [Flaky
Test\r\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\r\nused on any tests changed\r\n\r\n\r\n\r\n### For
maintainers\r\n\r\n- [ ] This was checked for breaking API changes and
was
[labeled\r\nappropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#_add_your_labels)\r\n-
[ ] This will appear in the **Release Notes** and follow
the\r\n[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)","sha":"771c139269ac0eac27274e0858be2fbcf87b31e3"}},{"branch":"8.x","label":"v8.17.0","labelRegex":"^v8.17.0$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"8.16","label":"v8.16.1","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->
  • Loading branch information
abhishekbhatia1710 authored Nov 18, 2024
1 parent e6faaa2 commit 0699913
Show file tree
Hide file tree
Showing 2 changed files with 106 additions and 22 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -120,30 +120,10 @@ export class RiskScoreDataClient {
const oldComponentTemplateExists = await esClient.cluster.existsComponentTemplate({
name: mappingComponentName,
});
// If present then copy the contents to a new component template with the namespace in the name
if (oldComponentTemplateExists) {
const oldComponentTemplateResponse = await esClient.cluster.getComponentTemplate(
{
name: mappingComponentName,
},
{ ignore: [404] }
);
const oldComponentTemplate = oldComponentTemplateResponse?.component_templates[0];
const newComponentTemplateName = nameSpaceAwareMappingsComponentName(namespace);
await esClient.cluster.putComponentTemplate({
name: newComponentTemplateName,
body: oldComponentTemplate.component_template,
});
await this.updateComponentTemplateNamewithNamespace(namespace);
}

// Delete the component template without the namespace in the name
await esClient.cluster.deleteComponentTemplate(
{
name: mappingComponentName,
},
{ ignore: [404] }
);

// Update the new component template with the required data
await Promise.all([
createOrUpdateComponentTemplate({
Expand Down Expand Up @@ -188,6 +168,14 @@ export class RiskScoreDataClient {
},
});

// Delete the component template without the namespace in the name
await esClient.cluster.deleteComponentTemplate(
{
name: mappingComponentName,
},
{ ignore: [404] }
);

await createDataStream({
logger: this.options.logger,
esClient,
Expand Down Expand Up @@ -327,4 +315,20 @@ export class RiskScoreDataClient {
{ logger: this.options.logger }
);
}

private async updateComponentTemplateNamewithNamespace(namespace: string): Promise<void> {
const esClient = this.options.esClient;
const oldComponentTemplateResponse = await esClient.cluster.getComponentTemplate(
{
name: mappingComponentName,
},
{ ignore: [404] }
);
const oldComponentTemplate = oldComponentTemplateResponse?.component_templates[0];
const newComponentTemplateName = nameSpaceAwareMappingsComponentName(namespace);
await esClient.cluster.putComponentTemplate({
name: newComponentTemplateName,
body: oldComponentTemplate.component_template,
});
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -540,7 +540,87 @@ export default ({ getService }: FtrProviderContext) => {
);
});

describe('remove legacy risk score transform', function () {
it('should update the existing component template and index template without any errors', async () => {
const componentTemplateName = '.risk-score-mappings';
const indexTemplateName = '.risk-score.risk-score-default-index-template';
const newComponentTemplateName = '.risk-score-mappings-default';

// Call API to put the component template and index template

await es.cluster.putComponentTemplate({
name: componentTemplateName,
body: {
template: {
settings: {
number_of_shards: 1,
},
mappings: {
properties: {
timestamp: {
type: 'date',
},
user: {
properties: {
id: {
type: 'keyword',
},
name: {
type: 'text',
},
},
},
},
},
},
version: 1,
},
});

// Call an API to put the index template

await es.indices.putIndexTemplate({
name: indexTemplateName,
body: {
index_patterns: [indexTemplateName],
composed_of: [componentTemplateName],
template: {
settings: {
number_of_shards: 1,
},
mappings: {
properties: {
timestamp: {
type: 'date',
},
user: {
properties: {
id: {
type: 'keyword',
},
name: {
type: 'text',
},
},
},
},
},
},
},
});

const response = await riskEngineRoutes.init();
expect(response.status).to.eql(200);
expect(response.body.result.errors).to.eql([]);

const response2 = await es.cluster.getComponentTemplate({
name: newComponentTemplateName,
});
expect(response2.component_templates.length).to.eql(1);
expect(response2.component_templates[0].name).to.eql(newComponentTemplateName);
});

// Failing: See https://github.com/elastic/kibana/issues/191637
describe.skip('remove legacy risk score transform', function () {
this.tags('skipFIPS');
it('should remove legacy risk score transform if it exists', async () => {
await installLegacyRiskScore({ supertest });
Expand Down

0 comments on commit 0699913

Please sign in to comment.