[Fleet] Add default http|https port to ES hosts (#99240)

x-pack/plugins/fleet/server/services/hosts_utils.test.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { normalizeHostsForAgents } from './hosts_utils';
+
+describe('normalizeHostsForAgents', () => {
+  const scenarios = [
+    { sourceUrl: 'http://test.fr', expectedUrl: 'http://test.fr:80' },
+    { sourceUrl: 'http://test.fr/test/toto', expectedUrl: 'http://test.fr:80/test/toto' },
+    { sourceUrl: 'https://test.fr', expectedUrl: 'https://test.fr:443' },
+    { sourceUrl: 'https://test.fr/test/toto', expectedUrl: 'https://test.fr:443/test/toto' },
+    { sourceUrl: 'https://test.fr:9243', expectedUrl: 'https://test.fr:9243' },
+    { sourceUrl: 'https://test.fr:9243/test/toto', expectedUrl: 'https://test.fr:9243/test/toto' },
+  ];
+
+  for (const scenario of scenarios) {
+    it(`should transform ${scenario.sourceUrl} correctly`, () => {
+      const url = normalizeHostsForAgents(scenario.sourceUrl);
+
+      expect(url).toEqual(scenario.expectedUrl);
+    });
+  }
+});

x-pack/plugins/fleet/server/services/hosts_utils.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+function getPortForURL(url: URL) {
+  if (url.port !== '') {
+    return url.port;
+  }
+
+  if (url.protocol === 'http:') {
+    return '80';
+  }
+
+  if (url.protocol === 'https:') {
+    return '443';
+  }
+}
+
+export function normalizeHostsForAgents(host: string) {
+  // Elastic Agent is not using default port for http|https for Fleet server and ES https://github.com/elastic/beats/issues/25420
+  const hostURL = new URL(host);
+
+  // We are building the URL manualy as url format will not include the port if the port is 80 or 443
+  return `${hostURL.protocol}//${hostURL.hostname}:${getPortForURL(hostURL)}${
+    hostURL.pathname === '/' ? '' : hostURL.pathname
+  }`;
+}

x-pack/plugins/fleet/server/services/output.ts

@@ -12,6 +12,7 @@ import { DEFAULT_OUTPUT, OUTPUT_SAVED_OBJECT_TYPE } from '../constants';
 import { decodeCloudId } from '../../common';
 
 import { appContextService } from './app_context';
+import { normalizeHostsForAgents } from './hosts_utils';
 
 const SAVED_OBJECT_TYPE = OUTPUT_SAVED_OBJECT_TYPE;
 
@@ -49,14 +50,6 @@ class OutputService {
     };
   }
 
-  public async updateOutput(
-    soClient: SavedObjectsClientContract,
-    id: string,
-    data: Partial<NewOutput>
-  ) {
-    await soClient.update<OutputSOAttributes>(SAVED_OBJECT_TYPE, id, data);
-  }
-
   public async getDefaultOutputId(soClient: SavedObjectsClientContract) {
     const outputs = await this.getDefaultOutput(soClient);
 
@@ -72,9 +65,15 @@ class OutputService {
     output: NewOutput,
     options?: { id?: string }
   ): Promise<Output> {
+    const data = { ...output };
+
+    if (data.hosts) {
+      data.hosts = data.hosts.map(normalizeHostsForAgents);
+    }
+
     const newSo = await soClient.create<OutputSOAttributes>(
       SAVED_OBJECT_TYPE,
-      output as Output,
+      data as Output,
       options
     );
 
@@ -98,7 +97,13 @@ class OutputService {
   }
 
   public async update(soClient: SavedObjectsClientContract, id: string, data: Partial<Output>) {
-    const outputSO = await soClient.update<OutputSOAttributes>(SAVED_OBJECT_TYPE, id, data);
+    const updateData = { ...data };
+
+    if (updateData.hosts) {
+      updateData.hosts = updateData.hosts.map(normalizeHostsForAgents);
+    }
+
+    const outputSO = await soClient.update<OutputSOAttributes>(SAVED_OBJECT_TYPE, id, updateData);
 
     if (outputSO.error) {
       throw new Error(outputSO.error.message);

x-pack/plugins/fleet/server/services/settings.test.ts

@@ -8,7 +8,7 @@
 import { savedObjectsClientMock } from 'src/core/server/mocks';
 
 import { appContextService } from './app_context';
-import { getCloudFleetServersHosts, normalizeFleetServerHost, settingsSetup } from './settings';
+import { getCloudFleetServersHosts, settingsSetup } from './settings';
 
 jest.mock('./app_context');
 
@@ -205,22 +205,3 @@ describe('settingsSetup', () => {
     expect(soClientMock.update).not.toBeCalled();
   });
 });
-
-describe('normalizeFleetServerHost', () => {
-  const scenarios = [
-    { sourceUrl: 'http://test.fr', expectedUrl: 'http://test.fr:80' },
-    { sourceUrl: 'http://test.fr/test/toto', expectedUrl: 'http://test.fr:80/test/toto' },
-    { sourceUrl: 'https://test.fr', expectedUrl: 'https://test.fr:443' },
-    { sourceUrl: 'https://test.fr/test/toto', expectedUrl: 'https://test.fr:443/test/toto' },
-    { sourceUrl: 'https://test.fr:9243', expectedUrl: 'https://test.fr:9243' },
-    { sourceUrl: 'https://test.fr:9243/test/toto', expectedUrl: 'https://test.fr:9243/test/toto' },
-  ];
-
-  for (const scenario of scenarios) {
-    it(`should transform ${scenario.sourceUrl} correctly`, () => {
-      const url = normalizeFleetServerHost(scenario.sourceUrl);
-
-      expect(url).toEqual(scenario.expectedUrl);
-    });
-  }
-});

x-pack/plugins/fleet/server/services/settings.ts

@@ -12,6 +12,7 @@ import { decodeCloudId, GLOBAL_SETTINGS_SAVED_OBJECT_TYPE } from '../../common';
 import type { SettingsSOAttributes, Settings, BaseSettings } from '../../common';
 
 import { appContextService } from './app_context';
+import { normalizeHostsForAgents } from './hosts_utils';
 
 export async function getSettings(soClient: SavedObjectsClientContract): Promise<Settings> {
   const res = await soClient.find<SettingsSOAttributes>({
@@ -51,37 +52,13 @@ export async function settingsSetup(soClient: SavedObjectsClientContract) {
   }
 }
 
-function getPortForURL(url: URL) {
-  if (url.port !== '') {
-    return url.port;
-  }
-
-  if (url.protocol === 'http:') {
-    return '80';
-  }
-
-  if (url.protocol === 'https:') {
-    return '443';
-  }
-}
-
-export function normalizeFleetServerHost(host: string) {
-  // Fleet server is not using default port for http|https https://github.com/elastic/beats/issues/25420
-  const fleetServerURL = new URL(host);
-
-  // We are building the URL manualy as url format will not include the port if the port is 80 or 443
-  return `${fleetServerURL.protocol}//${fleetServerURL.hostname}:${getPortForURL(fleetServerURL)}${
-    fleetServerURL.pathname === '/' ? '' : fleetServerURL.pathname
-  }`;
-}
-
 export async function saveSettings(
   soClient: SavedObjectsClientContract,
   newData: Partial<Omit<Settings, 'id'>>
 ): Promise<Partial<Settings> & Pick<Settings, 'id'>> {
   const data = { ...newData };
   if (data.fleet_server_hosts) {
-    data.fleet_server_hosts = data.fleet_server_hosts.map(normalizeFleetServerHost);
+    data.fleet_server_hosts = data.fleet_server_hosts.map(normalizeHostsForAgents);
   }
 
   try {

x-pack/test/fleet_api_integration/apis/index.js

@@ -46,5 +46,8 @@ export default function ({ loadTestFile }) {
 
     // Service tokens
     loadTestFile(require.resolve('./service_tokens'));
+
+    // Outputs
+    loadTestFile(require.resolve('./outputs'));
   });
 }

x-pack/test/fleet_api_integration/apis/outputs/crud.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
+import { skipIfNoDockerRegistry } from '../../helpers';
+import { setupFleetAndAgents } from '../agents/services';
+
+export default function (providerContext: FtrProviderContext) {
+  const { getService } = providerContext;
+  const supertest = getService('supertest');
+  const esArchiver = getService('esArchiver');
+
+  describe('fleet_output_crud', async function () {
+    skipIfNoDockerRegistry(providerContext);
+    before(async () => {
+      await esArchiver.load('fleet/empty_fleet_server');
+    });
+    setupFleetAndAgents(providerContext);
+
+    let defaultOutputId: string;
+
+    before(async function () {
+      const { body: getOutputsRes } = await supertest.get(`/api/fleet/outputs`).expect(200);
+
+      const defaultOutput = getOutputsRes.items.find((item: any) => item.is_default);
+      if (!defaultOutput) {
+        throw new Error('default output not set');
+      }
+
+      defaultOutputId = defaultOutput.id;
+    });
+
+    after(async () => {
+      await esArchiver.unload('fleet/empty_fleet_server');
+    });
+
+    it('GET /outputs should list the default output', async () => {
+      const { body: getOutputsRes } = await supertest.get(`/api/fleet/outputs`).expect(200);
+
+      expect(getOutputsRes.items.length).to.eql(1);
+    });
+
+    it('GET /outputs/{defaultOutputId} should return the default output', async () => {
+      const { body: getOutputRes } = await supertest
+        .get(`/api/fleet/outputs/${defaultOutputId}`)
+        .expect(200);
+
+      expect(getOutputRes.item).to.have.keys('id', 'name', 'type', 'is_default', 'hosts');
+    });
+
+    it('PUT /output/{defaultOutputId} should explicitly set port on ES hosts', async function () {
+      await supertest
+        .put(`/api/fleet/outputs/${defaultOutputId}`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({ hosts: ['https://test.fr'] })
+        .expect(200);
+
+      const { body: getSettingsRes } = await supertest
+        .get(`/api/fleet/outputs/${defaultOutputId}`)
+        .expect(200);
+      expect(getSettingsRes.item.hosts).to.eql(['https://test.fr:443']);
+    });
+  });
+}

x-pack/test/fleet_api_integration/apis/outputs/index.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export default function loadTests({ loadTestFile }) {
+  describe('Output Endpoints', () => {
+    loadTestFile(require.resolve('./crud'));
+  });
+}