[8.0] [Fleet] Avoid breaking setup when compatible package is not ava…

…ilable in registry (#125525) (#125672)

* [Fleet] Avoid breaking setup when compatible package is not available in registry (#125525)

(cherry picked from commit 928638e)

* [Fleet] Use registry version check on main (#125495)

* [Fleet] Get package info should not store the whole package (#123509)

Co-authored-by: Josh Dover <1813008+joshdover@users.noreply.github.com>
Co-authored-by: Nicolas Chaulet <nicolas.chaulet@elastic.co>

x-pack/plugins/fleet/common/openapi/bundled.json

@@ -646,6 +646,9 @@
                 "properties": {
                   "force": {
                     "type": "boolean"
+                  },
+                  "ignore_constraints": {
+                    "type": "boolean"
                   }
                 }
               }

x-pack/plugins/fleet/common/openapi/bundled.yaml

@@ -396,6 +396,8 @@ paths:
               properties:
                 force:
                   type: boolean
+                ignore_constraints:
+                  type: boolean
     put:
       summary: Packages - Update
       tags: []

x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml

@@ -78,6 +78,8 @@ post:
           properties:
             force:
               type: boolean
+            ignore_constraints:
+              type: boolean
 put:
   summary: Packages - Update
   tags: []

x-pack/plugins/fleet/server/routes/epm/handlers.ts

@@ -43,7 +43,7 @@ import {
   getCategories,
   getPackages,
   getFile,
-  getPackageInfo,
+  getPackageInfoFromRegistry,
   isBulkInstallError,
   installPackage,
   removeInstallation,
@@ -198,7 +198,7 @@ export const getInfoHandler: FleetRequestHandler<TypeOf<typeof GetInfoRequestSch
       if (pkgVersion && !semverValid(pkgVersion)) {
         throw new IngestManagerError('Package version is not a valid semver');
       }
-      const res = await getPackageInfo({
+      const res = await getPackageInfoFromRegistry({
         savedObjectsClient,
         pkgName,
         pkgVersion: pkgVersion || '',
@@ -263,6 +263,7 @@ export const installPackageFromRegistryHandler: FleetRequestHandler<
     esClient,
     spaceId,
     force: request.body?.force,
+    ignoreConstraints: request.body?.ignore_constraints,
   });
   if (!res.error) {
     const body: InstallPackageResponse = {

x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts

@@ -88,7 +88,7 @@ jest.mock(
 jest.mock('../../services/epm/packages', () => {
   return {
     ensureInstalledPackage: jest.fn(() => Promise.resolve()),
-    getPackageInfo: jest.fn(() => Promise.resolve()),
+    getPackageInfoFromRegistry: jest.fn(() => Promise.resolve()),
   };
 });
 

x-pack/plugins/fleet/server/services/epm/packages/get.test.ts

@@ -19,6 +19,8 @@ import * as Registry from '../registry';
 import { createAppContextStartContractMock } from '../../../mocks';
 import { appContextService } from '../../app_context';
 
+import { PackageNotFoundError } from '../../../errors';
+
 import { getPackageInfo, getPackageUsageStats } from './get';
 
 const MockRegistry = Registry as jest.Mocked<typeof Registry>;
@@ -279,5 +281,45 @@ describe('When using EPM `get` services', () => {
         });
       });
     });
+
+    describe('registry fetch errors', () => {
+      it('throws when a package that is not installed is not available in the registry', async () => {
+        MockRegistry.fetchFindLatestPackage.mockResolvedValue(undefined);
+        const soClient = savedObjectsClientMock.create();
+        soClient.get.mockRejectedValue(SavedObjectsErrorHelpers.createGenericNotFoundError());
+
+        await expect(
+          getPackageInfo({
+            savedObjectsClient: soClient,
+            pkgName: 'my-package',
+            pkgVersion: '1.0.0',
+          })
+        ).rejects.toThrowError(PackageNotFoundError);
+      });
+
+      it('sets the latestVersion to installed version when an installed package is not available in the registry', async () => {
+        MockRegistry.fetchFindLatestPackage.mockResolvedValue(undefined);
+        const soClient = savedObjectsClientMock.create();
+        soClient.get.mockResolvedValue({
+          id: 'my-package',
+          type: PACKAGES_SAVED_OBJECT_TYPE,
+          references: [],
+          attributes: {
+            install_status: 'installed',
+          },
+        });
+
+        await expect(
+          getPackageInfo({
+            savedObjectsClient: soClient,
+            pkgName: 'my-package',
+            pkgVersion: '1.0.0',
+          })
+        ).resolves.toMatchObject({
+          latestVersion: '1.0.0',
+          status: 'installed',
+        });
+      });
+    });
   });
 });

x-pack/plugins/fleet/server/services/epm/packages/get.ts

@@ -21,7 +21,7 @@ import type {
   GetCategoriesRequest,
 } from '../../../../common/types';
 import type { Installation, PackageInfo } from '../../../types';
-import { IngestManagerError } from '../../../errors';
+import { IngestManagerError, PackageNotFoundError } from '../../../errors';
 import { appContextService } from '../../';
 import * as Registry from '../registry';
 import { getEsPackage } from '../archive/storage';
@@ -98,7 +98,7 @@ export async function getPackageSavedObjects(
 
 export const getInstallations = getPackageSavedObjects;
 
-export async function getPackageInfo(options: {
+export async function getPackageInfoFromRegistry(options: {
   savedObjectsClient: SavedObjectsClientContract;
   pkgName: string;
   pkgVersion: string;
@@ -111,11 +111,50 @@ export async function getPackageInfo(options: {
 
   // If no package version is provided, use the installed version in the response
   let responsePkgVersion = pkgVersion || savedObject?.attributes.install_version;
-
   // If no installed version of the given package exists, default to the latest version of the package
   if (!responsePkgVersion) {
     responsePkgVersion = latestPackage.version;
   }
+  const packageInfo = await Registry.fetchInfo(pkgName, responsePkgVersion);
+
+  // Fix the paths
+  const paths =
+    packageInfo?.assets?.map((path) =>
+      path.replace(`/package/${pkgName}/${pkgVersion}`, `${pkgName}-${pkgVersion}`)
+    ) ?? [];
+
+  // add properties that aren't (or aren't yet) on the package
+  const additions: EpmPackageAdditions = {
+    latestVersion: latestPackage.version,
+    title: packageInfo.title || nameAsTitle(packageInfo.name),
+    assets: Registry.groupPathsByService(paths || []),
+    removable: true,
+    notice: Registry.getNoticePath(paths || []),
+    keepPoliciesUpToDate: savedObject?.attributes.keep_policies_up_to_date ?? false,
+  };
+  const updated = { ...packageInfo, ...additions };
+
+  return createInstallableFrom(updated, savedObject);
+}
+
+export async function getPackageInfo(options: {
+  savedObjectsClient: SavedObjectsClientContract;
+  pkgName: string;
+  pkgVersion: string;
+}): Promise<PackageInfo> {
+  const { savedObjectsClient, pkgName, pkgVersion } = options;
+  const [savedObject, latestPackage] = await Promise.all([
+    getInstallationObject({ savedObjectsClient, pkgName }),
+    Registry.fetchFindLatestPackage(pkgName, { throwIfNotFound: false }),
+  ]);
+
+  if (!savedObject && !latestPackage) {
+    throw new PackageNotFoundError(`[${pkgName}] package not installed or found in registry`);
+  }
+
+  // If no package version is provided, use the installed version in the response, fallback to package from registry
+  const responsePkgVersion =
+    pkgVersion ?? savedObject?.attributes.install_version ?? latestPackage!.version;
 
   const getPackageRes = await getPackageFromSource({
     pkgName,
@@ -127,7 +166,7 @@ export async function getPackageInfo(options: {
 
   // add properties that aren't (or aren't yet) on the package
   const additions: EpmPackageAdditions = {
-    latestVersion: latestPackage.version,
+    latestVersion: latestPackage?.version ?? responsePkgVersion,
     title: packageInfo.title || nameAsTitle(packageInfo.name),
     assets: Registry.groupPathsByService(paths || []),
     removable: !isUnremovablePackage(pkgName),

x-pack/plugins/fleet/server/services/epm/packages/index.ts

@@ -24,6 +24,7 @@ export {
   getInstallation,
   getInstallations,
   getPackageInfo,
+  getPackageInfoFromRegistry,
   getPackages,
   getLimitedPackages,
 } from './get';

x-pack/plugins/fleet/server/services/epm/packages/install.ts

@@ -209,6 +209,7 @@ interface InstallRegistryPackageParams {
   esClient: ElasticsearchClient;
   spaceId: string;
   force?: boolean;
+  ignoreConstraints?: boolean;
 }
 
 function getTelemetryEvent(pkgName: string, pkgVersion: string): PackageUpdateEvent {
@@ -237,6 +238,7 @@ async function installPackageFromRegistry({
   esClient,
   spaceId,
   force = false,
+  ignoreConstraints = false,
 }: InstallRegistryPackageParams): Promise<InstallResult> {
   const logger = appContextService.getLogger();
   // TODO: change epm API to /packageName/version so we don't need to do this
@@ -253,7 +255,7 @@ async function installPackageFromRegistry({
     installType = getInstallType({ pkgVersion, installedPkg });
 
     // get latest package version
-    const latestPackage = await Registry.fetchFindLatestPackage(pkgName);
+    const latestPackage = await Registry.fetchFindLatestPackage(pkgName, { ignoreConstraints });
 
     // let the user install if using the force flag or needing to reinstall or install a previous version due to failed update
     const installOutOfDateVersionOk =
@@ -476,14 +478,15 @@ export async function installPackage(args: InstallPackageParams) {
   const { savedObjectsClient, esClient } = args;
 
   if (args.installSource === 'registry') {
-    const { pkgkey, force, spaceId } = args;
+    const { pkgkey, force, ignoreConstraints, spaceId } = args;
     logger.debug(`kicking off install of ${pkgkey} from registry`);
     const response = installPackageFromRegistry({
       savedObjectsClient,
       pkgkey,
       esClient,
       spaceId,
       force,
+      ignoreConstraints,
     });
     return response;
   } else if (args.installSource === 'upload') {

x-pack/plugins/fleet/server/services/epm/registry/index.ts

@@ -65,18 +65,33 @@ export async function fetchList(params?: SearchParams): Promise<RegistrySearchRe
   return fetchUrl(url.toString()).then(JSON.parse);
 }
 
-export async function fetchFindLatestPackage(packageName: string): Promise<RegistrySearchResult> {
+// When `throwIfNotFound` is true or undefined, return type will never be undefined.
+export async function fetchFindLatestPackage(
+  packageName: string,
+  options?: { ignoreConstraints?: boolean; throwIfNotFound?: true }
+): Promise<RegistrySearchResult>;
+export async function fetchFindLatestPackage(
+  packageName: string,
+  options: { ignoreConstraints?: boolean; throwIfNotFound: false }
+): Promise<RegistrySearchResult | undefined>;
+export async function fetchFindLatestPackage(
+  packageName: string,
+  options?: { ignoreConstraints?: boolean; throwIfNotFound?: boolean }
+): Promise<RegistrySearchResult | undefined> {
+  const { ignoreConstraints = false, throwIfNotFound = true } = options ?? {};
   const registryUrl = getRegistryUrl();
   const url = new URL(`${registryUrl}/search?package=${packageName}&experimental=true`);
 
-  setKibanaVersion(url);
+  if (!ignoreConstraints) {
+    setKibanaVersion(url);
+  }
 
   const res = await fetchUrl(url.toString());
   const searchResults = JSON.parse(res);
   if (searchResults.length) {
     return searchResults[0];
-  } else {
-    throw new PackageNotFoundError(`${packageName} not found`);
+  } else if (throwIfNotFound) {
+    throw new PackageNotFoundError(`[${packageName}] package not found in registry`);
   }
 }
 
@@ -116,10 +131,8 @@ function setKibanaVersion(url: URL) {
   }
 
   const kibanaVersion = appContextService.getKibanaVersion().split('-')[0]; // may be x.y.z-SNAPSHOT
-  const kibanaBranch = appContextService.getKibanaBranch();
 
-  // on master, request all packages regardless of version
-  if (kibanaVersion && kibanaBranch !== 'master') {
+  if (kibanaVersion) {
     url.searchParams.set('kibana.version', kibanaVersion);
   }
 }

x-pack/plugins/fleet/server/types/rest_spec/epm.ts

@@ -74,7 +74,8 @@ export const InstallPackageFromRegistryRequestSchema = {
   }),
   body: schema.nullable(
     schema.object({
-      force: schema.boolean(),
+      force: schema.boolean({ defaultValue: false }),
+      ignore_constraints: schema.boolean({ defaultValue: false }),
     })
   ),
 };