feat(packagist): Support for v2 protocol (#20626)

lib/modules/datasource/packagist/index.spec.ts

@@ -84,6 +84,8 @@ describe('modules/datasource/packagist/index', () => {
         .replyWithError({ code: 'ETIMEDOUT' });
       httpMock
         .scope(baseUrl)
+        .get('/packages.json')
+        .reply(200, { 'metadata-url': '/p2/%package%.json' })
         .get('/p2/vendor/package-name2.json')
         .reply(200)
         .get('/p2/vendor/package-name2~dev.json')
@@ -104,6 +106,8 @@ describe('modules/datasource/packagist/index', () => {
         .reply(403);
       httpMock
         .scope(baseUrl)
+        .get('/packages.json')
+        .reply(200, { 'metadata-url': '/p2/%package%.json' })
         .get('/p2/vendor/package-name.json')
         .reply(200)
         .get('/p2/vendor/package-name~dev.json')
@@ -124,6 +128,8 @@ describe('modules/datasource/packagist/index', () => {
         .reply(404);
       httpMock
         .scope(baseUrl)
+        .get('/packages.json')
+        .reply(200, { 'metadata-url': '/p2/%package%.json' })
         .get('/p2/drewm/mailchimp-api.json')
         .reply(200)
         .get('/p2/drewm/mailchimp-api~dev.json')
@@ -287,6 +293,8 @@ describe('modules/datasource/packagist/index', () => {
         .reply(200, fileJson);
       httpMock
         .scope(baseUrl)
+        .get('/packages.json')
+        .reply(200, { 'metadata-url': '/p2/%package%.json' })
         .get('/p2/some/other.json')
         .reply(200, beytJson)
         .get('/p2/some/other~dev.json')
@@ -383,6 +391,8 @@ describe('modules/datasource/packagist/index', () => {
         .reply(200, packagesJson);
       httpMock
         .scope(baseUrl)
+        .get('/packages.json')
+        .reply(200, { 'metadata-url': '/p2/%package%.json' })
         .get('/p2/some/other.json')
         .reply(200, beytJson)
         .get('/p2/some/other~dev.json')
@@ -399,10 +409,10 @@ describe('modules/datasource/packagist/index', () => {
     it('processes real versioned data', async () => {
       httpMock
         .scope(baseUrl)
+        .get('/packages.json')
+        .reply(200, { 'metadata-url': '/p2/%package%.json' })
         .get('/p2/drewm/mailchimp-api.json')
-        .reply(200, mailchimpJson);
-      httpMock
-        .scope(baseUrl)
+        .reply(200, mailchimpJson)
         .get('/p2/drewm/mailchimp-api~dev.json')
         .reply(200, mailchimpDevJson);
       config.registryUrls = ['https://packagist.org'];
@@ -419,10 +429,10 @@ describe('modules/datasource/packagist/index', () => {
     it('adds packagist source implicitly', async () => {
       httpMock
         .scope(baseUrl)
+        .get('/packages.json')
+        .reply(200, { 'metadata-url': '/p2/%package%.json' })
         .get('/p2/drewm/mailchimp-api.json')
-        .reply(200, mailchimpJson);
-      httpMock
-        .scope(baseUrl)
+        .reply(200, mailchimpJson)
         .get('/p2/drewm/mailchimp-api~dev.json')
         .reply(200, mailchimpDevJson);
       config.registryUrls = [];
@@ -435,5 +445,41 @@ describe('modules/datasource/packagist/index', () => {
         })
       ).toMatchSnapshot();
     });
+
+    it('fetches packagist V2 packages', async () => {
+      httpMock
+        .scope('https://example.com')
+        .get('/packages.json')
+        .reply(200, {
+          'metadata-url': 'https://example.com/p2/%package%.json',
+        })
+        .get('/p2/drewm/mailchimp-api.json')
+        .reply(200, {
+          minified: 'composer/2.0',
+          packages: {
+            'drewm/mailchimp-api': [
+              {
+                name: 'drewm/mailchimp-api',
+                version: 'v2.5.4',
+              },
+            ],
+          },
+        })
+        .get('/p2/drewm/mailchimp-api~dev.json')
+        .reply(404);
+      config.registryUrls = ['https://example.com'];
+
+      const res = await getPkgReleases({
+        ...config,
+        datasource,
+        versioning,
+        depName: 'drewm/mailchimp-api',
+      });
+
+      expect(res).toEqual({
+        registryUrl: 'https://example.com',
+        releases: [{ gitRef: 'v2.5.4', version: '2.5.4' }],
+      });
+    });
   });
 });

lib/modules/datasource/packagist/index.ts

@@ -1,11 +1,11 @@
-import type { z } from 'zod';
+import { z } from 'zod';
 import { logger } from '../../../logger';
 import { ExternalHostError } from '../../../types/errors/external-host-error';
 import { cache } from '../../../util/cache/package/decorator';
 import * as hostRules from '../../../util/host-rules';
 import type { HttpOptions } from '../../../util/http/types';
 import * as p from '../../../util/promises';
-import { joinUrlParts, resolveBaseUrl } from '../../../util/url';
+import { parseUrl, resolveBaseUrl } from '../../../util/url';
 import * as composerVersioning from '../../versioning/composer';
 import { Datasource } from '../datasource';
 import type { GetReleasesConfig, ReleaseResult } from '../types';
@@ -115,17 +115,28 @@ export class PackagistDatasource extends Datasource {
 
   @cache({
     namespace: `datasource-${PackagistDatasource.id}-org`,
-    key: (regUrl: string) => regUrl,
+    key: (registryUrl: string, metadataUrl: string, packageName: string) =>
+      `${registryUrl}:${metadataUrl}:${packageName}`,
     ttlMinutes: 10,
   })
-  async packagistOrgLookup(name: string): Promise<ReleaseResult | null> {
-    const regUrl = 'https://packagist.org';
-    const pkgUrl = joinUrlParts(regUrl, `/p2/${name}.json`);
-    const devUrl = joinUrlParts(regUrl, `/p2/${name}~dev.json`);
-    const results = await p.map([pkgUrl, devUrl], (url) =>
-      this.http.getJson(url).then(({ body }) => body)
+  async packagistV2Lookup(
+    registryUrl: string,
+    metadataUrl: string,
+    packageName: string
+  ): Promise<ReleaseResult | null> {
+    let pkgUrl = metadataUrl.replace('%package%', packageName);
+    pkgUrl = parseUrl(pkgUrl) ? pkgUrl : resolveBaseUrl(registryUrl, pkgUrl);
+    const pkgPromise = this.getJson(pkgUrl, z.unknown());
+
+    let devUrl = metadataUrl.replace('%package%', `${packageName}~dev`);
+    devUrl = parseUrl(devUrl) ? devUrl : resolveBaseUrl(registryUrl, devUrl);
+    const devPromise = this.getJson(devUrl, z.unknown()).then(
+      (x) => x,
+      () => null
     );
-    return parsePackagesResponses(name, results);
+
+    const results = await Promise.all([pkgPromise, devPromise]);
+    return parsePackagesResponses(packageName, results);
   }
 
   public getPkgUrl(
@@ -169,13 +180,17 @@ export class PackagistDatasource extends Datasource {
     }
 
     try {
-      if (registryUrl === 'https://packagist.org') {
-        const packagistResult = await this.packagistOrgLookup(packageName);
+      const meta = await this.getRegistryMeta(registryUrl);
+
+      if (meta.metadataUrl) {
+        const packagistResult = await this.packagistV2Lookup(
+          registryUrl,
+          meta.metadataUrl,
+          packageName
+        );
         return packagistResult;
       }
 
-      const meta = await this.getRegistryMeta(registryUrl);
-
       if (meta.packages[packageName]) {
         const result = extractDepReleases(meta.packages[packageName]);
         return result;

lib/modules/datasource/packagist/schema.spec.ts

@@ -254,6 +254,7 @@ describe('modules/datasource/packagist/schema', () => {
         includesPackages: {},
         providersLazyUrl: null,
         providersUrl: null,
+        metadataUrl: null,
       });
     });
   });

lib/modules/datasource/packagist/schema.ts

@@ -205,6 +205,7 @@ export const RegistryMeta = z
         ),
         ['providers-lazy-url']: looseValue(z.string()),
         ['providers-url']: looseValue(z.string()),
+        ['metadata-url']: looseValue(z.string()),
       })
     )
   )
@@ -216,13 +217,15 @@ export const RegistryMeta = z
       ['providers']: providerPackages,
       ['providers-lazy-url']: providersLazyUrl,
       ['providers-url']: providersUrl,
+      ['metadata-url']: metadataUrl,
     }) => ({
       packages,
       includesFiles,
       providerPackages,
       files,
       providersUrl,
       providersLazyUrl,
+      metadataUrl,
       includesPackages: {} as Record<string, ReleaseResult | null>,
     })
   );