From c9fe3b99fadf6db92c2ce3cbc02e1189b8327b5f Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 28 Feb 2023 12:31:16 +0300 Subject: [PATCH] feat(packagist): Support for `v2` protocol (#20626) --- .../datasource/packagist/index.spec.ts | 58 +++++++++++++++++-- lib/modules/datasource/packagist/index.ts | 43 +++++++++----- .../datasource/packagist/schema.spec.ts | 1 + lib/modules/datasource/packagist/schema.ts | 3 + 4 files changed, 85 insertions(+), 20 deletions(-) diff --git a/lib/modules/datasource/packagist/index.spec.ts b/lib/modules/datasource/packagist/index.spec.ts index 3fe8837bf41cb5..845c19082fdd88 100644 --- a/lib/modules/datasource/packagist/index.spec.ts +++ b/lib/modules/datasource/packagist/index.spec.ts @@ -84,6 +84,8 @@ describe('modules/datasource/packagist/index', () => { .replyWithError({ code: 'ETIMEDOUT' }); httpMock .scope(baseUrl) + .get('/packages.json') + .reply(200, { 'metadata-url': '/p2/%package%.json' }) .get('/p2/vendor/package-name2.json') .reply(200) .get('/p2/vendor/package-name2~dev.json') @@ -104,6 +106,8 @@ describe('modules/datasource/packagist/index', () => { .reply(403); httpMock .scope(baseUrl) + .get('/packages.json') + .reply(200, { 'metadata-url': '/p2/%package%.json' }) .get('/p2/vendor/package-name.json') .reply(200) .get('/p2/vendor/package-name~dev.json') @@ -124,6 +128,8 @@ describe('modules/datasource/packagist/index', () => { .reply(404); httpMock .scope(baseUrl) + .get('/packages.json') + .reply(200, { 'metadata-url': '/p2/%package%.json' }) .get('/p2/drewm/mailchimp-api.json') .reply(200) .get('/p2/drewm/mailchimp-api~dev.json') @@ -287,6 +293,8 @@ describe('modules/datasource/packagist/index', () => { .reply(200, fileJson); httpMock .scope(baseUrl) + .get('/packages.json') + .reply(200, { 'metadata-url': '/p2/%package%.json' }) .get('/p2/some/other.json') .reply(200, beytJson) .get('/p2/some/other~dev.json') @@ -383,6 +391,8 @@ describe('modules/datasource/packagist/index', () => { .reply(200, packagesJson); httpMock .scope(baseUrl) + .get('/packages.json') + .reply(200, { 'metadata-url': '/p2/%package%.json' }) .get('/p2/some/other.json') .reply(200, beytJson) .get('/p2/some/other~dev.json') @@ -399,10 +409,10 @@ describe('modules/datasource/packagist/index', () => { it('processes real versioned data', async () => { httpMock .scope(baseUrl) + .get('/packages.json') + .reply(200, { 'metadata-url': '/p2/%package%.json' }) .get('/p2/drewm/mailchimp-api.json') - .reply(200, mailchimpJson); - httpMock - .scope(baseUrl) + .reply(200, mailchimpJson) .get('/p2/drewm/mailchimp-api~dev.json') .reply(200, mailchimpDevJson); config.registryUrls = ['https://packagist.org']; @@ -419,10 +429,10 @@ describe('modules/datasource/packagist/index', () => { it('adds packagist source implicitly', async () => { httpMock .scope(baseUrl) + .get('/packages.json') + .reply(200, { 'metadata-url': '/p2/%package%.json' }) .get('/p2/drewm/mailchimp-api.json') - .reply(200, mailchimpJson); - httpMock - .scope(baseUrl) + .reply(200, mailchimpJson) .get('/p2/drewm/mailchimp-api~dev.json') .reply(200, mailchimpDevJson); config.registryUrls = []; @@ -435,5 +445,41 @@ describe('modules/datasource/packagist/index', () => { }) ).toMatchSnapshot(); }); + + it('fetches packagist V2 packages', async () => { + httpMock + .scope('https://example.com') + .get('/packages.json') + .reply(200, { + 'metadata-url': 'https://example.com/p2/%package%.json', + }) + .get('/p2/drewm/mailchimp-api.json') + .reply(200, { + minified: 'composer/2.0', + packages: { + 'drewm/mailchimp-api': [ + { + name: 'drewm/mailchimp-api', + version: 'v2.5.4', + }, + ], + }, + }) + .get('/p2/drewm/mailchimp-api~dev.json') + .reply(404); + config.registryUrls = ['https://example.com']; + + const res = await getPkgReleases({ + ...config, + datasource, + versioning, + depName: 'drewm/mailchimp-api', + }); + + expect(res).toEqual({ + registryUrl: 'https://example.com', + releases: [{ gitRef: 'v2.5.4', version: '2.5.4' }], + }); + }); }); }); diff --git a/lib/modules/datasource/packagist/index.ts b/lib/modules/datasource/packagist/index.ts index 1f4897842ced56..8d902f99f39174 100644 --- a/lib/modules/datasource/packagist/index.ts +++ b/lib/modules/datasource/packagist/index.ts @@ -1,11 +1,11 @@ -import type { z } from 'zod'; +import { z } from 'zod'; import { logger } from '../../../logger'; import { ExternalHostError } from '../../../types/errors/external-host-error'; import { cache } from '../../../util/cache/package/decorator'; import * as hostRules from '../../../util/host-rules'; import type { HttpOptions } from '../../../util/http/types'; import * as p from '../../../util/promises'; -import { joinUrlParts, resolveBaseUrl } from '../../../util/url'; +import { parseUrl, resolveBaseUrl } from '../../../util/url'; import * as composerVersioning from '../../versioning/composer'; import { Datasource } from '../datasource'; import type { GetReleasesConfig, ReleaseResult } from '../types'; @@ -115,17 +115,28 @@ export class PackagistDatasource extends Datasource { @cache({ namespace: `datasource-${PackagistDatasource.id}-org`, - key: (regUrl: string) => regUrl, + key: (registryUrl: string, metadataUrl: string, packageName: string) => + `${registryUrl}:${metadataUrl}:${packageName}`, ttlMinutes: 10, }) - async packagistOrgLookup(name: string): Promise { - const regUrl = 'https://packagist.org'; - const pkgUrl = joinUrlParts(regUrl, `/p2/${name}.json`); - const devUrl = joinUrlParts(regUrl, `/p2/${name}~dev.json`); - const results = await p.map([pkgUrl, devUrl], (url) => - this.http.getJson(url).then(({ body }) => body) + async packagistV2Lookup( + registryUrl: string, + metadataUrl: string, + packageName: string + ): Promise { + let pkgUrl = metadataUrl.replace('%package%', packageName); + pkgUrl = parseUrl(pkgUrl) ? pkgUrl : resolveBaseUrl(registryUrl, pkgUrl); + const pkgPromise = this.getJson(pkgUrl, z.unknown()); + + let devUrl = metadataUrl.replace('%package%', `${packageName}~dev`); + devUrl = parseUrl(devUrl) ? devUrl : resolveBaseUrl(registryUrl, devUrl); + const devPromise = this.getJson(devUrl, z.unknown()).then( + (x) => x, + () => null ); - return parsePackagesResponses(name, results); + + const results = await Promise.all([pkgPromise, devPromise]); + return parsePackagesResponses(packageName, results); } public getPkgUrl( @@ -169,13 +180,17 @@ export class PackagistDatasource extends Datasource { } try { - if (registryUrl === 'https://packagist.org') { - const packagistResult = await this.packagistOrgLookup(packageName); + const meta = await this.getRegistryMeta(registryUrl); + + if (meta.metadataUrl) { + const packagistResult = await this.packagistV2Lookup( + registryUrl, + meta.metadataUrl, + packageName + ); return packagistResult; } - const meta = await this.getRegistryMeta(registryUrl); - if (meta.packages[packageName]) { const result = extractDepReleases(meta.packages[packageName]); return result; diff --git a/lib/modules/datasource/packagist/schema.spec.ts b/lib/modules/datasource/packagist/schema.spec.ts index 9e82858be9f945..035a8a1660cb21 100644 --- a/lib/modules/datasource/packagist/schema.spec.ts +++ b/lib/modules/datasource/packagist/schema.spec.ts @@ -254,6 +254,7 @@ describe('modules/datasource/packagist/schema', () => { includesPackages: {}, providersLazyUrl: null, providersUrl: null, + metadataUrl: null, }); }); }); diff --git a/lib/modules/datasource/packagist/schema.ts b/lib/modules/datasource/packagist/schema.ts index 32de3ca10c8360..d71a9b27da9cec 100644 --- a/lib/modules/datasource/packagist/schema.ts +++ b/lib/modules/datasource/packagist/schema.ts @@ -205,6 +205,7 @@ export const RegistryMeta = z ), ['providers-lazy-url']: looseValue(z.string()), ['providers-url']: looseValue(z.string()), + ['metadata-url']: looseValue(z.string()), }) ) ) @@ -216,6 +217,7 @@ export const RegistryMeta = z ['providers']: providerPackages, ['providers-lazy-url']: providersLazyUrl, ['providers-url']: providersUrl, + ['metadata-url']: metadataUrl, }) => ({ packages, includesFiles, @@ -223,6 +225,7 @@ export const RegistryMeta = z files, providersUrl, providersLazyUrl, + metadataUrl, includesPackages: {} as Record, }) );