Skip to content

Commit

Permalink
refactor(maven): Unified result type for http fetch (#32813)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zharinov
zharinov authored Dec 9, 2024
1 parent f1ffc5e commit 606ab43
Show file tree
Hide file tree
Showing 5 changed files with 110 additions and 80 deletions.
25 changes: 20 additions & 5 deletions lib/modules/datasource/maven/util.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,31 +71,43 @@ describe('modules/datasource/maven/util', () => {
get: () => Promise.reject(httpError({ message: HOST_DISABLED })),
});
const res = await downloadHttpProtocol(http, 'some://');
expect(res).toBeNull();
expect(res.unwrap()).toEqual({
ok: false,
err: { type: 'host-disabled' } satisfies MavenFetchError,
});
});

it('returns empty for host error', async () => {
const http = partial<Http>({
get: () => Promise.reject(httpError({ code: 'ETIMEDOUT' })),
});
const res = await downloadHttpProtocol(http, 'some://');
expect(res).toBeNull();
expect(res.unwrap()).toEqual({
ok: false,
err: { type: 'host-error' } satisfies MavenFetchError,
});
});

it('returns empty for temporary error', async () => {
const http = partial<Http>({
get: () => Promise.reject(httpError({ code: 'ECONNRESET' })),
});
const res = await downloadHttpProtocol(http, 'some://');
expect(res).toBeNull();
expect(res.unwrap()).toEqual({
ok: false,
err: { type: 'temporary-error' } satisfies MavenFetchError,
});
});

it('returns empty for connection error', async () => {
const http = partial<Http>({
get: () => Promise.reject(httpError({ code: 'ECONNREFUSED' })),
});
const res = await downloadHttpProtocol(http, 'some://');
expect(res).toBeNull();
expect(res.unwrap()).toEqual({
ok: false,
err: { type: 'connection-error' } satisfies MavenFetchError,
});
});

it('returns empty for unsupported error', async () => {
Expand All @@ -104,7 +116,10 @@ describe('modules/datasource/maven/util', () => {
Promise.reject(httpError({ name: 'UnsupportedProtocolError' })),
});
const res = await downloadHttpProtocol(http, 'some://');
expect(res).toBeNull();
expect(res.unwrap()).toEqual({
ok: false,
err: { type: 'unsupported-host' } satisfies MavenFetchError,
});
});
});

Expand Down
103 changes: 59 additions & 44 deletions lib/modules/datasource/maven/util.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,73 +71,92 @@ export async function downloadHttpProtocol(
http: Http,
pkgUrl: URL | string,
opts: HttpOptions = {},
): Promise<HttpResponse | null> {
): Promise<MavenFetchResult> {
const url = pkgUrl.toString();
const res = await Result.wrap(http.get(url, opts))
.onError((err) => {
const fetchResult = await Result.wrap<HttpResponse, Error>(
http.get(url, opts),
)
.transform((res): MavenFetchSuccess => {
const result: MavenFetchSuccess = { data: res.body };

if (!res.authorization) {
result.isCacheable = true;
}

const lastModified = normalizeDate(res?.headers?.['last-modified']);
if (lastModified) {
result.lastModified = lastModified;
}

return result;
})
.catch((err): MavenFetchResult => {
// istanbul ignore next: never happens, needs for type narrowing
if (!(err instanceof HttpError)) {
return;
return Result.err({ type: 'unknown', err });
}

const failedUrl = url;
if (err.message === HOST_DISABLED) {
logger.trace({ failedUrl }, 'Host disabled');
return;
return Result.err({ type: 'host-disabled' });
}

if (isNotFoundError(err)) {
logger.trace({ failedUrl }, `Url not found`);
return;
return Result.err({ type: 'not-found' });
}

if (isHostError(err)) {
logger.debug(`Cannot connect to host ${failedUrl}`);
return;
return Result.err({ type: 'host-error' });
}

if (isPermissionsIssue(err)) {
logger.debug(
`Dependency lookup unauthorized. Please add authentication with a hostRule for ${failedUrl}`,
);
return;
return Result.err({ type: 'permission-issue' });
}

if (isTemporaryError(err)) {
logger.debug({ failedUrl, err }, 'Temporary error');
return;
if (getHost(url) === getHost(MAVEN_REPO)) {
return Result.err({ type: 'maven-central-temporary-error', err });
} else {
return Result.err({ type: 'temporary-error' });
}
}

if (isConnectionError(err)) {
logger.debug(`Connection refused to maven registry ${failedUrl}`);
return;
return Result.err({ type: 'connection-error' });
}

if (isUnsupportedHostError(err)) {
logger.debug(`Unsupported host ${failedUrl}`);
return;
return Result.err({ type: 'unsupported-host' });
}

logger.info({ failedUrl, err }, 'Unknown HTTP download error');
})
.catch((err): Result<HttpResponse | 'silent-error', ExternalHostError> => {
if (
err instanceof HttpError &&
isTemporaryError(err) &&
getHost(url) === getHost(MAVEN_REPO)
) {
return Result.err(new ExternalHostError(err));
}

return Result.ok('silent-error');
})
.unwrapOrThrow();
return Result.err({ type: 'unknown', err });
});

if (res === 'silent-error') {
return null;
const { err } = fetchResult.unwrap();
if (err?.type === 'maven-central-temporary-error') {
throw new ExternalHostError(err.err);
}

return res;
return fetchResult;
}

export async function downloadHttpContent(
http: Http,
pkgUrl: URL | string,
opts: HttpOptions = {},
): Promise<string | null> {
const fetchResult = await downloadHttpProtocol(http, pkgUrl, opts);
return fetchResult.transform(({ data }) => data).unwrapOrNull();
}

function isS3NotFound(err: Error): boolean {
Expand Down Expand Up @@ -228,7 +247,7 @@ export async function downloadS3Protocol(
export async function downloadArtifactRegistryProtocol(
http: Http,
pkgUrl: URL,
): Promise<HttpResponse | null> {
): Promise<MavenFetchResult> {
const opts: HttpOptions = {};
const host = pkgUrl.host;
const path = pkgUrl.pathname;
Expand Down Expand Up @@ -357,25 +376,21 @@ export async function downloadMavenXml(
const protocol = pkgUrl.protocol;

if (protocol === 'http:' || protocol === 'https:') {
const res = await downloadHttpProtocol(http, pkgUrl);
const body = res?.body;
if (body) {
return {
xml: new XmlDocument(body),
isCacheable: !res.authorization,
};
}
const rawResult = await downloadHttpProtocol(http, pkgUrl);
const xmlResult = rawResult.transform(({ isCacheable, data }): MavenXml => {
const xml = new XmlDocument(data);
return { isCacheable, xml };
});
return xmlResult.unwrapOr({});
}

if (protocol === 'artifactregistry:') {
const res = await downloadArtifactRegistryProtocol(http, pkgUrl);
const body = res?.body;
if (body) {
return {
xml: new XmlDocument(body),
isCacheable: !res.authorization,
};
}
const rawResult = await downloadArtifactRegistryProtocol(http, pkgUrl);
const xmlResult = rawResult.transform(({ isCacheable, data }): MavenXml => {
const xml = new XmlDocument(data);
return { isCacheable, xml };
});
return xmlResult.unwrapOr({});
}

if (protocol === 's3:') {
Expand Down
6 changes: 3 additions & 3 deletions lib/modules/datasource/sbt-package/index.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,9 +149,9 @@ describe('modules/datasource/sbt-package/index', () => {
.get('/org/example/example_2.12/')
.reply(200, `<a href='1.2.3/'>1.2.3/</a>`)
.get('/org/example/example_2.12/1.2.3/example-1.2.3.pom')
.reply(200, ``)
.reply(404)
.get('/org/example/example_2.12/1.2.3/example_2.12-1.2.3.pom')
.reply(200, ``);
.reply(404);

const res = await getPkgReleases({
versioning: mavenVersioning.id,
Expand Down Expand Up @@ -267,7 +267,7 @@ describe('modules/datasource/sbt-package/index', () => {
`,
)
.get('/org/example/example_2.13/1.2.3/example_2.13-1.2.3.pom')
.reply(200);
.reply(404);

const res = await getPkgReleases({
versioning: mavenVersioning.id,
Expand Down
26 changes: 14 additions & 12 deletions lib/modules/datasource/sbt-package/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,7 @@ import * as ivyVersioning from '../../versioning/ivy';
import { compare } from '../../versioning/maven/compare';
import { MavenDatasource } from '../maven';
import { MAVEN_REPO } from '../maven/common';
import { downloadHttpProtocol } from '../maven/util';
import { normalizeDate } from '../metadata';
import { downloadHttpContent, downloadHttpProtocol } from '../maven/util';
import type {
GetReleasesConfig,
PostprocessReleaseConfig,
Expand Down Expand Up @@ -88,8 +87,11 @@ export class SbtPackageDatasource extends MavenDatasource {
let dependencyUrl: string | undefined;
let packageUrls: string[] | undefined;
for (const packageRootUrl of packageRootUrls) {
const res = await downloadHttpProtocol(this.http, packageRootUrl);
if (!res) {
const packageRootContent = await downloadHttpContent(
this.http,
packageRootUrl,
);
if (!packageRootContent) {
continue;
}

Expand All @@ -103,7 +105,7 @@ export class SbtPackageDatasource extends MavenDatasource {
dependencyUrl = trimTrailingSlash(packageRootUrl);

const rootPath = new URL(packageRootUrl).pathname;
const artifactSubdirs = extractPageLinks(res.body, (href) => {
const artifactSubdirs = extractPageLinks(packageRootContent, (href) => {
const path = href.replace(rootPath, '');

if (
Expand Down Expand Up @@ -149,15 +151,15 @@ export class SbtPackageDatasource extends MavenDatasource {

const allVersions = new Set<string>();
for (const pkgUrl of packageUrls) {
const res = await downloadHttpProtocol(this.http, pkgUrl);
const packageContent = await downloadHttpContent(this.http, pkgUrl);
// istanbul ignore if
if (!res) {
if (!packageContent) {
invalidPackageUrls.add(pkgUrl);
continue;
}

const rootPath = new URL(pkgUrl).pathname;
const versions = extractPageLinks(res.body, (href) => {
const versions = extractPageLinks(packageContent, (href) => {
const path = href.replace(rootPath, '');
if (path.startsWith('.')) {
return null;
Expand Down Expand Up @@ -275,20 +277,20 @@ export class SbtPackageDatasource extends MavenDatasource {
}

const res = await downloadHttpProtocol(this.http, pomUrl);
const content = res?.body;
if (!content) {
const { val } = res.unwrap();
if (!val) {
invalidPomFiles.add(pomUrl);
continue;
}

const result: PomInfo = {};

const releaseTimestamp = normalizeDate(res.headers['last-modified']);
const releaseTimestamp = val.lastModified;
if (releaseTimestamp) {
result.releaseTimestamp = releaseTimestamp;
}

const pomXml = new XmlDocument(content);
const pomXml = new XmlDocument(val.data);

const homepage = pomXml.valueWithPath('url');
if (homepage) {
Expand Down
Loading

0 comments on commit 606ab43

Please sign in to comment.