Skip to content

Commit

Permalink
Add Xray status aggregation option (#424)
Browse files Browse the repository at this point in the history 
* Change status reduction

* Add custom status reducer

* Simplify env name handling

* Add iterated unit tests

* Improve tests

* Exclude tests from coverage calculation

* Fix glob pattern

* Improve terminology
  • Loading branch information
@csvtuda
csvtuda authored Dec 21, 2024
1 parent 5061682 commit 5577df3
Show file tree
Hide file tree
Showing 13 changed files with 1,316 additions and 154 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
},
"scripts": {
"test": "node --require ./test/loader.js ./test/run-unit-tests.ts",
"test:coverage": "npx shx mkdir -p coverage && c8 -r html npm run test",
"test:coverage": "npx shx mkdir -p coverage && c8 -x '**/*.spec.ts' -r html npm run test",
"test:integration": "node --require ./test/loader.js ./test/run-integration-tests.ts",
"test:server": "node --require ./test/loader.js ./test/run-server.ts",
"build": "tsc --project tsconfig-build.json && shx cp package.json README.md LICENSE.md CHANGELOG.md dist/",
Expand Down
27 changes: 1 addition & 26 deletions src/env.ts
View file
Original file line number Diff line number Diff line change
@@ -1,32 +1,7 @@
import type { CypressXrayPluginOptions } from "./types/plugin";
import type { Remap } from "./types/util";

/**
* An interface containing all authentication options which can be provided via environment
* variables.
*/
interface Authentication {
authentication: {
jira: {
apiToken: string;
password: string;
username: string;
};
xray: {
clientId: string;
clientSecret: string;
};
};
}

/**
* Contains a mapping of all available options to their respective environment variable names.
*/
export const ENV_NAMES: Remap<
Omit<CypressXrayPluginOptions, "http"> & Authentication,
string,
["testExecutionIssue"]
> = {
export const ENV_NAMES = {
authentication: {
jira: {
apiToken: "JIRA_API_TOKEN",
Expand Down
305 changes: 305 additions & 0 deletions src/hooks/after/commands/conversion/cypress/convert-cypress-tests-command.spec.ts
View file

Large diffs are not rendered by default.

75 changes: 59 additions & 16 deletions src/hooks/after/commands/conversion/cypress/convert-cypress-tests-command.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { lt } from "semver";
import type { EvidenceCollection } from "../../../../../context";
import type { RunResult as RunResult_V12 } from "../../../../../types/cypress/12.0.0/api";
import type { CypressRunResultType } from "../../../../../types/cypress/cypress";
import { CypressStatus } from "../../../../../types/cypress/status";
import type { InternalXrayOptions } from "../../../../../types/plugin";
import type {
XrayEvidenceItem,
Expand Down Expand Up @@ -50,7 +51,7 @@ export class ConvertCypressTestsCommand extends Command<[XrayTest, ...XrayTest[]
const version = lt(results.cypressVersion, "13.0.0") ? "<13" : ">=13";
const testRunData = await this.getTestRunData(results, version);
const xrayTests: XrayTest[] = [];
const runsByKey = new Map<string, TestRunData[]>();
const runsByKey = new Map<string, [TestRunData, ...TestRunData[]]>();
testRunData.forEach((testData: TestRunData) => {
try {
const issueKeys = getTestIssueKeys(testData.title, this.parameters.projectKey);
Expand All @@ -77,7 +78,7 @@ export class ConvertCypressTestsCommand extends Command<[XrayTest, ...XrayTest[]
);
}
});
for (const [issueKey, testRuns] of runsByKey.entries()) {
for (const [issueKey, testRuns] of runsByKey) {
const test: XrayTest = this.getTest(testRuns, issueKey, this.getXrayEvidence(issueKey));
xrayTests.push(test);
}
Expand Down Expand Up @@ -113,9 +114,11 @@ export class ConvertCypressTestsCommand extends Command<[XrayTest, ...XrayTest[]
};
for (const run of cypressRuns) {
const testRuns = extractor(run);
testRuns.forEach((promise, index) =>
conversionPromises.push([run.tests[index].title.join(" "), promise])
);
for (const [title, promises] of testRuns) {
for (const promise of promises) {
conversionPromises.push([title, promise]);
}
}
}
if (this.parameters.uploadScreenshots) {
this.addScreenshotEvidence(runResults, version);
Expand Down Expand Up @@ -161,7 +164,7 @@ export class ConvertCypressTestsCommand extends Command<[XrayTest, ...XrayTest[]
const includedScreenshots: string[] = [];
for (const run of runResults.runs) {
const allScreenshots = extractor(run);
for (const [issueKey, screenshots] of allScreenshots.entries()) {
for (const [issueKey, screenshots] of allScreenshots) {
for (const screenshot of screenshots) {
let filename = path.basename(screenshot);
if (this.parameters.normalizeScreenshotNames) {
Expand Down Expand Up @@ -205,32 +208,28 @@ export class ConvertCypressTestsCommand extends Command<[XrayTest, ...XrayTest[]
}

private getTest(
tests: TestRunData[],
runs: [TestRunData, ...TestRunData[]],
issueKey: string,
evidence: XrayEvidenceItem[]
): XrayTest {
const xrayTest: XrayTest = {
finish: truncateIsoTime(
latestDate(
...tests.map((test) => new Date(test.startedAt.getTime() + test.duration))
...runs.map((test) => new Date(test.startedAt.getTime() + test.duration))
).toISOString()
),
start: truncateIsoTime(
earliestDate(...tests.map((test) => test.startedAt)).toISOString()
),
status: getXrayStatus(
tests.map((test) => test.status),
this.parameters.useCloudStatusFallback === true,
this.parameters.xrayStatus
earliestDate(...runs.map((test) => test.startedAt)).toISOString()
),
status: this.getXrayStatus(runs),
testKey: issueKey,
};
if (evidence.length > 0) {
xrayTest.evidence = evidence;
}
if (tests.length > 1) {
if (runs.length > 1) {
const iterations: XrayIterationResult[] = [];
for (const iteration of tests) {
for (const iteration of runs) {
iterations.push({
parameters: [{ name: "iteration", value: (iterations.length + 1).toString() }],
status: getXrayStatus(
Expand All @@ -245,6 +244,50 @@ export class ConvertCypressTestsCommand extends Command<[XrayTest, ...XrayTest[]
return xrayTest;
}

private getXrayStatus(tests: [TestRunData, ...TestRunData[]]): string {
const statuses = tests.map((test) => test.status);
if (statuses.length > 1) {
const passed = statuses.filter((s) => s === CypressStatus.PASSED).length;
const failed = statuses.filter((s) => s === CypressStatus.FAILED).length;
const pending = statuses.filter((s) => s === CypressStatus.PENDING).length;
const skipped = statuses.filter((s) => s === CypressStatus.SKIPPED).length;
if (this.parameters.xrayStatus.aggregate) {
return this.parameters.xrayStatus.aggregate({ failed, passed, pending, skipped });
}
if (passed > 0 && failed === 0 && skipped === 0) {
return getXrayStatus(
CypressStatus.PASSED,
this.parameters.useCloudStatusFallback === true,
this.parameters.xrayStatus
);
}
if (passed === 0 && failed === 0 && skipped === 0 && pending > 0) {
return getXrayStatus(
CypressStatus.PENDING,
this.parameters.useCloudStatusFallback === true,
this.parameters.xrayStatus
);
}
if (skipped > 0) {
return getXrayStatus(
CypressStatus.SKIPPED,
this.parameters.useCloudStatusFallback === true,
this.parameters.xrayStatus
);
}
return getXrayStatus(
CypressStatus.FAILED,
this.parameters.useCloudStatusFallback === true,
this.parameters.xrayStatus
);
}
return getXrayStatus(
statuses[0],
this.parameters.useCloudStatusFallback === true,
this.parameters.xrayStatus
);
}

private getXrayEvidence(issueKey: string): XrayEvidenceItem[] {
const evidence: XrayEvidenceItem[] = [];
this.parameters.evidenceCollection
Expand Down
27 changes: 21 additions & 6 deletions src/hooks/after/commands/conversion/cypress/util/run.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,10 @@ describe(relative(cwd(), __filename), async () => {
};

await it("returns test data for valid runs", async () => {
const testRuns = getTestRunData_V12(passedResult);
const map = getTestRunData_V12(passedResult);
assert.strictEqual(map.size, 1);
const testRuns = map.get("xray upload demo should look for paragraph elements");
assert.ok(testRuns);
const resolvedTestData = await Promise.all(testRuns);
assert.deepStrictEqual(resolvedTestData[0], {
duration: 244,
Expand All @@ -204,7 +207,10 @@ describe(relative(cwd(), __filename), async () => {
});

await it("rejects invalid runs", async () => {
const testRuns = getTestRunData_V12(invalidResult);
const map = getTestRunData_V12(invalidResult);
assert.strictEqual(map.size, 1);
const testRuns = map.get("xray upload demo should fail");
assert.ok(testRuns);
const resolvedTestData = await Promise.allSettled(testRuns);
assert.strictEqual(resolvedTestData[0].status, "rejected");
const reason = resolvedTestData[0].reason as Error;
Expand Down Expand Up @@ -381,9 +387,11 @@ describe(relative(cwd(), __filename), async () => {
};

await it("returns test data for valid runs", async () => {
const testRuns = getTestRunData_V13(passedResult);
const resolvedTestData = await Promise.all(testRuns);
assert.deepStrictEqual(resolvedTestData, [
const map = getTestRunData_V13(passedResult);
assert.strictEqual(map.size, 2);
let testRuns = map.get("something CYP-237 happens");
assert.ok(testRuns);
assert.deepStrictEqual(await Promise.all(testRuns), [
{
duration: 638,
spec: {
Expand All @@ -393,6 +401,10 @@ describe(relative(cwd(), __filename), async () => {
status: CypressStatus.PASSED,
title: "something CYP-237 happens",
},
]);
testRuns = map.get("something something");
assert.ok(testRuns);
assert.deepStrictEqual(await Promise.all(testRuns), [
{
duration: 123,
spec: {
Expand Down Expand Up @@ -422,7 +434,10 @@ describe(relative(cwd(), __filename), async () => {
});

await it("rejects invalid runs", async () => {
const testRuns = getTestRunData_V13(invalidResult);
const map = getTestRunData_V13(invalidResult);
assert.strictEqual(map.size, 1);
const testRuns = map.get("something CYP-237 happens");
assert.ok(testRuns);
const resolvedTestData = await Promise.allSettled(testRuns);
assert.strictEqual(resolvedTestData[0].status, "rejected");
const reason = resolvedTestData[0].reason as Error;
Expand Down
46 changes: 28 additions & 18 deletions src/hooks/after/commands/conversion/cypress/util/run.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,17 +59,17 @@ export interface TestRunData {
* ```
*
* @param runResult - the run result
* @returns an array of test data promises
* @returns a mapping of test titles to their test data promises
*/

// eslint-disable-next-line @typescript-eslint/naming-convention
export function getTestRunData_V12(runResult: RunResult_V12): Promise<TestRunData>[] {
const testRuns: Promise<TestRunData>[] = [];
export function getTestRunData_V12(runResult: RunResult_V12): Map<string, Promise<TestRunData>[]> {
const map = new Map<string, Promise<TestRunData>[]>();
runResult.tests.forEach((test: TestResult_V12) => {
const title = test.title.join(" ");
test.attempts.forEach((attempt) => {
testRuns.push(
new Promise((resolve) => {
const promises = test.attempts.map(
(attempt) =>
new Promise<TestRunData>((resolve) => {
resolve({
duration: attempt.duration,
spec: {
Expand All @@ -80,10 +80,15 @@ export function getTestRunData_V12(runResult: RunResult_V12): Promise<TestRunDat
title: title,
});
})
);
});
);
const testRuns = map.get(title);
if (testRuns) {
testRuns.push(...promises);
} else {
map.set(title, promises);
}
});
return testRuns;
return map;
}

/**
Expand All @@ -109,19 +114,19 @@ export function getTestRunData_V12(runResult: RunResult_V12): Promise<TestRunDat
*
* @param runResult - the run result
* @param options - additional extraction options to consider
* @returns an array of test data promises
* @returns a mapping of test titles to their test data promises
*/
// eslint-disable-next-line @typescript-eslint/naming-convention
export function getTestRunData_V13(
runResult: CypressCommandLine.RunResult
): Promise<TestRunData>[] {
const testRuns: Promise<TestRunData>[] = [];
): Map<string, Promise<TestRunData>[]> {
const map = new Map<string, Promise<TestRunData>[]>();
const testStarts = startTimesByTest(runResult);
runResult.tests.forEach((test: CypressCommandLine.TestResult) => {
const title = test.title.join(" ");
test.attempts.forEach((attempt) => {
testRuns.push(
new Promise((resolve) => {
const promises = test.attempts.map(
(attempt) =>
new Promise<TestRunData>((resolve) => {
resolve({
duration: test.duration,
spec: {
Expand All @@ -132,10 +137,15 @@ export function getTestRunData_V13(
title: title,
});
})
);
});
);
const testRuns = map.get(title);
if (testRuns) {
testRuns.push(...promises);
} else {
map.set(title, promises);
}
});
return testRuns;
return map;
}

function startTimesByTest(run: CypressCommandLine.RunResult): StringMap<Date> {
Expand Down
43 changes: 10 additions & 33 deletions src/hooks/after/commands/conversion/cypress/util/status.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ export function toCypressStatus(statusText: string): CypressStatus {
* @returns the Xray status
*/
export function getXrayStatus(
status: CypressStatus | CypressStatus[],
status: CypressStatus,
useCloudStatus: boolean,
statusOptions?: {
failed?: string;
Expand All @@ -47,37 +47,14 @@ export function getXrayStatus(
skipped?: string;
}
): string {
const lookupStatus = (cypressStatus: CypressStatus) => {
switch (cypressStatus) {
case CypressStatus.PASSED:
return statusOptions?.passed ?? (useCloudStatus ? "PASSED" : "PASS");
case CypressStatus.FAILED:
return statusOptions?.failed ?? (useCloudStatus ? "FAILED" : "FAIL");
case CypressStatus.PENDING:
return statusOptions?.pending ?? (useCloudStatus ? "TO DO" : "TODO");
case CypressStatus.SKIPPED:
return statusOptions?.skipped ?? (useCloudStatus ? "FAILED" : "FAIL");
}
};
if (typeof status === "string") {
return lookupStatus(status);
switch (status) {
case CypressStatus.PASSED:
return statusOptions?.passed ?? (useCloudStatus ? "PASSED" : "PASS");
case CypressStatus.FAILED:
return statusOptions?.failed ?? (useCloudStatus ? "FAILED" : "FAIL");
case CypressStatus.PENDING:
return statusOptions?.pending ?? (useCloudStatus ? "TO DO" : "TODO");
case CypressStatus.SKIPPED:
return statusOptions?.skipped ?? (useCloudStatus ? "FAILED" : "FAIL");
}
const hasPassed = status.some((cypressStatus) => cypressStatus === CypressStatus.PASSED);
const hasFailed = status.some((cypressStatus) => cypressStatus === CypressStatus.FAILED);
const hasPending = status.some((cypressStatus) => cypressStatus === CypressStatus.PENDING);
const hasSkipped = status.some((cypressStatus) => cypressStatus === CypressStatus.SKIPPED);
if (hasPassed && !hasFailed && !hasPending && !hasSkipped) {
return lookupStatus(CypressStatus.PASSED);
}
if (hasPending && !hasFailed && !hasSkipped) {
return lookupStatus(CypressStatus.PENDING);
}
if (hasFailed && hasPassed) {
// TODO: return FLAKY
return lookupStatus(CypressStatus.PASSED);
}
if (hasSkipped && !hasFailed) {
return lookupStatus(CypressStatus.SKIPPED);
}
return lookupStatus(CypressStatus.FAILED);
}
Loading

0 comments on commit 5577df3

Please sign in to comment.