From 49874991b9a71eb39ed10d34e0c566c977bb0455 Mon Sep 17 00:00:00 2001 From: Alexandru-Paul Dumitru Date: Tue, 21 Jan 2020 17:01:55 +0100 Subject: [PATCH 1/9] WIP: - added headers - fix current unit tests for download - update intdex.ts to export new interfaces and types - new interfaces created - update download to use new imperative rest method and return etag if needed - update upload to use new imperative rest method - pass and retreive etag if needed Signed-off-by: Alexandru-Paul Dumitru --- packages/rest/src/ZosmfHeaders.ts | 23 ++ .../methods/download/Download.unit.test.ts | 94 ++++-- .../src/api/doc/IOptionsFullResponse.ts | 85 ++++++ .../src/api/doc/IRestClientResponse.ts | 71 +++++ .../doc/types/ZosmfRestClientProperties.ts | 28 ++ packages/zosfiles/src/api/index.ts | 3 + .../src/api/methods/download/Download.ts | 61 +++- .../methods/download/doc/IDownloadOptions.ts | 8 + .../zosfiles/src/api/methods/upload/Upload.ts | 281 ++++++++++++++---- .../api/methods/upload/doc/IUploadOptions.ts | 19 ++ .../api/methods/upload/doc/IUploadResult.ts | 4 + .../zosfiles/src/api/methods/upload/index.ts | 1 + 12 files changed, 598 insertions(+), 80 deletions(-) create mode 100644 packages/zosfiles/src/api/doc/IOptionsFullResponse.ts create mode 100644 packages/zosfiles/src/api/doc/IRestClientResponse.ts create mode 100644 packages/zosfiles/src/api/doc/types/ZosmfRestClientProperties.ts diff --git a/packages/rest/src/ZosmfHeaders.ts b/packages/rest/src/ZosmfHeaders.ts index 0583573342..6be76a50ee 100644 --- a/packages/rest/src/ZosmfHeaders.ts +++ b/packages/rest/src/ZosmfHeaders.ts @@ -157,4 +157,27 @@ export class ZosmfHeaders { public static readonly X_IBM_MIGRATED_RECALL_NO_WAIT: IHeaderContent = {"X-IBM-Migrated-Recall": "nowait"}; public static readonly X_IBM_MIGRATED_RECALL_ERROR: IHeaderContent = {"X-IBM-Migrated-Recall": "error"}; + /** + * Header to check ETag on read + * Request returns HTTP 304 if not modified + * @static + * @memberof ZosmfHeaders + */ + public static readonly IF_NONE_MATCH = "If-None-Match"; + + /** + * Header to check ETag on write + * Request returns HTTP 412 if not matched + * @static + * @memberof ZosmfHeaders + */ + public static readonly IF_MATCH = "If-Match"; + + /** + * Header to force return of ETag in response regardless of file size + * By default Etag is returned only for files smaller than a system determined value (which is at least 8mb) + * @static + * @memberof ZosmfHeaders + */ + public static readonly X_IBM_RETURN_ETAG: IHeaderContent = {"X-IBM-Return-Etag": "true"}; } diff --git a/packages/zosfiles/__tests__/api/methods/download/Download.unit.test.ts b/packages/zosfiles/__tests__/api/methods/download/Download.unit.test.ts index 172a9a9e51..43cc17f528 100644 --- a/packages/zosfiles/__tests__/api/methods/download/Download.unit.test.ts +++ b/packages/zosfiles/__tests__/api/methods/download/Download.unit.test.ts @@ -26,6 +26,7 @@ describe("z/OS Files - Download", () => { const arrOfUssPath: string[] = ussname.split("/"); const localFileName = arrOfUssPath[arrOfUssPath.length - 1]; const ussFileContent = "Test data for unit test"; + const etag = "123ABC"; const dummySession = new Session({ user: "fake", @@ -42,10 +43,16 @@ describe("z/OS Files - Download", () => { const ioWriteFileSpy = jest.spyOn(IO, "writeFile"); const ioWriteStreamSpy = jest.spyOn(IO, "createWriteStream"); const fakeWriteStream: any = {fakeWriteStream: true}; + const zosmfGetFullSpy = jest.spyOn(ZosmfRestClient, "getExpectFullResponse"); + const fakeResponseWithEtag = {data: ussFileContent, response:{headers:{etag}}}; + beforeEach(() => { zosmfStreamSpy.mockClear(); zosmfStreamSpy.mockImplementation(() => null); + zosmfGetFullSpy.mockClear(); + zosmfGetFullSpy.mockImplementation(() => null); + ioCreateDirSpy.mockClear(); ioCreateDirSpy.mockImplementation(() => null); @@ -119,8 +126,12 @@ describe("z/OS Files - Download", () => { }); - expect(zosmfStreamSpy).toHaveBeenCalledTimes(1); - expect(zosmfStreamSpy).toHaveBeenCalledWith(dummySession, endpoint, [], fakeWriteStream, true, undefined); + expect(zosmfGetFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfGetFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders: [], + responseStream: fakeWriteStream, + normalizeResponseNewLines: true, + task: undefined}); expect(ioCreateDirSpy).toHaveBeenCalledTimes(1); expect(ioCreateDirSpy).toHaveBeenCalledWith(destination); @@ -151,8 +162,12 @@ describe("z/OS Files - Download", () => { apiResponse: {} }); - expect(zosmfStreamSpy).toHaveBeenCalledTimes(1); - expect(zosmfStreamSpy).toHaveBeenCalledWith(dummySession, endpoint, [], fakeWriteStream, true, undefined); + expect(zosmfGetFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfGetFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders: [], + responseStream: fakeWriteStream, + normalizeResponseNewLines: true, + task: undefined}); expect(ioCreateDirSpy).toHaveBeenCalledTimes(1); expect(ioCreateDirSpy).toHaveBeenCalledWith(destination); @@ -181,10 +196,12 @@ describe("z/OS Files - Download", () => { apiResponse: {} }); - expect(zosmfStreamSpy).toHaveBeenCalledTimes(1); - expect(zosmfStreamSpy).toHaveBeenCalledWith(dummySession, endpoint, [ZosmfHeaders.X_IBM_BINARY], fakeWriteStream, - false /* don't normalize newlines, binary mode*/, - undefined /* no progress task */); + expect(zosmfGetFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfGetFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders: [ZosmfHeaders.X_IBM_BINARY], + responseStream: fakeWriteStream, + normalizeResponseNewLines: false /* don't normalize newlines, binary mode*/, + task: undefined /* no progress task */}); expect(ioCreateDirSpy).toHaveBeenCalledTimes(1); expect(ioCreateDirSpy).toHaveBeenCalledWith(destination); @@ -213,10 +230,12 @@ describe("z/OS Files - Download", () => { apiResponse: {} }); - expect(zosmfStreamSpy).toHaveBeenCalledTimes(1); - expect(zosmfStreamSpy).toHaveBeenCalledWith(dummySession, endpoint, [ZosmfHeaders.X_IBM_BINARY], fakeWriteStream, - false, /* no normalizing new lines, binary mode*/ - undefined /*no progress task*/); + expect(zosmfGetFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfGetFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders: [ZosmfHeaders.X_IBM_BINARY], + responseStream: fakeWriteStream, + normalizeResponseNewLines: false, /* no normalizing new lines, binary mode*/ + task: undefined /*no progress task*/}); expect(ioCreateDirSpy).toHaveBeenCalledTimes(1); expect(ioCreateDirSpy).toHaveBeenCalledWith(file); @@ -230,7 +249,10 @@ describe("z/OS Files - Download", () => { let caughtError; const dummyError = new Error("test"); - zosmfStreamSpy.mockImplementation(() => { + // zosmfStreamSpy.mockImplementation(() => { + // throw dummyError; + // }); + zosmfGetFullSpy.mockImplementation(() => { throw dummyError; }); @@ -245,8 +267,12 @@ describe("z/OS Files - Download", () => { expect(response).toBeUndefined(); expect(caughtError).toEqual(dummyError); - expect(zosmfStreamSpy).toHaveBeenCalledTimes(1); - expect(zosmfStreamSpy).toHaveBeenCalledWith(dummySession, endpoint, [], fakeWriteStream, true, undefined /*no progress task*/); + expect(zosmfGetFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfGetFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders: [], + responseStream: fakeWriteStream, + normalizeResponseNewLines: true, + task: undefined /*no progress task*/}); }); }); @@ -523,11 +549,16 @@ describe("z/OS Files - Download", () => { const ioCreateDirSpy = jest.spyOn(IO, "createDirsSyncFromFilePath"); const ioWriteStreamSpy = jest.spyOn(IO, "createWriteStream"); const fakeStream: any = {fakeStream: true}; + const zosmfGetFullSpy = jest.spyOn(ZosmfRestClient, "getExpectFullResponse"); + const fakeResponseWithEtag = {data: ussFileContent, response:{headers:{etag}}}; beforeEach(() => { zosmfStreamSpy.mockClear(); zosmfStreamSpy.mockImplementation(() => ussFileContent); + zosmfGetFullSpy.mockClear(); + zosmfGetFullSpy.mockImplementation(() => ussFileContent); + zosmfExpectBufferSpy.mockClear(); zosmfExpectBufferSpy.mockImplementation(() => ussFileContent); @@ -597,8 +628,13 @@ describe("z/OS Files - Download", () => { apiResponse: {} }); - expect(zosmfStreamSpy).toHaveBeenCalledTimes(1); - expect(zosmfStreamSpy).toHaveBeenCalledWith(dummySession, endpoint, [], fakeStream, true, undefined); + expect(zosmfGetFullSpy).toHaveBeenCalledTimes(1); + // expect(zosmfGetFullSpy).toHaveBeenCalledWith(dummySession, endpoint, [], fakeStream, true, undefined); + expect(zosmfGetFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders: [], + responseStream: fakeStream, + normalizeResponseNewLines: true, + }); expect(ioCreateDirSpy).toHaveBeenCalledTimes(1); expect(ioCreateDirSpy).toHaveBeenCalledWith(destination); @@ -626,10 +662,15 @@ describe("z/OS Files - Download", () => { apiResponse: {} }); - expect(zosmfStreamSpy).toHaveBeenCalledTimes(1); - expect(zosmfStreamSpy).toHaveBeenCalledWith(dummySession, endpoint, [ZosmfHeaders.X_IBM_BINARY], fakeStream, - false, /* don't normalize new lines in binary*/ - undefined /* no progress task */); + expect(zosmfGetFullSpy).toHaveBeenCalledTimes(1); + // expect(zosmfStreamSpy).toHaveBeenCalledWith(dummySession, endpoint, [ZosmfHeaders.X_IBM_BINARY], fakeStream, + // false, /* don't normalize new lines in binary*/ + // undefined /* no progress task */); + expect(zosmfGetFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders: [ZosmfHeaders.X_IBM_BINARY], + responseStream: fakeStream, + normalizeResponseNewLines: false, /* don't normalize new lines in binary*/ + task: undefined /* no progress task */}); expect(ioCreateDirSpy).toHaveBeenCalledTimes(1); expect(ioCreateDirSpy).toHaveBeenCalledWith(destination); @@ -657,11 +698,12 @@ describe("z/OS Files - Download", () => { apiResponse: {} }); - expect(zosmfStreamSpy).toHaveBeenCalledTimes(1); - expect(zosmfStreamSpy).toHaveBeenCalledWith(dummySession, endpoint, [ZosmfHeaders.X_IBM_BINARY], - fakeStream, - false, /* don't normalize new lines in binary */ - undefined /* no progress task */); + expect(zosmfGetFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfGetFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders: [ZosmfHeaders.X_IBM_BINARY], + responseStream: fakeStream, + normalizeResponseNewLines: false, /* don't normalize new lines in binary */ + task: undefined /* no progress task */}); expect(ioCreateDirSpy).toHaveBeenCalledTimes(1); expect(ioCreateDirSpy).toHaveBeenCalledWith(file); diff --git a/packages/zosfiles/src/api/doc/IOptionsFullResponse.ts b/packages/zosfiles/src/api/doc/IOptionsFullResponse.ts new file mode 100644 index 0000000000..cbd8ace04e --- /dev/null +++ b/packages/zosfiles/src/api/doc/IOptionsFullResponse.ts @@ -0,0 +1,85 @@ +/* +* This program and the accompanying materials are made available under the terms of the +* Eclipse Public License v2.0 which accompanies this distribution, and is available at +* https://www.eclipse.org/legal/epl-v20.html +* +* SPDX-License-Identifier: EPL-2.0 +* +* Copyright Contributors to the Zowe Project. +* +*/ + +import { Writable, Readable } from "stream"; +import { ITaskWithStatus } from "@zowe/imperative"; +import { CLIENT_PROPERTY } from "./types/ZosmfRestClientProperties"; +/** + * Interface to define input options for RestClient GET|POST|PUT|DELETE ExpectFullResponse methods + * @export + * @interface IOptionsFullResponse + */ +export interface IOptionsFullResponse { + + /** + * URI for this request + * @type {string} + * @memberof IOptionsFullResponse + */ + resource: string; + /** + * List of properties to return from REST call + * @type {CLIENT_PROPERTY[]} + * @memberof IFullResponseOptions + */ + dataToReturn?: CLIENT_PROPERTY[]; + + /** + * Headers to include with request + * @type {any[]} + * @memberof IOptionsFullResponse + */ + reqHeaders?: any[]; + + /** + * Data to write on this REST request + * @type {*} + * @memberof IOptionsFullResponse + */ + writeData?: any; + + /** + * Stream for incoming response data from the server. If specified, response data will not be buffered + * @type {Writable} + * @memberof IOptionsFullResponse + */ + responseStream?: Writable; + + /** + * Stream for outgoing request data to the server + * @type {Readable} + * @memberof IOptionsFullResponse + */ + requestStream?: Readable; + + /** + * true if you want newlines to be \r\n on windows + * when receiving data from the server to responseStream. Don't set this for binary responses + * @type {boolean} + * @memberof IOptionsFullResponse + */ + normalizeResponseNewLines?: boolean; + + /** + * true if you want \r\n to be replaced with \n when sending + * data to the server from requestStream. Don't set this for binary requests + * @type {boolean} + * @memberof IOptionsFullResponse + */ + normalizeRequestNewLines?: boolean; + + /** + * Task that will automatically be updated to report progress of upload or download to user + * @type {ITaskWithStatus} + * @memberof IOptionsFullResponse + */ + task?: ITaskWithStatus; +} diff --git a/packages/zosfiles/src/api/doc/IRestClientResponse.ts b/packages/zosfiles/src/api/doc/IRestClientResponse.ts new file mode 100644 index 0000000000..31efe05774 --- /dev/null +++ b/packages/zosfiles/src/api/doc/IRestClientResponse.ts @@ -0,0 +1,71 @@ + +/* +* This program and the accompanying materials are made available under the terms of the +* Eclipse Public License v2.0 which accompanies this distribution, and is available at +* https://www.eclipse.org/legal/epl-v20.html +* +* SPDX-License-Identifier: EPL-2.0 +* +* Copyright Contributors to the Zowe Project. +* +*/ + +import { Session, Logger } from "@zowe/imperative"; + +/** + * Interface to map client's REST call response + * @export + * @interface IRestClientResponse + */ +export interface IRestClientResponse { + + /** + * Status whether or not a REST request was successful by HTTP status code + * @type {boolean} + * @memberof IRestClientResponse + */ + requestSuccess?: boolean; + + /** + * Status whether or not a REST request was successful by HTTP status code + * Reverse of requestSuccess + * @type {boolean} + * @memberof IRestClientResponse + */ + requestFailure?: boolean; + + /** + * Http(s) response body as a buffer + * @type {Buffer} + * @memberof IRestClientResponse + */ + data?: Buffer; + + /** + * Http(s) response body as a string + * @type {string} + * @memberof IRestClientResponse + */ + dataString?: string; + + /** + * Http(s) response object + * @type {any} + * @memberof IRestClientResponse + */ + response?: any; + + /** + * Session object + * @type {Session} + * @memberof IRestClientResponse + */ + session?: Session; + + /** + * Logger object + * @type {Logger} + * @memberof IRestClientResponse + */ + log?: Logger; +} diff --git a/packages/zosfiles/src/api/doc/types/ZosmfRestClientProperties.ts b/packages/zosfiles/src/api/doc/types/ZosmfRestClientProperties.ts new file mode 100644 index 0000000000..a34c541b05 --- /dev/null +++ b/packages/zosfiles/src/api/doc/types/ZosmfRestClientProperties.ts @@ -0,0 +1,28 @@ + +/* +* This program and the accompanying materials are made available under the terms of the +* Eclipse Public License v2.0 which accompanies this distribution, and is available at +* https://www.eclipse.org/legal/epl-v20.html +* +* SPDX-License-Identifier: EPL-2.0 +* +* Copyright Contributors to the Zowe Project. +* +*/ + +/** + * String type definition for properties of abstractRestClient that have a getter set. + * This can be safely used in a getter call as a variable for the abstractRestClient object. + * @export + * @typedef CLIENT_PROPERTY + */ +export type CLIENT_PROPERTY = "requestSuccess" | "requestFailure" | "data" | "dataString" | "response" | "session" | "log"; +export const CLIENT_PROPERTY = { + requestSuccess: "requestSuccess" as CLIENT_PROPERTY, + requestFailure: "requestFailure" as CLIENT_PROPERTY, + data: "data" as CLIENT_PROPERTY, + dataString: "dataString" as CLIENT_PROPERTY, + response: "response" as CLIENT_PROPERTY, + session: "session" as CLIENT_PROPERTY, + log: "log" as CLIENT_PROPERTY, +}; diff --git a/packages/zosfiles/src/api/index.ts b/packages/zosfiles/src/api/index.ts index 3e85704f95..bd928697f1 100644 --- a/packages/zosfiles/src/api/index.ts +++ b/packages/zosfiles/src/api/index.ts @@ -28,7 +28,10 @@ export * from "./utils/ZosFilesUtils"; export * from "./doc/IDataSet"; export * from "./doc/IZosFilesResponse"; +export * from "./doc/IOptionsFullResponse"; +export * from "./doc/IRestClientResponse"; export * from "./doc/types/ZosmfMigratedRecallOptions"; +export * from "./doc/types/ZosmfRestClientProperties"; export * from "./constants/ZosFiles.constants"; export * from "./constants/ZosFiles.messages"; diff --git a/packages/zosfiles/src/api/methods/download/Download.ts b/packages/zosfiles/src/api/methods/download/Download.ts index 83054b3219..c3876c9f7f 100644 --- a/packages/zosfiles/src/api/methods/download/Download.ts +++ b/packages/zosfiles/src/api/methods/download/Download.ts @@ -27,6 +27,9 @@ import { Get } from "../get/Get"; import { asyncPool } from "../../../../../utils"; import { IGetOptions } from "../get"; import { Writable } from "stream"; +import { IRestClientResponse } from "../../doc/IRestClientResponse"; +import { CLIENT_PROPERTY } from "../../doc/types/ZosmfRestClientProperties"; +import { IOptionsFullResponse } from "../../doc/IOptionsFullResponse"; /** * This class holds helper functions that are used to download data sets, members and more through the z/OS MF APIs @@ -97,11 +100,37 @@ export class Download { IO.createDirsSyncFromFilePath(destination); const writeStream = IO.createWriteStream(destination); - await ZosmfRestClient.getStreamed(session, endpoint, reqHeaders, writeStream, !options.binary, options.task); + // await ZosmfRestClient.getStreamed(session, endpoint, reqHeaders, writeStream, !options.binary, options.task); + + // Use specific options to mimic ZosmfRestClient.getStreamed() + const requestOptions: IOptionsFullResponse = { + resource: endpoint, + reqHeaders, + responseStream: writeStream, + normalizeResponseNewLines: !options.binary, + task: options.task, + }; + + // If requestor needs etag, add header + get "response" back + if (options.returnEtag) { + requestOptions.reqHeaders.push(ZosmfHeaders.X_IBM_RETURN_ETAG); + requestOptions.dataToReturn = [CLIENT_PROPERTY.response]; + } + + const request: IRestClientResponse = await ZosmfRestClient.getExpectFullResponse(session, requestOptions); + + // By default, apiResponse is empty when downloading + const apiResponse: any = {}; + + // Return Etag in apiResponse, if requested + if (options.returnEtag) { + apiResponse.etag = request.response.headers.etag; + } + return { success: true, commandResponse: util.format(ZosFilesMessages.datasetDownloadedSuccessfully.message, destination), - apiResponse: {} + apiResponse }; } catch (error) { Logger.getAppLogger().error(error); @@ -245,11 +274,35 @@ export class Download { reqHeaders = [ZosmfHeaders.X_IBM_BINARY]; } - await ZosmfRestClient.getStreamed(session, endpoint, reqHeaders, writeStream, !options.binary, options.task); + // Use specific options to mimic ZosmfRestClient.getStreamed() + const requestOptions: IOptionsFullResponse = { + resource: endpoint, + reqHeaders, + responseStream: writeStream, + normalizeResponseNewLines: !options.binary, + task: options.task, + }; + + // If requestor needs etag, add header + get "response" back + if (options.returnEtag) { + requestOptions.reqHeaders.push(ZosmfHeaders.X_IBM_RETURN_ETAG); + requestOptions.dataToReturn = [CLIENT_PROPERTY.response]; + } + + // const test = await ZosmfRestClient.getStreamed(session, endpoint, reqHeaders, writeStream, !options.binary, options.task); + const request = await ZosmfRestClient.getExpectFullResponse(session, requestOptions); + + // By default, apiResponse is empty when downloading + const apiResponse: any = {}; + + // Return Etag in apiResponse, if requested + if (options.returnEtag) { + apiResponse.etag = request.response.headers.etag; + } return { success: true, commandResponse: util.format(ZosFilesMessages.ussFileDownloadedSuccessfully.message, destination), - apiResponse: {} + apiResponse }; } catch (error) { Logger.getAppLogger().error(error); diff --git a/packages/zosfiles/src/api/methods/download/doc/IDownloadOptions.ts b/packages/zosfiles/src/api/methods/download/doc/IDownloadOptions.ts index fa8f979aa1..1468b1c22e 100644 --- a/packages/zosfiles/src/api/methods/download/doc/IDownloadOptions.ts +++ b/packages/zosfiles/src/api/methods/download/doc/IDownloadOptions.ts @@ -74,4 +74,12 @@ export interface IDownloadOptions { * Optional */ task?: ITaskWithStatus; + + /** + * The indicator to force return of ETag. + * If set to 'true' it forces the response to include an "ETag" header, regardless of the size of the response data. + * If it is not present, the the default is to only send an Etag for data sets smaller than a system determined length, + * which is at least 8MB. + */ + returnEtag?: boolean; } diff --git a/packages/zosfiles/src/api/methods/upload/Upload.ts b/packages/zosfiles/src/api/methods/upload/Upload.ts index a6ea4c5518..e93f64e91d 100644 --- a/packages/zosfiles/src/api/methods/upload/Upload.ts +++ b/packages/zosfiles/src/api/methods/upload/Upload.ts @@ -29,6 +29,10 @@ import { asyncPool } from "../../../../../utils"; import { ZosFilesAttributes, TransferMode } from "../../utils/ZosFilesAttributes"; import { Utilities, Tag } from "../utilities"; import { Readable } from "stream"; +import { IOptionsFullResponse } from "../../doc/IOptionsFullResponse"; +import { IRestClientResponse } from "../../doc/IRestClientResponse"; +import { CLIENT_PROPERTY } from "../../doc/types/ZosmfRestClientProperties"; + export class Upload { @@ -137,13 +141,13 @@ export class Upload { } /** - * Writting data buffer to a data set. + * Writing data buffer to a data set. * @param {AbstractSession} session - z/OS connection info * @param {Buffer} fileBuffer - Data buffer to be written * @param {string} dataSetName - Name of the data set to write to * @param {IUploadOptions} [options={}] - Uploading options * - * @return {Promise} A response indicating the out come + * @return {Promise} A response indicating the outcome * * @throws {ImperativeError} When encounter error scenarios. */ @@ -163,11 +167,11 @@ export class Upload { endpoint = path.posix.join(endpoint, dataSetName); // Construct request header parameters - const reqHeader: IHeaderContent[] = []; + const reqHeaders: IHeaderContent[] = []; if (options.binary) { - reqHeader.push(ZosmfHeaders.X_IBM_BINARY); + reqHeaders.push(ZosmfHeaders.X_IBM_BINARY); } else { - reqHeader.push(ZosmfHeaders.X_IBM_TEXT); + reqHeaders.push(ZosmfHeaders.X_IBM_TEXT); fileBuffer = ZosFilesUtils.normalizeNewline(fileBuffer); } @@ -175,25 +179,50 @@ export class Upload { if (options.recall) { switch (options.recall.toLowerCase()) { case "wait": - reqHeader.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_WAIT); + reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_WAIT); break; case "nowait": - reqHeader.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT); + reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT); break; case "error": - reqHeader.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_ERROR); + reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_ERROR); break; default: - reqHeader.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT); + reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT); break; } } - await ZosmfRestClient.putExpectString(session, endpoint, reqHeader, fileBuffer); + if (options.etag) { + reqHeaders.push({"If-Match" : options.etag}); + } + + // Options to use the buffer to write a file + const requestOptions: IOptionsFullResponse = { + resource: endpoint, + reqHeaders, + writeData: fileBuffer + }; + + // If requestor needs etag, add header + get "response" back + if (options.returnEtag) { + requestOptions.reqHeaders.push(ZosmfHeaders.X_IBM_RETURN_ETAG); + requestOptions.dataToReturn = [CLIENT_PROPERTY.response]; + } + const uploadRequest: IRestClientResponse = await ZosmfRestClient.putExpectFullResponse(session, requestOptions); + + // By default, apiResponse is empty when uploading + const apiResponse: any = {}; + + // Return Etag in apiResponse, if requested + if (options.returnEtag) { + apiResponse.etag = uploadRequest.response.headers.etag; + } return { success: true, - commandResponse: ZosFilesMessages.dataSetUploadedSuccessfully.message + commandResponse: ZosFilesMessages.dataSetUploadedSuccessfully.message, + apiResponse }; } catch (error) { throw error; @@ -201,7 +230,7 @@ export class Upload { } /** - * Writting data buffer to a data set. + * Writing data buffer to a data set. * @param {AbstractSession} session - z/OS connection info * @param {Buffer} fileBuffer - Data buffer to be written * @param {string} dataSetName - Name of the data set to write to @@ -227,39 +256,66 @@ export class Upload { endpoint = path.posix.join(endpoint, dataSetName); // Construct request header parameters - const reqHeader: IHeaderContent[] = []; + const reqHeaders: IHeaderContent[] = []; if (options.binary) { - reqHeader.push(ZosmfHeaders.X_IBM_BINARY); + reqHeaders.push(ZosmfHeaders.X_IBM_BINARY); } else { - reqHeader.push(ZosmfHeaders.X_IBM_TEXT); + reqHeaders.push(ZosmfHeaders.X_IBM_TEXT); } // Migrated recall options if (options.recall) { switch (options.recall.toLowerCase()) { case "wait": - reqHeader.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_WAIT); + reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_WAIT); break; case "nowait": - reqHeader.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT); + reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT); break; case "error": - reqHeader.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_ERROR); + reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_ERROR); break; default: - reqHeader.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT); + reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT); break; } } + if (options.etag) { + reqHeaders.push({"If-Match" : options.etag}); + } + + const requestOptions: IOptionsFullResponse = { + resource: endpoint, + reqHeaders, + requestStream: fileStream, + normalizeRequestNewLines: !options.binary /* only normalize newlines if we are not uploading in binary*/, + task: options.task + }; + + // If requestor needs etag, add header + get "response" back + if (options.returnEtag) { + requestOptions.reqHeaders.push(ZosmfHeaders.X_IBM_RETURN_ETAG); + requestOptions.dataToReturn = [CLIENT_PROPERTY.response]; + } + + const uploadRequest: IRestClientResponse = await ZosmfRestClient.putExpectFullResponse(session, requestOptions); + // await ZosmfRestClient.putStreamedRequestOnly(session, endpoint, reqHeaders, fileStream, + // !options.binary /* only normalize newlines if we are not uploading in binary*/, + // options.task); + + // By default, apiResponse is empty when uploading + const apiResponse: any = {}; - await ZosmfRestClient.putStreamedRequestOnly(session, endpoint, reqHeader, fileStream, - !options.binary /* only normalize newlines if we are not uploading in binary*/, - options.task); + // Return Etag in apiResponse, if requested + if (options.returnEtag) { + apiResponse.etag = uploadRequest.response.headers.etag; + } return { success: true, - commandResponse: ZosFilesMessages.dataSetUploadedSuccessfully.message + commandResponse: ZosFilesMessages.dataSetUploadedSuccessfully.message, + apiResponse }; } catch (error) { throw error; @@ -285,7 +341,7 @@ export class Upload { * * Note: * This method does everything needed to do from checking if path is file or directory - * and if data set is sequential file or PDS to determind what name to be used when + * and if data set is sequential file or PDS to determine what name to be used when * upload content to data set. All you have to specify is a directory and a dsname. */ public static async pathToDataSet(session: AbstractSession, @@ -407,6 +463,14 @@ export class Upload { } const result = await this.streamToDataSet(session, uploadStream, uploadingDsn, streamUploadOptions); this.log.info(`Success Uploaded data From ${uploadingFile} To ${uploadingDsn}`); + const toBePushed: IUploadResult = { + success: result.success, + from: uploadingFile, + to: uploadingDsn + }; + if (options.returnEtag) { + toBePushed.etag = result.apiResponse.etag; + } results.push({ success: result.success, from: uploadingFile, @@ -454,72 +518,156 @@ export class Upload { * @param {AbstractSession} session - z/OS connection info * @param {string} ussname - Name of the USS file to write to * @param {Buffer} buffer - Data to be written - * @param {boolean} binary - The indicator to upload the file in binary mode + * @param {IUploadOptions} [options={}] - Uploading options * @returns {Promise} */ - public static async bufferToUSSFile(session: AbstractSession, + public static async bufferToUssFile(session: AbstractSession, ussname: string, buffer: Buffer, - binary: boolean = false, - localEncoding?: string) { + options: IUploadOptions = {}) { + options.binary = false; ImperativeExpect.toNotBeNullOrUndefined(ussname, ZosFilesMessages.missingUSSFileName.message); ussname = ZosFilesUtils.sanitizeUssPathForRestCall(ussname); const parameters: string = ZosFilesConstants.RES_USS_FILES + "/" + ussname; const headers: any[] = []; - if (binary) { + if (options.binary) { headers.push(ZosmfHeaders.OCTET_STREAM); headers.push(ZosmfHeaders.X_IBM_BINARY); - } else if (localEncoding) { - headers.push({"Content-Type": localEncoding}); + } else if (options.localEncoding) { + headers.push({"Content-Type": options.localEncoding}); headers.push(ZosmfHeaders.X_IBM_TEXT); } else { headers.push(ZosmfHeaders.TEXT_PLAIN); } + if (options.etag) { + headers.push({"If-Match" : options.etag}); + } + return ZosmfRestClient.putExpectString(session, ZosFilesConstants.RESOURCE + parameters, headers, buffer); } - /** * Upload content to USS file + * @deprecated In favor of bufferToUssFile() which implements IUploadOptions * @param {AbstractSession} session - z/OS connection info * @param {string} ussname - Name of the USS file to write to - * @param {Buffer} uploadStream - Data to be written + * @param {Buffer} buffer - Data to be written * @param {boolean} binary - The indicator to upload the file in binary mode * @returns {Promise} */ - public static async streamToUSSFile(session: AbstractSession, + public static async bufferToUSSFile(session: AbstractSession, ussname: string, - uploadStream: Readable, + buffer: Buffer, binary: boolean = false, localEncoding?: string, - task?: ITaskWithStatus) { + etag?: string, + returnEtag?: boolean) { + return this.bufferToUssFile(session, ussname, buffer, { + binary, + localEncoding, + etag, + returnEtag + }); + } + + /** + * Upload content to USS file + * @param {AbstractSession} session - z/OS connection info + * @param {string} ussname - Name of the USS file to write to + * @param {Buffer} uploadStream - Data to be written + * @param {IUploadOptions} [options={}] - Uploading options + * @returns {Promise} - A response indicating the outcome + */ + + public static async streamToUssFile(session: AbstractSession, + ussname: string, + uploadStream: Readable, + options: IUploadOptions = {}) { ImperativeExpect.toNotBeNullOrUndefined(ussname, ZosFilesMessages.missingUSSFileName.message); ussname = path.posix.normalize(ussname); ussname = ZosFilesUtils.formatUnixFilepath(ussname); ussname = encodeURIComponent(ussname); const parameters: string = ZosFilesConstants.RES_USS_FILES + "/" + ussname; const headers: any[] = []; - if (binary) { + if (options.binary) { headers.push(ZosmfHeaders.OCTET_STREAM); headers.push(ZosmfHeaders.X_IBM_BINARY); - } else if (localEncoding) { - headers.push({"Content-Type": localEncoding}); + } else if (options.localEncoding) { + headers.push({"Content-Type": options.localEncoding}); headers.push(ZosmfHeaders.X_IBM_TEXT); } else { headers.push(ZosmfHeaders.TEXT_PLAIN); } - await ZosmfRestClient.putStreamedRequestOnly(session, ZosFilesConstants.RESOURCE + parameters, headers, uploadStream, - !binary /* only normalize newlines if we are not in binary mode*/, - task); + if (options.etag) { + headers.push({"If-Match" : options.etag}); + } + + // Options to use the stream to write a file + const restOptions: IOptionsFullResponse = { + resource: ZosFilesConstants.RESOURCE + parameters, + reqHeaders: headers, + requestStream: uploadStream, + normalizeRequestNewLines: !options.binary /* only normalize newlines if we are not in binary mode*/ + }; + + // If requestor needs etag, add header + get "response" back + if (options.returnEtag) { + restOptions.reqHeaders.push(ZosmfHeaders.X_IBM_RETURN_ETAG); + restOptions.dataToReturn = [CLIENT_PROPERTY.response]; + } + const uploadRequest: IRestClientResponse = await ZosmfRestClient.putExpectFullResponse(session, restOptions); + + // By default, apiResponse is empty when uploading + const apiResponse: any = {}; + + // Return Etag in apiResponse, if requested + if (options.returnEtag) { + apiResponse.etag = uploadRequest.response.headers.etag; + } + return { + success: true, + commandResponse: ZosFilesMessages.dataSetUploadedSuccessfully.message, + apiResponse + }; } - public static async fileToUSSFile(session: AbstractSession, + /** + * Upload content to USS file + * @deprecated - In favor of streamToUssFile() which implements IUploadOptions + * @param {AbstractSession} session - z/OS connection info + * @param {string} ussname - Name of the USS file to write to + * @param {Buffer} uploadStream - Data to be written + * @param {boolean} binary - The indicator to upload the file in binary mode + * @returns {Promise} + */ + public static async streamToUSSFile(session: AbstractSession, + ussname: string, + uploadStream: Readable, + binary: boolean = false, + localEncoding?: string, + task?: ITaskWithStatus, + etag?: string) { + return this.streamToUssFile(session, ussname, uploadStream, { + binary, + localEncoding, + task, + etag + }); + } + + /** + * Upload content from a local file to remote USS file + * @param session - z/OS connection info + * @param inputFile - Path to local file + * @param ussname - Name of USS file to write to + * @param options - Uploading options + * @returns {Promise} - A response indicating the outcome + */ + public static async fileToUssFile(session: AbstractSession, inputFile: string, ussname: string, - binary: boolean = false, - localEncoding?: string, - task?: ITaskWithStatus): Promise { + options: IUploadOptions = {}): Promise { ImperativeExpect.toNotBeNullOrUndefined(inputFile, ZosFilesMessages.missingInputFile.message); ImperativeExpect.toNotBeNullOrUndefined(ussname, ZosFilesMessages.missingUSSFileName.message); ImperativeExpect.toNotBeEqual(ussname, "", ZosFilesMessages.missingUSSFileName.message); @@ -551,12 +699,18 @@ export class Upload { let result: IUploadResult; // read payload from file const uploadStream = IO.createReadStream(inputFile); - await this.streamToUSSFile(session, ussname, uploadStream, binary, localEncoding, task); + + const request = await this.streamToUssFile(session, ussname, uploadStream, options); result = { success: true, from: inputFile, to: ussname }; + + // Return Etag in apiResponse, if requested + if (options.returnEtag) { + result.etag = request.apiResponse.etag; + } return { success: true, commandResponse: ZosFilesMessages.ussFileUploadedSuccessfully.message, @@ -564,6 +718,33 @@ export class Upload { }; } + /** + * @deprecated In favor of fileToUssFile() which implements IUploadOptions + * @param session + * @param inputFile + * @param ussname + * @param binary + * @param localEncoding + * @param task + * @param etag + */ + public static async fileToUSSFile(session: AbstractSession, + inputFile: string, + ussname: string, + binary: boolean = false, + localEncoding?: string, + task?: ITaskWithStatus, + etag?: string, + returnEtag?: boolean): Promise { + return this.fileToUssFile(session, inputFile, ussname, { + binary, + localEncoding, + task, + etag, + returnEtag + }); + } + /** * Upload local directory to USS directory * @param {AbstractSession} session - z/OS connection info @@ -826,7 +1007,7 @@ export class Upload { else { tempBinary = options.binary; } - await this.fileToUSSFile(session, localPath, ussPath, tempBinary); + await this.fileToUssFile(session, localPath, ussPath, {binary: tempBinary}); } } @@ -837,9 +1018,9 @@ export class Upload { if (attributes.fileShouldBeUploaded(localPath)) { const binary = attributes.getFileTransferMode(localPath) === TransferMode.BINARY; if (binary) { - await this.fileToUSSFile(session, localPath, ussPath, binary); + await this.fileToUssFile(session, localPath, ussPath, {binary}); } else { - await this.fileToUSSFile(session, localPath, ussPath, binary, attributes.getLocalEncoding(localPath)); + await this.fileToUssFile(session, localPath, ussPath, {binary, localEncoding: attributes.getLocalEncoding(localPath)}); } const tag = attributes.getRemoteEncoding(localPath); diff --git a/packages/zosfiles/src/api/methods/upload/doc/IUploadOptions.ts b/packages/zosfiles/src/api/methods/upload/doc/IUploadOptions.ts index 26a2e031e7..58e0fcad98 100644 --- a/packages/zosfiles/src/api/methods/upload/doc/IUploadOptions.ts +++ b/packages/zosfiles/src/api/methods/upload/doc/IUploadOptions.ts @@ -73,4 +73,23 @@ export interface IUploadOptions { * Default: 1 */ maxConcurrentRequests?: number; + + /** + * Etag value to pass to z/OSMF API request. + * It is used to check if the file was modified on target system before it is updated. + */ + etag?: string; + + /** + * The local file encoding to pass as a "Content-Type" header + */ + localEncoding?: string; + + /** + * The indicator to force return of ETag. + * If set to 'true' it forces the response to include an "ETag" header, regardless of the size of the response data. + * If it is not present, the the default is to only send an Etag for data sets smaller than a system determined length, + * which is at least 8MB. + */ + returnEtag?: boolean; } diff --git a/packages/zosfiles/src/api/methods/upload/doc/IUploadResult.ts b/packages/zosfiles/src/api/methods/upload/doc/IUploadResult.ts index 997de10a8d..9608ed62e9 100644 --- a/packages/zosfiles/src/api/methods/upload/doc/IUploadResult.ts +++ b/packages/zosfiles/src/api/methods/upload/doc/IUploadResult.ts @@ -26,4 +26,8 @@ export interface IUploadResult { * Optional, any error encounter while uploading the data */ error?: any; + /** + * Optional, etag set when writing the file + */ + etag?: string; } diff --git a/packages/zosfiles/src/api/methods/upload/index.ts b/packages/zosfiles/src/api/methods/upload/index.ts index 2248682c47..adc07bdb92 100644 --- a/packages/zosfiles/src/api/methods/upload/index.ts +++ b/packages/zosfiles/src/api/methods/upload/index.ts @@ -10,5 +10,6 @@ */ export * from "./doc/IUploadOptions"; +export * from "./doc/IUploadResult"; export * from "./Upload"; From b88041a1608a674d200df6db30b7f5265416c9fe Mon Sep 17 00:00:00 2001 From: Alexandru-Paul Dumitru Date: Wed, 22 Jan 2020 16:57:30 +0100 Subject: [PATCH 2/9] add unit tests for download + upload fix small bugs in upload Signed-off-by: Alexandru-Paul Dumitru --- .../methods/download/Download.unit.test.ts | 79 +++++++- .../api/methods/upload/Upload.unit.test.ts | 176 +++++++++++++++--- .../zosfiles/src/api/methods/upload/Upload.ts | 8 +- 3 files changed, 225 insertions(+), 38 deletions(-) diff --git a/packages/zosfiles/__tests__/api/methods/download/Download.unit.test.ts b/packages/zosfiles/__tests__/api/methods/download/Download.unit.test.ts index 43cc17f528..d39581cbd6 100644 --- a/packages/zosfiles/__tests__/api/methods/download/Download.unit.test.ts +++ b/packages/zosfiles/__tests__/api/methods/download/Download.unit.test.ts @@ -17,6 +17,7 @@ import { posix } from "path"; import { ZosFilesConstants } from "../../../../src/api/constants/ZosFiles.constants"; import * as util from "util"; import { List } from "../../../../src/api/methods/list"; +import { CLIENT_PROPERTY } from "../../../../src/api/doc/types/ZosmfRestClientProperties"; describe("z/OS Files - Download", () => { const dsname = "USER.DATA.SET"; @@ -26,7 +27,7 @@ describe("z/OS Files - Download", () => { const arrOfUssPath: string[] = ussname.split("/"); const localFileName = arrOfUssPath[arrOfUssPath.length - 1]; const ussFileContent = "Test data for unit test"; - const etag = "123ABC"; + const etagValue = "123ABC"; const dummySession = new Session({ user: "fake", @@ -44,7 +45,7 @@ describe("z/OS Files - Download", () => { const ioWriteStreamSpy = jest.spyOn(IO, "createWriteStream"); const fakeWriteStream: any = {fakeWriteStream: true}; const zosmfGetFullSpy = jest.spyOn(ZosmfRestClient, "getExpectFullResponse"); - const fakeResponseWithEtag = {data: ussFileContent, response:{headers:{etag}}}; + const fakeResponseWithEtag = {data: ussFileContent, response:{headers:{etag: etagValue}}}; beforeEach(() => { zosmfStreamSpy.mockClear(); @@ -244,6 +245,44 @@ describe("z/OS Files - Download", () => { expect(ioWriteStreamSpy).toHaveBeenCalledWith(file); }); + it("should download a data set and return Etag", async () => { + zosmfGetFullSpy.mockImplementationOnce(() => fakeResponseWithEtag); + let response; + let caughtError; + const volume = "testVs"; + const extension = ".test"; + const destination = dsFolder + extension; + + + try { + response = await Download.dataSet(dummySession, dsname, {volume, extension, returnEtag: true}); + } catch (e) { + caughtError = e; + } + const endpoint = posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_DS_FILES, `-(${volume})`, dsname); + + expect(caughtError).toBeUndefined(); + expect(response).toEqual({ + success: true, + commandResponse: util.format(ZosFilesMessages.datasetDownloadedSuccessfully.message, destination), + apiResponse: {etag: etagValue} + }); + + + expect(zosmfGetFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfGetFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders: [ZosmfHeaders.X_IBM_RETURN_ETAG], + responseStream: fakeWriteStream, + normalizeResponseNewLines: true, + task: undefined, + dataToReturn: [CLIENT_PROPERTY.response]}); // import and use proper property + + expect(ioCreateDirSpy).toHaveBeenCalledTimes(1); + expect(ioCreateDirSpy).toHaveBeenCalledWith(destination); + + expect(ioWriteStreamSpy).toHaveBeenCalledTimes(1); + }); + it("should handle a z/OS MF error", async () => { let response; let caughtError; @@ -550,7 +589,7 @@ describe("z/OS Files - Download", () => { const ioWriteStreamSpy = jest.spyOn(IO, "createWriteStream"); const fakeStream: any = {fakeStream: true}; const zosmfGetFullSpy = jest.spyOn(ZosmfRestClient, "getExpectFullResponse"); - const fakeResponseWithEtag = {data: ussFileContent, response:{headers:{etag}}}; + const fakeResponseWithEtag = {data: ussFileContent, response:{headers:{etag: etagValue}}}; beforeEach(() => { zosmfStreamSpy.mockClear(); @@ -711,5 +750,39 @@ describe("z/OS Files - Download", () => { expect(ioWriteStreamSpy).toHaveBeenCalledTimes(1); expect(ioWriteStreamSpy).toHaveBeenCalledWith(file); }); + + it("should download uss file and return Etag", async () => { + zosmfGetFullSpy.mockImplementationOnce(() => fakeResponseWithEtag); + let response; + let caughtError; + const destination = localFileName; + try { + response = await Download.ussFile(dummySession, ussname, {returnEtag: true}); + } catch (e) { + caughtError = e; + } + + const endpoint = posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_USS_FILES, encodeURIComponent(ussname.substr(1))); + + expect(caughtError).toBeUndefined(); + expect(response).toEqual({ + success: true, + commandResponse: util.format(ZosFilesMessages.ussFileDownloadedSuccessfully.message, destination), + apiResponse: {etag: etagValue} + }); + + expect(zosmfGetFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfGetFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders: [ZosmfHeaders.X_IBM_RETURN_ETAG], + responseStream: fakeStream, + normalizeResponseNewLines: true, + dataToReturn: [CLIENT_PROPERTY.response]}); + + expect(ioCreateDirSpy).toHaveBeenCalledTimes(1); + expect(ioCreateDirSpy).toHaveBeenCalledWith(destination); + + expect(ioWriteStreamSpy).toHaveBeenCalledTimes(1); + expect(ioWriteStreamSpy).toHaveBeenCalledWith(destination); + }); }); }); diff --git a/packages/zosfiles/__tests__/api/methods/upload/Upload.unit.test.ts b/packages/zosfiles/__tests__/api/methods/upload/Upload.unit.test.ts index ada057a4f6..6f9f159736 100644 --- a/packages/zosfiles/__tests__/api/methods/upload/Upload.unit.test.ts +++ b/packages/zosfiles/__tests__/api/methods/upload/Upload.unit.test.ts @@ -28,10 +28,12 @@ import { ZosFilesUtils } from "../../../../src/api/utils/ZosFilesUtils"; import { stripNewLines } from "../../../../../../__tests__/__src__/TestUtils"; import { Create } from "../../../../src/api/methods/create"; import { ZosFilesAttributes, TransferMode, Tag } from "../../../../src/api"; +import { CLIENT_PROPERTY } from "../../../../src/api/doc/types/ZosmfRestClientProperties"; describe("z/OS Files - Upload", () => { const dsName = "UNIT.TEST"; + const etagValue = "123ABC"; const dummySession = new Session({ user: "fake", password: "fake", @@ -246,12 +248,17 @@ describe("z/OS Files - Upload", () => { describe("bufferToDataSet", () => { const zosmfExpectSpy = jest.spyOn(ZosmfRestClient, "putExpectString"); + const zosmfPutFullSpy = jest.spyOn(ZosmfRestClient, "putExpectFullResponse"); + const fakeResponseWithEtag = {data: dsName, response:{headers:{etag: etagValue}}}; beforeEach(() => { response = undefined; error = undefined; zosmfExpectSpy.mockClear(); zosmfExpectSpy.mockImplementation(() => null); + + zosmfPutFullSpy.mockClear(); + zosmfPutFullSpy.mockImplementation(() => null); }); it("should throw error if data set name is not specified", async () => { @@ -273,7 +280,7 @@ describe("z/OS Files - Upload", () => { msg: "test error" }); - zosmfExpectSpy.mockRejectedValueOnce(testError); + zosmfPutFullSpy.mockRejectedValueOnce(testError); try { response = await Upload.bufferToDataSet(dummySession, buffer, dsName); @@ -288,7 +295,7 @@ describe("z/OS Files - Upload", () => { it("return with proper response when upload buffer to a data set", async () => { const buffer: Buffer = Buffer.from("testing"); const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_DS_FILES, dsName); - const options = [ZosmfHeaders.X_IBM_TEXT]; + const reqHeaders = [ZosmfHeaders.X_IBM_TEXT]; try { response = await Upload.bufferToDataSet(dummySession, buffer, dsName); @@ -299,14 +306,16 @@ describe("z/OS Files - Upload", () => { expect(error).toBeUndefined(); expect(response).toBeDefined(); - expect(zosmfExpectSpy).toHaveBeenCalledTimes(1); - expect(zosmfExpectSpy).toHaveBeenCalledWith(dummySession, endpoint, options, buffer); + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders, + writeData: buffer}); }); it("return with proper response when upload buffer to a PDS member", async () => { const buffer: Buffer = Buffer.from("testing"); const testDsName = `${dsName}(member)`; const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_DS_FILES, testDsName); - const options = [ZosmfHeaders.X_IBM_TEXT]; + const reqHeaders = [ZosmfHeaders.X_IBM_TEXT]; try { response = await Upload.bufferToDataSet(dummySession, buffer, testDsName); @@ -317,8 +326,10 @@ describe("z/OS Files - Upload", () => { expect(error).toBeUndefined(); expect(response).toBeDefined(); - expect(zosmfExpectSpy).toHaveBeenCalledTimes(1); - expect(zosmfExpectSpy).toHaveBeenCalledWith(dummySession, endpoint, options, buffer); + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource:endpoint, + reqHeaders, + writeData: buffer}); }); it("return with proper response when upload buffer to a data set with optional parameters", async () => { const buffer: Buffer = Buffer.from("testing"); @@ -326,7 +337,7 @@ describe("z/OS Files - Upload", () => { binary: true }; const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_DS_FILES, dsName); - let options = [ZosmfHeaders.X_IBM_BINARY]; + let reqHeaders = [ZosmfHeaders.X_IBM_BINARY]; try { response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); @@ -337,13 +348,15 @@ describe("z/OS Files - Upload", () => { expect(error).toBeUndefined(); expect(response).toBeDefined(); - expect(zosmfExpectSpy).toHaveBeenCalledTimes(1); - expect(zosmfExpectSpy).toHaveBeenCalledWith(dummySession, endpoint, options, buffer); - zosmfExpectSpy.mockClear(); + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders, + writeData: buffer}); + zosmfPutFullSpy.mockClear(); // Unit test for wait option uploadOptions.recall = "wait"; - options = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_WAIT]; + reqHeaders = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_WAIT]; try { response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); @@ -354,13 +367,15 @@ describe("z/OS Files - Upload", () => { expect(error).toBeUndefined(); expect(response).toBeDefined(); - expect(zosmfExpectSpy).toHaveBeenCalledTimes(1); - expect(zosmfExpectSpy).toHaveBeenCalledWith(dummySession, endpoint, options, buffer); - zosmfExpectSpy.mockClear(); + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders, + writeData: buffer}); + zosmfPutFullSpy.mockClear(); // Unit test for no wait option uploadOptions.recall = "nowait"; - options = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT]; + reqHeaders = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT]; try { response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); @@ -371,13 +386,15 @@ describe("z/OS Files - Upload", () => { expect(error).toBeUndefined(); expect(response).toBeDefined(); - expect(zosmfExpectSpy).toHaveBeenCalledTimes(1); - expect(zosmfExpectSpy).toHaveBeenCalledWith(dummySession, endpoint, options, buffer); - zosmfExpectSpy.mockClear(); + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders, + writeData: buffer}); + zosmfPutFullSpy.mockClear(); // Unit test for no error option uploadOptions.recall = "error"; - options = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_ERROR]; + reqHeaders = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_ERROR]; try { response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); @@ -388,13 +405,15 @@ describe("z/OS Files - Upload", () => { expect(error).toBeUndefined(); expect(response).toBeDefined(); - expect(zosmfExpectSpy).toHaveBeenCalledTimes(1); - expect(zosmfExpectSpy).toHaveBeenCalledWith(dummySession, endpoint, options, buffer); - zosmfExpectSpy.mockClear(); + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders, + writeData: buffer}); + zosmfPutFullSpy.mockClear(); // Unit test default value uploadOptions.recall = "non-existing"; - options = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT]; + reqHeaders = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT]; try { response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); @@ -405,13 +424,56 @@ describe("z/OS Files - Upload", () => { expect(error).toBeUndefined(); expect(response).toBeDefined(); - expect(zosmfExpectSpy).toHaveBeenCalledTimes(1); - expect(zosmfExpectSpy).toHaveBeenCalledWith(dummySession, endpoint, options, buffer); + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders, + writeData: buffer}); + zosmfPutFullSpy.mockClear(); + + // Unit test for pass etag option + uploadOptions.etag = etagValue; + reqHeaders = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT, {"If-Match" : uploadOptions.etag}]; + + try { + response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(response).toBeDefined(); + + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders, + writeData: buffer}); + zosmfPutFullSpy.mockClear(); + zosmfPutFullSpy.mockImplementationOnce(() => fakeResponseWithEtag); + // Unit test for return etag option + reqHeaders = [ZosmfHeaders.X_IBM_BINARY, + ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT, + {"If-Match" : uploadOptions.etag}, + ZosmfHeaders.X_IBM_RETURN_ETAG]; + uploadOptions.returnEtag = true; + try { + response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(response).toBeDefined(); + + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders, + writeData: buffer, + dataToReturn: [CLIENT_PROPERTY.response]}); }); it("return with proper response when upload dataset with specify volume option", async () => { const buffer: Buffer = Buffer.from("testing"); const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_DS_FILES, `-(TEST)`, dsName); - const options = [ZosmfHeaders.X_IBM_TEXT]; + const reqHeaders = [ZosmfHeaders.X_IBM_TEXT]; const uploadOptions: IUploadOptions = { volume: "TEST" }; @@ -424,8 +486,10 @@ describe("z/OS Files - Upload", () => { expect(error).toBeUndefined(); expect(response).toBeDefined(); - expect(zosmfExpectSpy).toHaveBeenCalledTimes(1); - expect(zosmfExpectSpy).toHaveBeenCalledWith(dummySession, endpoint, options, buffer); + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders, + writeData: buffer}); }); }); @@ -597,6 +661,44 @@ describe("z/OS Files - Upload", () => { expect(response.success).toBeTruthy(); expect(response.apiResponse[0].success).toBeTruthy(); }); + it("should return etag when requested", async () => { + const mockFileList = ["file1"]; + const mockListResponse: IZosFilesResponse = { + success: true, + commandResponse: "dummy response", + apiResponse: { + items: [{ + dsname: dsName, + dsorg: "PS" + }], + returnedRows: 1 + } + }; + listDatasetSpy.mockResolvedValueOnce(mockListResponse); + getFileListSpy.mockReturnValueOnce(mockFileList); + writeZosmfDatasetSpy.mockResolvedValueOnce({ + success: true, + CommandResponse: "dummy", + apiResponse: { + etag: etagValue + } + }); + const uploadOptions: IUploadOptions = { + returnEtag: true + }; + + try { + response = await Upload.pathToDataSet(dummySession, "dummyPath", dsName, uploadOptions); + } catch (err) { + error = err; + } + expect(error).toBeUndefined(); + expect(response).toBeDefined(); + expect(response.success).toBeTruthy(); + expect(response.apiResponse[0].success).toBeTruthy(); + expect(response.apiResponse[0].etag).toBeDefined(); + expect(response.apiResponse[0].etag).toEqual(etagValue); + }); it("should return information when successfully uploaded to a PDS member", async () => { const mockFileList = ["file1"]; const pdsMem = `${dsName}(MEMBER)`; @@ -775,7 +877,23 @@ describe("z/OS Files - Upload", () => { expect(zosmfExpectSpy).toHaveBeenCalledTimes(1); expect(zosmfExpectSpy).toHaveBeenCalledWith(dummySession, endpoint, headers, data); }); + it("return with proper response when upload USS file with Etag", async () => { + const data: Buffer = Buffer.from("testing"); + const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_USS_FILES, dsName); + const headers = [ZosmfHeaders.TEXT_PLAIN, {"If-Match": etagValue}]; + + try { + USSresponse = await Upload.bufferToUSSFile(dummySession, dsName, data, false, undefined, etagValue); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(USSresponse).toBeDefined(); + expect(zosmfExpectSpy).toHaveBeenCalledTimes(1); + expect(zosmfExpectSpy).toHaveBeenCalledWith(dummySession, endpoint, headers, data); + }); it("should set local encoding if specified", async () => { const data: Buffer = Buffer.from("testing"); const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_USS_FILES, dsName); diff --git a/packages/zosfiles/src/api/methods/upload/Upload.ts b/packages/zosfiles/src/api/methods/upload/Upload.ts index e93f64e91d..23e997aeb7 100644 --- a/packages/zosfiles/src/api/methods/upload/Upload.ts +++ b/packages/zosfiles/src/api/methods/upload/Upload.ts @@ -471,11 +471,7 @@ export class Upload { if (options.returnEtag) { toBePushed.etag = result.apiResponse.etag; } - results.push({ - success: result.success, - from: uploadingFile, - to: uploadingDsn - }); + results.push(toBePushed); } catch (err) { this.log.error(`Failure Uploading data From ${uploadingFile} To ${uploadingDsn}`); results.push({ @@ -525,7 +521,7 @@ export class Upload { ussname: string, buffer: Buffer, options: IUploadOptions = {}) { - options.binary = false; + options.binary = options.binary? options.binary : false; ImperativeExpect.toNotBeNullOrUndefined(ussname, ZosFilesMessages.missingUSSFileName.message); ussname = ZosFilesUtils.sanitizeUssPathForRestCall(ussname); const parameters: string = ZosFilesConstants.RES_USS_FILES + "/" + ussname; From 503398a499c74e128b60935e1ca49375a02a1efa Mon Sep 17 00:00:00 2001 From: Alexandru-Paul Dumitru Date: Thu, 23 Jan 2020 15:25:52 +0100 Subject: [PATCH 3/9] adding system tests Signed-off-by: Alexandru-Paul Dumitru --- .../methods/download/Download.system.test.ts | 91 ++++++++++++++++ .../api/methods/upload/Upload.system.test.ts | 100 +++++++++++++++++- 2 files changed, 189 insertions(+), 2 deletions(-) diff --git a/packages/zosfiles/__tests__/__system__/api/methods/download/Download.system.test.ts b/packages/zosfiles/__tests__/__system__/api/methods/download/Download.system.test.ts index c8a69a88b3..0d8e8c7ccc 100644 --- a/packages/zosfiles/__tests__/__system__/api/methods/download/Download.system.test.ts +++ b/packages/zosfiles/__tests__/__system__/api/methods/download/Download.system.test.ts @@ -14,6 +14,7 @@ import { CreateDataSetTypeEnum, Delete, Download, + Upload, IDownloadOptions, IZosFilesResponse, ZosFilesConstants, @@ -148,6 +149,41 @@ describe("Download Data Set", () => { file = dsname.replace(regex, "/") + ".txt"; }); + it("should download a data set and return Etag", async () => { + let error; + let response: IZosFilesResponse; + + const data: string = "abcdefghijklmnopqrstuvwxyz"; + const endpoint: string = ZosFilesConstants.RESOURCE + ZosFilesConstants.RES_DS_FILES + "/" + dsname; + await ZosmfRestClient.putExpectString(REAL_SESSION, endpoint, [], data); + + const options: IDownloadOptions = { + returnEtag: true + }; + + try { + response = await Download.dataSet(REAL_SESSION, dsname, options); + Imperative.console.info("Response: " + inspect(response)); + } catch (err) { + error = err; + Imperative.console.info("Error: " + inspect(error)); + } + + expect(error).toBeFalsy(); + expect(response).toBeTruthy(); + expect(response.success).toBeTruthy(); + expect(response.commandResponse).toContain( + ZosFilesMessages.datasetDownloadedSuccessfully.message.substring(0, "Data set downloaded successfully".length + 1)); + expect(response.apiResponse.etag).toBeDefined(); + // convert the data set name to use as a path/file for clean up in AfterEach + const regex = /\./gi; + file = dsname.replace(regex, "/") + ".txt"; + + // Compare the downloaded contents to those uploaded + const fileContents = stripNewLines(readFileSync(`${file}`).toString()); + expect(fileContents).toEqual(data); + }); + it("should download a data set that has been populated by upload and use file extension specified", async () => { let error; let response: IZosFilesResponse; @@ -448,6 +484,61 @@ describe("Download Data Set", () => { }); + it("should download uss file and return Etag", async () => { + let error; + let response: IZosFilesResponse; + + const data: string = "abcdefghijklmnopqrstuvwxyz"; + const endpoint: string = ZosFilesConstants.RESOURCE + ZosFilesConstants.RES_USS_FILES + ussname; + + (await ZosmfRestClient.putExpectString(REAL_SESSION, endpoint, [], data)); + + const options: IDownloadOptions = { + returnEtag: true + }; + + try { + response = await Download.ussFile(REAL_SESSION, ussname, options); + } catch (err) { + error = err; + } + expect(error).toBeFalsy(); + expect(response).toBeTruthy(); + expect(response.apiResponse.etag).toBeDefined(); + // Compare the downloaded contents to those uploaded + const fileContents = stripNewLines(readFileSync(`./${posix.basename(ussname)}`).toString()); + expect(fileContents).toEqual(data); + + }); + + // When requesting etag, z/OSMF has a limit on file size when it stops to return etag by default (>8mb) + // We are passing X-IBM-Return-Etag to force z/OSMF to always return etag, but testing here for case where it would be optional + it("should download a 10mb uss file and return Etag", async () => { + let error; + let response: IZosFilesResponse; + + // Create a 10 mb buffer + const bufferSize = 10000000; + const buffer = new ArrayBuffer(bufferSize); + const data = Buffer.from(buffer); + + (await Upload.bufferToUSSFile(REAL_SESSION, ussname, data)); + + const options: IDownloadOptions = { + returnEtag: true + }; + + try { + response = await Download.ussFile(REAL_SESSION, ussname, options); + } catch (err) { + error = err; + } + expect(error).toBeFalsy(); + expect(response).toBeTruthy(); + expect(response.apiResponse.etag).toBeDefined(); + Imperative.console.info(response.apiResponse.etag); + }); + it("should download uss file content in binary", async () => { let error; let response: IZosFilesResponse; diff --git a/packages/zosfiles/__tests__/__system__/api/methods/upload/Upload.system.test.ts b/packages/zosfiles/__tests__/__system__/api/methods/upload/Upload.system.test.ts index 4766b202b3..d738ab7ca9 100644 --- a/packages/zosfiles/__tests__/__system__/api/methods/upload/Upload.system.test.ts +++ b/packages/zosfiles/__tests__/__system__/api/methods/upload/Upload.system.test.ts @@ -9,7 +9,7 @@ * */ -import { Create, CreateDataSetTypeEnum, Delete, IUploadOptions, IZosFilesResponse, Upload, ZosFilesMessages } from "../../../../../"; +import { Create, CreateDataSetTypeEnum, Delete, IUploadOptions, IZosFilesResponse, Upload, ZosFilesMessages, Download } from "../../../../../"; import { Imperative, Session } from "@zowe/imperative"; import { inspect } from "util"; import { ITestEnvironment } from "../../../../../../../__tests__/__src__/environment/doc/response/ITestEnvironment"; @@ -19,6 +19,7 @@ import { getUniqueDatasetName, stripNewLines } from "../../../../../../../__test import { Get, ZosFilesConstants } from "../../../../../index"; import { ZosmfRestClient } from "../../../../../../rest"; import { IUploadMap } from "../../../../../src/api/methods/upload/doc/IUploadMap"; +import * as fs from "fs"; let REAL_SESSION: Session; let testEnvironment: ITestEnvironment; @@ -55,6 +56,8 @@ describe("Upload Data Set", () => { beforeEach(async () => { let error; let response; + uploadOptions.etag = undefined; + uploadOptions.returnEtag = undefined; try { response = await Create.dataSet(REAL_SESSION, @@ -94,6 +97,54 @@ describe("Upload Data Set", () => { expect(response.commandResponse).toContain(ZosFilesMessages.dataSetUploadedSuccessfully.message); }); + it("should upload a file to a physical sequential data set while passing correct Etag", async () => { + let error; + let response: IZosFilesResponse; + + // first we have to get the Etag, so we can compare it. We do it by preemtively downloading the file and requesting Etag + await Upload.fileToDataset(REAL_SESSION, __dirname + "/testfiles/upload.txt", dsname); + const downloadOptions = {file: __dirname + "/testfiles/upload.txt", returnEtag: true}; + const downloadResponse = await Download.dataSet(REAL_SESSION, dsname, downloadOptions); + expect(downloadResponse.success).toBeTruthy(); + expect(downloadResponse.apiResponse.etag).toBeDefined(); + + try { + uploadOptions.etag = downloadResponse.apiResponse.etag; + // packages/zosfiles/__tests__/__system__/api/methods/upload/ + response = await Upload.fileToDataset(REAL_SESSION, __dirname + "/testfiles/upload.txt", dsname, uploadOptions); + Imperative.console.info("Response: " + inspect(response)); + } catch (err) { + error = err; + Imperative.console.info("Error: " + inspect(error)); + } + + expect(error).toBeFalsy(); + expect(response).toBeTruthy(); + expect(response.success).toBeTruthy(); + expect(response.commandResponse).toContain(ZosFilesMessages.dataSetUploadedSuccessfully.message); + }); + + it("should upload a file to a physical sequential data set and return the Etag", async () => { + let error; + let response: IZosFilesResponse; + uploadOptions.returnEtag = true; + + try { + // packages/zosfiles/__tests__/__system__/api/methods/upload/ + response = await Upload.fileToDataset(REAL_SESSION, + __dirname + "/testfiles/upload.txt", dsname, uploadOptions); + Imperative.console.info("Response: " + inspect(response)); + } catch (err) { + error = err; + Imperative.console.info("Error: " + inspect(error)); + } + expect(error).toBeFalsy(); + expect(response).toBeTruthy(); + expect(response.success).toBeTruthy(); + expect(response.commandResponse).toContain(ZosFilesMessages.dataSetUploadedSuccessfully.message); + expect(response.apiResponse[0].etag).toBeDefined(); + }); + it("should upload a file to a physical sequential data set using relative path", async () => { let error; let response: IZosFilesResponse; @@ -448,6 +499,11 @@ describe("Upload USS file", () => { describe("Success Scenarios", () => { + beforeEach(async () => { + uploadOptions.etag = undefined; + fs.writeFileSync(inputfile, testdata); + }); + afterEach(async () => { let error; let response; @@ -502,7 +558,6 @@ describe("Upload USS file", () => { let uploadResponse; let getResponse; - try { uploadResponse = await Upload.fileToUSSFile(REAL_SESSION, inputfile, ussname); getResponse = await Get.USSFile(REAL_SESSION, ussname); @@ -533,6 +588,47 @@ describe("Upload USS file", () => { expect(getResponse).toEqual(Buffer.from(testdata)); }); + it("should upload a USS file while passing correct Etag", async () => { + let error; + let uploadResponse; + + // first we have to get the Etag, so we can compare it. We do it by preemtively downloading the file and requesting Etag + await Upload.fileToUssFile(REAL_SESSION, inputfile, ussname, {returnEtag: false}); + const downloadResponse = await Download.ussFile(REAL_SESSION, ussname, {file: inputfile, returnEtag: true}); + expect(downloadResponse.success).toBeTruthy(); + expect(downloadResponse.apiResponse.etag).toBeDefined(); + + try { + uploadResponse = await Upload.fileToUssFile(REAL_SESSION, inputfile, ussname, {etag: downloadResponse.apiResponse.etag}); + Imperative.console.info("Response: " + inspect(uploadResponse)); + } catch (err) { + error = err; + Imperative.console.info("Error: " + inspect(error)); + } + + expect(error).toBeFalsy(); + expect(uploadResponse).toBeTruthy(); + expect(uploadResponse.success).toBeTruthy(); + expect(uploadResponse.commandResponse).toContain(ZosFilesMessages.ussFileUploadedSuccessfully.message); + }); + it("should upload a USS file and return Etag", async () => { + let error; + let uploadResponse; + let getResponse; + + try { + uploadResponse = await Upload.fileToUssFile(REAL_SESSION, inputfile, ussname, {returnEtag: true}); + getResponse = await Get.USSFile(REAL_SESSION, ussname); + } catch (err) { + error = err; + Imperative.console.info("Error: " + inspect(error)); + } + + expect(error).toBeFalsy(); + expect(uploadResponse).toBeTruthy(); + expect(uploadResponse.success).toBeTruthy(); + expect(uploadResponse.apiResponse.etag).toBeDefined(); + }); }); }); From b4a2e37fcdbfc79ad5885864809acbc7cc7a1b14 Mon Sep 17 00:00:00 2001 From: Alexandru-Paul Dumitru Date: Thu, 23 Jan 2020 16:07:59 +0100 Subject: [PATCH 4/9] fix unit tests for Upload Signed-off-by: Alexandru-Paul Dumitru --- .../api/methods/upload/Upload.unit.test.ts | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/packages/zosfiles/__tests__/api/methods/upload/Upload.unit.test.ts b/packages/zosfiles/__tests__/api/methods/upload/Upload.unit.test.ts index 6f9f159736..61cf59cf43 100644 --- a/packages/zosfiles/__tests__/api/methods/upload/Upload.unit.test.ts +++ b/packages/zosfiles/__tests__/api/methods/upload/Upload.unit.test.ts @@ -920,8 +920,9 @@ describe("z/OS Files - Upload", () => { const getFileListFromPathSpy = jest.spyOn(ZosFilesUtils, "getFileListFromPath"); const getFileListWithFsSpy = jest.spyOn(fs, "readdirSync"); const createUssDirSpy = jest.spyOn(Create, "uss"); - const fileToUSSFileSpy = jest.spyOn(Upload, "fileToUSSFile"); + const fileToUssFileSpy = jest.spyOn(Upload, "fileToUssFile"); const zosmfExpectSpy = jest.spyOn(ZosmfRestClient, "putExpectString"); + const zosmfExpectFullSpy = jest.spyOn(ZosmfRestClient, "putExpectFullResponse"); const pathJoinSpy = jest.spyOn(path, "join"); const pathNormalizeSpy = jest.spyOn(path, "normalize"); const filterDirectoriesSpy = jest.spyOn(Array.prototype, "filter"); @@ -932,7 +933,7 @@ describe("z/OS Files - Upload", () => { beforeEach(() => { USSresponse = undefined; error = undefined; - fileToUSSFileSpy.mockClear(); + fileToUssFileSpy.mockClear(); createUssDirSpy.mockClear(); isDirectoryExistsSpy.mockClear(); getFileListFromPathSpy.mockClear(); @@ -941,8 +942,10 @@ describe("z/OS Files - Upload", () => { pathJoinSpy.mockClear(); pathNormalizeSpy.mockClear(); zosmfExpectSpy.mockClear(); + zosmfExpectFullSpy.mockClear(); filterDirectoriesSpy.mockClear(); zosmfExpectSpy.mockImplementation(() => null); + zosmfExpectFullSpy.mockImplementation(() => null); }); it("should upload recursively if option is specified", async () => { @@ -953,7 +956,7 @@ describe("z/OS Files - Upload", () => { getFileListWithFsSpy.mockReturnValueOnce(["test", "file1.txt", "file2.txt"]).mockReturnValueOnce(["test", "file1.txt", "file2.txt"]).mockReturnValueOnce([]); filterDirectoriesSpy.mockReturnValueOnce(["test"]).mockReturnValueOnce(["test"]); getFileListFromPathSpy.mockReturnValueOnce(["file1.txt", "file2.txt"]).mockReturnValueOnce([]); - fileToUSSFileSpy.mockReturnValue({}); + fileToUssFileSpy.mockReturnValue({}); try { USSresponse = await Upload.dirToUSSDirRecursive(dummySession, testPath, dsName); } catch (err) { @@ -1034,8 +1037,8 @@ describe("z/OS Files - Upload", () => { getFileListWithFsSpy.mockReturnValueOnce(["file1", "file2"]); filterDirectoriesSpy.mockReturnValueOnce([]); getFileListFromPathSpy.mockReturnValueOnce(["file1", "file2"]); - fileToUSSFileSpy.mockReturnValue(testReturn); - fileToUSSFileSpy.mockReturnValue(testReturn); + fileToUssFileSpy.mockReturnValue(testReturn); + fileToUssFileSpy.mockReturnValue(testReturn); promiseSpy.mockReturnValueOnce({}); try { @@ -1057,8 +1060,9 @@ describe("z/OS Files - Upload", () => { const getFileListFromPathSpy = jest.spyOn(ZosFilesUtils, "getFileListFromPath"); const getFileListWithFsSpy = jest.spyOn(fs, "readdirSync"); const createUssDirSpy = jest.spyOn(Create, "uss"); - const fileToUSSFileSpy = jest.spyOn(Upload, "fileToUSSFile"); + const fileToUssFileSpy = jest.spyOn(Upload, "fileToUssFile"); const zosmfExpectSpy = jest.spyOn(ZosmfRestClient, "putExpectString"); + const zosmfExpectFullSpy = jest.spyOn(ZosmfRestClient, "putExpectFullResponse"); const pathJoinSpy = jest.spyOn(path, "join"); const pathNormalizeSpy = jest.spyOn(path, "normalize"); const promiseSpy = jest.spyOn(Promise, "all"); @@ -1069,7 +1073,7 @@ describe("z/OS Files - Upload", () => { beforeEach(() => { USSresponse = undefined; error = undefined; - fileToUSSFileSpy.mockClear(); + fileToUssFileSpy.mockClear(); createUssDirSpy.mockClear(); isDirectoryExistsSpy.mockClear(); getFileListFromPathSpy.mockClear(); @@ -1078,8 +1082,10 @@ describe("z/OS Files - Upload", () => { pathJoinSpy.mockClear(); pathNormalizeSpy.mockClear(); zosmfExpectSpy.mockClear(); + zosmfExpectFullSpy.mockClear(); filterDirectoriesSpy.mockClear(); zosmfExpectSpy.mockImplementation(() => null); + zosmfExpectFullSpy.mockImplementation(() => null); }); it("should throw an error if local directory is not specified", async () => { @@ -1148,8 +1154,8 @@ describe("z/OS Files - Upload", () => { isDirSpy.mockReturnValueOnce(true); isDirectoryExistsSpy.mockReturnValueOnce(true); getFileListFromPathSpy.mockReturnValueOnce(["file1", "file2"]); - fileToUSSFileSpy.mockReturnValue(testReturn); - fileToUSSFileSpy.mockReturnValue(testReturn); + fileToUssFileSpy.mockReturnValue(testReturn); + fileToUssFileSpy.mockReturnValue(testReturn); promiseSpy.mockReturnValueOnce({}); try { @@ -1176,7 +1182,7 @@ describe("z/OS Files - Upload", () => { isDirSpy.mockReturnValueOnce(true) .mockReturnValue(false); isDirectoryExistsSpy.mockReturnValue(true); - fileToUSSFileSpy.mockReturnValue(testReturn); + fileToUssFileSpy.mockReturnValue(testReturn); attributesMock.getFileTransferMode = jest.fn((filePath: string) => { if (filePath.endsWith("textfile")) { @@ -1215,10 +1221,10 @@ describe("z/OS Files - Upload", () => { expect(attributesMock.fileShouldBeUploaded).toHaveBeenCalledWith(path.normalize(path.join(testPath,"uploadme"))); expect(attributesMock.fileShouldBeUploaded).toHaveBeenCalledWith(path.normalize(path.join(testPath,"ignoreme"))); - expect(fileToUSSFileSpy).toHaveBeenCalledTimes(1); - expect(fileToUSSFileSpy).toHaveBeenCalledWith(dummySession, + expect(fileToUssFileSpy).toHaveBeenCalledTimes(1); + expect(fileToUssFileSpy).toHaveBeenCalledWith(dummySession, path.normalize(path.join(testPath,"uploadme")), - `${dsName}/uploadme`, true); + `${dsName}/uploadme`, {binary: true}); }); it("should not upload ignored directories", async () => { @@ -1262,10 +1268,10 @@ describe("z/OS Files - Upload", () => { expect(USSresponse).toBeDefined(); expect(USSresponse.success).toBeTruthy(); expect(attributesMock.fileShouldBeUploaded).toHaveBeenCalledWith(path.normalize(path.join(testPath,"uploaddir"))); - expect(fileToUSSFileSpy).toHaveBeenCalledTimes(1); - expect(fileToUSSFileSpy).toHaveBeenCalledWith(dummySession, + expect(fileToUssFileSpy).toHaveBeenCalledTimes(1); + expect(fileToUssFileSpy).toHaveBeenCalledWith(dummySession, path.normalize(path.join(testPath, "uploaddir", "uploadedfile")), - `${dsName}/uploaddir/uploadedfile`, true); + `${dsName}/uploaddir/uploadedfile`, {binary: true}); }); it("should upload files in text or binary according to attributes", async () => { getFileListFromPathSpy.mockReturnValue(["textfile", "binaryfile"]); @@ -1275,15 +1281,15 @@ describe("z/OS Files - Upload", () => { expect(USSresponse).toBeDefined(); expect(USSresponse.success).toBeTruthy(); - expect(fileToUSSFileSpy).toHaveBeenCalledTimes(2); - expect(fileToUSSFileSpy).toHaveBeenCalledWith(dummySession, + expect(fileToUssFileSpy).toHaveBeenCalledTimes(2); + expect(fileToUssFileSpy).toHaveBeenCalledWith(dummySession, path.normalize(path.join(testPath,"textfile")), `${dsName}/textfile`, - false, - "ISO8859-1"); - expect(fileToUSSFileSpy).toHaveBeenCalledWith(dummySession, + {binary: false, + localEncoding: "ISO8859-1"}); + expect(fileToUssFileSpy).toHaveBeenCalledWith(dummySession, path.normalize(path.join(testPath,"binaryfile")), - `${dsName}/binaryfile`, true); + `${dsName}/binaryfile`, {binary: true}); }); it("should call API to tag files accord to remote encoding", async () => { From 2f7c18761c448cfc3049196690bbc2ac1d5b78df Mon Sep 17 00:00:00 2001 From: Alexandru-Paul Dumitru Date: Thu, 23 Jan 2020 16:55:39 +0100 Subject: [PATCH 5/9] remove comments Signed-off-by: Alexandru-Paul Dumitru --- packages/zosfiles/src/api/methods/download/Download.ts | 2 -- packages/zosfiles/src/api/methods/upload/Upload.ts | 3 --- 2 files changed, 5 deletions(-) diff --git a/packages/zosfiles/src/api/methods/download/Download.ts b/packages/zosfiles/src/api/methods/download/Download.ts index c3876c9f7f..4d6054a8ec 100644 --- a/packages/zosfiles/src/api/methods/download/Download.ts +++ b/packages/zosfiles/src/api/methods/download/Download.ts @@ -100,7 +100,6 @@ export class Download { IO.createDirsSyncFromFilePath(destination); const writeStream = IO.createWriteStream(destination); - // await ZosmfRestClient.getStreamed(session, endpoint, reqHeaders, writeStream, !options.binary, options.task); // Use specific options to mimic ZosmfRestClient.getStreamed() const requestOptions: IOptionsFullResponse = { @@ -289,7 +288,6 @@ export class Download { requestOptions.dataToReturn = [CLIENT_PROPERTY.response]; } - // const test = await ZosmfRestClient.getStreamed(session, endpoint, reqHeaders, writeStream, !options.binary, options.task); const request = await ZosmfRestClient.getExpectFullResponse(session, requestOptions); // By default, apiResponse is empty when downloading diff --git a/packages/zosfiles/src/api/methods/upload/Upload.ts b/packages/zosfiles/src/api/methods/upload/Upload.ts index 23e997aeb7..59266fb04d 100644 --- a/packages/zosfiles/src/api/methods/upload/Upload.ts +++ b/packages/zosfiles/src/api/methods/upload/Upload.ts @@ -300,9 +300,6 @@ export class Upload { } const uploadRequest: IRestClientResponse = await ZosmfRestClient.putExpectFullResponse(session, requestOptions); - // await ZosmfRestClient.putStreamedRequestOnly(session, endpoint, reqHeaders, fileStream, - // !options.binary /* only normalize newlines if we are not uploading in binary*/, - // options.task); // By default, apiResponse is empty when uploading const apiResponse: any = {}; From e220c4fc0ca8242b286d237e9412ca579e91c8d0 Mon Sep 17 00:00:00 2001 From: Alexandru-Paul Dumitru Date: Sat, 25 Jan 2020 01:17:18 +0100 Subject: [PATCH 6/9] add more tests to increase coverage Signed-off-by: Alexandru-Paul Dumitru --- .../api/methods/upload/Upload.unit.test.ts | 597 ++++++++++++++++++ 1 file changed, 597 insertions(+) diff --git a/packages/zosfiles/__tests__/api/methods/upload/Upload.unit.test.ts b/packages/zosfiles/__tests__/api/methods/upload/Upload.unit.test.ts index 61cf59cf43..97c102f820 100644 --- a/packages/zosfiles/__tests__/api/methods/upload/Upload.unit.test.ts +++ b/packages/zosfiles/__tests__/api/methods/upload/Upload.unit.test.ts @@ -29,6 +29,7 @@ import { stripNewLines } from "../../../../../../__tests__/__src__/TestUtils"; import { Create } from "../../../../src/api/methods/create"; import { ZosFilesAttributes, TransferMode, Tag } from "../../../../src/api"; import { CLIENT_PROPERTY } from "../../../../src/api/doc/types/ZosmfRestClientProperties"; +import { Readable } from "stream"; describe("z/OS Files - Upload", () => { @@ -492,7 +493,256 @@ describe("z/OS Files - Upload", () => { writeData: buffer}); }); }); + describe("streamToDataSet", () => { + const zosmfPutFullSpy = jest.spyOn(ZosmfRestClient, "putExpectFullResponse"); + const fakeResponseWithEtag = {data: dsName, response:{headers:{etag: etagValue}}}; + const inputStream = new Readable(); + inputStream.push("testing"); + inputStream.push(null); + + beforeEach(() => { + response = undefined; + error = undefined; + + zosmfPutFullSpy.mockClear(); + zosmfPutFullSpy.mockImplementation(() => null); + }); + + it("should throw error if data set name is not specified", async () => { + try { + response = await Upload.streamToDataSet(dummySession, inputStream, undefined); + } catch (err) { + error = err; + } + + expect(response).toBeUndefined(); + expect(error).toBeDefined(); + expect(error.message).toContain(ZosFilesMessages.missingDatasetName.message); + }); + it("return error that throw by the ZosmfRestClient", async () => { + const testError = new ImperativeError({ + msg: "test error" + }); + + zosmfPutFullSpy.mockRejectedValueOnce(testError); + + try { + response = await Upload.streamToDataSet(dummySession, inputStream, dsName); + } catch (err) { + error = err; + } + + expect(response).toBeUndefined(); + expect(error).toBeDefined(); + expect(error).toBe(testError); + }); + it("return with proper response when upload stream to a data set", async () => { + const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_DS_FILES, dsName); + const reqHeaders = [ZosmfHeaders.X_IBM_TEXT]; + + try { + response = await Upload.streamToDataSet(dummySession, inputStream, dsName); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(response).toBeDefined(); + + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + normalizeRequestNewLines: true, + reqHeaders, + requestStream: inputStream}); + }); + it("return with proper response when upload stream to a PDS member", async () => { + const testDsName = `${dsName}(member)`; + const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_DS_FILES, testDsName); + const reqHeaders = [ZosmfHeaders.X_IBM_TEXT]; + + try { + response = await Upload.streamToDataSet(dummySession, inputStream, testDsName); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(response).toBeDefined(); + + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource:endpoint, + normalizeRequestNewLines: true, + reqHeaders, + requestStream: inputStream}); + }); + it("return with proper response when upload stream to a data set with optional parameters", async () => { + const uploadOptions: IUploadOptions = { + binary: true + }; + const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_DS_FILES, dsName); + let reqHeaders = [ZosmfHeaders.X_IBM_BINARY]; + + try { + response = await Upload.streamToDataSet(dummySession, inputStream, dsName, uploadOptions); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(response).toBeDefined(); + + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + normalizeRequestNewLines: false, + reqHeaders, + requestStream: inputStream}); + zosmfPutFullSpy.mockClear(); + + // Unit test for wait option + uploadOptions.recall = "wait"; + reqHeaders = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_WAIT]; + + try { + response = await Upload.streamToDataSet(dummySession, inputStream, dsName, uploadOptions); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(response).toBeDefined(); + + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + normalizeRequestNewLines: false, + reqHeaders, + requestStream: inputStream}); + zosmfPutFullSpy.mockClear(); + + // Unit test for no wait option + uploadOptions.recall = "nowait"; + reqHeaders = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT]; + + try { + response = await Upload.streamToDataSet(dummySession, inputStream, dsName, uploadOptions); + } catch (err) { + error = err; + } + expect(error).toBeUndefined(); + expect(response).toBeDefined(); + + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + normalizeRequestNewLines: false, + reqHeaders, + requestStream: inputStream}); + zosmfPutFullSpy.mockClear(); + + // Unit test for no error option + uploadOptions.recall = "error"; + reqHeaders = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_ERROR]; + + try { + response = await Upload.streamToDataSet(dummySession, inputStream, dsName, uploadOptions); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(response).toBeDefined(); + + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + normalizeRequestNewLines: false, + reqHeaders, + requestStream: inputStream}); + zosmfPutFullSpy.mockClear(); + + // Unit test default value + uploadOptions.recall = "non-existing"; + reqHeaders = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT]; + + try { + response = await Upload.streamToDataSet(dummySession, inputStream, dsName, uploadOptions); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(response).toBeDefined(); + + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + normalizeRequestNewLines: false, + reqHeaders, + requestStream: inputStream}); + zosmfPutFullSpy.mockClear(); + + // Unit test for pass etag option + uploadOptions.etag = etagValue; + reqHeaders = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT, {"If-Match" : uploadOptions.etag}]; + + try { + response = await Upload.streamToDataSet(dummySession, inputStream, dsName, uploadOptions); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(response).toBeDefined(); + + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + normalizeRequestNewLines: false, + reqHeaders, + requestStream: inputStream}); + zosmfPutFullSpy.mockClear(); + zosmfPutFullSpy.mockImplementationOnce(() => fakeResponseWithEtag); + + // Unit test for return etag option + reqHeaders = [ZosmfHeaders.X_IBM_BINARY, + ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT, + {"If-Match" : uploadOptions.etag}, + ZosmfHeaders.X_IBM_RETURN_ETAG]; + uploadOptions.returnEtag = true; + try { + response = await Upload.streamToDataSet(dummySession, inputStream, dsName, uploadOptions); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(response).toBeDefined(); + + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + normalizeRequestNewLines: false, + reqHeaders, + requestStream: inputStream, + dataToReturn: [CLIENT_PROPERTY.response]}); + }); + it("return with proper response when upload dataset with specify volume option", async () => { + const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_DS_FILES, `-(TEST)`, dsName); + const reqHeaders = [ZosmfHeaders.X_IBM_TEXT]; + const uploadOptions: IUploadOptions = { + volume: "TEST" + }; + try { + response = await Upload.streamToDataSet(dummySession, inputStream, dsName, uploadOptions); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(response).toBeDefined(); + + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + normalizeRequestNewLines: true, + reqHeaders, + requestStream: inputStream}); + }); + }); describe("pathToDataSet", () => { const listDatasetSpy = jest.spyOn(List, "dataSet"); const getFileListSpy = jest.spyOn(ZosFilesUtils, "getFileListFromPath"); @@ -594,6 +844,40 @@ describe("z/OS Files - Upload", () => { expect(error).toBeDefined(); expect(error.message).toContain(ZosFilesMessages.uploadDirectoryToDatasetMember.message); }); + it("should throw error when trying to upload multiple files to a PS data set", async () => { + const mockFileList = ["file1", "file2"]; + const mockListResponse: IZosFilesResponse = { + success: true, + commandResponse: "dummy response", + apiResponse: { + items: [{ + dsname: dsName, + dsorg: "PS" + }], + returnedRows: 1 + } + }; + listDatasetSpy.mockResolvedValueOnce(mockListResponse); + getFileListSpy.mockReturnValueOnce(mockFileList); + writeZosmfDatasetSpy.mockResolvedValueOnce({ + success: true, + CommandResponse: "dummy" + }); + writeZosmfDatasetSpy.mockResolvedValueOnce({ + success: true, + CommandResponse: "dummy" + }); + + try { + response = await Upload.pathToDataSet(dummySession, "dummyPath", dsName); + } catch (err) { + error = err; + } + + expect(error).toBeDefined(); + expect(response).toBeUndefined(); + expect(error.message).toContain(ZosFilesMessages.uploadDirectoryToPhysicalSequentialDataSet.message); + }); it("should return information when successfully uploaded multiple files", async () => { const mockFileList = ["file1", "file2"]; const mockListResponse: IZosFilesResponse = { @@ -913,6 +1197,311 @@ describe("z/OS Files - Upload", () => { }); }); + describe("streamToUssFile", () => { + let USSresponse: IZosFilesResponse; + const zosmfExpectFullSpy = jest.spyOn(ZosmfRestClient, "putExpectFullResponse"); + const fakeResponseWithEtag = {data: dsName, response:{headers:{etag: etagValue}}}; + const inputStream = new Readable(); + inputStream.push("testing"); + inputStream.push(null); + + beforeEach(() => { + USSresponse = undefined; + error = undefined; + + zosmfExpectFullSpy.mockClear(); + zosmfExpectFullSpy.mockImplementation(() => null); + }); + + afterAll(() => { + zosmfExpectFullSpy.mockRestore(); + }); + + it("should throw an error if USS file name is not specified", async () => { + try { + USSresponse = await Upload.streamToUssFile(dummySession, undefined, inputStream); + } catch (err) { + error = err; + } + + expect(USSresponse).toBeUndefined(); + expect(error).toBeDefined(); + expect(error.message).toContain(ZosFilesMessages.missingUSSFileName.message); + }); + it("return error that is thrown by the ZosmfRestClient", async () => { + const testError = new ImperativeError({ + msg: "test error" + }); + + zosmfExpectFullSpy.mockRejectedValueOnce(testError); + + try { + USSresponse = await Upload.streamToUssFile(dummySession, dsName, inputStream); + } catch (err) { + error = err; + } + + expect(USSresponse).toBeUndefined(); + expect(error).toBeDefined(); + expect(error).toBe(testError); + }); + it("return with proper response when upload USS file", async () => { + const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_USS_FILES, dsName); + const reqHeaders = [ZosmfHeaders.TEXT_PLAIN]; + + try { + USSresponse = await Upload.streamToUssFile(dummySession, dsName, inputStream); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(USSresponse).toBeDefined(); + expect(USSresponse.success).toBeTruthy(); + + expect(zosmfExpectFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfExpectFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders, + requestStream: inputStream, + normalizeRequestNewLines: true}); + }); + // it("return with proper response when upload USS file using deprecated function", async () => { + // const uploadStreamToUssFileSpy = jest.spyOn(Upload, "streamToUssFile"); + // try { + // USSresponse = await Upload.streamToUSSFile(dummySession, dsName, inputStream); + // } catch (err) { + // error = err; + // } + + // expect(error).toBeUndefined(); + // expect(USSresponse).toBeDefined(); + // expect(USSresponse.success).toBeTruthy(); + + // expect(uploadStreamToUssFileSpy).toHaveBeenCalledTimes(1); + // expect(uploadStreamToUssFileSpy).toHaveBeenCalledWith(dummySession, dsName, inputStream, { + // binary: false}); + // }); + it("return with proper response when upload USS file in binary", async () => { + const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_USS_FILES, dsName); + const reqHeaders = [ZosmfHeaders.OCTET_STREAM, ZosmfHeaders.X_IBM_BINARY]; + + try { + USSresponse = await Upload.streamToUssFile(dummySession, dsName, inputStream, {binary: true}); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(USSresponse).toBeDefined(); + expect(USSresponse.success).toBeTruthy(); + + expect(zosmfExpectFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfExpectFullSpy).toHaveBeenCalledWith(dummySession, { + resource: endpoint, + reqHeaders, + requestStream: inputStream, + normalizeRequestNewLines: false}); + }); + it("return with proper response when upload USS file with Etag", async () => { + const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_USS_FILES, dsName); + const reqHeaders = [ZosmfHeaders.TEXT_PLAIN, {"If-Match": etagValue}]; + + try { + USSresponse = await Upload.streamToUssFile(dummySession, dsName, inputStream, {etag: etagValue}); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(USSresponse).toBeDefined(); + expect(USSresponse.success).toBeTruthy(); + + expect(zosmfExpectFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfExpectFullSpy).toHaveBeenCalledWith(dummySession, { + resource: endpoint, + reqHeaders, + requestStream: inputStream, + normalizeRequestNewLines: true}); + }); + it("return with proper response when upload USS file and request Etag back", async () => { + const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_USS_FILES, dsName); + const reqHeaders = [ZosmfHeaders.TEXT_PLAIN, ZosmfHeaders.X_IBM_RETURN_ETAG]; + zosmfExpectFullSpy.mockImplementationOnce(() => fakeResponseWithEtag); + try { + USSresponse = await Upload.streamToUssFile(dummySession, dsName, inputStream, {returnEtag: true}); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(USSresponse).toBeDefined(); + expect(USSresponse.success).toBeTruthy(); + expect(USSresponse.apiResponse.etag).toBeDefined(); + expect(USSresponse.apiResponse.etag).toEqual(etagValue); + + expect(zosmfExpectFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfExpectFullSpy).toHaveBeenCalledWith(dummySession, { + resource: endpoint, + reqHeaders, + requestStream: inputStream, + normalizeRequestNewLines: true, + dataToReturn: [CLIENT_PROPERTY.response]}); + }); + it("should set local encoding if specified", async () => { + const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_USS_FILES, dsName); + const reqHeaders = [{"Content-Type": "UCS-2"}, ZosmfHeaders.X_IBM_TEXT]; + + try { + USSresponse = await Upload.streamToUssFile(dummySession, dsName, inputStream, {localEncoding: "UCS-2"}); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(USSresponse).toBeDefined(); + + expect(zosmfExpectFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfExpectFullSpy).toHaveBeenCalledWith(dummySession, { + resource: endpoint, + reqHeaders, + requestStream: inputStream, + normalizeRequestNewLines: true}); + }); + }); + + describe("fileToUssFile", () => { + let USSresponse: IZosFilesResponse; + const createReadStreamSpy = jest.spyOn(IO, "createReadStream"); + const streamToUssFileSpy = jest.spyOn(Upload, "streamToUssFile"); + const lsStatSpy = jest.spyOn(fs, "lstat"); + const inputFile = "/path/to/file1.txt"; + + beforeEach(() => { + USSresponse = undefined; + error = undefined; + + createReadStreamSpy.mockReset(); + createReadStreamSpy.mockImplementation(() => null); + + streamToUssFileSpy.mockReset(); + streamToUssFileSpy.mockImplementation(() => null); + + lsStatSpy.mockClear(); + lsStatSpy.mockImplementation((somePath, callback) => { + callback(null, {isFile: () => true}); + }); + }); + + it("should throw an error if local file name is not specified", async () => { + try { + USSresponse = await Upload.fileToUssFile(dummySession, undefined, "file"); + } catch (err) { + error = err; + } + + expect(USSresponse).toBeUndefined(); + expect(error).toBeDefined(); + expect(error.message).toContain(ZosFilesMessages.missingInputFile.message); + }); + it("should throw an error if USS file name is not specified", async () => { + try { + USSresponse = await Upload.fileToUssFile(dummySession, inputFile, undefined); + } catch (err) { + error = err; + } + + expect(USSresponse).toBeUndefined(); + expect(error).toBeDefined(); + expect(error.message).toContain(ZosFilesMessages.missingUSSFileName.message); + }); + it("should throw an error if USS file name is an empty string", async () => { + try { + USSresponse = await Upload.fileToUssFile(dummySession, inputFile, ""); + } catch (err) { + error = err; + } + + expect(USSresponse).toBeUndefined(); + expect(error).toBeDefined(); + expect(error.message).toContain(ZosFilesMessages.missingUSSFileName.message); + }); + it("should throw underlying fs error", async () => { + const rootError = { + code: "test", + toString() { + return this.code; + } + }; + + lsStatSpy.mockImplementationOnce((somePath, callback) => { + callback(rootError); + }); + try { + USSresponse = await Upload.fileToUssFile(dummySession, inputFile, "file"); + } catch (err) { + error = err; + } + + expect(USSresponse).toBeUndefined(); + expect(error).toBeDefined(); + expect(error.message).toContain(ZosFilesMessages.nodeJsFsError.message); + expect(error.additionalDetails).toEqual(rootError.toString()); + expect(error.causeErrors).toBe(rootError); + }); + it("return with proper response when upload USS file", async () => { + try { + USSresponse = await Upload.fileToUssFile(dummySession, inputFile, "file"); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(USSresponse).toBeDefined(); + expect(USSresponse.success).toBeTruthy(); + + expect(createReadStreamSpy).toHaveBeenCalledTimes(1); + expect(createReadStreamSpy).toHaveBeenCalledWith(inputFile); + expect(streamToUssFileSpy).toHaveBeenCalledTimes(1); + expect(streamToUssFileSpy).toHaveBeenCalledWith(dummySession, "file", null, {}); + }); + it("return with proper response when upload USS file including Etag", async () => { + const streamResponse: IZosFilesResponse = { + success: true, + commandResponse: undefined, + apiResponse: {etag: etagValue}}; + streamToUssFileSpy.mockImplementationOnce(() => streamResponse); + try { + USSresponse = await Upload.fileToUssFile(dummySession, inputFile, "file", {returnEtag: true}); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(USSresponse).toBeDefined(); + expect(USSresponse.success).toBeTruthy(); + expect(USSresponse.apiResponse.etag).toEqual(etagValue); + + expect(createReadStreamSpy).toHaveBeenCalledTimes(1); + expect(createReadStreamSpy).toHaveBeenCalledWith(inputFile); + expect(streamToUssFileSpy).toHaveBeenCalledTimes(1); + expect(streamToUssFileSpy).toHaveBeenCalledWith(dummySession, "file", null, {returnEtag: true}); + }); + it("should throw an error if local file name is not a valid file path", async () => { + lsStatSpy.mockImplementationOnce((somePath, callback) => { + callback(null, {isFile: () => false}); + }); + try { + USSresponse = await Upload.fileToUssFile(dummySession, undefined, "file"); + } catch (err) { + error = err; + } + + expect(USSresponse).toBeUndefined(); + expect(error).toBeDefined(); + expect(error.message).toContain(ZosFilesMessages.missingInputFile.message); + lsStatSpy.mockClear(); + }); + }); describe("dirToUSSDirRecursive", () => { let USSresponse: IZosFilesResponse; const isDirSpy = jest.spyOn(IO, "isDir"); @@ -948,6 +1537,10 @@ describe("z/OS Files - Upload", () => { zosmfExpectFullSpy.mockImplementation(() => null); }); + afterAll(() => { + zosmfExpectFullSpy.mockRestore(); + }); + it("should upload recursively if option is specified", async () => { isDirSpy.mockReturnValue(true); isDirectoryExistsSpy.mockReturnValueOnce(false).mockReturnValueOnce(true); @@ -1088,6 +1681,10 @@ describe("z/OS Files - Upload", () => { zosmfExpectFullSpy.mockImplementation(() => null); }); + afterAll(() => { + zosmfExpectFullSpy.mockRestore(); + }); + it("should throw an error if local directory is not specified", async () => { try { USSresponse = await Upload.dirToUSSDir(dummySession, undefined, dsName); From eae942158830691a4189d84a95d02500979f2c1f Mon Sep 17 00:00:00 2001 From: Alexandru-Paul Dumitru Date: Sat, 25 Jan 2020 02:13:18 +0100 Subject: [PATCH 7/9] attempt to remove code duplication #1 - adding function that generates headers based on options passed and context (buffer, stream, default) Signed-off-by: Alexandru-Paul Dumitru --- .../zosfiles/src/api/methods/upload/Upload.ts | 152 ++++++++---------- 1 file changed, 65 insertions(+), 87 deletions(-) diff --git a/packages/zosfiles/src/api/methods/upload/Upload.ts b/packages/zosfiles/src/api/methods/upload/Upload.ts index 59266fb04d..a314af24ee 100644 --- a/packages/zosfiles/src/api/methods/upload/Upload.ts +++ b/packages/zosfiles/src/api/methods/upload/Upload.ts @@ -167,34 +167,10 @@ export class Upload { endpoint = path.posix.join(endpoint, dataSetName); // Construct request header parameters - const reqHeaders: IHeaderContent[] = []; - if (options.binary) { - reqHeaders.push(ZosmfHeaders.X_IBM_BINARY); - } else { - reqHeaders.push(ZosmfHeaders.X_IBM_TEXT); - fileBuffer = ZosFilesUtils.normalizeNewline(fileBuffer); - } + const reqHeaders: IHeaderContent[] = this.generateHeadersBasedOnOptions(options); - // Migrated recall options - if (options.recall) { - switch (options.recall.toLowerCase()) { - case "wait": - reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_WAIT); - break; - case "nowait": - reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT); - break; - case "error": - reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_ERROR); - break; - default: - reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT); - break; - } - } - - if (options.etag) { - reqHeaders.push({"If-Match" : options.etag}); + if (!options.binary) { + fileBuffer = ZosFilesUtils.normalizeNewline(fileBuffer); } // Options to use the buffer to write a file @@ -206,7 +182,6 @@ export class Upload { // If requestor needs etag, add header + get "response" back if (options.returnEtag) { - requestOptions.reqHeaders.push(ZosmfHeaders.X_IBM_RETURN_ETAG); requestOptions.dataToReturn = [CLIENT_PROPERTY.response]; } const uploadRequest: IRestClientResponse = await ZosmfRestClient.putExpectFullResponse(session, requestOptions); @@ -256,34 +231,7 @@ export class Upload { endpoint = path.posix.join(endpoint, dataSetName); // Construct request header parameters - const reqHeaders: IHeaderContent[] = []; - if (options.binary) { - reqHeaders.push(ZosmfHeaders.X_IBM_BINARY); - } else { - reqHeaders.push(ZosmfHeaders.X_IBM_TEXT); - } - - // Migrated recall options - if (options.recall) { - switch (options.recall.toLowerCase()) { - case "wait": - reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_WAIT); - break; - case "nowait": - reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT); - break; - case "error": - reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_ERROR); - break; - default: - reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT); - break; - } - } - - if (options.etag) { - reqHeaders.push({"If-Match" : options.etag}); - } + const reqHeaders: IHeaderContent[] = this.generateHeadersBasedOnOptions(options); const requestOptions: IOptionsFullResponse = { resource: endpoint, @@ -295,7 +243,6 @@ export class Upload { // If requestor needs etag, add header + get "response" back if (options.returnEtag) { - requestOptions.reqHeaders.push(ZosmfHeaders.X_IBM_RETURN_ETAG); requestOptions.dataToReturn = [CLIENT_PROPERTY.response]; } @@ -522,20 +469,7 @@ export class Upload { ImperativeExpect.toNotBeNullOrUndefined(ussname, ZosFilesMessages.missingUSSFileName.message); ussname = ZosFilesUtils.sanitizeUssPathForRestCall(ussname); const parameters: string = ZosFilesConstants.RES_USS_FILES + "/" + ussname; - const headers: any[] = []; - if (options.binary) { - headers.push(ZosmfHeaders.OCTET_STREAM); - headers.push(ZosmfHeaders.X_IBM_BINARY); - } else if (options.localEncoding) { - headers.push({"Content-Type": options.localEncoding}); - headers.push(ZosmfHeaders.X_IBM_TEXT); - } else { - headers.push(ZosmfHeaders.TEXT_PLAIN); - } - - if (options.etag) { - headers.push({"If-Match" : options.etag}); - } + const headers: IHeaderContent[] = this.generateHeadersBasedOnOptions(options, "buffer"); return ZosmfRestClient.putExpectString(session, ZosFilesConstants.RESOURCE + parameters, headers, buffer); } @@ -581,32 +515,18 @@ export class Upload { ussname = ZosFilesUtils.formatUnixFilepath(ussname); ussname = encodeURIComponent(ussname); const parameters: string = ZosFilesConstants.RES_USS_FILES + "/" + ussname; - const headers: any[] = []; - if (options.binary) { - headers.push(ZosmfHeaders.OCTET_STREAM); - headers.push(ZosmfHeaders.X_IBM_BINARY); - } else if (options.localEncoding) { - headers.push({"Content-Type": options.localEncoding}); - headers.push(ZosmfHeaders.X_IBM_TEXT); - } else { - headers.push(ZosmfHeaders.TEXT_PLAIN); - } - - if (options.etag) { - headers.push({"If-Match" : options.etag}); - } + const reqHeaders: IHeaderContent[] = this.generateHeadersBasedOnOptions(options, "stream"); // Options to use the stream to write a file const restOptions: IOptionsFullResponse = { resource: ZosFilesConstants.RESOURCE + parameters, - reqHeaders: headers, + reqHeaders, requestStream: uploadStream, normalizeRequestNewLines: !options.binary /* only normalize newlines if we are not in binary mode*/ }; // If requestor needs etag, add header + get "response" back if (options.returnEtag) { - restOptions.reqHeaders.push(ZosmfHeaders.X_IBM_RETURN_ETAG); restOptions.dataToReturn = [CLIENT_PROPERTY.response]; } const uploadRequest: IRestClientResponse = await ZosmfRestClient.putExpectFullResponse(session, restOptions); @@ -1082,4 +1002,62 @@ export class Upload { return result; } + + /** + * helper function to generate the headers based on the options used + * @param {IUploadOptions} options - upload options + * @param {string} context - context method from where you call this function (can be "buffer", "stream" or undefined) + */ + private static generateHeadersBasedOnOptions(options: IUploadOptions, context?: string): IHeaderContent[] { + const reqHeaders: IHeaderContent[] = []; + + switch (context) { + case "stream": + case "buffer": + if (options.binary) { + reqHeaders.push(ZosmfHeaders.OCTET_STREAM); + reqHeaders.push(ZosmfHeaders.X_IBM_BINARY); + } else if (options.localEncoding) { + reqHeaders.push({"Content-Type": options.localEncoding}); + reqHeaders.push(ZosmfHeaders.X_IBM_TEXT); + } else { + reqHeaders.push(ZosmfHeaders.TEXT_PLAIN); + } + break; + default: + if (options.binary) { + reqHeaders.push(ZosmfHeaders.X_IBM_BINARY); + } else { + reqHeaders.push(ZosmfHeaders.X_IBM_TEXT); + } + break; + } + + // Migrated recall options + if (options.recall) { + switch (options.recall.toLowerCase()) { + case "wait": + reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_WAIT); + break; + case "nowait": + reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT); + break; + case "error": + reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_ERROR); + break; + default: + reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT); + break; + } + } + + if (options.etag) { + reqHeaders.push({"If-Match" : options.etag}); + } + + if (options.returnEtag) { + reqHeaders.push(ZosmfHeaders.X_IBM_RETURN_ETAG); + } + return reqHeaders; + } } From 7fb7199aa3c8c178e4c78bee958f148d4ce5c1d4 Mon Sep 17 00:00:00 2001 From: zFernand0 Date: Fri, 7 Feb 2020 10:16:43 -0500 Subject: [PATCH 8/9] remove trailing commas Signed-off-by: zFernand0 --- .../__tests__/api/methods/download/Download.unit.test.ts | 2 +- .../zosfiles/src/api/doc/types/ZosmfRestClientProperties.ts | 2 +- packages/zosfiles/src/api/methods/download/Download.ts | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/zosfiles/__tests__/api/methods/download/Download.unit.test.ts b/packages/zosfiles/__tests__/api/methods/download/Download.unit.test.ts index d7ac0e3c54..8d37e4cf96 100644 --- a/packages/zosfiles/__tests__/api/methods/download/Download.unit.test.ts +++ b/packages/zosfiles/__tests__/api/methods/download/Download.unit.test.ts @@ -672,7 +672,7 @@ describe("z/OS Files - Download", () => { expect(zosmfGetFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, reqHeaders: [], responseStream: fakeStream, - normalizeResponseNewLines: true, + normalizeResponseNewLines: true }); expect(ioCreateDirSpy).toHaveBeenCalledTimes(1); diff --git a/packages/zosfiles/src/api/doc/types/ZosmfRestClientProperties.ts b/packages/zosfiles/src/api/doc/types/ZosmfRestClientProperties.ts index a34c541b05..6a688e467e 100644 --- a/packages/zosfiles/src/api/doc/types/ZosmfRestClientProperties.ts +++ b/packages/zosfiles/src/api/doc/types/ZosmfRestClientProperties.ts @@ -24,5 +24,5 @@ export const CLIENT_PROPERTY = { dataString: "dataString" as CLIENT_PROPERTY, response: "response" as CLIENT_PROPERTY, session: "session" as CLIENT_PROPERTY, - log: "log" as CLIENT_PROPERTY, + log: "log" as CLIENT_PROPERTY }; diff --git a/packages/zosfiles/src/api/methods/download/Download.ts b/packages/zosfiles/src/api/methods/download/Download.ts index 4d6054a8ec..2ee888cef4 100644 --- a/packages/zosfiles/src/api/methods/download/Download.ts +++ b/packages/zosfiles/src/api/methods/download/Download.ts @@ -107,7 +107,7 @@ export class Download { reqHeaders, responseStream: writeStream, normalizeResponseNewLines: !options.binary, - task: options.task, + task: options.task }; // If requestor needs etag, add header + get "response" back @@ -279,7 +279,7 @@ export class Download { reqHeaders, responseStream: writeStream, normalizeResponseNewLines: !options.binary, - task: options.task, + task: options.task }; // If requestor needs etag, add header + get "response" back From 4be6a96c7afeb37d818eb6bfb618af7fff154ca5 Mon Sep 17 00:00:00 2001 From: Alexandru-Paul Dumitru Date: Mon, 10 Feb 2020 14:52:54 +0100 Subject: [PATCH 9/9] refactor test - split optional params into unit tests - create group of tests for deprecated function Signed-off-by: Alexandru-Paul Dumitru --- .../api/methods/upload/Upload.unit.test.ts | 330 ++++++++++-------- 1 file changed, 194 insertions(+), 136 deletions(-) diff --git a/packages/zosfiles/__tests__/api/methods/upload/Upload.unit.test.ts b/packages/zosfiles/__tests__/api/methods/upload/Upload.unit.test.ts index 97c102f820..a3928d5967 100644 --- a/packages/zosfiles/__tests__/api/methods/upload/Upload.unit.test.ts +++ b/packages/zosfiles/__tests__/api/methods/upload/Upload.unit.test.ts @@ -14,7 +14,7 @@ jest.mock("fs"); import * as path from "path"; import * as fs from "fs"; -import { ImperativeError, IO, Session } from "@zowe/imperative"; +import { ImperativeError, IO, Session, IHeaderContent } from "@zowe/imperative"; import { ZosFilesMessages } from "../../../../"; import { ZosmfHeaders, ZosmfRestClient } from "../../../../../rest"; import { IZosFilesResponse } from "../../../../../zosfiles"; @@ -332,144 +332,156 @@ describe("z/OS Files - Upload", () => { reqHeaders, writeData: buffer}); }); - it("return with proper response when upload buffer to a data set with optional parameters", async () => { - const buffer: Buffer = Buffer.from("testing"); - const uploadOptions: IUploadOptions = { - binary: true - }; - const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_DS_FILES, dsName); - let reqHeaders = [ZosmfHeaders.X_IBM_BINARY]; - - try { - response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); - } catch (err) { - error = err; - } - - expect(error).toBeUndefined(); - expect(response).toBeDefined(); - - expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); - expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, - reqHeaders, - writeData: buffer}); - zosmfPutFullSpy.mockClear(); - - // Unit test for wait option - uploadOptions.recall = "wait"; - reqHeaders = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_WAIT]; - - try { - response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); - } catch (err) { - error = err; - } - - expect(error).toBeUndefined(); - expect(response).toBeDefined(); + describe("Using optional parameters", () => { + let buffer: Buffer; + let uploadOptions: IUploadOptions; + let reqHeaders: IHeaderContent[]; + let endpoint: string; + beforeAll(() => { + buffer = Buffer.from("testing"); + endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_DS_FILES, dsName); + }); - expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); - expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, - reqHeaders, - writeData: buffer}); - zosmfPutFullSpy.mockClear(); + beforeEach(() => { + uploadOptions = {}; + reqHeaders = [ZosmfHeaders.X_IBM_TEXT]; + zosmfPutFullSpy.mockClear(); + }); - // Unit test for no wait option - uploadOptions.recall = "nowait"; - reqHeaders = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT]; + it("return with proper response when uploading with 'binary' option", async () => { + uploadOptions.binary = true; + reqHeaders = [ZosmfHeaders.X_IBM_BINARY]; - try { - response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); - } catch (err) { - error = err; - } + try { + response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); + } catch (err) { + error = err; + } - expect(error).toBeUndefined(); - expect(response).toBeDefined(); + expect(error).toBeUndefined(); + expect(response).toBeDefined(); - expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); - expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, - reqHeaders, - writeData: buffer}); - zosmfPutFullSpy.mockClear(); + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders, + writeData: buffer}); + }); + it("return with proper response when uploading with 'recall wait' option", async () => { - // Unit test for no error option - uploadOptions.recall = "error"; - reqHeaders = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_ERROR]; + // Unit test for wait option + uploadOptions.recall = "wait"; + reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_WAIT); - try { - response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); - } catch (err) { - error = err; - } + try { + response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); + } catch (err) { + error = err; + } - expect(error).toBeUndefined(); - expect(response).toBeDefined(); + expect(error).toBeUndefined(); + expect(response).toBeDefined(); - expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); - expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, - reqHeaders, - writeData: buffer}); - zosmfPutFullSpy.mockClear(); + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders, + writeData: buffer}); + }); + it("return with proper response when uploading with 'recall nowait' option", async () => { + // Unit test for no wait option + uploadOptions.recall = "nowait"; + reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT); + + try { + response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); + } catch (err) { + error = err; + } - // Unit test default value - uploadOptions.recall = "non-existing"; - reqHeaders = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT]; + expect(error).toBeUndefined(); + expect(response).toBeDefined(); - try { - response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); - } catch (err) { - error = err; - } + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders, + writeData: buffer}); + }); + it("return with proper response when uploading with 'recall error' option", async () => { + // Unit test for no error option + uploadOptions.recall = "error"; + reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_ERROR); + + try { + response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); + } catch (err) { + error = err; + } - expect(error).toBeUndefined(); - expect(response).toBeDefined(); + expect(error).toBeUndefined(); + expect(response).toBeDefined(); - expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); - expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, - reqHeaders, - writeData: buffer}); - zosmfPutFullSpy.mockClear(); + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders, + writeData: buffer}); + }); + it("return with proper response when uploading with non-exiting recall option", async () => { + // Unit test default value + uploadOptions.recall = "non-existing"; + reqHeaders.push(ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT); + + try { + response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); + } catch (err) { + error = err; + } - // Unit test for pass etag option - uploadOptions.etag = etagValue; - reqHeaders = [ZosmfHeaders.X_IBM_BINARY, ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT, {"If-Match" : uploadOptions.etag}]; + expect(error).toBeUndefined(); + expect(response).toBeDefined(); - try { - response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); - } catch (err) { - error = err; - } + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders, + writeData: buffer}); + }); + it("return with proper response when uploading with pass 'etag' option", async () => { + // Unit test for pass etag option + uploadOptions.etag = etagValue; + reqHeaders.push({"If-Match" : uploadOptions.etag}); + + try { + response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); + } catch (err) { + error = err; + } - expect(error).toBeUndefined(); - expect(response).toBeDefined(); + expect(error).toBeUndefined(); + expect(response).toBeDefined(); - expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); - expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, - reqHeaders, - writeData: buffer}); - zosmfPutFullSpy.mockClear(); - zosmfPutFullSpy.mockImplementationOnce(() => fakeResponseWithEtag); - // Unit test for return etag option - reqHeaders = [ZosmfHeaders.X_IBM_BINARY, - ZosmfHeaders.X_IBM_MIGRATED_RECALL_NO_WAIT, - {"If-Match" : uploadOptions.etag}, - ZosmfHeaders.X_IBM_RETURN_ETAG]; - uploadOptions.returnEtag = true; - try { - response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); - } catch (err) { - error = err; - } + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders, + writeData: buffer}); + }); + it("return with proper response when uploading with return 'etag' option", async () => { + zosmfPutFullSpy.mockImplementationOnce(() => fakeResponseWithEtag); + // Unit test for return etag option + reqHeaders.push(ZosmfHeaders.X_IBM_RETURN_ETAG); + uploadOptions.returnEtag = true; + try { + response = await Upload.bufferToDataSet(dummySession, buffer, dsName, uploadOptions); + } catch (err) { + error = err; + } - expect(error).toBeUndefined(); - expect(response).toBeDefined(); + expect(error).toBeUndefined(); + expect(response).toBeDefined(); - expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); - expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, - reqHeaders, - writeData: buffer, - dataToReturn: [CLIENT_PROPERTY.response]}); + expect(zosmfPutFullSpy).toHaveBeenCalledTimes(1); + expect(zosmfPutFullSpy).toHaveBeenCalledWith(dummySession, {resource: endpoint, + reqHeaders, + writeData: buffer, + dataToReturn: [CLIENT_PROPERTY.response]}); + }); }); it("return with proper response when upload dataset with specify volume option", async () => { const buffer: Buffer = Buffer.from("testing"); @@ -1265,22 +1277,6 @@ describe("z/OS Files - Upload", () => { requestStream: inputStream, normalizeRequestNewLines: true}); }); - // it("return with proper response when upload USS file using deprecated function", async () => { - // const uploadStreamToUssFileSpy = jest.spyOn(Upload, "streamToUssFile"); - // try { - // USSresponse = await Upload.streamToUSSFile(dummySession, dsName, inputStream); - // } catch (err) { - // error = err; - // } - - // expect(error).toBeUndefined(); - // expect(USSresponse).toBeDefined(); - // expect(USSresponse.success).toBeTruthy(); - - // expect(uploadStreamToUssFileSpy).toHaveBeenCalledTimes(1); - // expect(uploadStreamToUssFileSpy).toHaveBeenCalledWith(dummySession, dsName, inputStream, { - // binary: false}); - // }); it("return with proper response when upload USS file in binary", async () => { const endpoint = path.posix.join(ZosFilesConstants.RESOURCE, ZosFilesConstants.RES_USS_FILES, dsName); const reqHeaders = [ZosmfHeaders.OCTET_STREAM, ZosmfHeaders.X_IBM_BINARY]; @@ -1914,4 +1910,66 @@ describe("z/OS Files - Upload", () => { }); }); }); + + describe("Deprecated Functions", () => { + let USSresponse: IZosFilesResponse; + const streamToUssFileSpy = jest.spyOn(Upload, "streamToUssFile"); + const fileToUssFileSpy = jest.spyOn(Upload, "fileToUssFile"); + + beforeEach(() => { + USSresponse = undefined; + error = undefined; + + streamToUssFileSpy.mockReset(); + streamToUssFileSpy.mockImplementation(() => null); + + fileToUssFileSpy.mockClear(); + fileToUssFileSpy.mockImplementation(() => null); + }); + + it("return with proper response when upload USS file using deprecated function - streamToUSSFile", async () => { + const inputStream = new Readable(); + inputStream.push("testing"); + inputStream.push(null); + const streamResponse: IZosFilesResponse = { + success: true, + commandResponse: undefined, + apiResponse: undefined }; + streamToUssFileSpy.mockReturnValueOnce(streamResponse); + try { + USSresponse = await Upload.streamToUSSFile(dummySession, dsName, inputStream); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(USSresponse).toBeDefined(); + expect(USSresponse.success).toBeTruthy(); + + expect(streamToUssFileSpy).toHaveBeenCalledTimes(1); + expect(streamToUssFileSpy).toHaveBeenCalledWith(dummySession, dsName, inputStream, { + binary: false}); + }); + it("return with proper response when upload USS file using deprecated function - fileToUSSFile", async () => { + const inputFile = "/path/to/file1.txt"; + const streamResponse: IZosFilesResponse = { + success: true, + commandResponse: undefined, + apiResponse: undefined }; + fileToUssFileSpy.mockReturnValueOnce(streamResponse); + try { + USSresponse = await Upload.fileToUSSFile(dummySession, dsName, inputFile); + } catch (err) { + error = err; + } + + expect(error).toBeUndefined(); + expect(USSresponse).toBeDefined(); + expect(USSresponse.success).toBeTruthy(); + + expect(fileToUssFileSpy).toHaveBeenCalledTimes(1); + expect(fileToUssFileSpy).toHaveBeenCalledWith(dummySession, dsName, inputFile, { + binary: false}); + }); + }); });