From 7c260e41803df43138c8ce01114369748debab0b Mon Sep 17 00:00:00 2001 From: Igor Shadurin Date: Fri, 6 Oct 2023 14:34:10 +0300 Subject: [PATCH] feat: download data progress (#268) --- README.md | 7 ++ src/file/file.ts | 13 ++-- src/file/handler.ts | 49 +++++++----- src/file/types.ts | 77 +++++++++++++++---- src/file/utils.ts | 24 +++++- src/types.ts | 10 ++- .../node/download-progress.spec.ts | 40 ++++++++++ 7 files changed, 179 insertions(+), 41 deletions(-) create mode 100644 test/integration/node/download-progress.spec.ts diff --git a/README.md b/README.md index 4cc2f752..2c94eaba 100644 --- a/README.md +++ b/README.md @@ -265,6 +265,13 @@ Downloading data from a file path const data = await fdp.file.downloadData('my-new-pod', '/myfile.txt') console.log(data.text()) // prints data content in text format 'Hello world!' +// you can also track the progress of data download +// using the callback, you can track not only the progress of downloaded blocks but also other time-consuming operations required for data download +await fdp.file.downloadData('my-new-pod', '/myfile.txt', { + progressCallback: event => { + console.log(event) + } +}) ``` Deleting a pod diff --git a/src/file/file.ts b/src/file/file.ts index 39580ecc..55c80a80 100644 --- a/src/file/file.ts +++ b/src/file/file.ts @@ -14,7 +14,7 @@ import { import { writeFeedData } from '../feed/api' import { downloadData, uploadData } from './handler' import { getFileMetadataRawBytes, rawFileMetadataToFileMetadata } from './adapter' -import { DataUploadOptions, FileReceiveOptions, FileShareInfo } from './types' +import { DataDownloadOptions, DataUploadOptions, FileReceiveOptions, FileShareInfo } from './types' import { addEntryToDirectory, DEFAULT_UPLOAD_OPTIONS, removeEntryFromDirectory } from '../content-items/handler' import { Reference } from '@ethersphere/bee-js' import { getRawMetadata } from '../content-items/utils' @@ -32,20 +32,19 @@ export class File { * * @param podName pod where file is stored * @param fullPath full path of the file + * @param options download options */ - async downloadData(podName: string, fullPath: string): Promise { + async downloadData(podName: string, fullPath: string, options?: DataDownloadOptions): Promise { assertAccount(this.accountData) assertPodName(podName) assertFullPathWithName(fullPath) - assertPodName(podName) - const { podAddress, pod } = await getExtendedPodsListByAccountData(this.accountData, podName) return downloadData( - this.accountData.connection.bee, + this.accountData, + podName, fullPath, - podAddress, - pod.password, this.accountData.connection.options?.requestOptions, + options, ) } diff --git a/src/file/handler.ts b/src/file/handler.ts index 647d57cd..031ad094 100644 --- a/src/file/handler.ts +++ b/src/file/handler.ts @@ -8,6 +8,7 @@ import { downloadBlocksManifest, extractPathInfo, getFileMode, + updateDownloadProgress, updateUploadProgress, uploadBytes, } from './utils' @@ -16,7 +17,7 @@ import { blocksToManifest, getFileMetadataRawBytes, rawFileMetadataToFileMetadat import { assertRawFileMetadata } from '../directory/utils' import { getCreationPathInfo, getRawMetadata } from '../content-items/utils' import { PodPasswordBytes } from '../utils/encryption' -import { Blocks, DataUploadOptions, UploadProgressType } from './types' +import { Blocks, DataDownloadOptions, DataUploadOptions, DownloadProgressType, UploadProgressType } from './types' import { assertPodName, getExtendedPodsListByAccountData, META_VERSION } from '../pod/utils' import { getUnixTimestamp } from '../utils/time' import { addEntryToDirectory, DEFAULT_UPLOAD_OPTIONS } from '../content-items/handler' @@ -60,26 +61,32 @@ export async function getFileMetadata( /** * Downloads file parts and compile them into Data * - * @param bee Bee client + * @param accountData account data + * @param podName pod name * @param fullPath full path to the file - * @param address address of the pod - * @param podPassword bytes for data encryption from pod metadata * @param downloadOptions download options + * @param dataDownloadOptions data download options */ export async function downloadData( - bee: Bee, + accountData: AccountData, + podName: string, fullPath: string, - address: EthAddress, - podPassword: PodPasswordBytes, downloadOptions?: BeeRequestOptions, + dataDownloadOptions?: DataDownloadOptions, ): Promise { - const fileMetadata = await getFileMetadata(bee, fullPath, address, podPassword, downloadOptions) + dataDownloadOptions = dataDownloadOptions ?? {} + const bee = accountData.connection.bee + updateDownloadProgress(dataDownloadOptions, DownloadProgressType.GetPodInfo) + const { podAddress, pod } = await getExtendedPodsListByAccountData(accountData, podName) + updateDownloadProgress(dataDownloadOptions, DownloadProgressType.GetPathInfo) + const fileMetadata = await getFileMetadata(bee, fullPath, podAddress, pod.password, downloadOptions) if (fileMetadata.compression) { // TODO: implement compression support throw new Error('Compressed data is not supported yet') } + updateDownloadProgress(dataDownloadOptions, DownloadProgressType.DownloadBlocksMeta) const blocks = await downloadBlocksManifest(bee, fileMetadata.blocksReference, downloadOptions) let totalLength = 0 @@ -89,12 +96,22 @@ export async function downloadData( const result = new Uint8Array(totalLength) let offset = 0 - for (const block of blocks.blocks) { + const totalBlocks = blocks.blocks.length + for (const [currentBlockId, block] of blocks.blocks.entries()) { + const blockData = { + totalBlocks, + currentBlockId, + percentage: calcUploadBlockPercentage(currentBlockId, totalBlocks), + } + updateDownloadProgress(dataDownloadOptions, DownloadProgressType.DownloadBlockStart, blockData) const data = await bee.downloadData(block.reference, downloadOptions) + updateDownloadProgress(dataDownloadOptions, DownloadProgressType.DownloadBlockEnd, blockData) result.set(data, offset) offset += data.length } + updateDownloadProgress(dataDownloadOptions, DownloadProgressType.Done) + return wrapBytesWithHelpers(result) } @@ -123,7 +140,6 @@ export async function uploadData( ): Promise { assertPodName(podName) assertFullPathWithName(fullPath) - assertPodName(podName) assertWallet(accountData.wallet) const blockSize = options.blockSize ?? Number(DEFAULT_UPLOAD_OPTIONS!.blockSize) @@ -146,11 +162,12 @@ export async function uploadData( const totalBlocks = Math.ceil(data.length / blockSize) const blocks: Blocks = { blocks: [] } for (let i = 0; i < totalBlocks; i++) { - updateUploadProgress(options, UploadProgressType.UploadBlockStart, { + const blockData = { totalBlocks, currentBlockId: i, - uploadPercentage: calcUploadBlockPercentage(i, totalBlocks), - }) + percentage: calcUploadBlockPercentage(i, totalBlocks), + } + updateUploadProgress(options, UploadProgressType.UploadBlockStart, blockData) const currentBlock = data.slice(i * blockSize, (i + 1) * blockSize) const result = await uploadBytes(connection, currentBlock) blocks.blocks.push({ @@ -158,11 +175,7 @@ export async function uploadData( compressedSize: currentBlock.length, reference: result.reference, }) - updateUploadProgress(options, UploadProgressType.UploadBlockEnd, { - totalBlocks, - currentBlockId: i, - uploadPercentage: calcUploadBlockPercentage(i, totalBlocks), - }) + updateUploadProgress(options, UploadProgressType.UploadBlockEnd, blockData) } updateUploadProgress(options, UploadProgressType.UploadBlocksMeta) diff --git a/src/file/types.ts b/src/file/types.ts index 06a62ab1..c9dd643d 100644 --- a/src/file/types.ts +++ b/src/file/types.ts @@ -1,6 +1,55 @@ import { Reference } from '@ethersphere/bee-js' import { RawFileMetadata } from '../pod/types' +/** + * Download progress info + */ +export interface DownloadProgressInfo { + /** + * Type of the progress + */ + progressType: DownloadProgressType + /** + * Data of the progress + */ + data?: ProgressBlockData +} + +/** + * Data download options + */ +export type DataDownloadOptions = ProgressCallback + +/** + * Download progress types + */ +export enum DownloadProgressType { + /** + * Getting pod info + */ + GetPodInfo = 'get-pod-info', + /** + * Getting path info + */ + GetPathInfo = 'get-path-info', + /** + * Downloading file blocks meta + */ + DownloadBlocksMeta = 'download-blocks-meta', + /** + * Downloading a file block start + */ + DownloadBlockStart = 'download-block-start', + /** + * Downloading a file block end + */ + DownloadBlockEnd = 'download-block-end', + /** + * Done + */ + Done = 'done', +} + /** * Uploading progress types */ @@ -14,7 +63,7 @@ export enum UploadProgressType { */ GetPathInfo = 'get-path-info', /** - * Uploading file block start + * Uploading a file block start */ UploadBlockStart = 'upload-block-start', /** @@ -40,11 +89,11 @@ export enum UploadProgressType { } /** - * Uploading progress block data + * Processing progress block data */ -export interface UploadProgressBlockData { +export interface ProgressBlockData { /** - * Total number of blocks that will be uploaded + * Total number of blocks that will be processed */ totalBlocks: number /** @@ -52,9 +101,9 @@ export interface UploadProgressBlockData { */ currentBlockId: number /** - * Percentage of blocks uploaded + * Percentage of blocks processed */ - uploadPercentage: number + percentage: number } /** @@ -68,13 +117,20 @@ export interface UploadProgressInfo { /** * Data of the progress */ - data?: UploadProgressBlockData + data?: ProgressBlockData +} + +/** + * Progress callback + */ +export interface ProgressCallback { + progressCallback?: (info: T) => void } /** * File upload options */ -export interface DataUploadOptions { +export interface DataUploadOptions extends ProgressCallback { /** * Size of blocks in bytes will the file be divided */ @@ -83,11 +139,6 @@ export interface DataUploadOptions { * Content type of the file */ contentType?: string - /** - * Progress callback - * @param info progress info - */ - progressCallback?: (info: UploadProgressInfo) => void } /** diff --git a/src/file/utils.ts b/src/file/utils.ts index 75b03c52..8394b6ef 100644 --- a/src/file/utils.ts +++ b/src/file/utils.ts @@ -7,8 +7,10 @@ import { FileShareInfo, RawBlock, RawBlocks, - UploadProgressBlockData, + ProgressBlockData, UploadProgressType, + DataDownloadOptions, + DownloadProgressType, } from './types' import { rawBlocksToBlocks } from './adapter' import CryptoJS from 'crypto-js' @@ -257,7 +259,25 @@ export function getFileMode(mode: number): number { export function updateUploadProgress( options: DataUploadOptions, progressType: UploadProgressType, - data?: UploadProgressBlockData, + data?: ProgressBlockData, +): void { + if (!options.progressCallback) { + return + } + + options.progressCallback({ progressType, data }) +} + +/** + * Updates download progress + * @param options download options + * @param progressType progress type + * @param data progress data + */ +export function updateDownloadProgress( + options: DataDownloadOptions, + progressType: DownloadProgressType, + data?: ProgressBlockData, ): void { if (!options.progressCallback) { return diff --git a/src/types.ts b/src/types.ts index 88b201de..8bce5835 100644 --- a/src/types.ts +++ b/src/types.ts @@ -3,7 +3,15 @@ import { EnsEnvironment } from '@fairdatasociety/fdp-contracts-js' import { CacheOptions } from './cache/types' export { DirectoryItem, FileItem } from './content-items/types' -export { UploadProgressType, UploadProgressBlockData, UploadProgressInfo, DataUploadOptions } from './file/types' +export { + UploadProgressType, + DownloadProgressType, + ProgressBlockData, + UploadProgressInfo, + DownloadProgressInfo, + DataUploadOptions, + DataDownloadOptions, +} from './file/types' /** * Fair Data Protocol options diff --git a/test/integration/node/download-progress.spec.ts b/test/integration/node/download-progress.spec.ts new file mode 100644 index 00000000..8df3c266 --- /dev/null +++ b/test/integration/node/download-progress.spec.ts @@ -0,0 +1,40 @@ +import { createFdp, generateRandomHexString, generateUser } from '../../utils' +import { wrapBytesWithHelpers } from '../../../src/utils/bytes' +import { DownloadProgressInfo } from '../../../src' +import { DEFAULT_UPLOAD_OPTIONS } from '../../../src/content-items/handler' + +jest.setTimeout(400000) +it('Fair Data Protocol download progress', async () => { + const fdp = createFdp() + generateUser(fdp) + const pod = generateRandomHexString() + const fileSizeBig = 5000005 + const blocksCount = Math.ceil(fileSizeBig / DEFAULT_UPLOAD_OPTIONS.blockSize!) + const contentBig = generateRandomHexString(fileSizeBig) + const filenameBig = generateRandomHexString() + '.txt' + const fullFilenameBigPath = '/' + filenameBig + const callbackData = [] + + const progressCallback = (progressInfo: DownloadProgressInfo) => { + callbackData.push(progressInfo) + } + + await fdp.personalStorage.create(pod) + await fdp.file.uploadData(pod, fullFilenameBigPath, contentBig) + const dataBig = wrapBytesWithHelpers( + await fdp.file.downloadData(pod, fullFilenameBigPath, { + progressCallback, + }), + ).text() + expect(dataBig).toEqual(contentBig) + const fdpList = await fdp.directory.read(pod, '/', true) + expect(fdpList.files.length).toEqual(1) + const fileInfoBig = fdpList.files[0] + expect(fileInfoBig.name).toEqual(filenameBig) + expect(fileInfoBig.size).toEqual(fileSizeBig) + + // multiply `blocksCount` by 2 because each block has two events. + // the 4 other events from `DownloadProgressType` occur once each + const totalEvents = blocksCount * 2 + 4 + expect(callbackData.length).toEqual(totalEvents) +})