feat: download data progress (#268)

README.md

@@ -265,6 +265,13 @@ Downloading data from a file path
 const data = await fdp.file.downloadData('my-new-pod', '/myfile.txt')
 console.log(data.text()) // prints data content in text format 'Hello world!'
 
+// you can also track the progress of data download
+// using the callback, you can track not only the progress of downloaded blocks but also other time-consuming operations required for data download
+await fdp.file.downloadData('my-new-pod', '/myfile.txt', {
+  progressCallback: event => {
+    console.log(event)
+  }
+})
 ```
 
 Deleting a pod

src/file/file.ts

@@ -14,7 +14,7 @@ import {
 import { writeFeedData } from '../feed/api'
 import { downloadData, uploadData } from './handler'
 import { getFileMetadataRawBytes, rawFileMetadataToFileMetadata } from './adapter'
-import { DataUploadOptions, FileReceiveOptions, FileShareInfo } from './types'
+import { DataDownloadOptions, DataUploadOptions, FileReceiveOptions, FileShareInfo } from './types'
 import { addEntryToDirectory, DEFAULT_UPLOAD_OPTIONS, removeEntryFromDirectory } from '../content-items/handler'
 import { Reference } from '@ethersphere/bee-js'
 import { getRawMetadata } from '../content-items/utils'
@@ -32,20 +32,19 @@ export class File {
    *
    * @param podName pod where file is stored
    * @param fullPath full path of the file
+   * @param options download options
    */
-  async downloadData(podName: string, fullPath: string): Promise<Uint8Array> {
+  async downloadData(podName: string, fullPath: string, options?: DataDownloadOptions): Promise<Uint8Array> {
     assertAccount(this.accountData)
     assertPodName(podName)
     assertFullPathWithName(fullPath)
-    assertPodName(podName)
-    const { podAddress, pod } = await getExtendedPodsListByAccountData(this.accountData, podName)
 
     return downloadData(
-      this.accountData.connection.bee,
+      this.accountData,
+      podName,
       fullPath,
-      podAddress,
-      pod.password,
       this.accountData.connection.options?.requestOptions,
+      options,
     )
   }
 

src/file/handler.ts

@@ -8,6 +8,7 @@ import {
   downloadBlocksManifest,
   extractPathInfo,
   getFileMode,
+  updateDownloadProgress,
   updateUploadProgress,
   uploadBytes,
 } from './utils'
@@ -16,7 +17,7 @@ import { blocksToManifest, getFileMetadataRawBytes, rawFileMetadataToFileMetadat
 import { assertRawFileMetadata } from '../directory/utils'
 import { getCreationPathInfo, getRawMetadata } from '../content-items/utils'
 import { PodPasswordBytes } from '../utils/encryption'
-import { Blocks, DataUploadOptions, UploadProgressType } from './types'
+import { Blocks, DataDownloadOptions, DataUploadOptions, DownloadProgressType, UploadProgressType } from './types'
 import { assertPodName, getExtendedPodsListByAccountData, META_VERSION } from '../pod/utils'
 import { getUnixTimestamp } from '../utils/time'
 import { addEntryToDirectory, DEFAULT_UPLOAD_OPTIONS } from '../content-items/handler'
@@ -60,26 +61,32 @@ export async function getFileMetadata(
 /**
  * Downloads file parts and compile them into Data
  *
- * @param bee Bee client
+ * @param accountData account data
+ * @param podName pod name
  * @param fullPath full path to the file
- * @param address address of the pod
- * @param podPassword bytes for data encryption from pod metadata
  * @param downloadOptions download options
+ * @param dataDownloadOptions data download options
  */
 export async function downloadData(
-  bee: Bee,
+  accountData: AccountData,
+  podName: string,
   fullPath: string,
-  address: EthAddress,
-  podPassword: PodPasswordBytes,
   downloadOptions?: BeeRequestOptions,
+  dataDownloadOptions?: DataDownloadOptions,
 ): Promise<Data> {
-  const fileMetadata = await getFileMetadata(bee, fullPath, address, podPassword, downloadOptions)
+  dataDownloadOptions = dataDownloadOptions ?? {}
+  const bee = accountData.connection.bee
+  updateDownloadProgress(dataDownloadOptions, DownloadProgressType.GetPodInfo)
+  const { podAddress, pod } = await getExtendedPodsListByAccountData(accountData, podName)
+  updateDownloadProgress(dataDownloadOptions, DownloadProgressType.GetPathInfo)
+  const fileMetadata = await getFileMetadata(bee, fullPath, podAddress, pod.password, downloadOptions)
 
   if (fileMetadata.compression) {
     // TODO: implement compression support
     throw new Error('Compressed data is not supported yet')
   }
 
+  updateDownloadProgress(dataDownloadOptions, DownloadProgressType.DownloadBlocksMeta)
   const blocks = await downloadBlocksManifest(bee, fileMetadata.blocksReference, downloadOptions)
 
   let totalLength = 0
@@ -89,12 +96,22 @@ export async function downloadData(
 
   const result = new Uint8Array(totalLength)
   let offset = 0
-  for (const block of blocks.blocks) {
+  const totalBlocks = blocks.blocks.length
+  for (const [currentBlockId, block] of blocks.blocks.entries()) {
+    const blockData = {
+      totalBlocks,
+      currentBlockId,
+      percentage: calcUploadBlockPercentage(currentBlockId, totalBlocks),
+    }
+    updateDownloadProgress(dataDownloadOptions, DownloadProgressType.DownloadBlockStart, blockData)
     const data = await bee.downloadData(block.reference, downloadOptions)
+    updateDownloadProgress(dataDownloadOptions, DownloadProgressType.DownloadBlockEnd, blockData)
     result.set(data, offset)
     offset += data.length
   }
 
+  updateDownloadProgress(dataDownloadOptions, DownloadProgressType.Done)
+
   return wrapBytesWithHelpers(result)
 }
 
@@ -123,7 +140,6 @@ export async function uploadData(
 ): Promise<FileMetadata> {
   assertPodName(podName)
   assertFullPathWithName(fullPath)
-  assertPodName(podName)
   assertWallet(accountData.wallet)
 
   const blockSize = options.blockSize ?? Number(DEFAULT_UPLOAD_OPTIONS!.blockSize)
@@ -146,23 +162,20 @@ export async function uploadData(
   const totalBlocks = Math.ceil(data.length / blockSize)
   const blocks: Blocks = { blocks: [] }
   for (let i = 0; i < totalBlocks; i++) {
-    updateUploadProgress(options, UploadProgressType.UploadBlockStart, {
+    const blockData = {
       totalBlocks,
       currentBlockId: i,
-      uploadPercentage: calcUploadBlockPercentage(i, totalBlocks),
-    })
+      percentage: calcUploadBlockPercentage(i, totalBlocks),
+    }
+    updateUploadProgress(options, UploadProgressType.UploadBlockStart, blockData)
     const currentBlock = data.slice(i * blockSize, (i + 1) * blockSize)
     const result = await uploadBytes(connection, currentBlock)
     blocks.blocks.push({
       size: currentBlock.length,
       compressedSize: currentBlock.length,
       reference: result.reference,
     })
-    updateUploadProgress(options, UploadProgressType.UploadBlockEnd, {
-      totalBlocks,
-      currentBlockId: i,
-      uploadPercentage: calcUploadBlockPercentage(i, totalBlocks),
-    })
+    updateUploadProgress(options, UploadProgressType.UploadBlockEnd, blockData)
   }
 
   updateUploadProgress(options, UploadProgressType.UploadBlocksMeta)

src/file/types.ts

@@ -1,6 +1,55 @@
 import { Reference } from '@ethersphere/bee-js'
 import { RawFileMetadata } from '../pod/types'
 
+/**
+ * Download progress info
+ */
+export interface DownloadProgressInfo {
+  /**
+   * Type of the progress
+   */
+  progressType: DownloadProgressType
+  /**
+   * Data of the progress
+   */
+  data?: ProgressBlockData
+}
+
+/**
+ * Data download options
+ */
+export type DataDownloadOptions = ProgressCallback<DownloadProgressInfo>
+
+/**
+ * Download progress types
+ */
+export enum DownloadProgressType {
+  /**
+   * Getting pod info
+   */
+  GetPodInfo = 'get-pod-info',
+  /**
+   * Getting path info
+   */
+  GetPathInfo = 'get-path-info',
+  /**
+   * Downloading file blocks meta
+   */
+  DownloadBlocksMeta = 'download-blocks-meta',
+  /**
+   * Downloading a file block start
+   */
+  DownloadBlockStart = 'download-block-start',
+  /**
+   * Downloading a file block end
+   */
+  DownloadBlockEnd = 'download-block-end',
+  /**
+   * Done
+   */
+  Done = 'done',
+}
+
 /**
  * Uploading progress types
  */
@@ -14,7 +63,7 @@ export enum UploadProgressType {
    */
   GetPathInfo = 'get-path-info',
   /**
-   * Uploading file block start
+   * Uploading a file block start
    */
   UploadBlockStart = 'upload-block-start',
   /**
@@ -40,21 +89,21 @@ export enum UploadProgressType {
 }
 
 /**
- * Uploading progress block data
+ * Processing progress block data
  */
-export interface UploadProgressBlockData {
+export interface ProgressBlockData {
   /**
-   * Total number of blocks that will be uploaded
+   * Total number of blocks that will be processed
    */
   totalBlocks: number
   /**
    * ID of the currently processing block starting from 0
    */
   currentBlockId: number
   /**
-   * Percentage of blocks uploaded
+   * Percentage of blocks processed
    */
-  uploadPercentage: number
+  percentage: number
 }
 
 /**
@@ -68,13 +117,20 @@ export interface UploadProgressInfo {
   /**
    * Data of the progress
    */
-  data?: UploadProgressBlockData
+  data?: ProgressBlockData
+}
+
+/**
+ * Progress callback
+ */
+export interface ProgressCallback<T> {
+  progressCallback?: (info: T) => void
 }
 
 /**
  * File upload options
  */
-export interface DataUploadOptions {
+export interface DataUploadOptions extends ProgressCallback<UploadProgressInfo> {
   /**
    * Size of blocks in bytes will the file be divided
    */
@@ -83,11 +139,6 @@ export interface DataUploadOptions {
    * Content type of the file
    */
   contentType?: string
-  /**
-   * Progress callback
-   * @param info progress info
-   */
-  progressCallback?: (info: UploadProgressInfo) => void
 }
 
 /**

src/file/utils.ts

@@ -7,8 +7,10 @@ import {
   FileShareInfo,
   RawBlock,
   RawBlocks,
-  UploadProgressBlockData,
+  ProgressBlockData,
   UploadProgressType,
+  DataDownloadOptions,
+  DownloadProgressType,
 } from './types'
 import { rawBlocksToBlocks } from './adapter'
 import CryptoJS from 'crypto-js'
@@ -257,7 +259,25 @@ export function getFileMode(mode: number): number {
 export function updateUploadProgress(
   options: DataUploadOptions,
   progressType: UploadProgressType,
-  data?: UploadProgressBlockData,
+  data?: ProgressBlockData,
+): void {
+  if (!options.progressCallback) {
+    return
+  }
+
+  options.progressCallback({ progressType, data })
+}
+
+/**
+ * Updates download progress
+ * @param options download options
+ * @param progressType progress type
+ * @param data progress data
+ */
+export function updateDownloadProgress(
+  options: DataDownloadOptions,
+  progressType: DownloadProgressType,
+  data?: ProgressBlockData,
 ): void {
   if (!options.progressCallback) {
     return

src/types.ts

@@ -3,7 +3,15 @@ import { EnsEnvironment } from '@fairdatasociety/fdp-contracts-js'
 import { CacheOptions } from './cache/types'
 
 export { DirectoryItem, FileItem } from './content-items/types'
-export { UploadProgressType, UploadProgressBlockData, UploadProgressInfo, DataUploadOptions } from './file/types'
+export {
+  UploadProgressType,
+  DownloadProgressType,
+  ProgressBlockData,
+  UploadProgressInfo,
+  DownloadProgressInfo,
+  DataUploadOptions,
+  DataDownloadOptions,
+} from './file/types'
 
 /**
  * Fair Data Protocol options

test/integration/node/download-progress.spec.ts

@@ -0,0 +1,40 @@
+import { createFdp, generateRandomHexString, generateUser } from '../../utils'
+import { wrapBytesWithHelpers } from '../../../src/utils/bytes'
+import { DownloadProgressInfo } from '../../../src'
+import { DEFAULT_UPLOAD_OPTIONS } from '../../../src/content-items/handler'
+
+jest.setTimeout(400000)
+it('Fair Data Protocol download progress', async () => {
+  const fdp = createFdp()
+  generateUser(fdp)
+  const pod = generateRandomHexString()
+  const fileSizeBig = 5000005
+  const blocksCount = Math.ceil(fileSizeBig / DEFAULT_UPLOAD_OPTIONS.blockSize!)
+  const contentBig = generateRandomHexString(fileSizeBig)
+  const filenameBig = generateRandomHexString() + '.txt'
+  const fullFilenameBigPath = '/' + filenameBig
+  const callbackData = []
+
+  const progressCallback = (progressInfo: DownloadProgressInfo) => {
+    callbackData.push(progressInfo)
+  }
+
+  await fdp.personalStorage.create(pod)
+  await fdp.file.uploadData(pod, fullFilenameBigPath, contentBig)
+  const dataBig = wrapBytesWithHelpers(
+    await fdp.file.downloadData(pod, fullFilenameBigPath, {
+      progressCallback,
+    }),
+  ).text()
+  expect(dataBig).toEqual(contentBig)
+  const fdpList = await fdp.directory.read(pod, '/', true)
+  expect(fdpList.files.length).toEqual(1)
+  const fileInfoBig = fdpList.files[0]
+  expect(fileInfoBig.name).toEqual(filenameBig)
+  expect(fileInfoBig.size).toEqual(fileSizeBig)
+
+  // multiply `blocksCount` by 2 because each block has two events.
+  // the 4 other events from `DownloadProgressType` occur once each
+  const totalEvents = blocksCount * 2 + 4
+  expect(callbackData.length).toEqual(totalEvents)
+})