add functions to lib

cli/src/lib/website/chunk.ts

@@ -3,6 +3,8 @@ import { sha256 } from 'js-sha256'
 import { storageCostForEntry } from '../utils/storage'
 import { fileChunkKey } from './storageKeys'
 import { FileChunkPost } from './models/FileChunkPost'
+import { getFileFromAddress } from './read'
+import { Provider, SmartContract } from '@massalabs/massa-web3'
 
 /**
  * Divide a data array into chunks of a given size.
@@ -74,3 +76,46 @@ export function toChunkPosts(
     return new FileChunkPost(filepath, BigInt(index), chunk)
   })
 }
+
+
+
+
+/**
+ * Check if the file requires update
+ * @param provider - the web3 provider
+ * @param location - the file path
+ * @param localFileContent - the local file content
+ * @param sc - the smart contract
+ * @returns true if the file requires update, false otherwise
+ */
+export async function requiresUpdate(
+  provider: Provider,
+  location: string,
+  localFileContent: Uint8Array,
+  sc?: SmartContract
+): Promise<boolean> {
+  if (localFileContent.length === 0) {
+    return false
+  }
+
+  if (!sc) {
+    return true
+  }
+
+  var onChainFileContent: Uint8Array
+  try {
+    onChainFileContent = await getFileFromAddress(provider, sc, location)
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  } catch (_) {
+    return true
+  }
+
+  if (onChainFileContent.length !== localFileContent.length) {
+    return true
+  }
+
+  const localFileHash = sha256(localFileContent)
+  const onChainFileHash = sha256(onChainFileContent)
+
+  return localFileHash !== onChainFileHash
+}

cli/src/lib/website/filesInit.ts

@@ -5,6 +5,7 @@ import {
   MIN_GAS_CALL,
   minBigInt,
   Operation,
+  Provider,
   SmartContract,
   strToBytes,
   U32,
@@ -13,6 +14,7 @@ import { storageCostForEntry } from '../utils/storage'
 import { FileInit } from './models/FileInit'
 import { Metadata } from './models/Metadata'
 import {
+  FILE_TAG,
   fileChunkCountKey,
   fileLocationKey,
   globalMetadataKey,
@@ -313,3 +315,62 @@ class Batch {
     )
   }
 }
+
+/**
+ * Filter out pre-stores that are already stored on the blockchain
+ * @param provider - the web3 provider
+ * @param scAddress - the smart contract address
+ * @param fileInits - the pre-stores to filter
+ * @returns the pre-stores that are not stored on the blockchain
+ */
+export async function filterUselessFileInits(
+  provider: Provider,
+  scAddress: string,
+  fileInits: FileInit[]
+): Promise<FileInit[]> {
+  const fileInitsWithKey = fileInits.map((preStore) => {
+    return {
+      preStore: preStore,
+      totalChunkKey: fileChunkCountKey(preStore.hashLocation),
+    }
+  })
+
+  const batches: {
+    preStore: FileInit
+    totalChunkKey: Uint8Array
+  }[][] = []
+
+  for (let i = 0; i < fileInitsWithKey.length; i += 100) {
+    batches.push(fileInitsWithKey.slice(i, i + 100))
+  }
+
+  const fileInitsToKeep: FileInit[] = []
+
+  for (const batch of batches) {
+    const keys = await provider.getStorageKeys(scAddress, FILE_TAG)
+
+    // Remove missing keys from the batch and add them to the list of files to keep
+    for (let i = batch.length - 1; i >= 0; i--) {
+      if (!keys.includes(batch[i].totalChunkKey)) {
+        fileInitsToKeep.push(batch[i].preStore)
+        batch.splice(i, 1)
+      }
+    }
+
+    const results = await provider.readStorage(
+      scAddress,
+      batch.map((key) => key.totalChunkKey)
+    )
+
+    for (let i = 0; i < batch.length; i++) {
+      if (
+        results[i].length !== U32.SIZE_BYTE ||
+        U32.fromBytes(results[i]) !== batch[i].preStore.totalChunk
+      ) {
+        fileInitsToKeep.push(batch[i].preStore)
+      }
+    }
+  }
+
+  return fileInitsToKeep
+}

cli/src/tasks/prepareChunk.ts

@@ -7,7 +7,7 @@ import { Provider, SmartContract, U32 } from '@massalabs/massa-web3'
 
 import { batcher } from '../lib/batcher'
 
-import { divideIntoChunks, toChunkPosts } from '../lib/website/chunk'
+import { divideIntoChunks, requiresUpdate, toChunkPosts } from '../lib/website/chunk'
 import { getFileFromAddress, listFiles } from '../lib/website/read'
 import { FILE_TAG, fileChunkCountKey } from '../lib/website/storageKeys'
 
@@ -16,6 +16,7 @@ import { FileInit } from '../lib/website/models/FileInit'
 
 import { FileDelete } from '../lib/website/models/FileDelete'
 import { UploadCtx } from './tasks'
+import { filterUselessFileInits } from '../lib/website/filesInit'
 
 /**
  * Create a task to prepare batches from the website file
@@ -86,13 +87,15 @@ async function prepareChunks(
   fileInits: FileInit[]
   localFiles: string[]
 }> {
+  //init variables
   const files = readdirSync(dirPath)
 
   const chunks: FileChunkPost[] = []
   const fileInits: FileInit[] = []
 
   const localFiles: string[] = []
 
+  // iterate over the files
   for (const file of files) {
     const fullPath = path.join(dirPath, file)
     const stats = statSync(fullPath)
@@ -131,101 +134,4 @@ async function prepareChunks(
   return { chunks, fileInits, localFiles }
 }
 
-/**
- * Check if the file requires update
- * @param provider - the web3 provider
- * @param location - the file path
- * @param localFileContent - the local file content
- * @param sc - the smart contract
- * @returns true if the file requires update, false otherwise
- */
-async function requiresUpdate(
-  provider: Provider,
-  location: string,
-  localFileContent: Uint8Array,
-  sc?: SmartContract
-): Promise<boolean> {
-  if (localFileContent.length === 0) {
-    return false
-  }
-
-  if (!sc) {
-    return true
-  }
-
-  var onChainFileContent: Uint8Array
-  try {
-    onChainFileContent = await getFileFromAddress(provider, sc, location)
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  } catch (_) {
-    return true
-  }
-
-  if (onChainFileContent.length !== localFileContent.length) {
-    return true
-  }
-
-  const localFileHash = sha256(localFileContent)
-  const onChainFileHash = sha256(onChainFileContent)
-
-  return localFileHash !== onChainFileHash
-}
-
-/**
- * Filter out pre-stores that are already stored on the blockchain
- * @param provider - the web3 provider
- * @param scAddress - the smart contract address
- * @param fileInits - the pre-stores to filter
- * @returns the pre-stores that are not stored on the blockchain
- */
-async function filterUselessFileInits(
-  provider: Provider,
-  scAddress: string,
-  fileInits: FileInit[]
-): Promise<FileInit[]> {
-  const fileInitsWithKey = fileInits.map((preStore) => {
-    return {
-      preStore: preStore,
-      totalChunkKey: fileChunkCountKey(preStore.hashLocation),
-    }
-  })
-
-  const batches: {
-    preStore: FileInit
-    totalChunkKey: Uint8Array
-  }[][] = []
-
-  for (let i = 0; i < fileInitsWithKey.length; i += 100) {
-    batches.push(fileInitsWithKey.slice(i, i + 100))
-  }
-
-  const fileInitsToKeep: FileInit[] = []
-
-  for (const batch of batches) {
-    const keys = await provider.getStorageKeys(scAddress, FILE_TAG)
 
-    // Remove missing keys from the batch and add them to the list of files to keep
-    for (let i = batch.length - 1; i >= 0; i--) {
-      if (!keys.includes(batch[i].totalChunkKey)) {
-        fileInitsToKeep.push(batch[i].preStore)
-        batch.splice(i, 1)
-      }
-    }
-
-    const results = await provider.readStorage(
-      scAddress,
-      batch.map((key) => key.totalChunkKey)
-    )
-
-    for (let i = 0; i < batch.length; i++) {
-      if (
-        results[i].length !== U32.SIZE_BYTE ||
-        U32.fromBytes(results[i]) !== batch[i].preStore.totalChunk
-      ) {
-        fileInitsToKeep.push(batch[i].preStore)
-      }
-    }
-  }
-
-  return fileInitsToKeep
-}