Add docker fix and Parallel processing implementation.

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18.15.0 as build
+FROM node:18.16.0 as build
 ARG REACT_APP_TG_API_ID
 ARG REACT_APP_TG_API_HASH
 
@@ -10,7 +10,9 @@ COPY api/package.json api/package.json
 COPY web/package.json web/package.json
 COPY docker/.env .
 RUN yarn cache clean
-RUN yarn install --network-timeout 1000000
+RUN yarn install
+RUN yarn global add prisma
+RUN npx browserslist@latest --update-db
 COPY . .
 RUN export NODE_OPTIONS="--openssl-legacy-provider --no-experimental-fetch"
 RUN yarn workspaces run build

docker/docker-compose.yml

@@ -2,7 +2,7 @@ version: '3'
 
 services:
   teledrive:
-    command: bash -c "export NODE_OPTIONS='--openssl-legacy-provider --no-experimental-fetch' && yarn api prisma migrate deploy || yarn api prisma migrate resolve --applied 20220420012853_init && node api/dist/index.js"
+    command: bash -c "export NODE_OPTIONS='--openssl-legacy-provider --no-experimental-fetch' && yarn api prisma migrate deploy || yarn api prisma migrate resolve --applied "20220420012853_init" && node api/dist/index.js"
     labels:
       traefik.http.routers.server.rule: Host(`teledrive.localhost`)
       traefik.port: 4000

install.docker.new.sh

@@ -1,70 +1,55 @@
 #!/bin/bash
 
-set -e -x
+set -e
 
-# Check if configuration file exists
-if [ ! -f docker/.env ]; then
-  echo "Error: Configuration file not found."
-  exit 1
-fi
+echo "Node Version: $(node -v)"
+echo "cURL Version: $(curl --version | head -n 1)"
+echo "Docker Version: $(docker -v)"
+echo "Docker Compose Version: $(docker compose version)"
 
-# Create /docker directory if it doesn't exist
-if [ ! -d docker ]; then
-  mkdir docker
-fi
+if [ ! -f docker/.env ]
+then
+  echo "Generating .env file..."
 
-# Configure Node.js and cURL
-export NODE_OPTIONS="--openssl-legacy-provider --no-experimental-fetch"
-printf "Node.js Version: %s\n" "$(node -v)"
-printf "cURL Version: %s\n" "$(curl --version | head -n 1)"
+  ENV="develop"
 
-# Check Docker and Docker Compose versions
-printf "Docker Version: %s\n" "$(docker -v)"
-printf "Docker Compose Version: %s\n" "$(docker compose -v)"
+  echo "Preparing your keys from https://my.telegram.org/"
+  read -p "Enter your TG_API_ID: " TG_API_ID
+  read -p "Enter your TG_API_HASH: " TG_API_HASH
 
-# Disable Git-related functionality in BuildKit
-export DOCKER_BUILDKIT=1
-export BUILDKIT_PROGRESS=plain
-export BUILDKIT_INLINE_CACHE=1
-export BUILDKIT_ENABLE_LEGACY_GIT=0
+  echo
+  read -p "Enter your ADMIN_USERNAME: " ADMIN_USERNAME
+  read -p "Enter your PORT: " PORT
+  PORT="${PORT:=4000}"
 
-# If no parameters are provided, start services using Docker Compose
-if [ $# -eq 0 ]; then
-  # Stop and start services using Docker Compose
-  docker compose down
-  docker compose up --build --force-recreate -d
+  DB_PASSWORD=$(openssl rand -hex 48)
+
+  echo "ENV=$ENV" > docker/.env
+  echo "PORT=$PORT" >> docker/.env
+  echo "TG_API_ID=$TG_API_ID" >> docker/.env
+  echo "TG_API_HASH=$TG_API_HASH" >> docker/.env
+  echo "ADMIN_USERNAME=$ADMIN_USERNAME" >> docker/.env
+  export DATABASE_URL=postgresql://postgres:$DB_PASSWORD@db:5432/teledrive
+  echo "DB_PASSWORD=$DB_PASSWORD" >> docker/.env
+
+  cd docker
+  docker compose build teledrive
+  docker compose up -d
   sleep 2
+  docker compose exec teledrive yarn workspace api prisma migrate reset
   docker compose exec teledrive yarn workspace api prisma migrate deploy
-  # Update PostgreSQL password
-  DB_PASSWORD=$(openssl rand -hex 16)
-  docker compose exec docker-db-1 psql -U postgres -c "ALTER USER postgres PASSWORD '${DB_PASSWORD}';"
-  printf "Generated random DB_PASSWORD: %s\n" "$DB_PASSWORD"
-fi
+else
+  git pull origin experiment
+
+  export $(cat docker/.env | xargs)
 
-# If "update" parameter is provided, update services using Docker Compose
-if [ "$1" == "update" ]; then
   cd docker
-  if ! git branch --list experiment >/dev/null; then
-    git branch experiment origin/experiment
-  fi
-  git checkout experiment
-  export $(grep -v '^#' docker/.env | xargs)
   docker compose down
   docker compose up --build --force-recreate -d
   sleep 2
+  docker compose up -d
   docker compose exec teledrive yarn workspace api prisma migrate deploy
   git reset --hard
   git clean -f
   git pull origin experiment
 fi
-
-# If "permissions" parameter is provided, check if the current user has permission to modify necessary directories and files
-if [ "$1" == "permissions" ]; then
-  if [ ! -w /var/run/docker.sock ] || [ ! -w ./docker/.env ] || [ ! -w ./docker/data ]; then
-    printf "This script requires permission to modify some files and directories.\n"
-    printf "Giving permission to the current user...\n"
-    sudo chown -R "$(whoami)" /var/run/docker.sock ./docker/.env ./docker/data
-  else
-    printf "No permissions required.\n"
-  fi
-fi

web/package.json

@@ -50,7 +50,9 @@
     "use-debounce": "^7.0.1",
     "web-vitals": "^0.2.4",
     "world_countries_lists": "^2.5.1",
-    "concat-stream": "^2.0.0"
+    "concat-stream": "^2.0.0",
+    "axios-parallel": "^1.1.2",
+    "fastpriorityqueue": "0.7.4"
   },
   "scripts": {
     "start": "react-scripts --openssl-legacy-provider --no-experimental-fetch start",

web/src/pages/dashboard/components/Upload.tsx

@@ -37,14 +37,16 @@ const Upload: React.FC<Props> = ({ dataFileList: [fileList, setFileList], parent
         await new Promise(res => setTimeout(res, 3000 * ++retry))
         await cb?.()
         if (retry === RETRY_COUNT) {
-          notification.error({ message: 'Failed to upload file', description: <>
-            <Typography.Paragraph>
-              {error?.response?.data?.error || error.message || 'Something error'}
-            </Typography.Paragraph>
-            <Typography.Paragraph code>
-              {JSON.stringify(error?.response?.data || error?.data || error, null, 2)}
-            </Typography.Paragraph>
-          </> })
+          notification.error({
+            message: 'Failed to upload file', description: <>
+              <Typography.Paragraph>
+                {error?.response?.data?.error || error.message || 'Something error'}
+              </Typography.Paragraph>
+              <Typography.Paragraph code>
+                {JSON.stringify(error?.response?.data || error?.data || error, null, 2)}
+              </Typography.Paragraph>
+            </>
+          })
           throw error
         }
       }
@@ -138,14 +140,17 @@ const Upload: React.FC<Props> = ({ dataFileList: [fileList, setFileList], parent
                         if (type === 'channel') {
                           peer = new Api.InputPeerChannel({
                             channelId: BigInt(peerId) as any,
-                            accessHash: BigInt(accessHash as string) as any })
+                            accessHash: BigInt(accessHash as string) as any
+                          })
                         } else if (type === 'user') {
                           peer = new Api.InputPeerUser({
                             userId: BigInt(peerId.toString()) as any,
-                            accessHash: BigInt(accessHash.toString()) as any })
+                            accessHash: BigInt(accessHash.toString()) as any
+                          })
                         } else if (type === 'chat') {
                           peer = new Api.InputPeerChat({
-                            chatId: BigInt(peerId) as any })
+                            chatId: BigInt(peerId) as any
+                          })
                         }
                       }
                       return await client.sendFile(peer || 'me', {
@@ -160,7 +165,7 @@ const Upload: React.FC<Props> = ({ dataFileList: [fileList, setFileList], parent
                         attributes: forceDocument ? [
                           new Api.DocumentAttributeFilename({ fileName: response.file.name })
                         ] : undefined,
-                        workers: 6
+                        workers: 1
                       })
                     }
 
@@ -203,72 +208,98 @@ const Upload: React.FC<Props> = ({ dataFileList: [fileList, setFileList], parent
           }
         }
       } else {
-        for (let j = 0; j < fileParts; j++) {
-          const fileBlob = file.slice(j * MAX_UPLOAD_SIZE, Math.min(j * MAX_UPLOAD_SIZE + MAX_UPLOAD_SIZE, file.size))
-          const parts = Math.ceil(fileBlob.size / CHUNK_SIZE)
+        const PARALLELISM = 8
 
-          if (!deleted) {
-            const uploadPart = async (i: number) => {
-              if (responses?.length && cancelUploading.current && file.uid === cancelUploading.current) {
-                await Promise.all(responses.map(async response => {
-                  try {
-                    await req.delete(`/files/${response?.file.id}`)
-                  } catch (error) {
-                    // ignore
-                  }
-                }))
-                cancelUploading.current = null
-                deleted = true
-                window.onbeforeunload = undefined as any
-              } else {
-                const blobPart = fileBlob.slice(i * CHUNK_SIZE, Math.min(i * CHUNK_SIZE + CHUNK_SIZE, file.size))
-                const data = new FormData()
-                data.append('upload', blobPart)
+        interface Response {
+          file?: {
+            id: string
+          }
+        }
 
-                const beginUpload = async () => {
-                  const { data: response } = await req.post(`/files/upload${i > 0 && responses[j]?.file?.id ? `/${responses[j]?.file.id}` : ''}`, data, {
-                    params: {
-                      ...parent?.id ? { parent_id: parent.id } : {},
-                      relative_path: file.webkitRelativePath || null,
-                      name: `${file.name}${fileParts > 1 ? `.part${String(j + 1).padStart(3, '0')}` : ''}`,
-                      size: fileBlob.size,
-                      mime_type: file.type || mime.lookup(file.name) || 'application/octet-stream',
-                      part: i,
-                      total_part: parts,
-                    },
-                  })
-                  return response
+        const uploadFile = async (file: File, parent?: { id: string }, onProgress?: (progress: any, file: File) => void) => {
+          const fileParts = Math.ceil(file.size / MAX_UPLOAD_SIZE)
+          const totalAllParts = fileParts * Math.ceil(file.size / CHUNK_SIZE)
+          const totalParts = 0
+          let deleted = false
+          const responses: Response[] = []
+
+          const workerScript = `
+            const uploadPart = async (file, j, i) => {
+              const fileBlob = file.slice(
+                j * MAX_UPLOAD_SIZE,
+                Math.min(j * MAX_UPLOAD_SIZE + MAX_UPLOAD_SIZE, file.size)
+              )
+              const blobPart = fileBlob.slice(i * CHUNK_SIZE, Math.min(i * CHUNK_SIZE + CHUNK_SIZE, file.size))
+              const data = new FormData()
+              data.append('upload', blobPart)
+              const { data: response } = await axios.post(
+                \`/files/upload\${i > 0 && responses[j]?.file?.id ? \`/\${responses[j]?.file?.id}\` : ''}\`,
+                data,
+                {
+                  params: {
+                    ...parent?.id ? { parent_id: parent.id } : {},
+                    relative_path: file.webkitRelativePath || null,
+                    name: \`\${file.name}\${fileParts > 1 ? \`.part\${String(j + 1).padStart(3, '0')}\` : ''}\`,
+                    size: fileBlob.size,
+                    mime_type: file.type || mime.lookup(file.name) || 'application/octet-stream',
+                    part: i,
+                    total_part: fileParts,
+                  },
                 }
+              )
+              responses[j] = response
+              const percent = (++totalParts / totalAllParts * 100).toFixed(1)
+              self.postMessage({ percent })
+            }
+
+            self.onmessage = (event) => {
+              const { file, j, partIndex } = event.data
+              uploadPart(file, j, partIndex)
+            }
+          `
+          const workerBlob = new Blob([workerScript], { type: 'application/javascript' })
+          const workerUrl = URL.createObjectURL(workerBlob)
 
-                let trial = 0
-                while (trial < RETRY_COUNT) {
+          const uploadPart = async (j: number, i: number) => {
+            if (responses?.length && cancelUploading.current && file.name === cancelUploading.current) {
+              await Promise.all(
+                responses.map(async (response) => {
                   try {
-                    responses[j] = await beginUpload()
-                    trial = RETRY_COUNT
+                    await req.delete(`/files/${response?.file?.id}`)
                   } catch (error) {
-                    if (trial >= RETRY_COUNT) {
-                      throw error
-                    }
-                    await new Promise(res => setTimeout(res, ++trial * 3000))
+                    // ignore
                   }
+                })
+              )
+              cancelUploading.current = null
+              deleted = true
+              window.onbeforeunload = undefined as any
+            } else {
+              const worker = new Worker(workerUrl)
+              worker.postMessage({ file, j, partIndex: i })
+              worker.onmessage = (event) => {
+                const { percent } = event.data
+                onProgress?.({ percent }, file)
+                if (++i < fileParts) {
+                  uploadPart(j, i)
                 }
-
-                const percent = (++totalParts / totalAllParts * 100).toFixed(1)
-                onProgress({ percent }, file)
               }
             }
+          }
 
-            const group = 2
-            await uploadPart(0)
-            for (let i = 1; i < parts - 1; i += group) {
-              if (deleted) break
-              const others = Array.from(Array(i + group).keys()).slice(i, Math.min(parts - 1, i + group))
-              await Promise.all(others.map(async j => await uploadPart(j)))
-            }
-            if (!deleted && parts - 1 > 0) {
-              await uploadPart(parts - 1)
-            }
+          const promises: Promise<void>[] = []
+
+          for (let j = 0; j < fileParts; j++) {
+            if (deleted) break
+            const groupIndex = Math.floor(j / PARALLELISM)
+            const partIndex = j % PARALLELISM
+            promises[groupIndex] = promises[groupIndex] || Promise.resolve()
+            promises[groupIndex] = promises[groupIndex].then(() => uploadPart(j, 0))
           }
+
+          await Promise.all(promises)
+
+          URL.revokeObjectURL(workerUrl)
         }
       }
 
@@ -290,14 +321,16 @@ const Upload: React.FC<Props> = ({ dataFileList: [fileList, setFileList], parent
       notification.error({
         key: 'fileUploadError',
         message: error?.response?.status || 'Something error',
-        ...error?.response?.data ? { description: <>
-          <Typography.Paragraph>
-            {error?.response?.data?.error || error.message || 'Something error'}
-          </Typography.Paragraph>
-          <Typography.Paragraph code>
-            {JSON.stringify(error?.response?.data || error?.data || error, null, 2)}
-          </Typography.Paragraph>
-        </> } : {}
+        ...error?.response?.data ? {
+          description: <>
+            <Typography.Paragraph>
+              {error?.response?.data?.error || error.message || 'Something error'}
+            </Typography.Paragraph>
+            <Typography.Paragraph code>
+              {JSON.stringify(error?.response?.data || error?.data || error, null, 2)}
+            </Typography.Paragraph>
+          </>
+        } : {}
       })
       // filesWantToUpload.current = filesWantToUpload.current?.map(f => f.uid === file.uid ? { ...f, status: 'done' } : f)
       filesWantToUpload.current = filesWantToUpload.current?.map(f => f.uid === file.uid ? null : f).filter(Boolean)