Mirror orig upload in parallel (#7630)

mediorum/cidutil/cidutil.go

@@ -3,7 +3,6 @@ package cidutil
 import (
 	"fmt"
 	"io"
-	"mime/multipart"
 	"net/url"
 	"path/filepath"
 	"strings"
@@ -27,15 +26,6 @@ func UnescapeCidParam(next echo.HandlerFunc) echo.HandlerFunc {
 	}
 }
 
-func ComputeFileHeaderCID(fh *multipart.FileHeader) (string, error) {
-	f, err := fh.Open()
-	if err != nil {
-		return "", err
-	}
-	defer f.Close()
-	return ComputeFileCID(f)
-}
-
 func ComputeFileCID(f io.ReadSeeker) (string, error) {
 	defer f.Seek(0, 0)
 	builder := cid.V1Builder{}

mediorum/server/serve_upload.go

@@ -3,7 +3,10 @@ package server
 import (
 	"database/sql"
 	"fmt"
+	"io"
+	"mime/multipart"
 	"net/http"
+	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -172,28 +175,33 @@ func (ss *MediorumServer) postUpload(c echo.Context) error {
 			}
 			uploads[idx] = upload
 
-			formFileCID, err := cidutil.ComputeFileHeaderCID(formFile)
+			tmpFile, err := copyUploadToTempFile(formFile)
 			if err != nil {
 				upload.Error = err.Error()
 				return err
 			}
+			defer os.Remove(tmpFile.Name())
 
-			upload.OrigFileCID = formFileCID
-			upload.FFProbe, err = ffprobeUpload(formFile)
-			if err != nil && upload.Template == JobTemplateAudio {
-				// fail audio upload if ffprobe fails
+			formFileCID, err := cidutil.ComputeFileCID(tmpFile)
+			if err != nil {
 				upload.Error = err.Error()
 				return err
 			}
 
-			// mirror to n peers
-			file, err := formFile.Open()
-			if err != nil {
+			upload.OrigFileCID = formFileCID
+
+			// ffprobe:
+			upload.FFProbe, err = ffprobe(tmpFile.Name())
+			if err != nil && upload.Template == JobTemplateAudio {
+				// fail audio upload if ffprobe fails
 				upload.Error = err.Error()
 				return err
 			}
 
-			upload.Mirrors, err = ss.replicateFile(formFileCID, file)
+			// ffprobe: restore orig filename
+			upload.FFProbe.Format.Filename = formFile.Filename
+
+			upload.Mirrors, err = ss.replicateFileParallel(formFileCID, tmpFile.Name())
 			if err != nil {
 				upload.Error = err.Error()
 				return err
@@ -216,3 +224,25 @@ func (ss *MediorumServer) postUpload(c echo.Context) error {
 
 	return c.JSON(status, uploads)
 }
+
+func copyUploadToTempFile(file *multipart.FileHeader) (*os.File, error) {
+	temp, err := os.CreateTemp("", "mediorumUpload")
+	if err != nil {
+		return nil, err
+	}
+
+	r, err := file.Open()
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+
+	_, err = io.Copy(temp, r)
+	if err != nil {
+		return nil, err
+	}
+	temp.Sync()
+	temp.Seek(0, 0)
+
+	return temp, nil
+}

mediorum/server/transcode.go

@@ -12,7 +12,6 @@ import (
 	"image/png"
 	"io"
 	"log"
-	"mime/multipart"
 	"os"
 	"os/exec"
 	"strings"
@@ -619,35 +618,6 @@ type FFProbeResult struct {
 	} `json:"format"`
 }
 
-func ffprobeUpload(file *multipart.FileHeader) (*FFProbeResult, error) {
-	temp, err := os.CreateTemp("", "mediorumProbe")
-	if err != nil {
-		return nil, err
-	}
-
-	r, err := file.Open()
-	if err != nil {
-		return nil, err
-	}
-	defer r.Close()
-
-	_, err = io.Copy(temp, r)
-	if err != nil {
-		return nil, err
-	}
-	temp.Close()
-	defer os.Remove(temp.Name())
-
-	probe, err := ffprobe(temp.Name())
-	if err != nil {
-		return nil, err
-	}
-
-	// restore orig filename
-	probe.Format.Filename = file.Filename
-	return probe, nil
-}
-
 func ffprobe(sourcePath string) (*FFProbeResult, error) {
 	probe, err := exec.Command("ffprobe",
 		"-v", "quiet",