Merge pull request #193 from Yancey1989/upload_file_recursion

Upload files with recursion

go/paddlecloud/simplefile.go

@@ -10,6 +10,8 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"path/filepath"
+	"strings"
 
 	"github.com/PaddlePaddle/cloud/go/utils/restclient"
 	"github.com/google/subcommands"
@@ -45,7 +47,7 @@ func (p *SimpleFileCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interfa
 	}
 	switch f.Arg(0) {
 	case "put":
-		err := putFile(f.Arg(1), f.Arg(2))
+		err := putFiles(f.Arg(1), f.Arg(2))
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "put file error: %s\n", err)
 			return subcommands.ExitFailure
@@ -87,13 +89,39 @@ func lsFile(dst string) error {
 		return errors.New("list file error: " + errMsg)
 	}
 	items := respObj.(map[string]interface{})["items"].([]interface{})
-	for _, i := range items {
-		fmt.Println(i.(string))
+	for _, fn := range items {
+		fmt.Println(fn.(string))
+	}
+	return nil
+}
+
+func putFiles(src string, dest string) error {
+	f, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+	switch mode := f.Mode(); {
+	case mode.IsDir():
+		if err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
+			if info.Mode().IsRegular() {
+				srcs := strings.Split(filepath.Clean(src), string(os.PathSeparator))
+				paths := strings.Split(path, string(os.PathSeparator))
+				destFile := filepath.Join(dest, strings.Join(paths[len(srcs)-1:len(paths)], string(os.PathSeparator)))
+				putFile(path, destFile)
+			}
+			return nil
+		}); err != nil {
+			return err
+		}
+
+	case mode.IsRegular():
+		return putFile(src, dest)
 	}
 	return nil
 }
 
 func putFile(src string, dest string) error {
+	fmt.Printf("Uploading ... %s %s\n", src, dest)
 	query := url.Values{}
 	_, srcFile := path.Split(src)
 	destDir, destFile := path.Split(dest)

go/paddlecloud/submit.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"os"
 	"path"
-	"path/filepath"
 
 	"github.com/PaddlePaddle/cloud/go/utils/config"
 	"github.com/PaddlePaddle/cloud/go/utils/restclient"
@@ -108,19 +107,8 @@ func (s *Submitter) Submit(jobPackage string, jobName string) error {
 	// if jobPackage is not a local dir, skip uploading package.
 	_, pkgerr := os.Stat(jobPackage)
 	if pkgerr == nil {
-		// 1. upload user job package to pfs.
-		err := filepath.Walk(jobPackage, func(filePath string, info os.FileInfo, err error) error {
-			if info.IsDir() {
-				return nil
-			}
-			glog.V(10).Infof("Uploading %s...\n", filePath)
-			dest := path.Join("/pfs", Config.ActiveConfig.Name, "home", Config.ActiveConfig.Username, "jobs", jobName, filepath.Base(filePath))
-			fmt.Printf("uploading: %s...\n", filePath)
-			return putFile(filePath, dest)
-		})
-		if err != nil {
-			return err
-		}
+		dest := path.Join("/pfs", Config.ActiveConfig.Name, "home", Config.ActiveConfig.Username, "jobs", jobName)
+		return putFiles(jobPackage, dest)
 	} else if os.IsNotExist(pkgerr) {
 		glog.Warning("jobpackage not a local dir, skip upload.")
 	}