Add support for .ipfsignore and hidden files

commands/files/is_hidden.go

@@ -0,0 +1,19 @@
+// +build !windows
+
+package files
+
+import (
+	"path/filepath"
+	"strings"
+)
+
+func IsHidden(f File) bool {
+
+	fName := filepath.Base(f.FileName())
+
+	if strings.HasPrefix(fName, ".") && len(fName) > 1 {
+		return true, nil
+	}
+
+	return false, nil
+}

commands/files/is_hidden_windows.go

@@ -0,0 +1,29 @@
+// +build windows
+
+package files
+
+import (
+	"path/filepath"
+	"strings"
+	"syscall"
+)
+
+func IsHidden(f File) bool {
+
+	fName := filepath.Base(f.FileName())
+
+	if strings.HasPrefix(fName, ".") && len(fName) > 1 {
+		return true
+	}
+
+	p, e := syscall.UTF16PtrFromString(f.FileName())
+	if e != nil {
+		return false
+	}
+
+	attrs, e := syscall.GetFileAttributes(p)
+	if e != nil {
+		return false
+	}
+	return attrs&syscall.FILE_ATTRIBUTE_HIDDEN != 0
+}

commands/files/serialfile.go

@@ -3,7 +3,7 @@ package files
 import (
 	"io"
 	"os"
-	fp "path"
+	fp "path/filepath"
 	"sort"
 	"syscall"
 )

core/commands/add.go

@@ -3,11 +3,13 @@ package commands
 import (
 	"fmt"
 	"io"
+	"os"
 	"path"
 	"strings"
 
 	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
 	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
+	ignore "github.com/sabhiram/go-git-ignore"
 
 	cmds "github.com/ipfs/go-ipfs/commands"
 	files "github.com/ipfs/go-ipfs/commands/files"
@@ -29,6 +31,7 @@ const progressReaderIncrement = 1024 * 256
 const (
 	progressOptionName = "progress"
 	wrapOptionName     = "wrap-with-directory"
+	hiddenOptionName   = "hidden"
 )
 
 type AddedObject struct {
@@ -57,6 +60,7 @@ remains to be implemented.
 		cmds.BoolOption(progressOptionName, "p", "Stream progress data"),
 		cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object"),
 		cmds.BoolOption("t", "trickle", "Use trickle-dag format for dag generation"),
+		cmds.BoolOption(hiddenOptionName, "Include files that are hidden"),
 	},
 	PreRun: func(req cmds.Request) error {
 		if quiet, _, _ := req.Option("quiet").Bool(); quiet {
@@ -90,6 +94,17 @@ remains to be implemented.
 
 		progress, _, _ := req.Option(progressOptionName).Bool()
 		wrap, _, _ := req.Option(wrapOptionName).Bool()
+		hidden, _, _ := req.Option(hiddenOptionName).Bool()
+
+		var ignoreFilePatterns []ignore.GitIgnore
+
+		// Check the $IPFS_PATH
+		if ipfs_path := os.Getenv("$IPFS_PATH"); len(ipfs_path) > 0 {
+			baseFilePattern, err := ignore.CompileIgnoreFile(path.Join(ipfs_path, ".ipfsignore"))
+			if err == nil && baseFilePattern != nil {
+				ignoreFilePatterns = append(ignoreFilePatterns, *baseFilePattern)
+			}
+		}
 
 		outChan := make(chan interface{})
 		res.SetOutput((<-chan interface{})(outChan))
@@ -107,7 +122,15 @@ remains to be implemented.
 					return
 				}
 
-				rootnd, err := addFile(n, file, outChan, progress, wrap)
+				// If the file is not a folder, then let's get the root of that
+				// folder and attempt to load the appropriate .ipfsignore.
+				localIgnorePatterns := ignoreFilePatterns
+				if !file.IsDirectory() {
+					parentPath := path.Dir(file.FileName())
+					localIgnorePatterns = checkForLocalIgnorePatterns(parentPath, ignoreFilePatterns)
+				}
+
+				rootnd, err := addFile(n, file, outChan, progress, wrap, hidden, localIgnorePatterns)
 				if err != nil {
 					res.SetError(err, cmds.ErrNormal)
 					return
@@ -227,9 +250,24 @@ func add(n *core.IpfsNode, reader io.Reader) (*dag.Node, error) {
 	return node, nil
 }
 
-func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress bool, wrap bool) (*dag.Node, error) {
+func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress bool, wrap bool, hidden bool, ignoreFilePatterns []ignore.GitIgnore) (*dag.Node, error) {
+	// Check if file is hidden
+	if fileIsHidden := files.IsHidden(file); fileIsHidden && !hidden {
+		log.Debugf("%s is hidden, skipping", file.FileName())
+		return nil, &hiddenFileError{file.FileName()}
+	}
+
+	// Check for ignore files matches
+	for i := range ignoreFilePatterns {
+		if ignoreFilePatterns[i].MatchesPath(file.FileName()) {
+			log.Debugf("%s is ignored file, skipping", file.FileName())
+			return nil, &ignoreFileError{file.FileName()}
+		}
+	}
+
+	// Check if "file" is actually a directory
 	if file.IsDirectory() {
-		return addDir(n, file, out, progress)
+		return addDir(n, file, out, progress, hidden, ignoreFilePatterns)
 	}
 
 	// if the progress flag was specified, wrap the file so that we can send
@@ -263,10 +301,13 @@ func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress b
 	return dagnode, nil
 }
 
-func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress bool) (*dag.Node, error) {
-	log.Infof("adding directory: %s", dir.FileName())
+func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress bool, hidden bool, ignoreFilePatterns []ignore.GitIgnore) (*dag.Node, error) {
 
 	tree := &dag.Node{Data: ft.FolderPBData()}
+	log.Infof("adding directory: %s", dir.FileName())
+
+	// Check for an .ipfsignore file that is local to this Dir and append to the incoming
+	localIgnorePatterns := checkForLocalIgnorePatterns(dir.FileName(), ignoreFilePatterns)
 
 	for {
 		file, err := dir.NextFile()
@@ -277,16 +318,24 @@ func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress boo
 			break
 		}
 
-		node, err := addFile(n, file, out, progress, false)
-		if err != nil {
+		node, err := addFile(n, file, out, progress, false, hidden, localIgnorePatterns)
+		if _, ok := err.(*hiddenFileError); ok {
+			// hidden file error, set the node to nil for below
+			node = nil
+		} else if _, ok := err.(*ignoreFileError); ok {
+			// ignore file error, set the node to nil for below
+			node = nil
+		} else if err != nil {
 			return nil, err
 		}
 
-		_, name := path.Split(file.FileName())
+		if node != nil {
+			_, name := path.Split(file.FileName())
 
-		err = tree.AddNodeLink(name, node)
-		if err != nil {
-			return nil, err
+			err = tree.AddNodeLink(name, node)
+			if err != nil {
+				return nil, err
+			}
 		}
 	}
 
@@ -303,6 +352,23 @@ func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress boo
 	return tree, nil
 }
 
+func checkForLocalIgnorePatterns(dir string, ignoreFilePatterns []ignore.GitIgnore) []ignore.GitIgnore {
+	var ignorePathname string
+	if dir == "." {
+		ignorePathname = ".ipfsignore"
+	} else {
+		ignorePathname = path.Join(dir, ".ipfsignore")
+	}
+
+	localIgnore, ignoreErr := ignore.CompileIgnoreFile(ignorePathname)
+	if ignoreErr == nil && localIgnore != nil {
+		log.Debugf("found ignore file: %s", dir)
+		return append(ignoreFilePatterns, *localIgnore)
+	} else {
+		return ignoreFilePatterns
+	}
+}
+
 // outputDagnode sends dagnode info over the output channel
 func outputDagnode(out chan interface{}, name string, dn *dag.Node) error {
 	o, err := getOutput(dn)
@@ -318,6 +384,22 @@ func outputDagnode(out chan interface{}, name string, dn *dag.Node) error {
 	return nil
 }
 
+type hiddenFileError struct {
+	fileName string
+}
+
+func (e *hiddenFileError) Error() string {
+	return fmt.Sprintf("%s is a hidden file", e.fileName)
+}
+
+type ignoreFileError struct {
+	fileName string
+}
+
+func (e *ignoreFileError) Error() string {
+	return fmt.Sprintf("%s is an ignored file", e.fileName)
+}
+
 type progressReader struct {
 	file         files.File
 	out          chan interface{}