Add fixed-period repo GC

License: MIT Signed-off-by: rht <rhtbot@gmail.com>
ipfs · Jul 19, 2015 · 7ed7f35 · 7ed7f35
1 parent 8668f40
commit 7ed7f35
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 2 deletions.
diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	_ "expvar"
 	"fmt"
 	"net/http"
@@ -9,15 +10,18 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"time"
 
 	_ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/codahale/metrics/runtime"
 	ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
 	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net"
+	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 
 	cmds "github.com/ipfs/go-ipfs/commands"
 	"github.com/ipfs/go-ipfs/core"
 	commands "github.com/ipfs/go-ipfs/core/commands"
 	corehttp "github.com/ipfs/go-ipfs/core/corehttp"
+	corerepo "github.com/ipfs/go-ipfs/core/corerepo"
 	"github.com/ipfs/go-ipfs/core/corerouting"
 	conn "github.com/ipfs/go-ipfs/p2p/net/conn"
 	peer "github.com/ipfs/go-ipfs/p2p/peer"
@@ -246,6 +250,41 @@ func daemonFunc(req cmds.Request, res cmds.Response) {
 		}
 	}
 
+	// repo blockstore GC
+	const storageMax = 1024 * 1024 * 1024 // 1 GB
+	const storageGcWatermark = storageMax / 10 * 9
+
+	go func() {
+		for {
+			<-time.After(30 * time.Minute)
+
+			// Check fs repo disk usage
+			diskUsage, err := fsrepo.GetDiskUsage()
+			if err != nil {
+				res.SetError(err, cmds.ErrNormal)
+			}
+
+			if diskUsage > storageMax {
+				res.SetError(errors.New("Maximum storage limit exceeded. Maybe unpin some files?"), cmds.ErrNormal)
+			} else if diskUsage > storageGcWatermark {
+				fmt.Println("Starting repo GC...")
+				// 1 minute is sufficient for ~1GB unlink() blocks each of 100kb in SSD
+				// TODO: scale this, e.g. multiply by (diskUsage / 10e9)
+				ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
+				defer cancel()
+				if err := corerepo.GarbageCollect(node, ctx); err != nil {
+					res.SetError(err, cmds.ErrNormal)
+				}
+				newDiskUsage, err := fsrepo.GetDiskUsage()
+				if err != nil {
+					res.SetError(err, cmds.ErrNormal)
+				}
+				// TODO: Add unit to "Released x KB/MB/GB/TB/PB"
+				fmt.Println("Repo GC done. Released %d", newDiskUsage-diskUsage)
+			}
+		}
+	}()
+
 	// collect long-running errors and block for shutdown
 	// TODO(cryptix): our fuse currently doesnt follow this pattern for graceful shutdown
 	for err := range merge(apiErrc, gwErrc) {

diff --git a/core/corerepo/gc.go b/core/corerepo/gc.go
@@ -23,8 +23,7 @@ func GarbageCollect(n *core.IpfsNode, ctx context.Context) error {
 	}
 	for k := range keychan { // rely on AllKeysChan to close chan
 		if !n.Pinning.IsPinned(k) {
-			err := n.Blockstore.DeleteBlock(k)
-			if err != nil {
+			if err := n.Blockstore.DeleteBlock(k); err != nil {
 				return err
 			}
 		}

diff --git a/repo/fsrepo/fsrepo.go b/repo/fsrepo/fsrepo.go
@@ -9,6 +9,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"syscall"
 
 	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
 	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs"
@@ -588,3 +589,16 @@ func isInitializedUnsynced(repoPath string) bool {
 	}
 	return true
 }
+
+// GetDiskUsage computes the disk space taken by the repo
+func GetDiskUsage() (int64, error) {
+	// TODO: This should be cross-platform
+	// See http://wendal.net/2012/1224.html
+	pth, err := config.PathRoot()
+	if err != nil {
+		return 0, err
+	}
+	var stat syscall.Statfs_t
+	syscall.Statfs(pth, &stat)
+	return int64(stat.Bsize) * int64(stat.Bavail), nil
+}