Sector check command

api/api_storage.go

@@ -19,6 +19,7 @@ import (
 	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	"github.com/filecoin-project/specs-storage/storage"
 )
 
 // StorageMiner is a low-level interface to the Filecoin network storage miner node
@@ -116,6 +117,8 @@ type StorageMiner interface {
 	// LOTUS_BACKUP_BASE_PATH environment variable set to some path, and that
 	// the path specified when calling CreateBackup is within the base path
 	CreateBackup(ctx context.Context, fpath string) error
+
+	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorNumber]string, error)
 }
 
 type SealRes struct {

api/apistruct/struct.go

@@ -363,6 +363,8 @@ type StorageMinerStruct struct {
 		PiecesGetCIDInfo   func(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) `perm:"read"`
 
 		CreateBackup func(ctx context.Context, fpath string) error `perm:"admin"`
+
+		CheckProvable func(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorNumber]string, error) `perm:"admin"`
 	}
 }
 
@@ -1510,6 +1512,10 @@ func (c *StorageMinerStruct) CreateBackup(ctx context.Context, fpath string) err
 	return c.Internal.CreateBackup(ctx, fpath)
 }
 
+func (c *StorageMinerStruct) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorNumber]string, error) {
+	return c.Internal.CheckProvable(ctx, pp, sectors)
+}
+
 // WorkerStruct
 
 func (w *WorkerStruct) Version(ctx context.Context) (build.Version, error) {

api/docgen/docgen.go

@@ -234,6 +234,9 @@ func init() {
 		},
 	})
 	addExample(storiface.ErrorCode(0))
+	addExample(map[abi.SectorNumber]string{
+		123: "can't acquire read lock",
+	})
 
 	// worker specific
 	addExample(storiface.AcquireMove)

cmd/lotus-storage-miner/proving.go

@@ -10,11 +10,14 @@ import (
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/api/apibstore"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/specs-storage/storage"
 )
 
 var provingCmd = &cli.Command{
@@ -25,6 +28,7 @@ var provingCmd = &cli.Command{
 		provingDeadlinesCmd,
 		provingDeadlineInfoCmd,
 		provingFaultsCmd,
+		provingCheckProvableCmd,
 	},
 }
 
@@ -371,3 +375,104 @@ var provingDeadlineInfoCmd = &cli.Command{
 		return nil
 	},
 }
+
+var provingCheckProvableCmd = &cli.Command{
+	Name:      "check",
+	Usage:     "Check sectors provable",
+	ArgsUsage: "<deadlineIdx>",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "only-bad",
+			Usage: "print only bad sectors",
+			Value: false,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() != 1 {
+			return xerrors.Errorf("must pass deadline index")
+		}
+
+		dlIdx, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64)
+		if err != nil {
+			return xerrors.Errorf("could not parse deadline index: %w", err)
+		}
+
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		sapi, scloser, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer scloser()
+
+		ctx := lcli.ReqContext(cctx)
+
+		addr, err := sapi.ActorAddress(ctx)
+		if err != nil {
+			return err
+		}
+
+		mid, err := address.IDFromAddress(addr)
+		if err != nil {
+			return err
+		}
+
+		info, err := api.StateMinerInfo(ctx, addr, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		pf, err := info.SealProofType.RegisteredWindowPoStProof()
+		if err != nil {
+			return err
+		}
+
+		partitions, err := api.StateMinerPartitions(ctx, addr, dlIdx, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		tw := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0)
+		_, _ = fmt.Fprintln(tw, "deadline\tpartition\tsector\tstatus")
+
+		for parIdx, par := range partitions {
+			sectors := make(map[abi.SectorNumber]struct{})
+
+			sectorInfos, err := api.StateMinerSectors(ctx, addr, &par.AllSectors, types.EmptyTSK)
+			if err != nil {
+				return err
+			}
+
+			var tocheck []storage.SectorRef
+			for _, info := range sectorInfos {
+				sectors[info.SectorNumber] = struct{}{}
+				tocheck = append(tocheck, storage.SectorRef{
+					ProofType: info.SealProof,
+					ID: abi.SectorID{
+						Miner:  abi.ActorID(mid),
+						Number: info.SectorNumber,
+					},
+				})
+			}
+
+			bad, err := sapi.CheckProvable(ctx, pf, tocheck)
+			if err != nil {
+				return err
+			}
+
+			for s := range sectors {
+				if err, exist := bad[s]; exist {
+					_, _ = fmt.Fprintf(tw, "%d\t%d\t%d\t%s\n", dlIdx, parIdx, s, color.RedString("bad")+fmt.Sprintf(" (%s)", err))
+				} else if !cctx.Bool("only-bad") {
+					_, _ = fmt.Fprintf(tw, "%d\t%d\t%d\t%s\n", dlIdx, parIdx, s, color.GreenString("good"))
+				}
+			}
+		}
+
+		return tw.Flush()
+	},
+}

documentation/en/api-methods-miner.md

@@ -10,6 +10,8 @@
 * [Auth](#Auth)
   * [AuthNew](#AuthNew)
   * [AuthVerify](#AuthVerify)
+* [Check](#Check)
+  * [CheckProvable](#CheckProvable)
 * [Create](#Create)
   * [CreateBackup](#CreateBackup)
 * [Deals](#Deals)
@@ -218,6 +220,29 @@ Inputs:
 
 Response: `null`
 
+## Check
+
+
+### CheckProvable
+There are not yet any comments for this method.
+
+Perms: admin
+
+Inputs:
+```json
+[
+  8,
+  null
+]
+```
+
+Response:
+```json
+{
+  "123": "can't acquire read lock"
+}
+```
+
 ## Create
 
 

extern/sector-storage/faults.go

@@ -16,12 +16,12 @@ import (
 
 // FaultTracker TODO: Track things more actively
 type FaultTracker interface {
-	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) ([]abi.SectorID, error)
+	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error)
 }
 
 // CheckProvable returns unprovable sectors
-func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) ([]abi.SectorID, error) {
-	var bad []abi.SectorID
+func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error) {
+	var bad = make(map[abi.SectorID]string)
 
 	ssize, err := pp.SectorSize()
 	if err != nil {
@@ -41,20 +41,20 @@ func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof,
 
 			if !locked {
 				log.Warnw("CheckProvable Sector FAULT: can't acquire read lock", "sector", sector)
-				bad = append(bad, sector.ID)
+				bad[sector.ID] = fmt.Sprint("can't acquire read lock")
 				return nil
 			}
 
 			lp, _, err := m.localStore.AcquireSector(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove)
 			if err != nil {
 				log.Warnw("CheckProvable Sector FAULT: acquire sector in checkProvable", "sector", sector, "error", err)
-				bad = append(bad, sector.ID)
+				bad[sector.ID] = fmt.Sprintf("acquire sector failed: %s", err)
 				return nil
 			}
 
 			if lp.Sealed == "" || lp.Cache == "" {
-				log.Warnw("CheckProvable Sector FAULT: cache an/or sealed paths not found", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache)
-				bad = append(bad, sector.ID)
+				log.Warnw("CheckProvable Sector FAULT: cache and/or sealed paths not found", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache)
+				bad[sector.ID] = fmt.Sprintf("cache and/or sealed paths not found, cache %q, sealed %q", lp.Cache, lp.Sealed)
 				return nil
 			}
 
@@ -70,14 +70,14 @@ func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof,
 				st, err := os.Stat(p)
 				if err != nil {
 					log.Warnw("CheckProvable Sector FAULT: sector file stat error", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "file", p, "err", err)
-					bad = append(bad, sector.ID)
+					bad[sector.ID] = fmt.Sprintf("%s", err)
 					return nil
 				}
 
 				if sz != 0 {
 					if st.Size() != int64(ssize)*sz {
 						log.Warnw("CheckProvable Sector FAULT: sector file is wrong size", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "file", p, "size", st.Size(), "expectSize", int64(ssize)*sz)
-						bad = append(bad, sector.ID)
+						bad[sector.ID] = fmt.Sprintf("%s is wrong size (got %d, expect %d)", p, st.Size(), int64(ssize)*sz)
 						return nil
 					}
 				}

extern/sector-storage/mock/mock.go

@@ -405,14 +405,14 @@ func (mgr *SectorMgr) Remove(ctx context.Context, sector storage.SectorRef) erro
 	return nil
 }
 
-func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []storage.SectorRef) ([]abi.SectorID, error) {
-	var bad []abi.SectorID
+func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []storage.SectorRef) (map[abi.SectorID]string, error) {
+	bad := map[abi.SectorID]string{}
 
 	for _, sid := range ids {
 		_, found := mgr.sectors[sid.ID]
 
 		if !found || mgr.sectors[sid.ID].failed {
-			bad = append(bad, sid.ID)
+			bad[sid.ID] = "mock fail"
 		}
 	}
 

node/impl/storminer.go

@@ -37,6 +37,7 @@ import (
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/storage"
 	"github.com/filecoin-project/lotus/storage/sectorblocks"
+	sto "github.com/filecoin-project/specs-storage/storage"
 )
 
 type StorageMinerAPI struct {
@@ -543,4 +544,18 @@ func (sm *StorageMinerAPI) CreateBackup(ctx context.Context, fpath string) error
 	return backup(sm.DS, fpath)
 }
 
+func (sm *StorageMinerAPI) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []sto.SectorRef) (map[abi.SectorNumber]string, error) {
+	bad, err := sm.StorageMgr.CheckProvable(ctx, pp, sectors)
+	if err != nil {
+		return nil, err
+	}
+
+	var out = make(map[abi.SectorNumber]string)
+	for sid, err := range bad {
+		out[sid.Number] = err
+	}
+
+	return out, nil
+}
+
 var _ api.StorageMiner = &StorageMinerAPI{}

storage/wdpost_run.go

@@ -212,14 +212,13 @@ func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check bitfield.B
 				Number: info.SectorNumber,
 			},
 		})
-
 	}
 
 	bad, err := s.faultTracker.CheckProvable(ctx, s.proofType, tocheck)
 	if err != nil {
 		return bitfield.BitField{}, xerrors.Errorf("checking provable sectors: %w", err)
 	}
-	for _, id := range bad {
+	for id := range bad {
 		delete(sectors, id.Number)
 	}
 

storage/wdpost_run_test.go

@@ -125,9 +125,9 @@ func (m *mockProver) GenerateWindowPoSt(ctx context.Context, aid abi.ActorID, si
 type mockFaultTracker struct {
 }
 
-func (m mockFaultTracker) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) ([]abi.SectorID, error) {
+func (m mockFaultTracker) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error) {
 	// Returns "bad" sectors so just return nil meaning all sectors are good
-	return nil, nil
+	return map[abi.SectorID]string{}, nil
 }
 
 // TestWDPostDoPost verifies that doPost will send the correct number of window