Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

booster-http: disable on-the-fly indexing by default #646

Merged
merged 1 commit into from
Aug 24, 2022
Merged

booster-http: disable on-the-fly indexing by default #646

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion cmd/booster-http/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@ var runCmd = &cli.Command{
Usage: "the port the web server listens on",
Value: 7777,
},
&cli.BoolFlag{
Name: "allow-indexing",
Usage: "allow booster-http to build an index for a CAR file on the fly if necessary (requires doing an extra pass over the CAR file)",
Value: false,
},
&cli.StringFlag{
Name: "api-boost",
Usage: "the endpoint for the boost API",
Expand Down Expand Up @@ -135,13 +140,26 @@ var runCmd = &cli.Command{
// Create the piece provider and sector accessors
pp := sealer.NewPieceProvider(storage, sealingService, sealingService)
sa := sectoraccessor.NewSectorAccessor(dtypes.MinerAddress(maddr), sealingService, pp, fullnodeApi)
allowIndexing := cctx.Bool("allow-indexing")
// Create the server API
sapi := serverApi{ctx: ctx, bapi: bapi, sa: sa}
server := NewHttpServer(cctx.String("base-path"), cctx.Int("port"), sapi)
server := NewHttpServer(
cctx.String("base-path"),
cctx.Int("port"),
allowIndexing,
sapi,
)

// Start the server
log.Infof("Starting booster-http node on port %d with base path '%s'",
cctx.Int("port"), cctx.String("base-path"))
var indexingStr string
if allowIndexing {
indexingStr = "Enabled"
} else {
indexingStr = "Disabled"
}
log.Info("On-the-fly indexing of CAR files is " + indexingStr)
server.Start(ctx)

// Monitor for shutdown.
Expand Down
20 changes: 12 additions & 8 deletions cmd/booster-http/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,10 @@ var ErrNotFound = errors.New("not found")
const carSuffix = ".car"

type HttpServer struct {
path string
port int
api HttpServerApi
path string
port int
allowIndexing bool
api HttpServerApi

ctx context.Context
cancel context.CancelFunc
Expand All @@ -47,8 +48,8 @@ type HttpServerApi interface {
UnsealSectorAt(ctx context.Context, sectorID abi.SectorNumber, pieceOffset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (mount.Reader, error)
}

func NewHttpServer(path string, port int, api HttpServerApi) *HttpServer {
return &HttpServer{path: path, port: port, api: api}
func NewHttpServer(path string, port int, allowIndexing bool, api HttpServerApi) *HttpServer {
return &HttpServer{path: path, port: port, allowIndexing: allowIndexing, api: api}
}

func (s *HttpServer) payloadBasePath() string {
Expand Down Expand Up @@ -307,9 +308,12 @@ func (s *HttpServer) getPieceContent(ctx context.Context, pieceCid cid.Cid) (io.
func (s *HttpServer) getCarContent(pieceCid cid.Cid, pieceReader io.ReadSeeker) (io.ReadSeeker, error) {
maxOffset, err := s.api.GetMaxPieceOffset(pieceCid)
if err != nil {
// If it's not possible to get the max piece offset it may be because
// the CAR file hasn't been indexed yet. So try to index it in real time.
maxOffset, err = getMaxPieceOffset(pieceReader)
if s.allowIndexing {
// If it's not possible to get the max piece offset it may be because
// the CAR file hasn't been indexed yet. So try to index it in real time.
alog("%s\tbuilding index for %s", color.New(color.FgBlue).Sprintf("INFO"), pieceCid)
maxOffset, err = getMaxPieceOffset(pieceReader)
}
if err != nil {
return nil, fmt.Errorf("getting max offset for piece %s: %w", pieceCid, err)
}
Expand Down