Skip to content

Commit

Permalink
Merge branch 'main' into feat/no-size-for-want-have
Browse files Browse the repository at this point in the history
  • Loading branch information
gammazero authored Sep 27, 2024
2 parents db2d8bc + 4af06fd commit d7f122c
Show file tree
Hide file tree
Showing 11 changed files with 906 additions and 51 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,14 @@ The following emojis are used to highlight certain changes:

* `boxo/bitswap/server`:
* A new [`WithWantHaveReplaceSize(n)`](https://pkg.go.dev/github.com/ipfs/boxo/bitswap/server/#WithWantHaveReplaceSize) option can be used with `bitswap.New` to fine-tune cost-vs-performance. It sets the maximum size of a block in bytes up to which the bitswap server will replace a WantHave with a WantBlock response. Setting this to 0 disables this WantHave replacement and means that block sizes are not read when processing WantHave requests. [#672](https://github.com/ipfs/boxo/pull/672)
- `routing/http`: added support for address and protocol filtering to the delegated routing server ([IPIP-484](https://github.com/ipfs/specs/pull/484)) [#671](https://github.com/ipfs/boxo/pull/671)

### Changed

### Removed

### Fixed
= `unixfs/hamt` Log error instead of panic if both link and shard are nil [#393](https://github.com/ipfs/boxo/pull/393)

### Security

Expand Down
16 changes: 11 additions & 5 deletions ipld/unixfs/hamt/hamt.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,19 @@ import (
"os"
"sync"

"golang.org/x/sync/errgroup"

format "github.com/ipfs/boxo/ipld/unixfs"
"github.com/ipfs/boxo/ipld/unixfs/internal"

dag "github.com/ipfs/boxo/ipld/merkledag"
bitfield "github.com/ipfs/go-bitfield"
cid "github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
logging "github.com/ipfs/go-log/v2"
"golang.org/x/sync/errgroup"
)

var log = logging.Logger("unixfs-hamt")

const (
// HashMurmur3 is the multiformats identifier for Murmur3
HashMurmur3 uint64 = 0x22
Expand Down Expand Up @@ -430,8 +432,13 @@ type listCidsAndShards struct {
func (ds *Shard) walkChildren(processLinkValues func(formattedLink *ipld.Link) error) (*listCidsAndShards, error) {
res := &listCidsAndShards{}

for idx, lnk := range ds.childer.links {
if nextShard := ds.childer.children[idx]; nextShard == nil {
for i, nextShard := range ds.childer.children {
if nextShard == nil {
lnk := ds.childer.link(i)
if lnk == nil {
log.Warnf("internal HAMT error: both link and shard nil at pos %d, dumping shard: %+v", i, *ds)
return nil, fmt.Errorf("internal HAMT error: both link and shard nil, check log")
}
lnkLinkType, err := ds.childLinkType(lnk)
if err != nil {
return nil, err
Expand All @@ -454,7 +461,6 @@ func (ds *Shard) walkChildren(processLinkValues func(formattedLink *ipld.Link) e
default:
return nil, errors.New("unsupported shard link type")
}

} else {
if nextShard.val != nil {
formattedLink := &ipld.Link{
Expand Down
21 changes: 21 additions & 0 deletions ipld/unixfs/hamt/hamt_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -749,3 +749,24 @@ func TestHamtBadSize(t *testing.T) {
}
}
}

func TestHamtNilLinkAndShard(t *testing.T) {
shard, err := NewShard(nil, 1024)
if err != nil {
t.Fatal(err)
}
shard.childer = shard.childer.makeChilder(nil, []*ipld.Link{nil})
nextShard, err := shard.walkChildren(func(_ *ipld.Link) error {
t.Fatal("processLinkValues function should not have been called")
return nil
})
if err == nil {
t.Fatal("expected error")
}
if err.Error() != "internal HAMT error: both link and shard nil, check log" {
t.Fatal("did not get expected error")
}
if nextShard != nil {
t.Fatal("nextShard should be nil")
}
}
8 changes: 3 additions & 5 deletions routing/http/client/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -228,9 +228,7 @@ func TestClient_FindProviders(t *testing.T) {
}

bitswapRecord := makeBitswapRecord()
bitswapProviders := []iter.Result[types.Record]{
{Val: &bitswapRecord},
}
peerRecordFromBitswapRecord := types.FromBitswapRecord(&bitswapRecord)

cases := []struct {
name string
Expand All @@ -254,8 +252,8 @@ func TestClient_FindProviders(t *testing.T) {
},
{
name: "happy case (with deprecated bitswap schema)",
routerResult: bitswapProviders,
expResult: bitswapProviders,
routerResult: []iter.Result[types.Record]{{Val: &bitswapRecord}},
expResult: []iter.Result[types.Record]{{Val: peerRecordFromBitswapRecord}},
expStreamingResponse: true,
},
{
Expand Down
197 changes: 197 additions & 0 deletions routing/http/server/filters.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,197 @@
package server

import (
"reflect"
"slices"
"strings"

"github.com/ipfs/boxo/routing/http/types"
"github.com/ipfs/boxo/routing/http/types/iter"
"github.com/multiformats/go-multiaddr"
)

// filters implements IPIP-0484

func parseFilter(param string) []string {
if param == "" {
return nil
}
return strings.Split(strings.ToLower(param), ",")
}

// applyFiltersToIter applies the filters to the given iterator and returns a new iterator.
//
// The function iterates over the input iterator, applying the specified filters to each record.
// It supports both positive and negative filters for both addresses and protocols.
//
// Parameters:
// - recordsIter: An iterator of types.Record to be filtered.
// - filterAddrs: A slice of strings representing the address filter criteria.
// - filterProtocols: A slice of strings representing the protocol filter criteria.
func applyFiltersToIter(recordsIter iter.ResultIter[types.Record], filterAddrs, filterProtocols []string) iter.ResultIter[types.Record] {
mappedIter := iter.Map(recordsIter, func(v iter.Result[types.Record]) iter.Result[types.Record] {
if v.Err != nil || v.Val == nil {
return v
}

switch v.Val.GetSchema() {
case types.SchemaPeer:
record, ok := v.Val.(*types.PeerRecord)
if !ok {
logger.Errorw("problem casting find providers record", "Schema", v.Val.GetSchema(), "Type", reflect.TypeOf(v).String())
// drop failed type assertion
return iter.Result[types.Record]{}
}

record = applyFilters(record, filterAddrs, filterProtocols)
if record == nil {
return iter.Result[types.Record]{}
}
v.Val = record

//lint:ignore SA1019 // ignore staticcheck
case types.SchemaBitswap:
//lint:ignore SA1019 // ignore staticcheck
record, ok := v.Val.(*types.BitswapRecord)
if !ok {
logger.Errorw("problem casting find providers record", "Schema", v.Val.GetSchema(), "Type", reflect.TypeOf(v).String())
// drop failed type assertion
return iter.Result[types.Record]{}
}
peerRecord := types.FromBitswapRecord(record)
peerRecord = applyFilters(peerRecord, filterAddrs, filterProtocols)
if peerRecord == nil {
return iter.Result[types.Record]{}
}
v.Val = peerRecord
}
return v
})

// filter out nil results and errors
filteredIter := iter.Filter(mappedIter, func(v iter.Result[types.Record]) bool {
return v.Err == nil && v.Val != nil
})

return filteredIter
}

// Applies the filters. Returns nil if the provider does not pass the protocols filter
// The address filter is more complicated because it potentially modifies the Addrs slice.
func applyFilters(provider *types.PeerRecord, filterAddrs, filterProtocols []string) *types.PeerRecord {
if len(filterAddrs) == 0 && len(filterProtocols) == 0 {
return provider
}

if !protocolsAllowed(provider.Protocols, filterProtocols) {
// If the provider doesn't match any of the passed protocols, the provider is omitted from the response.
return nil
}

// return untouched if there's no filter or filterAddrsQuery contains "unknown" and provider has no addrs
if len(filterAddrs) == 0 || (len(provider.Addrs) == 0 && slices.Contains(filterAddrs, "unknown")) {
return provider
}

filteredAddrs := applyAddrFilter(provider.Addrs, filterAddrs)

// If filtering resulted in no addrs, omit the provider
if len(filteredAddrs) == 0 {
return nil
}

provider.Addrs = filteredAddrs
return provider
}

// applyAddrFilter filters a list of multiaddresses based on the provided filter query.
//
// Parameters:
// - addrs: A slice of types.Multiaddr to be filtered.
// - filterAddrsQuery: A slice of strings representing the filter criteria.
//
// The function supports both positive and negative filters:
// - Positive filters (e.g., "tcp", "udp") include addresses that match the specified protocols.
// - Negative filters (e.g., "!tcp", "!udp") exclude addresses that match the specified protocols.
//
// If no filters are provided, the original list of addresses is returned unchanged.
// If only negative filters are provided, addresses not matching any negative filter are included.
// If positive filters are provided, only addresses matching at least one positive filter (and no negative filters) are included.
// If both positive and negative filters are provided, the address must match at least one positive filter and no negative filters to be included.
//
// Returns:
// A new slice of types.Multiaddr containing only the addresses that pass the filter criteria.
func applyAddrFilter(addrs []types.Multiaddr, filterAddrsQuery []string) []types.Multiaddr {
if len(filterAddrsQuery) == 0 {
return addrs
}

var filteredAddrs []types.Multiaddr
var positiveFilters, negativeFilters []multiaddr.Protocol

// Separate positive and negative filters
for _, filter := range filterAddrsQuery {
if strings.HasPrefix(filter, "!") {
negativeFilters = append(negativeFilters, multiaddr.ProtocolWithName(filter[1:]))
} else {
positiveFilters = append(positiveFilters, multiaddr.ProtocolWithName(filter))
}
}

for _, addr := range addrs {
protocols := addr.Protocols()

// Check negative filters
if containsAny(protocols, negativeFilters) {
continue
}

// If no positive filters or matches a positive filter, include the address
if len(positiveFilters) == 0 || containsAny(protocols, positiveFilters) {
filteredAddrs = append(filteredAddrs, addr)
}
}

return filteredAddrs
}

// Helper function to check if protocols contain any of the filters
func containsAny(protocols []multiaddr.Protocol, filters []multiaddr.Protocol) bool {
for _, filter := range filters {
if containsProtocol(protocols, filter) {
return true
}
}
return false
}

func containsProtocol(protos []multiaddr.Protocol, proto multiaddr.Protocol) bool {
for _, p := range protos {
if p.Code == proto.Code {
return true
}
}
return false
}

// protocolsAllowed returns true if the peerProtocols are allowed by the filter protocols.
func protocolsAllowed(peerProtocols []string, filterProtocols []string) bool {
if len(filterProtocols) == 0 {
// If no filter is passed, do not filter
return true
}

for _, filterProtocol := range filterProtocols {
if filterProtocol == "unknown" && len(peerProtocols) == 0 {
return true
}

for _, peerProtocol := range peerProtocols {
if strings.EqualFold(peerProtocol, filterProtocol) {
return true
}

}
}
return false
}
Loading

0 comments on commit d7f122c

Please sign in to comment.