Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Parallel decryptions #128

Merged
merged 84 commits into from
Sep 30, 2024
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
84 commits
Select commit Hold shift + click to select a range
3504d8c
Parallel decryptions: DecryptionResults (#124)
toml01 Aug 13, 2024
d9d0d3b
just minor import changes
toml01 Aug 21, 2024
d9160aa
geth version with custom txprocessinghooks
toml01 Aug 25, 2024
c9e2f86
Add ParallelTxHooks to TxParams
toml01 Aug 25, 2024
8574901
fix build?
toml01 Aug 25, 2024
ab99728
update overrides
toml01 Aug 27, 2024
d17f239
fix build
toml01 Sep 2, 2024
2da1391
update the Get() function to delegate the type assertion to the caller
toml01 Sep 3, 2024
169b585
type compatibility
toml01 Sep 3, 2024
e1d1cdf
fix tests
toml01 Sep 3, 2024
8c34dde
implement the decrypt precomp
toml01 Sep 3, 2024
323192f
added fheos serialization for decryption results
eshelB Sep 8, 2024
c02cea3
serialize flow finished+ added loadMultiple results
eshelB Sep 8, 2024
ddad1ff
added first serialization test
eshelB Sep 8, 2024
22ce2bc
update geth
toml01 Sep 9, 2024
7ed36a3
update the Get() function to delegate the type assertion to the caller
toml01 Sep 3, 2024
a839ada
type compatibility
toml01 Sep 3, 2024
24170e7
fix tests
toml01 Sep 3, 2024
4bd4b42
implement the decrypt precomp
toml01 Sep 3, 2024
8215b54
implement req() and sealoutput() as well
toml01 Sep 4, 2024
2c19a1e
just override stuff for CI
toml01 Sep 5, 2024
a5944c2
added compatibility for non-sequencer nodes too
toml01 Sep 5, 2024
d77e5b6
restructure req, sealoutput and decrypt to allow sync operation in qu…
toml01 Sep 5, 2024
268aaeb
Change precompiles to play nice with contracts_parser.ts
toml01 Sep 5, 2024
a8c556a
fix expected type for sealoutput
toml01 Sep 9, 2024
018f3d9
overrides for CI
toml01 Sep 9, 2024
c039422
use deep copies for function arguments
toml01 Sep 9, 2024
797e369
Create a unique PendingDecryption key for sealing output for each pub…
toml01 Sep 10, 2024
31d1f02
fix build
toml01 Sep 10, 2024
db7f1d4
better comment
toml01 Sep 10, 2024
51938f4
update the Get() function to delegate the type assertion to the caller
toml01 Sep 3, 2024
e6b1b16
type compatibility
toml01 Sep 3, 2024
b86a54a
fix tests
toml01 Sep 3, 2024
938d579
remove comment
toml01 Sep 10, 2024
5f2e90d
Merge branch 'pd-sequencer-flow' into sync-tx-results
eshelB Sep 11, 2024
9e132ee
Merge pull request #133 from FhenixProtocol/pd-sequencer-flow
toml01 Sep 11, 2024
f00a87d
Merge pull request #137 from FhenixProtocol/pd-new-structs-2
toml01 Sep 11, 2024
61191b1
Merge branch 'parallel-decrypts' into pd-precompiles
toml01 Sep 11, 2024
b97abe9
Merge branch 'pd-precompiles' into sync-tx-results-3
eshelB Sep 12, 2024
095558c
adding nitro-overrides parse_l2.go and executionegine.go, updating bl…
eshelB Sep 11, 2024
7113bda
added nitro-overrides to dockerfile
eshelB Sep 11, 2024
7a51719
using the go-ethereum version of pd-precompiles
eshelB Sep 12, 2024
6eb2be7
removing ontxsuccess
eshelB Sep 12, 2024
8d6c500
changed expected type for sealoutput result serialization to be strin…
eshelB Sep 12, 2024
eab219c
Merge branch 'pd-precompiles' into sync-tx-results
eshelB Sep 12, 2024
53c6324
now parallel goroutine waits for txqueueitem to exist
eshelB Sep 15, 2024
5890c58
adding a nitro override
eshelB Sep 15, 2024
34f4da5
fixing sequencer bug on notifyDecResult and documenting potential issue
eshelB Sep 15, 2024
aa454ee
fixing re-close channel bug
eshelB Sep 15, 2024
38c80e9
Merge branch 'pd-delegate-test-fix' into sync-tx-results
eshelB Sep 15, 2024
180df97
restoring dockerfile local change
eshelB Sep 15, 2024
2671908
adding a nitro override
eshelB Sep 15, 2024
0b2fb8f
updating overrides
eshelB Sep 15, 2024
79f74d5
forgot one bytes to string conversion
eshelB Sep 15, 2024
ca4324a
removing todo
eshelB Sep 15, 2024
2ece2f5
Merge pull request #138 from FhenixProtocol/pd-precompiles
toml01 Sep 16, 2024
53d0a5f
nitro overrides
eshelB Sep 16, 2024
7b47439
Merge branch 'parallel-decrypts' of github.com:FhenixProtocol/fheos i…
eshelB Sep 16, 2024
9f0146f
adding debug log
eshelB Sep 18, 2024
fea1d1d
CI: refactoring common code into function
eshelB Sep 18, 2024
42bb41a
adding minor change in override
eshelB Sep 18, 2024
a807ac1
resolve conflicts
toml01 Sep 18, 2024
f9b7b8d
fix log lib
toml01 Sep 18, 2024
b924b74
propagate tx hash to precompiles through TxContext
toml01 Sep 19, 2024
1143e85
Use txHash instead of blockHash in auto-seeding
toml01 Sep 19, 2024
1387526
Update the nitro fix (override)
toml01 Sep 18, 2024
1fd351b
update geth
toml01 Sep 22, 2024
d99280d
use block hash for non-commit calls
toml01 Sep 22, 2024
0d5f6ec
remove lastSavedHash from fheos state
toml01 Sep 22, 2024
324bb2e
Merge branch 'parallel-decrypts' into pd-sync-results
eshelB Sep 23, 2024
0001764
Merge pull request #152 from FhenixProtocol/rand-ct-deterministic-hash
toml01 Sep 23, 2024
0e0939a
updating nitro-override
eshelB Sep 23, 2024
e167b94
Merge remote-tracking branch 'origin/parallel-decrypts' into pd-sync-…
eshelB Sep 23, 2024
266d5c8
Merge pull request #146 from FhenixProtocol/pd-sync-results
toml01 Sep 24, 2024
9712c01
Update go-ethereum
toml01 Sep 24, 2024
0839294
fix nitro tests
toml01 Sep 24, 2024
d6f2813
sync with master
toml01 Sep 24, 2024
345632d
mastermain-synced geth
toml01 Sep 24, 2024
d2f81a1
update geth to support celestia
toml01 Sep 25, 2024
023b4e7
update geth with celestia merge
toml01 Sep 25, 2024
ff101ca
update geth
toml01 Sep 26, 2024
46ccfc5
Update docker base image for tests
toml01 Sep 29, 2024
74c7994
Merge pull request #155 from FhenixProtocol/celestia-merge
toml01 Sep 30, 2024
7e8efa2
update geth
toml01 Sep 30, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
*/node_modules/*
precompiles/cache/*
precompiles/types/*
precompiles/artifacts/*
solgen/**/*.js
build/
Expand Down
7 changes: 4 additions & 3 deletions precompiles/contracts.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@ package precompiles
import (
"encoding/hex"
"fmt"
"github.com/fhenixprotocol/fheos/precompiles/types"
storage2 "github.com/fhenixprotocol/fheos/storage"
"github.com/fhenixprotocol/warp-drive/fhe-driver"
"math/big"
"os"
"strings"

"github.com/fhenixprotocol/fheos/precompiles/types"
storage2 "github.com/fhenixprotocol/fheos/storage"
"github.com/fhenixprotocol/warp-drive/fhe-driver"

"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/log"
)
Expand Down
11 changes: 7 additions & 4 deletions precompiles/state.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,19 @@ package precompiles
import (
"errors"
"fmt"
"os"
"time"

"github.com/ethereum/go-ethereum/metrics"
"github.com/fhenixprotocol/fheos/precompiles/types"
storage2 "github.com/fhenixprotocol/fheos/storage"
"github.com/fhenixprotocol/warp-drive/fhe-driver"
"os"
"time"
)

type FheosState struct {
FheosVersion uint64
Storage storage2.FheosStorage
FheosVersion uint64
Storage storage2.FheosStorage
DecryptResults *types.DecryptionResults
//MaxUintValue *big.Int // This should contain the max value of the supported uint type
}

Expand Down Expand Up @@ -72,6 +74,7 @@ func createFheosState(storage storage2.FheosStorage, version uint64) {
State = &FheosState{
version,
storage,
types.NewDecryptionResultsMap(),
}
}

Expand Down
102 changes: 102 additions & 0 deletions precompiles/types/decryption_results.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package types

import (
"fmt"
"math/big"
"sync"
"time"
)

type PendingDecryption struct {
Hash Hash
Type PrecompileName
}

type DecryptionRecord struct {
Value any
Timestamp time.Time
}

type DecryptionResults struct {
data map[PendingDecryption]DecryptionRecord
mu sync.RWMutex
}

func NewDecryptionResultsMap() *DecryptionResults {
return &DecryptionResults{
data: make(map[PendingDecryption]DecryptionRecord),
}
}

func (dr *DecryptionResults) CreateEmptyRecord(key PendingDecryption) {
dr.mu.Lock()
defer dr.mu.Unlock()
if _, exists := dr.data[key]; !exists {
dr.data[key] = DecryptionRecord{Value: nil, Timestamp: time.Now()}
}
}

func (dr *DecryptionResults) SetValue(key PendingDecryption, value any) error {
dr.mu.Lock()
defer dr.mu.Unlock()

switch key.Type {
case SealOutput:
if _, ok := value.([]byte); !ok {
return fmt.Errorf("value for SealOutput must be []byte")
}
case Require:
if _, ok := value.(bool); !ok {
return fmt.Errorf("value for Require must be bool")
}
case Decrypt:
if _, ok := value.(*big.Int); !ok {
return fmt.Errorf("value for Decrypt must be *big.Int")
}
default:
return fmt.Errorf("unknown PrecompileName")
}

dr.data[key] = DecryptionRecord{Value: value, Timestamp: time.Now()}
return nil
}

func (dr *DecryptionResults) Get(key PendingDecryption) (any, bool, time.Time, error) {
dr.mu.RLock()
defer dr.mu.RUnlock()

record, exists := dr.data[key]
if !exists {
return nil, false, time.Time{}, nil
}

if record.Value == nil {
return nil, true, record.Timestamp, nil // Exists but no value
}

switch key.Type {
case SealOutput:
if bytes, ok := record.Value.([]byte); ok {
return bytes, true, record.Timestamp, nil
}
return nil, true, record.Timestamp, fmt.Errorf("value is not []byte as expected for SealOutput")
case Require:
if boolValue, ok := record.Value.(bool); ok {
return boolValue, true, record.Timestamp, nil
}
return nil, true, record.Timestamp, fmt.Errorf("value is not bool as expected for Require")
case Decrypt:
if bigInt, ok := record.Value.(*big.Int); ok {
return bigInt, true, record.Timestamp, nil
}
return nil, true, record.Timestamp, fmt.Errorf("value is not *big.Int as expected for Decrypt")
default:
return nil, true, record.Timestamp, fmt.Errorf("unknown PrecompileName")
}
}

func (dr *DecryptionResults) Remove(key PendingDecryption) {
dr.mu.Lock()
defer dr.mu.Unlock()
delete(dr.data, key)
}
158 changes: 158 additions & 0 deletions precompiles/types/decryption_results_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
package types

import (
"math/big"
"testing"
"time"

"github.com/stretchr/testify/assert"
)

func TestDecryptionResults(t *testing.T) {
t.Run("NewDecryptionResultsMap", func(t *testing.T) {
dr := NewDecryptionResultsMap()
assert.NotNil(t, dr)
assert.Empty(t, dr.data)
})

t.Run("CreateEmptyRecord", func(t *testing.T) {
dr := NewDecryptionResultsMap()
key := PendingDecryption{Hash: Hash{1, 2, 3}, Type: SealOutput}

dr.CreateEmptyRecord(key)
record, exists := dr.data[key]
assert.True(t, exists)
assert.Nil(t, record.Value)
assert.WithinDuration(t, time.Now(), record.Timestamp, time.Second)

// Creating again should not overwrite
time.Sleep(time.Millisecond * 10)
dr.CreateEmptyRecord(key)
newRecord, _ := dr.data[key]
assert.Equal(t, record.Timestamp, newRecord.Timestamp)
})

t.Run("SetValue", func(t *testing.T) {
dr := NewDecryptionResultsMap()
key := PendingDecryption{Hash: Hash{1, 2, 3}, Type: SealOutput}

// Set for SealOutput
err := dr.SetValue(key, []byte{4, 5, 6})
assert.NoError(t, err)
record, exists := dr.data[key]
assert.True(t, exists)
assert.Equal(t, []byte{4, 5, 6}, record.Value)

// Set for Require
keyRequire := PendingDecryption{Hash: Hash{4, 5, 6}, Type: Require}
err = dr.SetValue(keyRequire, true)
assert.NoError(t, err)

// Set for Decrypt
keyDecrypt := PendingDecryption{Hash: Hash{7, 8, 9}, Type: Decrypt}
err = dr.SetValue(keyDecrypt, big.NewInt(123))
assert.NoError(t, err)

// Set with wrong type
err = dr.SetValue(key, true)
assert.Error(t, err)
})

t.Run("Get", func(t *testing.T) {
dr := NewDecryptionResultsMap()
key := PendingDecryption{Hash: Hash{1, 2, 3}, Type: SealOutput}

// Get non-existent key
value, exists, timestamp, err := dr.Get(key)
assert.Nil(t, value)
assert.False(t, exists)
assert.True(t, timestamp.IsZero())
assert.NoError(t, err)

// Get empty record
dr.CreateEmptyRecord(key)
value, exists, timestamp, err = dr.Get(key)
assert.Nil(t, value)
assert.True(t, exists)
assert.False(t, timestamp.IsZero())
assert.NoError(t, err)

// Get SealOutput
dr.SetValue(key, []byte{4, 5, 6})
value, exists, timestamp, err = dr.Get(key)
assert.Equal(t, []byte{4, 5, 6}, value)
assert.True(t, exists)
assert.False(t, timestamp.IsZero())
assert.NoError(t, err)

// Get Require
keyRequire := PendingDecryption{Hash: Hash{4, 5, 6}, Type: Require}
dr.SetValue(keyRequire, true)
value, exists, timestamp, err = dr.Get(keyRequire)
assert.Equal(t, true, value)
assert.True(t, exists)
assert.False(t, timestamp.IsZero())
assert.NoError(t, err)

// Get Decrypt
keyDecrypt := PendingDecryption{Hash: Hash{7, 8, 9}, Type: Decrypt}
dr.SetValue(keyDecrypt, big.NewInt(123))
value, exists, timestamp, err = dr.Get(keyDecrypt)
assert.Equal(t, big.NewInt(123), value)
assert.True(t, exists)
assert.False(t, timestamp.IsZero())
assert.NoError(t, err)

// Get with wrong type
keyWrong := PendingDecryption{Hash: Hash{10, 11, 12}, Type: PrecompileName(99)}
dr.data[keyWrong] = DecryptionRecord{Value: "wrong", Timestamp: time.Now()}
value, exists, timestamp, err = dr.Get(keyWrong)
assert.Nil(t, value)
assert.True(t, exists)
assert.False(t, timestamp.IsZero())
assert.Error(t, err)
})

t.Run("Remove", func(t *testing.T) {
dr := NewDecryptionResultsMap()
key := PendingDecryption{Hash: Hash{1, 2, 3}, Type: SealOutput}

dr.SetValue(key, []byte{4, 5, 6})
assert.Len(t, dr.data, 1)

dr.Remove(key)
assert.Len(t, dr.data, 0)

// Removing non-existent key should not panic
dr.Remove(key)
})

t.Run("Concurrency", func(t *testing.T) {
dr := NewDecryptionResultsMap()
key := PendingDecryption{Hash: Hash{1, 2, 3}, Type: SealOutput}

done := make(chan bool)
go func() {
for i := 0; i < 1000; i++ {
dr.CreateEmptyRecord(key)
dr.SetValue(key, []byte{byte(i)})
dr.Get(key)
}
done <- true
}()

go func() {
for i := 0; i < 1000; i++ {
dr.CreateEmptyRecord(key)
dr.SetValue(key, []byte{byte(i)})
dr.Get(key)
}
done <- true
}()

<-done
<-done

// No race condition should occur
})
}
Loading