From bfef4b4b1cba94483c69d4944854aba36771ca2e Mon Sep 17 00:00:00 2001 From: tarassh Date: Sat, 24 Aug 2024 10:25:46 -0700 Subject: [PATCH] update c-kzg dep. remove old kzg libs --- c-kzg-4844 | 2 +- go.mod | 3 - go.sum | 7 - internal/pinner/kzg/confing.go | 58 ------- internal/pinner/kzg/encoder.go | 255 ---------------------------- internal/pinner/kzg/encoder_test.go | 137 --------------- internal/pinner/kzg/process.go | 26 --- internal/pinner/kzg/setup.go | 220 ------------------------ internal/pinner/kzg/utils.go | 137 --------------- 9 files changed, 1 insertion(+), 844 deletions(-) delete mode 100644 internal/pinner/kzg/confing.go delete mode 100644 internal/pinner/kzg/encoder.go delete mode 100644 internal/pinner/kzg/encoder_test.go delete mode 100644 internal/pinner/kzg/process.go delete mode 100644 internal/pinner/kzg/setup.go delete mode 100644 internal/pinner/kzg/utils.go diff --git a/c-kzg-4844 b/c-kzg-4844 index 62cbdb1..252a6b2 160000 --- a/c-kzg-4844 +++ b/c-kzg-4844 @@ -1 +1 @@ -Subproject commit 62cbdb17f97be7461e5d0c14c5f7c927d577025d +Subproject commit 252a6b2c2c6fd478bd0ccc0edaf846e5d83f9f73 diff --git a/go.mod b/go.mod index d171a18..1cdc208 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,6 @@ require ( github.com/ipld/go-ipld-prime v0.21.0 github.com/multiformats/go-multicodec v0.9.0 github.com/multiformats/go-multihash v0.2.3 - github.com/protolambda/go-kzg v0.0.0-20221224134646-c91cee5e954e github.com/spf13/cobra v1.8.1 github.com/web3-storage/go-ucanto v0.1.0 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 @@ -89,7 +88,6 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect - github.com/herumi/bls-eth-go-binary v1.28.1 // indirect github.com/holiman/uint256 v1.3.1 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -127,7 +125,6 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect - github.com/kilic/bls12-381 v0.1.1-0.20220929213557-ca162e8a70f4 // indirect github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/koron/go-ssdp v0.0.4 // indirect diff --git a/go.sum b/go.sum index ae05096..dad202d 100644 --- a/go.sum +++ b/go.sum @@ -338,8 +338,6 @@ github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/herumi/bls-eth-go-binary v1.28.1 h1:fcIZ48y5EE9973k05XjE8+P3YiQgjZz4JI/YabAm8KA= -github.com/herumi/bls-eth-go-binary v1.28.1/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 h1:X4egAf/gcS1zATw6wn4Ej8vjuVGxeHdan+bRb2ebyv4= github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= @@ -487,8 +485,6 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= -github.com/kilic/bls12-381 v0.1.1-0.20220929213557-ca162e8a70f4 h1:xWK4TZ4bRL05WQUU/3x6TG1l+IYAqdXpAeSLt/zZJc4= -github.com/kilic/bls12-381 v0.1.1-0.20220929213557-ca162e8a70f4/go.mod h1:tlkavyke+Ac7h8R3gZIjI5LKBcvMlSWnXNMgT3vZXo8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -764,8 +760,6 @@ github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.15.0 h1:A82kmvXJq2jTu5YUhSGNlYoxh85zLnKgPz4bMZgI5Ek= github.com/prometheus/procfs v0.15.0/go.mod h1:Y0RJ/Y5g5wJpkTisOtqwDSo4HwhGmLB4VQSw2sQJLHk= -github.com/protolambda/go-kzg v0.0.0-20221224134646-c91cee5e954e h1:Wed8Zc7HuSVNDJQy6ybac5/1fpRoyhMDcPvGlMjyIiY= -github.com/protolambda/go-kzg v0.0.0-20221224134646-c91cee5e954e/go.mod h1:7EhkBJFo/qJ9sToiW5baPqbyPo/TadVHn4iNdpwEW/w= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/quic-go v0.44.0 h1:So5wOr7jyO4vzL2sd8/pD9Kesciv91zSk8BoFngItQ0= @@ -1140,7 +1134,6 @@ golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/internal/pinner/kzg/confing.go b/internal/pinner/kzg/confing.go deleted file mode 100644 index 451a05a..0000000 --- a/internal/pinner/kzg/confing.go +++ /dev/null @@ -1,58 +0,0 @@ -package kzg - -import ( - "log" - "os" - "path/filepath" - "strconv" -) - -// Config holds the configuration for the KZG trusted setup. -type Config struct { - SubGroupLen uint64 - SubGroupCount uint64 - EvalLen uint64 - Seed string - StorageDir string -} - -// DefaultConfig provides default values for the KZG setup. -var DefaultConfig = Config{ - SubGroupLen: 1 << 6, - SubGroupCount: 1 << 8, - Seed: "1927409816240961209460912649124", - StorageDir: filepath.Join(os.Getenv("HOME"), ".pinner"), // Default storage directory -} - -// LoadConfig loads the configuration from environment variables or uses default values. -func LoadConfig() Config { - config := DefaultConfig - - if envSubGroupLen := os.Getenv("SUB_GROUP_LEN"); envSubGroupLen != "" { - if val, err := strconv.ParseUint(envSubGroupLen, 10, 64); err == nil { - config.SubGroupLen = val - } else { - log.Printf("Invalid SUB_GROUP_LEN value, using default: %v\n", DefaultConfig.SubGroupLen) - } - } - - if envSubGroupCount := os.Getenv("SUB_GROUP_COUNT"); envSubGroupCount != "" { - if val, err := strconv.ParseUint(envSubGroupCount, 10, 64); err == nil { - config.SubGroupCount = val - } else { - log.Printf("Invalid SUB_GROUP_COUNT value, using default: %v\n", DefaultConfig.SubGroupCount) - } - } - - if envSeed := os.Getenv("SEED"); envSeed != "" { - config.Seed = envSeed - } - - if envStorageDir := os.Getenv("STORAGE_DIR"); envStorageDir != "" { - config.StorageDir = envStorageDir - } - - config.EvalLen = config.SubGroupLen * config.SubGroupCount - - return config -} diff --git a/internal/pinner/kzg/encoder.go b/internal/pinner/kzg/encoder.go deleted file mode 100644 index 98778d7..0000000 --- a/internal/pinner/kzg/encoder.go +++ /dev/null @@ -1,255 +0,0 @@ -package kzg - -import ( - "errors" - "fmt" - "sync" - - "github.com/protolambda/go-kzg/bls" -) - -// Custom errors for encoding and decoding -var ( - ErrNoPolynomials = errors.New("no polynomials found for decoding") - ErrInvalidPolynomial = errors.New("invalid polynomial data") - ErrEncodingFailed = errors.New("failed to encode data into polynomials") - ErrDataLengthMismatch = errors.New("data length does not match expected size") - ErrInsufficientFieldSize = errors.New("insufficient field size for data") - ErrInvalidDegree = errors.New("invalid degree for polynomial encoding") - ErrDataExceedsCapacity = errors.New("data exceeds the capacity of the polynomial array") - ErrCopyingData = errors.New("error while copying data to bytes32 array") -) - -// DataBlock represents encoded data with polynomials and related cryptographic proofs. -type DataBlock struct { - Degree uint64 // Degree of the polynomials - FieldSize uint64 // Size of the field elements - TotalSize uint64 // Total size of the encoded data - Polynomials [][]bls.Fr // Encoded polynomial cells - Extended [][]bls.Fr // Extended polynomial cells - Proofs [][]bls.G1Point // Proofs associated with the polynomials - Commitments []bls.G1Point // Commitments for the polynomials -} - -// NewDataBlock creates a new instance of DataBlock with the specified degree and field size. -func NewDataBlock(degree, fieldSize uint64) (*DataBlock, error) { - if degree == 0 { - return nil, ErrInvalidDegree - } - return &DataBlock{ - Degree: degree, - FieldSize: fieldSize, - }, nil -} - -// Encode converts raw data into a series of polynomials. -func (db *DataBlock) Encode(data []byte) error { - if db.Degree == 0 { - return ErrInvalidDegree - } - - dataLen := uint64(len(data)) - if dataLen == 0 { - return ErrDataLengthMismatch - } - - // Calculate the number of polynomials required to encode the data - polynomialCount := (dataLen + db.Degree*db.FieldSize - 1) / (db.Degree * db.FieldSize) - - // Initialize the polynomials slice - db.Polynomials = make([][]bls.Fr, polynomialCount) - for i := range db.Polynomials { - db.Polynomials[i] = make([]bls.Fr, db.Degree) - } - - // Encode the data into polynomials - for i, offset := uint64(0), uint64(0); offset < dataLen; offset += db.FieldSize { - availableBytes := dataLen - offset - copySize := db.FieldSize - if availableBytes < db.FieldSize { - copySize = availableBytes - } - - var bytes32 [32]byte - if copySize > 32 { - return ErrInsufficientFieldSize - } - - copy(bytes32[:copySize], data[offset:offset+copySize]) - - if !bls.FrFrom32(&db.Polynomials[offset/(db.Degree*db.FieldSize)][i%db.Degree], bytes32) { - return fmt.Errorf("%w at index %d", ErrEncodingFailed, offset) - } - - i++ - } - - db.TotalSize = dataLen - return nil -} - -// Decode converts the stored polynomials back into the original data. -func (db *DataBlock) Decode() ([]byte, error) { - if len(db.Polynomials) == 0 { - return nil, ErrNoPolynomials - } - - polynomialCount := uint64(len(db.Polynomials)) - degree := uint64(len(db.Polynomials[0])) - - if degree*polynomialCount*db.FieldSize < db.TotalSize { - return nil, ErrInvalidPolynomial - } - - data := make([]byte, db.TotalSize) - dataIndex := uint64(0) - - // Decode the polynomials back into the original data - for polyIndex := uint64(0); polyIndex < polynomialCount; polyIndex++ { - for degIndex := uint64(0); degIndex < degree; degIndex++ { - if dataIndex >= db.TotalSize { - break - } - - bytes32 := bls.FrTo32(&db.Polynomials[polyIndex][degIndex]) - - // Calculate the number of bytes to copy, considering the potential end of the data - bytesToCopy := db.FieldSize - if dataIndex+db.FieldSize > db.TotalSize { - bytesToCopy = db.TotalSize - dataIndex - } - - copy(data[dataIndex:dataIndex+bytesToCopy], bytes32[:bytesToCopy]) - dataIndex += bytesToCopy - } - } - - return data, nil -} - -// Commit generates the commitments and proofs for the polynomials. -func (db *DataBlock) Commit(setup *TrustedSetup) error { - var wg sync.WaitGroup - resultCh := make(chan struct { - index int - commitment bls.G1Point - proofs []bls.G1Point - err error - }, len(db.Polynomials)) - - commitments := make([]bls.G1Point, len(db.Polynomials)) - proofs := make([][]bls.G1Point, len(db.Polynomials)) - - for i, poly := range db.Polynomials { - wg.Add(1) - go func(index int, data []bls.Fr) { - defer wg.Done() - - coefficients, err := setup.Fk20Settings.KZGSettings.FFTSettings.FFT(data, true) - if err != nil { - resultCh <- struct { - index int - commitment bls.G1Point - proofs []bls.G1Point - err error - }{index, bls.G1Point{}, nil, err} - return - } - - commitment := setup.Fk20Settings.KZGSettings.CommitToPoly(coefficients) - proof := setup.Fk20Settings.DAUsingFK20Multi(coefficients) - - resultCh <- struct { - index int - commitment bls.G1Point - proofs []bls.G1Point - err error - }{index, *commitment, proof, nil} - }(i, poly) - } - - wg.Wait() - close(resultCh) - - for result := range resultCh { - if result.err != nil { - return result.err - } - commitments[result.index] = result.commitment - proofs[result.index] = result.proofs - } - - db.Commitments = commitments - db.Proofs = proofs - - return nil -} - -// Extend extends the polynomials by zero-padding and performing forward and inverse FFT. -func (db *DataBlock) Extend(setup *TrustedSetup) error { - if len(db.Polynomials) == 0 { - return ErrNoPolynomials - } - - var wg sync.WaitGroup - resultCh := make(chan struct { - index int - extended []bls.Fr - err error - }, len(db.Polynomials)) - - for i, poly := range db.Polynomials { - wg.Add(1) - go func(index int, data []bls.Fr) { - defer wg.Done() - - // Perform forward FFT on the polynomial data - coefficients, err := setup.Fk20Settings.KZGSettings.FFTSettings.FFT(data, true) - if err != nil { - resultCh <- struct { - index int - extended []bls.Fr - err error - }{index, nil, err} - return - } - - // Prepare extended coefficients with zero padding - extendedCoeffs := make([]bls.Fr, setup.EvalLen*2) - copy(extendedCoeffs[:setup.EvalLen], coefficients) - // Zero padding is automatically done as the rest of extendedCoeffs is zero-initialized - - // Perform inverse FFT to obtain the extended polynomial - extended, err := setup.Fk20Settings.KZGSettings.FFTSettings.FFT(extendedCoeffs, false) - if err != nil { - resultCh <- struct { - index int - extended []bls.Fr - err error - }{index, nil, err} - return - } - reverseBitOrderFr(extended) - - resultCh <- struct { - index int - extended []bls.Fr - err error - }{index, extended, nil} - }(i, poly) - } - - wg.Wait() - close(resultCh) - - db.Extended = make([][]bls.Fr, len(db.Polynomials)) - - for result := range resultCh { - if result.err != nil { - return result.err - } - db.Extended[result.index] = result.extended - } - - return nil -} diff --git a/internal/pinner/kzg/encoder_test.go b/internal/pinner/kzg/encoder_test.go deleted file mode 100644 index 82b2714..0000000 --- a/internal/pinner/kzg/encoder_test.go +++ /dev/null @@ -1,137 +0,0 @@ -package kzg - -import ( - "bytes" - "testing" -) - -// TestEncodeDecode tests the encoding and decoding process. -func TestEncodeDecode(t *testing.T) { - // Initialize the test data - originalData := []byte("test data for encoding and decoding") - - // Define the parameters for the DataBlock - degree := uint64(4) - fieldSize := uint64(32) - - // Create a new DataBlock - db, err := NewDataBlock(degree, fieldSize) - if err != nil { - t.Fatalf("Failed to create DataBlock: %v", err) - } - - // Encode the data - err = db.Encode(originalData) - if err != nil { - t.Fatalf("Encoding failed: %v", err) - } - - // Decode the data - decodedData, err := db.Decode() - if err != nil { - t.Fatalf("Decoding failed: %v", err) - } - - // Verify the decoded data matches the original data - if !bytes.Equal(originalData, decodedData) { - t.Errorf("Decoded data does not match original. Got: %v, Want: %v", decodedData, originalData) - } -} - -// Additional tests for edge cases and error handling can be added here. -func TestLargeData(t *testing.T) { - originalData := make([]byte, 1024*1024*4+7) // 4MB of data - for i := range originalData { - originalData[i] = byte(i % 256) - } - - // Define the parameters for the DataBlock - degree := uint64(4) - fieldSize := uint64(31) - - // Create a new DataBlock - db, err := NewDataBlock(degree, fieldSize) - if err != nil { - t.Fatalf("Failed to create DataBlock: %v", err) - } - - // Encode the data - err = db.Encode(originalData) - if err != nil { - t.Fatalf("Encoding failed: %v", err) - } - - // Decode the data - decodedData, err := db.Decode() - if err != nil { - t.Fatalf("Decoding failed: %v", err) - } - - // Verify the decoded data matches the original data - if !bytes.Equal(originalData, decodedData) { - t.Errorf("Decoded data does not match original. Got: %v, Want: %v", decodedData, originalData) - } -} - -func TestInvalidDegree(t *testing.T) { - // Define the parameters for the DataBlock with degree 0 - degree := uint64(0) - fieldSize := uint64(32) - - // Try to create a new DataBlock - _, err := NewDataBlock(degree, fieldSize) - if err == nil { - t.Fatalf("Expected error when degree is invalid, but got nil") - } -} - -func TestPartialFieldSizeData(t *testing.T) { - originalData := []byte("incomplete field data") - - // Define the parameters for the DataBlock - degree := uint64(4) - fieldSize := uint64(32) - - // Create a new DataBlock - db, err := NewDataBlock(degree, fieldSize) - if err != nil { - t.Fatalf("Failed to create DataBlock: %v", err) - } - - // Encode the data - err = db.Encode(originalData) - if err != nil { - t.Fatalf("Encoding failed: %v", err) - } - - // Decode the data - decodedData, err := db.Decode() - if err != nil { - t.Fatalf("Decoding failed: %v", err) - } - - // Verify the decoded data matches the original data - if !bytes.Equal(originalData, decodedData) { - t.Errorf("Decoded data does not match original. Got: %v, Want: %v", decodedData, originalData) - } -} - -func TestOversizedFieldSize(t *testing.T) { - originalData := []byte("data that will be oversized when encoded") - - // Define the parameters for the DataBlock - degree := uint64(4) - fieldSize := uint64(64) // Oversized field size - - // Create a new DataBlock - db, err := NewDataBlock(degree, fieldSize) - if err != nil { - t.Fatalf("Failed to create DataBlock: %v", err) - } - - // Encode the data and expect an error - err = db.Encode(originalData) - if err == nil { - t.Fatalf("Expected encoding to fail due to oversized field size, but it succeeded") - } -} diff --git a/internal/pinner/kzg/process.go b/internal/pinner/kzg/process.go deleted file mode 100644 index 3f139fe..0000000 --- a/internal/pinner/kzg/process.go +++ /dev/null @@ -1,26 +0,0 @@ -package kzg - -// Encode converts raw data into a series of polynomials. -func Encode(data []byte) (*DataBlock, error) { - - // Encode the data into polynomials - encoded, err := NewDataBlock(setup.EvalLen, 31) - if err != nil { - return nil, err - } - - err = encoded.Encode(data) - if err != nil { - return nil, err - } - - decoded, err := encoded.Decode() - if err != nil { - return nil, err - } - - println("decoded", string(decoded)) - - // Commit to the polynomials - return nil, nil -} diff --git a/internal/pinner/kzg/setup.go b/internal/pinner/kzg/setup.go deleted file mode 100644 index 13a2622..0000000 --- a/internal/pinner/kzg/setup.go +++ /dev/null @@ -1,220 +0,0 @@ -package kzg - -import ( - "fmt" - "log" - "math/big" - "os" - "path/filepath" - "sync" - - "github.com/protolambda/go-kzg" - "github.com/protolambda/go-kzg/bls" -) - -const ( - - // G1CompressedSize is the size of a compressed G1 point in bytes. - G1CompressedSize = 48 - // G2CompressedSize is the size of a compressed G2 point in bytes. - G2CompressedSize = 96 - // splitFactor is the number of goroutines to use for concurrent reading. - splitFactor = 8 -) - -// TrustedSetup contains the necessary parameters and precomputed values for KZG commitments. -type TrustedSetup struct { - SubGroupLen uint64 - SubGroupCount uint64 - EvalLen uint64 - Fk20Settings *kzg.FK20MultiSettings -} - -var setup *TrustedSetup - -// InitTrustedSetup initializes the trusted setup for KZG commitments using the provided configuration. -// It attempts to load cached setup data if available and verifies the seed matches. -func InitTrustedSetup(config Config) error { - if setup != nil { - return fmt.Errorf("trusted setup already initialized") - } - - // Attempt to load the setup from the binary files - s1, s2, err := LoadSetup(config) - if err == nil && len(s1) > 0 && len(s2) > 0 { - // Successfully loaded setup, now use it - log.Println("Loaded trusted setup from cache") - setup = initializeFromPoints(s1, s2, config) - return nil - } - - // If loading fails, generate the setup and cache it - log.Println("Generating new trusted setup") - s1, s2 = kzg.GenerateTestingSetup(config.Seed, config.EvalLen*2) - if err := StoreSetup(s1, s2, config); err != nil { - return fmt.Errorf("failed to store trusted setup: %w", err) - } - - setup = initializeFromPoints(s1, s2, config) - return nil -} - -func initializeFromPoints(s1 []bls.G1Point, s2 []bls.G2Point, config Config) *TrustedSetup { - scaleBits := bitsNeeded(config.EvalLen) - fftSettings := kzg.NewFFTSettings(scaleBits) - kzgSettings := kzg.NewKZGSettings(fftSettings, s1, s2) - fk20Settings := kzg.NewFK20MultiSettings(kzgSettings, config.EvalLen*2, config.SubGroupLen) - - return &TrustedSetup{ - SubGroupLen: config.SubGroupLen, - SubGroupCount: config.SubGroupCount, - EvalLen: config.EvalLen, - Fk20Settings: fk20Settings, - } -} - -// StoreSetup stores the KZG setup (s1, s2) to binary files. -func StoreSetup(s1 []bls.G1Point, s2 []bls.G2Point, config Config) error { - if len(s1) != len(s2) { - return fmt.Errorf("s1 and s2 must have the same length") - } - - // Ensure storage directory exists - if err := os.MkdirAll(config.StorageDir, os.ModePerm); err != nil { - return fmt.Errorf("failed to create storage directory: %w", err) - } - - // Paths for storing the setup points - s1Path := filepath.Join(config.StorageDir, "s1.bin") - s2Path := filepath.Join(config.StorageDir, "s2.bin") - - // Create files for s1 and s2 - f1, err := os.Create(s1Path) - if err != nil { - return fmt.Errorf("failed to create file for s1: %w", err) - } - defer f1.Close() - - f2, err := os.Create(s2Path) - if err != nil { - return fmt.Errorf("failed to create file for s2: %w", err) - } - defer f2.Close() - - // Write s1 and s2 points to files in binary format - for i := 0; i < len(s1); i++ { - if _, err := f1.Write(bls.ToCompressedG1(&s1[i])); err != nil { - return fmt.Errorf("failed to write s1[%d]: %w", i, err) - } - if _, err := f2.Write(bls.ToCompressedG2(&s2[i])); err != nil { - return fmt.Errorf("failed to write s2[%d]: %w", i, err) - } - } - - return nil -} - -// LoadSetup loads the KZG setup (s1, s2) from binary files using concurrent reading. -func LoadSetup(config Config) ([]bls.G1Point, []bls.G2Point, error) { - s1Path := filepath.Join(config.StorageDir, "s1.bin") - s2Path := filepath.Join(config.StorageDir, "s2.bin") - - // Open the files for reading - f1, err := os.Open(s1Path) - if err != nil { - return nil, nil, fmt.Errorf("failed to open file for s1: %w", err) - } - defer f1.Close() - - f2, err := os.Open(s2Path) - if err != nil { - return nil, nil, fmt.Errorf("failed to open file for s2: %w", err) - } - defer f2.Close() - - // Determine the number of points - info, err := f1.Stat() - if err != nil { - return nil, nil, fmt.Errorf("failed to stat file for s1: %w", err) - } - n := uint64(info.Size() / G1CompressedSize) // Assuming each G1 point is of fixed compressed size - - // Allocate slices for points - s1 := make([]bls.G1Point, n) - s2 := make([]bls.G2Point, n) - - chunkSize := n / splitFactor - - var wg sync.WaitGroup - errCh := make(chan error, splitFactor) - defer close(errCh) - - readChunk := func(start, end uint64) { - defer wg.Done() - for i := start; i < end; i++ { - // Read G1 point - s1Data := make([]byte, G1CompressedSize) - _, err := f1.ReadAt(s1Data, int64(i*G1CompressedSize)) - if err != nil { - errCh <- fmt.Errorf("failed to read s1[%d]: %w", i, err) - return - } - point, err := bls.FromCompressedG1(s1Data) - if err != nil { - errCh <- fmt.Errorf("failed to decompress s1[%d]: %w", i, err) - return - } - s1[i] = *point - - // Read G2 point - s2Data := make([]byte, G2CompressedSize) - _, err = f2.ReadAt(s2Data, int64(i*G2CompressedSize)) - if err != nil { - errCh <- fmt.Errorf("failed to read s2[%d]: %w", i, err) - return - } - point2, err := bls.FromCompressedG2(s2Data) - if err != nil { - errCh <- fmt.Errorf("failed to decompress s2[%d]: %w", i, err) - return - } - s2[i] = *point2 - } - } - - // Launch concurrent reading tasks - for i := uint64(0); i < splitFactor; i++ { - start := i * chunkSize - end := start + chunkSize - if i == splitFactor-1 { - end = n // Ensure the last chunk covers any remaining items - } - wg.Add(1) - go readChunk(start, end) - } - - // Wait for all goroutines to complete - wg.Wait() - - // Check for errors from goroutines - select { - case err := <-errCh: - return nil, nil, err - default: - } - - return s1, s2, nil -} - -// bitsNeeded calculates the number of bits needed to represent the given value. -func bitsNeeded(x uint64) uint8 { - return uint8((big.NewInt(0).SetUint64(x).BitLen()+7)/8*8) - 1 -} - -// GetTrustedSetup returns the initialized trusted setup. -func GetTrustedSetup() (*TrustedSetup, error) { - if setup == nil { - return nil, fmt.Errorf("trusted setup not initialized") - } - return setup, nil -} diff --git a/internal/pinner/kzg/utils.go b/internal/pinner/kzg/utils.go deleted file mode 100644 index de0777e..0000000 --- a/internal/pinner/kzg/utils.go +++ /dev/null @@ -1,137 +0,0 @@ -package kzg - -import "github.com/protolambda/go-kzg/bls" - -var revByte = [256]byte{ - 0b00000000, 0b10000000, 0b01000000, 0b11000000, 0b00100000, 0b10100000, 0b01100000, 0b11100000, - 0b00010000, 0b10010000, 0b01010000, 0b11010000, 0b00110000, 0b10110000, 0b01110000, 0b11110000, - 0b00001000, 0b10001000, 0b01001000, 0b11001000, 0b00101000, 0b10101000, 0b01101000, 0b11101000, - 0b00011000, 0b10011000, 0b01011000, 0b11011000, 0b00111000, 0b10111000, 0b01111000, 0b11111000, - 0b00000100, 0b10000100, 0b01000100, 0b11000100, 0b00100100, 0b10100100, 0b01100100, 0b11100100, - 0b00010100, 0b10010100, 0b01010100, 0b11010100, 0b00110100, 0b10110100, 0b01110100, 0b11110100, - 0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100, - 0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100, - 0b00000010, 0b10000010, 0b01000010, 0b11000010, 0b00100010, 0b10100010, 0b01100010, 0b11100010, - 0b00010010, 0b10010010, 0b01010010, 0b11010010, 0b00110010, 0b10110010, 0b01110010, 0b11110010, - 0b00001010, 0b10001010, 0b01001010, 0b11001010, 0b00101010, 0b10101010, 0b01101010, 0b11101010, - 0b00011010, 0b10011010, 0b01011010, 0b11011010, 0b00111010, 0b10111010, 0b01111010, 0b11111010, - 0b00000110, 0b10000110, 0b01000110, 0b11000110, 0b00100110, 0b10100110, 0b01100110, 0b11100110, - 0b00010110, 0b10010110, 0b01010110, 0b11010110, 0b00110110, 0b10110110, 0b01110110, 0b11110110, - 0b00001110, 0b10001110, 0b01001110, 0b11001110, 0b00101110, 0b10101110, 0b01101110, 0b11101110, - 0b00011110, 0b10011110, 0b01011110, 0b11011110, 0b00111110, 0b10111110, 0b01111110, 0b11111110, - 0b00000001, 0b10000001, 0b01000001, 0b11000001, 0b00100001, 0b10100001, 0b01100001, 0b11100001, - 0b00010001, 0b10010001, 0b01010001, 0b11010001, 0b00110001, 0b10110001, 0b01110001, 0b11110001, - 0b00001001, 0b10001001, 0b01001001, 0b11001001, 0b00101001, 0b10101001, 0b01101001, 0b11101001, - 0b00011001, 0b10011001, 0b01011001, 0b11011001, 0b00111001, 0b10111001, 0b01111001, 0b11111001, - 0b00000101, 0b10000101, 0b01000101, 0b11000101, 0b00100101, 0b10100101, 0b01100101, 0b11100101, - 0b00010101, 0b10010101, 0b01010101, 0b11010101, 0b00110101, 0b10110101, 0b01110101, 0b11110101, - 0b00001101, 0b10001101, 0b01001101, 0b11001101, 0b00101101, 0b10101101, 0b01101101, 0b11101101, - 0b00011101, 0b10011101, 0b01011101, 0b11011101, 0b00111101, 0b10111101, 0b01111101, 0b11111101, - 0b00000011, 0b10000011, 0b01000011, 0b11000011, 0b00100011, 0b10100011, 0b01100011, 0b11100011, - 0b00010011, 0b10010011, 0b01010011, 0b11010011, 0b00110011, 0b10110011, 0b01110011, 0b11110011, - 0b00001011, 0b10001011, 0b01001011, 0b11001011, 0b00101011, 0b10101011, 0b01101011, 0b11101011, - 0b00011011, 0b10011011, 0b01011011, 0b11011011, 0b00111011, 0b10111011, 0b01111011, 0b11111011, - 0b00000111, 0b10000111, 0b01000111, 0b11000111, 0b00100111, 0b10100111, 0b01100111, 0b11100111, - 0b00010111, 0b10010111, 0b01010111, 0b11010111, 0b00110111, 0b10110111, 0b01110111, 0b11110111, - 0b00001111, 0b10001111, 0b01001111, 0b11001111, 0b00101111, 0b10101111, 0b01101111, 0b11101111, - 0b00011111, 0b10011111, 0b01011111, 0b11011111, 0b00111111, 0b10111111, 0b01111111, 0b11111111, -} - -// Bit masks for efficient bit indexing -const ( - mask0 = ^uint32((1 << (1 << iota)) - 1) - mask1 - mask2 - mask3 - mask4 - // Uncomment the following line if needed - //mask5 -) - -// Bit positions for setting specific bits in a uint8 value -const ( - bit0 = uint8(1 << iota) - bit1 - bit2 - bit3 - bit4 - // Uncomment the following line if needed - //bit5 -) - -// bitIndex finds the index of the first set bit in a uint32 value. -// Special case: zero has a 0 bit index. -// Example outputs: (0,0), (1,0), (2,1), (3,1), (4,2), etc. -func bitIndex(v uint32) uint8 { - if v == 0 { - return 0 - } - var out uint8 - // Uncomment the following block if mask5 - //if v&mask5 != 0 { - // v >>= bit5 - // out |= bit5 - //} - if v&mask4 != 0 { - v >>= bit4 - out |= bit4 - } - if v&mask3 != 0 { - v >>= bit3 - out |= bit3 - } - if v&mask2 != 0 { - v >>= bit2 - out |= bit2 - } - if v&mask1 != 0 { - v >>= bit1 - out |= bit1 - } - if v&mask0 != 0 { - out |= bit0 - } - return out -} - -// reverseBitsLimited reverses the bits in value up to the length specified. -func reverseBitsLimited(length uint32, value uint32) uint32 { - unusedBitLen := 32 - bitIndex(length) - return reverseBits(value) >> unusedBitLen -} - -// ReverseBitsLimited is a public function that calls reverseBitsLimited internally. -func ReverseBitsLimited(length uint32, value uint32) uint32 { - return reverseBitsLimited(length, value) -} - -// reverseBits reverses the bits in a uint32 value using a lookup table for performance. -func reverseBits(b uint32) uint32 { - return (uint32(revByte[uint8(b)]) << 24) | - (uint32(revByte[uint8(b>>8)]) << 16) | - (uint32(revByte[uint8(b>>16)]) << 8) | - uint32(revByte[uint8(b>>24)]) -} - -// reverseBitOrder reverses the bit order of indices and applies a swap function for each pair. -func reverseBitOrder(length uint32, swap func(i, j uint32)) { - if length == 0 || length&(length-1) != 0 { - panic("length must be a positive power of 2") - } - unusedBitLen := 32 - bitIndex(length) - for i := uint32(0); i < length; i++ { - if r := reverseBits(i) >> unusedBitLen; r > i { - swap(r, i) - } - } -} - -// reverseBitOrderFr rearranges elements in a slice of bls.Fr in reverse bit order. -func reverseBitOrderFr(values []bls.Fr) { - if len(values) > (1 << 31) { - panic("list too large") - } - reverseBitOrder(uint32(len(values)), func(i, j uint32) { - values[i], values[j] = values[j], values[i] - }) -}