diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 6f06ae67c22..c7771f9fc67 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -41,11 +41,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@c7f9125735019aa87cfc361530512d50ea439c71 # v3.25.1 + uses: github/codeql-action/init@d39d31e687223d841ef683f52467bd88e9b21c14 # v3.25.3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -59,7 +59,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@c7f9125735019aa87cfc361530512d50ea439c71 # v3.25.1 + uses: github/codeql-action/autobuild@d39d31e687223d841ef683f52467bd88e9b21c14 # v3.25.3 # ℹī¸ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -72,6 +72,6 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@c7f9125735019aa87cfc361530512d50ea439c71 # v3.25.1 + uses: github/codeql-action/analyze@d39d31e687223d841ef683f52467bd88e9b21c14 # v3.25.3 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 700ed2f4c07..9fd57fbdf81 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -12,12 +12,12 @@ jobs: name: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3 - - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 with: go-version: "1.21" - name: golangci-lint - uses: golangci/golangci-lint-action@3cfe3a4abbb849e10058ce4af15d205b6da42804 # v4.0.0 + uses: golangci/golangci-lint-action@9d1e0624a798bb64f6c3cea93db47765312263dc # v5.1.0 with: version: v1.55.1 args: --timeout=5m diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 5cc30e704b5..187f96a3f85 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -32,7 +32,7 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 with: persist-credentials: false @@ -59,7 +59,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4.3.2 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: name: SARIF file path: results.sarif @@ -67,6 +67,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@c7f9125735019aa87cfc361530512d50ea439c71 # v3.25.1 + uses: github/codeql-action/upload-sarif@d39d31e687223d841ef683f52467bd88e9b21c14 # v3.25.3 with: sarif_file: results.sarif diff --git a/go.mod b/go.mod index eae8c5ba465..b31df81e7d8 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,16 @@ go 1.21 require ( github.com/fullstorydev/grpcurl v1.9.1 + github.com/go-sql-driver/mysql v1.7.1 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.6.0 github.com/google/trillian v1.6.0 github.com/gorilla/mux v1.8.1 + github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-sqlite3 v1.14.22 github.com/prometheus/client_golang v1.19.0 - github.com/rs/cors v1.10.1 + github.com/rs/cors v1.11.0 github.com/sergi/go-diff v1.3.1 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 @@ -20,11 +22,11 @@ require ( go.etcd.io/etcd/client/v3 v3.5.13 go.etcd.io/etcd/etcdctl/v3 v3.5.13 go.etcd.io/etcd/v3 v3.5.13 - golang.org/x/crypto v0.22.0 - golang.org/x/net v0.24.0 + golang.org/x/crypto v0.23.0 + golang.org/x/net v0.25.0 golang.org/x/time v0.5.0 google.golang.org/grpc v1.63.2 - google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002 + google.golang.org/protobuf v1.34.1 gopkg.in/yaml.v3 v3.0.1 k8s.io/klog/v2 v2.120.1 ) @@ -53,7 +55,6 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-sql-driver/mysql v1.7.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect @@ -114,8 +115,8 @@ require ( golang.org/x/mod v0.14.0 // indirect golang.org/x/oauth2 v0.17.0 // indirect golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/tools v0.16.1 // indirect google.golang.org/api v0.162.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/go.sum b/go.sum index 87678cd5b19..2d48655d31d 100644 --- a/go.sum +++ b/go.sum @@ -140,6 +140,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 h1:RtRsiaGvWxcwd8y3BiRZxsylPT8hLWZ5SPcfI+3IDNk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0/go.mod h1:TzP6duP4Py2pHLVPPQp42aoYI92+PCrVotyR5e8Vqlk= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jhump/protoreflect v1.16.0 h1:54fZg+49widqXYQ0b+usAFHbMkBGR4PpXrsHc8+TBDg= @@ -210,8 +212,8 @@ github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= -github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rs/cors v1.11.0 h1:0B9GE/r9Bc2UxRMMtymBkHTenPkHDv0CW4Y98GBY+po= +github.com/rs/cors v1.11.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= @@ -312,8 +314,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -342,8 +344,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ= @@ -375,8 +377,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -386,8 +388,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -444,8 +446,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002 h1:V7Da7qt0MkY3noVANIMVBk28nOnijADeOR3i5Hcvpj4= -google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/integration/Dockerfile b/integration/Dockerfile index 48c29d75da3..5cb7f216a94 100644 --- a/integration/Dockerfile +++ b/integration/Dockerfile @@ -1,6 +1,6 @@ # This Dockerfile builds a base image for the certificate-transparency-go CloudBuild integration testing. # See https://hub.docker.com/_/golang for the set of golang base images. -FROM golang:1.22.2-bookworm@sha256:b03f3ba515751657c75475b20941fef47341fccb3341c3c0b64283ff15d3fb46 as ct_testbase +FROM golang:1.22.2-bookworm@sha256:d0902bacefdde1cf45528c098d14e55d78c107def8a22d148eabd71582d7a99f as ct_testbase WORKDIR /testbase diff --git a/internal/witness/cmd/feeder/Dockerfile b/internal/witness/cmd/feeder/Dockerfile index 9b72557dd16..1837ad88a96 100644 --- a/internal/witness/cmd/feeder/Dockerfile +++ b/internal/witness/cmd/feeder/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22.2-bookworm@sha256:b03f3ba515751657c75475b20941fef47341fccb3341c3c0b64283ff15d3fb46 as builder +FROM golang:1.22.2-bookworm@sha256:d0902bacefdde1cf45528c098d14e55d78c107def8a22d148eabd71582d7a99f as builder ARG GOFLAGS="" ENV GOFLAGS=$GOFLAGS diff --git a/internal/witness/cmd/witness/Dockerfile b/internal/witness/cmd/witness/Dockerfile index 147c13ebf12..b2fa43d9d6e 100644 --- a/internal/witness/cmd/witness/Dockerfile +++ b/internal/witness/cmd/witness/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22.2-bookworm@sha256:b03f3ba515751657c75475b20941fef47341fccb3341c3c0b64283ff15d3fb46 AS builder +FROM golang:1.22.2-bookworm@sha256:d0902bacefdde1cf45528c098d14e55d78c107def8a22d148eabd71582d7a99f AS builder ARG GOFLAGS="" ENV GOFLAGS=$GOFLAGS @@ -18,7 +18,7 @@ COPY . . RUN go build -o /build/bin/witness ./internal/witness/cmd/witness # Build release image -FROM golang:1.22.2-bookworm@sha256:b03f3ba515751657c75475b20941fef47341fccb3341c3c0b64283ff15d3fb46 +FROM golang:1.22.2-bookworm@sha256:d0902bacefdde1cf45528c098d14e55d78c107def8a22d148eabd71582d7a99f COPY --from=builder /build/bin/witness /bin/witness ENTRYPOINT ["/bin/witness"] diff --git a/trillian/ctfe/cache/cache.go b/trillian/ctfe/cache/cache.go new file mode 100644 index 00000000000..3ef06cb9702 --- /dev/null +++ b/trillian/ctfe/cache/cache.go @@ -0,0 +1,27 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cache defines the IssuanceChainCache type, which allows different cache implementation with Get and Set operations. +package cache + +import "context" + +// IssuanceChainCache is an interface which allows CTFE binaries to use different cache implementations for issuance chains. +type IssuanceChainCache interface { + // Get returns the issuance chain associated with the provided hash. + Get(ctx context.Context, key []byte) ([]byte, error) + + // Set inserts the key-value pair of issuance chain. + Set(ctx context.Context, key []byte, chain []byte) error +} diff --git a/trillian/ctfe/cache/lru/lru.go b/trillian/ctfe/cache/lru/lru.go new file mode 100644 index 00000000000..400e8df84e3 --- /dev/null +++ b/trillian/ctfe/cache/lru/lru.go @@ -0,0 +1,51 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package lru defines the IssuanceChainCache type, which implements IssuanceChainCache interface with Get and Set operations. +package lru + +import ( + "context" + "time" + + "github.com/hashicorp/golang-lru/v2/expirable" +) + +type CacheOption struct { + Size int + TTL time.Duration +} + +type IssuanceChainCache struct { + opt CacheOption + cache *expirable.LRU[string, []byte] +} + +func NewIssuanceChainCache(opt CacheOption) *IssuanceChainCache { + cache := expirable.NewLRU[string, []byte](int(opt.Size), nil, opt.TTL) + return &IssuanceChainCache{ + opt: opt, + cache: cache, + } +} + +func (c *IssuanceChainCache) Get(_ context.Context, key []byte) ([]byte, error) { + chain, _ := c.cache.Get(string(key)) + return chain, nil +} + +func (c *IssuanceChainCache) Set(_ context.Context, key []byte, chain []byte) error { + c.cache.Add(string(key), chain) + return nil +} diff --git a/trillian/ctfe/cache/lru/lru_test.go b/trillian/ctfe/cache/lru/lru_test.go new file mode 100644 index 00000000000..19e615675dc --- /dev/null +++ b/trillian/ctfe/cache/lru/lru_test.go @@ -0,0 +1,68 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lru + +import ( + "bytes" + "context" + "crypto/sha256" + "os" + "testing" +) + +func TestLRUIssuanceChainCache(t *testing.T) { + cacheSize := 3 + + tests := setupTestData(t, + "leaf00.chain", + "leaf01.chain", + "leaf02.chain", + ) + + cache := NewIssuanceChainCache(CacheOption{Size: cacheSize}) + + for key, val := range tests { + if err := cache.Set(context.Background(), []byte(key), val); err != nil { + t.Errorf("cache.Set: %v", err) + } + } + + for key, want := range tests { + got, err := cache.Get(context.Background(), []byte(key)) + if err != nil { + t.Errorf("cache.Get: %v", err) + } + if !bytes.Equal(got, want) { + t.Errorf("got: %v, want: %v", got, want) + } + } +} + +func setupTestData(t *testing.T, filenames ...string) map[string][]byte { + t.Helper() + + data := make(map[string][]byte, len(filenames)) + + for _, filename := range filenames { + val, err := os.ReadFile("../../../testdata/" + filename) + if err != nil { + t.Fatal(err) + } + key := sha256.Sum256(val) + data[string(key[:])] = val + } + + return data +} diff --git a/trillian/ctfe/config.go b/trillian/ctfe/config.go index 4656c288feb..d3788f5ef00 100644 --- a/trillian/ctfe/config.go +++ b/trillian/ctfe/config.go @@ -19,8 +19,10 @@ import ( "errors" "fmt" "os" + "strings" "time" + "github.com/go-sql-driver/mysql" ct "github.com/google/certificate-transparency-go" "github.com/google/certificate-transparency-go/trillian/ctfe/configpb" "github.com/google/certificate-transparency-go/x509" @@ -32,13 +34,15 @@ import ( // ValidatedLogConfig represents the LogConfig with the information that has // been successfully parsed as a result of validating it. type ValidatedLogConfig struct { - Config *configpb.LogConfig - PubKey crypto.PublicKey - PrivKey proto.Message - KeyUsages []x509.ExtKeyUsage - NotAfterStart *time.Time - NotAfterLimit *time.Time - FrozenSTH *ct.SignedTreeHead + Config *configpb.LogConfig + PubKey crypto.PublicKey + PrivKey proto.Message + KeyUsages []x509.ExtKeyUsage + NotAfterStart *time.Time + NotAfterLimit *time.Time + FrozenSTH *ct.SignedTreeHead + CTFEStorageConnectionString string + ExtraDataIssuanceChainStorageBackend configpb.LogConfig_IssuanceChainStorageBackend } // LogConfigFromFile creates a slice of LogConfig options from the given @@ -210,6 +214,25 @@ func ValidateLogConfig(cfg *configpb.LogConfig) (*ValidatedLogConfig, error) { } } + switch cfg.ExtraDataIssuanceChainStorageBackend { + case configpb.LogConfig_ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE: + // Validate the combination of CtfeStorageConnectionString and ExtraDataIssuanceChainStorageBackend + if len(cfg.CtfeStorageConnectionString) == 0 { + return nil, errors.New("missing ctfe_storage_connection_string when issuance chain storage backend is CTFE") + } + // Validate CTFEStorageConnectionString + if !strings.HasPrefix(cfg.CtfeStorageConnectionString, "mysql") { + return nil, errors.New("unsupported driver in ctfe_storage_connection_string") + } + if _, err := mysql.ParseDSN(strings.Split(cfg.CtfeStorageConnectionString, "://")[1]); err != nil { + return nil, errors.New("failed to parse ctfe_storage_connection_string for mysql driver") + } + vCfg.CTFEStorageConnectionString = cfg.CtfeStorageConnectionString + case configpb.LogConfig_ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC: + // Nothing to validate for Trillian gRPC + } + vCfg.ExtraDataIssuanceChainStorageBackend = cfg.ExtraDataIssuanceChainStorageBackend + return &vCfg, nil } diff --git a/trillian/ctfe/config_test.go b/trillian/ctfe/config_test.go index 4e63dd5030a..104a1e509a0 100644 --- a/trillian/ctfe/config_test.go +++ b/trillian/ctfe/config_test.go @@ -259,6 +259,35 @@ func TestValidateLogConfig(t *testing.T) { FrozenSth: corruptedSTH, }, }, + { + desc: "invalid-ctfe-storage-connection-string-mysql", + wantErr: "failed to parse ctfe_storage_connection_string for mysql driver", + cfg: &configpb.LogConfig{ + LogId: 123, + PrivateKey: privKey, + CtfeStorageConnectionString: "mysql://test:zaphod@localhost:3306/test", + ExtraDataIssuanceChainStorageBackend: configpb.LogConfig_ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE, + }, + }, + { + desc: "unsupported-driver-in-ctfe-storage-connection-string", + wantErr: "unsupported driver in ctfe_storage_connection_string", + cfg: &configpb.LogConfig{ + LogId: 123, + PrivateKey: privKey, + CtfeStorageConnectionString: "spanner://test:zaphod@tcp(localhost:3306)/test", + ExtraDataIssuanceChainStorageBackend: configpb.LogConfig_ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE, + }, + }, + { + desc: "missing-ctfe-storage-connection-string-when-issuance-chain-storage-backend-ctfe", + wantErr: "missing ctfe_storage_connection_string when issuance chain storage backend is CTFE", + cfg: &configpb.LogConfig{ + LogId: 123, + PrivateKey: privKey, + ExtraDataIssuanceChainStorageBackend: configpb.LogConfig_ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE, + }, + }, { desc: "ok", cfg: &configpb.LogConfig{ @@ -336,6 +365,31 @@ func TestValidateLogConfig(t *testing.T) { FrozenSth: validSTH, }, }, + { + desc: "ok-ctfe-storage-connection-string-mysql", + cfg: &configpb.LogConfig{ + LogId: 123, + PrivateKey: privKey, + CtfeStorageConnectionString: "mysql://test:zaphod@tcp(localhost:3306)/test", + }, + }, + { + desc: "ok-extra-data-issuance-chain-storage-backend-trillian", + cfg: &configpb.LogConfig{ + LogId: 123, + PrivateKey: privKey, + ExtraDataIssuanceChainStorageBackend: configpb.LogConfig_ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC, + }, + }, + { + desc: "ok-extra-data-issuance-chain-storage-backend-ctfe", + cfg: &configpb.LogConfig{ + LogId: 123, + PrivateKey: privKey, + ExtraDataIssuanceChainStorageBackend: configpb.LogConfig_ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE, + CtfeStorageConnectionString: "mysql://test:zaphod@tcp(localhost:3306)/test", + }, + }, } { t.Run(tc.desc, func(t *testing.T) { vc, err := ValidateLogConfig(tc.cfg) diff --git a/trillian/ctfe/configpb/config.pb.go b/trillian/ctfe/configpb/config.pb.go index 37134fabaf8..cf38922a429 100644 --- a/trillian/ctfe/configpb/config.pb.go +++ b/trillian/ctfe/configpb/config.pb.go @@ -37,6 +37,56 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// An optional storage backend for the issuance chain in ExtraData. +// By default, the storage backend is Trillian GRPC. To use CTFE as the +// storage backend, the CTFE storage connection string needs to be specified. +// Do not change this value during the log's lifetime. +type LogConfig_IssuanceChainStorageBackend int32 + +const ( + LogConfig_ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC LogConfig_IssuanceChainStorageBackend = 0 + LogConfig_ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE LogConfig_IssuanceChainStorageBackend = 1 +) + +// Enum value maps for LogConfig_IssuanceChainStorageBackend. +var ( + LogConfig_IssuanceChainStorageBackend_name = map[int32]string{ + 0: "ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC", + 1: "ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE", + } + LogConfig_IssuanceChainStorageBackend_value = map[string]int32{ + "ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC": 0, + "ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE": 1, + } +) + +func (x LogConfig_IssuanceChainStorageBackend) Enum() *LogConfig_IssuanceChainStorageBackend { + p := new(LogConfig_IssuanceChainStorageBackend) + *p = x + return p +} + +func (x LogConfig_IssuanceChainStorageBackend) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (LogConfig_IssuanceChainStorageBackend) Descriptor() protoreflect.EnumDescriptor { + return file_trillian_ctfe_configpb_config_proto_enumTypes[0].Descriptor() +} + +func (LogConfig_IssuanceChainStorageBackend) Type() protoreflect.EnumType { + return &file_trillian_ctfe_configpb_config_proto_enumTypes[0] +} + +func (x LogConfig_IssuanceChainStorageBackend) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use LogConfig_IssuanceChainStorageBackend.Descriptor instead. +func (LogConfig_IssuanceChainStorageBackend) EnumDescriptor() ([]byte, []int) { + return file_trillian_ctfe_configpb_config_proto_rawDescGZIP(), []int{3, 0} +} + type LogBackend struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -196,7 +246,7 @@ func (x *LogConfigSet) GetConfig() []*LogConfig { // LogConfig describes the configuration options for a log instance. // -// NEXT_ID: 20 +// NEXT_ID: 22 type LogConfig struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -283,6 +333,18 @@ type LogConfig struct { // A list of X.509 extension OIDs, in dotted string form (e.g. "2.3.4.5") // which should cause submissions to be rejected. RejectExtensions []string `protobuf:"bytes,18,rep,name=reject_extensions,json=rejectExtensions,proto3" json:"reject_extensions,omitempty"` + // CTFE storage connection string in the following format in general: + // driver://[username[:password]@][protocol[(host[:port])]][/[schema|database][?options]] + // + // MySQL/MariaDB: + // mysql://[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...¶mN=valueN] + // + // This is required when the issuance chain storage backend is CTFE. + // + // Warning: CT log operators are advised not to re-use the same connection + // string across multiple LogConfigs due to the log lifecycle. + CtfeStorageConnectionString string `protobuf:"bytes,20,opt,name=ctfe_storage_connection_string,json=ctfeStorageConnectionString,proto3" json:"ctfe_storage_connection_string,omitempty"` + ExtraDataIssuanceChainStorageBackend LogConfig_IssuanceChainStorageBackend `protobuf:"varint,21,opt,name=extra_data_issuance_chain_storage_backend,json=extraDataIssuanceChainStorageBackend,proto3,enum=configpb.LogConfig_IssuanceChainStorageBackend" json:"extra_data_issuance_chain_storage_backend,omitempty"` } func (x *LogConfig) Reset() { @@ -450,6 +512,20 @@ func (x *LogConfig) GetRejectExtensions() []string { return nil } +func (x *LogConfig) GetCtfeStorageConnectionString() string { + if x != nil { + return x.CtfeStorageConnectionString + } + return "" +} + +func (x *LogConfig) GetExtraDataIssuanceChainStorageBackend() LogConfig_IssuanceChainStorageBackend { + if x != nil { + return x.ExtraDataIssuanceChainStorageBackend + } + return LogConfig_ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC +} + // LogMultiConfig wraps up a LogBackendSet and corresponding LogConfigSet so // that they can easily be parsed as a single proto. type LogMultiConfig struct { @@ -608,7 +684,7 @@ var file_trillian_ctfe_configpb_config_proto_rawDesc = []byte{ 0x0c, 0x4c, 0x6f, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x65, 0x74, 0x12, 0x2b, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x67, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0xdd, 0x06, 0x0a, 0x09, 0x4c, + 0x69, 0x67, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0xa7, 0x09, 0x0a, 0x09, 0x4c, 0x6f, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x15, 0x0a, 0x06, 0x6c, 0x6f, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x6c, 0x6f, 0x67, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, @@ -662,31 +738,51 @@ var file_trillian_ctfe_configpb_config_proto_rawDesc = []byte{ 0x61, 0x64, 0x52, 0x09, 0x66, 0x72, 0x6f, 0x7a, 0x65, 0x6e, 0x53, 0x74, 0x68, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x12, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x72, 0x65, 0x6a, 0x65, 0x63, 0x74, - 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x7e, 0x0a, 0x0e, 0x4c, 0x6f, - 0x67, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x33, 0x0a, 0x08, - 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, - 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x67, 0x42, 0x61, 0x63, - 0x6b, 0x65, 0x6e, 0x64, 0x53, 0x65, 0x74, 0x52, 0x08, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, - 0x73, 0x12, 0x37, 0x0a, 0x0b, 0x6c, 0x6f, 0x67, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x70, - 0x62, 0x2e, 0x4c, 0x6f, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x65, 0x74, 0x52, 0x0a, - 0x6c, 0x6f, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x22, 0xa5, 0x01, 0x0a, 0x0e, 0x53, - 0x69, 0x67, 0x6e, 0x65, 0x64, 0x54, 0x72, 0x65, 0x65, 0x48, 0x65, 0x61, 0x64, 0x12, 0x1b, 0x0a, - 0x09, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x08, 0x74, 0x72, 0x65, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x68, 0x61, 0x32, - 0x35, 0x36, 0x5f, 0x72, 0x6f, 0x6f, 0x74, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x0e, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, 0x52, 0x6f, 0x6f, 0x74, 0x48, 0x61, - 0x73, 0x68, 0x12, 0x2e, 0x0a, 0x13, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x68, 0x65, 0x61, 0x64, 0x5f, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x11, 0x74, 0x72, 0x65, 0x65, 0x48, 0x65, 0x61, 0x64, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x42, 0x46, 0x5a, 0x44, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, - 0x61, 0x74, 0x65, 0x2d, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x63, 0x79, - 0x2d, 0x67, 0x6f, 0x2f, 0x74, 0x72, 0x69, 0x6c, 0x6c, 0x69, 0x61, 0x6e, 0x2f, 0x63, 0x74, 0x66, - 0x65, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x43, 0x0a, 0x1e, 0x63, 0x74, + 0x66, 0x65, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x14, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x1b, 0x63, 0x74, 0x66, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x12, + 0x88, 0x01, 0x0a, 0x29, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x69, + 0x73, 0x73, 0x75, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x73, 0x74, + 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x18, 0x15, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x2f, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x70, 0x62, 0x2e, 0x4c, + 0x6f, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x49, 0x73, 0x73, 0x75, 0x61, 0x6e, 0x63, + 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x61, 0x63, + 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x24, 0x65, 0x78, 0x74, 0x72, 0x61, 0x44, 0x61, 0x74, 0x61, 0x49, + 0x73, 0x73, 0x75, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x53, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x22, 0x78, 0x0a, 0x1b, 0x49, 0x73, + 0x73, 0x75, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x53, 0x74, 0x6f, 0x72, 0x61, + 0x67, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x30, 0x0a, 0x2c, 0x49, 0x53, 0x53, + 0x55, 0x41, 0x4e, 0x43, 0x45, 0x5f, 0x43, 0x48, 0x41, 0x49, 0x4e, 0x5f, 0x53, 0x54, 0x4f, 0x52, + 0x41, 0x47, 0x45, 0x5f, 0x42, 0x41, 0x43, 0x4b, 0x45, 0x4e, 0x44, 0x5f, 0x54, 0x52, 0x49, 0x4c, + 0x4c, 0x49, 0x41, 0x4e, 0x5f, 0x47, 0x52, 0x50, 0x43, 0x10, 0x00, 0x12, 0x27, 0x0a, 0x23, 0x49, + 0x53, 0x53, 0x55, 0x41, 0x4e, 0x43, 0x45, 0x5f, 0x43, 0x48, 0x41, 0x49, 0x4e, 0x5f, 0x53, 0x54, + 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x42, 0x41, 0x43, 0x4b, 0x45, 0x4e, 0x44, 0x5f, 0x43, 0x54, + 0x46, 0x45, 0x10, 0x01, 0x22, 0x7e, 0x0a, 0x0e, 0x4c, 0x6f, 0x67, 0x4d, 0x75, 0x6c, 0x74, 0x69, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x33, 0x0a, 0x08, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, + 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x67, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x53, 0x65, + 0x74, 0x52, 0x08, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x12, 0x37, 0x0a, 0x0b, 0x6c, + 0x6f, 0x67, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x67, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x65, 0x74, 0x52, 0x0a, 0x6c, 0x6f, 0x67, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x73, 0x22, 0xa5, 0x01, 0x0a, 0x0e, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x54, + 0x72, 0x65, 0x65, 0x48, 0x65, 0x61, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x72, 0x65, 0x65, 0x5f, + 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x74, 0x72, 0x65, 0x65, + 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, 0x5f, 0x72, 0x6f, 0x6f, + 0x74, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0e, 0x73, 0x68, + 0x61, 0x32, 0x35, 0x36, 0x52, 0x6f, 0x6f, 0x74, 0x48, 0x61, 0x73, 0x68, 0x12, 0x2e, 0x0a, 0x13, + 0x74, 0x72, 0x65, 0x65, 0x5f, 0x68, 0x65, 0x61, 0x64, 0x5f, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x11, 0x74, 0x72, 0x65, 0x65, 0x48, + 0x65, 0x61, 0x64, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x42, 0x46, 0x5a, 0x44, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2f, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x2d, 0x74, 0x72, + 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x2d, 0x67, 0x6f, 0x2f, 0x74, 0x72, + 0x69, 0x6c, 0x6c, 0x69, 0x61, 0x6e, 0x2f, 0x63, 0x74, 0x66, 0x65, 0x2f, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -701,33 +797,36 @@ func file_trillian_ctfe_configpb_config_proto_rawDescGZIP() []byte { return file_trillian_ctfe_configpb_config_proto_rawDescData } +var file_trillian_ctfe_configpb_config_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_trillian_ctfe_configpb_config_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_trillian_ctfe_configpb_config_proto_goTypes = []interface{}{ - (*LogBackend)(nil), // 0: configpb.LogBackend - (*LogBackendSet)(nil), // 1: configpb.LogBackendSet - (*LogConfigSet)(nil), // 2: configpb.LogConfigSet - (*LogConfig)(nil), // 3: configpb.LogConfig - (*LogMultiConfig)(nil), // 4: configpb.LogMultiConfig - (*SignedTreeHead)(nil), // 5: configpb.SignedTreeHead - (*anypb.Any)(nil), // 6: google.protobuf.Any - (*keyspb.PublicKey)(nil), // 7: keyspb.PublicKey - (*timestamppb.Timestamp)(nil), // 8: google.protobuf.Timestamp + (LogConfig_IssuanceChainStorageBackend)(0), // 0: configpb.LogConfig.IssuanceChainStorageBackend + (*LogBackend)(nil), // 1: configpb.LogBackend + (*LogBackendSet)(nil), // 2: configpb.LogBackendSet + (*LogConfigSet)(nil), // 3: configpb.LogConfigSet + (*LogConfig)(nil), // 4: configpb.LogConfig + (*LogMultiConfig)(nil), // 5: configpb.LogMultiConfig + (*SignedTreeHead)(nil), // 6: configpb.SignedTreeHead + (*anypb.Any)(nil), // 7: google.protobuf.Any + (*keyspb.PublicKey)(nil), // 8: keyspb.PublicKey + (*timestamppb.Timestamp)(nil), // 9: google.protobuf.Timestamp } var file_trillian_ctfe_configpb_config_proto_depIdxs = []int32{ - 0, // 0: configpb.LogBackendSet.backend:type_name -> configpb.LogBackend - 3, // 1: configpb.LogConfigSet.config:type_name -> configpb.LogConfig - 6, // 2: configpb.LogConfig.private_key:type_name -> google.protobuf.Any - 7, // 3: configpb.LogConfig.public_key:type_name -> keyspb.PublicKey - 8, // 4: configpb.LogConfig.not_after_start:type_name -> google.protobuf.Timestamp - 8, // 5: configpb.LogConfig.not_after_limit:type_name -> google.protobuf.Timestamp - 5, // 6: configpb.LogConfig.frozen_sth:type_name -> configpb.SignedTreeHead - 1, // 7: configpb.LogMultiConfig.backends:type_name -> configpb.LogBackendSet - 2, // 8: configpb.LogMultiConfig.log_configs:type_name -> configpb.LogConfigSet - 9, // [9:9] is the sub-list for method output_type - 9, // [9:9] is the sub-list for method input_type - 9, // [9:9] is the sub-list for extension type_name - 9, // [9:9] is the sub-list for extension extendee - 0, // [0:9] is the sub-list for field type_name + 1, // 0: configpb.LogBackendSet.backend:type_name -> configpb.LogBackend + 4, // 1: configpb.LogConfigSet.config:type_name -> configpb.LogConfig + 7, // 2: configpb.LogConfig.private_key:type_name -> google.protobuf.Any + 8, // 3: configpb.LogConfig.public_key:type_name -> keyspb.PublicKey + 9, // 4: configpb.LogConfig.not_after_start:type_name -> google.protobuf.Timestamp + 9, // 5: configpb.LogConfig.not_after_limit:type_name -> google.protobuf.Timestamp + 6, // 6: configpb.LogConfig.frozen_sth:type_name -> configpb.SignedTreeHead + 0, // 7: configpb.LogConfig.extra_data_issuance_chain_storage_backend:type_name -> configpb.LogConfig.IssuanceChainStorageBackend + 2, // 8: configpb.LogMultiConfig.backends:type_name -> configpb.LogBackendSet + 3, // 9: configpb.LogMultiConfig.log_configs:type_name -> configpb.LogConfigSet + 10, // [10:10] is the sub-list for method output_type + 10, // [10:10] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name } func init() { file_trillian_ctfe_configpb_config_proto_init() } @@ -814,13 +913,14 @@ func file_trillian_ctfe_configpb_config_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_trillian_ctfe_configpb_config_proto_rawDesc, - NumEnums: 0, + NumEnums: 1, NumMessages: 6, NumExtensions: 0, NumServices: 0, }, GoTypes: file_trillian_ctfe_configpb_config_proto_goTypes, DependencyIndexes: file_trillian_ctfe_configpb_config_proto_depIdxs, + EnumInfos: file_trillian_ctfe_configpb_config_proto_enumTypes, MessageInfos: file_trillian_ctfe_configpb_config_proto_msgTypes, }.Build() File_trillian_ctfe_configpb_config_proto = out.File diff --git a/trillian/ctfe/configpb/config.proto b/trillian/ctfe/configpb/config.proto index 6f78a9bab5d..8b0a2d03ffe 100644 --- a/trillian/ctfe/configpb/config.proto +++ b/trillian/ctfe/configpb/config.proto @@ -45,7 +45,7 @@ message LogConfigSet { // LogConfig describes the configuration options for a log instance. // -// NEXT_ID: 20 +// NEXT_ID: 22 message LogConfig { // The ID of a Trillian tree that stores the log data. The tree type must be // LOG for regular CT logs. For mirror logs it must be either PREORDERED_LOG @@ -132,6 +132,28 @@ message LogConfig { // A list of X.509 extension OIDs, in dotted string form (e.g. "2.3.4.5") // which should cause submissions to be rejected. repeated string reject_extensions = 18; + + // CTFE storage connection string in the following format in general: + // driver://[username[:password]@][protocol[(host[:port])]][/[schema|database][?options]] + // + // MySQL/MariaDB: + // mysql://[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...¶mN=valueN] + // + // This is required when the issuance chain storage backend is CTFE. + // + // Warning: CT log operators are advised not to re-use the same connection + // string across multiple LogConfigs due to the log lifecycle. + string ctfe_storage_connection_string = 20; + + // An optional storage backend for the issuance chain in ExtraData. + // By default, the storage backend is Trillian GRPC. To use CTFE as the + // storage backend, the CTFE storage connection string needs to be specified. + // Do not change this value during the log's lifetime. + enum IssuanceChainStorageBackend { + ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC = 0; + ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE = 1; + } + IssuanceChainStorageBackend extra_data_issuance_chain_storage_backend = 21; } // LogMultiConfig wraps up a LogBackendSet and corresponding LogConfigSet so diff --git a/trillian/ctfe/services.go b/trillian/ctfe/services.go new file mode 100644 index 00000000000..580cf8763eb --- /dev/null +++ b/trillian/ctfe/services.go @@ -0,0 +1,89 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ctfe + +import ( + "context" + "crypto/sha256" + + "github.com/google/certificate-transparency-go/trillian/ctfe/cache" + "github.com/google/certificate-transparency-go/trillian/ctfe/storage" + "k8s.io/klog/v2" +) + +type issuanceChainService struct { + storage storage.IssuanceChainStorage + cache cache.IssuanceChainCache +} + +func newIssuanceChainService(s storage.IssuanceChainStorage, c cache.IssuanceChainCache) *issuanceChainService { + service := &issuanceChainService{ + storage: s, + cache: c, + } + + return service +} + +// GetByHash returns the issuance chain with hash as the input. +func (s *issuanceChainService) GetByHash(ctx context.Context, hash []byte) ([]byte, error) { + // Return if found in cache. + chain, err := s.cache.Get(ctx, hash) + if chain != nil || err != nil { + return chain, err + } + + // Find in storage if cache miss. + chain, err = s.storage.FindByKey(ctx, hash) + if err != nil { + return nil, err + } + + // If there is any error from cache set, do not return the error because + // the chain is still available for read. + go func(ctx context.Context, hash, chain []byte) { + if err := s.cache.Set(ctx, hash, chain); err != nil { + klog.Errorf("failed to set hash and chain into cache: %v", err) + } + }(ctx, hash, chain) + + return chain, nil +} + +// Add adds the issuance chain into the storage and cache and returns the hash +// of the chain. +func (s *issuanceChainService) Add(ctx context.Context, chain []byte) ([]byte, error) { + hash := issuanceChainHash(chain) + + if err := s.storage.Add(ctx, hash, chain); err != nil { + return nil, err + } + + // If there is any error from cache set, do not return the error because + // the chain is already stored. + go func(ctx context.Context, hash, chain []byte) { + if err := s.cache.Set(ctx, hash, chain); err != nil { + klog.Errorf("failed to set hash and chain into cache: %v", err) + } + }(ctx, hash, chain) + + return hash, nil +} + +// issuanceChainHash returns the SHA-256 hash of the chain. +func issuanceChainHash(chain []byte) []byte { + checksum := sha256.Sum256(chain) + return checksum[:] +} diff --git a/trillian/ctfe/services_test.go b/trillian/ctfe/services_test.go new file mode 100644 index 00000000000..3a9f7c12b41 --- /dev/null +++ b/trillian/ctfe/services_test.go @@ -0,0 +1,116 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ctfe + +import ( + "bytes" + "context" + "crypto/sha256" + "os" + "sync" + "testing" +) + +func TestIssuanceChainServiceAddAndGet(t *testing.T) { + tests := []struct { + chain []byte + }{ + {readTestData(t, "leaf00.chain")}, + {readTestData(t, "leaf01.chain")}, + {readTestData(t, "leaf02.chain")}, + {nil}, + } + + ctx := context.Background() + storage := &fakeIssuanceChainStorage{} + cache := &fakeIssuanceChainCache{} + issuanceChainService := newIssuanceChainService(storage, cache) + + for _, test := range tests { + hash, err := issuanceChainService.Add(ctx, test.chain) + if err != nil { + t.Errorf("IssuanceChainService.Add(): %v", err) + } + + got, err := issuanceChainService.GetByHash(ctx, hash) + if err != nil { + t.Errorf("IssuanceChainService.GetByHash(): %v", err) + } + + if !bytes.Equal(got, test.chain) { + t.Errorf("GetByHash = %v, want %v", got, test.chain) + } + } +} + +func TestIssuanceChainHashLen(t *testing.T) { + want := sha256.Size + tests := []struct { + chain []byte + }{ + {readTestData(t, "leaf00.chain")}, + {readTestData(t, "leaf01.chain")}, + {readTestData(t, "leaf02.chain")}, + {nil}, + } + + for _, test := range tests { + got := len(issuanceChainHash(test.chain)) + if got != want { + t.Errorf("len(issuanceChainHash(%v)) = %d, want %d", test.chain, got, want) + } + } +} + +func readTestData(t *testing.T, filename string) []byte { + t.Helper() + + data, err := os.ReadFile("../testdata/" + filename) + if err != nil { + t.Fatal(err) + } + + return data +} + +type fakeIssuanceChainStorage struct { + chains sync.Map +} + +func (s *fakeIssuanceChainStorage) FindByKey(_ context.Context, key []byte) ([]byte, error) { + val, _ := s.chains.Load(string(key)) + chain, _ := val.([]byte) + return chain, nil +} + +func (s *fakeIssuanceChainStorage) Add(_ context.Context, key []byte, chain []byte) error { + s.chains.Store(string(key), chain) + return nil +} + +type fakeIssuanceChainCache struct { + chains sync.Map +} + +func (c *fakeIssuanceChainCache) Get(_ context.Context, key []byte) ([]byte, error) { + val, _ := c.chains.Load(string(key)) + chain, _ := val.([]byte) + return chain, nil +} + +func (c *fakeIssuanceChainCache) Set(_ context.Context, key []byte, chain []byte) error { + c.chains.Store(string(key), chain) + return nil +} diff --git a/trillian/ctfe/storage/storage.go b/trillian/ctfe/storage/storage.go new file mode 100644 index 00000000000..5fccb72419f --- /dev/null +++ b/trillian/ctfe/storage/storage.go @@ -0,0 +1,29 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package storage defines the IssuanceChainStorage type, which allows different storage implementation for the key-value pairs of issuance chains. +package storage + +import ( + "context" +) + +// IssuanceChainStorage is an interface which allows CTFE binaries to use different storage implementations for issuance chains. +type IssuanceChainStorage interface { + // FindByKey returns the issuance chain associated with the provided key. + FindByKey(ctx context.Context, key []byte) ([]byte, error) + + // Add inserts the key-value pair of issuance chain. + Add(ctx context.Context, key []byte, chain []byte) error +} diff --git a/trillian/examples/deployment/docker/ctfe/Dockerfile b/trillian/examples/deployment/docker/ctfe/Dockerfile index 3069dbce882..447227a5f5f 100644 --- a/trillian/examples/deployment/docker/ctfe/Dockerfile +++ b/trillian/examples/deployment/docker/ctfe/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22.2-bookworm@sha256:b03f3ba515751657c75475b20941fef47341fccb3341c3c0b64283ff15d3fb46 as build +FROM golang:1.22.2-bookworm@sha256:d0902bacefdde1cf45528c098d14e55d78c107def8a22d148eabd71582d7a99f as build ARG GOFLAGS="" ENV GOFLAGS=$GOFLAGS @@ -12,7 +12,7 @@ COPY . . RUN go build ./trillian/ctfe/ct_server -FROM gcr.io/distroless/base-debian12@sha256:611d30d7f6d9992c37b1e1a212eefdf1f7c671deb56db3707e24eb01da8c4c2a +FROM gcr.io/distroless/base-debian12@sha256:786007f631d22e8a1a5084c5b177352d9dcac24b1e8c815187750f70b24a9fc6 COPY --from=build /build/ct_server / diff --git a/types.go b/types.go index b6af606bff7..2a96f6a09f5 100644 --- a/types.go +++ b/types.go @@ -248,6 +248,21 @@ type CertificateChain struct { Entries []ASN1Cert `tls:"minlen:0,maxlen:16777215"` } +// PrecertChainEntryHash is an extended PrecertChainEntry type with the +// IssuanceChainHash field added to store the hash of the +// CertificateChain field of PrecertChainEntry. +type PrecertChainEntryHash struct { + PreCertificate ASN1Cert `tls:"minlen:1,maxlen:16777215"` + IssuanceChainHash []byte `tls:"minlen:0,maxlen:256"` +} + +// CertificateChainHash is an extended CertificateChain type with the +// IssuanceChainHash field added to store the hash of the +// Entries field of CertificateChain. +type CertificateChainHash struct { + IssuanceChainHash []byte `tls:"minlen:0,maxlen:256"` +} + // JSONDataEntry holds arbitrary data. type JSONDataEntry struct { Data []byte `tls:"minlen:0,maxlen:1677215"`