From a930c7e57e84ce3a470c5cf1fd9c95f96e3434e1 Mon Sep 17 00:00:00 2001 From: wuhuizuo Date: Mon, 30 Sep 2024 07:40:45 +0000 Subject: [PATCH] feat(tiup-publisher): init implement Signed-off-by: wuhuizuo --- tiup-publisher/go.mod | 31 +++++ tiup-publisher/go.sum | 118 ++++++++++++++++++ tiup-publisher/handler.go | 211 +++++++++++++++++++++++++++++++++ tiup-publisher/handler_test.go | 45 +++++++ tiup-publisher/main.go | 64 ++++++++++ tiup-publisher/types.go | 61 ++++++++++ 6 files changed, 530 insertions(+) create mode 100644 tiup-publisher/go.mod create mode 100644 tiup-publisher/go.sum create mode 100644 tiup-publisher/handler.go create mode 100644 tiup-publisher/handler_test.go create mode 100644 tiup-publisher/main.go create mode 100644 tiup-publisher/types.go diff --git a/tiup-publisher/go.mod b/tiup-publisher/go.mod new file mode 100644 index 0000000..61307ab --- /dev/null +++ b/tiup-publisher/go.mod @@ -0,0 +1,31 @@ +module github.com/pingcap-qe/ee-apps/tiup-publisher + +go 1.23.1 + +require ( + github.com/PingCAP-QE/ee-apps/dl v0.0.0-20240926143418-da3f56b05e46 + github.com/cloudevents/sdk-go/v2 v2.15.2 + github.com/segmentio/kafka-go v0.4.47 + github.com/stretchr/testify v1.9.0 + gopkg.in/yaml.v3 v3.0.1 + oras.land/oras-go/v2 v2.5.0 +) + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/net v0.28.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/time v0.5.0 // indirect +) diff --git a/tiup-publisher/go.sum b/tiup-publisher/go.sum new file mode 100644 index 0000000..2e8cf2b --- /dev/null +++ b/tiup-publisher/go.sum @@ -0,0 +1,118 @@ +github.com/PingCAP-QE/ee-apps/dl v0.0.0-20240926143418-da3f56b05e46 h1:VsOcy58bTAFgXngZl2DgWyzLp22lPOFITGBl0Cpg/aU= +github.com/PingCAP-QE/ee-apps/dl v0.0.0-20240926143418-da3f56b05e46/go.mod h1:SgEiJheCBFDHDeMSK5hB/xZtGAJBM/L6f6YU40Gg7wo= +github.com/cloudevents/sdk-go/v2 v2.15.2 h1:54+I5xQEnI73RBhWHxbI1XJcqOFOVJN85vb41+8mHUc= +github.com/cloudevents/sdk-go/v2 v2.15.2/go.mod h1:lL7kSWAE/V8VI4Wh0jbL2v/jvqsm6tjmaQBSvxcv4uE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0= +github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c= +oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg= diff --git a/tiup-publisher/handler.go b/tiup-publisher/handler.go new file mode 100644 index 0000000..59ce438 --- /dev/null +++ b/tiup-publisher/handler.go @@ -0,0 +1,211 @@ +package main + +import ( + "context" + "crypto/sha256" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "slices" + "strings" + + cloudevents "github.com/cloudevents/sdk-go/v2" + "github.com/rs/zerolog/log" + "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/retry" + + "github.com/PingCAP-QE/ee-apps/dl/pkg/oci" +) + +type Handler struct { + mirrorsURL string +} + +func NewHandler(mirror string) (*Handler, error) { + return &Handler{mirror}, nil +} + +func (h *Handler) SupportEventTypes() []string { + return []string{EventTypeTiupPublishRequest} +} + +// Handle for test case run events +func (h *Handler) Handle(event cloudevents.Event) cloudevents.Result { + if !slices.Contains(h.SupportEventTypes(), event.Type()) { + return cloudevents.ResultNACK + } + + data := new(PublishRequestEvent) + if err := event.DataAs(&data); err != nil { + return cloudevents.NewReceipt(false, "invalid data: %v", err) + } + + return h.handleImpl(data) +} + +func (h *Handler) handleImpl(data *PublishRequestEvent) cloudevents.Result { + // 1. get the the tarball from data.From. + saveTo, err := h.downloadFile(data) + if err != nil { + return cloudevents.NewReceipt(true, "download file failed: %v", err) + } + + // 2. publish the tarball to the mirror. + if err := publish(saveTo, &data.Publish, h.mirrorsURL); err != nil { + return cloudevents.NewReceipt(true, "publish to mirror failed: %v", err) + } + + // 3. check the package is in the mirror. + // printf 'post_check "$(tiup mirror show)/%s-%s-%s-%s.tar.gz" "%s"\n' \ + remoteURL := fmt.Sprintf("%s/%s-%s-%s-%s.tar.gz", h.mirrorsURL, data.Publish.Name, data.Publish.Version, data.Publish.OS, data.Publish.Arch) + if err := postCheck(saveTo, remoteURL); err != nil { + return cloudevents.NewReceipt(true, "post check failed: %v", err) + } + return cloudevents.ResultACK +} + +func (h *Handler) downloadFile(data *PublishRequestEvent) (string, error) { + switch data.From.Type { + case FromTypeOci: + // save to file with `saveTo` param: + return downloadOCIFile(data.From.Oci) + case FromTypeHTTP: + return downloadHTTPFile(data.From.HTTP.URL) + default: + return "", nil + } +} + +func publish(file string, info *PublishInfo, to string) error { + args := []string{"mirror", "publish", info.Name, info.Version, file, info.EntryPoint, "--os", info.OS, "--arch", info.Arch, "--desc", info.Description} + if info.Standalone { + args = append(args, "--standalone") + } + command := exec.Command("tiup", args...) + command.Env = os.Environ() + command.Env = append(command.Env, "TIUP_MIRRORS="+to) + log.Debug().Any("args", command.Args).Any("env", command.Args).Msg("will execute tiup command") + output, err := command.Output() + if err != nil { + log.Err(err).Msg("failed to execute tiup command") + return err + } + log.Info(). + Str("mirror", to). + Str("output", string(output)). + Msg("tiup package publish success") + + return nil +} + +// check the remote file after published. +func postCheck(localFile, remoteFileURL string) error { + // 1. Calculate the sha256sum of the local file + localSum, err := calculateSHA256(localFile) + if err != nil { + return fmt.Errorf("failed to calculate local file sha256: %v", err) + } + + // 2. Download the remote file + tempFile, err := downloadHTTPFile(remoteFileURL) + if err != nil { + return fmt.Errorf("failed to download remote file: %v", err) + } + defer os.Remove(tempFile) + + // 3. Calculate the sha256sum of the remote file + remoteSum, err := calculateSHA256(tempFile) + if err != nil { + return fmt.Errorf("failed to calculate remote file sha256: %v", err) + } + + // 4. Compare the two sha256sums + if localSum != remoteSum { + return fmt.Errorf("sha256 mismatch: local %s, remote %s", localSum, remoteSum) + } + + return nil +} + +func calculateSHA256(filePath string) (string, error) { + f, err := os.Open(filePath) + if err != nil { + return "", err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +func downloadHTTPFile(url string) (string, error) { + resp, err := http.Get(url) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + tempFile, err := os.CreateTemp("", "remote_file_*") + if err != nil { + return "", err + } + defer tempFile.Close() + + _, err = io.Copy(tempFile, resp.Body) + if err != nil { + return "", err + } + + return tempFile.Name(), nil +} + +func downloadOCIFile(from *FromOci) (string, error) { + repo, err := getOciRepo(from.Repo) + if err != nil { + return "", err + } + rc, _, err := oci.NewFileReadCloser(context.Background(), repo, from.Tag, from.File) + if err != nil { + return "", err + } + defer rc.Close() + + tempFile, err := os.CreateTemp("", "oci_download_*.tar.gz") + if err != nil { + return "", err + } + defer tempFile.Close() + + if _, err := io.Copy(tempFile, rc); err != nil { + return "", err + } + + return tempFile.Name(), nil +} + +func getOciRepo(repo string) (*remote.Repository, error) { + repository, err := remote.NewRepository(repo) + if err != nil { + return nil, err + } + + reg := strings.SplitN(repo, "/", 2)[0] + repository.Client = &auth.Client{ + Client: retry.DefaultClient, + Cache: auth.DefaultCache, + Credential: auth.StaticCredential(reg, auth.EmptyCredential), + } + + return repository, nil +} diff --git a/tiup-publisher/handler_test.go b/tiup-publisher/handler_test.go new file mode 100644 index 0000000..333d3fe --- /dev/null +++ b/tiup-publisher/handler_test.go @@ -0,0 +1,45 @@ +package main + +import ( + "testing" + + cloudevents "github.com/cloudevents/sdk-go/v2" + "github.com/cloudevents/sdk-go/v2/event" + "github.com/stretchr/testify/assert" +) + +const testMirrorURL = "http://tiup.pingcap.net:8988" + +func TestHandler_Handle(t *testing.T) { + t.Skip("manual test case") + + t.Run("Valid event data", func(tt *testing.T) { + event := event.New() + event.SetType(EventTypeTiupPublishRequest) + event.SetSource("testSource") + event.SetData("application/json", &PublishRequestEvent{ + From: From{ + Type: "oci", + Oci: &FromOci{ + Repo: "hub.pingcap.net/pingcap/tidb/package", + Tag: "master_darwin_amd64", + File: "tidb-v8.4.0-alpha-312-g8f0baf4444-darwin-amd64.tar.gz", + }, + }, + Publish: PublishInfo{ + Name: "tidb", + Version: "v8.4.0-alpha-n2", + EntryPoint: "tidb-server", + OS: "darwin", + Arch: "amd64", + Description: "TiDB is an open source distributed HTAP database compatible with the MySQL protocol.", + Standalone: false, + }, + }) + + h := &Handler{testMirrorURL} + result := h.Handle(event) + assert.True(tt, cloudevents.IsACK(result)) + tt.Fail() + }) +} diff --git a/tiup-publisher/main.go b/tiup-publisher/main.go new file mode 100644 index 0000000..1dee688 --- /dev/null +++ b/tiup-publisher/main.go @@ -0,0 +1,64 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "os" + + "github.com/cloudevents/sdk-go/v2/event" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/segmentio/kafka-go" + "gopkg.in/yaml.v3" +) + +func main() { + zerolog.TimeFieldFormat = zerolog.TimeFormatUnix + log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) + + configFile := flag.String("config", "./config.yaml", "Path to config file") + flag.Parse() + + configData, err := os.ReadFile(*configFile) + if err != nil { + log.Fatal().Err(err).Msg("Error reading config file") + } + + var config Config + if err := yaml.Unmarshal(configData, &config); err != nil { + log.Fatal().Err(err).Msg("Error parsing config file") + } + handler, err := NewHandler(config.MirrorUrl) + if err != nil { + log.Fatal().Err(err).Msg("Error creating handler") + } + + reader := kafka.NewReader(kafka.ReaderConfig{ + Brokers: config.Brokers, + Topic: config.Topic, + GroupID: config.ConsumerGroup, + MinBytes: 10e3, + MaxBytes: 10e6, + CommitInterval: 5000, + Logger: kafka.LoggerFunc(log.Printf), + }) + defer reader.Close() + + for { + msg, err := reader.ReadMessage(context.Background()) + if err != nil { + log.Err(err).Msg("Error reading message") + continue + } + + var cloudEvent event.Event + if err := json.Unmarshal(msg.Value, &cloudEvent); err != nil { + log.Err(err).Msg("Error unmarshaling CloudEvent") + continue + } + + log.Debug().Str("ce-id", cloudEvent.ID()).Str("ce-type", cloudEvent.Type()).Msg("received cloud event") + handler.Handle(cloudEvent) + } +} diff --git a/tiup-publisher/types.go b/tiup-publisher/types.go new file mode 100644 index 0000000..baf7629 --- /dev/null +++ b/tiup-publisher/types.go @@ -0,0 +1,61 @@ +package main + +const ( + EventTypeTiupPublishRequest = "tiup-publish-request" + FromTypeOci = "oci" + FromTypeHTTP = "http" +) + +type Config struct { + Brokers []string `yaml:"brokers"` + Topic string `yaml:"topic"` + Credentials struct { + Type string `yaml:"type"` + Username string `yaml:"username"` + Password string `yaml:"password"` + } `yaml:"credentials"` + ConsumerGroup string `yaml:"consumerGroup"` + MirrorUrl string `yaml:"mirrorUrl"` +} + +type PublishRequestEvent struct { + MirrorName string `json:"mirror_name,omitempty"` // staging or production. + From From `json:"from,omitempty"` + Publish PublishInfo `json:"publish,omitempty"` +} +type From struct { + Type string `json:"type,omitempty"` + Oci *FromOci `json:"oci,omitempty"` + HTTP *FromHTTP `json:"http,omitempty"` +} + +type PublishInfo struct { + Name string `json:"name,omitempty"` + OS string `json:"os,omitempty"` + Arch string `json:"arch,omitempty"` + Version string `json:"version,omitempty"` + Description string `json:"description,omitempty"` + EntryPoint string `json:"entry_point,omitempty"` + Standalone bool `json:"standalone,omitempty"` +} + +type FromOci struct { + Repo string `json:"repo,omitempty"` + Tag string `json:"tag,omitempty"` + File string `json:"file,omitempty"` +} + +type FromHTTP struct { + URL string `json:"url,omitempty"` +} + +// 2024-09-23T20:02:29.932583969+08:00 tiup mirror publish ctl v8.4.0-alpha-nightly ctl-v8.4.0-alpha-41-gedb43c053-darwin-amd64.tar.gz ctl --os darwin --arch amd64 --desc "TiDB controller suite" + +type TiupMirror struct { + Name string `yaml:"name,omitempty" json:"name,omitempty"` + URL string `yaml:"url,omitempty" json:"url,omitempty"` + Auth struct { + Username string `yaml:"username,omitempty" json:"username,omitempty"` + Password string `yaml:"password,omitempty" json:"password,omitempty"` + } `yaml:"auth,omitempty" json:"auth,omitempty"` +}