From ef3a14460cfef9bad4f3deeb3e6e031dc95817f3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sat, 9 Sep 2023 23:27:57 -0600 Subject: [PATCH 1/9] Add support for MSC3916 --- api/_apimeta/auth.go | 4 + api/_auth_cache/auth_cache.go | 2 +- api/_routers/97-require-server-auth.go | 30 +++++ api/_routers/98-use-rcontext.go | 32 +++--- api/custom/federation.go | 3 + api/routes.go | 18 ++- api/unstable/msc3916_download.go | 37 ++++++ api/unstable/msc3916_thumbnail.go | 32 ++++++ matrix/requests_signing.go | 150 +++++++++++++++++++++++++ matrix/xmatrix.go | 65 +++++++++++ util/http.go | 77 +++++++++++++ util/readers/multipart_reader.go | 57 ++++++++++ 12 files changed, 490 insertions(+), 17 deletions(-) create mode 100644 api/_routers/97-require-server-auth.go create mode 100644 api/unstable/msc3916_download.go create mode 100644 api/unstable/msc3916_thumbnail.go create mode 100644 matrix/requests_signing.go create mode 100644 matrix/xmatrix.go create mode 100644 util/readers/multipart_reader.go diff --git a/api/_apimeta/auth.go b/api/_apimeta/auth.go index 5bd5dfd6..d16df56f 100644 --- a/api/_apimeta/auth.go +++ b/api/_apimeta/auth.go @@ -16,6 +16,10 @@ type UserInfo struct { IsShared bool } +type ServerInfo struct { + ServerName string +} + func GetRequestUserAdminStatus(r *http.Request, rctx rcontext.RequestContext, user UserInfo) (bool, bool) { isGlobalAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared isLocalAdmin, err := matrix.IsUserAdmin(rctx, r.Host, user.AccessToken, r.RemoteAddr) diff --git a/api/_auth_cache/auth_cache.go b/api/_auth_cache/auth_cache.go index da1f2d5c..8ac47758 100644 --- a/api/_auth_cache/auth_cache.go +++ b/api/_auth_cache/auth_cache.go @@ -12,7 +12,7 @@ import ( "github.com/t2bot/matrix-media-repo/matrix" ) -var tokenCache = cache.New(0*time.Second, 30*time.Second) +var tokenCache = cache.New(cache.NoExpiration, 30*time.Second) var rwLock = &sync.RWMutex{} var regexCache = make(map[string]*regexp.Regexp) diff --git a/api/_routers/97-require-server-auth.go b/api/_routers/97-require-server-auth.go new file mode 100644 index 00000000..b7ca3bac --- /dev/null +++ b/api/_routers/97-require-server-auth.go @@ -0,0 +1,30 @@ +package _routers + +import ( + "net/http" + + "github.com/turt2live/matrix-media-repo/api/_apimeta" + "github.com/turt2live/matrix-media-repo/api/_responses" + "github.com/turt2live/matrix-media-repo/common" + "github.com/turt2live/matrix-media-repo/common/rcontext" + "github.com/turt2live/matrix-media-repo/matrix" +) + +type GeneratorWithServerFn = func(r *http.Request, ctx rcontext.RequestContext, server _apimeta.ServerInfo) interface{} + +func RequireServerAuth(generator GeneratorWithServerFn) GeneratorFn { + return func(r *http.Request, ctx rcontext.RequestContext) interface{} { + serverName, err := matrix.ValidateXMatrixAuth(r, true) + if err != nil { + ctx.Log.Debug("Error with X-Matrix auth: ", err) + return &_responses.ErrorResponse{ + Code: common.ErrCodeForbidden, + Message: "no auth provided (required)", + InternalCode: common.ErrCodeMissingToken, + } + } + return generator(r, ctx, _apimeta.ServerInfo{ + ServerName: serverName, + }) + } +} diff --git a/api/_routers/98-use-rcontext.go b/api/_routers/98-use-rcontext.go index 8776fd21..2852f2b0 100644 --- a/api/_routers/98-use-rcontext.go +++ b/api/_routers/98-use-rcontext.go @@ -95,20 +95,24 @@ func (c *RContextRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { beforeParseDownload: log.Infof("Replying with result: %T %+v", res, res) if downloadRes, isDownload := res.(*_responses.DownloadResponse); isDownload { - ranges, err := http_range.ParseRange(r.Header.Get("Range"), downloadRes.SizeBytes, rctx.Config.Downloads.DefaultRangeChunkSizeBytes) - if errors.Is(err, http_range.ErrInvalid) { - proposedStatusCode = http.StatusRequestedRangeNotSatisfiable - res = _responses.BadRequest("invalid range header") - goto beforeParseDownload // reprocess `res` - } else if errors.Is(err, http_range.ErrNoOverlap) { - proposedStatusCode = http.StatusRequestedRangeNotSatisfiable - res = _responses.BadRequest("out of range") - goto beforeParseDownload // reprocess `res` - } - if len(ranges) > 1 { - proposedStatusCode = http.StatusRequestedRangeNotSatisfiable - res = _responses.BadRequest("only 1 range is supported") - goto beforeParseDownload // reprocess `res` + var ranges []http_range.Range + var err error + if downloadRes.SizeBytes > 0 { + ranges, err = http_range.ParseRange(r.Header.Get("Range"), downloadRes.SizeBytes, rctx.Config.Downloads.DefaultRangeChunkSizeBytes) + if errors.Is(err, http_range.ErrInvalid) { + proposedStatusCode = http.StatusRequestedRangeNotSatisfiable + res = _responses.BadRequest("invalid range header") + goto beforeParseDownload // reprocess `res` + } else if errors.Is(err, http_range.ErrNoOverlap) { + proposedStatusCode = http.StatusRequestedRangeNotSatisfiable + res = _responses.BadRequest("out of range") + goto beforeParseDownload // reprocess `res` + } + if len(ranges) > 1 { + proposedStatusCode = http.StatusRequestedRangeNotSatisfiable + res = _responses.BadRequest("only 1 range is supported") + goto beforeParseDownload // reprocess `res` + } } contentType = downloadRes.ContentType diff --git a/api/custom/federation.go b/api/custom/federation.go index 48a0fb92..ceee40a4 100644 --- a/api/custom/federation.go +++ b/api/custom/federation.go @@ -34,6 +34,9 @@ func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user _apim versionUrl := url + "/_matrix/federation/v1/version" versionResponse, err := matrix.FederatedGet(versionUrl, hostname, rctx) + if versionResponse != nil { + defer versionResponse.Body.Close() + } if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) diff --git a/api/routes.go b/api/routes.go index 423a6e32..376d2d69 100644 --- a/api/routes.go +++ b/api/routes.go @@ -18,6 +18,7 @@ import ( const PrefixMedia = "/_matrix/media" const PrefixClient = "/_matrix/client" +const PrefixFederation = "/_matrix/federation" func buildRoutes() http.Handler { counter := &_routers.RequestCounter{} @@ -36,13 +37,25 @@ func buildRoutes() http.Handler { register([]string{"GET"}, PrefixMedia, "download/:server/:mediaId/:filename", mxSpecV3Transition, router, downloadRoute) register([]string{"GET"}, PrefixMedia, "download/:server/:mediaId", mxSpecV3Transition, router, downloadRoute) register([]string{"GET"}, PrefixMedia, "thumbnail/:server/:mediaId", mxSpecV3Transition, router, makeRoute(_routers.OptionalAccessToken(r0.ThumbnailMedia), "thumbnail", counter)) - register([]string{"GET"}, PrefixMedia, "preview_url", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.PreviewUrl), "url_preview", counter)) + previewUrlRoute := makeRoute(_routers.RequireAccessToken(r0.PreviewUrl), "url_preview", counter) + register([]string{"GET"}, PrefixMedia, "preview_url", mxSpecV3TransitionCS, router, previewUrlRoute) register([]string{"GET"}, PrefixMedia, "identicon/*seed", mxR0, router, makeRoute(_routers.OptionalAccessToken(r0.Identicon), "identicon", counter)) - register([]string{"GET"}, PrefixMedia, "config", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.PublicConfig), "config", counter)) + configRoute := makeRoute(_routers.RequireAccessToken(r0.PublicConfig), "config", counter) + register([]string{"GET"}, PrefixMedia, "config", mxSpecV3TransitionCS, router, configRoute) register([]string{"POST"}, PrefixClient, "logout", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.Logout), "logout", counter)) register([]string{"POST"}, PrefixClient, "logout/all", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.LogoutAll), "logout_all", counter)) register([]string{"POST"}, PrefixMedia, "create", mxV1, router, makeRoute(_routers.RequireAccessToken(v1.CreateMedia), "create", counter)) + // MSC3916 - Authentication & endpoint API separation + register([]string{"GET"}, PrefixClient, "media/preview_url", msc3916, router, previewUrlRoute) + register([]string{"GET"}, PrefixClient, "media/config", msc3916, router, configRoute) + authedDownloadRoute := makeRoute(_routers.RequireAccessToken(unstable.ClientDownloadMedia), "download", counter) + register([]string{"GET"}, PrefixClient, "media/download/:server/:mediaId/:filename", msc3916, router, authedDownloadRoute) + register([]string{"GET"}, PrefixClient, "media/download/:server/:mediaId", msc3916, router, authedDownloadRoute) + register([]string{"GET"}, PrefixClient, "media/thumbnail/:server/:mediaId", msc3916, router, makeRoute(_routers.RequireAccessToken(r0.ThumbnailMedia), "thumbnail", counter)) + register([]string{"GET"}, PrefixFederation, "media/download/:server/:mediaId", msc3916, router, makeRoute(_routers.RequireServerAuth(unstable.FederationDownloadMedia), "download", counter)) + register([]string{"GET"}, PrefixFederation, "media/thumbnail/:server/:mediaId", msc3916, router, makeRoute(_routers.RequireServerAuth(unstable.FederationThumbnailMedia), "thumbnail", counter)) + // Custom features register([]string{"GET"}, PrefixMedia, "local_copy/:server/:mediaId", mxUnstable, router, makeRoute(_routers.RequireAccessToken(unstable.LocalCopy), "local_copy", counter)) register([]string{"GET"}, PrefixMedia, "info/:server/:mediaId", mxUnstable, router, makeRoute(_routers.RequireAccessToken(unstable.MediaInfo), "info", counter)) @@ -129,6 +142,7 @@ var ( //mxAllSpec matrixVersions = []string{"r0", "v1", "v3", "unstable", "unstable/io.t2bot.media" /* and MSC routes */} mxUnstable matrixVersions = []string{"unstable", "unstable/io.t2bot.media"} msc4034 matrixVersions = []string{"unstable/org.matrix.msc4034"} + msc3916 matrixVersions = []string{"unstable/org.matrix.msc3916"} mxSpecV3Transition matrixVersions = []string{"r0", "v1", "v3"} mxSpecV3TransitionCS matrixVersions = []string{"r0", "v3"} mxR0 matrixVersions = []string{"r0"} diff --git a/api/unstable/msc3916_download.go b/api/unstable/msc3916_download.go new file mode 100644 index 00000000..64d9d5b1 --- /dev/null +++ b/api/unstable/msc3916_download.go @@ -0,0 +1,37 @@ +package unstable + +import ( + "bytes" + "net/http" + + "github.com/turt2live/matrix-media-repo/api/_apimeta" + "github.com/turt2live/matrix-media-repo/api/_responses" + "github.com/turt2live/matrix-media-repo/api/r0" + "github.com/turt2live/matrix-media-repo/common/rcontext" + "github.com/turt2live/matrix-media-repo/util/readers" +) + +func ClientDownloadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { + r.URL.Query().Set("allow_remote", "true") + return r0.DownloadMedia(r, rctx, user) +} + +func FederationDownloadMedia(r *http.Request, rctx rcontext.RequestContext, server _apimeta.ServerInfo) interface{} { + r.URL.Query().Set("allow_remote", "false") + + res := r0.DownloadMedia(r, rctx, _apimeta.UserInfo{}) + if dl, ok := res.(*_responses.DownloadResponse); ok { + return &_responses.DownloadResponse{ + ContentType: "multipart/mixed", + Filename: "", + SizeBytes: 0, + Data: readers.NewMultipartReader( + &readers.MultipartPart{ContentType: "application/json", Reader: readers.MakeCloser(bytes.NewReader([]byte("{}")))}, + &readers.MultipartPart{ContentType: dl.ContentType, FileName: dl.Filename, Reader: dl.Data}, + ), + TargetDisposition: "attachment", + } + } else { + return res + } +} diff --git a/api/unstable/msc3916_thumbnail.go b/api/unstable/msc3916_thumbnail.go new file mode 100644 index 00000000..81b77d20 --- /dev/null +++ b/api/unstable/msc3916_thumbnail.go @@ -0,0 +1,32 @@ +package unstable + +import ( + "bytes" + "net/http" + + "github.com/turt2live/matrix-media-repo/api/_apimeta" + "github.com/turt2live/matrix-media-repo/api/_responses" + "github.com/turt2live/matrix-media-repo/api/r0" + "github.com/turt2live/matrix-media-repo/common/rcontext" + "github.com/turt2live/matrix-media-repo/util/readers" +) + +func FederationThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, server _apimeta.ServerInfo) interface{} { + r.URL.Query().Set("allow_remote", "false") + + res := r0.ThumbnailMedia(r, rctx, _apimeta.UserInfo{}) + if dl, ok := res.(*_responses.DownloadResponse); ok { + return &_responses.DownloadResponse{ + ContentType: "multipart/mixed", + Filename: "", + SizeBytes: 0, + Data: readers.NewMultipartReader( + &readers.MultipartPart{ContentType: "application/json", Reader: readers.MakeCloser(bytes.NewReader([]byte("{}")))}, + &readers.MultipartPart{ContentType: dl.ContentType, FileName: dl.Filename, Reader: dl.Data}, + ), + TargetDisposition: "attachment", + } + } else { + return res + } +} diff --git a/matrix/requests_signing.go b/matrix/requests_signing.go new file mode 100644 index 00000000..15e37997 --- /dev/null +++ b/matrix/requests_signing.go @@ -0,0 +1,150 @@ +package matrix + +import ( + "crypto/ed25519" + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + "github.com/patrickmn/go-cache" + "github.com/sirupsen/logrus" + "github.com/t2bot/go-typed-singleflight" + "github.com/turt2live/matrix-media-repo/common/rcontext" + "github.com/turt2live/matrix-media-repo/database" + "github.com/turt2live/matrix-media-repo/util" +) + +type signingKey struct { + Key string `json:"key"` +} + +type serverKeyResult struct { + ServerName string `json:"server_name"` + ValidUntilTs int64 `json:"valid_until_ts"` + VerifyKeys map[string]signingKey `json:"verify_keys"` // unpadded base64 + OldVerifyKeys map[string]signingKey `json:"old_verify_keys"` // unpadded base64 + Signatures map[string]map[string]string `json:"signatures"` // unpadded base64; > +} + +type ServerSigningKeys map[string]ed25519.PublicKey + +var signingKeySf = new(typedsf.Group[*ServerSigningKeys]) +var signingKeyCache = cache.New(cache.NoExpiration, 30*time.Second) +var signingKeyRWLock = new(sync.RWMutex) + +func querySigningKeyCache(serverName string) *ServerSigningKeys { + if val, ok := signingKeyCache.Get(serverName); ok { + return val.(*ServerSigningKeys) + } + return nil +} + +func QuerySigningKeys(serverName string) (*ServerSigningKeys, error) { + signingKeyRWLock.RLock() + keys := querySigningKeyCache(serverName) + signingKeyRWLock.RUnlock() + if keys != nil { + return keys, nil + } + + keys, err, _ := signingKeySf.Do(serverName, func() (*ServerSigningKeys, error) { + ctx := rcontext.Initial().LogWithFields(logrus.Fields{ + "keysForServer": serverName, + }) + + signingKeyRWLock.Lock() + defer signingKeyRWLock.Unlock() + + // check cache once more, just in case the locks overlapped + cachedKeys := querySigningKeyCache(serverName) + if keys != nil { + return cachedKeys, nil + } + + // now we can try to get the keys from the source + url, hostname, err := GetServerApiUrl(serverName) + if err != nil { + return nil, err + } + + keysUrl := url + "/_matrix/key/v2/server" + keysResponse, err := FederatedGet(keysUrl, hostname, ctx) + if keysResponse != nil { + defer keysResponse.Body.Close() + } + if err != nil { + return nil, err + } + + decoder := json.NewDecoder(keysResponse.Body) + raw := database.AnonymousJson{} + if err = decoder.Decode(&raw); err != nil { + return nil, err + } + keyInfo := new(serverKeyResult) + if err = raw.ApplyTo(keyInfo); err != nil { + return nil, err + } + + // Check validity before we go much further + if keyInfo.ServerName != serverName { + return nil, fmt.Errorf("got keys for '%s' but expected '%s'", keyInfo.ServerName, serverName) + } + if keyInfo.ValidUntilTs <= util.NowMillis() { + return nil, errors.New("returned server keys are expired") + } + cacheUntil := time.Until(time.UnixMilli(keyInfo.ValidUntilTs)) / 2 + if cacheUntil <= (6 * time.Second) { + return nil, errors.New("returned server keys would expire too quickly") + } + + // Convert to something useful + serverKeys := make(ServerSigningKeys) + for keyId, keyObj := range keyInfo.VerifyKeys { + b, err := util.DecodeUnpaddedBase64String(keyObj.Key) + if err != nil { + return nil, errors.Join(fmt.Errorf("bad base64 for key ID '%s' for '%s'", keyId, serverName), err) + } + + serverKeys[keyId] = b + } + + // Check signatures + if len(keyInfo.Signatures) == 0 || len(keyInfo.Signatures[serverName]) == 0 { + return nil, fmt.Errorf("missing signatures from '%s'", serverName) + } + delete(raw, "signatures") + canonical, err := util.EncodeCanonicalJson(raw) + if err != nil { + return nil, err + } + for domain, sig := range keyInfo.Signatures { + if domain != serverName { + return nil, fmt.Errorf("unexpected signature from '%s' (expected '%s')", domain, serverName) + } + + for keyId, b64 := range sig { + signatureBytes, err := util.DecodeUnpaddedBase64String(b64) + if err != nil { + return nil, errors.Join(fmt.Errorf("bad base64 signature for key ID '%s' for '%s'", keyId, serverName), err) + } + + key, ok := serverKeys[keyId] + if !ok { + return nil, fmt.Errorf("unknown key ID '%s' for signature from '%s'", keyId, serverName) + } + + if !ed25519.Verify(key, canonical, signatureBytes) { + return nil, fmt.Errorf("invalid signature '%s' from key ID '%s' for '%s'", b64, keyId, serverName) + } + } + } + + // Cache & return (unlock was deferred) + signingKeyCache.Set(serverName, &serverKeys, cacheUntil) + return &serverKeys, nil + }) + return keys, err +} diff --git a/matrix/xmatrix.go b/matrix/xmatrix.go new file mode 100644 index 00000000..72482f13 --- /dev/null +++ b/matrix/xmatrix.go @@ -0,0 +1,65 @@ +package matrix + +import ( + "crypto/ed25519" + "errors" + "fmt" + "github.com/turt2live/matrix-media-repo/util" + "net/http" +) + +var ErrNoXMatrixAuth = errors.New("no X-Matrix auth headers") + +func ValidateXMatrixAuth(request *http.Request, expectNoContent bool) (string, error) { + if !expectNoContent { + panic("development error: X-Matrix auth validation can only be done with an empty body for now") + } + + auths, err := util.GetXMatrixAuth(request) + if err != nil { + return "", err + } + + if len(auths) == 0 { + return "", ErrNoXMatrixAuth + } + + obj := map[string]interface{}{ + "method": request.Method, + "uri": request.RequestURI, + "origin": auths[0].Origin, + "destination": auths[0].Destination, + "content": "{}", + } + canonical, err := util.EncodeCanonicalJson(obj) + if err != nil { + return "", err + } + + keys, err := QuerySigningKeys(auths[0].Origin) + if err != nil { + return "", err + } + + for _, h := range auths { + if h.Origin != obj["origin"] { + return "", errors.New("auth is from multiple servers") + } + if h.Destination != obj["destination"] { + return "", errors.New("auth is for multiple servers") + } + if h.Destination != "" && !util.IsServerOurs(h.Destination) { + return "", errors.New("unknown destination") + } + + if key, ok := (*keys)[h.KeyId]; ok { + if !ed25519.Verify(key, canonical, h.Signature) { + return "", fmt.Errorf("failed signatures on '%s'", h.KeyId) + } + } else { + return "", fmt.Errorf("unknown key '%s'", h.KeyId) + } + } + + return auths[0].Origin, nil +} diff --git a/util/http.go b/util/http.go index 2188feb3..ac5144a4 100644 --- a/util/http.go +++ b/util/http.go @@ -1,11 +1,19 @@ package util import ( + "fmt" "net/http" "net/url" "strings" ) +type XMatrixAuth struct { + Origin string + Destination string + KeyId string + Signature []byte +} + func GetAccessTokenFromRequest(request *http.Request) string { token := request.Header.Get("Authorization") @@ -40,3 +48,72 @@ func GetLogSafeUrl(r *http.Request) string { copyUrl.RawQuery = GetLogSafeQueryString(r) return copyUrl.String() } + +func GetXMatrixAuth(request *http.Request) ([]XMatrixAuth, error) { + headers := request.Header.Values("Authorization") + auths := make([]XMatrixAuth, 0) + for _, h := range headers { + if !strings.HasPrefix(h, "X-Matrix ") { + continue + } + + paramCsv := h[len("X-Matrix "):] + params := make(map[string]string) + isKey := true + keyName := "" + keyValue := "" + escape := false + for _, c := range paramCsv { + if c == ',' && isKey { + params[strings.TrimSpace(strings.ToLower(keyName))] = keyValue + keyName = "" + keyValue = "" + continue + } + if c == '=' { + isKey = false + continue + } + + if isKey { + keyName = fmt.Sprintf("%s%s", keyName, string(c)) + } else { + if c == '\\' && !escape { + escape = true + continue + } + if c == '"' && !escape { + escape = false + if len(keyValue) > 0 { + isKey = true + } + continue + } + if escape { + escape = false + } + keyValue = fmt.Sprintf("%s%s", keyValue, string(c)) + } + } + if len(keyName) > 0 && isKey { + params[strings.TrimSpace(strings.ToLower(keyName))] = keyValue + } + + sig, err := DecodeUnpaddedBase64String(params["sig"]) + if err != nil { + return nil, err + } + auth := XMatrixAuth{ + Origin: params["origin"], + Destination: params["destination"], + KeyId: params["key"], + Signature: sig, + } + if auth.Origin == "" || auth.KeyId == "" || len(auth.Signature) == 0 { + continue + } + auths = append(auths, auth) + } + + return auths, nil +} diff --git a/util/readers/multipart_reader.go b/util/readers/multipart_reader.go new file mode 100644 index 00000000..ea3c901d --- /dev/null +++ b/util/readers/multipart_reader.go @@ -0,0 +1,57 @@ +package readers + +import ( + "io" + "mime/multipart" + "net/textproto" + "net/url" + + "github.com/alioygur/is" +) + +type MultipartPart struct { + ContentType string + FileName string + Reader io.ReadCloser +} + +func NewMultipartReader(parts ...*MultipartPart) io.ReadCloser { + r, w := io.Pipe() + go func() { + mpw := multipart.NewWriter(w) + + for _, part := range parts { + headers := textproto.MIMEHeader{} + if part.ContentType != "" { + headers.Set("Content-Type", part.ContentType) + } + if part.FileName != "" { + if is.ASCII(part.FileName) { + headers.Set("Content-Disposition", "attachment; filename="+url.QueryEscape(part.FileName)) + } else { + headers.Set("Content-Disposition", "attachment; filename*=utf-8''"+url.QueryEscape(part.FileName)) + } + } + + partW, err := mpw.CreatePart(headers) + if err != nil { + _ = w.CloseWithError(err) + return + } + if _, err = io.Copy(partW, part.Reader); err != nil { + _ = w.CloseWithError(err) + return + } + if err = part.Reader.Close(); err != nil { + _ = w.CloseWithError(err) + return + } + } + + if err := mpw.Close(); err != nil { + _ = w.CloseWithError(err) + } + _ = w.Close() + }() + return MakeCloser(r) +} From 1f3fa383da58282e90b3cc0a0b97c74ccbd32e8e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sat, 25 Nov 2023 14:42:32 -0700 Subject: [PATCH 2/9] Add changelog --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 299c3c58..dbd032a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,17 +7,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] -*Nothing yet.* +### Added + +* Add *unstable* support for [MSC3916: Authentication for media](https://github.com/matrix-org/matrix-spec-proposals/pull/3916). + * **Note**: MMR will *not* attempt to use authentication to download media over federation in this version. + * ***Subject to change during development.*** ## [1.3.4] - February 9, 2024 ### Added * Dendrite homeservers can now have their media imported safely, and `adminApiKind` may be set to `dendrite`. -* Exporting MMR's data to Synapse is now possible with `import_to_synapse`. To use it, first run `gdpr_export` or similar. -* Errors encountered during a background task, such as an API-induced export, are exposed as `error_message` in the admin API. -* MMR will follow redirects on federated downloads up to 5 hops. -* S3-backed datastores can have download requests redirected to a public-facing CDN rather than being proxied through MMR. See `publicBaseUrl` under the S3 datastore config. ### Changed From 90599ff7ff85e5758dfd2bbe4dffe43c81d9d5ab Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sat, 25 Nov 2023 15:54:02 -0700 Subject: [PATCH 3/9] Add tests for preview_url and config authenticated endpoints --- ...sc3916_misc_client_endpoints_suite_test.go | 91 +++++++++++++++++++ test/templates/mmr.config.yaml | 17 ++++ test/test_internals/inline_dep_host_file.go | 82 +++++++++++++++++ 3 files changed, 190 insertions(+) create mode 100644 test/msc3916_misc_client_endpoints_suite_test.go create mode 100644 test/test_internals/inline_dep_host_file.go diff --git a/test/msc3916_misc_client_endpoints_suite_test.go b/test/msc3916_misc_client_endpoints_suite_test.go new file mode 100644 index 00000000..3d392b6b --- /dev/null +++ b/test/msc3916_misc_client_endpoints_suite_test.go @@ -0,0 +1,91 @@ +package test + +import ( + "log" + "net/http" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "github.com/turt2live/matrix-media-repo/test/test_internals" +) + +type MSC3916MiscClientEndpointsSuite struct { + suite.Suite + deps *test_internals.ContainerDeps + htmlPage *test_internals.HostedFile +} + +func (s *MSC3916MiscClientEndpointsSuite) SetupSuite() { + deps, err := test_internals.MakeTestDeps() + if err != nil { + log.Fatal(err) + } + s.deps = deps + + file, err := test_internals.ServeFile("index.html", deps, "

This is a test file

") + if err != nil { + log.Fatal(err) + } + s.htmlPage = file +} + +func (s *MSC3916MiscClientEndpointsSuite) TearDownSuite() { + if s.htmlPage != nil { + s.htmlPage.Teardown() + } + if s.deps != nil { + if s.T().Failed() { + s.deps.Debug() + } + s.deps.Teardown() + } +} + +func (s *MSC3916MiscClientEndpointsSuite) TestPreviewUrlRequiresAuth() { + t := s.T() + + client1 := s.deps.Homeservers[0].UnprivilegedUsers[0].WithCsUrl(s.deps.Machines[0].HttpUrl) + client2 := &test_internals.MatrixClient{ + ClientServerUrl: s.deps.Machines[0].HttpUrl, + ServerName: s.deps.Homeservers[0].ServerName, + AccessToken: "", // no auth on this client + UserId: "", // no auth on this client + } + + qs := url.Values{ + "url": []string{s.htmlPage.PublicUrl}, + } + raw, err := client2.DoRaw("GET", "/_matrix/client/unstable/org.matrix.msc3916/media/preview_url", qs, "", nil) + assert.NoError(t, err) + assert.Equal(t, http.StatusUnauthorized, raw.StatusCode) + + raw, err = client1.DoRaw("GET", "/_matrix/client/unstable/org.matrix.msc3916/media/preview_url", qs, "", nil) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, raw.StatusCode) +} + +func (s *MSC3916MiscClientEndpointsSuite) TestConfigRequiresAuth() { + t := s.T() + + client1 := s.deps.Homeservers[0].UnprivilegedUsers[0].WithCsUrl(s.deps.Machines[0].HttpUrl) + client2 := &test_internals.MatrixClient{ + ClientServerUrl: s.deps.Machines[0].HttpUrl, + ServerName: s.deps.Homeservers[0].ServerName, + AccessToken: "", // no auth on this client + UserId: "", // no auth on this client + } + + raw, err := client2.DoRaw("GET", "/_matrix/client/unstable/org.matrix.msc3916/media/config", nil, "", nil) + assert.NoError(t, err) + assert.Equal(t, http.StatusUnauthorized, raw.StatusCode) + + raw, err = client1.DoRaw("GET", "/_matrix/client/unstable/org.matrix.msc3916/media/config", nil, "", nil) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, raw.StatusCode) +} + +func TestMSC3916MiscClientEndpointsSuite(t *testing.T) { + suite.Run(t, new(MSC3916MiscClientEndpointsSuite)) +} diff --git a/test/templates/mmr.config.yaml b/test/templates/mmr.config.yaml index 40fbf6d6..4d82dd3a 100644 --- a/test/templates/mmr.config.yaml +++ b/test/templates/mmr.config.yaml @@ -40,3 +40,20 @@ datastores: ssl: false rateLimit: enabled: false # we've got tests which intentionally spam +urlPreviews: + enabled: true + maxPageSizeBytes: 10485760 + previewUnsafeCertificates: false + numWords: 50 + maxLength: 200 + numTitleWords: 30 + maxTitleLength: 150 + filePreviewTypes: + - "image/*" + numWorkers: 10 + disallowedNetworks: [] + allowedNetworks: ["0.0.0.0/0"] + expireAfterDays: 0 + defaultLanguage: "en-US,en" + userAgent: "matrix-media-repo" + oEmbed: true diff --git a/test/test_internals/inline_dep_host_file.go b/test/test_internals/inline_dep_host_file.go new file mode 100644 index 00000000..dd486052 --- /dev/null +++ b/test/test_internals/inline_dep_host_file.go @@ -0,0 +1,82 @@ +package test_internals + +import ( + "fmt" + "log" + "os" + "path" + + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +type HostedFile struct { + upstream *ContainerDeps + nginx testcontainers.Container + tempDirectoryPath string + + PublicUrl string +} + +func ServeFile(fileName string, deps *ContainerDeps, contents string) (*HostedFile, error) { + tmp, err := os.MkdirTemp(os.TempDir(), "mmr-nginx") + if err != nil { + return nil, err + } + + f, err := os.Create(path.Join(tmp, fileName)) + if err != nil { + return nil, err + } + defer func(f *os.File) { + _ = f.Close() + }(f) + + _, err = f.Write([]byte(contents)) + if err != nil { + return nil, err + } + + err = f.Close() + if err != nil { + return nil, err + } + + nginx, err := testcontainers.GenericContainer(deps.ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: "docker.io/library/nginx:latest", + ExposedPorts: []string{"80/tcp"}, + Mounts: []testcontainers.ContainerMount{ + testcontainers.BindMount(tmp, "/usr/share/nginx/html"), + }, + Networks: []string{deps.depNet.NetId}, + WaitingFor: wait.ForListeningPort("80/tcp"), + }, + Started: true, + }) + if err != nil { + return nil, err + } + + nginxIp, err := nginx.ContainerIP(deps.ctx) + if err != nil { + return nil, err + } + + //goland:noinspection HttpUrlsUsage + return &HostedFile{ + upstream: deps, + nginx: nginx, + tempDirectoryPath: tmp, + PublicUrl: fmt.Sprintf("http://%s:%d/%s", nginxIp, 80, fileName), + }, nil +} + +func (f *HostedFile) Teardown() { + if err := f.nginx.Terminate(f.upstream.ctx); err != nil { + log.Fatalf("Error shutting down nginx container: %s", err.Error()) + } + if err := os.RemoveAll(f.tempDirectoryPath); err != nil { + log.Fatalf("Error cleaning up temporarily hosted file: %s", err.Error()) + } +} From ec222c219e08f40af5b1e27bd349fa4226054621 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sat, 25 Nov 2023 16:27:28 -0700 Subject: [PATCH 4/9] Add placeholder tests for downloads and thumbnails --- test/msc3916_downloads_suite_test.go | 87 +++++++++++++++++++++++++++ test/msc3916_thumbnails_suite_test.go | 87 +++++++++++++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 test/msc3916_downloads_suite_test.go create mode 100644 test/msc3916_thumbnails_suite_test.go diff --git a/test/msc3916_downloads_suite_test.go b/test/msc3916_downloads_suite_test.go new file mode 100644 index 00000000..fc0edf83 --- /dev/null +++ b/test/msc3916_downloads_suite_test.go @@ -0,0 +1,87 @@ +package test + +import ( + "fmt" + "log" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "github.com/turt2live/matrix-media-repo/test/test_internals" + "github.com/turt2live/matrix-media-repo/util" +) + +type MSC3916DownloadsSuite struct { + suite.Suite + deps *test_internals.ContainerDeps +} + +func (s *MSC3916DownloadsSuite) SetupSuite() { + deps, err := test_internals.MakeTestDeps() + if err != nil { + log.Fatal(err) + } + s.deps = deps +} + +func (s *MSC3916DownloadsSuite) TearDownSuite() { + if s.deps != nil { + if s.T().Failed() { + s.deps.Debug() + } + s.deps.Teardown() + } +} + +func (s *MSC3916DownloadsSuite) TestClientDownloads() { + t := s.T() + + client1 := s.deps.Homeservers[0].UnprivilegedUsers[0].WithCsUrl(s.deps.Machines[0].HttpUrl) + client2 := &test_internals.MatrixClient{ + ClientServerUrl: s.deps.Machines[0].HttpUrl, + ServerName: s.deps.Homeservers[0].ServerName, + AccessToken: "", // this client isn't authed + UserId: "", // this client isn't authed + } + + contentType, img, err := test_internals.MakeTestImage(512, 512) + assert.NoError(t, err) + fname := "image" + util.ExtensionForContentType(contentType) + + //res := new(test_internals.MatrixUploadResponse) + //err = client1.DoReturnJson("POST", "/_matrix/client/unstable/org.matrix.msc3916/media/upload", url.Values{"filename": []string{fname}}, contentType, img, res) + res, err := client1.Upload(fname, contentType, img) + assert.NoError(t, err) + assert.NotEmpty(t, res.MxcUri) + + origin, mediaId, err := util.SplitMxc(res.MxcUri) + assert.NoError(t, err) + assert.Equal(t, client1.ServerName, origin) + assert.NotEmpty(t, mediaId) + + raw, err := client2.DoRaw("GET", fmt.Sprintf("/_matrix/client/unstable/org.matrix.msc3916/media/download/%s/%s", origin, mediaId), nil, "", nil) + assert.NoError(t, err) + assert.Equal(t, http.StatusUnauthorized, raw.StatusCode) + raw, err = client2.DoRaw("GET", fmt.Sprintf("/_matrix/client/unstable/org.matrix.msc3916/media/download/%s/%s/whatever.png", origin, mediaId), nil, "", nil) + assert.NoError(t, err) + assert.Equal(t, http.StatusUnauthorized, raw.StatusCode) + + raw, err = client1.DoRaw("GET", fmt.Sprintf("/_matrix/client/unstable/org.matrix.msc3916/media/download/%s/%s", origin, mediaId), nil, "", nil) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, raw.StatusCode) + test_internals.AssertIsTestImage(t, raw.Body) + raw, err = client1.DoRaw("GET", fmt.Sprintf("/_matrix/client/unstable/org.matrix.msc3916/media/download/%s/%s/whatever.png", origin, mediaId), nil, "", nil) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, raw.StatusCode) + test_internals.AssertIsTestImage(t, raw.Body) +} + +func (s *MSC3916DownloadsSuite) TestFederationDownloads() { + t := s.T() + t.Error("Not yet implemented") +} + +func TestMSC3916DownloadsSuite(t *testing.T) { + suite.Run(t, new(MSC3916DownloadsSuite)) +} diff --git a/test/msc3916_thumbnails_suite_test.go b/test/msc3916_thumbnails_suite_test.go new file mode 100644 index 00000000..43cf5c77 --- /dev/null +++ b/test/msc3916_thumbnails_suite_test.go @@ -0,0 +1,87 @@ +package test + +import ( + "fmt" + "log" + "net/http" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "github.com/turt2live/matrix-media-repo/test/test_internals" + "github.com/turt2live/matrix-media-repo/util" +) + +type MSC3916ThumbnailsSuite struct { + suite.Suite + deps *test_internals.ContainerDeps +} + +func (s *MSC3916ThumbnailsSuite) SetupSuite() { + deps, err := test_internals.MakeTestDeps() + if err != nil { + log.Fatal(err) + } + s.deps = deps +} + +func (s *MSC3916ThumbnailsSuite) TearDownSuite() { + if s.deps != nil { + if s.T().Failed() { + s.deps.Debug() + } + s.deps.Teardown() + } +} + +func (s *MSC3916ThumbnailsSuite) TestClientThumbnails() { + t := s.T() + + client1 := s.deps.Homeservers[0].UnprivilegedUsers[0].WithCsUrl(s.deps.Machines[0].HttpUrl) + client2 := &test_internals.MatrixClient{ + ClientServerUrl: s.deps.Machines[0].HttpUrl, + ServerName: s.deps.Homeservers[0].ServerName, + AccessToken: "", // this client isn't authed + UserId: "", // this client isn't authed + } + + contentType, img, err := test_internals.MakeTestImage(512, 512) + assert.NoError(t, err) + fname := "image" + util.ExtensionForContentType(contentType) + + //res := new(test_internals.MatrixUploadResponse) + //err = client1.DoReturnJson("POST", "/_matrix/client/unstable/org.matrix.msc3916/media/upload", url.Values{"filename": []string{fname}}, contentType, img, res) + res, err := client1.Upload(fname, contentType, img) + assert.NoError(t, err) + assert.NotEmpty(t, res.MxcUri) + + origin, mediaId, err := util.SplitMxc(res.MxcUri) + assert.NoError(t, err) + assert.Equal(t, client1.ServerName, origin) + assert.NotEmpty(t, mediaId) + + qs := url.Values{ + "width": []string{"96"}, + "height": []string{"96"}, + "method": []string{"scale"}, + } + + raw, err := client2.DoRaw("GET", fmt.Sprintf("/_matrix/client/unstable/org.matrix.msc3916/media/thumbnail/%s/%s", origin, mediaId), qs, "", nil) + assert.NoError(t, err) + assert.Equal(t, http.StatusUnauthorized, raw.StatusCode) + + raw, err = client1.DoRaw("GET", fmt.Sprintf("/_matrix/client/unstable/org.matrix.msc3916/media/thumbnail/%s/%s", origin, mediaId), qs, "", nil) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, raw.StatusCode) + //test_internals.AssertIsTestImage(t, raw.Body) // we can't verify that the resulting image is correct +} + +func (s *MSC3916ThumbnailsSuite) TestFederationThumbnails() { + t := s.T() + t.Error("Not yet implemented") +} + +func TestMSC3916ThumbnailsSuite(t *testing.T) { + suite.Run(t, new(MSC3916ThumbnailsSuite)) +} From 96fcd374f80867a943115d63703c1c79e88a2ced Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sat, 25 Nov 2023 20:08:32 -0700 Subject: [PATCH 5/9] Test X-Matrix auth header stuff --- common/config/access.go | 9 +++++ matrix/requests_signing.go | 12 +++---- matrix/xmatrix.go | 72 ++++++++++++++++++++++++++----------- test/xmatrix_header_test.go | 37 +++++++++++++++++++ util/canonical_json.go | 2 +- util/http.go | 3 +- 6 files changed, 106 insertions(+), 29 deletions(-) create mode 100644 test/xmatrix_header_test.go diff --git a/common/config/access.go b/common/config/access.go index 6b1bab44..b95a613c 100644 --- a/common/config/access.go +++ b/common/config/access.go @@ -222,6 +222,15 @@ func GetDomain(domain string) *DomainRepoConfig { return domains[domain] } +func AddDomainForTesting(domain string, config *DomainRepoConfig) { + Get() // Ensure the "main" config was loaded first + if config == nil { + c := NewDefaultDomainConfig() + config = &c + } + domains[domain] = config +} + func DomainConfigFrom(c MainRepoConfig) DomainRepoConfig { // HACK: We should be better at this kind of inheritance dc := NewDefaultDomainConfig() diff --git a/matrix/requests_signing.go b/matrix/requests_signing.go index 15e37997..8694139f 100644 --- a/matrix/requests_signing.go +++ b/matrix/requests_signing.go @@ -30,18 +30,18 @@ type serverKeyResult struct { type ServerSigningKeys map[string]ed25519.PublicKey -var signingKeySf = new(typedsf.Group[*ServerSigningKeys]) +var signingKeySf = new(typedsf.Group[ServerSigningKeys]) var signingKeyCache = cache.New(cache.NoExpiration, 30*time.Second) var signingKeyRWLock = new(sync.RWMutex) -func querySigningKeyCache(serverName string) *ServerSigningKeys { +func querySigningKeyCache(serverName string) ServerSigningKeys { if val, ok := signingKeyCache.Get(serverName); ok { - return val.(*ServerSigningKeys) + return val.(ServerSigningKeys) } return nil } -func QuerySigningKeys(serverName string) (*ServerSigningKeys, error) { +func QuerySigningKeys(serverName string) (ServerSigningKeys, error) { signingKeyRWLock.RLock() keys := querySigningKeyCache(serverName) signingKeyRWLock.RUnlock() @@ -49,7 +49,7 @@ func QuerySigningKeys(serverName string) (*ServerSigningKeys, error) { return keys, nil } - keys, err, _ := signingKeySf.Do(serverName, func() (*ServerSigningKeys, error) { + keys, err, _ := signingKeySf.Do(serverName, func() (ServerSigningKeys, error) { ctx := rcontext.Initial().LogWithFields(logrus.Fields{ "keysForServer": serverName, }) @@ -144,7 +144,7 @@ func QuerySigningKeys(serverName string) (*ServerSigningKeys, error) { // Cache & return (unlock was deferred) signingKeyCache.Set(serverName, &serverKeys, cacheUntil) - return &serverKeys, nil + return serverKeys, nil }) return keys, err } diff --git a/matrix/xmatrix.go b/matrix/xmatrix.go index 72482f13..7b9889d3 100644 --- a/matrix/xmatrix.go +++ b/matrix/xmatrix.go @@ -4,8 +4,10 @@ import ( "crypto/ed25519" "errors" "fmt" - "github.com/turt2live/matrix-media-repo/util" "net/http" + + "github.com/turt2live/matrix-media-repo/database" + "github.com/turt2live/matrix-media-repo/util" ) var ErrNoXMatrixAuth = errors.New("no X-Matrix auth headers") @@ -15,51 +17,81 @@ func ValidateXMatrixAuth(request *http.Request, expectNoContent bool) (string, e panic("development error: X-Matrix auth validation can only be done with an empty body for now") } - auths, err := util.GetXMatrixAuth(request) + auths, err := util.GetXMatrixAuth(request.Header.Values("Authorization")) if err != nil { return "", err } - if len(auths) == 0 { return "", ErrNoXMatrixAuth } - obj := map[string]interface{}{ - "method": request.Method, - "uri": request.RequestURI, - "origin": auths[0].Origin, - "destination": auths[0].Destination, - "content": "{}", - } - canonical, err := util.EncodeCanonicalJson(obj) + keys, err := QuerySigningKeys(auths[0].Origin) if err != nil { return "", err } - keys, err := QuerySigningKeys(auths[0].Origin) + err = ValidateXMatrixAuthHeader(request.Method, request.RequestURI, &database.AnonymousJson{}, auths, keys) if err != nil { return "", err } + return auths[0].Origin, nil +} + +func ValidateXMatrixAuthHeader(requestMethod string, requestUri string, content any, headers []util.XMatrixAuth, originKeys ServerSigningKeys) error { + if len(headers) == 0 { + return ErrNoXMatrixAuth + } + + obj := map[string]interface{}{ + "method": requestMethod, + "uri": requestUri, + "origin": headers[0].Origin, + "destination": headers[0].Destination, + "content": content, + } + canonical, err := util.EncodeCanonicalJson(obj) + if err != nil { + return err + } - for _, h := range auths { + for _, h := range headers { if h.Origin != obj["origin"] { - return "", errors.New("auth is from multiple servers") + return errors.New("auth is from multiple servers") } if h.Destination != obj["destination"] { - return "", errors.New("auth is for multiple servers") + return errors.New("auth is for multiple servers") } if h.Destination != "" && !util.IsServerOurs(h.Destination) { - return "", errors.New("unknown destination") + return errors.New("unknown destination") } - if key, ok := (*keys)[h.KeyId]; ok { + if key, ok := (originKeys)[h.KeyId]; ok { if !ed25519.Verify(key, canonical, h.Signature) { - return "", fmt.Errorf("failed signatures on '%s'", h.KeyId) + return fmt.Errorf("failed signatures on '%s'", h.KeyId) } } else { - return "", fmt.Errorf("unknown key '%s'", h.KeyId) + return fmt.Errorf("unknown key '%s'", h.KeyId) } } - return auths[0].Origin, nil + return nil +} + +func CreateXMatrixHeader(origin string, destination string, requestMethod string, requestUri string, content any, key *ed25519.PrivateKey, keyVersion string) (string, error) { + obj := map[string]interface{}{ + "method": requestMethod, + "uri": requestUri, + "origin": origin, + "destination": destination, + "content": content, + } + canonical, err := util.EncodeCanonicalJson(obj) + if err != nil { + return "", err + } + + b := ed25519.Sign(*key, canonical) + sig := util.EncodeUnpaddedBase64ToString(b) + + return fmt.Sprintf("X-Matrix origin=\"%s\",destination=\"%s\",key=\"ed25519:%s\",sig=\"%s\"", origin, destination, keyVersion, sig), nil } diff --git a/test/xmatrix_header_test.go b/test/xmatrix_header_test.go new file mode 100644 index 00000000..17019108 --- /dev/null +++ b/test/xmatrix_header_test.go @@ -0,0 +1,37 @@ +package test + +import ( + "crypto/ed25519" + "testing" + + "github.com/turt2live/matrix-media-repo/common/config" + "github.com/turt2live/matrix-media-repo/database" + "github.com/turt2live/matrix-media-repo/matrix" + "github.com/turt2live/matrix-media-repo/util" +) + +func TestXMatrixAuthHeader(t *testing.T) { + config.AddDomainForTesting("localhost", nil) + + pub, priv, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + + header, err := matrix.CreateXMatrixHeader("localhost:8008", "localhost", "GET", "/_matrix/media/v3/download/example.org/abc", &database.AnonymousJson{}, &priv, "0") + if err != nil { + t.Fatal(err) + } + + auths, err := util.GetXMatrixAuth([]string{header}) + if err != nil { + t.Fatal(err) + } + + keys := make(matrix.ServerSigningKeys) + keys["ed25519:0"] = pub + err = matrix.ValidateXMatrixAuthHeader("GET", "/_matrix/media/v3/download/example.org/abc", &database.AnonymousJson{}, auths, keys) + if err != nil { + t.Error(err) + } +} diff --git a/util/canonical_json.go b/util/canonical_json.go index c514f311..939da9f6 100644 --- a/util/canonical_json.go +++ b/util/canonical_json.go @@ -5,7 +5,7 @@ import ( "encoding/json" ) -func EncodeCanonicalJson(obj map[string]interface{}) ([]byte, error) { +func EncodeCanonicalJson(obj any) ([]byte, error) { b, err := json.Marshal(obj) if err != nil { return nil, err diff --git a/util/http.go b/util/http.go index ac5144a4..0892f661 100644 --- a/util/http.go +++ b/util/http.go @@ -49,8 +49,7 @@ func GetLogSafeUrl(r *http.Request) string { return copyUrl.String() } -func GetXMatrixAuth(request *http.Request) ([]XMatrixAuth, error) { - headers := request.Header.Values("Authorization") +func GetXMatrixAuth(headers []string) ([]XMatrixAuth, error) { auths := make([]XMatrixAuth, 0) for _, h := range headers { if !strings.HasPrefix(h, "X-Matrix ") { From f972216d682b1045fe275cc0c3321bd3d642cf89 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sat, 25 Nov 2023 20:58:23 -0700 Subject: [PATCH 6/9] Validate signing keys more correctly --- matrix/requests_signing.go | 93 ++++++++++++++++++++++---------------- test/signing_keys_test.go | 73 ++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 39 deletions(-) create mode 100644 test/signing_keys_test.go diff --git a/matrix/requests_signing.go b/matrix/requests_signing.go index 8694139f..57bb1677 100644 --- a/matrix/requests_signing.go +++ b/matrix/requests_signing.go @@ -20,7 +20,7 @@ type signingKey struct { Key string `json:"key"` } -type serverKeyResult struct { +type ServerKeyResult struct { ServerName string `json:"server_name"` ValidUntilTs int64 `json:"valid_until_ts"` VerifyKeys map[string]signingKey `json:"verify_keys"` // unpadded base64 @@ -83,7 +83,7 @@ func QuerySigningKeys(serverName string) (ServerSigningKeys, error) { if err = decoder.Decode(&raw); err != nil { return nil, err } - keyInfo := new(serverKeyResult) + keyInfo := new(ServerKeyResult) if err = raw.ApplyTo(keyInfo); err != nil { return nil, err } @@ -100,51 +100,66 @@ func QuerySigningKeys(serverName string) (ServerSigningKeys, error) { return nil, errors.New("returned server keys would expire too quickly") } - // Convert to something useful - serverKeys := make(ServerSigningKeys) - for keyId, keyObj := range keyInfo.VerifyKeys { - b, err := util.DecodeUnpaddedBase64String(keyObj.Key) - if err != nil { - return nil, errors.Join(fmt.Errorf("bad base64 for key ID '%s' for '%s'", keyId, serverName), err) - } - - serverKeys[keyId] = b + // Convert keys to something useful, and check signatures + serverKeys, err := CheckSigningKeySignatures(serverName, keyInfo, raw) + if err != nil { + return nil, err } - // Check signatures - if len(keyInfo.Signatures) == 0 || len(keyInfo.Signatures[serverName]) == 0 { - return nil, fmt.Errorf("missing signatures from '%s'", serverName) - } - delete(raw, "signatures") - canonical, err := util.EncodeCanonicalJson(raw) + // Cache & return (unlock was deferred) + signingKeyCache.Set(serverName, &serverKeys, cacheUntil) + return serverKeys, nil + }) + return keys, err +} + +func CheckSigningKeySignatures(serverName string, keyInfo *ServerKeyResult, raw database.AnonymousJson) (ServerSigningKeys, error) { + serverKeys := make(ServerSigningKeys) + for keyId, keyObj := range keyInfo.VerifyKeys { + b, err := util.DecodeUnpaddedBase64String(keyObj.Key) if err != nil { - return nil, err + return nil, errors.Join(fmt.Errorf("bad base64 for key ID '%s' for '%s'", keyId, serverName), err) } - for domain, sig := range keyInfo.Signatures { - if domain != serverName { - return nil, fmt.Errorf("unexpected signature from '%s' (expected '%s')", domain, serverName) - } - for keyId, b64 := range sig { - signatureBytes, err := util.DecodeUnpaddedBase64String(b64) - if err != nil { - return nil, errors.Join(fmt.Errorf("bad base64 signature for key ID '%s' for '%s'", keyId, serverName), err) - } + serverKeys[keyId] = b + } - key, ok := serverKeys[keyId] - if !ok { - return nil, fmt.Errorf("unknown key ID '%s' for signature from '%s'", keyId, serverName) - } + if len(keyInfo.Signatures) == 0 || len(keyInfo.Signatures[serverName]) == 0 { + return nil, fmt.Errorf("missing signatures from '%s'", serverName) + } + delete(raw, "signatures") + canonical, err := util.EncodeCanonicalJson(raw) + if err != nil { + return nil, err + } + for domain, sig := range keyInfo.Signatures { + if domain != serverName { + return nil, fmt.Errorf("unexpected signature from '%s' (expected '%s')", domain, serverName) + } + + for keyId, b64 := range sig { + signatureBytes, err := util.DecodeUnpaddedBase64String(b64) + if err != nil { + return nil, errors.Join(fmt.Errorf("bad base64 signature for key ID '%s' for '%s'", keyId, serverName), err) + } + + key, ok := serverKeys[keyId] + if !ok { + return nil, fmt.Errorf("unknown key ID '%s' for signature from '%s'", keyId, serverName) + } - if !ed25519.Verify(key, canonical, signatureBytes) { - return nil, fmt.Errorf("invalid signature '%s' from key ID '%s' for '%s'", b64, keyId, serverName) - } + if !ed25519.Verify(key, canonical, signatureBytes) { + return nil, fmt.Errorf("invalid signature '%s' from key ID '%s' for '%s'", b64, keyId, serverName) } } + } + + // Ensure *all* keys have signed the response + for keyId, _ := range serverKeys { + if _, ok := keyInfo.Signatures[serverName][keyId]; !ok { + return nil, fmt.Errorf("missing signature from key '%s'", keyId) + } + } - // Cache & return (unlock was deferred) - signingKeyCache.Set(serverName, &serverKeys, cacheUntil) - return serverKeys, nil - }) - return keys, err + return serverKeys, nil } diff --git a/test/signing_keys_test.go b/test/signing_keys_test.go new file mode 100644 index 00000000..13edf013 --- /dev/null +++ b/test/signing_keys_test.go @@ -0,0 +1,73 @@ +package test + +import ( + "crypto/ed25519" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/turt2live/matrix-media-repo/database" + "github.com/turt2live/matrix-media-repo/matrix" + "github.com/turt2live/matrix-media-repo/util" +) + +func TestFailInjectedKeys(t *testing.T) { + raw := database.AnonymousJson{ + "old_verify_keys": database.AnonymousJson{}, + "server_name": "x.resolvematrix.dev", + "signatures": database.AnonymousJson{ + "x.resolvematrix.dev": database.AnonymousJson{ + "ed25519:injected": "FB93YAF+fOPyWcsx285Q/xFzRiG5sr7/u1iX9XWaIcOwDyDDwx7daS1eYxuM9PfosWE5vqUyTsCxmB40JTzdCw", + }, + }, + "valid_until_ts": 1701055573679, + "verify_keys": database.AnonymousJson{ + "ed25519:AY4k3ADlto8": database.AnonymousJson{"key": "VF7dl9W/tFWAjZSXm42Ef22k3v4WKBYLXZF9I7ErU00"}, + "ed25519:injected": database.AnonymousJson{"key": "w48CLiV1IkWoEbqJLFmniGUYtxwT+c2zm87X8oEpRO8"}, + }, + } + keyInfo := new(matrix.ServerKeyResult) + err := raw.ApplyTo(keyInfo) + if err != nil { + t.Fatal(err) + } + + _, err = matrix.CheckSigningKeySignatures("x.resolvematrix.dev", keyInfo, raw) + assert.Error(t, err) + assert.Equal(t, "missing signature from key 'ed25519:AY4k3ADlto8'", err.Error()) +} + +func TestRegularKeys(t *testing.T) { + raw := database.AnonymousJson{ + "old_verify_keys": database.AnonymousJson{}, + "server_name": "x.resolvematrix.dev", + "signatures": database.AnonymousJson{ + "x.resolvematrix.dev": database.AnonymousJson{ + "ed25519:AY4k3ADlto8": "3WlsmHFTVjywCoDYyrtx3ies+VufTuBuw1Prlgmoqh+a4XrJT+isEwhTX+I5FBvtJTKTt6vLH3gaP7BA6712CA", + }, + }, + "valid_until_ts": 1701057124839, + "verify_keys": database.AnonymousJson{ + "ed25519:AY4k3ADlto8": database.AnonymousJson{"key": "VF7dl9W/tFWAjZSXm42Ef22k3v4WKBYLXZF9I7ErU00"}, + }, + } + keyInfo := new(matrix.ServerKeyResult) + err := raw.ApplyTo(keyInfo) + if err != nil { + t.Fatal(err) + } + + keys, err := matrix.CheckSigningKeySignatures("x.resolvematrix.dev", keyInfo, raw) + assert.NoError(t, err) + for keyId, keyVal := range keys { + if b64, ok := keyInfo.VerifyKeys[keyId]; !ok { + t.Errorf("got key for '%s' but wasn't expecting it", keyId) + } else { + keySelf, err := util.DecodeUnpaddedBase64String(b64.Key) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, ed25519.PublicKey(keySelf), keyVal) + } + } +} From 4a665e9985c0792a51e9ceda2e85bd77cf08c07e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 26 Nov 2023 22:53:33 -0700 Subject: [PATCH 7/9] Add early documentation for what this setup will look like --- docs/msc3916.md | 51 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 docs/msc3916.md diff --git a/docs/msc3916.md b/docs/msc3916.md new file mode 100644 index 00000000..c1f9755e --- /dev/null +++ b/docs/msc3916.md @@ -0,0 +1,51 @@ +# MSC3916 Support in MMR + +[MSC3916](https://github.com/matrix-org/matrix-spec-proposals/pull/3916) is a proposal to add new endpoints to the Matrix +Client-Server API for authenticated media downloads. The MSC itself does not implement restrictions on who can download +a given piece of media, but does require that both users and servers identify themselves when downloading media. + +MMR supports the MSC by default, allowing the following additional endpoints to be routed to it: + +* `GET /_matrix/client/unstable/org.matrix.msc3916/media/download/:origin/:mediaId` +* `GET /_matrix/client/unstable/org.matrix.msc3916/media/download/:origin/:mediaId/:fileName` +* `GET /_matrix/client/unstable/org.matrix.msc3916/media/thumbnail/:origin/:mediaId` +* `GET /_matrix/client/unstable/org.matrix.msc3916/media/preview_url` +* `GET /_matrix/client/unstable/org.matrix.msc3916/media/config` +* `GET /_matrix/federation/unstable/org.matrix.msc3916/media/download/:origin/:mediaId` +* `GET /_matrix/federation/unstable/org.matrix.msc3916/media/thumbnail/:origin/:mediaId` + +Note that the new endpoints are *not* in the traditional `/_matrix/media` namespace. Note also that the upload endpoint +does not appear in this list - the endpoint is modified by [MSC3911](https://github.com/matrix-org/matrix-spec-proposals/pull/3911) +instead. + +MMR will allow the new `/_matrix/client` endpoints to be used if it is configured with a signing key for the server the +request is being made to. If MMR is not configured with a signing key, clients will receive the normal "not implemented" +error responses. When a signing key is configured for the server, MMR will *always* try to use the new federation endpoints +to download media, regardless as to how it was approached. This will cause a fallback to the legacy `/_matrix/media` +endpoints if the remote server does not support the new endpoints. + +**It is strongly recommended that a signing key be configured.** Future versions of MMR may refuse to start up without a +signing key configured. + +To set up a signing key for MMR: + +1. Back up your existing homeserver signing key, and store it in a safe place. The signing key effectively grants full + access to your server and events, and should not be disclosed to anyone. +2. Download the `generate_signing_key` and `combine_signing_keys` tools for your version of MMR from the + [GitHub releases page](https://github.com/turt2live/matrix-media-repo/releases). +3. Run `./generate_signing_key -output mmr.signing.key` to create a signing key usable with MMR. +4. If you're using Synapse as your homeserver software, run `./combine_signing_keys -format synapse -output ./merged.signing.key ./existing.signing.key ./mmr.signing.key` + to combine the signing keys, being sure to list Synapse's existing signing key *first* in the arguments. For all other + homeserver software, please consult the homeserver documentation for how to deploy multiple signing keys. Note that + not all homeserver software options support multiple signing keys. +5. Run `cat ./merged.signing.key` to verify that your existing signing key ID is on the first line. You can get your key + ID from `GET /_matrix/key/v2/server` against your homeserver in a web browser. If your existing signing key is *not* + first, re-run the steps above, noting the order of keys supplied to `./combine_signing_keys` is important. +6. Deploy `./merged.signing.key` to your Synapse server in place of the existing signing key, restarting it. +7. Deploy `./mmr.signing.key` alongside MMR and specified as `signingKeyPath` for that homeserver in your MMR config. + +In the event that you ever need to revoke MMR's signing key from your homeserver, restore your signing key from the most +recent backup. If your homeserver's signing key changes after running the above steps, re-run the steps above to set up +your server with the new key. Note that it's considered good practice to list your old signing keys, including MMR's +revoked keys, under `old_verify_keys` on `GET /_matrix/key/v2/server` - many homeservers offer a config option to +populate this field. From 85802b45b458b7a31e3509c686f45293b1fdd765 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 9 Feb 2024 22:13:44 -0700 Subject: [PATCH 8/9] Fix imports --- api/_routers/97-require-server-auth.go | 10 +++++----- api/unstable/msc3916_download.go | 10 +++++----- api/unstable/msc3916_thumbnail.go | 10 +++++----- docs/msc3916.md | 2 +- matrix/requests_signing.go | 6 +++--- matrix/xmatrix.go | 4 ++-- test/msc3916_downloads_suite_test.go | 4 ++-- test/msc3916_misc_client_endpoints_suite_test.go | 2 +- test/msc3916_thumbnails_suite_test.go | 4 ++-- test/signing_keys_test.go | 6 +++--- test/xmatrix_header_test.go | 8 ++++---- 11 files changed, 33 insertions(+), 33 deletions(-) diff --git a/api/_routers/97-require-server-auth.go b/api/_routers/97-require-server-auth.go index b7ca3bac..0b128523 100644 --- a/api/_routers/97-require-server-auth.go +++ b/api/_routers/97-require-server-auth.go @@ -3,11 +3,11 @@ package _routers import ( "net/http" - "github.com/turt2live/matrix-media-repo/api/_apimeta" - "github.com/turt2live/matrix-media-repo/api/_responses" - "github.com/turt2live/matrix-media-repo/common" - "github.com/turt2live/matrix-media-repo/common/rcontext" - "github.com/turt2live/matrix-media-repo/matrix" + "github.com/t2bot/matrix-media-repo/api/_apimeta" + "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/common" + "github.com/t2bot/matrix-media-repo/common/rcontext" + "github.com/t2bot/matrix-media-repo/matrix" ) type GeneratorWithServerFn = func(r *http.Request, ctx rcontext.RequestContext, server _apimeta.ServerInfo) interface{} diff --git a/api/unstable/msc3916_download.go b/api/unstable/msc3916_download.go index 64d9d5b1..6089976a 100644 --- a/api/unstable/msc3916_download.go +++ b/api/unstable/msc3916_download.go @@ -4,11 +4,11 @@ import ( "bytes" "net/http" - "github.com/turt2live/matrix-media-repo/api/_apimeta" - "github.com/turt2live/matrix-media-repo/api/_responses" - "github.com/turt2live/matrix-media-repo/api/r0" - "github.com/turt2live/matrix-media-repo/common/rcontext" - "github.com/turt2live/matrix-media-repo/util/readers" + "github.com/t2bot/matrix-media-repo/api/_apimeta" + "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/r0" + "github.com/t2bot/matrix-media-repo/common/rcontext" + "github.com/t2bot/matrix-media-repo/util/readers" ) func ClientDownloadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { diff --git a/api/unstable/msc3916_thumbnail.go b/api/unstable/msc3916_thumbnail.go index 81b77d20..e6de68ec 100644 --- a/api/unstable/msc3916_thumbnail.go +++ b/api/unstable/msc3916_thumbnail.go @@ -4,11 +4,11 @@ import ( "bytes" "net/http" - "github.com/turt2live/matrix-media-repo/api/_apimeta" - "github.com/turt2live/matrix-media-repo/api/_responses" - "github.com/turt2live/matrix-media-repo/api/r0" - "github.com/turt2live/matrix-media-repo/common/rcontext" - "github.com/turt2live/matrix-media-repo/util/readers" + "github.com/t2bot/matrix-media-repo/api/_apimeta" + "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/r0" + "github.com/t2bot/matrix-media-repo/common/rcontext" + "github.com/t2bot/matrix-media-repo/util/readers" ) func FederationThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, server _apimeta.ServerInfo) interface{} { diff --git a/docs/msc3916.md b/docs/msc3916.md index c1f9755e..5d41d293 100644 --- a/docs/msc3916.md +++ b/docs/msc3916.md @@ -32,7 +32,7 @@ To set up a signing key for MMR: 1. Back up your existing homeserver signing key, and store it in a safe place. The signing key effectively grants full access to your server and events, and should not be disclosed to anyone. 2. Download the `generate_signing_key` and `combine_signing_keys` tools for your version of MMR from the - [GitHub releases page](https://github.com/turt2live/matrix-media-repo/releases). + [GitHub releases page](https://github.com/t2bot/matrix-media-repo/releases). 3. Run `./generate_signing_key -output mmr.signing.key` to create a signing key usable with MMR. 4. If you're using Synapse as your homeserver software, run `./combine_signing_keys -format synapse -output ./merged.signing.key ./existing.signing.key ./mmr.signing.key` to combine the signing keys, being sure to list Synapse's existing signing key *first* in the arguments. For all other diff --git a/matrix/requests_signing.go b/matrix/requests_signing.go index 57bb1677..f013f047 100644 --- a/matrix/requests_signing.go +++ b/matrix/requests_signing.go @@ -11,9 +11,9 @@ import ( "github.com/patrickmn/go-cache" "github.com/sirupsen/logrus" "github.com/t2bot/go-typed-singleflight" - "github.com/turt2live/matrix-media-repo/common/rcontext" - "github.com/turt2live/matrix-media-repo/database" - "github.com/turt2live/matrix-media-repo/util" + "github.com/t2bot/matrix-media-repo/common/rcontext" + "github.com/t2bot/matrix-media-repo/database" + "github.com/t2bot/matrix-media-repo/util" ) type signingKey struct { diff --git a/matrix/xmatrix.go b/matrix/xmatrix.go index 7b9889d3..dd8c662e 100644 --- a/matrix/xmatrix.go +++ b/matrix/xmatrix.go @@ -6,8 +6,8 @@ import ( "fmt" "net/http" - "github.com/turt2live/matrix-media-repo/database" - "github.com/turt2live/matrix-media-repo/util" + "github.com/t2bot/matrix-media-repo/database" + "github.com/t2bot/matrix-media-repo/util" ) var ErrNoXMatrixAuth = errors.New("no X-Matrix auth headers") diff --git a/test/msc3916_downloads_suite_test.go b/test/msc3916_downloads_suite_test.go index fc0edf83..5cc12256 100644 --- a/test/msc3916_downloads_suite_test.go +++ b/test/msc3916_downloads_suite_test.go @@ -8,8 +8,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "github.com/turt2live/matrix-media-repo/test/test_internals" - "github.com/turt2live/matrix-media-repo/util" + "github.com/t2bot/matrix-media-repo/test/test_internals" + "github.com/t2bot/matrix-media-repo/util" ) type MSC3916DownloadsSuite struct { diff --git a/test/msc3916_misc_client_endpoints_suite_test.go b/test/msc3916_misc_client_endpoints_suite_test.go index 3d392b6b..dac970e3 100644 --- a/test/msc3916_misc_client_endpoints_suite_test.go +++ b/test/msc3916_misc_client_endpoints_suite_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "github.com/turt2live/matrix-media-repo/test/test_internals" + "github.com/t2bot/matrix-media-repo/test/test_internals" ) type MSC3916MiscClientEndpointsSuite struct { diff --git a/test/msc3916_thumbnails_suite_test.go b/test/msc3916_thumbnails_suite_test.go index 43cf5c77..89132fac 100644 --- a/test/msc3916_thumbnails_suite_test.go +++ b/test/msc3916_thumbnails_suite_test.go @@ -9,8 +9,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "github.com/turt2live/matrix-media-repo/test/test_internals" - "github.com/turt2live/matrix-media-repo/util" + "github.com/t2bot/matrix-media-repo/test/test_internals" + "github.com/t2bot/matrix-media-repo/util" ) type MSC3916ThumbnailsSuite struct { diff --git a/test/signing_keys_test.go b/test/signing_keys_test.go index 13edf013..b23ad54b 100644 --- a/test/signing_keys_test.go +++ b/test/signing_keys_test.go @@ -5,9 +5,9 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/turt2live/matrix-media-repo/database" - "github.com/turt2live/matrix-media-repo/matrix" - "github.com/turt2live/matrix-media-repo/util" + "github.com/t2bot/matrix-media-repo/database" + "github.com/t2bot/matrix-media-repo/matrix" + "github.com/t2bot/matrix-media-repo/util" ) func TestFailInjectedKeys(t *testing.T) { diff --git a/test/xmatrix_header_test.go b/test/xmatrix_header_test.go index 17019108..e3038c39 100644 --- a/test/xmatrix_header_test.go +++ b/test/xmatrix_header_test.go @@ -4,10 +4,10 @@ import ( "crypto/ed25519" "testing" - "github.com/turt2live/matrix-media-repo/common/config" - "github.com/turt2live/matrix-media-repo/database" - "github.com/turt2live/matrix-media-repo/matrix" - "github.com/turt2live/matrix-media-repo/util" + "github.com/t2bot/matrix-media-repo/common/config" + "github.com/t2bot/matrix-media-repo/database" + "github.com/t2bot/matrix-media-repo/matrix" + "github.com/t2bot/matrix-media-repo/util" ) func TestXMatrixAuthHeader(t *testing.T) { From 8b13680cbbdd8dcb36e7b11b16a91b97ffa55f2c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 9 Feb 2024 23:00:12 -0700 Subject: [PATCH 9/9] Testing the Fastly API --- cdn/fastly.go | 57 ++++++++++++++++++++++++ config.sample.yaml | 4 ++ datastores/download.go | 6 +++ datastores/s3.go | 12 +++++ go.mod | 3 ++ go.sum | 8 ++++ pipelines/_steps/download/open_stream.go | 11 +++-- 7 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 cdn/fastly.go diff --git a/cdn/fastly.go b/cdn/fastly.go new file mode 100644 index 00000000..8ff2ecb0 --- /dev/null +++ b/cdn/fastly.go @@ -0,0 +1,57 @@ +package cdn + +import ( + "context" + "errors" + + "github.com/fastly/fastly-go/fastly" +) + +type FastlyCdn struct { + cli *fastly.APIClient + ctx context.Context + serviceId string + dictionaryName string +} + +func NewFastlyCdn(apiKey string, serviceId string, dictionaryName string) *FastlyCdn { + if apiKey == "" || serviceId == "" || dictionaryName == "" { + return nil + } + + client := fastly.NewAPIClient(fastly.NewConfiguration()) + ctx := fastly.NewAPIKeyContext(apiKey) + return &FastlyCdn{ + cli: client, + ctx: ctx, + serviceId: serviceId, + dictionaryName: dictionaryName, + } +} + +func (f *FastlyCdn) SetDictionaryItem(key string, value string) error { + // Find the latest service version + versions, _, err := f.cli.VersionAPI.ListServiceVersions(f.ctx, f.serviceId).Execute() + if err != nil { + return err + } + var latestVersion *fastly.VersionResponse + for _, v := range versions { + if v.GetActive() && (latestVersion == nil || latestVersion.GetNumber() < v.GetNumber()) { + latestVersion = &v + } + } + if latestVersion == nil { + return errors.New("no active service versions to configure") + } + + // Find and update the dictionary + d, _, err := f.cli.DictionaryAPI.GetDictionary(f.ctx, f.serviceId, latestVersion.GetNumber(), f.dictionaryName).Execute() + if err != nil { + return err + } + req := f.cli.DictionaryItemAPI.UpsertDictionaryItem(f.ctx, f.serviceId, *d.ID, key) + req.ItemValue(value) + _, _, err = req.Execute() + return err +} diff --git a/config.sample.yaml b/config.sample.yaml index 3f892881..df05f35e 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -208,6 +208,10 @@ datastores: # capability. MMR may still be responsible for bandwidth charges incurred from going to # the bucket directly. #publicBaseUrl: "https://mycdn.example.org/" + cdn: "fastly" + fastlyApiToken: "FASTLY_API_TOKEN" + fastlyServiceId: "some_string" + fastlyDictionaryName: "mmr_encrypt_keys" # Options for controlling archives. Archives are exports of a particular user's content for # the purpose of GDPR or moving media to a different server. diff --git a/datastores/download.go b/datastores/download.go index 281b3085..b0f29677 100644 --- a/datastores/download.go +++ b/datastores/download.go @@ -48,6 +48,12 @@ func DownloadOrRedirect(ctx rcontext.RequestContext, ds config.DatastoreConfig, } if s3c.publicBaseUrl != "" { + if s3c.fastlyApi != nil { + err = s3c.fastlyApi.SetDictionaryItem("test", "hello world") + if err != nil { + ctx.Log.Warn("Error updating Fastly API:", err) + } + } metrics.S3Operations.With(prometheus.Labels{"operation": "RedirectGetObject"}).Inc() return nil, redirect(fmt.Sprintf("%s%s", s3c.publicBaseUrl, dsFileName)) } diff --git a/datastores/s3.go b/datastores/s3.go index 9064433f..4873ac6a 100644 --- a/datastores/s3.go +++ b/datastores/s3.go @@ -9,6 +9,7 @@ import ( "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/t2bot/matrix-media-repo/cdn" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -20,6 +21,7 @@ type s3 struct { storageClass string bucket string publicBaseUrl string + fastlyApi *cdn.FastlyCdn } func ResetS3Clients() { @@ -39,6 +41,11 @@ func getS3(ds config.DatastoreConfig) (*s3, error) { storageClass, hasStorageClass := ds.Options["storageClass"] useSslStr, hasSsl := ds.Options["ssl"] publicBaseUrl := ds.Options["publicBaseUrl"] + cdnType := ds.Options["cdn"] + + fastlyApiToken := ds.Options["fastlyApiToken"] + fastlyDictionaryName := ds.Options["fastlyDictionaryName"] + fastlyServiceId := ds.Options["fastlyServiceId"] if !hasStorageClass { storageClass = "STANDARD" @@ -66,6 +73,11 @@ func getS3(ds config.DatastoreConfig) (*s3, error) { bucket: bucket, publicBaseUrl: publicBaseUrl, } + + if cdnType == "fastly" { + s3c.fastlyApi = cdn.NewFastlyCdn(fastlyApiToken, fastlyServiceId, fastlyDictionaryName) + } + s3clients.Store(ds.Id, s3c) return s3c, nil } diff --git a/go.mod b/go.mod index 0ef832c5..a440a98c 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( require ( github.com/docker/go-connections v0.5.0 + github.com/fastly/fastly-go v1.0.0-beta.25 github.com/go-redsync/redsync/v4 v4.11.0 github.com/julienschmidt/httprouter v1.3.0 github.com/minio/minio-go/v7 v7.0.66 @@ -135,9 +136,11 @@ require ( go.opentelemetry.io/otel/trace v1.23.1 // indirect golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 // indirect golang.org/x/mod v0.15.0 // indirect + golang.org/x/oauth2 v0.16.0 // indirect golang.org/x/sys v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.17.0 // indirect + google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240205150955-31a09d347014 // indirect google.golang.org/protobuf v1.32.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 21a3537e..9be922e8 100644 --- a/go.sum +++ b/go.sum @@ -100,6 +100,8 @@ github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yi github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= github.com/faiface/beep v1.1.0 h1:A2gWP6xf5Rh7RG/p9/VAW2jRSDEGQm5sbOb38sf5d4c= github.com/faiface/beep v1.1.0/go.mod h1:6I8p6kK2q4opL/eWb+kAkk38ehnTunWeToJB+s51sT4= +github.com/fastly/fastly-go v1.0.0-beta.25 h1:gzcpAN3qS55A01wFVo9PMeoySyrIHdMZRuj0ZgFpdZU= +github.com/fastly/fastly-go v1.0.0-beta.25/go.mod h1:ev9Xm6svDmNXg9VVHCGCEkq2fWnhPWnol+M4ncl7w6I= github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239 h1:Ghm4eQYC0nEPnSJdVkTrXpu9KtoVCSo1hg7mtI7G9KU= github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239/go.mod h1:Gdwt2ce0yfBxPvZrHkprdPPTTS3N5rwmLE8T22KBXlw= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= @@ -155,6 +157,7 @@ github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgR github.com/golang/geo v0.0.0-20230421003525-6adc56603217 h1:HKlyj6in2JV6wVkmQ4XmG/EIm+SCYlPZ+V4GWit7Z+I= github.com/golang/geo v0.0.0-20230421003525-6adc56603217/go.mod h1:8wI0hitZ3a1IxZfeH3/5I97CI8i5cLGsYe7xNhQGs9U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws= @@ -454,6 +457,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -512,6 +517,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= @@ -533,6 +539,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe h1:USL2DhxfgRchafRvt/wYyyQNzwgL7ZiURcozOE/Pkvo= google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo= google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4= diff --git a/pipelines/_steps/download/open_stream.go b/pipelines/_steps/download/open_stream.go index 51c0c7e9..58838be5 100644 --- a/pipelines/_steps/download/open_stream.go +++ b/pipelines/_steps/download/open_stream.go @@ -8,7 +8,6 @@ import ( "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/datastores" - "github.com/t2bot/matrix-media-repo/redislib" "github.com/t2bot/matrix-media-repo/util/readers" ) @@ -39,11 +38,11 @@ func OpenOrRedirect(ctx rcontext.RequestContext, media *database.Locatable) (io. } func doOpenStream(ctx rcontext.RequestContext, media *database.Locatable) (io.ReadSeekCloser, config.DatastoreConfig, error) { - reader, err := redislib.TryGetMedia(ctx, media.Sha256Hash) - if err != nil || reader != nil { - ctx.Log.Debugf("Got %s from cache", media.Sha256Hash) - return readers.NopSeekCloser(reader), config.DatastoreConfig{}, err - } + //reader, err := redislib.TryGetMedia(ctx, media.Sha256Hash) + //if err != nil || reader != nil { + // ctx.Log.Debugf("Got %s from cache", media.Sha256Hash) + // return readers.NopSeekCloser(reader), config.DatastoreConfig{}, err + //} ds, ok := datastores.Get(ctx, media.DatastoreId) if !ok {