From 0258b155f66f3602116377e0fc9e06d4e34d909b Mon Sep 17 00:00:00 2001 From: Dmitry Verkhoturov Date: Thu, 20 Jul 2023 19:16:37 +0200 Subject: [PATCH] remove proxied images from sanity check Previously, proxied and local images were checked for presence in the storage before previewing or posting the comment. That logic resulted in an inability to post with an image when a proxy for images is enabled, as proxied images are not downloaded to disk before the first time someone loads them, which could only happen after the user either previews or posts the message. After this change, preview and post only checks the local images' presence and ignore the proxied ones. --- backend/app/rest/api/rest_private.go | 8 ++++---- backend/app/store/image/image.go | 7 +++++-- backend/app/store/image/image_test.go | 15 +++++++++------ backend/app/store/service/service.go | 2 +- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go index 5385f756d9..4185e38eb4 100644 --- a/backend/app/rest/api/rest_private.go +++ b/backend/app/rest/api/rest_private.go @@ -90,8 +90,8 @@ func (s *private) previewCommentCtrl(w http.ResponseWriter, r *http.Request) { comment = s.commentFormatter.Format(comment) comment.Sanitize() - // check if images are valid - for _, id := range s.imageService.ExtractPictures(comment.Text) { + // check if images are valid, omit proxied images as they are lazy-loaded + for _, id := range s.imageService.ExtractPictures(comment.Text, false) { err := s.imageService.ResetCleanupTimer(id) if err != nil { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't load picture from the comment", rest.ErrImgNotFound) @@ -129,8 +129,8 @@ func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) { } comment = s.commentFormatter.Format(comment) - // check if images are valid - for _, id := range s.imageService.ExtractPictures(comment.Text) { + // check if images are valid, omit proxied images as they are lazy-loaded + for _, id := range s.imageService.ExtractPictures(comment.Text, false) { _, err := s.imageService.Load(id) if err != nil { rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't load picture from the comment", rest.ErrImgNotFound) diff --git a/backend/app/store/image/image.go b/backend/app/store/image/image.go index a5dfaee138..9365fc1625 100644 --- a/backend/app/store/image/image.go +++ b/backend/app/store/image/image.go @@ -140,7 +140,10 @@ func (s *Service) Submit(idsFn func() []string) { } // ExtractPictures gets list of images from the doc html and convert from urls to ids, i.e. user/pic.png -func (s *Service) ExtractPictures(commentHTML string) (ids []string) { +// Proxied images are returned only if the flag is set: this is used in image check on post preview and load, +// as proxied images have lazy loading and wouldn't be present on disk but still valid as they will be loaded +// the first time someone requests them. +func (s *Service) ExtractPictures(commentHTML string, returnProxied bool) (ids []string) { doc, err := goquery.NewDocumentFromReader(strings.NewReader(commentHTML)) if err != nil { log.Printf("[ERROR] can't parse commentHTML to parse images: %q, error: %v", commentHTML, err) @@ -155,7 +158,7 @@ func (s *Service) ExtractPictures(commentHTML string) (ids []string) { ids = append(ids, id) } } - if strings.Contains(im, s.ProxyAPI) { + if strings.Contains(im, s.ProxyAPI) && returnProxied { proxiedURL, err := url.Parse(im) if err != nil { return diff --git a/backend/app/store/image/image_test.go b/backend/app/store/image/image_test.go index 274a6fc88e..e10d783e82 100644 --- a/backend/app/store/image/image_test.go +++ b/backend/app/store/image/image_test.go @@ -84,7 +84,7 @@ func TestService_ExtractPictures(t *testing.T) { svc := Service{ServiceParams: ServiceParams{ImageAPI: "/blah/", ProxyAPI: "/non_existent"}} html := `blah foo xyz

123

` - ids := svc.ExtractPictures(html) + ids := svc.ExtractPictures(html, false) require.Equal(t, 2, len(ids), "two images") assert.Equal(t, "user1/pic1.png", ids[0]) assert.Equal(t, "user2/pic3.png", ids[1]) @@ -96,30 +96,33 @@ func TestService_ExtractPictures(t *testing.T) {

bjtr0-201906-08110846-i324c.png

\n\n

По форме все верно, это все packages, но по сути это все одна библиотека организованная таким образом. При ее импорте, например посредством go mod, она выглядит как один модуль, т.е. github.com/go-pkgz/auth v0.5.2.

\n` - ids = svc.ExtractPictures(html) + ids = svc.ExtractPictures(html, false) require.Equal(t, 1, len(ids), "one image in") assert.Equal(t, "github_ef0f706a79cc24b17bbbb374cd234a691d034128/bjttt8ahajfmrhsula10.png", ids[0]) // proxied image html = `cat.png` - ids = svc.ExtractPictures(html) + ids = svc.ExtractPictures(html, true) require.Equal(t, 1, len(ids), "one image in") assert.Equal(t, "cached_images/12318fbd4c55e9d177b8b5ae197bc89c5afd8e07-a41fcb00643f28d700504256ec81cbf2e1aac53e", ids[0]) + // same proxied image, with returnProxied set to false + ids = svc.ExtractPictures(html, false) + require.Empty(t, ids, "no images expected to be found") // bad url html = `` - ids = svc.ExtractPictures(html) + ids = svc.ExtractPictures(html, false) require.Empty(t, ids) // bad src html = `` - ids = svc.ExtractPictures(html) + ids = svc.ExtractPictures(html, false) require.Empty(t, ids) // good src with bad content badURL := base64.URLEncoding.EncodeToString([]byte(" http://foo.bar")) html = fmt.Sprintf(``, badURL) - ids = svc.ExtractPictures(html) + ids = svc.ExtractPictures(html, false) require.Empty(t, ids) } diff --git a/backend/app/store/service/service.go b/backend/app/store/service/service.go index 5503712b52..345e2e18a6 100644 --- a/backend/app/store/service/service.go +++ b/backend/app/store/service/service.go @@ -270,7 +270,7 @@ func (s *DataStore) submitImages(comment store.Comment) { log.Printf("[WARN] can't get comment's %s text for image extraction, %v", comment.ID, err) return nil } - imgIDs := s.ImageService.ExtractPictures(cc.Text) + imgIDs := s.ImageService.ExtractPictures(cc.Text, true) if len(imgIDs) > 0 { log.Printf("[DEBUG] image ids extracted from %s - %+v", comment.ID, imgIDs) }