Skip to content

Commit

Permalink
implement thumbnail support for txt files
Browse files Browse the repository at this point in the history
  • Loading branch information
David Christofas committed Apr 29, 2021
1 parent 9ddfab8 commit 12b4a26
Show file tree
Hide file tree
Showing 16 changed files with 160 additions and 59 deletions.
5 changes: 5 additions & 0 deletions changelog/unreleased/thumbnails-for-txt.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Enhancement: Support thumbnails for txt files

Implemented support for thumbnails for txt files in the thumbnails service.

https://github.com/owncloud/ocis/pull/1988
1 change: 1 addition & 0 deletions ocis/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -669,6 +669,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/golang/gddo v0.0.0-20180828051604-96d2a289f41e/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4=
github.com/golang/gddo v0.0.0-20190904175337-72a348e765d2/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4=
Expand Down
4 changes: 2 additions & 2 deletions ocs/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,14 @@ require (
github.com/owncloud/ocis/proxy v0.0.0-20210412105747-9b95e9b1191b
github.com/owncloud/ocis/settings v0.0.0-20210413063522-955bd60edf33
github.com/owncloud/ocis/store v0.0.0-20210413063522-955bd60edf33
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.10.0
github.com/spf13/viper v1.7.1
github.com/stretchr/testify v1.7.0
github.com/thejerf/suture/v4 v4.0.0
go.opencensus.io v0.23.0
google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea
google.golang.org/grpc v1.37.0 // indirect
google.golang.org/grpc v1.37.0
google.golang.org/protobuf v1.26.0
)

Expand Down
6 changes: 2 additions & 4 deletions tests/acceptance/expected-failures-API-on-OCIS-storage.md
Original file line number Diff line number Diff line change
Expand Up @@ -801,16 +801,14 @@ cannot share a folder with create permission
#### [Previews via webDAV API tests fail on OCIS](https://github.com/owncloud/ocis/issues/187)

- [apiWebdavPreviews/previews.feature:15](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L15)
- [apiWebdavPreviews/previews.feature:16](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L16)
- [apiWebdavPreviews/previews.feature:17](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L17)
- [apiWebdavPreviews/previews.feature:18](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L18)
- [apiWebdavPreviews/previews.feature:19](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L19)
- [apiWebdavPreviews/previews.feature:56](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L56)
- [apiWebdavPreviews/previews.feature:87](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L87)
- [apiWebdavPreviews/previews.feature:95](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L95)
- [apiWebdavPreviews/previews.feature:104](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L104)
- [apiWebdavPreviews/previews.feature:113](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L113)
- [apiWebdavPreviews/previews.feature:163](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L163)
- [apiWebdavPreviews/previews.feature:127](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L127)
- [apiWebdavPreviews/previews.feature:135](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L135)
- [apiWebdavPreviews/previews.feature:164](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L164)
- [apiWebdavPreviews/previews.feature:165](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L165)
- [apiWebdavPreviews/previews.feature:166](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L166)
Expand Down
6 changes: 2 additions & 4 deletions tests/acceptance/expected-failures-API-on-OWNCLOUD-storage.md
Original file line number Diff line number Diff line change
Expand Up @@ -825,16 +825,14 @@ cannot share a folder with create permission
#### [Previews via webDAV API tests fail on OCIS](https://github.com/owncloud/ocis/issues/187)

- [apiWebdavPreviews/previews.feature:15](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L15)
- [apiWebdavPreviews/previews.feature:16](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L16)
- [apiWebdavPreviews/previews.feature:17](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L17)
- [apiWebdavPreviews/previews.feature:18](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L18)
- [apiWebdavPreviews/previews.feature:19](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L19)
- [apiWebdavPreviews/previews.feature:56](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L56)
- [apiWebdavPreviews/previews.feature:87](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L87)
- [apiWebdavPreviews/previews.feature:95](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L95)
- [apiWebdavPreviews/previews.feature:104](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L104)
- [apiWebdavPreviews/previews.feature:113](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L113)
- [apiWebdavPreviews/previews.feature:163](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L163)
- [apiWebdavPreviews/previews.feature:127](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L127)
- [apiWebdavPreviews/previews.feature:135](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L135)
- [apiWebdavPreviews/previews.feature:164](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L164)
- [apiWebdavPreviews/previews.feature:165](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L165)
- [apiWebdavPreviews/previews.feature:166](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavPreviews/previews.feature#L166)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,6 @@ Feature: previews of files downloaded through the webdav API
| A |
| %2F |

@issue-ocis-187
# after fixing all issues delete this Scenario and use the one from oC10 core
Scenario: download previews of image after renaming it
Given user "Alice" has uploaded file "filesForUpload/testavatar.jpg" to "/testimage.jpg"
When user "Alice" moves file "/testimage.jpg" to "/testimage.txt" using the WebDAV API
And user "Alice" downloads the preview of "/testimage.txt" with width "32" and height "32" using the WebDAV API
Then the HTTP status code should be "404"
# And the downloaded image should be "1240" pixels wide and "648" pixels high

@issue-ocis-thumbnails-191 @skipOnOcis-EOS-Storage @issue-ocis-reva-308
# after fixing all issues delete this Scenario and use the one from oC10 core
Scenario: download previews of other users files
Expand All @@ -66,7 +57,7 @@ Feature: previews of files downloaded through the webdav API
Given the administrator has updated system config key "enable_previews" with value "false" and type "boolean"
And user "Alice" has uploaded file "filesForUpload/lorem.txt" to "/parent.txt"
When user "Alice" downloads the preview of "/parent.txt" with width "32" and height "32" using the WebDAV API
Then the HTTP status code should be "404"
Then the HTTP status code should be "200"

@issue-ocis-193
# after fixing all issues delete this Scenario and use the one from oC10 core
Expand All @@ -75,7 +66,7 @@ Feature: previews of files downloaded through the webdav API
And the administrator has updated system config key "preview_max_x" with value "null"
And the administrator has updated system config key "preview_max_y" with value "null"
When user "Alice" downloads the preview of "/parent.txt" with width "32" and height "32" using the WebDAV API
Then the HTTP status code should be "404"
Then the HTTP status code should be "200"

@issue-ocis-193
# after fixing all issues delete this Scenario and use the one from oC10 core
Expand Down
2 changes: 2 additions & 0 deletions thumbnails/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ require (
github.com/cs3org/go-cs3apis v0.0.0-20210325133324-32b03d75a535
github.com/cs3org/reva v1.6.1-0.20210414111318-a4b5148cbfb2
github.com/disintegration/imaging v1.6.2
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0
github.com/golang/protobuf v1.5.2
github.com/grpc-ecosystem/grpc-gateway/v2 v2.2.0
github.com/micro/cli/v2 v2.1.2
Expand All @@ -23,6 +24,7 @@ require (
github.com/stretchr/testify v1.7.0
github.com/thejerf/suture/v4 v4.0.0
go.opencensus.io v0.23.0
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8
google.golang.org/grpc v1.37.0
google.golang.org/protobuf v1.26.0
)
Expand Down
1 change: 1 addition & 0 deletions thumbnails/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -665,6 +665,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/golang/gddo v0.0.0-20180828051604-96d2a289f41e/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4=
github.com/golang/gddo v0.0.0-20190904175337-72a348e765d2/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4=
Expand Down
106 changes: 106 additions & 0 deletions thumbnails/pkg/preprocessor/preprocessor.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
package preprocessor

import (
"bufio"
"github.com/golang/freetype"
"github.com/golang/freetype/truetype"
"github.com/pkg/errors"
"golang.org/x/image/font"
"golang.org/x/image/font/gofont/goregular"
"image"
"image/draw"
"io"
"mime"
"strings"
)

const (
fontSize = 12
spacing float64 = 1.5
)

type FileConverter interface {
Convert(r io.Reader) (image.Image, error)
}

type ImageDecoder struct{}

func (i ImageDecoder) Convert(r io.Reader) (image.Image, error) {
img, _, err := image.Decode(r)
if err != nil {
return nil, errors.Wrap(err, `could not decode the image`)
}
return img, nil
}

type TxtToImageConverter struct{}

func (t TxtToImageConverter) Convert(r io.Reader) (image.Image, error) {
img := image.NewRGBA(image.Rect(0, 0, 640, 480))
draw.Draw(img, img.Bounds(), image.White, image.Point{}, draw.Src)

c := freetype.NewContext()
// Ignoring the error since we are using the embedded Golang font.
// This shouldn't return an error.
f, _ := truetype.Parse(goregular.TTF)
c.SetFont(f)
c.SetFontSize(fontSize)
c.SetClip(img.Bounds())
c.SetDst(img)
c.SetSrc(image.Black)
c.SetHinting(font.HintingFull)
pt := freetype.Pt(10, 10+int(c.PointToFixed(fontSize)>>6))

scanner := bufio.NewScanner(r)
for scanner.Scan() {
txt := scanner.Text()
cs := chunks(txt, 80)
for _, s := range cs {
_, err := c.DrawString(strings.TrimSpace(s), pt)
if err != nil {
return nil, err
}
pt.Y += c.PointToFixed(fontSize * spacing)
if pt.Y.Round() >= img.Bounds().Dy() {
return img, scanner.Err()
}
}

}
return img, scanner.Err()
}

// Code from https://stackoverflow.com/a/61469854
// Written By Igor Mikushkin
func chunks(s string, chunkSize int) []string {
if chunkSize >= len(s) {
return []string{s}
}
var chunks []string
chunk := make([]rune, chunkSize)
length := 0
for _, r := range s {
chunk[length] = r
length++
if length == chunkSize {
chunks = append(chunks, string(chunk))
length = 0
}
}
if length > 0 {
chunks = append(chunks, string(chunk[:length]))
}
return chunks
}

func ForType(mimeType string) FileConverter {
// We can ignore the error here because we parse it in IsMimeTypeSupported before and if it fails
// return the service call. So we should only get here when the mimeType parses fine.
mimeType, _, _ = mime.ParseMediaType(mimeType)
switch mimeType {
case "text/plain":
return TxtToImageConverter{}
default:
return ImageDecoder{}
}
}
22 changes: 13 additions & 9 deletions thumbnails/pkg/service/v0/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/pkg/token"
"github.com/owncloud/ocis/ocis-pkg/log"
"github.com/owncloud/ocis/thumbnails/pkg/preprocessor"
v0proto "github.com/owncloud/ocis/thumbnails/pkg/proto/v0"
"github.com/owncloud/ocis/thumbnails/pkg/thumbnail"
"github.com/owncloud/ocis/thumbnails/pkg/thumbnail/imgsource"
Expand Down Expand Up @@ -105,12 +106,15 @@ func (g Thumbnail) handleCS3Source(ctx context.Context, req *v0proto.GetThumbnai
}

ctx = imgsource.ContextSetAuthorization(ctx, src.Authorization)
img, err := g.cs3Source.Get(ctx, src.Path)
r, err := g.cs3Source.Get(ctx, src.Path)
if err != nil {
return nil, merrors.InternalServerError(g.serviceID, "could not get image from source: %s", err.Error())
}
if img == nil {
return nil, merrors.InternalServerError(g.serviceID, "could not get image from source")
defer r.Close() // nolint:errcheck
pp := preprocessor.ForType(sRes.GetInfo().GetMimeType())
img, err := pp.Convert(r)
if img == nil || err != nil {
return nil, merrors.InternalServerError(g.serviceID, "could not get image")
}
if thumb, err = g.manager.Generate(tr, img); err != nil {
return nil, err
Expand Down Expand Up @@ -176,12 +180,15 @@ func (g Thumbnail) handleWebdavSource(ctx context.Context, req *v0proto.GetThumb
ctx = imgsource.ContextSetAuthorization(ctx, src.WebdavAuthorization)
}
imgURL.RawQuery = ""
img, err := g.webdavSource.Get(ctx, imgURL.String())
r, err := g.webdavSource.Get(ctx, imgURL.String())
if err != nil {
return nil, merrors.InternalServerError(g.serviceID, "could not get image from source: %s", err.Error())
}
if img == nil {
return nil, merrors.InternalServerError(g.serviceID, "could not get image from source")
defer r.Close() // nolint:errcheck
pp := preprocessor.ForType(sRes.GetInfo().GetMimeType())
img, err := pp.Convert(r)
if img == nil || err != nil {
return nil, merrors.InternalServerError(g.serviceID, "could not get image")
}
if thumb, err = g.manager.Generate(tr, img); err != nil {
return nil, err
Expand Down Expand Up @@ -223,8 +230,5 @@ func (g Thumbnail) stat(path, auth string) (*provider.StatResponse, error) {
if !thumbnail.IsMimeTypeSupported(rsp.Info.MimeType) {
return nil, merrors.NotFound(g.serviceID, "Unsupported file type")
}



return rsp, nil
}
15 changes: 6 additions & 9 deletions thumbnails/pkg/thumbnail/imgsource/cs3.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
"github.com/cs3org/reva/pkg/token"
"github.com/pkg/errors"
"google.golang.org/grpc/metadata"
"image"
"io"
"net/http"
)

Expand All @@ -25,7 +25,9 @@ func NewCS3Source(c gateway.GatewayAPIClient) CS3 {
}
}

func (s CS3) Get(ctx context.Context, path string) (image.Image, error) {
// Get downloads the file from a cs3 service
// The caller MUST make sure to close the returned ReadCloser
func (s CS3) Get(ctx context.Context, path string) (io.ReadCloser, error) {
auth, ok := ContextGetAuthorization(ctx)
if !ok {
return nil, errors.New("cs3source: authorization missing")
Expand Down Expand Up @@ -63,19 +65,14 @@ func (s CS3) Get(ctx context.Context, path string) (image.Image, error) {
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec
client := &http.Client{}

resp, err := client.Do(httpReq)
resp, err := client.Do(httpReq) // nolint:bodyclose
if err != nil {
return nil, err
}
defer resp.Body.Close() //nolint:errcheck

if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("could not get the image \"%s\". Request returned with statuscode %d ", path, resp.StatusCode)
}

img, _, err := image.Decode(resp.Body)
if err != nil {
return nil, errors.Wrapf(err, `could not decode the image "%s"`, path)
}
return img, nil
return resp.Body, nil
}
11 changes: 3 additions & 8 deletions thumbnails/pkg/thumbnail/imgsource/filesystem.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package imgsource

import (
"context"
"image"
"io"
"os"
"path/filepath"

Expand All @@ -23,17 +23,12 @@ type FileSystem struct {
}

// Get retrieves an image from the filesystem.
func (s FileSystem) Get(ctx context.Context, file string) (image.Image, error) {
func (s FileSystem) Get(ctx context.Context, file string) (io.ReadCloser, error) {
imgPath := filepath.Join(s.basePath, file)
f, err := os.Open(filepath.Clean(imgPath))
if err != nil {
return nil, errors.Wrapf(err, "failed to load the file %s from %s", file, imgPath)
}

img, _, err := image.Decode(f)
if err != nil {
return nil, errors.Wrap(err, "Get: Decode:")
}

return img, nil
return f, nil
}
4 changes: 2 additions & 2 deletions thumbnails/pkg/thumbnail/imgsource/imgsource.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package imgsource

import (
"context"
"image"
"io"
)

type key int
Expand All @@ -13,7 +13,7 @@ const (

// Source defines the interface for image sources
type Source interface {
Get(ctx context.Context, path string) (image.Image, error)
Get(ctx context.Context, path string) (io.ReadCloser, error)
}

// ContextSetAuthorization puts the authorization in the context.
Expand Down
Loading

0 comments on commit 12b4a26

Please sign in to comment.