Add custom S3 backend support (eg: Minio) to media uploads

- Introduce a new S3 backend URL on the settings UI - Add DB migration to populate S3 URL for existing S3 settings - Refactor and fix URL formatting Closes #139
knadh · Aug 15, 2021 · d6d1883 · d6d1883
1 parent 923b882
commit d6d1883
Show file tree

Hide file tree

Showing 20 changed files with 102 additions and 41 deletions.
diff --git a/cmd/init.go b/cmd/init.go
@@ -452,7 +452,7 @@ func initPostbackMessengers(m *manager.Manager) []messenger.Messenger {
 func initMediaStore() media.Store {
 	switch provider := ko.String("upload.provider"); provider {
 	case "s3":
-		var o s3.Opts
+		var o s3.Opt
 		ko.Unmarshal("upload.s3", &o)
 		up, err := s3.NewS3Store(o)
 		if err != nil {

diff --git a/cmd/settings.go b/cmd/settings.go
@@ -42,6 +42,7 @@ type settings struct {
 	UploadProvider             string `json:"upload.provider"`
 	UploadFilesystemUploadPath string `json:"upload.filesystem.upload_path"`
 	UploadFilesystemUploadURI  string `json:"upload.filesystem.upload_uri"`
+	UploadS3URL                string `json:"upload.s3.url"`
 	UploadS3AwsAccessKeyID     string `json:"upload.s3.aws_access_key_id"`
 	UploadS3AwsDefaultRegion   string `json:"upload.s3.aws_default_region"`
 	UploadS3AwsSecretAccessKey string `json:"upload.s3.aws_secret_access_key,omitempty"`

diff --git a/frontend/src/views/Settings.vue b/frontend/src/views/Settings.vue
@@ -18,7 +18,7 @@
     <hr />
 
     <section class="wrap-small">
-      <form @submit.prevent="onSubmit" v-if="!isLoading">
+      <form @submit.prevent="onSubmit">
         <b-tabs type="is-boxed" :animated="false">
           <b-tab-item :label="$t('settings.general.name')" label-position="on-border">
             <div class="items">
@@ -213,7 +213,7 @@
                   <div class="column is-3">
                     <b-field :label="$t('settings.media.s3.region')"
                       label-position="on-border" expanded>
-                      <b-input v-model="form['upload.s3.aws_default_region']"
+                      <b-input v-model="form['upload.s3.aws_default_region']" @input="onS3URLChange"
                           name="upload.s3.aws_default_region"
                           :maxlength="200" placeholder="ap-south-1" />
                     </b-field>
@@ -254,7 +254,7 @@
                     <b-field grouped>
                       <b-field :label="$t('settings.media.s3.bucket')"
                         label-position="on-border" expanded>
-                        <b-input v-model="form['upload.s3.bucket']"
+                        <b-input v-model="form['upload.s3.bucket']" @input="onS3URLChange"
                             name="upload.s3.bucket" :maxlength="200" placeholder="" />
                       </b-field>
                       <b-field :label="$t('settings.media.s3.bucketPath')"
@@ -276,6 +276,16 @@
                         placeholder="14d" :pattern="regDuration" :maxlength="10" />
                     </b-field>
                   </div>
+                  <div class="column">
+                    <b-field :label="$t('settings.media.s3.url')"
+                      label-position="on-border"
+                      :message="$t('settings.media.s3.urlHelp')" expanded>
+                      <b-input v-model="form['upload.s3.url']"
+                        name="upload.s3.url"
+                        :disabled="!form['upload.s3.bucket']"
+                        placeholder="https://s3.region.amazonaws.com" :maxlength="200" />
+                    </b-field>
+                  </div>
                 </div>
               </div><!-- s3 -->
             </div>
@@ -786,6 +796,13 @@ export default Vue.extend({
       this.form.messengers.splice(i, 1);
     },
 
+    onS3URLChange() {
+      // If a custom non-AWS URL has been entered, don't update it automatically.
+      if (this.form['upload.s3.url'] !== '' && !this.form['upload.s3.url'].match(/amazonaws\.com/)) {
+        return;
+      }
+      this.form['upload.s3.url'] = `https://s3.${this.form['upload.s3.aws_default_region']}.amazonaws.com`;
+    },
 
     onSubmit() {
       const form = JSON.parse(JSON.stringify(this.form));

diff --git a/go.mod b/go.mod
@@ -19,7 +19,7 @@ require (
 	github.com/mailru/easyjson v0.7.6
 	github.com/mitchellh/copystructure v1.1.2 // indirect
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
-	github.com/rhnvrm/simples3 v0.5.0
+	github.com/rhnvrm/simples3 v0.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/yuin/goldmark v1.3.4
 	golang.org/x/mod v0.3.0

diff --git a/go.sum b/go.sum
@@ -84,6 +84,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rhnvrm/simples3 v0.5.0 h1:X+WX0hqoKScdoJAw/G3GArfZ6Ygsn8q+6MdocTMKXOw=
 github.com/rhnvrm/simples3 v0.5.0/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA=
+github.com/rhnvrm/simples3 v0.7.0 h1:KSEuKw0eGC5vltLW8ChLvjko+aUr0HbGet+bZHdwfMo=
+github.com/rhnvrm/simples3 v0.7.0/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA=
 github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=

diff --git a/i18n/de.json b/i18n/de.json
@@ -354,6 +354,8 @@
     "settings.media.s3.secret": "AWS Access Secret",
     "settings.media.s3.uploadExpiry": "Upload Ablaufdatum",
     "settings.media.s3.uploadExpiryHelp": "(Optional) Zeit bis zum Ablauf (in Sekunden) für die generierte URL. Nur für private Buckets. (s, m, h, d für Sekunden, Minuten, Stunden, Tage).",
+    "settings.media.s3.url": "S3 backend URL",
+    "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
     "settings.media.title": "Medien Uploads",
     "settings.media.upload.path": "Upload Pfad",
     "settings.media.upload.pathHelp": "Pfad zum Upload Verzeichnis.",

diff --git a/i18n/en.json b/i18n/en.json
@@ -354,6 +354,8 @@
     "settings.media.s3.secret": "AWS access secret",
     "settings.media.s3.uploadExpiry": "Upload expiry",
     "settings.media.s3.uploadExpiryHelp": "(Optional) Specify TTL (in seconds) for the generated presigned URL. Only applicable for private buckets (s, m, h, d for seconds, minutes, hours, days).",
+    "settings.media.s3.url": "S3 backend URL",
+    "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
     "settings.media.title": "Media uploads",
     "settings.media.upload.path": "Upload path",
     "settings.media.upload.pathHelp": "Path to the directory where media will be uploaded.",

diff --git a/i18n/es.json b/i18n/es.json
@@ -354,6 +354,8 @@
     "settings.media.s3.secret": "Secrete de acceso a AWS (secret)",
     "settings.media.s3.uploadExpiry": "Expiración de carga",
     "settings.media.s3.uploadExpiryHelp": "(Opcional) TTL específico (en segundos) para la URL pre firmada generada. Solo es aplicable para contenedores privados (s, m, h, d para segundos, minutos, horas, días)",
+    "settings.media.s3.url": "S3 backend URL",
+    "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
     "settings.media.title": "Cargas de media",
     "settings.media.upload.path": "Ruta de carga",
     "settings.media.upload.pathHelp": "Ruta al directorio donde la media será cargada.",

diff --git a/i18n/fr.json b/i18n/fr.json
@@ -354,6 +354,8 @@
     "settings.media.s3.secret": "Mot de passe d'accès AWS",
     "settings.media.s3.uploadExpiry": "Durée de validité",
     "settings.media.s3.uploadExpiryHelp": "(Facultatif) Spécifiez la durée de validité (en secondes) pour l'URL prédéfinie générée. Uniquement applicable pour les compartiments privés (s, m, h, d pour les secondes, minutes, heures, jours).",
+    "settings.media.s3.url": "S3 backend URL",
+    "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
     "settings.media.title": "Mise en ligne de fichiers",
     "settings.media.upload.path": "Emplacement d'envoi des fichiers",
     "settings.media.upload.pathHelp": "Chemin vers le répertoire où les médias seront mis en ligne",

diff --git a/i18n/it.json b/i18n/it.json
@@ -354,6 +354,8 @@
     "settings.media.s3.secret": "Accesso segreto AWS",
     "settings.media.s3.uploadExpiry": "Caricamento scaduto",
     "settings.media.s3.uploadExpiryHelp": "(Facoltativo) Specifica il TTL (in secondi) per l'URL predefinito generato. Applicabile solo per i buckets privati (s, m, h, d per i secondi, minuti, ore e giorni).",
+    "settings.media.s3.url": "S3 backend URL",
+    "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
     "settings.media.title": "Caricamento dei media",
     "settings.media.upload.path": "Percorso del caricamento",
     "settings.media.upload.pathHelp": "Percorso verso il repertorio dove i media saranno caricati.",

diff --git a/i18n/ml.json b/i18n/ml.json
@@ -354,6 +354,8 @@
     "settings.media.s3.secret": "AWS പ്രവേശന രഹസ്യം",
     "settings.media.s3.uploadExpiry": "അപ്ലോഡിന്റെ കാലാവധി",
     "settings.media.s3.uploadExpiryHelp": "(ഐച്ഛികം) മുൻകൂട്ടി നിർമ്മിക്കുന്ന യൂ. ആർ. എല്ലിനുള്ള സെക്കന്റിലുള്ള TTL വ്യക്തമാക്കുക . സ്വകാര്യ ബക്കറ്റുകൾക്ക് മാത്രമേ ബാധകമാകൂ (s, m, h, d എന്നിവ യഥാക്രമം സെക്കന്റ്, മിനുട്ട്, മണിക്കൂർ, ദിവസങ്ങൾ എന്നിവയെ സൂചിപ്പിക്കുന്നു).",
+    "settings.media.s3.url": "S3 backend URL",
+    "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
     "settings.media.title": "മീഡിയാ അപ്ലോഡുകൾ",
     "settings.media.upload.path": "അപ്ലോഡ് പാത്ത്",
     "settings.media.upload.pathHelp": "മീഡിയ അപ്ലോഡ് ചെയ്യുന്നതിനുള്ള ഡയറക്ടറിയിലേക്കുള്ള പാത്ത്.",

diff --git a/i18n/pl.json b/i18n/pl.json
@@ -354,6 +354,8 @@
     "settings.media.s3.secret": "Sekret dostępu AWS",
     "settings.media.s3.uploadExpiry": "Wygaśnięcie przesyłania",
     "settings.media.s3.uploadExpiryHelp": "(Opcjonalne) Zdefiniuj TTL (w sekundach) dla wygenerowanego podpisanego URL. Tylko dla prywatnych komór (bucketów) (s, m, h, d dla sekund, minut, godzin, dni).",
+    "settings.media.s3.url": "S3 backend URL",
+    "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
     "settings.media.title": "Wysyłka mediów",
     "settings.media.upload.path": "Ścieżka do wysyłki",
     "settings.media.upload.pathHelp": "Ścieżka do folderu do którego media będą wrzucane.",

diff --git a/i18n/pt-BR.json b/i18n/pt-BR.json
@@ -354,6 +354,8 @@
     "settings.media.s3.secret": "Segredo de acesso AWS",
     "settings.media.s3.uploadExpiry": "Expiração do arquivo enviado",
     "settings.media.s3.uploadExpiryHelp": "(Opcional) Especificar TTL (em segundos) para a URL pré-assinada gerada. Apenas aplicável para buckets privados (s, m, h, d para segundos, minutos, horas e dias).",
+    "settings.media.s3.url": "S3 backend URL",
+    "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
     "settings.media.title": "Envios de mídias",
     "settings.media.upload.path": "Caminho de envio",
     "settings.media.upload.pathHelp": "Caminho para o diretório onde a mídia será enviado.",

diff --git a/i18n/pt.json b/i18n/pt.json
@@ -354,6 +354,8 @@
     "settings.media.s3.secret": "Segredo de acesso AWS",
     "settings.media.s3.uploadExpiry": "Validade do upload",
     "settings.media.s3.uploadExpiryHelp": "(Opcional) Especifica TTL (em segundos) para o URL pré-assinado gerado. Apenas aplicável a buckets privados (s, m, h, d para segundos, minutos, horas e dias).",
+    "settings.media.s3.url": "S3 backend URL",
+    "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
     "settings.media.title": "Upload de mídia",
     "settings.media.upload.path": "Caminho de upload",
     "settings.media.upload.pathHelp": "Caminho para a pasta onde será enviada a mídia.",

diff --git a/i18n/ru.json b/i18n/ru.json
@@ -354,6 +354,8 @@
     "settings.media.s3.secret": "Секретаня фраза AWS",
     "settings.media.s3.uploadExpiry": "Срок жизни выгрузки",
     "settings.media.s3.uploadExpiryHelp": "(Необязательно) Укажите TTL (в секундах) сгенерированного подписанного URL. Применимо только для приватных bucket (s, m, h, d соответствует секундам, минутам, часам и дням).",
+    "settings.media.s3.url": "S3 backend URL",
+    "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
     "settings.media.title": "Выгрузки медиа",
     "settings.media.upload.path": "Путь для выгрузок",
     "settings.media.upload.pathHelp": "Путь до каталога, куда будут выгружаться медиа-файлы.",

diff --git a/i18n/tr.json b/i18n/tr.json
@@ -354,6 +354,8 @@
     "settings.media.s3.secret": "AWS erişim şifresi(secret)",
     "settings.media.s3.uploadExpiry": "Yükleme sona erme",
     "settings.media.s3.uploadExpiryHelp": "(İsteğe bağlı) Oluşturulan önceden imzalanmış URL için TTL'yi (saniye cinsinden) belirtin. Yalnızca özel paketler için geçerlidir (saniye, dakika, saat, gün için s, m, h, d).",
+    "settings.media.s3.url": "S3 backend URL",
+    "settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
     "settings.media.title": "Medya yüklemeleri",
     "settings.media.upload.path": "Yükleme yolu",
     "settings.media.upload.pathHelp": "Medyanın yükleneceği dizinin yolu.",

diff --git a/internal/media/providers/s3/s3.go b/internal/media/providers/s3/s3.go
@@ -2,7 +2,6 @@ package s3
 
 import (
 	"errors"
-	"fmt"
 	"io"
 	"strings"
 	"time"
@@ -11,47 +10,51 @@ import (
 	"github.com/rhnvrm/simples3"
 )
 
-const amznS3PublicURL = "https://%s.s3.%s.amazonaws.com%s"
-
-// Opts represents AWS S3 specific params
-type Opts struct {
+// Opt represents AWS S3 specific params
+type Opt struct {
+	URL        string        `koanf:"url"`
 	AccessKey  string        `koanf:"aws_access_key_id"`
 	SecretKey  string        `koanf:"aws_secret_access_key"`
 	Region     string        `koanf:"aws_default_region"`
 	Bucket     string        `koanf:"bucket"`
 	BucketPath string        `koanf:"bucket_path"`
-	BucketURL  string        `koanf:"bucket_url"`
 	BucketType string        `koanf:"bucket_type"`
 	Expiry     time.Duration `koanf:"expiry"`
 }
 
 // Client implements `media.Store` for S3 provider
 type Client struct {
 	s3   *simples3.S3
-	opts Opts
+	opts Opt
 }
 
 // NewS3Store initialises store for S3 provider. It takes in the AWS configuration
 // and sets up the `simples3` client to interact with AWS APIs for all bucket operations.
-func NewS3Store(opts Opts) (media.Store, error) {
-	var s3svc *simples3.S3
-	var err error
-	if opts.Region == "" {
-		return nil, errors.New("Invalid AWS Region specified. Please check `upload.s3` config")
+func NewS3Store(opt Opt) (media.Store, error) {
+	var (
+		cl  *simples3.S3
+		err error
+	)
+	if opt.URL == "" {
+		return nil, errors.New("Invalid AWS URL in settings.")
 	}
+	opt.URL = strings.TrimRight(opt.URL, "/")
+
 	// Use Access Key/Secret Key if specified in config.
-	if opts.AccessKey != "" && opts.SecretKey != "" {
-		s3svc = simples3.New(opts.Region, opts.AccessKey, opts.SecretKey)
+	if opt.AccessKey != "" && opt.SecretKey != "" {
+		cl = simples3.New(opt.Region, opt.AccessKey, opt.SecretKey)
 	} else {
 		// fallback to IAM role if no access key/secret key is provided.
-		s3svc, err = simples3.NewUsingIAM(opts.Region)
+		cl, err = simples3.NewUsingIAM(opt.Region)
 		if err != nil {
 			return nil, err
 		}
 	}
+	cl.SetEndpoint(opt.URL)
+
 	return &Client{
-		s3:   s3svc,
-		opts: opts,
+		s3:   cl,
+		opts: opt,
 	}, nil
 }
 
@@ -65,7 +68,7 @@ func (c *Client) Put(name string, cType string, file io.ReadSeeker) (string, err
 		Body:        file,
 
 		// Paths inside the bucket should not start with /.
-		ObjectKey: strings.TrimPrefix(makeBucketPath(c.opts.BucketPath, name), "/"),
+		ObjectKey: c.makeBucketPath(name),
 	}
 	// Perform an upload.
 	if _, err := c.s3.FileUpload(upParams); err != nil {
@@ -78,39 +81,42 @@ func (c *Client) Put(name string, cType string, file io.ReadSeeker) (string, err
 func (c *Client) Get(name string) string {
 	// Generate a private S3 pre-signed URL if it's a private bucket.
 	if c.opts.BucketType == "private" {
-		url := c.s3.GeneratePresignedURL(simples3.PresignedInput{
+		u := c.s3.GeneratePresignedURL(simples3.PresignedInput{
 			Bucket:        c.opts.Bucket,
-			ObjectKey:     makeBucketPath(c.opts.BucketPath, name),
+			ObjectKey:     c.makeBucketPath(name),
 			Method:        "GET",
 			Timestamp:     time.Now(),
 			ExpirySeconds: int(c.opts.Expiry.Seconds()),
 		})
-		return url
+		return u
 	}
 
 	// Generate a public S3 URL if it's a public bucket.
-	url := ""
-	if c.opts.BucketURL != "" {
-		url = c.opts.BucketURL + makeBucketPath(c.opts.BucketPath, name)
-	} else {
-		url = fmt.Sprintf(amznS3PublicURL, c.opts.Bucket, c.opts.Region,
-			makeBucketPath(c.opts.BucketPath, name))
-	}
-	return url
+	return c.makeFileURL(name)
 }
 
 // Delete accepts the filename of the object and deletes from S3.
 func (c *Client) Delete(name string) error {
 	err := c.s3.FileDelete(simples3.DeleteInput{
 		Bucket:    c.opts.Bucket,
-		ObjectKey: strings.TrimPrefix(makeBucketPath(c.opts.BucketPath, name), "/"),
+		ObjectKey: c.makeBucketPath(name),
 	})
 	return err
 }
 
-func makeBucketPath(bucketPath string, name string) string {
-	if bucketPath == "/" {
-		return "/" + name
+// makeBucketPath returns the file path inside the bucket. The path should not
+// start with a /.
+func (c *Client) makeBucketPath(name string) string {
+	// If the path is root (/), return the filename without the preceding slash.
+	p := strings.TrimPrefix(strings.TrimSuffix(c.opts.BucketPath, "/"), "/")
+	if p == "" {
+		return name
 	}
-	return fmt.Sprintf("%s/%s", bucketPath, name)
+
+	// whatever/bucket/path/filename.jpg: No preceding slash.
+	return p + "/" + name
+}
+
+func (c *Client) makeFileURL(name string) string {
+	return c.opts.URL + "/" + c.opts.Bucket + "/" + c.makeBucketPath(name)
 }
diff --git a/internal/migrations/v0.7.0.go b/internal/migrations/v0.7.0.go
@@ -85,7 +85,7 @@ func V0_7_0(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf) error {
 		('upload.filesystem.upload_uri', '"/uploads"'),
 		('upload.s3.aws_access_key_id', '""'),
 		('upload.s3.aws_secret_access_key', '""'),
-		('upload.s3.aws_default_region', '"ap-south-b"'),
+		('upload.s3.aws_default_region', '"ap-south-1"'),
 		('upload.s3.bucket', '""'),
 		('upload.s3.bucket_domain', '""'),
 		('upload.s3.bucket_path', '"/"'),

diff --git a/internal/migrations/v2.0.0.go b/internal/migrations/v2.0.0.go
@@ -43,5 +43,17 @@ func V2_0_0(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf) error {
 		return err
 	}
 
+	// S3 URL i snow a settings field. Prepare S3 URL based on region and bucket.
+	if _, err := db.Exec(`
+		WITH region AS (
+			SELECT value#>>'{}' AS value FROM settings WHERE key='upload.s3.aws_default_region'
+		), s3url AS (
+			SELECT FORMAT('https://s3.%s.amazonaws.com', (SELECT value FROM region)) AS value
+		)
+
+		INSERT INTO settings (key, value) VALUES ('upload.s3.url', TO_JSON((SELECT * FROM s3url))) ON CONFLICT DO NOTHING;`); err != nil {
+		return err
+	}
+
 	return nil
 }
diff --git a/schema.sql b/schema.sql
@@ -190,9 +190,10 @@ INSERT INTO settings (key, value) VALUES
     ('upload.provider', '"filesystem"'),
     ('upload.filesystem.upload_path', '"uploads"'),
     ('upload.filesystem.upload_uri', '"/uploads"'),
+    ('upload.s3.url', '"https://ap-south-1.s3.amazonaws.com"'),
     ('upload.s3.aws_access_key_id', '""'),
     ('upload.s3.aws_secret_access_key', '""'),
-    ('upload.s3.aws_default_region', '"ap-south-b"'),
+    ('upload.s3.aws_default_region', '"ap-south-1"'),
     ('upload.s3.bucket', '""'),
     ('upload.s3.bucket_domain', '""'),
     ('upload.s3.bucket_path', '"/"'),