image plugin prototype

.gitignore

@@ -26,3 +26,4 @@ coverage.out
 testdata/acceptance/tmp/
 completions/
 .envrc
+images_impl

cmd/images/images_impl.go

@@ -0,0 +1,195 @@
+package main
+
+import (
+	"errors"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/cueblox/blox/plugins"
+	"github.com/disintegration/imaging"
+	"github.com/h2non/filetype"
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-plugin"
+	"github.com/pterm/pterm"
+	"gopkg.in/yaml.v2"
+)
+
+// Here is a real implementation of Greeter
+type ImageScanner struct {
+	logger hclog.Logger
+}
+
+func (g *ImageScanner) Process() error {
+	pterm.Info.Println("PROCESS()")
+	g.logger.Debug("message from ImageScanner.Process")
+	return g.processImages()
+}
+
+func (g *ImageScanner) processImages() error {
+	// hard coded, need to pass in config? read config?
+	staticDir := "static"
+
+	g.logger.Debug("processing images", "dir", staticDir)
+	fi, err := os.Stat(staticDir)
+	if errors.Is(err, os.ErrNotExist) {
+		g.logger.Debug("no image directory found, skipping")
+		return nil
+	}
+	if !fi.IsDir() {
+		return errors.New("given static directory is not a directory")
+	}
+	imagesDirectory := filepath.Join(staticDir, "images")
+
+	fi, err = os.Stat(imagesDirectory)
+	if errors.Is(err, os.ErrNotExist) {
+		g.logger.Debug("no image directory found, skipping")
+		return nil
+	}
+	if !fi.IsDir() {
+		return errors.New("given images directory is not a directory")
+	}
+	err = filepath.Walk(imagesDirectory,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			g.logger.Debug("Processing", "path", path)
+			if !info.IsDir() {
+				buf, err := ioutil.ReadFile(path)
+				if err != nil {
+					return err
+				}
+				if filetype.IsImage(buf) {
+
+					src, err := imaging.Open(path)
+					if err != nil {
+						return err
+					}
+
+					relpath, err := filepath.Rel(staticDir, path)
+					if err != nil {
+						return err
+					}
+					g.logger.Debug("File is an image", "path", relpath)
+					kind, err := filetype.Match(buf)
+					if err != nil {
+						return err
+					}
+					g.logger.Debug("\t\tFile type: %s. MIME: %s\n", kind.Extension, kind.MIME.Value)
+					if err != nil {
+						return err
+					}
+					/*	if cloud {
+							pterm.Info.Println("Synchronizing images to cloud provider")
+							bucketURL := os.Getenv("IMAGE_BUCKET")
+							if bucketURL == "" {
+								return errors.New("image sync enabled (-s,--sync), but no IMAGE_BUCKET environment variable set")
+							}
+
+							ctx := context.Background()
+							// Open a connection to the bucket.
+							b, err := blob.OpenBucket(ctx, bucketURL)
+							if err != nil {
+								return fmt.Errorf("failed to setup bucket: %s", err)
+							}
+							defer b.Close()
+
+							w, err := b.NewWriter(ctx, relpath, nil)
+							if err != nil {
+								return fmt.Errorf("cloud sync failed to obtain writer: %s", err)
+							}
+							_, err = w.Write(buf)
+							if err != nil {
+								return fmt.Errorf("cloud sync failed to write to bucket: %s", err)
+							}
+							if err = w.Close(); err != nil {
+								return fmt.Errorf("cloud sync failed to close: %s", err)
+							}
+						}
+					*/
+					cdnEndpoint := os.Getenv("CDN_URL")
+
+					bi := &BloxImage{
+						FileName: relpath,
+						Height:   src.Bounds().Dy(),
+						Width:    src.Bounds().Dx(),
+						CDN:      cdnEndpoint,
+					}
+					bytes, err := yaml.Marshal(bi)
+					if err != nil {
+						return err
+					}
+					dataDir := "data"
+
+					ext := strings.TrimPrefix(filepath.Ext(relpath), ".")
+					slug := strings.TrimSuffix(relpath, "."+ext)
+
+					outputPath := filepath.Join(dataDir, slug+".yaml")
+					err = os.MkdirAll(filepath.Dir(outputPath), 0o755)
+					if err != nil {
+						pterm.Error.Println(err)
+						return err
+					}
+					// only write the yaml file if it doesn't exist.
+					// don't overwrite existing records.
+					_, err = os.Stat(outputPath)
+					if err != nil && errors.Is(err, os.ErrNotExist) {
+						err = os.WriteFile(outputPath, bytes, 0o755)
+						if err != nil {
+							g.logger.Debug(err.Error())
+							return err
+						}
+					}
+				} else {
+					g.logger.Debug("File is not an image",
+						"path", path)
+				}
+			}
+
+			return nil
+		})
+	return err
+}
+
+type BloxImage struct {
+	FileName string `yaml:"file_name"`
+	Height   int    `yaml:"height"`
+	Width    int    `yaml:"width"`
+	CDN      string `yaml:"cdn"`
+}
+
+// handshakeConfigs are used to just do a basic handshake between
+// a plugin and host. If the handshake fails, a user friendly error is shown.
+// This prevents users from executing bad plugins or executing a plugin
+// directory. It is a UX feature, not a security feature.
+var handshakeConfig = plugin.HandshakeConfig{
+	ProtocolVersion:  1,
+	MagicCookieKey:   "BLOX_PLUGIN",
+	MagicCookieValue: "image",
+}
+
+func main() {
+	logger := hclog.New(&hclog.LoggerOptions{
+		Level:      hclog.Trace,
+		Output:     os.Stderr,
+		JSONFormat: true,
+	})
+
+	imageScanner := &ImageScanner{
+		logger: logger,
+	}
+	// pluginMap is the map of plugins we can dispense.
+	var pluginMap = map[string]plugin.Plugin{
+		"images": &plugins.PrebuildPlugin{Impl: imageScanner},
+	}
+
+	logger.Debug("message from plugin", "hello", "images")
+
+	plugin.Serve(&plugin.ServeConfig{
+		HandshakeConfig: handshakeConfig,
+		Plugins:         pluginMap,
+	})
+}

content/repository.go

@@ -6,8 +6,10 @@ import (
 	"fmt"
 	"io/fs"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"os"
+	"os/exec"
 	"path"
 	"path/filepath"
 	"sort"
@@ -23,8 +25,11 @@ import (
 	"github.com/cueblox/blox/internal/cuedb"
 	"github.com/cueblox/blox/internal/encoding/markdown"
 	"github.com/cueblox/blox/internal/repository"
+	"github.com/cueblox/blox/plugins"
 	"github.com/goccy/go-yaml"
+	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-plugin"
 	"github.com/pterm/pterm"
 )
 
@@ -552,7 +557,7 @@ func (s *Service) parseRemotes() error {
 // when it finds an image, it reads the image metadata
 // and saves a corresponding YAML file describing the image
 // in the 'images' data directory.
-func (s *Service) processImages() error {
+func (s *Service) oldProcessImages() error {
 	staticDir, err := s.Cfg.GetString("static_dir")
 	if err != nil {
 		pterm.Info.Printf("no static directory present, skipping image linking")
@@ -684,6 +689,46 @@ func (s *Service) processImages() error {
 	return err
 }
 
+// processImages scans the static dir for images
+// when it finds an image, it reads the image metadata
+// and saves a corresponding YAML file describing the image
+// in the 'images' data directory.
+func (s *Service) processImages() error {
+	pterm.Info.Println("calling the plugin")
+	// Create an hclog.Logger
+	logger := hclog.New(&hclog.LoggerOptions{
+		Name:   "plugin",
+		Output: os.Stdout,
+		Level:  hclog.Debug,
+	})
+
+	// We're a host! Start by launching the plugin process.
+	client := plugin.NewClient(&plugin.ClientConfig{
+		HandshakeConfig: handshakeConfig,
+		Plugins:         pluginMap,
+		Cmd:             exec.Command("../images_impl"),
+		Logger:          logger,
+	})
+	defer client.Kill()
+
+	// Connect via RPC
+	rpcClient, err := client.Client()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Request the plugin
+	raw, err := rpcClient.Dispense("images")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// We should have a Greeter now! This feels like a normal interface
+	// implementation but is in fact over an RPC connection.
+	imgs := raw.(plugins.Prebuild)
+	return imgs.Process()
+}
+
 // ensureRemote downloads a remote schema at a specific
 // version if it doesn't exist locally
 func (s *Service) ensureRemote(name, version, repo string) error {
@@ -747,3 +792,18 @@ type BloxImage struct {
 	Width    int    `yaml:"width"`
 	CDN      string `yaml:"cdn"`
 }
+
+// handshakeConfigs are used to just do a basic handshake between
+// a plugin and host. If the handshake fails, a user friendly error is shown.
+// This prevents users from executing bad plugins or executing a plugin
+// directory. It is a UX feature, not a security feature.
+var handshakeConfig = plugin.HandshakeConfig{
+	ProtocolVersion:  1,
+	MagicCookieKey:   "BLOX_PLUGIN",
+	MagicCookieValue: "image",
+}
+
+// pluginMap is the map of plugins we can dispense.
+var pluginMap = map[string]plugin.Plugin{
+	"images": &plugins.PrebuildPlugin{},
+}

dogfood/data/images/jpg/main-img-preview.yaml

@@ -1,3 +1,4 @@
 file_name: images/jpg/main-img-preview.jpg
 height: 1188
 width: 2272
+cdn: ""

dogfood/data/images/png/bketelsens-report.yaml

@@ -1,3 +1,4 @@
 file_name: images/png/bketelsens-report.png
 height: 700
 width: 600
+cdn: ""

dogfood/schemata/image_v1.cue

@@ -17,6 +17,7 @@
 		caption?:			string
 		attribution?:		string
 		attribution_link?:	string
+		cdn?: 				string
 	}
 
 }

go.mod

@@ -12,7 +12,9 @@ require (
 	github.com/graphql-go/graphql v0.7.9
 	github.com/graphql-go/handler v0.2.3
 	github.com/h2non/filetype v1.1.1
+	github.com/hashicorp/go-hclog v0.16.2 // indirect
 	github.com/hashicorp/go-multierror v1.0.0
+	github.com/hashicorp/go-plugin v1.4.2 // indirect
 	github.com/heimdalr/dag v1.0.1
 	github.com/otiai10/copy v1.5.0
 	github.com/pterm/pterm v0.12.13