cli: manage seed migrations as sql files (close hasura#2431) (hasura#…

…3763)

cli/cli.go

@@ -52,6 +52,10 @@ const (
 
 	// Name of the cli extension plugin
 	CLIExtPluginName = "cli-ext"
+
+	DefaultMigrationsDirectory = "migrations"
+	DefaultMetadataDirectory   = "metadata"
+	DefaultSeedsDirectory      = "seeds"
 )
 
 const (
@@ -290,6 +294,8 @@ type Config struct {
 	MetadataDirectory string `yaml:"metadata_directory,omitempty"`
 	// MigrationsDirectory defines the directory where the migration files were stored.
 	MigrationsDirectory string `yaml:"migrations_directory,omitempty"`
+	// SeedsDirectory defines the directory where seed files will be stored
+	SeedsDirectory string `yaml:"seeds_directory,omitempty"`
 	// ActionConfig defines the config required to create or generate codegen for an action.
 	ActionConfig *types.ActionExecutionConfig `yaml:"actions,omitempty"`
 }
@@ -322,6 +328,8 @@ type ExecutionContext struct {
 	MigrationDir string
 	// MetadataDir is the name of directory where metadata files are stored.
 	MetadataDir string
+	// Seed directory -- directory in which seed files are to be stored
+	SeedsDirectory string
 	// ConfigFile is the file where endpoint etc. are stored.
 	ConfigFile string
 	// HGE Headers, are the custom headers which can be passed to HGE API
@@ -548,6 +556,14 @@ func (ec *ExecutionContext) Validate() error {
 		}
 	}
 
+	ec.SeedsDirectory = filepath.Join(ec.ExecutionDirectory, ec.Config.SeedsDirectory)
+	if _, err := os.Stat(ec.SeedsDirectory); os.IsNotExist(err) {
+		err = os.MkdirAll(ec.SeedsDirectory, os.ModePerm)
+		if err != nil {
+			return errors.Wrap(err, "cannot create seeds directory")
+		}
+	}
+
 	if ec.Config.Version == V2 && ec.Config.MetadataDirectory != "" {
 		// set name of metadata directory
 		ec.MetadataDir = filepath.Join(ec.ExecutionDirectory, ec.Config.MetadataDirectory)
@@ -638,7 +654,8 @@ func (ec *ExecutionContext) readConfig() error {
 	v.SetDefault("api_paths.pg_dump", "v1alpha1/pg_dump")
 	v.SetDefault("api_paths.version", "v1/version")
 	v.SetDefault("metadata_directory", "")
-	v.SetDefault("migrations_directory", "migrations")
+	v.SetDefault("migrations_directory", DefaultMigrationsDirectory)
+	v.SetDefault("seeds_directory", DefaultSeedsDirectory)
 	v.SetDefault("actions.kind", "synchronous")
 	v.SetDefault("actions.handler_webhook_baseurl", "http://localhost:3000")
 	v.SetDefault("actions.codegen.framework", "")
@@ -671,6 +688,7 @@ func (ec *ExecutionContext) readConfig() error {
 		},
 		MetadataDirectory:   v.GetString("metadata_directory"),
 		MigrationsDirectory: v.GetString("migrations_directory"),
+		SeedsDirectory:      v.GetString("seeds_directory"),
 		ActionConfig: &types.ActionExecutionConfig{
 			Kind:                  v.GetString("actions.kind"),
 			HandlerWebhookBaseURL: v.GetString("actions.handler_webhook_baseurl"),

cli/commands/help.go

@@ -49,6 +49,7 @@ func (o *helpOptions) run() {
 				NewMetadataCmd(o.EC),
 				NewConsoleCmd(o.EC),
 				NewActionsCmd(o.EC),
+				NewSeedCmd(o.EC),
 			},
 		},
 		{

cli/commands/init.go

@@ -230,15 +230,15 @@ func (o *InitOptions) createFiles() error {
 	}
 
 	// create migrations directory
-	o.EC.MigrationDir = filepath.Join(o.EC.ExecutionDirectory, "migrations")
+	o.EC.MigrationDir = filepath.Join(o.EC.ExecutionDirectory, cli.DefaultMigrationsDirectory)
 	err = os.MkdirAll(o.EC.MigrationDir, os.ModePerm)
 	if err != nil {
 		return errors.Wrap(err, "cannot write migration directory")
 	}
 
 	if config.Version == cli.V2 {
 		// create metadata directory
-		o.EC.MetadataDir = filepath.Join(o.EC.ExecutionDirectory, "metadata")
+		o.EC.MetadataDir = filepath.Join(o.EC.ExecutionDirectory, cli.DefaultMetadataDirectory)
 		err = os.MkdirAll(o.EC.MetadataDir, os.ModePerm)
 		if err != nil {
 			return errors.Wrap(err, "cannot write migration directory")
@@ -261,6 +261,13 @@ func (o *InitOptions) createFiles() error {
 			}
 		}
 	}
+
+	// create seeds directory
+	o.EC.SeedsDirectory = filepath.Join(o.EC.ExecutionDirectory, cli.DefaultSeedsDirectory)
+	err = os.MkdirAll(o.EC.SeedsDirectory, os.ModePerm)
+	if err != nil {
+		return errors.Wrap(err, "cannot write seeds directory")
+	}
 	return nil
 }
 

cli/commands/root.go

@@ -63,6 +63,7 @@ func init() {
 		NewConsoleCmd(ec),
 		NewMetadataCmd(ec),
 		NewMigrateCmd(ec),
+		NewSeedCmd(ec),
 		NewActionsCmd(ec),
 		NewPluginsCmd(ec),
 		NewVersionCmd(ec),

cli/commands/seed.go

@@ -0,0 +1,41 @@
+package commands
+
+import (
+	"github.com/hasura/graphql-engine/cli"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+// NewSeedCmd will return the seed command
+func NewSeedCmd(ec *cli.ExecutionContext) *cobra.Command {
+	v := viper.New()
+	ec.Viper = v
+	seedCmd := &cobra.Command{
+		Use:          "seeds",
+		Aliases:      []string{"sd"},
+		Short:        "Manage seed data",
+		SilenceUsage: true,
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			err := ec.Prepare()
+			if err != nil {
+				return err
+			}
+			return ec.Validate()
+		},
+	}
+
+	seedCmd.AddCommand(
+		newSeedCreateCmd(ec),
+		newSeedApplyCmd(ec),
+	)
+	seedCmd.PersistentFlags().String("endpoint", "", "http(s) endpoint for Hasura GraphQL Engine")
+	seedCmd.PersistentFlags().String("admin-secret", "", "admin secret for Hasura GraphQL Engine")
+	seedCmd.PersistentFlags().String("access-key", "", "access key for Hasura GraphQL Engine")
+	seedCmd.PersistentFlags().MarkDeprecated("access-key", "use --admin-secret instead")
+
+	v.BindPFlag("endpoint", seedCmd.PersistentFlags().Lookup("endpoint"))
+	v.BindPFlag("admin_secret", seedCmd.PersistentFlags().Lookup("admin-secret"))
+	v.BindPFlag("access_key", seedCmd.PersistentFlags().Lookup("access-key"))
+
+	return seedCmd
+}

cli/commands/seed_apply.go

@@ -0,0 +1,57 @@
+package commands
+
+import (
+	"github.com/hasura/graphql-engine/cli/migrate"
+	"github.com/spf13/afero"
+	"github.com/spf13/cobra"
+
+	"github.com/hasura/graphql-engine/cli"
+	"github.com/hasura/graphql-engine/cli/seed"
+)
+
+type SeedApplyOptions struct {
+	EC *cli.ExecutionContext
+
+	// seed file to apply
+	FileNames []string
+}
+
+func newSeedApplyCmd(ec *cli.ExecutionContext) *cobra.Command {
+	opts := SeedApplyOptions{
+		EC: ec,
+	}
+	cmd := &cobra.Command{
+		Use:   "apply",
+		Short: "Apply seed data",
+		Example: `  # Apply all seeds on the database:
+  hasura seed apply
+
+  # Apply only a particular file:
+  hasura seed apply --file seeds/1234_add_some_seed_data.sql`,
+		SilenceUsage: false,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			return ec.Validate()
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			opts.EC.Spin("Applying seeds...")
+			err := opts.Run()
+			opts.EC.Spinner.Stop()
+			if err != nil {
+				return err
+			}
+			opts.EC.Logger.Info("Seeds planted")
+			return nil
+		},
+	}
+	cmd.Flags().StringArrayVarP(&opts.FileNames, "file", "f", []string{}, "seed file to apply")
+	return cmd
+}
+
+func (o *SeedApplyOptions) Run() error {
+	migrateDriver, err := migrate.NewMigrate(o.EC, true)
+	if err != nil {
+		return err
+	}
+	fs := afero.NewOsFs()
+	return seed.ApplySeedsToDatabase(o.EC, fs, migrateDriver, o.FileNames)
+}

cli/commands/seed_create.go

@@ -0,0 +1,104 @@
+package commands
+
+import (
+	"bytes"
+
+	"github.com/hasura/graphql-engine/cli/migrate"
+	"github.com/pkg/errors"
+	"github.com/spf13/afero"
+	"github.com/spf13/cobra"
+
+	"github.com/hasura/graphql-engine/cli"
+	"github.com/hasura/graphql-engine/cli/metadata/actions/editor"
+	"github.com/hasura/graphql-engine/cli/seed"
+)
+
+type SeedNewOptions struct {
+	EC *cli.ExecutionContext
+
+	// filename for the new seed file
+	SeedName string
+	// table name if seed file has to be created from a database table
+	FromTableNames []string
+
+	// seed file that was created
+	FilePath string
+}
+
+func newSeedCreateCmd(ec *cli.ExecutionContext) *cobra.Command {
+	opts := SeedNewOptions{
+		EC: ec,
+	}
+	cmd := &cobra.Command{
+		Use:   "create seed_name",
+		Short: "create a new seed file",
+		Example: `  # Create a new seed file and use editor to add SQL:
+  hasura seed create new_table_seed
+
+  # Create a new seed by exporting data from tables already present in the database:
+  hasura seed create table1_seed --from-table table1
+
+  # Export data from multiple tables:
+  hasura seed create tables_seed --from-table table1 --from-table table2`,
+		Args:         cobra.ExactArgs(1),
+		SilenceUsage: false,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			return ec.Validate()
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			opts.SeedName = args[0]
+			err := opts.Run()
+			if err != nil {
+				return err
+			}
+			ec.Logger.WithField("file", opts.FilePath).Info("created seed file successfully")
+			return nil
+		},
+	}
+
+	cmd.Flags().StringArrayVar(&opts.FromTableNames, "from-table", []string{}, "name of table from which seed file has to be initialized")
+
+	return cmd
+}
+
+func (o *SeedNewOptions) Run() error {
+	createSeedOpts := seed.CreateSeedOpts{
+		UserProvidedSeedName: o.SeedName,
+		DirectoryPath:        o.EC.SeedsDirectory,
+	}
+
+	// If we are initializing from a database table
+	// create a hasura client and add table name opts
+	if createSeedOpts.Data == nil {
+		if len(o.FromTableNames) > 0 {
+			migrateDriver, err := migrate.NewMigrate(ec, true)
+			if err != nil {
+				return errors.Wrap(err, "cannot initialize migrate driver")
+			}
+			// Send the query
+			body, err := migrateDriver.ExportDataDump(o.FromTableNames)
+			if err != nil {
+				return errors.Wrap(err, "exporting seed data")
+			}
+
+			createSeedOpts.Data = bytes.NewReader(body)
+		} else {
+			const defaultText = ""
+			data, err := editor.CaptureInputFromEditor(editor.GetPreferredEditorFromEnvironment, defaultText, "*.sql")
+			if err != nil {
+				return errors.Wrap(err, "cannot find default editor from env")
+			}
+			createSeedOpts.Data = bytes.NewReader(data)
+		}
+	}
+
+	fs := afero.NewOsFs()
+	filepath, err := seed.CreateSeedFile(fs, createSeedOpts)
+	if err != nil || filepath == nil {
+		return errors.Wrap(err, "failed to create seed file")
+	}
+
+	o.FilePath = *filepath
+
+	return nil
+}

cli/go.sum

@@ -260,12 +260,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
+github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=

cli/integration_test/cli_test.go

@@ -132,6 +132,16 @@ func TestCommands(t *testing.T) {
 		t.Run("metadata commands", func(t *testing.T) {
 			v2.TestMetadataCmd(t, ec)
 		})
+
+		skip(t)
+		t.Run("seed create command", func(t *testing.T) {
+			v2.TestSeedsCreateCmd(t, ec)
+		})
+
+		skip(t)
+		t.Run("seed apply commands", func(t *testing.T) {
+			v2.TestSeedsApplyCmd(t, ec)
+		})
 	})
 }
 

cli/integration_test/v1/seeds/1591867862409_test.sql

@@ -0,0 +1,8 @@
+CREATE TABLE IF NOT EXISTS  account(
+	user_id serial PRIMARY KEY,
+	username VARCHAR (50) UNIQUE NOT NULL,
+	password VARCHAR (50) NOT NULL,
+	email VARCHAR (355) UNIQUE NOT NULL,
+	created_on TIMESTAMP NOT NULL,
+	last_login TIMESTAMP
+);

cli/integration_test/v1/seeds/1591867862419_test2.sql

@@ -0,0 +1,8 @@
+CREATE TABLE  account2(
+	user_id serial PRIMARY KEY,
+	username VARCHAR (50) UNIQUE NOT NULL,
+	password VARCHAR (50) NOT NULL,
+	email VARCHAR (355) UNIQUE NOT NULL,
+	created_on TIMESTAMP NOT NULL,
+	last_login TIMESTAMP
+);