diff --git a/internal/cmd/fmt.go b/internal/cmd/fmt.go index 6a7e2787..bd5b744c 100644 --- a/internal/cmd/fmt.go +++ b/internal/cmd/fmt.go @@ -6,71 +6,95 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" - "github.com/stateful/runme/v3/pkg/document/identity" + idt "github.com/stateful/runme/v3/pkg/document/identity" "github.com/stateful/runme/v3/pkg/project" ) -func fmtCmd() *cobra.Command { - var ( - flatten bool - formatJSON bool - identityStr string - write bool - ) +var ( + flatten bool + formatJSON bool + identityStr string + write bool +) - cmd := cobra.Command{ - Use: "fmt", - Short: "Format a Markdown file into canonical format", - Args: cobra.MaximumNArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - if formatJSON { - if write { - return errors.New("invalid usage of --json with --write") - } - if !flatten { - return errors.New("invalid usage of --json without --flatten") - } +func buildFmtCmd(cmd *cobra.Command, reset bool) *cobra.Command { + cmd.RunE = func(cmd *cobra.Command, args []string) error { + if formatJSON { + if write { + return errors.New("invalid usage of --json with --write") } + if !flatten { + return errors.New("invalid usage of --json without --flatten") + } + } - files := args + files := args - if len(args) == 0 { - var err error - files, err = getProjectFiles(cmd) - if err != nil { - return err - } + if len(args) == 0 { + var err error + files, err = getProjectFiles(cmd) + if err != nil { + return err } + } - var identityResolver *identity.IdentityResolver - switch strings.ToLower(identityStr) { - case "all": - identityResolver = identity.NewResolver(identity.AllLifecycleIdentity) - case "doc", "document": - identityResolver = identity.NewResolver(identity.DocumentLifecycleIdentity) - case "cell": - identityResolver = identity.NewResolver(identity.CellLifecycleIdentity) - default: - identityResolver = identity.NewResolver(identity.DefaultLifecycleIdentity) - } + identityResolver := idt.NewResolver(idt.UnspecifiedLifecycleIdentity) + if reset { + identityResolver = strToIdentityResolver(identityStr) + } - return project.FormatFiles(files, identityResolver, formatJSON, write, func(file string, formatted []byte) error { + return project.FormatFiles(files, &project.FormatOptions{ + FormatJSON: formatJSON, + IdentityResolver: identityResolver, + Outputter: func(file string, formatted []byte) error { out := cmd.OutOrStdout() _, _ = fmt.Fprintf(out, "===== %s =====\n", file) _, _ = out.Write(formatted) _, _ = fmt.Fprint(out, "\n") return nil - }) - }, + }, + Reset: reset, + Write: write, + }) } - - setDefaultFlags(&cmd) + setDefaultFlags(cmd) cmd.Flags().BoolVar(&flatten, "flatten", true, "Flatten nested blocks in the output. WARNING: This can currently break frontmatter if turned off.") cmd.Flags().BoolVar(&formatJSON, "json", false, "Print out data as JSON. Only possible with --flatten and not allowed with --write.") cmd.Flags().BoolVarP(&write, "write", "w", false, "Write result to the source file instead of stdout.") - cmd.Flags().StringVar(&identityStr, "identity", "", "Set the lifecycle identity. Overrides the default.") + cmd.Flags().StringVar(&identityStr, "identity", "", "Set the lifecycle identity, \"doc\", \"cell\", \"all\", or \"\" (default).") _ = cmd.Flags().MarkDeprecated("flatten", "This flag is now default and no longer has any other effect.") - return &cmd + return cmd +} + +func fmtCmd() *cobra.Command { + cmd := buildFmtCmd(&cobra.Command{ + Use: "fmt", + Short: "Format a Markdown file into canonical format", + Args: cobra.MaximumNArgs(1), + }, false) + + cmd.AddCommand(buildFmtCmd(&cobra.Command{ + Use: "reset", + Short: "Format a Markdown file and reset all lifecycle metadata", + Args: cobra.MaximumNArgs(1), + }, true)) + + return cmd +} + +func strToIdentityResolver(identity string) *idt.IdentityResolver { + var identityResolver *idt.IdentityResolver + switch strings.ToLower(identity) { + case "all": + identityResolver = idt.NewResolver(idt.AllLifecycleIdentity) + case "doc", "document": + identityResolver = idt.NewResolver(idt.DocumentLifecycleIdentity) + case "cell": + identityResolver = idt.NewResolver(idt.CellLifecycleIdentity) + default: + identityResolver = idt.NewResolver(idt.DefaultLifecycleIdentity) + } + return identityResolver } diff --git a/pkg/document/document.go b/pkg/document/document.go index 6f5b3068..5aba49b9 100644 --- a/pkg/document/document.go +++ b/pkg/document/document.go @@ -120,7 +120,7 @@ func (d *Document) FrontmatterRaw() []byte { return d.frontmatterRaw } -// splitSource splits source into FrontMatter and content. +// splitSource splits source into Frontmatter and content. // TODO(adamb): replace it with an extension to goldmark. // Example: https://github.com/abhinav/goldmark-frontmatter func (d *Document) splitSource() { @@ -131,7 +131,7 @@ func (d *Document) splitSource() { for _, item := range l.items { switch item.Type() { - case parsedItemFrontMatter: + case parsedItemFrontmatter: d.frontmatterRaw = item.Value(d.source) case parsedItemContent: d.content = item.Value(d.source) diff --git a/pkg/document/editor/editor.go b/pkg/document/editor/editor.go index 7a1d0409..c2c0a05e 100644 --- a/pkg/document/editor/editor.go +++ b/pkg/document/editor/editor.go @@ -22,6 +22,7 @@ const ( type Options struct { IdentityResolver *identity.IdentityResolver LoggerInstance *zap.Logger + Reset bool } func (o Options) Logger() *zap.Logger { @@ -41,8 +42,16 @@ func Deserialize(data []byte, opts Options) (*Notebook, error) { frontmatter, fmErr := doc.FrontmatterWithError() // non-fatal error - if fmErr != nil { + switch { + case fmErr != nil: opts.Logger().Warn("failed to parse frontmatter", zap.Error(fmErr)) + case opts.Reset: + // reset Runme part of frontmatter if required + f, err := frontmatter.ResetRunme(opts.IdentityResolver.DocumentEnabled()) + if err != nil { + return nil, err + } + frontmatter = f } cacheID := opts.IdentityResolver.CacheID() @@ -69,36 +78,45 @@ func Deserialize(data []byte, opts Options) (*Notebook, error) { // todo(sebastian): faux document-level `runme.dev/id` to not break existing clients; revert in near future notebook.Metadata[PrefixAttributeName(InternalAttributePrefix, "id")] = cacheID - // Make ephemeral cell IDs permanent if the cell lifecycle ID is enabled. - if opts.IdentityResolver.CellEnabled() { - err := applyCellLifecycleIdentity(notebook) - if err != nil { - return nil, err - } + // apply lifecycle identity to cells; reset first if required + if err := applyCellLifecycleIdentity(notebook, &opts); err != nil { + return nil, err } return notebook, nil } -func applyCellLifecycleIdentity(notebook *Notebook) error { +func applyCellLifecycleIdentity(notebook *Notebook, opts *Options) error { + if opts == nil { + return nil + } + ephCellIDKey := PrefixAttributeName(InternalAttributePrefix, CellID) + for _, cell := range notebook.Cells { if cell.Kind != CodeKind { continue } - // don't overwrite existing cell ID - if _, ok := cell.Metadata["id"]; ok { - continue + if opts.Reset { + delete(cell.Metadata, CellID) } - // make sure we have an ephemeral cell ID - if _, ok := cell.Metadata[ephCellIDKey]; !ok { - return errors.Errorf("missing ephemeral cell ID") - } + if opts.IdentityResolver.CellEnabled() { + // don't overwrite existing cell ID + if _, ok := cell.Metadata["id"]; ok { + continue + } - cell.Metadata[CellID] = cell.Metadata[ephCellIDKey] + // make sure we have an ephemeral cell ID + if _, ok := cell.Metadata[ephCellIDKey]; !ok { + return errors.Errorf("missing ephemeral cell ID") + } + + cell.Metadata[CellID] = cell.Metadata[ephCellIDKey] + } } + return nil } diff --git a/pkg/document/editor/editor_test.go b/pkg/document/editor/editor_test.go index d372f931..64c5754b 100644 --- a/pkg/document/editor/editor_test.go +++ b/pkg/document/editor/editor_test.go @@ -21,6 +21,7 @@ var ( identityResolverNone = identity.NewResolver(identity.UnspecifiedLifecycleIdentity) identityResolverAll = identity.NewResolver(identity.AllLifecycleIdentity) identityResolverCell = identity.NewResolver(identity.CellLifecycleIdentity) + identityResolverDoc = identity.NewResolver(identity.DocumentLifecycleIdentity) testMockID = ulid.GenerateID() ) @@ -383,3 +384,212 @@ This will test final line breaks`) ) }) } + +func TestEditor_ResetIdentity(t *testing.T) { + codeCell := "```sh {\"id\":\"abcdefg\"}\necho 1\n```" + runmeYamlFm := fmt.Sprintf(`runme: + id: %s + version: %s`, testMockID, version.BaseVersion()) + data := []byte(fmt.Sprintf(`--- +prop1: val1 +prop2: val2 +%s +--- + +# Example + +A paragraph + +%s +`, runmeYamlFm, codeCell)) + + t.Run("WithoutResetNoIdentity", func(t *testing.T) { + notebook, err := Deserialize(data, Options{IdentityResolver: identityResolverNone, Reset: false}) + require.NoError(t, err) + result, err := Serialize(notebook, nil, Options{}) + require.NoError(t, err) + assert.Equal( + t, + string(data), + string(result), + ) + }) + + t.Run("WithResetNoIdentity", func(t *testing.T) { + notebook, err := Deserialize(data, Options{IdentityResolver: identityResolverNone, Reset: true}) + require.NoError(t, err) + + // notebook-level + require.NotNil(t, notebook.Metadata["runme.dev/cache-id"]) + require.Empty(t, notebook.Metadata["id"]) + require.EqualValues(t, "---\nprop1: val1\nprop2: val2\n---", notebook.Metadata["runme.dev/frontmatter"]) + + // cell-level + cell := notebook.Cells[2] + require.EqualValues(t, 2, cell.Kind) + require.EqualValues(t, "echo 1", cell.Value) + require.EqualValues(t, "sh", cell.LanguageID) + require.Empty(t, cell.Metadata["id"]) + }) + + t.Run("WithResetWithIdentity", func(t *testing.T) { + notebook, err := Deserialize(data, Options{IdentityResolver: identityResolverDoc, Reset: true}) + require.NoError(t, err) + + // notebook-level + require.NotNil(t, notebook.Metadata["runme.dev/cache-id"]) + require.Empty(t, notebook.Metadata["id"]) + expected := fmt.Sprintf("---\nprop1: val1\nprop2: val2\nrunme:\n id: %s\n version: %s\n---", testMockID, version.BaseVersion()) + require.EqualValues(t, expected, notebook.Metadata["runme.dev/frontmatter"]) + + // cell-level + cell := notebook.Cells[2] + require.EqualValues(t, 2, cell.Kind) + require.EqualValues(t, "echo 1", cell.Value) + require.EqualValues(t, "sh", cell.LanguageID) + require.Empty(t, cell.Metadata["id"]) + }) +} + +func TestEditor_ResetRunmeFrontmatterYAML(t *testing.T) { + t.Run("RetainUnrelated", func(t *testing.T) { + data := []byte(`--- +runme: + id: 01HFVTDYA775K2HREH9ZGQJ75B + version: v3 +unrelated: frontmatter +--- + +# Example + +A paragraph +`) + notebook, err := Deserialize(data, Options{IdentityResolver: identityResolverNone, Reset: true}) + require.NoError(t, err) + result, err := Serialize(notebook, nil, Options{}) + require.NoError(t, err) + assert.Equal( + t, + "---\nunrelated: frontmatter\n---\n\n# Example\n\nA paragraph\n", + string(result), + ) + }) + + t.Run("RemoveEmpty", func(t *testing.T) { + data := []byte(`--- +runme: + id: 01HFVTDYA775K2HREH9ZGQJ75B + version: v3 +--- + +# Example + +A paragraph +`) + notebook, err := Deserialize(data, Options{IdentityResolver: identityResolverNone, Reset: true}) + require.NoError(t, err) + result, err := Serialize(notebook, nil, Options{}) + require.NoError(t, err) + assert.Equal( + t, + "# Example\n\nA paragraph\n", + string(result), + ) + }) +} + +func TestEditor_ResetRunmeFrontmatterJSON(t *testing.T) { + t.Run("RetainUnrelated", func(t *testing.T) { + data := []byte(`{ + "runme": { + "id": "01HF7YYYYYYYYYYYYMQ2KEEYGM", + "version": "v3" + }, + "unrelated": "frontmatter" +} + +# Example + +A paragraph +`) + notebook, err := Deserialize(data, Options{IdentityResolver: identityResolverNone, Reset: true}) + require.NoError(t, err) + result, err := Serialize(notebook, nil, Options{}) + require.NoError(t, err) + assert.Equal( + t, + "{\n \"unrelated\": \"frontmatter\"\n}\n\n# Example\n\nA paragraph\n", + string(result), + ) + }) + + t.Run("RemoveEmpty", func(t *testing.T) { + data := []byte(`{ + "runme": { + "id": "01HF7YYYYYYYYYYYYMQ2KEEYGM", + "version": "v3" + } +} + +# Example + +A paragraph +`) + notebook, err := Deserialize(data, Options{IdentityResolver: identityResolverNone, Reset: true}) + require.NoError(t, err) + result, err := Serialize(notebook, nil, Options{}) + require.NoError(t, err) + assert.Equal( + t, + "# Example\n\nA paragraph\n", + string(result), + ) + }) +} + +func TestEditor_ResetRunmeFrontmatterTOML(t *testing.T) { + t.Run("RetainUnrelated", func(t *testing.T) { + data := []byte(`+++ +unrelated = 'frontmatter' +[runme] +id = '01HRA297WC2XXXXXX8FM3DR1V0' +version = 'v3' ++++ + +# Example + +A paragraph +`) + notebook, err := Deserialize(data, Options{IdentityResolver: identityResolverNone, Reset: true}) + require.NoError(t, err) + result, err := Serialize(notebook, nil, Options{}) + require.NoError(t, err) + assert.Equal( + t, + "+++\nunrelated = 'frontmatter'\n+++\n\n# Example\n\nA paragraph\n", + string(result), + ) + }) + + t.Run("RemoveEmpty", func(t *testing.T) { + data := []byte(`+++ +[runme] +id = '01HRA297WC2XXXXXX8FM3DR1V0' +version = 'v3' ++++ + +# Example + +A paragraph +`) + notebook, err := Deserialize(data, Options{IdentityResolver: identityResolverNone, Reset: true}) + require.NoError(t, err) + result, err := Serialize(notebook, nil, Options{}) + require.NoError(t, err) + assert.Equal( + t, + "# Example\n\nA paragraph\n", + string(result), + ) + }) +} diff --git a/pkg/document/editor/editorservice/service.go b/pkg/document/editor/editorservice/service.go index bf189d91..096f0b31 100644 --- a/pkg/document/editor/editorservice/service.go +++ b/pkg/document/editor/editorservice/service.go @@ -27,7 +27,7 @@ func (s *parserServiceServer) Deserialize(_ context.Context, req *parserv1.Deser s.logger.Info("Deserialize", zap.ByteString("source", req.Source[:min(len(req.Source), 64)])) identityResolver := identity.NewResolver(fromProtoDeserializeReqOptionsToLifecycleIdentity(req.Options)) - notebook, err := editor.Deserialize(req.Source, editor.Options{LoggerInstance: s.logger, IdentityResolver: identityResolver}) + notebook, err := editor.Deserialize(req.Source, editor.Options{LoggerInstance: s.logger, IdentityResolver: identityResolver, Reset: false}) if err != nil { s.logger.Info("failed to call Deserialize", zap.Error(err)) return nil, err diff --git a/pkg/document/frontmatter.go b/pkg/document/frontmatter.go index bbd7cfc8..10af9080 100644 --- a/pkg/document/frontmatter.go +++ b/pkg/document/frontmatter.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" stderrors "errors" + "strings" "github.com/pelletier/go-toml/v2" "github.com/pkg/errors" @@ -21,6 +22,88 @@ const ( frontmatterFormatTOML = "toml" ) +type formatter func(f *Frontmatter, reset bool) ([]byte, error) + +var formatters = map[string]formatter{ + frontmatterFormatYAML: func(f *Frontmatter, reset bool) ([]byte, error) { + m := make(map[string]interface{}) + unmarshalYaml := consumeFrontmatterSeparators(yaml.Unmarshal, false) + if err := unmarshalYaml([]byte(f.raw), &m); err != nil { + return nil, errors.WithStack(err) + } + + switch { + case reset: + f.Runme = nil + delete(m, "runme") + case !f.Runme.IsEmpty(): + m["runme"] = f.Runme + } + + var buf bytes.Buffer + encoder := yaml.NewEncoder(&buf) + encoder.SetIndent(2) + if err := encoder.Encode(m); err != nil { + return nil, errors.WithStack(err) + } + if err := encoder.Close(); err != nil { + return nil, errors.WithStack(err) + } + if buf.Len() < 5 && strings.Trim(buf.String(), "\r\n") == "{}" { + return nil, nil + } + return append(append([]byte("---\n"), buf.Bytes()...), []byte("---")...), nil + }, + frontmatterFormatJSON: func(f *Frontmatter, reset bool) ([]byte, error) { + m := make(map[string]interface{}) + unmarshalJSON := consumeFrontmatterSeparators(json.Unmarshal, false) + if err := unmarshalJSON([]byte(f.raw), &m); err != nil { + return nil, errors.WithStack(err) + } + + switch { + case reset: + f.Runme = nil + delete(m, "runme") + case !f.Runme.IsEmpty(): + m["runme"] = f.Runme + } + + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + return nil, errors.WithStack(err) + } + if len(data) < 5 && bytes.Equal(bytes.Trim(data, "\r\n"), []byte("{}")) { + return nil, nil + } + return data, nil + }, + frontmatterFormatTOML: func(f *Frontmatter, reset bool) ([]byte, error) { + m := make(map[string]interface{}) + unmarshalTOML := consumeFrontmatterSeparators(toml.Unmarshal, false) + if err := unmarshalTOML([]byte(f.raw), &m); err != nil { + return nil, errors.WithStack(err) + } + + switch { + case reset: + f.Runme = nil + delete(m, "runme") + case !f.Runme.IsEmpty(): + m["runme"] = f.Runme + } + + data, err := toml.Marshal(m) + if err != nil { + return nil, errors.WithStack(err) + } + if len(data) == 0 { + return nil, nil + } + return append(append([]byte("+++\n"), data...), []byte("+++")...), nil + }, +} + type RunmeMetadataDocument struct { RelativePath string `yaml:"relativePath,omitempty" json:"relativePath,omitempty" toml:"relativePath,omitempty"` } @@ -90,6 +173,33 @@ func newFrontmatter() *Frontmatter { } } +func (f *Frontmatter) ResetRunme(requireIdentity bool) (*Frontmatter, error) { + if f == nil { + return nil, nil + } + + if _, err := f.Marshal(requireIdentity); err != nil { + return f, err + } + + // remove runme frontmatter + remove := !requireIdentity + + formatter := formatters[f.format] + resetRaw, err := formatter(f, remove) + if err != nil { + return nil, err + } + + // return nil if frontmatter is actually empty + if resetRaw == nil { + return nil, nil + } + f.raw = string(resetRaw) + + return f, nil +} + // Marshal returns a marshaled frontmatter including triple-dashed lines. // If the identity is required, but Frontmatter is nil, a new one is created. func (f *Frontmatter) Marshal(requireIdentity bool) ([]byte, error) { @@ -107,67 +217,12 @@ func (f *Frontmatter) marshal(requireIdentity bool) ([]byte, error) { f.ensureID() } - switch f.format { - case frontmatterFormatYAML: - m := make(map[string]interface{}) - - if err := yaml.Unmarshal([]byte(f.raw), &m); err != nil { - return nil, errors.WithStack(err) - } - - if !f.Runme.IsEmpty() { - m["runme"] = f.Runme - } - - var buf bytes.Buffer - encoder := yaml.NewEncoder(&buf) - encoder.SetIndent(2) - if err := encoder.Encode(m); err != nil { - return nil, errors.WithStack(err) - } - if err := encoder.Close(); err != nil { - return nil, errors.WithStack(err) - } - - return append(append([]byte("---\n"), buf.Bytes()...), []byte("---")...), nil - - case frontmatterFormatJSON: - m := make(map[string]interface{}) - - if err := json.Unmarshal([]byte(f.raw), &m); err != nil { - return nil, errors.WithStack(err) - } - - if !f.Runme.IsEmpty() { - m["runme"] = f.Runme - } - - data, err := json.Marshal(m) - if err != nil { - return nil, errors.WithStack(err) - } - return append(append([]byte("---\n"), data...), []byte("\n---")...), nil - - case frontmatterFormatTOML: - m := make(map[string]interface{}) - - if err := toml.Unmarshal([]byte(f.raw), &m); err != nil { - return nil, errors.WithStack(err) - } - - if !f.Runme.IsEmpty() { - m["runme"] = f.Runme - } - - data, err := toml.Marshal(m) - if err != nil { - return nil, errors.WithStack(err) - } - return append(append([]byte("+++\n"), data...), []byte("+++")...), nil - - default: + formatter, ok := formatters[f.format] + if !ok { panic("invariant: Frontmatter created with invalid format") } + + return formatter(f, false) } func (f *Frontmatter) ensureID() { @@ -217,6 +272,8 @@ func ParseFrontmatter(raw []byte) (*Frontmatter, error) { return parseFrontmatter(raw) } +type parserFunc func([]byte, any) error + func parseFrontmatter(raw []byte) (*Frontmatter, error) { if len(raw) == 0 { return nil, nil @@ -228,25 +285,24 @@ func parseFrontmatter(raw []byte) (*Frontmatter, error) { // this detail will be in d.parseFrontmatterErr. var f Frontmatter - lines := bytes.Split(raw, []byte{'\n'}) - - if len(lines) < 2 || !bytes.Equal(bytes.TrimSpace(lines[0]), bytes.TrimSpace(lines[len(lines)-1])) { - return nil, errors.WithStack(ErrFrontmatterInvalid) - } - - raw = bytes.Join(lines[1:len(lines)-1], []byte{'\n'}) + // raw, err := consumeFrontmatterSeparators(raw) + // if err != nil { + // return nil, err + // } // TODO(adamb): discuss how to approach this in the most sensible way. // It can always return to the initial idea of returning all errors, // but the client will be left with the same problem. - parsers := []func([]byte, any) error{ - yaml.Unmarshal, + parsers := []parserFunc{ + consumeFrontmatterSeparators(yaml.Unmarshal, true), json.Unmarshal, - toml.Unmarshal, + consumeFrontmatterSeparators(json.Unmarshal, true), + consumeFrontmatterSeparators(toml.Unmarshal, true), } parsersNames := []string{ frontmatterFormatYAML, frontmatterFormatJSON, + frontmatterFormatJSON, frontmatterFormatTOML, } errorsCount := 0 @@ -274,3 +330,17 @@ func parseFrontmatter(raw []byte) (*Frontmatter, error) { return &f, nil } + +func consumeFrontmatterSeparators(parser parserFunc, strict bool) parserFunc { + return func(raw []byte, f any) error { + lines := bytes.Split(raw, []byte{'\n'}) + + if len(lines) >= 2 && bytes.Equal(bytes.TrimSpace(lines[0]), bytes.TrimSpace(lines[len(lines)-1])) { + raw = bytes.Join(lines[1:len(lines)-1], []byte{'\n'}) + } else if strict { + return errors.WithStack(ErrFrontmatterInvalid) + } + + return parser(raw, f) + } +} diff --git a/pkg/document/parser.go b/pkg/document/parser.go index 78a9d442..94be5198 100644 --- a/pkg/document/parser.go +++ b/pkg/document/parser.go @@ -9,7 +9,7 @@ import ( ) type ParsedSections struct { - FrontMatter []byte + Frontmatter []byte Content []byte ContentOffset int } @@ -19,8 +19,8 @@ func ParseSections(source []byte) (result ParsedSections, _ error) { runItemParser(l, parseInit) for _, item := range l.items { switch item.Type() { - case parsedItemFrontMatter: - result.FrontMatter = item.Value(source) + case parsedItemFrontmatter: + result.Frontmatter = item.Value(source) case parsedItemContent: result.ContentOffset = item.start result.Content = item.Value(source) @@ -48,7 +48,7 @@ func isEOL(r rune) bool { type parsedItemType int const ( - parsedItemFrontMatter parsedItemType = iota + 1 + parsedItemFrontmatter parsedItemType = iota + 1 parsedItemContent parsedItemError ) diff --git a/pkg/document/parser_frontmatter.go b/pkg/document/parser_frontmatter.go index 30622a79..2c737f65 100644 --- a/pkg/document/parser_frontmatter.go +++ b/pkg/document/parser_frontmatter.go @@ -31,7 +31,7 @@ func parseRawFrontmatter(l *itemParser, delimiter byte) parserStateFunc { if wasEndOfLine || isEOL(r) { if l.hasPrefix(bytes.Repeat([]byte{delimiter}, 3)) { l.pos += 3 - l.emit(parsedItemFrontMatter) + l.emit(parsedItemFrontmatter) l.consume(crlf) l.ignore() break @@ -79,7 +79,7 @@ func parseRawFrontmatterJSON(l *itemParser) parserStateFunc { } } - l.emit(parsedItemFrontMatter) + l.emit(parsedItemFrontmatter) l.consume(crlf) l.ignore() diff --git a/pkg/document/parser_test.go b/pkg/document/parser_test.go index d292fa80..19644f28 100644 --- a/pkg/document/parser_test.go +++ b/pkg/document/parser_test.go @@ -18,11 +18,11 @@ a: b require.NoError(t, err) assert.Equal(t, string(data), string(sections.Content)) - assert.Equal(t, "", string(sections.FrontMatter)) + assert.Equal(t, "", string(sections.Frontmatter)) assert.Equal(t, 0, sections.ContentOffset) } -func TestParseSections_WithoutFrontMatter(t *testing.T) { +func TestParseSections_WithoutFrontmatter(t *testing.T) { data := []byte(`# Example A paragraph @@ -32,7 +32,7 @@ A paragraph assert.Equal(t, string(data), string(sections.Content)) } -func TestParseSections_WithFrontMatterYAML(t *testing.T) { +func TestParseSections_WithFrontmatterYAML(t *testing.T) { data := []byte(`--- prop1: val1 prop2: val2 @@ -44,11 +44,11 @@ A paragraph `) sections, err := ParseSections(data) require.NoError(t, err) - assert.Equal(t, "---\nprop1: val1\nprop2: val2\n---", string(sections.FrontMatter)) + assert.Equal(t, "---\nprop1: val1\nprop2: val2\n---", string(sections.Frontmatter)) assert.Equal(t, "# Example\n\nA paragraph\n", string(sections.Content)) } -func TestParseSections_WithFrontMatterTOML(t *testing.T) { +func TestParseSections_WithFrontmatterTOML(t *testing.T) { data := []byte(`+++ prop1 = "val1" prop2 = "val2" @@ -60,11 +60,11 @@ A paragraph `) sections, err := ParseSections(data) require.NoError(t, err) - assert.Equal(t, "+++\nprop1 = \"val1\"\nprop2 = \"val2\"\n+++", string(sections.FrontMatter)) + assert.Equal(t, "+++\nprop1 = \"val1\"\nprop2 = \"val2\"\n+++", string(sections.Frontmatter)) assert.Equal(t, "# Example\n\nA paragraph\n", string(sections.Content)) } -func TestParseSections_WithFrontMatterJSON(t *testing.T) { +func TestParseSections_WithFrontmatterJSON(t *testing.T) { data := []byte(`{ "prop1": "val1", "prop2": "val2" @@ -76,7 +76,7 @@ A paragraph `) sections, err := ParseSections(data) require.NoError(t, err) - assert.Equal(t, "{\n \"prop1\": \"val1\",\n \"prop2\": \"val2\"\n}", string(sections.FrontMatter)) + assert.Equal(t, "{\n \"prop1\": \"val1\",\n \"prop2\": \"val2\"\n}", string(sections.Frontmatter)) assert.Equal(t, "# Example\n\nA paragraph\n", string(sections.Content)) } @@ -89,7 +89,7 @@ You can configure the rollout-duration parameter by modifying the config-network `) sections, err := ParseSections(data) require.NoError(t, err) - assert.Equal(t, "", string(sections.FrontMatter)) + assert.Equal(t, "", string(sections.Frontmatter)) assert.Equal(t, "{% include \"gradual-rollout-intro.md\" %}\n\n## Procedure\n\nYou can configure the rollout-duration parameter by modifying the config-network ConfigMap, or by using the Operator.\n", string(sections.Content)) } @@ -100,6 +100,6 @@ All notable changes to this project will be documented in this file. `) sections, err := ParseSections(data) require.NoError(t, err) - assert.Equal(t, "", string(sections.FrontMatter)) + assert.Equal(t, "", string(sections.Frontmatter)) assert.Equal(t, "{{- $repourl := $.Info.RepositoryURL -}}\n# CHANGELOG\nAll notable changes to this project will be documented in this file.\n", string(sections.Content)) } diff --git a/pkg/project/formatter.go b/pkg/project/formatter.go index 6cb60bb8..2843afe6 100644 --- a/pkg/project/formatter.go +++ b/pkg/project/formatter.go @@ -11,21 +11,29 @@ import ( type FuncOutput func(string, []byte) error -func FormatFiles(files []string, identityResolver *identity.IdentityResolver, formatJSON bool, write bool, outputter FuncOutput) error { +type FormatOptions struct { + IdentityResolver *identity.IdentityResolver + FormatJSON bool + Write bool + Outputter FuncOutput + Reset bool +} + +func FormatFiles(files []string, options *FormatOptions) error { for _, file := range files { data, err := readMarkdown(file) if err != nil { return err } - formatted, err := formatFile(data, identityResolver, formatJSON) + formatted, err := formatFile(data, options) if err != nil { return err } - if write { + if options.Write { err = writeMarkdown(file, formatted) - } else { + } else if outputter := options.Outputter; outputter != nil { err = outputter(file, formatted) } if err != nil { @@ -36,15 +44,15 @@ func FormatFiles(files []string, identityResolver *identity.IdentityResolver, fo return nil } -func formatFile(data []byte, identityResolver *identity.IdentityResolver, formatJSON bool) ([]byte, error) { +func formatFile(data []byte, options *FormatOptions) ([]byte, error) { var formatted []byte - notebook, err := editor.Deserialize(data, editor.Options{IdentityResolver: identityResolver}) + notebook, err := editor.Deserialize(data, editor.Options{IdentityResolver: options.IdentityResolver, Reset: options.Reset}) if err != nil { return nil, errors.Wrap(err, "failed to deserialize") } - if formatJSON { + if options.FormatJSON { var buf bytes.Buffer enc := json.NewEncoder(&buf) enc.SetIndent("", " ") @@ -53,7 +61,7 @@ func formatFile(data []byte, identityResolver *identity.IdentityResolver, format } formatted = buf.Bytes() } else { - formatted, err = editor.Serialize(notebook, nil, editor.Options{IdentityResolver: identityResolver}) + formatted, err = editor.Serialize(notebook, nil, editor.Options{IdentityResolver: options.IdentityResolver}) if err != nil { return nil, errors.Wrap(err, "failed to serialize") } @@ -61,7 +69,7 @@ func formatFile(data []byte, identityResolver *identity.IdentityResolver, format // todo(sebastian): remove moving to beta? it's neither used nor maintained // { - // doc := document.New(data, identityResolver) + // doc := document.New(data, options.IdentityResolver) // astNode, err := doc.RootAST() // if err != nil { // return nil, errors.Wrap(err, "failed to parse source") diff --git a/testdata/flags/fmt.txtar b/testdata/flags/fmt.txtar index 62ceb517..397498cc 100644 --- a/testdata/flags/fmt.txtar +++ b/testdata/flags/fmt.txtar @@ -3,6 +3,11 @@ exec runme fmt --write cmp README-FORMATTED.md README.md ! stderr . +env SHELL=/bin/bash +exec runme fmt reset --write +cmp LCID-all.md LCID-none.md +! stderr . + -- README.md -- --- runme: @@ -39,3 +44,62 @@ echo 1 ```{"name":"bash-echo-2"} echo 2 ``` +-- LCID-all.md -- +{ + "runme": { + "id": "01HF7AX2R37KPNPH1MQ2KEEYGM", + "version": "v3" + } +} + +```sh {"id":"01HF7BT3HBDTRGQAQMGP4A5DAJ"} +cat > heredoc << EOF +TEST=123 +HELLO=WORLD +NAME=$NAME +EOF +``` + +```sh {"id":"01HF7BT3HBDTRGQAQMGQCHXPXH"} +$ cat > somefile << EOF +line1 +line2 +line3 +EOF +$ cat somefile +``` + +```sh {"id":"01HF7BT3HBDTRGQAQMGSKMPTNS"} +$ rm -f somefile +$ cat << EOF +line1 +line2 +line3 +EOF > somefile +``` +-- LCID-none.md -- +```sh +cat > heredoc << EOF +TEST=123 +HELLO=WORLD +NAME=$NAME +EOF +``` + +```sh +$ cat > somefile << EOF +line1 +line2 +line3 +EOF +$ cat somefile +``` + +```sh +$ rm -f somefile +$ cat << EOF +line1 +line2 +line3 +EOF > somefile +```