API rewrite work in progress

See #32.

.gitattributes

@@ -1,2 +1,5 @@
 # To prevent CRLF breakages on Windows for fragile files, like testdata.
 * -text
+
+# Don't show bundled base64 strings in git diff/grep.
+bundle.go -diff -text

.gitignore

@@ -0,0 +1 @@
+/gogrep

README.md

@@ -4,9 +4,7 @@
 
 Search for Go code using syntax trees. Work in progress.
 
-	gogrep -x 'if $x != nil { return $x, $*_ }'
-
-### Instructions
+### Instructions (TODO: update withthe new interface)
 
 	usage: gogrep commands [packages]
 

builtin/tests.go

@@ -0,0 +1,27 @@
+// +build gogrep
+
+package builtin
+
+import "mvdan.cc/gogrep/nls"
+
+func excludeParallel(g *nls.G) {
+	// g.With(nls.All(`$*body`))
+	// g.With(nls.All(`$*body`), nls.All(`{ $*_; }`), nls.Replace(`{}`))
+	g.Excluding(`t.Parallel()`)
+}
+
+func NonParallelTests(g *nls.G) {
+	// TODO: include the case where a sub-test calls Parallel, but the
+	// parent test does not.
+	g.All(`func $f(t *testing.T) { $*body }`)
+	excludeParallel(g)
+	g.Report("$f is not parallel")
+}
+
+func NonParallelSubtests(g *nls.G) {
+	// TODO: include the case where a sub-test calls Parallel, but the
+	// parent test does not.
+	g.All(`t.Run($n, func(t *testing.T) { $*body })`)
+	excludeParallel(g)
+	g.Report("subtest $n is not parallel")
+}

gen_bundle.go

@@ -0,0 +1,76 @@
+// +build ignore
+
+package main
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+var globs = []string{
+	"go.*",
+	// TODO: use 'go list' to include deps automatically
+	"mainpkg/*.go",
+	"nls/*.go",
+	"gsyntax/*.go",
+	"internal/load/*.go",
+}
+
+type bundledFile struct{ Name, Content string }
+
+var tmpl = template.Must(template.New("").Parse(`
+package main
+
+var bundledFiles = [...]struct{
+	name, encContent string
+}{
+{{ range $_, $f := . }} { {{ printf "%q" $f.Name }}, {{ printf "%q" $f.Content }} },
+{{ end }}
+}
+`))
+
+func run() error {
+	var files []bundledFile
+	for _, glob := range globs {
+		matches, err := filepath.Glob(glob)
+		if err != nil {
+			return err
+		}
+		if len(matches) == 0 {
+			return fmt.Errorf("no matches for %q", glob)
+		}
+		for _, match := range matches {
+			if strings.HasSuffix(match, "_test.go") {
+				continue
+			}
+			content, err := ioutil.ReadFile(match)
+			if err != nil {
+				return err
+			}
+			files = append(files, bundledFile{
+				Name:    match,
+				Content: base64.RawStdEncoding.EncodeToString(content),
+			})
+		}
+	}
+	f, err := os.Create("bundle.go")
+	if err != nil {
+		return err
+	}
+	if err := tmpl.Execute(f, files); err != nil {
+		return err
+	}
+	return f.Close()
+}

go.mod

@@ -1,5 +1,8 @@
 module mvdan.cc/gogrep
 
-require golang.org/x/tools v0.0.0-20191223235410-3721262b3e7c
+require (
+	github.com/rogpeppe/go-internal v1.5.1
+	golang.org/x/tools v0.0.0-20191223235410-3721262b3e7c
+)
 
 go 1.13

go.sum

@@ -1,3 +1,8 @@
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/rogpeppe/go-internal v1.5.1 h1:asQ0uD7BN9RU5Im41SEEZTwCi/zAXdMOLS3npYaos2g=
+github.com/rogpeppe/go-internal v1.5.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
@@ -10,3 +15,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20191223235410-3721262b3e7c h1:PeFrxQ8YTAKg53UR8aP/nxa82lQYIdb+pd1bfg3dBDM=
 golang.org/x/tools v0.0.0-20191223235410-3721262b3e7c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=

gsyntax/parse.go

@@ -0,0 +1,170 @@
+// Copyright (c) 2019, Daniel Martí <mvdan@mvdan.cc>
+// See LICENSE for licensing information
+
+package gsyntax
+
+import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/scanner"
+	"go/token"
+	"text/template"
+)
+
+var tmplDecl = template.Must(template.New("").Parse(`` +
+	`package p; {{ . }}`))
+
+var tmplBlock = template.Must(template.New("").Parse(`` +
+	`package p; func _() { if true {{ . }} else {} }`))
+
+var tmplExprs = template.Must(template.New("").Parse(`` +
+	`package p; var _ = []interface{}{ {{ . }}, }`))
+
+var tmplStmts = template.Must(template.New("").Parse(`` +
+	`package p; func _() { {{ . }} }`))
+
+var tmplType = template.Must(template.New("").Parse(`` +
+	`package p; var _ {{ . }}`))
+
+var tmplValSpec = template.Must(template.New("").Parse(`` +
+	`package p; var {{ . }}`))
+
+func execTmpl(tmpl *template.Template, src string) string {
+	var buf bytes.Buffer
+	if err := tmpl.Execute(&buf, src); err != nil {
+		panic(err)
+	}
+	return buf.String()
+}
+
+func noBadNodes(node ast.Node) bool {
+	any := false
+	ast.Inspect(node, func(n ast.Node) bool {
+		if any {
+			return false
+		}
+		switch n.(type) {
+		case *ast.BadExpr, *ast.BadDecl:
+			any = true
+		}
+		return true
+	})
+	return !any
+}
+
+func ParseType(fset *token.FileSet, src string) (ast.Expr, *ast.File, error) {
+	asType := execTmpl(tmplType, src)
+	f, err := parser.ParseFile(fset, "", asType, 0)
+	if err != nil {
+		err = SubPosOffsets(err, PosOffset{1, 1, 17})
+		return nil, nil, err
+	}
+	vs := f.Decls[0].(*ast.GenDecl).Specs[0].(*ast.ValueSpec)
+	return vs.Type, f, nil
+}
+
+// ParseAny tries its best to parse the ast.Node contained in src, as
+// one of: *ast.File, ast.Decl, ast.Expr, ast.Stmt, *ast.ValueSpec.
+// It also returns the *ast.File used for the parsing, so that the returned node
+// can be easily type-checked.
+func ParseAny(fset *token.FileSet, src string) (ast.Node, *ast.File, error) {
+	file := fset.AddFile("", fset.Base(), len(src))
+	scan := scanner.Scanner{}
+	scan.Init(file, []byte(src), nil, 0)
+	if _, tok, _ := scan.Scan(); tok == token.EOF {
+		return nil, nil, fmt.Errorf("empty source code")
+	}
+	var mainErr error
+
+	// first try as a whole file
+	if f, err := parser.ParseFile(fset, "", src, 0); err == nil && noBadNodes(f) {
+		return f, f, nil
+	}
+
+	// then as a single declaration, or many
+	asDecl := execTmpl(tmplDecl, src)
+	if f, err := parser.ParseFile(fset, "", asDecl, 0); err == nil && noBadNodes(f) {
+		if len(f.Decls) == 1 {
+			return f.Decls[0], f, nil
+		}
+		return f, f, nil
+	}
+
+	// then as a block; otherwise blocks might be mistaken for composite
+	// literals further below
+	asBlock := execTmpl(tmplBlock, src)
+	if f, err := parser.ParseFile(fset, "", asBlock, 0); err == nil && noBadNodes(f) {
+		bl := f.Decls[0].(*ast.FuncDecl).Body
+		if len(bl.List) == 1 {
+			ifs := bl.List[0].(*ast.IfStmt)
+			return ifs.Body, f, nil
+		}
+	}
+
+	// then as value expressions
+	asExprs := execTmpl(tmplExprs, src)
+	if f, err := parser.ParseFile(fset, "", asExprs, 0); err == nil && noBadNodes(f) {
+		vs := f.Decls[0].(*ast.GenDecl).Specs[0].(*ast.ValueSpec)
+		cl := vs.Values[0].(*ast.CompositeLit)
+		if len(cl.Elts) == 1 {
+			return cl.Elts[0], f, nil
+		}
+		return ExprList(cl.Elts), f, nil
+	}
+
+	// then try as statements
+	asStmts := execTmpl(tmplStmts, src)
+	if f, err := parser.ParseFile(fset, "", asStmts, 0); err == nil && noBadNodes(f) {
+		bl := f.Decls[0].(*ast.FuncDecl).Body
+		if len(bl.List) == 1 {
+			return bl.List[0], f, nil
+		}
+		return StmtList(bl.List), f, nil
+	} else {
+		// Statements is what covers most cases, so it will give
+		// the best overall error message. Show positions
+		// relative to where the user's code is put in the
+		// template.
+		mainErr = SubPosOffsets(err, PosOffset{1, 1, 22})
+	}
+
+	// type expressions not yet picked up, for e.g. chans and interfaces
+	if typ, f, err := ParseType(fset, src); err == nil && noBadNodes(f) {
+		return typ, f, nil
+	}
+
+	// value specs
+	asValSpec := execTmpl(tmplValSpec, src)
+	if f, err := parser.ParseFile(fset, "", asValSpec, 0); err == nil && noBadNodes(f) {
+		vs := f.Decls[0].(*ast.GenDecl).Specs[0].(*ast.ValueSpec)
+		return vs, f, nil
+	}
+	return nil, nil, mainErr
+}
+
+type PosOffset struct {
+	AtLine, AtCol int
+	Offset        int
+}
+
+func SubPosOffsets(err error, offs ...PosOffset) error {
+	list, ok := err.(scanner.ErrorList)
+	if !ok {
+		return err
+	}
+	for i, err := range list {
+		for _, off := range offs {
+			if err.Pos.Line != off.AtLine {
+				continue
+			}
+			if err.Pos.Column < off.AtCol {
+				continue
+			}
+			err.Pos.Column -= off.Offset
+		}
+		list[i] = err
+	}
+	return list
+}