From f7e8581dd9a337fc1c0ea7069595dcd082f1784f Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Tue, 22 Nov 2022 09:43:54 +0200 Subject: [PATCH] js: refactor how modules are loaded This refactor tries to simplify the implementation of `require` and connected code by splitting it heavily into different parts. These changes are similar to what will be needed for native ESM support as shown in https://github.com/grafana/k6/pull/2563 , but without any of the native parts and without doing anything that isn't currently needed. This will hopefully make ESM PR much smaller and less intrusive. This includes still keeping the wrong relativity of `require` as explained in https://github.com/grafana/k6/issues/2674. It also tries to simplify connected code, but due to this being very sensitive code and the changes already being quite big, this is done only to an extent. The lack of new tests is mostly due to there not being really any new code and the tests that were created along these changes already being merged months ago with https://github.com/grafana/k6/pull/2782. Future changes will try to address the above as well as potentially moving the whole module types and logic in separate package to be reused in tests. --- js/bundle.go | 288 +++++++++++++++++++++------------------- js/bundle_test.go | 2 +- js/cjsmodule.go | 70 ++++++++++ js/gomodule.go | 93 +++++++++++++ js/initcontext.go | 294 +++++++++-------------------------------- js/initcontext_test.go | 25 ++-- js/modules.go | 122 +++++++++++++++++ js/runner.go | 4 +- 8 files changed, 512 insertions(+), 386 deletions(-) create mode 100644 js/cjsmodule.go create mode 100644 js/gomodule.go create mode 100644 js/modules.go diff --git a/js/bundle.go b/js/bundle.go index 4dbf251cf37..b0f191ac0fd 100644 --- a/js/bundle.go +++ b/js/bundle.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "net/url" + "path/filepath" "runtime" "github.com/dop251/goja" @@ -27,17 +28,19 @@ import ( // You can use this to produce identical BundleInstance objects. type Bundle struct { Filename *url.URL - Source string - Program *goja.Program + Source []byte Options lib.Options - - BaseInitContext *InitContext + logger *logrus.Logger RuntimeOptions lib.RuntimeOptions CompatibilityMode lib.CompatibilityMode // parsed value registry *metrics.Registry - exports map[string]goja.Callable + filesystems map[string]afero.Fs + pwd *url.URL + + callableExports map[string]struct{} + modResolution *modulesResolution } // A BundleInstance is a self-contained instance of a Bundle. @@ -47,17 +50,18 @@ type BundleInstance struct { // TODO: maybe just have a reference to the Bundle? or save and pass rtOpts? env map[string]string - exports map[string]goja.Callable - moduleVUImpl *moduleVUImpl - pgm programWithSource + mainModuleInstance moduleInstance + moduleVUImpl *moduleVUImpl } func (bi *BundleInstance) getCallableExport(name string) goja.Callable { - return bi.exports[name] + fn, ok := goja.AssertFunction(bi.mainModuleInstance.exports().Get(name)) + _ = ok // TODO maybe return it + return fn } func (bi *BundleInstance) getExported(name string) goja.Value { - return bi.pgm.exports.Get(name) + return bi.mainModuleInstance.exports().ToObject(bi.Runtime).Get(name) } // NewBundle creates a new bundle from a source file and a filesystem. @@ -76,36 +80,35 @@ func newBundle( return nil, err } - // Compile sources, both ES5 and ES6 are supported. - code := string(src.Data) - c := compiler.New(piState.Logger) - c.Options = compiler.Options{ - CompatibilityMode: compatMode, - Strict: true, - SourceMapLoader: generateSourceMapLoader(piState.Logger, filesystems), - } - pgm, _, err := c.Compile(code, src.URL.String(), false) - if err != nil { - return nil, err - } // Make a bundle, instantiate it into a throwaway VM to populate caches. - rt := goja.New() bundle := Bundle{ Filename: src.URL, - Source: code, - Program: pgm, - BaseInitContext: NewInitContext(piState.Logger, rt, c, compatMode, filesystems, loader.Dir(src.URL)), + Source: src.Data, RuntimeOptions: piState.RuntimeOptions, Options: options, CompatibilityMode: compatMode, - exports: make(map[string]goja.Callable), + callableExports: make(map[string]struct{}), registry: piState.Registry, + modResolution: newModuleResolution(getJSModules()), + filesystems: filesystems, + pwd: loader.Dir(src.URL), + logger: piState.Logger, } - if err = bundle.instantiate(bundle.BaseInitContext, 0); err != nil { + c := bundle.newCompiler(piState.Logger) + if err = bundle.modResolution.setMain(src, c); err != nil { + return nil, err + } + // Instantiate the bundle into a new VM using a bound init context. This uses a context with a + // runtime, but no state, to allow module-provided types to function within the init context. + // TODO use a real context + vuImpl := &moduleVUImpl{ctx: context.Background(), runtime: goja.New()} + vuImpl.eventLoop = eventloop.New(vuImpl) + instance, err := bundle.instantiate(piState.Logger, vuImpl, 0, c) + if err != nil { return nil, err } - err = bundle.getExports(piState.Logger, rt, updateOptions) + err = bundle.populateExports(piState.Logger, updateOptions, instance) if err != nil { return nil, err } @@ -143,11 +146,11 @@ func NewBundleFromArchive(piState *lib.TestPreInitState, arc *lib.Archive) (*Bun func (b *Bundle) makeArchive() *lib.Archive { arc := &lib.Archive{ Type: "js", - Filesystems: b.BaseInitContext.filesystems, + Filesystems: b.filesystems, Options: b.Options, FilenameURL: b.Filename, - Data: []byte(b.Source), - PwdURL: b.BaseInitContext.pwd, + Data: b.Source, + PwdURL: b.pwd, Env: make(map[string]string, len(b.RuntimeOptions.Env)), CompatibilityMode: b.CompatibilityMode.String(), K6Version: consts.Version, @@ -161,19 +164,16 @@ func (b *Bundle) makeArchive() *lib.Archive { return arc } -// getExports validates and extracts exported objects -func (b *Bundle) getExports(logger logrus.FieldLogger, rt *goja.Runtime, updateOptions bool) error { - pgm := b.BaseInitContext.programs[b.Filename.String()] // this is the main script and it's always present - exportsV := pgm.module.Get("exports") - if goja.IsNull(exportsV) || goja.IsUndefined(exportsV) { +// populateExports validates and extracts exported objects +func (b *Bundle) populateExports(logger logrus.FieldLogger, updateOptions bool, instance moduleInstance) error { + exports := instance.exports() + if exports == nil { return errors.New("exports must be an object") } - exports := exportsV.ToObject(rt) - for _, k := range exports.Keys() { v := exports.Get(k) - if fn, ok := goja.AssertFunction(v); ok && k != consts.Options { - b.exports[k] = fn + if _, ok := goja.AssertFunction(v); ok && k != consts.Options { + b.callableExports[k] = struct{}{} continue } switch k { @@ -200,7 +200,7 @@ func (b *Bundle) getExports(logger logrus.FieldLogger, rt *goja.Runtime, updateO } } - if len(b.exports) == 0 { + if len(b.callableExports) == 0 { return errors.New("no exported functions in script") } @@ -211,43 +211,34 @@ func (b *Bundle) getExports(logger logrus.FieldLogger, rt *goja.Runtime, updateO func (b *Bundle) Instantiate(ctx context.Context, vuID uint64) (*BundleInstance, error) { // Instantiate the bundle into a new VM using a bound init context. This uses a context with a // runtime, but no state, to allow module-provided types to function within the init context. - rt := goja.New() - vuImpl := &moduleVUImpl{ - ctx: ctx, - runtime: rt, - } - init := newBoundInitContext(b.BaseInitContext, vuImpl) - if err := b.instantiate(init, vuID); err != nil { + vuImpl := &moduleVUImpl{ctx: ctx, runtime: goja.New()} + vuImpl.eventLoop = eventloop.New(vuImpl) + instance, err := b.instantiate(b.logger, vuImpl, vuID, b.newCompiler(b.logger)) + if err != nil { return nil, err } - pgm := init.programs[b.Filename.String()] // this is the main script and it's always present bi := &BundleInstance{ - Runtime: rt, - exports: make(map[string]goja.Callable), - env: b.RuntimeOptions.Env, - moduleVUImpl: vuImpl, - pgm: pgm, + Runtime: vuImpl.runtime, + env: b.RuntimeOptions.Env, + moduleVUImpl: vuImpl, + mainModuleInstance: instance, } // Grab any exported functions that could be executed. These were // already pre-validated in cmd.validateScenarioConfig(), just get them here. - exports := pgm.module.Get("exports").ToObject(rt) - for k := range b.exports { - fn, _ := goja.AssertFunction(exports.Get(k)) - bi.exports[k] = fn - } + exports := instance.exports() jsOptions := exports.Get("options") var jsOptionsObj *goja.Object if jsOptions == nil || goja.IsNull(jsOptions) || goja.IsUndefined(jsOptions) { - jsOptionsObj = rt.NewObject() + jsOptionsObj = vuImpl.runtime.NewObject() err := exports.Set("options", jsOptionsObj) if err != nil { return nil, fmt.Errorf("couldn't set exported options with merged values: %w", err) } } else { - jsOptionsObj = jsOptions.ToObject(rt) + jsOptionsObj = jsOptions.ToObject(vuImpl.runtime) } var instErr error @@ -260,78 +251,72 @@ func (b *Bundle) Instantiate(ctx context.Context, vuID uint64) (*BundleInstance, return bi, instErr } -// Instantiates the bundle into an existing runtime. Not public because it also messes with a bunch -// of other things, will potentially thrash data and makes a mess in it if the operation fails. - -func (b *Bundle) initializeProgramObject(rt *goja.Runtime, init *InitContext) programWithSource { - pgm := programWithSource{ - pgm: b.Program, - src: b.Source, - exports: rt.NewObject(), - module: rt.NewObject(), +func (b *Bundle) newCompiler(logger logrus.FieldLogger) *compiler.Compiler { + c := compiler.New(logger) + c.Options = compiler.Options{ + CompatibilityMode: b.CompatibilityMode, + Strict: true, + SourceMapLoader: generateSourceMapLoader(logger, b.filesystems), } - _ = pgm.module.Set("exports", pgm.exports) - init.programs[b.Filename.String()] = pgm - return pgm + return c } -//nolint:funlen -func (b *Bundle) instantiate(init *InitContext, vuID uint64) (err error) { - rt := init.moduleVUImpl.runtime - logger := init.logger - rt.SetFieldNameMapper(common.FieldNameMapper{}) - rt.SetRandSource(common.NewRandSource()) - - env := make(map[string]string, len(b.RuntimeOptions.Env)) - for key, value := range b.RuntimeOptions.Env { - env[key] = value +func (b *Bundle) instantiate( + logger logrus.FieldLogger, vuImpl *moduleVUImpl, vuID uint64, c *compiler.Compiler, +) (instance moduleInstance, err error) { + rt := vuImpl.runtime + err = b.setupJSRuntime(rt, int64(vuID), logger) + if err != nil { + return nil, err } - rt.Set("__ENV", env) - rt.Set("__VU", vuID) - _ = rt.Set("console", newConsole(logger)) - if init.compatibilityMode == lib.CompatibilityModeExtended { - rt.Set("global", rt.GlobalObject()) + initenv := &common.InitEnvironment{Logger: logger, FileSystems: b.filesystems, CWD: b.pwd, Registry: b.registry} + cjsLoad := func(specifier *url.URL, name string) (*cjsModule, error) { + if filepath.IsAbs(name) && runtime.GOOS == "windows" { + logger.Warnf("'%s' was imported with an absolute path - this won't be cross-platform and won't work if"+ + " you move the script between machines or run it with `k6 cloud`; if absolute paths are required,"+ + " import them with the `file://` schema for slightly better compatibility", + name) + } + //nolint:shadow,govet // here we shadow err on purpose + d, err := loader.Load(logger, b.filesystems, specifier, name) + if err != nil { + return nil, err + } + return cjsmoduleFromString(specifier, d.Data, c) } - initenv := &common.InitEnvironment{ - Logger: logger, - FileSystems: init.filesystems, - CWD: init.pwd, - Registry: b.registry, - } - unbindInit := b.setInitGlobals(rt, init) - init.moduleVUImpl.initEnv = initenv - init.moduleVUImpl.eventLoop = eventloop.New(init.moduleVUImpl) - pgm := b.initializeProgramObject(rt, init) + modSys := newModuleSystem(b.modResolution, vuImpl, cjsLoad) + unbindInit := b.setInitGlobals(rt, modSys) + vuImpl.initEnv = initenv + defer func() { + unbindInit() + vuImpl.initEnv = nil + }() // TODO: make something cleaner for interrupting scripts, and more unified // (e.g. as a part of the event loop or RunWithPanicCatching()? - initCtxDone := init.moduleVUImpl.ctx.Done() + initCtxDone := vuImpl.ctx.Done() initDone := make(chan struct{}) watchDone := make(chan struct{}) go func() { select { case <-initCtxDone: - rt.Interrupt(init.moduleVUImpl.ctx.Err()) + rt.Interrupt(vuImpl.ctx.Err()) case <-initDone: // do nothing } close(watchDone) }() err = common.RunWithPanicCatching(logger, rt, func() error { - return init.moduleVUImpl.eventLoop.Start(func() error { - f, errRun := rt.RunProgram(b.Program) - if errRun != nil { - return errRun - } - if call, ok := goja.AssertFunction(f); ok { - if _, errRun = call(pgm.exports, pgm.module, pgm.exports); errRun != nil { - return errRun - } - return nil + return vuImpl.eventLoop.Start(func() error { + //nolint:shadow,govet // here we shadow err on purpose + mod, err := b.modResolution.resolve(b.pwd, b.Filename.String(), cjsLoad) + if err != nil { + return err // TODO wrap as this should never happen } - panic("Somehow a commonjs main module is not wrapped in a function") + instance = mod.Instantiate(vuImpl) + return instance.execute() }) }) close(initDone) @@ -342,54 +327,79 @@ func (b *Bundle) instantiate(init *InitContext, vuID uint64) (err error) { if errors.As(err, &exception) { err = &scriptException{inner: exception} } - return err + return nil, err } - exportsV := pgm.module.Get("exports") - if goja.IsNull(exportsV) { - return errors.New("exports must be an object") + if exports := instance.exports(); exports == nil { + return nil, errors.New("exports must be an object") } - pgm.exports = exportsV.ToObject(rt) - init.programs[b.Filename.String()] = pgm - unbindInit() - init.moduleVUImpl.ctx = nil - init.moduleVUImpl.initEnv = nil // If we've already initialized the original VU init context, forbid // any subsequent VUs to open new files if vuID == 0 { - init.allowOnlyOpenedFiles() + allowOnlyOpenedFiles(b.filesystems["file"]) } rt.SetRandSource(common.NewRandSource()) + return instance, nil +} + +func (b *Bundle) setupJSRuntime(rt *goja.Runtime, vuID int64, logger logrus.FieldLogger) error { + rt.SetFieldNameMapper(common.FieldNameMapper{}) + rt.SetRandSource(common.NewRandSource()) + + env := make(map[string]string, len(b.RuntimeOptions.Env)) + for key, value := range b.RuntimeOptions.Env { + env[key] = value + } + err := rt.Set("__ENV", env) + if err != nil { + return err + } + err = rt.Set("__VU", vuID) + if err != nil { + return err + } + err = rt.Set("console", newConsole(logger)) + if err != nil { + return err + } + + if b.CompatibilityMode == lib.CompatibilityModeExtended { + err = rt.Set("global", rt.GlobalObject()) + if err != nil { + return err + } + } return nil } -func (b *Bundle) setInitGlobals(rt *goja.Runtime, init *InitContext) (unset func()) { +func (b *Bundle) setInitGlobals(rt *goja.Runtime, modSys *moduleSystem) (unset func()) { mustSet := func(k string, v interface{}) { if err := rt.Set(k, v); err != nil { panic(fmt.Errorf("failed to set '%s' global object: %w", k, err)) } } - mustSet("require", init.Require) - mustSet("open", init.Open) - return func() { - mustSet("require", goja.Undefined()) - mustSet("open", goja.Undefined()) + r := requireImpl{ + vu: modSys.vu, + modules: modSys, + pwd: b.pwd, } -} + mustSet("require", r.require) -func generateSourceMapLoader(logger logrus.FieldLogger, filesystems map[string]afero.Fs, -) func(path string) ([]byte, error) { - return func(path string) ([]byte, error) { - u, err := url.Parse(path) - if err != nil { - return nil, err + mustSet("open", func(filename string, args ...string) (goja.Value, error) { + if modSys.vu.State() != nil { // fix + return nil, fmt.Errorf(cantBeUsedOutsideInitContextMsg, "open") } - data, err := loader.Load(logger, filesystems, u, path) - if err != nil { - return nil, err + + if filename == "" { + return nil, errors.New("open() can't be used with an empty filename") } - return data.Data, nil + // This uses the pwd from the requireImpl + return openImpl(rt, b.filesystems["file"], r.pwd, filename, args...) + }) + return func() { + mustSet("require", goja.Undefined()) + mustSet("open", goja.Undefined()) } } diff --git a/js/bundle_test.go b/js/bundle_test.go index 658523bb7b5..377009d6433 100644 --- a/js/bundle_test.go +++ b/js/bundle_test.go @@ -124,7 +124,7 @@ func TestNewBundle(t *testing.T) { b, err := getSimpleBundle(t, "-", `export default function() {};`) require.NoError(t, err) assert.Equal(t, "file://-", b.Filename.String()) - assert.Equal(t, "file:///", b.BaseInitContext.pwd.String()) + assert.Equal(t, "file:///", b.pwd.String()) }) t.Run("CompatibilityMode", func(t *testing.T) { t.Parallel() diff --git a/js/cjsmodule.go b/js/cjsmodule.go new file mode 100644 index 00000000000..21d6d7736b4 --- /dev/null +++ b/js/cjsmodule.go @@ -0,0 +1,70 @@ +package js + +import ( + "fmt" + "net/url" + + "github.com/dop251/goja" + "go.k6.io/k6/js/compiler" + "go.k6.io/k6/js/modules" +) + +// cjsModule represents a commonJS module +type cjsModule struct { + prg *goja.Program + url *url.URL +} + +var _ module = &cjsModule{} + +type cjsModuleInstance struct { + mod *cjsModule + moduleObj *goja.Object + vu modules.VU +} + +func (c *cjsModule) Instantiate(vu modules.VU) moduleInstance { + return &cjsModuleInstance{vu: vu, mod: c} +} + +func (c *cjsModuleInstance) execute() error { + rt := c.vu.Runtime() + exports := rt.NewObject() + c.moduleObj = rt.NewObject() + err := c.moduleObj.Set("exports", exports) + if err != nil { + return fmt.Errorf("error while getting ready to import commonJS, couldn't set exports property of module: %w", + err) + } + + // Run the program. + f, err := rt.RunProgram(c.mod.prg) + if err != nil { + return err + } + if call, ok := goja.AssertFunction(f); ok { + if _, err = call(exports, c.moduleObj, exports); err != nil { + return err + } + } + + return nil +} + +func (c *cjsModuleInstance) exports() *goja.Object { + exportsV := c.moduleObj.Get("exports") + if goja.IsNull(exportsV) || goja.IsUndefined(exportsV) { + return nil + } + return exportsV.ToObject(c.vu.Runtime()) +} + +type cjsModuleLoader func(specifier *url.URL, name string) (*cjsModule, error) + +func cjsmoduleFromString(fileURL *url.URL, data []byte, c *compiler.Compiler) (*cjsModule, error) { + pgm, _, err := c.Compile(string(data), fileURL.String(), false) + if err != nil { + return nil, err + } + return &cjsModule{prg: pgm, url: fileURL}, nil +} diff --git a/js/gomodule.go b/js/gomodule.go new file mode 100644 index 00000000000..1abe4742acc --- /dev/null +++ b/js/gomodule.go @@ -0,0 +1,93 @@ +package js + +import ( + "github.com/dop251/goja" + "go.k6.io/k6/js/modules" +) + +// baseGoModule is a go module that does not implement modules.Module interface +// TODO maybe depracate those in the future +type baseGoModule struct { + mod interface{} +} + +var _ module = &baseGoModule{} + +func (b *baseGoModule) Instantiate(vu modules.VU) moduleInstance { + return &baseGoModuleInstance{mod: b.mod, vu: vu} +} + +type baseGoModuleInstance struct { + mod interface{} + vu modules.VU + exportsO *goja.Object // this is so we only initialize the exports once per instance +} + +func (b *baseGoModuleInstance) execute() error { + return nil +} + +func (b *baseGoModuleInstance) exports() *goja.Object { + if b.exportsO == nil { + // TODO check this does not panic a lot + rt := b.vu.Runtime() + b.exportsO = rt.ToValue(b.mod).ToObject(rt) + } + return b.exportsO +} + +// goModule is a go module which implements modules.Module +type goModule struct { + modules.Module +} + +var _ module = &goModule{} + +func (g *goModule) Instantiate(vu modules.VU) moduleInstance { + return &goModuleInstance{vu: vu, module: g} +} + +type goModuleInstance struct { + modules.Instance + module *goModule + vu modules.VU + exportsO *goja.Object // this is so we only initialize the exports once per instance +} + +var _ moduleInstance = &goModuleInstance{} + +func (gi *goModuleInstance) execute() error { + gi.Instance = gi.module.NewModuleInstance(gi.vu) + return nil +} + +func (gi *goModuleInstance) exports() *goja.Object { + if gi.exportsO == nil { + rt := gi.vu.Runtime() + gi.exportsO = rt.ToValue(toESModuleExports(gi.Instance.Exports())).ToObject(rt) + } + return gi.exportsO +} + +func toESModuleExports(exp modules.Exports) interface{} { + if exp.Named == nil { + return exp.Default + } + if exp.Default == nil { + return exp.Named + } + + result := make(map[string]interface{}, len(exp.Named)+2) + + for k, v := range exp.Named { + result[k] = v + } + // Maybe check that those weren't set + result["default"] = exp.Default + // this so babel works with the `default` when it transpiles from ESM to commonjs. + // This should probably be removed once we have support for ESM directly. So that require doesn't get support for + // that while ESM has. + result["__esModule"] = true + + return result +} diff --git a/js/initcontext.go b/js/initcontext.go index 8b8083ea2c3..ca00a7fd9ae 100644 --- a/js/initcontext.go +++ b/js/initcontext.go @@ -1,254 +1,37 @@ package js import ( - "context" "errors" "fmt" "net/url" "path/filepath" - "runtime" "strings" "github.com/dop251/goja" "github.com/sirupsen/logrus" "github.com/spf13/afero" - "go.k6.io/k6/js/common" - "go.k6.io/k6/js/compiler" "go.k6.io/k6/js/modules" - "go.k6.io/k6/lib" "go.k6.io/k6/lib/fsext" "go.k6.io/k6/loader" ) -type programWithSource struct { - pgm *goja.Program - src string - module *goja.Object - exports *goja.Object -} - -const openCantBeUsedOutsideInitContextMsg = `The "open()" function is only available in the init stage ` + +const cantBeUsedOutsideInitContextMsg = `the "%s" function is only available in the init stage ` + `(i.e. the global scope), see https://k6.io/docs/using-k6/test-life-cycle for more information` -// InitContext provides APIs for use in the init context. -// -// TODO: refactor most/all of this state away, use common.InitEnvironment instead -type InitContext struct { - // Bound runtime; used to instantiate objects. - compiler *compiler.Compiler - - moduleVUImpl *moduleVUImpl - - // Filesystem to load files and scripts from with the map key being the scheme - filesystems map[string]afero.Fs - pwd *url.URL - - // Cache of loaded programs and files. - programs map[string]programWithSource - exportsCache map[string]goja.Value - - compatibilityMode lib.CompatibilityMode - - logger logrus.FieldLogger - - moduleRegistry map[string]interface{} -} - -// NewInitContext creates a new initcontext with the provided arguments -func NewInitContext( - logger logrus.FieldLogger, rt *goja.Runtime, c *compiler.Compiler, compatMode lib.CompatibilityMode, - filesystems map[string]afero.Fs, pwd *url.URL, -) *InitContext { - return &InitContext{ - compiler: c, - filesystems: filesystems, - pwd: pwd, - programs: make(map[string]programWithSource), - compatibilityMode: compatMode, - logger: logger, - moduleRegistry: getJSModules(), - exportsCache: make(map[string]goja.Value), - moduleVUImpl: &moduleVUImpl{ - // TODO: pass a real context as we did for https://github.com/grafana/k6/pull/2800, - // also see https://github.com/grafana/k6/issues/2804 - ctx: context.Background(), - runtime: rt, - }, - } -} - -func newBoundInitContext(base *InitContext, vuImpl *moduleVUImpl) *InitContext { - // we don't copy the exports as otherwise they will be shared and we don't want this. - // this means that all the files will be executed again but once again only once per compilation - // of the main file. - programs := make(map[string]programWithSource, len(base.programs)) - for key, program := range base.programs { - programs[key] = programWithSource{ - src: program.src, - pgm: program.pgm, - } - } - return &InitContext{ - filesystems: base.filesystems, - pwd: base.pwd, - compiler: base.compiler, - - programs: programs, - compatibilityMode: base.compatibilityMode, - exportsCache: make(map[string]goja.Value), - logger: base.logger, - moduleRegistry: base.moduleRegistry, - moduleVUImpl: vuImpl, - } -} - -// Require is called when a module/file needs to be loaded by a script -func (i *InitContext) Require(arg string) (export goja.Value) { - var ok bool - if export, ok = i.exportsCache[arg]; ok { - return export - } - defer func() { i.exportsCache[arg] = export }() - switch { - case arg == "k6", strings.HasPrefix(arg, "k6/"): - // Builtin or external modules ("k6", "k6/*", or "k6/x/*") are handled - // specially, as they don't exist on the filesystem. This intentionally - // shadows attempts to name your own modules this. - v, err := i.requireModule(arg) - if err != nil { - common.Throw(i.moduleVUImpl.runtime, err) - } - return v - default: - // Fall back to loading from the filesystem. - v, err := i.requireFile(arg) - if err != nil { - common.Throw(i.moduleVUImpl.runtime, err) - } - return v - } -} - -func toESModuleExports(exp modules.Exports) interface{} { - if exp.Named == nil { - return exp.Default - } - if exp.Default == nil { - return exp.Named - } - - result := make(map[string]interface{}, len(exp.Named)+2) - - for k, v := range exp.Named { - result[k] = v - } - // Maybe check that those weren't set - result["default"] = exp.Default - // this so babel works with the `default` when it transpiles from ESM to commonjs. - // This should probably be removed once we have support for ESM directly. So that require doesn't get support for - // that while ESM has. - result["__esModule"] = true - - return result -} - -func (i *InitContext) requireModule(name string) (goja.Value, error) { - mod, ok := i.moduleRegistry[name] - if !ok { - return nil, fmt.Errorf("unknown module: %s", name) - } - if m, ok := mod.(modules.Module); ok { - instance := m.NewModuleInstance(i.moduleVUImpl) - return i.moduleVUImpl.runtime.ToValue(toESModuleExports(instance.Exports())), nil - } - - return i.moduleVUImpl.runtime.ToValue(mod), nil -} - -func (i *InitContext) requireFile(name string) (goja.Value, error) { - // Resolve the file path, push the target directory as pwd to make relative imports work. - pwd := i.pwd - fileURL, err := loader.Resolve(pwd, name) - if err != nil { - return nil, err - } - - // First, check if we have a cached program already. - pgm, ok := i.programs[fileURL.String()] - if !ok || pgm.module == nil { - if filepath.IsAbs(name) && runtime.GOOS == "windows" { - i.logger.Warnf("'%s' was imported with an absolute path - this won't be cross-platform and won't work if"+ - " you move the script between machines or run it with `k6 cloud`; if absolute paths are required,"+ - " import them with the `file://` schema for slightly better compatibility", - name) - } - i.pwd = loader.Dir(fileURL) - defer func() { i.pwd = pwd }() - exports := i.moduleVUImpl.runtime.NewObject() - pgm.module = i.moduleVUImpl.runtime.NewObject() - _ = pgm.module.Set("exports", exports) - - if pgm.pgm == nil { - // Load the sources; the loader takes care of remote loading, etc. - data, err := loader.Load(i.logger, i.filesystems, fileURL, name) - if err != nil { - return goja.Undefined(), err - } - - pgm.src = string(data.Data) - - // Compile the sources; this handles ES5 vs ES6 automatically. - pgm.pgm, err = i.compileImport(pgm.src, data.URL.String()) - if err != nil { - return goja.Undefined(), err - } - } - - i.programs[fileURL.String()] = pgm - - // Run the program. - f, err := i.moduleVUImpl.runtime.RunProgram(pgm.pgm) - if err != nil { - delete(i.programs, fileURL.String()) - return goja.Undefined(), err - } - if call, ok := goja.AssertFunction(f); ok { - if _, err = call(exports, pgm.module, exports); err != nil { - return nil, err - } - } - } - - return pgm.module.Get("exports"), nil -} - -func (i *InitContext) compileImport(src, filename string) (*goja.Program, error) { - pgm, _, err := i.compiler.Compile(src, filename, false) - return pgm, err -} - -// Open implements open() in the init context and will read and return the +// openImpl implements openImpl() in the init context and will read and return the // contents of a file. If the second argument is "b" it returns an ArrayBuffer // instance, otherwise a string representation. -func (i *InitContext) Open(filename string, args ...string) (goja.Value, error) { - if i.moduleVUImpl.State() != nil { - return nil, errors.New(openCantBeUsedOutsideInitContextMsg) - } - - if filename == "" { - return nil, errors.New("open() can't be used with an empty filename") - } - +func openImpl(rt *goja.Runtime, fs afero.Fs, basePWD *url.URL, filename string, args ...string) (goja.Value, error) { // Here IsAbs should be enough but unfortunately it doesn't handle absolute paths starting from // the current drive on windows like `\users\noname\...`. Also it makes it more easy to test and // will probably be need for archive execution under windows if always consider '/...' as an // absolute path. if filename[0] != '/' && filename[0] != '\\' && !filepath.IsAbs(filename) { - filename = filepath.Join(i.pwd.Path, filename) + filename = filepath.Join(basePWD.Path, filename) } filename = filepath.Clean(filename) - fs := i.filesystems["file"] + if filename[0:1] != afero.FilePathSeparator { filename = afero.FilePathSeparator + filename } @@ -259,10 +42,10 @@ func (i *InitContext) Open(filename string, args ...string) (goja.Value, error) } if len(args) > 0 && args[0] == "b" { - ab := i.moduleVUImpl.runtime.NewArrayBuffer(data) - return i.moduleVUImpl.runtime.ToValue(&ab), nil + ab := rt.NewArrayBuffer(data) + return rt.ToValue(&ab), nil } - return i.moduleVUImpl.runtime.ToValue(string(data)), nil + return rt.ToValue(string(data)), nil } func readFile(fileSystem afero.Fs, filename string) (data []byte, err error) { @@ -288,9 +71,7 @@ func readFile(fileSystem afero.Fs, filename string) (data []byte, err error) { } // allowOnlyOpenedFiles enables seen only files -func (i *InitContext) allowOnlyOpenedFiles() { - fs := i.filesystems["file"] - +func allowOnlyOpenedFiles(fs afero.Fs) { alreadyOpenedFS, ok := fs.(fsext.OnlyCachedEnabler) if !ok { return @@ -298,3 +79,60 @@ func (i *InitContext) allowOnlyOpenedFiles() { alreadyOpenedFS.AllowOnlyCached() } + +type requireImpl struct { + vu modules.VU + modules *moduleSystem + pwd *url.URL +} + +func (r *requireImpl) require(specifier string) (*goja.Object, error) { + // TODO remove this in the future when we address https://github.com/grafana/k6/issues/2674 + // This is currently needed as each time require is called we need to record it's new pwd + // to be used if a require *or* open is used within the file as they are relative to the + // latest call to require. + // This is *not* the actual require behaviour defined in commonJS as it is actually always relative + // to the file it is in. This is unlikely to be an issue but this code is here to keep backwards + // compatibility *for now*. + // With native ESM this won't even be possible as `require` might not be called - instead an import + // might be used in which case we won't be able to be doing this hack. In that case we either will + // need some goja specific helper or to use stack traces as goja_nodejs does. + currentPWD := r.pwd + if specifier != "k6" && !strings.HasPrefix(specifier, "k6/") { + defer func() { + r.pwd = currentPWD + }() + // In theory we can give that downwards, but this makes the code more tightly coupled + // plus as explained above this will be removed in the future so the code reflects more + // closely what will be needed then + fileURL, err := loader.Resolve(r.pwd, specifier) + if err != nil { + return nil, err + } + r.pwd = loader.Dir(fileURL) + } + + if r.vu.State() != nil { // fix + return nil, fmt.Errorf(cantBeUsedOutsideInitContextMsg, "require") + } + if specifier == "" { + return nil, errors.New("require() can't be used with an empty specifier") + } + + return r.modules.Require(currentPWD, specifier) +} + +func generateSourceMapLoader(logger logrus.FieldLogger, filesystems map[string]afero.Fs, +) func(path string) ([]byte, error) { + return func(path string) ([]byte, error) { + u, err := url.Parse(path) + if err != nil { + return nil, err + } + data, err := loader.Load(logger, filesystems, u, path) + if err != nil { + return nil, err + } + return data.Data, nil + } +} diff --git a/js/initcontext_test.go b/js/initcontext_test.go index 00ef84ec387..180d62dd3a6 100644 --- a/js/initcontext_test.go +++ b/js/initcontext_test.go @@ -25,7 +25,7 @@ import ( "go.k6.io/k6/metrics" ) -func TestInitContextRequire(t *testing.T) { +func TestRequire(t *testing.T) { t.Parallel() t.Run("Modules", func(t *testing.T) { t.Run("Nonexistent", func(t *testing.T) { @@ -48,13 +48,11 @@ func TestInitContextRequire(t *testing.T) { bi, err := b.Instantiate(context.Background(), 0) assert.NoError(t, err, "instance error") - exports := bi.pgm.exports - require.NotNil(t, exports) - _, defaultOk := goja.AssertFunction(exports.Get("default")) + _, defaultOk := goja.AssertFunction(bi.getExported("default")) assert.True(t, defaultOk, "default export is not a function") - assert.Equal(t, "abc123", exports.Get("dummy").String()) + assert.Equal(t, "abc123", bi.getExported("dummy").String()) - k6 := exports.Get("_k6").ToObject(bi.Runtime) + k6 := bi.getExported("_k6").ToObject(bi.Runtime) require.NotNil(t, k6) _, groupOk := goja.AssertFunction(k6.Get("group")) assert.True(t, groupOk, "k6.group is not a function") @@ -73,13 +71,11 @@ func TestInitContextRequire(t *testing.T) { bi, err := b.Instantiate(context.Background(), 0) require.NoError(t, err) - exports := bi.pgm.exports - require.NotNil(t, exports) - _, defaultOk := goja.AssertFunction(exports.Get("default")) + _, defaultOk := goja.AssertFunction(bi.getExported("default")) assert.True(t, defaultOk, "default export is not a function") - assert.Equal(t, "abc123", exports.Get("dummy").String()) + assert.Equal(t, "abc123", bi.getExported("dummy").String()) - _, groupOk := goja.AssertFunction(exports.Get("_group")) + _, groupOk := goja.AssertFunction(bi.getExported("_group")) assert.True(t, groupOk, "{ group } is not a function") }) }) @@ -107,7 +103,7 @@ func TestInitContextRequire(t *testing.T) { require.NoError(t, afero.WriteFile(fs, "/file.js", []byte(`throw new Error("aaaa")`), 0o755)) _, err := getSimpleBundle(t, "/script.js", `import "/file.js"; export default function() {}`, fs) assert.EqualError(t, err, - "Error: aaaa\n\tat file:///file.js:2:7(3)\n\tat go.k6.io/k6/js.(*InitContext).Require-fm (native)\n\tat file:///script.js:1:0(15)\n") + "Error: aaaa\n\tat file:///file.js:2:7(3)\n\tat go.k6.io/k6/js.(*requireImpl).require-fm (native)\n\tat file:///script.js:1:0(15)\n") }) imports := map[string]struct { @@ -175,9 +171,6 @@ func TestInitContextRequire(t *testing.T) { libName) b, err := getSimpleBundle(t, "/path/to/script.js", data, fs) require.NoError(t, err) - if constPath != "" { - assert.Contains(t, b.BaseInitContext.programs, "file://"+constPath) - } _, err = b.Instantiate(context.Background(), 0) require.NoError(t, err) @@ -538,7 +531,7 @@ func TestRequestWithMultipleBinaryFiles(t *testing.T) { <-ch } -func TestInitContextVU(t *testing.T) { +func Test__VU(t *testing.T) { t.Parallel() b, err := getSimpleBundle(t, "/script.js", ` let vu = __VU; diff --git a/js/modules.go b/js/modules.go new file mode 100644 index 00000000000..06e2c954e47 --- /dev/null +++ b/js/modules.go @@ -0,0 +1,122 @@ +package js + +import ( + "fmt" + "net/url" + "strings" + + "github.com/dop251/goja" + "go.k6.io/k6/js/compiler" + "go.k6.io/k6/js/modules" + "go.k6.io/k6/loader" +) + +type module interface { + Instantiate(vu modules.VU) moduleInstance +} + +type moduleInstance interface { + execute() error + exports() *goja.Object +} +type moduleCacheElement struct { + mod module + err error +} + +type modulesResolution struct { + cache map[string]moduleCacheElement + goModules map[string]interface{} +} + +func newModuleResolution(goModules map[string]interface{}) *modulesResolution { + return &modulesResolution{goModules: goModules, cache: make(map[string]moduleCacheElement)} +} + +func (mr *modulesResolution) setMain(main *loader.SourceData, c *compiler.Compiler) error { + mod, err := cjsmoduleFromString(main.URL, main.Data, c) + mr.cache[main.URL.String()] = moduleCacheElement{mod: mod, err: err} + return err +} + +func (mr *modulesResolution) resolveSpecifier(basePWD *url.URL, arg string) (*url.URL, error) { + specifier, err := loader.Resolve(basePWD, arg) + if err != nil { + return nil, err + } + return specifier, nil +} + +func (mr *modulesResolution) requireModule(name string) (module, error) { + mod, ok := mr.goModules[name] + if !ok { + return nil, fmt.Errorf("unknown module: %s", name) + } + if m, ok := mod.(modules.Module); ok { + return &goModule{Module: m}, nil + } + + return &baseGoModule{mod: mod}, nil +} + +func (mr *modulesResolution) resolve(basePWD *url.URL, arg string, loadCJS cjsModuleLoader) (module, error) { + if cached, ok := mr.cache[arg]; ok { + return cached.mod, cached.err + } + switch { + case arg == "k6", strings.HasPrefix(arg, "k6/"): + // Builtin or external modules ("k6", "k6/*", or "k6/x/*") are handled + // specially, as they don't exist on the filesystem. + mod, err := mr.requireModule(arg) + mr.cache[arg] = moduleCacheElement{mod: mod, err: err} + return mod, err + default: + specifier, err := mr.resolveSpecifier(basePWD, arg) + if err != nil { + return nil, err + } + // try cache with the final specifier + if cached, ok := mr.cache[specifier.String()]; ok { + return cached.mod, cached.err + } + // Fall back to loading from the filesystem. + mod, err := loadCJS(specifier, arg) + mr.cache[specifier.String()] = moduleCacheElement{mod: mod, err: err} + return mod, err + } +} + +type moduleSystem struct { + vu modules.VU + instanceCache map[module]moduleInstance + resolution *modulesResolution + cjsLoad cjsModuleLoader +} + +func newModuleSystem(resolution *modulesResolution, vu modules.VU, cjsLoad cjsModuleLoader) *moduleSystem { + return &moduleSystem{ + resolution: resolution, + instanceCache: make(map[module]moduleInstance), + vu: vu, + cjsLoad: cjsLoad, + } +} + +// Require is called when a module/file needs to be loaded by a script +func (ms *moduleSystem) Require(pwd *url.URL, arg string) (*goja.Object, error) { + mod, err := ms.resolution.resolve(pwd, arg, ms.cjsLoad) + if err != nil { + return nil, err + } + if instance, ok := ms.instanceCache[mod]; ok { + return instance.exports(), nil + } + + instance := mod.Instantiate(ms.vu) + ms.instanceCache[mod] = instance + if err = instance.execute(); err != nil { + return nil, err + } + + return instance.exports(), nil +} diff --git a/js/runner.go b/js/runner.go index edb1d43c648..c925782a8c0 100644 --- a/js/runner.go +++ b/js/runner.go @@ -253,7 +253,7 @@ func (r *Runner) newVU( // instead of "Value is not an object: undefined ..." _ = vu.Runtime.GlobalObject().Set("open", func() { - common.Throw(vu.Runtime, errors.New(openCantBeUsedOutsideInitContextMsg)) + common.Throw(vu.Runtime, fmt.Errorf(cantBeUsedOutsideInitContextMsg, "open")) }) return vu, nil @@ -353,7 +353,7 @@ func (r *Runner) GetOptions() lib.Options { // IsExecutable returns whether the given name is an exported and // executable function in the script. func (r *Runner) IsExecutable(name string) bool { - _, exists := r.Bundle.exports[name] + _, exists := r.Bundle.callableExports[name] return exists }