From 01bf098ed4d07951730369707b4357558cfbb1cb Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 11 Sep 2024 21:14:51 +0700 Subject: [PATCH 01/12] refactor(headless): use `WaitStable` for `waitload` action Signed-off-by: Dwi Siswanto --- pkg/protocols/headless/engine/page_actions.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index b34ed18b94..6481b1d483 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -520,16 +520,7 @@ func (p *Page) SelectInputElement(act *Action, out ActionData) error { // WaitLoad waits for the page to load func (p *Page) WaitLoad(act *Action, out ActionData) error { - p.page.Timeout(2 * time.Second).WaitNavigation(proto.PageLifecycleEventNameFirstMeaningfulPaint)() - - // Wait for the window.onload event and also wait for the network requests - // to become idle for a maximum duration of 3 seconds. If the requests - // do not finish, - if err := p.page.WaitLoad(); err != nil { - return errors.Wrap(err, "could not wait load event") - } - _ = p.page.WaitIdle(1 * time.Second) - return nil + return p.page.WaitStable(1) } // GetResource gets a resource from an element from page. From 499806fd32cee11b3caad3f0141f51f54bd5098b Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Fri, 13 Sep 2024 19:20:38 +0700 Subject: [PATCH 02/12] feat(headless): add `getNavigationFunc` Signed-off-by: Dwi Siswanto --- pkg/protocols/headless/engine/page_actions.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index 6481b1d483..fe3fa5d990 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -204,6 +204,17 @@ func createBackOffSleeper(pollTimeout, timeout time.Duration) utils.Sleeper { } } +func getNavigationFunc(p *Page, act *Action, event proto.PageLifecycleEventName) (func(), error) { + dur, err := getTimeout(p, act) + if err != nil { + return nil, errors.Wrap(err, "Wrong timeout given") + } + + fn := p.page.Timeout(dur).WaitNavigation(event) + + return fn, nil +} + func getTimeout(p *Page, act *Action) (time.Duration, error) { return geTimeParameter(p, act, "timeout", 3, time.Second) } From 6d41af6783bd452d33621febdb83a043008a7960 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Fri, 13 Sep 2024 19:41:15 +0700 Subject: [PATCH 03/12] feat(headless): add `WaitDOM` action Signed-off-by: Dwi Siswanto --- pkg/protocols/headless/engine/action_types.go | 5 +++++ pkg/protocols/headless/engine/page_actions.go | 15 +++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/pkg/protocols/headless/engine/action_types.go b/pkg/protocols/headless/engine/action_types.go index d61e377431..1000330674 100644 --- a/pkg/protocols/headless/engine/action_types.go +++ b/pkg/protocols/headless/engine/action_types.go @@ -45,6 +45,9 @@ const ( // ActionFilesInput performs an action on a file input. // name:files ActionFilesInput + // ActionWaitDOM waits for the HTML document has been completely loaded & parsed. + // name:waitdom + ActionWaitDOM // ActionWaitLoad waits for the page to stop loading. // name:waitload ActionWaitLoad @@ -102,6 +105,7 @@ var ActionStringToAction = map[string]ActionType{ "time": ActionTimeInput, "select": ActionSelectInput, "files": ActionFilesInput, + "waitdom": ActionWaitDOM, "waitload": ActionWaitLoad, "getresource": ActionGetResource, "extract": ActionExtract, @@ -129,6 +133,7 @@ var ActionToActionString = map[ActionType]string{ ActionTimeInput: "time", ActionSelectInput: "select", ActionFilesInput: "files", + ActionWaitDOM: "waitdom", ActionWaitLoad: "waitload", ActionGetResource: "getresource", ActionExtract: "extract", diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index fe3fa5d990..3dfda58daa 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -94,6 +94,9 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var err = p.TimeInputElement(act, outData) case ActionSelectInput: err = p.SelectInputElement(act, outData) + case ActionWaitDOM: + event := proto.PageLifecycleEventNameDOMContentLoaded + err = p.WaitPageLifecycleEvent(act, outData, event) case ActionWaitLoad: err = p.WaitLoad(act, outData) case ActionGetResource: @@ -529,6 +532,18 @@ func (p *Page) SelectInputElement(act *Action, out ActionData) error { return nil } +// WaitPageLifecycleEvent waits for specified page lifecycle event name +func (p *Page) WaitPageLifecycleEvent(act *Action, out ActionData, event proto.PageLifecycleEventName) error { + fn, err := getNavigationFunc(p, act, event) + if err != nil { + return err + } + + fn() + + return nil +} + // WaitLoad waits for the page to load func (p *Page) WaitLoad(act *Action, out ActionData) error { return p.page.WaitStable(1) From a5da79020d495cfef3bc2b9881e47b760e30fc03 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Fri, 13 Sep 2024 19:41:35 +0700 Subject: [PATCH 04/12] feat(headless): add `WaitFMP` action Signed-off-by: Dwi Siswanto --- pkg/protocols/headless/engine/action_types.go | 5 +++++ pkg/protocols/headless/engine/page_actions.go | 3 +++ 2 files changed, 8 insertions(+) diff --git a/pkg/protocols/headless/engine/action_types.go b/pkg/protocols/headless/engine/action_types.go index 1000330674..84a2e8db4c 100644 --- a/pkg/protocols/headless/engine/action_types.go +++ b/pkg/protocols/headless/engine/action_types.go @@ -48,6 +48,9 @@ const ( // ActionWaitDOM waits for the HTML document has been completely loaded & parsed. // name:waitdom ActionWaitDOM + // ActionWaitFMP waits for page has rendered enough meaningful content to be useful to the user. + // name:waitfmp + ActionWaitFMP // ActionWaitLoad waits for the page to stop loading. // name:waitload ActionWaitLoad @@ -106,6 +109,7 @@ var ActionStringToAction = map[string]ActionType{ "select": ActionSelectInput, "files": ActionFilesInput, "waitdom": ActionWaitDOM, + "waitfmp": ActionWaitFMP, "waitload": ActionWaitLoad, "getresource": ActionGetResource, "extract": ActionExtract, @@ -134,6 +138,7 @@ var ActionToActionString = map[ActionType]string{ ActionSelectInput: "select", ActionFilesInput: "files", ActionWaitDOM: "waitdom", + ActionWaitFMP: "waitfmp", ActionWaitLoad: "waitload", ActionGetResource: "getresource", ActionExtract: "extract", diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index 3dfda58daa..685f008d23 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -97,6 +97,9 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var case ActionWaitDOM: event := proto.PageLifecycleEventNameDOMContentLoaded err = p.WaitPageLifecycleEvent(act, outData, event) + case ActionWaitFMP: + event := proto.PageLifecycleEventNameFirstMeaningfulPaint + err = p.WaitPageLifecycleEvent(act, outData, event) case ActionWaitLoad: err = p.WaitLoad(act, outData) case ActionGetResource: From 65b397529feecaa6bd70a90c6f8eeaf5cbf695c1 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Fri, 13 Sep 2024 19:49:10 +0700 Subject: [PATCH 05/12] feat(headless): add `WaitFCP` action Signed-off-by: Dwi Siswanto --- pkg/protocols/headless/engine/action_types.go | 5 +++++ pkg/protocols/headless/engine/page_actions.go | 3 +++ 2 files changed, 8 insertions(+) diff --git a/pkg/protocols/headless/engine/action_types.go b/pkg/protocols/headless/engine/action_types.go index 84a2e8db4c..f258ba4960 100644 --- a/pkg/protocols/headless/engine/action_types.go +++ b/pkg/protocols/headless/engine/action_types.go @@ -48,6 +48,9 @@ const ( // ActionWaitDOM waits for the HTML document has been completely loaded & parsed. // name:waitdom ActionWaitDOM + // ActionWaitFCP waits for the first piece of content (text, image, etc.) is painted on the screen. + // name:waitfcp + ActionWaitFCP // ActionWaitFMP waits for page has rendered enough meaningful content to be useful to the user. // name:waitfmp ActionWaitFMP @@ -109,6 +112,7 @@ var ActionStringToAction = map[string]ActionType{ "select": ActionSelectInput, "files": ActionFilesInput, "waitdom": ActionWaitDOM, + "waitfcp": ActionWaitFCP, "waitfmp": ActionWaitFMP, "waitload": ActionWaitLoad, "getresource": ActionGetResource, @@ -138,6 +142,7 @@ var ActionToActionString = map[ActionType]string{ ActionSelectInput: "select", ActionFilesInput: "files", ActionWaitDOM: "waitdom", + ActionWaitFCP: "waitfcp", ActionWaitFMP: "waitfmp", ActionWaitLoad: "waitload", ActionGetResource: "getresource", diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index 685f008d23..5298b52f1f 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -97,6 +97,9 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var case ActionWaitDOM: event := proto.PageLifecycleEventNameDOMContentLoaded err = p.WaitPageLifecycleEvent(act, outData, event) + case ActionWaitFCP: + event := proto.PageLifecycleEventNameFirstContentfulPaint + err = p.WaitPageLifecycleEvent(act, outData, event) case ActionWaitFMP: event := proto.PageLifecycleEventNameFirstMeaningfulPaint err = p.WaitPageLifecycleEvent(act, outData, event) From 2b23a298daa82401d6fbe62292cc20e7400127b6 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Fri, 13 Sep 2024 19:56:56 +0700 Subject: [PATCH 06/12] feat(headless): add `WaitIdle` action Signed-off-by: Dwi Siswanto --- pkg/protocols/headless/engine/action_types.go | 5 +++++ pkg/protocols/headless/engine/page_actions.go | 3 +++ 2 files changed, 8 insertions(+) diff --git a/pkg/protocols/headless/engine/action_types.go b/pkg/protocols/headless/engine/action_types.go index f258ba4960..cc1fd843d0 100644 --- a/pkg/protocols/headless/engine/action_types.go +++ b/pkg/protocols/headless/engine/action_types.go @@ -54,6 +54,9 @@ const ( // ActionWaitFMP waits for page has rendered enough meaningful content to be useful to the user. // name:waitfmp ActionWaitFMP + // ActionWaitIdle waits for the network is completely idle (no ongoing network requests). + // name:waitidle + ActionWaitIdle // ActionWaitLoad waits for the page to stop loading. // name:waitload ActionWaitLoad @@ -114,6 +117,7 @@ var ActionStringToAction = map[string]ActionType{ "waitdom": ActionWaitDOM, "waitfcp": ActionWaitFCP, "waitfmp": ActionWaitFMP, + "waitidle": ActionWaitIdle, "waitload": ActionWaitLoad, "getresource": ActionGetResource, "extract": ActionExtract, @@ -144,6 +148,7 @@ var ActionToActionString = map[ActionType]string{ ActionWaitDOM: "waitdom", ActionWaitFCP: "waitfcp", ActionWaitFMP: "waitfmp", + ActionWaitIdle: "waitidle", ActionWaitLoad: "waitload", ActionGetResource: "getresource", ActionExtract: "extract", diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index 5298b52f1f..6772392589 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -103,6 +103,9 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var case ActionWaitFMP: event := proto.PageLifecycleEventNameFirstMeaningfulPaint err = p.WaitPageLifecycleEvent(act, outData, event) + case ActionWaitIdle: + event := proto.PageLifecycleEventNameNetworkIdle + err = p.WaitPageLifecycleEvent(act, outData, event) case ActionWaitLoad: err = p.WaitLoad(act, outData) case ActionGetResource: From 6ac42dd33103d913bba5bfcbca062753d4ff450d Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Fri, 13 Sep 2024 20:25:26 +0700 Subject: [PATCH 07/12] refactor(headless): `ActionWaitLoad` waits for `proto.PageLifecycleEventNameLoad` also rename `Page.WaitLoad` to `Page.WaitStable` method. Signed-off-by: Dwi Siswanto --- pkg/protocols/headless/engine/action_types.go | 2 +- pkg/protocols/headless/engine/page_actions.go | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/protocols/headless/engine/action_types.go b/pkg/protocols/headless/engine/action_types.go index cc1fd843d0..2f015ef4d7 100644 --- a/pkg/protocols/headless/engine/action_types.go +++ b/pkg/protocols/headless/engine/action_types.go @@ -57,7 +57,7 @@ const ( // ActionWaitIdle waits for the network is completely idle (no ongoing network requests). // name:waitidle ActionWaitIdle - // ActionWaitLoad waits for the page to stop loading. + // ActionWaitLoad waits for the page and all its resources (like stylesheets and images) have finished loading. // name:waitload ActionWaitLoad // ActionGetResource performs a get resource action on an element diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index 6772392589..64b9d469d2 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -107,7 +107,8 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var event := proto.PageLifecycleEventNameNetworkIdle err = p.WaitPageLifecycleEvent(act, outData, event) case ActionWaitLoad: - err = p.WaitLoad(act, outData) + event := proto.PageLifecycleEventNameLoad + err = p.WaitPageLifecycleEvent(act, outData, event) case ActionGetResource: err = p.GetResource(act, outData) case ActionExtract: @@ -553,9 +554,9 @@ func (p *Page) WaitPageLifecycleEvent(act *Action, out ActionData, event proto.P return nil } -// WaitLoad waits for the page to load -func (p *Page) WaitLoad(act *Action, out ActionData) error { - return p.page.WaitStable(1) +// WaitStable waits until the page is stable +func (p *Page) WaitStable(act *Action, out ActionData) error { + return p.page.WaitStable(1) // 1ns } // GetResource gets a resource from an element from page. From 0d5181359c543df3e20a22cb6050421df4d973e1 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Fri, 13 Sep 2024 20:40:37 +0700 Subject: [PATCH 08/12] feat(headless): add `WaitStable` action Signed-off-by: Dwi Siswanto --- pkg/protocols/headless/engine/action_types.go | 5 +++++ pkg/protocols/headless/engine/page_actions.go | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/pkg/protocols/headless/engine/action_types.go b/pkg/protocols/headless/engine/action_types.go index 2f015ef4d7..d66fe91130 100644 --- a/pkg/protocols/headless/engine/action_types.go +++ b/pkg/protocols/headless/engine/action_types.go @@ -60,6 +60,9 @@ const ( // ActionWaitLoad waits for the page and all its resources (like stylesheets and images) have finished loading. // name:waitload ActionWaitLoad + // ActionWaitStable waits until the page is stable. + // name:waitstable + ActionWaitStable // ActionGetResource performs a get resource action on an element // name:getresource ActionGetResource @@ -119,6 +122,7 @@ var ActionStringToAction = map[string]ActionType{ "waitfmp": ActionWaitFMP, "waitidle": ActionWaitIdle, "waitload": ActionWaitLoad, + "waitstable": ActionWaitStable, "getresource": ActionGetResource, "extract": ActionExtract, "setmethod": ActionSetMethod, @@ -150,6 +154,7 @@ var ActionToActionString = map[ActionType]string{ ActionWaitFMP: "waitfmp", ActionWaitIdle: "waitidle", ActionWaitLoad: "waitload", + ActionWaitStable: "waitstable", ActionGetResource: "getresource", ActionExtract: "extract", ActionSetMethod: "setmethod", diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index 64b9d469d2..c5fc7996bd 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -109,6 +109,13 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action, var case ActionWaitLoad: event := proto.PageLifecycleEventNameLoad err = p.WaitPageLifecycleEvent(act, outData, event) + case ActionWaitStable: + err = p.WaitStable(act, outData) + // NOTE(dwisiswant0): Mapping `ActionWaitLoad` to `Page.WaitStable`, + // just in case waiting for the `proto.PageLifecycleEventNameLoad` event + // doesn't meet expectations. + // case ActionWaitLoad, ActionWaitStable: + // err = p.WaitStable(act, outData) case ActionGetResource: err = p.GetResource(act, outData) case ActionExtract: From ffcd387aeea3a881bc1d68d7a379f45c5b1dab7e Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Fri, 13 Sep 2024 23:47:52 +0700 Subject: [PATCH 09/12] refactor(headless): supporting `duration` arg for `WaitStable` action Signed-off-by: Dwi Siswanto --- pkg/protocols/headless/engine/page_actions.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index c5fc7996bd..a3b709839f 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -563,7 +563,18 @@ func (p *Page) WaitPageLifecycleEvent(act *Action, out ActionData, event proto.P // WaitStable waits until the page is stable func (p *Page) WaitStable(act *Action, out ActionData) error { - return p.page.WaitStable(1) // 1ns + var err error + var dur time.Duration = time.Second // default 1s + + argDur := act.Data["duration"] + if argDur != "" { + dur, err = time.ParseDuration(argDur) + if err != nil { + dur = time.Second + } + } + + return p.page.WaitStable(dur) } // GetResource gets a resource from an element from page. From ce09c0e09ae471246ef75b761fe57d68e3d7752f Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sat, 14 Sep 2024 04:12:26 +0700 Subject: [PATCH 10/12] chore: ignore `*.png` Signed-off-by: Dwi Siswanto --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5a50568910..146b0a892c 100644 --- a/.gitignore +++ b/.gitignore @@ -39,4 +39,7 @@ dist pkg/protocols/common/helpers/deserialization/testdata/Deserialize.class pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser -vendor \ No newline at end of file +vendor + +# Headless `screenshot` action +*.png \ No newline at end of file From 9f9a03164a044863bf15078d9e4aa7a8c9046587 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sat, 14 Sep 2024 04:19:01 +0700 Subject: [PATCH 11/12] test(headless): update `TestActionScreenshot*` call `ActionWaitFMP` instead of `WaitLoad` before take screenshot Signed-off-by: Dwi Siswanto --- pkg/protocols/headless/engine/page_actions_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/protocols/headless/engine/page_actions_test.go b/pkg/protocols/headless/engine/page_actions_test.go index 3620bb2bb4..bc85ea7417 100644 --- a/pkg/protocols/headless/engine/page_actions_test.go +++ b/pkg/protocols/headless/engine/page_actions_test.go @@ -201,7 +201,7 @@ func TestActionScreenshot(t *testing.T) { filePath := filepath.Join(os.TempDir(), "test.png") actions := []*Action{ {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitFMP}}, {ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: map[string]string{"to": filePath}}, } @@ -229,7 +229,7 @@ func TestActionScreenshotToDir(t *testing.T) { actions := []*Action{ {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitFMP}}, {ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: map[string]string{"to": filePath, "mkdir": "true"}}, } From 91320f676fd68025ecc6b55a3e334bfb167a628f Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 19 Sep 2024 18:30:37 +0700 Subject: [PATCH 12/12] feat(headless): chained with `Timeout` when `WaitStable` Signed-off-by: Dwi Siswanto --- pkg/protocols/headless/engine/page_actions.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index a3b709839f..18ed515662 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -563,8 +563,12 @@ func (p *Page) WaitPageLifecycleEvent(act *Action, out ActionData, event proto.P // WaitStable waits until the page is stable func (p *Page) WaitStable(act *Action, out ActionData) error { - var err error - var dur time.Duration = time.Second // default 1s + var dur time.Duration = time.Second // default stable page duration: 1s + + timeout, err := getTimeout(p, act) + if err != nil { + return errors.Wrap(err, "Wrong timeout given") + } argDur := act.Data["duration"] if argDur != "" { @@ -574,7 +578,7 @@ func (p *Page) WaitStable(act *Action, out ActionData) error { } } - return p.page.WaitStable(dur) + return p.page.Timeout(timeout).WaitStable(dur) } // GetResource gets a resource from an element from page.