diff --git a/lib/basedriver/commands/find.js b/lib/basedriver/commands/find.js index ba7de6ef8..a0faf843c 100644 --- a/lib/basedriver/commands/find.js +++ b/lib/basedriver/commands/find.js @@ -363,38 +363,50 @@ helpers.getScreenshotForImageFind = async function (screenWidth, screenHeight) { // of coordinates returned by the image match algorithm, since we match based // on the screenshot coordinates not the device coordinates themselves. There // are two potential types of mismatch: aspect ratio mismatch and scale - // mismatch. + // mismatch. we need to detect and fix both + + let scale = {xScale: 1.0, yScale: 1.0}; const screenAR = screenWidth / screenHeight; const shotAR = shotWidth / shotHeight; - if (screenAR !== shotAR) { + if (screenAR === shotAR) { + log.info('Screenshot aspect ratio matched screen aspect ratio'); + } else { + log.warn(`When trying to find an element, determined that the screen ` + + `aspect ratio and screenshot aspect ratio are different. Screen ` + + `is ${screenWidth}x${screenHeight} whereas screenshot is ` + + `${shotWidth}x${shotHeight}.`); + + // Select smaller ratio + // this.getScreenshot is 540x397, this.getDeviceSize is 1080x1920. + // The ratio is {xScale: 2, yScale: 4.83}. + // In this case, we must choose `xScale: 2` as scaleFactor. + // Because if Appium selects the both ratio, the screenshot will be distorted. + // If Appium selects the yScale, width will be bigger than real screenshot. const _xScale = (1.0 * shotWidth) / screenWidth; const _yScale = (1.0 * shotHeight) / screenHeight; + const scaleFactor = _xScale >= _yScale ? _yScale : _xScale; - let scaleFactor; - if (_xScale >= _yScale) { - scaleFactor = _yScale; - } else { - scaleFactor = _xScale; - } - log.info(`Scaling screenshot from ${screenWidth}x${screenHeight} to match ` + - `screen at ${screenWidth * scaleFactor}x${screenHeight * scaleFactor}. Scale factor is ${scaleFactor}`); + log.warn(`Resizing screenshot to ${shotWidth * scaleFactor}x${shotHeight * scaleFactor} to match ` + + `screen aspect ratio so that image element coordinates have a ` + + `greater chance of being correct.`); imgObj = imgObj.resize(shotWidth * scaleFactor, shotHeight * scaleFactor); - // shotWidth = imgObj.bitmap.width; - // shotHeight = imgObj.bitmap.height; + + scale.xScale *= scaleFactor; + scale.yScale *= scaleFactor; } // Resize based on the screen dimensions only if both width and height are mismatched // since except for that, it might be a situation which is different window rect and // screenshot size like `@driver.window_rect #=>x=0, y=0, width=1080, height=1794` and // `"deviceScreenSize"=>"1080x1920"` - let scale; if (screenWidth !== shotWidth && screenHeight !== shotHeight) { log.info(`Scaling screenshot from ${shotWidth}x${shotHeight} to match ` + `screen at ${screenWidth}x${screenHeight}`); imgObj = imgObj.resize(screenWidth, screenHeight); - scale = {xScale: (1.0 * screenWidth) / shotWidth, yScale: (1.0 * screenHeight) / shotHeight}; + scale.xScale *= (1.0 * screenWidth) / shotWidth; + scale.yScale *= (1.0 * screenHeight) / shotHeight; } b64Screenshot = (await imgObj.getBuffer(imageUtil.MIME_PNG)).toString('base64'); diff --git a/test/basedriver/commands/find-specs.js b/test/basedriver/commands/find-specs.js index 3169ef65c..c8e5a3c6c 100644 --- a/test/basedriver/commands/find-specs.js +++ b/test/basedriver/commands/find-specs.js @@ -281,21 +281,27 @@ describe('finding elements by image', function () { // try first with portrait screen let screen = [TINY_PNG_DIMS[0] * 2, TINY_PNG_DIMS[1] * 3]; + let expectedScale = { xScale: 0.67, yScale: 1 }; + const {b64Screenshot, scale} = await d.getScreenshotForImageFind(...screen); b64Screenshot.should.not.eql(TINY_PNG); let screenshotObj = await imageUtil.getJimpImage(b64Screenshot); screenshotObj.bitmap.width.should.eql(screen[0]); screenshotObj.bitmap.height.should.eql(screen[1]); - scale.should.eql({ xScale: 2, yScale: 3 }); + scale.xScale.toFixed(2).should.eql(expectedScale.xScale.toString()); + scale.yScale.should.eql(expectedScale.yScale); // then with landscape screen screen = [TINY_PNG_DIMS[0] * 3, TINY_PNG_DIMS[1] * 2]; + expectedScale = { xScale: 1, yScale: 0.67 }; + const {b64Screenshot: newScreen, scale: newScale} = await d.getScreenshotForImageFind(...screen); newScreen.should.not.eql(TINY_PNG); screenshotObj = await imageUtil.getJimpImage(newScreen); screenshotObj.bitmap.width.should.eql(screen[0]); screenshotObj.bitmap.height.should.eql(screen[1]); - newScale.should.eql({ xScale: 3, yScale: 2 }); + newScale.xScale.should.eql(expectedScale.xScale); + newScale.yScale.toFixed(2).should.eql(expectedScale.yScale.toString()); }); }); });