From b01831e623ee26167880245eb9696ddf94dc2c08 Mon Sep 17 00:00:00 2001 From: rishigupta1599 Date: Thu, 28 Nov 2024 12:37:13 +0530 Subject: [PATCH 1/6] Fixing logic of resources with size > 25MB not skipping --- packages/client/src/client.js | 9 +++------ packages/client/src/utils.js | 8 ++++++++ packages/core/src/network.js | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/packages/client/src/client.js b/packages/client/src/client.js index 7bee9cce5..01bac4f6a 100644 --- a/packages/client/src/client.js +++ b/packages/client/src/client.js @@ -2,7 +2,6 @@ import fs from 'fs'; import PercyEnv from '@percy/env'; import { git } from '@percy/env/utils'; import logger from '@percy/logger'; -import Pako from 'pako'; import { pool, @@ -14,7 +13,8 @@ import { waitForTimeout, validateTiles, formatLogErrors, - tagsList + tagsList, + getEncodedContent } from './utils.js'; // Default client API URL can be set with an env var for API development @@ -342,11 +342,8 @@ export class PercyClient { validateId('build', buildId); if (filepath) { content = await fs.promises.readFile(filepath); - if (process.env.PERCY_GZIP) { - content = Pako.gzip(content); - } } - let encodedContent = base64encode(content); + let encodedContent = getEncodedContent(content); this.log.debug(`Uploading ${formatBytes(encodedContent.length)} resource: ${url}`, meta); this.mayBeLogUploadSize(encodedContent.length, meta); diff --git a/packages/client/src/utils.js b/packages/client/src/utils.js index 9bb0f81fd..a103b700d 100644 --- a/packages/client/src/utils.js +++ b/packages/client/src/utils.js @@ -4,6 +4,7 @@ import url from 'url'; import path from 'path'; import crypto from 'crypto'; import logger from '@percy/logger'; +import Pako from 'pako'; // Formats a raw byte integer as a string export function formatBytes(int) { @@ -260,6 +261,13 @@ export function tagsList(tags) { return tagsArr; } +export function getEncodedContent(content) { + if (process.env.PERCY_GZIP) { + content = Pako.gzip(content); + } + return base64encode(content); +} + export { hostnameMatches, getProxy, diff --git a/packages/core/src/network.js b/packages/core/src/network.js index cd7b22f3c..77bc15548 100644 --- a/packages/core/src/network.js +++ b/packages/core/src/network.js @@ -1,4 +1,4 @@ -import { request as makeRequest } from '@percy/client/utils'; +import { request as makeRequest, getEncodedContent } from '@percy/client/utils'; import logger from '@percy/logger'; import mime from 'mime-types'; import { DefaultMap, createResource, hostnameMatches, normalizeURL, waitFor, decodeAndEncodeURLWithLogging } from './utils.js'; @@ -471,7 +471,7 @@ async function saveResponseResource(network, request, session) { return log.debug('- Skipping remote resource', meta); } else if (!body.length) { return log.debug('- Skipping empty response', meta); - } else if (body.length > MAX_RESOURCE_SIZE) { + } else if (getEncodedContent(body).length > MAX_RESOURCE_SIZE) { return log.debug('- Skipping resource larger than 25MB', meta); } else if (!ALLOWED_STATUSES.includes(response.status)) { return log.debug(`- Skipping disallowed status [${response.status}]`, meta); From d94184b77daae70754146fbcd6a17f195edf1238 Mon Sep 17 00:00:00 2001 From: rishigupta1599 Date: Thu, 28 Nov 2024 13:20:51 +0530 Subject: [PATCH 2/6] removing pako gzip in client --- packages/client/src/utils.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/client/src/utils.js b/packages/client/src/utils.js index a103b700d..5db108026 100644 --- a/packages/client/src/utils.js +++ b/packages/client/src/utils.js @@ -262,9 +262,6 @@ export function tagsList(tags) { } export function getEncodedContent(content) { - if (process.env.PERCY_GZIP) { - content = Pako.gzip(content); - } return base64encode(content); } From c18b045d8209bce8bc7b8de8447e73dc88765b53 Mon Sep 17 00:00:00 2001 From: rishigupta1599 Date: Thu, 28 Nov 2024 13:22:35 +0530 Subject: [PATCH 3/6] lint fix --- packages/client/src/utils.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/client/src/utils.js b/packages/client/src/utils.js index 5db108026..0c4e5f21a 100644 --- a/packages/client/src/utils.js +++ b/packages/client/src/utils.js @@ -4,7 +4,6 @@ import url from 'url'; import path from 'path'; import crypto from 'crypto'; import logger from '@percy/logger'; -import Pako from 'pako'; // Formats a raw byte integer as a string export function formatBytes(int) { From 038ee327d514e30df933885464836bd4f68c5577 Mon Sep 17 00:00:00 2001 From: rishigupta1599 Date: Thu, 28 Nov 2024 13:58:42 +0530 Subject: [PATCH 4/6] Making 0.67 factor --- packages/client/src/client.js | 9 ++++++--- packages/client/src/utils.js | 4 ---- packages/core/src/network.js | 6 +++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/client/src/client.js b/packages/client/src/client.js index 01bac4f6a..7bee9cce5 100644 --- a/packages/client/src/client.js +++ b/packages/client/src/client.js @@ -2,6 +2,7 @@ import fs from 'fs'; import PercyEnv from '@percy/env'; import { git } from '@percy/env/utils'; import logger from '@percy/logger'; +import Pako from 'pako'; import { pool, @@ -13,8 +14,7 @@ import { waitForTimeout, validateTiles, formatLogErrors, - tagsList, - getEncodedContent + tagsList } from './utils.js'; // Default client API URL can be set with an env var for API development @@ -342,8 +342,11 @@ export class PercyClient { validateId('build', buildId); if (filepath) { content = await fs.promises.readFile(filepath); + if (process.env.PERCY_GZIP) { + content = Pako.gzip(content); + } } - let encodedContent = getEncodedContent(content); + let encodedContent = base64encode(content); this.log.debug(`Uploading ${formatBytes(encodedContent.length)} resource: ${url}`, meta); this.mayBeLogUploadSize(encodedContent.length, meta); diff --git a/packages/client/src/utils.js b/packages/client/src/utils.js index 0c4e5f21a..9bb0f81fd 100644 --- a/packages/client/src/utils.js +++ b/packages/client/src/utils.js @@ -260,10 +260,6 @@ export function tagsList(tags) { return tagsArr; } -export function getEncodedContent(content) { - return base64encode(content); -} - export { hostnameMatches, getProxy, diff --git a/packages/core/src/network.js b/packages/core/src/network.js index 77bc15548..532e909f5 100644 --- a/packages/core/src/network.js +++ b/packages/core/src/network.js @@ -1,9 +1,9 @@ -import { request as makeRequest, getEncodedContent } from '@percy/client/utils'; +import { request as makeRequest } from '@percy/client/utils'; import logger from '@percy/logger'; import mime from 'mime-types'; import { DefaultMap, createResource, hostnameMatches, normalizeURL, waitFor, decodeAndEncodeURLWithLogging } from './utils.js'; -const MAX_RESOURCE_SIZE = 25 * (1024 ** 2); // 25MB +const MAX_RESOURCE_SIZE = 25 * (1024 ** 2) * 0.67; // 25MB, 0.67 factor for accounting for base64 encoding const ALLOWED_STATUSES = [200, 201, 301, 302, 304, 307, 308]; const ALLOWED_RESOURCES = ['Document', 'Stylesheet', 'Image', 'Media', 'Font', 'Other']; const ABORTED_MESSAGE = 'Request was aborted by browser'; @@ -471,7 +471,7 @@ async function saveResponseResource(network, request, session) { return log.debug('- Skipping remote resource', meta); } else if (!body.length) { return log.debug('- Skipping empty response', meta); - } else if (getEncodedContent(body).length > MAX_RESOURCE_SIZE) { + } else if (body.length > MAX_RESOURCE_SIZE) { return log.debug('- Skipping resource larger than 25MB', meta); } else if (!ALLOWED_STATUSES.includes(response.status)) { return log.debug(`- Skipping disallowed status [${response.status}]`, meta); From a567e18b6c0b2871914acfde11fd27ea21db801f Mon Sep 17 00:00:00 2001 From: rishigupta1599 Date: Thu, 28 Nov 2024 14:00:18 +0530 Subject: [PATCH 5/6] changing factor --- packages/core/src/network.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/network.js b/packages/core/src/network.js index 532e909f5..8626de493 100644 --- a/packages/core/src/network.js +++ b/packages/core/src/network.js @@ -3,7 +3,7 @@ import logger from '@percy/logger'; import mime from 'mime-types'; import { DefaultMap, createResource, hostnameMatches, normalizeURL, waitFor, decodeAndEncodeURLWithLogging } from './utils.js'; -const MAX_RESOURCE_SIZE = 25 * (1024 ** 2) * 0.67; // 25MB, 0.67 factor for accounting for base64 encoding +const MAX_RESOURCE_SIZE = 25 * (1024 ** 2) * 0.63; // 25MB, 0.67 factor for accounting for base64 encoding const ALLOWED_STATUSES = [200, 201, 301, 302, 304, 307, 308]; const ALLOWED_RESOURCES = ['Document', 'Stylesheet', 'Image', 'Media', 'Font', 'Other']; const ABORTED_MESSAGE = 'Request was aborted by browser'; From 336aea438da4699da951e3276aa9fce84eff0636 Mon Sep 17 00:00:00 2001 From: rishigupta1599 Date: Thu, 28 Nov 2024 14:01:01 +0530 Subject: [PATCH 6/6] changing comment --- packages/core/src/network.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/network.js b/packages/core/src/network.js index 8626de493..ccdfb059e 100644 --- a/packages/core/src/network.js +++ b/packages/core/src/network.js @@ -3,7 +3,7 @@ import logger from '@percy/logger'; import mime from 'mime-types'; import { DefaultMap, createResource, hostnameMatches, normalizeURL, waitFor, decodeAndEncodeURLWithLogging } from './utils.js'; -const MAX_RESOURCE_SIZE = 25 * (1024 ** 2) * 0.63; // 25MB, 0.67 factor for accounting for base64 encoding +const MAX_RESOURCE_SIZE = 25 * (1024 ** 2) * 0.63; // 25MB, 0.63 factor for accounting for base64 encoding const ALLOWED_STATUSES = [200, 201, 301, 302, 304, 307, 308]; const ALLOWED_RESOURCES = ['Document', 'Stylesheet', 'Image', 'Media', 'Font', 'Other']; const ABORTED_MESSAGE = 'Request was aborted by browser';