diff --git a/__tests__/server/utils/__snapshots__/devCdnFactory.spec.js.snap b/__tests__/server/utils/__snapshots__/devCdnFactory.spec.js.snap index 496d0a515..d7dc682b9 100644 --- a/__tests__/server/utils/__snapshots__/devCdnFactory.spec.js.snap +++ b/__tests__/server/utils/__snapshots__/devCdnFactory.spec.js.snap @@ -26,6 +26,8 @@ exports[`one-app-dev-cdn module-map.json uses the local map overriding the cdn u } `; +exports[`one-app-dev-cdn modules gets remote modules from cached data if incoming url is matching: module map response 1`] = `"{"key":"not-used-in-development","modules":{"module-b":{"node":{"url":"http://localhost:3001/static/cdn/module-b/1.0.0/module-b.node.js","integrity":"123"},"browser":{"url":"http://localhost:3001/static/cdn/module-b/1.0.0/module-b.browser.js","integrity":"234"},"legacyBrowser":{"url":"http://localhost:3001/static/cdn/module-b/1.0.0/module-b.legacy.browser.js","integrity":"345"}}}}"`; + exports[`one-app-dev-cdn modules gets remote modules: module map response 1`] = `"{"key":"not-used-in-development","modules":{"module-b":{"node":{"url":"http://localhost:3001/static/cdn/module-b/1.0.0/module-b.node.js","integrity":"123"},"browser":{"url":"http://localhost:3001/static/cdn/module-b/1.0.0/module-b.browser.js","integrity":"234"},"legacyBrowser":{"url":"http://localhost:3001/static/cdn/module-b/1.0.0/module-b.legacy.browser.js","integrity":"345"}}}}"`; exports[`one-app-dev-cdn modules returns a 404 if a request for something not known as a module from the module map comes in: module map response 1`] = `"{"key":"not-used-in-development","modules":{"module-b":{"node":{"url":"http://localhost:3001/static/cdn/module-b/1.0.0/module-b.node.js","integrity":"123"},"browser":{"url":"http://localhost:3001/static/cdn/module-b/1.0.0/module-b.browser.js","integrity":"234"},"legacyBrowser":{"url":"http://localhost:3001/static/cdn/module-b/1.0.0/module-b.legacy.browser.js","integrity":"345"}}}}"`; diff --git a/__tests__/server/utils/cacheCDNModules.spec.js b/__tests__/server/utils/cacheCDNModules.spec.js new file mode 100644 index 000000000..6353fac62 --- /dev/null +++ b/__tests__/server/utils/cacheCDNModules.spec.js @@ -0,0 +1,251 @@ +/* + * Copyright 2023 American Express Travel Related Services Company, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,either express + * or implied. See the License for the specific language governing permissions and limitations + * under the License. + */ + +import fs from 'fs'; + +import { + getUserHomeDirectory, + getCachedModules, + writeToCache, + removeDuplicatedModules, + showCacheInfo, + setupCacheFile, + cacheFileName, + oneAppDirectoryName, + oneAppDirectoryPath, + oneAppModuleCachePath, +} from '../../../src/server/utils/cacheCDNModules'; + +jest.mock('fs'); +jest.mock('chalk', () => ({ + bold: { + greenBright: (txt) => txt, + cyanBright: (txt) => txt, + redBright: (txt) => txt, + }, +})); + +describe('Cache module utils', () => { + let logSpy; + let errorSpy; + + beforeEach(() => { + logSpy = jest.spyOn(console, 'log'); + errorSpy = jest.spyOn(console, 'error'); + process.env.HOME = ''; + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('should get USERPROFILE for windows user', () => { + delete process.env.HOME; + process.env.USERPROFILE = 'Users/windows'; + expect(getUserHomeDirectory()).toBe('Users/windows'); + }); + + describe('showCacheInfo', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should display the cache info when there is no error', () => { + const mockStats = { + size: 1048576 * 5, // 5 MB + }; + + fs.stat.mockImplementation((_path, callback) => { + callback(null, mockStats); + }); + + showCacheInfo(); + + expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('CACHE INFORMATION')); + expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('File size of .one-app-module-cache: 5.00')); + expect(errorSpy).not.toHaveBeenCalled(); + }); + + it('should display an error when there is an error checking file stats', () => { + const mockError = new Error('Test error'); + + fs.stat.mockImplementation((_path, callback) => { + callback(mockError, null); + }); + + showCacheInfo(); + + expect(errorSpy).toHaveBeenCalledWith('There was error checking file stat', mockError); + }); + }); + + describe('setupCacheFile', () => { + beforeEach(() => { + jest.clearAllMocks(); + fs.mkdir.mockRestore(); + fs.writeFileSync.mockRestore(); + }); + + it('should log success message when directory and file are created', () => { + fs.mkdir.mockImplementationOnce((_filePath, _options, cb) => cb(null)); + fs.writeFileSync.mockImplementationOnce(() => ({})); + + setupCacheFile(); + + expect(logSpy).toHaveBeenCalledWith(`Successfully created ${oneAppDirectoryPath}`); + expect(logSpy).toHaveBeenCalledWith(`Creating ${cacheFileName}`); + expect(logSpy).toHaveBeenCalledWith(`${cacheFileName} created successfully on ${oneAppModuleCachePath}`); + }); + + it('should log error when unable to create a directory', () => { + fs.mkdir.mockImplementationOnce((_filePath, _options, cb) => cb(new Error('Error creating directory'))); + fs.writeFileSync.mockImplementationOnce(() => {}); + + setupCacheFile(); + expect(errorSpy).toHaveBeenCalledWith(`There was error creating ${oneAppDirectoryName} directory`); + fs.mkdir.mockRestore(); + }); + + it('should log error when unable to create a file', () => { + const error = new Error('Cannot create file'); + + fs.mkdir.mockImplementationOnce((_filePath, _options, cb) => cb(null)); + fs.writeFileSync.mockImplementationOnce(() => { throw error; }); + setupCacheFile(); + expect(errorSpy).toHaveBeenCalledWith(`Error creating ${cacheFileName} on ${oneAppModuleCachePath}, \n${error}`); + }); + }); + + describe('getCachedModules', () => { + it('should return an empty object if the cache file does not exist', () => { + fs.existsSync.mockImplementationOnce(() => false); + + const result = getCachedModules(); + + expect(result).toEqual({}); + }); + + it('should create a new cache file and return an empty object if the cache file does not exist', () => { + fs.existsSync.mockImplementationOnce(() => false); + fs.mkdir.mockImplementationOnce((_filePath, _options, cb) => cb(null)); + fs.writeFileSync.mockImplementationOnce(() => {}); + + const result = getCachedModules(); + + expect(logSpy).toHaveBeenCalledWith(`Successfully created ${oneAppDirectoryPath}`); + expect(logSpy).toHaveBeenCalledWith(`${cacheFileName} created successfully on ${oneAppModuleCachePath}`); + expect(result).toEqual({}); + }); + + it('should return an empty object if the cache file contains invalid JSON', () => { + const invalidJSON = 'invalid JSON'; + fs.existsSync.mockImplementationOnce(() => true); + fs.readFileSync.mockImplementationOnce(() => invalidJSON); + + const result = getCachedModules(); + let error; + try { + JSON.parse(invalidJSON); + } catch (err) { + error = err; + } + expect(errorSpy).toHaveBeenCalledWith('Could not parse JSON content', error); + expect(result).toEqual({}); + }); + + it('should return the content of the cache file as a JSON object if the cache file exists and contains valid JSON', () => { + const validJSON = '{"module":"test"}'; + fs.existsSync.mockImplementationOnce(() => true); + fs.readFileSync.mockImplementationOnce(() => validJSON); + + const result = getCachedModules(); + + expect(result).toEqual(JSON.parse(validJSON)); + }); + }); + + describe('writeToCache', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('should set content on cache after a delay', () => { + fs.writeFile.mockImplementation((_filePath, _content, callback) => callback(null)); + + const content = { module: 'test' }; + writeToCache(content); + + expect(fs.writeFile).not.toHaveBeenCalled(); + + jest.runAllTimers(); + + expect(fs.writeFile).toHaveBeenCalled(); + expect(fs.writeFile.mock.calls[0][1]).toBe(JSON.stringify(content, null, 2)); + }); + + it('should handle error when writing to file fails', () => { + const error = new Error('write error'); + fs.writeFile.mockImplementation((_filePath, _content, callback) => callback(error)); + + const content = { module: 'test' }; + writeToCache(content); + + jest.runAllTimers(); + + expect(fs.writeFile).toHaveBeenCalled(); + expect(logSpy).toHaveBeenCalledWith(`There was an error updating content \n ${error}`); + }); + }); + + describe('removeDuplicatedModules', () => { + it('removes the matching modules from cachedModules', () => { + const url = '/somepath/moduleA/someotherpath'; + const cachedModules = { + '/path/to/moduleA/1': 'data', + '/path/to/moduleA/2': 'data', + '/path/to/moduleB/1': 'data', + }; + const moduleNames = ['moduleA', 'moduleB', 'moduleC']; + + const result = removeDuplicatedModules(url, cachedModules, moduleNames); + + expect(result).toEqual({ + '/path/to/moduleB/1': 'data', + }); + + expect(logSpy).toHaveBeenCalledWith('Deleted /path/to/moduleA/1 from cache'); + expect(logSpy).toHaveBeenCalledWith('Deleted /path/to/moduleA/2 from cache'); + }); + + it('returns cachedModules unchanged if no module matches', () => { + const url = '/somepath/moduleX/someotherpath'; + const cachedModules = { + '/path/to/moduleA/1': 'data', + '/path/to/moduleA/2': 'data', + '/path/to/moduleB/1': 'data', + }; + const moduleNames = ['moduleA', 'moduleB', 'moduleC']; + + const result = removeDuplicatedModules(url, cachedModules, moduleNames); + + expect(result).toEqual(cachedModules); + expect(logSpy).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/__tests__/server/utils/devCdnFactory.spec.js b/__tests__/server/utils/devCdnFactory.spec.js index 7094b7f9d..470236b6f 100644 --- a/__tests__/server/utils/devCdnFactory.spec.js +++ b/__tests__/server/utils/devCdnFactory.spec.js @@ -20,9 +20,18 @@ import path from 'path'; import mkdirp from 'mkdirp'; import ProxyAgent from 'proxy-agent'; import oneAppDevCdn from '../../../src/server/utils/devCdnFactory'; +import { removeDuplicatedModules } from '../../../src/server/utils/cacheCDNModules'; jest.mock('node-fetch'); +jest.mock('../../../src/server/utils/cacheCDNModules', () => ({ + getCachedModules: jest.fn(() => ({ + '/cdn/module-b/1.0.0/module-c.node.js': 'console.log("c");', + })), + writeToCache: jest.fn(() => ({})), + removeDuplicatedModules: jest.fn(() => ({})), +})); + const pathToStubs = path.join(__dirname, 'stubs'); const pathToCache = path.join(__dirname, '..', '.cache'); const mockLocalDevPublicPath = path.join(pathToStubs, 'public'); @@ -33,6 +42,7 @@ describe('one-app-dev-cdn', () => { jest.spyOn(console, 'warn'); jest.spyOn(console, 'log'); jest.spyOn(console, 'error'); + const defaultLocalMap = { key: 'not-used-in-development', modules: { @@ -140,6 +150,7 @@ describe('one-app-dev-cdn', () => { }, }, }; + removeDuplicatedModules.mockImplementation(() => ({})); fetch.mockImplementation((url) => Promise.reject(new Error(`no mock for ${url} set up`))); }); @@ -547,6 +558,32 @@ describe('one-app-dev-cdn', () => { ]); }); + it('gets remote modules from cached data if incoming url is matching', async () => { + expect.assertions(6); + const fcdn = setupTest({ + useLocalModules: false, + appPort: 3000, + remoteModuleMapUrl: 'https://example.com/module-map.json', + }); + fetch.mockReturnJsonOnce(defaultRemoteMap); + fetch.mockReturnFileOnce('console.log("a");'); + + const moduleMapResponse = await fcdn.inject() + .get('/module-map.json'); + + expect(moduleMapResponse.statusCode).toBe(200); + expect(moduleMapResponse.headers['content-type']).toMatch(/^application\/json/); + expect( + sanitizeModuleMapForSnapshot(moduleMapResponse.body) + ).toMatchSnapshot('module map response'); + + const moduleResponse = await fcdn.inject() + .get('/cdn/module-b/1.0.0/module-c.node.js'); + expect(moduleResponse.statusCode).toBe(200); + expect(moduleResponse.headers['content-type']).toMatch('.js'); + expect(moduleResponse.body).toBe('console.log("c");'); + }); + it('returns a 404 if a request for something not known as a module from the module map comes in', async () => { expect.assertions(5); diff --git a/src/server/utils/cacheCDNModules.js b/src/server/utils/cacheCDNModules.js new file mode 100644 index 000000000..e86a9cf5e --- /dev/null +++ b/src/server/utils/cacheCDNModules.js @@ -0,0 +1,108 @@ +/* + * Copyright 2023 American Express Travel Related Services Company, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,either express + * or implied. See the License for the specific language governing permissions and limitations + * under the License. + */ + +import path from 'path'; +import fs from 'fs'; +import chalk from 'chalk'; + +export const getUserHomeDirectory = () => process.env.HOME || process.env.USERPROFILE; +export const cacheFileName = '.one-app-module-cache'; +export const moduleCacheFileName = '.one-app-module-map-cache'; +export const oneAppDirectoryName = '.one-app'; +export const oneAppDirectoryPath = path.join(getUserHomeDirectory(), oneAppDirectoryName); +export const oneAppModuleCachePath = path.join(oneAppDirectoryPath, cacheFileName); + +// show cache size and how to delete info on start +export const showCacheInfo = () => { + fs.stat(oneAppModuleCachePath, (error, stats) => { + if (error) { + console.error('There was error checking file stat', error); + } else { + const fileSizeOnMB = stats.size / (1024 * 1024); // bytes to mb + const message = `File size of ${cacheFileName}: ${chalk.bold.greenBright(fileSizeOnMB.toFixed(2), 'MB')}`; + const separator = '*'.repeat(message.length); + console.log(chalk.bold.cyanBright(separator)); + console.log(chalk.bold.redBright('CACHE INFORMATION')); + console.log(message); + console.log(`To delete cache, please run \n ${chalk.bold.redBright(' rm ', oneAppModuleCachePath)}`); + console.log(chalk.bold.cyanBright(separator)); + } + }); +}; + +// setup folder and file +export const setupCacheFile = () => { + fs.mkdir(oneAppDirectoryPath, { recursive: true }, (error) => { + if (error) { + console.error(`There was error creating ${oneAppDirectoryName} directory`); + } else { + console.log(`Successfully created ${oneAppDirectoryPath}`); + console.log(`Creating ${cacheFileName}`); + try { + fs.writeFileSync(oneAppModuleCachePath, JSON.stringify('{}')); + console.log(`${cacheFileName} created successfully on ${oneAppModuleCachePath}`); + } catch (err) { + console.error(`Error creating ${cacheFileName} on ${oneAppModuleCachePath}, \n${err}`); + } + } + }); +}; + +// gets cached module from ~/.one-app/.one-app-module-cache +export const getCachedModules = () => { + let hasCachedFile = false; + if (fs.existsSync(oneAppModuleCachePath)) { + hasCachedFile = true; + } else { + setupCacheFile(); + } + if (hasCachedFile) { + try { + showCacheInfo(); + const cachedContent = fs.readFileSync(oneAppModuleCachePath, 'utf8'); + return JSON.parse(cachedContent); + } catch (err) { + console.error('Could not parse JSON content', err); + } + } + return {}; +}; + +let timerId = null; + +export const writeToCache = (content, delay = 500) => { + // added debounce + clearTimeout(timerId); + timerId = setTimeout(() => { + fs.writeFile(oneAppModuleCachePath, JSON.stringify(content, null, 2), (error) => { + if (error) { + console.log(`There was an error updating content \n ${error}`); + } + }); + timerId = null; + }, delay); +}; + +export const removeDuplicatedModules = (url, cachedModules, moduleNames) => { + const matchingModule = moduleNames.find((moduleName) => url.match(new RegExp(`\\b\\/${moduleName}\\/\\b`))); + + const updatedCachedModules = cachedModules; + Object.keys(updatedCachedModules).forEach((cachedModuleKey) => { + if (cachedModuleKey.match(new RegExp(`\\b\\/${matchingModule}\\/\\b`))) { + delete updatedCachedModules[cachedModuleKey]; + console.log(`Deleted ${cachedModuleKey} from cache`); + } + }); + return updatedCachedModules; +}; diff --git a/src/server/utils/devCdnFactory.js b/src/server/utils/devCdnFactory.js index 1898bd3eb..eb60a53f5 100644 --- a/src/server/utils/devCdnFactory.js +++ b/src/server/utils/devCdnFactory.js @@ -24,6 +24,10 @@ import Fastify from 'fastify'; import ip from 'ip'; import ProxyAgent from 'proxy-agent'; import fetch from 'node-fetch'; +import { getCachedModules, writeToCache, removeDuplicatedModules } from './cacheCDNModules'; + +let moduleNames = []; +const cachedModules = getCachedModules(); const getLocalModuleMap = ({ pathToModuleMap, oneAppDevCdnAddress }) => { const moduleMap = JSON.parse(fs.readFileSync(pathToModuleMap, 'utf8').toString()); @@ -155,7 +159,7 @@ export const oneAppDevCdnFactory = ({ ...localMap.modules, }, }; - + moduleNames = Object.keys(map.modules); reply .code(200) .send(map); @@ -170,15 +174,29 @@ export const oneAppDevCdnFactory = ({ remoteModuleBaseUrls ); const remoteModuleBaseUrlOrigin = new URL(knownRemoteModuleBaseUrl).origin; + if (cachedModules[incomingRequestPath]) { + return reply + .code(200) + .type(path.extname(incomingRequestPath)) + .send(cachedModules[incomingRequestPath]); + } const remoteModuleResponse = await fetch(`${remoteModuleBaseUrlOrigin}/${incomingRequestPath}`, { headers: { connection: 'keep-alive' }, agent: new ProxyAgent(), }); const { status, type } = remoteModuleResponse; + const responseText = await remoteModuleResponse.text(); + const updatedCachedModules = removeDuplicatedModules( + incomingRequestPath, + cachedModules, + moduleNames + ); + updatedCachedModules[incomingRequestPath] = responseText; + writeToCache(updatedCachedModules); reply .code(status) .type(type) - .send(await remoteModuleResponse.text()); + .send(responseText); } else { reply .code(404)