From da0ec3c5a58ccfcfd99ed7906de0bedbba8b2a4b Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Tue, 17 Sep 2024 21:43:26 -0500 Subject: [PATCH 01/10] Add lite api --- graphql-yoga.mjs | 2 + index.mjs | 59 +++++++-------- plugins/plugin-lite-api.mjs | 145 ++++++++++++++++++++++++++++++++++++ plugins/plugin-nightbot.mjs | 4 +- 4 files changed, 174 insertions(+), 36 deletions(-) create mode 100644 plugins/plugin-lite-api.mjs diff --git a/graphql-yoga.mjs b/graphql-yoga.mjs index d0a49ef8..b6f4e069 100644 --- a/graphql-yoga.mjs +++ b/graphql-yoga.mjs @@ -13,6 +13,7 @@ import useTwitch from './plugins/plugin-twitch.mjs'; import useNightbot from './plugins/plugin-nightbot.mjs'; import usePlayground from './plugins/plugin-playground.mjs'; import useOptionMethod from './plugins/plugin-option-method.mjs'; +import useLiteApi from './plugins/plugin-lite-api.mjs'; let dataAPI, yoga; @@ -46,6 +47,7 @@ export default async function getYoga(env) { useTwitch(env), usePlayground(), useNightbot(env), + useLiteApi(env), useHttpServer(env), useCacheMachine(env), ], diff --git a/index.mjs b/index.mjs index 5e2ce4e2..4e769bd5 100644 --- a/index.mjs +++ b/index.mjs @@ -26,8 +26,9 @@ import graphQLOptions from './utils/graphql-options.mjs'; import cacheMachine from './utils/cache-machine.mjs'; import fetchWithTimeout from './utils/fetch-with-timeout.mjs'; -import { getNightbotResponse } from './plugins/plugin-nightbot.mjs'; +import { getNightbotResponse, nightbotPaths } from './plugins/plugin-nightbot.mjs'; import { getTwitchResponse } from './plugins/plugin-twitch.mjs'; +import { getLiteApiResponse, liteApiPathRegex } from './plugins/plugin-lite-api.mjs'; let dataAPI; @@ -108,32 +109,6 @@ async function graphqlHandler(request, env, ctx) { //console.log(`Skipping cache in ${ENVIRONMENT} environment`); } - // if an HTTP GraphQL server is configured, pass the request to that - if (env.USE_ORIGIN === 'true') { - try { - const serverUrl = `https://api.tarkov.dev${graphQLOptions.baseEndpoint}`; - const queryResult = await fetchWithTimeout(serverUrl, { - method: request.method, - body: JSON.stringify({ - query, - variables, - }), - headers: { - 'Content-Type': 'application/json', - 'cache-check-complete': 'true', - }, - timeout: 20000 - }); - if (queryResult.status !== 200) { - throw new Error(`${queryResult.status} ${await queryResult.text()}`); - } - console.log('Request served from graphql server'); - return new Response(await queryResult.text(), responseOptions); - } catch (error) { - console.error(`Error getting response from GraphQL server: ${error}`); - } - } - const context = graphqlUtil.getDefaultContext(dataAPI, requestId); let result, ttl; try { @@ -221,18 +196,34 @@ export default { response = graphiql(graphQLOptions); } - if (graphQLOptions.forwardUnmatchedRequestsToOrigin) { - return fetch(request); + // if an origin server is configured, pass the request + if (env.USE_ORIGIN === 'true') { + try { + response = await fetchWithTimeout(request, { + headers: { + 'cache-check-complete': 'true', + }, + timeout: 20000 + }); + if (response.status !== 200) { + throw new Error(`${response.status} ${await response.text()}`); + } + console.log('Request served from origin server'); + } catch (error) { + console.error(`Error getting response from origin server: ${error}`); + response = undefined; + } } - if (url.pathname === '/webhook/nightbot' || - url.pathname === '/webhook/stream-elements' || - url.pathname === '/webhook/moobot' - ) { + if (!response && nightbotPaths.includes(url.pathname)) { response = await getNightbotResponse(request, url, env, ctx); } - if (url.pathname === graphQLOptions.baseEndpoint) { + if (!response && url.pathname.match(liteApiPathRegex)) { + response = await getLiteApiResponse(request, url, env, ctx); + } + + if (!response && url.pathname === graphQLOptions.baseEndpoint) { response = await graphqlHandler(request, env, ctx); if (graphQLOptions.cors) { setCors(response, graphQLOptions.cors); diff --git a/plugins/plugin-lite-api.mjs b/plugins/plugin-lite-api.mjs new file mode 100644 index 00000000..c9f1ae4a --- /dev/null +++ b/plugins/plugin-lite-api.mjs @@ -0,0 +1,145 @@ +import cacheMachine from '../utils/cache-machine.mjs'; +import DataSource from '../datasources/index.mjs'; +import graphqlUtil from '../utils/graphql-util.mjs'; + +export const liteApiPathRegex = /\/api\/v1(?\/\w+)?\/(?item[\w\/]*)/; + +const currencyMap = { + RUB: '₽', + USD: '$', + EUR: '€', +}; + +export async function getLiteApiResponse(request, url, env, serverContext) { + let q, lang; + if (request.method.toUpperCase() === 'GET') { + q = url.searchParams.get('q'); + lang = url.searchParams.get('lang') ?? 'en'; + } else if (request.method.toUpperCase() === 'POST') { + const body = await request.json(); + q = body.q; + lang = body.lang ?? 'en'; + } else { + return new Response(null, { + status: 405, + headers: { 'cache-control': 'public, max-age=2592000' }, + }); + } + + const pathInfo = url.pathname.match(liteApiPathRegex); + + const gameMode = pathInfo.groups.gameMode || 'regular'; + + const endpoint = pathInfo.groups.endpoint; + + let key; + if (env.SKIP_CACHE !== 'true' && !request.headers.has('cache-check-complete')) { + const requestStart = new Date(); + key = await cacheMachine.createKey(env, url.pathname, { q, lang, gameMode }); + const cachedResponse = await cacheMachine.get(env, {key}); + if (cachedResponse) { + // Construct a new response with the cached data + const newResponse = new Response(cachedResponse); + // Add a custom 'X-CACHE: HIT' header so we know the request hit the cache + newResponse.headers.append('X-CACHE', 'HIT'); + console.log(`Request served from cache: ${new Date() - requestStart} ms`); + // Return the new cached response + request.cached = true; + return newResponse; + } else { + console.log('no cached response'); + } + } else { + //console.log(`Skipping cache in ${ENVIRONMENT} environment`); + } + const data = new DataSource(env); + const context = graphqlUtil.getDefaultContext(data); + + const info = graphqlUtil.getGenericInfo(lang, gameMode); + + const traders = await data.worker.trader.getList(context, info); + function toLiteApiItem(item) { + const bestTraderSell = item.traderPrices.reduce((best, current) => { + if (!best || current.priceRUB > best.priceRUB) { + return current; + } + return best; + }, undefined); + const bestTrader = bestTraderSell ? traders.find(t => t.id === bestTraderSell.trader) : undefined; + return { + uid: item.id, + name: data.worker.item.getLocale(item.name, context, info), + tags: item.types, + shortName: data.worker.item.getLocale(item.shortName, context, info), + price: item.lastLowPrice, + basePrice: item.basePrice, + avg24hPrice: item.avg24hPrice, + //avg7daysPrice: null, + traderName: bestTrader ? data.worker.trader.getLocale(bestTrader.name, context, info) : null, + traderPrice: bestTraderSell ? bestTraderSell.price : null, + traderPriceCur: bestTraderSell ? currencyMap[bestTraderSell.currency] : null, + updated: item.updated, + slots: item.width * item.height, + diff24h: item.changeLast48h, + //diff7days: null, + icon: item.iconLink, + link: item.link, + wikiLink: item.wikiLink, + img: item.gridImageLink, + imgBig: item.inspectImageLink, + img512: item.image512pxLink, + image8x: item.image8xLink, + bsgId: item.id, + isFunctional: !item.types.includes('gun'), + reference: 'https://tarkov.dev', + }; + } + + let items, ttl; + try { + if (endpoint.startsWith('items')) { + items = await data.worker.item.getAllItems(context, info); + } + if (!items && endpoint.startsWith('item')) { + if (!q) { + throw new Error('No q parameter provided'); + } + items = await data.worker.item.getItemsByName(context, info, q); + } + items = items.map(toLiteApiItem); + ttl = data.getRequestTtl(context.requestId); + } catch (error) { + throw (error); + } finally { + data.clearRequestData(context.requestId); + } + const responseBody = JSON.stringify(items ?? [], null, 4); + + // Update the cache with the results of the query + if (env.SKIP_CACHE !== 'true' && ttl > 0) { + const putCachePromise = cacheMachine.put(env, responseBody, { key, query: url.pathname, variables: { q, lang, gameMode }, ttl: String(ttl)}); + // using waitUntil doens't hold up returning a response but keeps the worker alive as long as needed + if (request.ctx?.waitUntil) { + request.ctx.waitUntil(putCachePromise); + } else if (serverContext.waitUntil) { + serverContext.waitUntil(putCachePromise); + } + } + return new Response(responseBody) +} + +export default function useLiteApi(env) { + return { + async onRequest({ request, url, endResponse, serverContext, fetchAPI }) { + request.headers.forEach((hVal, hKey) => { + console.log(hKey, hVal); + }); + if (!url.pathname.match(liteApiPathRegex)) { + return; + } + const response = await getLiteApiResponse(request, url, env, serverContext); + + endResponse(response); + }, + } +} diff --git a/plugins/plugin-nightbot.mjs b/plugins/plugin-nightbot.mjs index 72f67b33..5f2d3da2 100644 --- a/plugins/plugin-nightbot.mjs +++ b/plugins/plugin-nightbot.mjs @@ -6,7 +6,7 @@ function capitalize(s) { return s && s[0].toUpperCase() + s.slice(1); } -const usePaths = [ +export const nightbotPaths = [ '/webhook/nightbot', '/webhook/stream-elements', '/webhook/moobot', @@ -88,7 +88,7 @@ export async function getNightbotResponse(request, url, env, serverContext) { export default function useNightbot(env) { return { async onRequest({ request, url, endResponse, serverContext, fetchAPI }) { - if (!usePaths.includes(url.pathname)) { + if (!nightbotPaths.includes(url.pathname)) { return; } const response = await getNightbotResponse(request, url, env, serverContext); From dc55182521f66cd90ea3caac302609b2e5c91ffa Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Wed, 18 Sep 2024 08:37:51 -0500 Subject: [PATCH 02/10] improve response headers --- plugins/plugin-lite-api.mjs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/plugins/plugin-lite-api.mjs b/plugins/plugin-lite-api.mjs index c9f1ae4a..d110bf10 100644 --- a/plugins/plugin-lite-api.mjs +++ b/plugins/plugin-lite-api.mjs @@ -57,7 +57,6 @@ export async function getLiteApiResponse(request, url, env, serverContext) { const info = graphqlUtil.getGenericInfo(lang, gameMode); - const traders = await data.worker.trader.getList(context, info); function toLiteApiItem(item) { const bestTraderSell = item.traderPrices.reduce((best, current) => { if (!best || current.priceRUB > best.priceRUB) { @@ -65,7 +64,6 @@ export async function getLiteApiResponse(request, url, env, serverContext) { } return best; }, undefined); - const bestTrader = bestTraderSell ? traders.find(t => t.id === bestTraderSell.trader) : undefined; return { uid: item.id, name: data.worker.item.getLocale(item.name, context, info), @@ -75,7 +73,7 @@ export async function getLiteApiResponse(request, url, env, serverContext) { basePrice: item.basePrice, avg24hPrice: item.avg24hPrice, //avg7daysPrice: null, - traderName: bestTrader ? data.worker.trader.getLocale(bestTrader.name, context, info) : null, + traderName: bestTraderSell ? bestTraderSell.name : null, traderPrice: bestTraderSell ? bestTraderSell.price : null, traderPriceCur: bestTraderSell ? currencyMap[bestTraderSell.currency] : null, updated: item.updated, @@ -90,15 +88,23 @@ export async function getLiteApiResponse(request, url, env, serverContext) { img512: item.image512pxLink, image8x: item.image8xLink, bsgId: item.id, - isFunctional: !item.types.includes('gun'), + isFunctional: true, // !item.types.includes('gun'), reference: 'https://tarkov.dev', }; } let items, ttl; + const responseOptions = { + headers: { + 'Content-Type': 'application/json', + }, + }; try { if (endpoint.startsWith('items')) { items = await data.worker.item.getAllItems(context, info); + if (endpoint.endsWith('/download')) { + responseOptions.headers['Content-Disposition'] = 'attachment; filename="items.json"'; + } } if (!items && endpoint.startsWith('item')) { if (!q) { @@ -125,15 +131,13 @@ export async function getLiteApiResponse(request, url, env, serverContext) { serverContext.waitUntil(putCachePromise); } } - return new Response(responseBody) + + return new Response(responseBody, responseOptions); } export default function useLiteApi(env) { return { async onRequest({ request, url, endResponse, serverContext, fetchAPI }) { - request.headers.forEach((hVal, hKey) => { - console.log(hKey, hVal); - }); if (!url.pathname.match(liteApiPathRegex)) { return; } From 0c8c9f8c4199bf8c54f80bdec1c5d79ef2d721da Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Wed, 18 Sep 2024 08:40:29 -0500 Subject: [PATCH 03/10] fix origin forward --- index.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.mjs b/index.mjs index 4e769bd5..84afed77 100644 --- a/index.mjs +++ b/index.mjs @@ -199,7 +199,7 @@ export default { // if an origin server is configured, pass the request if (env.USE_ORIGIN === 'true') { try { - response = await fetchWithTimeout(request, { + response = await fetchWithTimeout(request.clone(), { headers: { 'cache-check-complete': 'true', }, From 6f970dbf441ca7b95920567c8efd8386fa91324f Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Wed, 18 Sep 2024 10:33:34 -0500 Subject: [PATCH 04/10] support more parameters --- plugins/plugin-lite-api.mjs | 43 +++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/plugins/plugin-lite-api.mjs b/plugins/plugin-lite-api.mjs index d110bf10..cd6dc8e4 100644 --- a/plugins/plugin-lite-api.mjs +++ b/plugins/plugin-lite-api.mjs @@ -11,14 +11,22 @@ const currencyMap = { }; export async function getLiteApiResponse(request, url, env, serverContext) { - let q, lang; + let q, lang, uid, tags, sort, sort_direction; if (request.method.toUpperCase() === 'GET') { q = url.searchParams.get('q'); lang = url.searchParams.get('lang') ?? 'en'; + uid = url.searchParams.get('uid'); + tags = url.searchParams.get('tags')?.split(',') ?? []; + sort = url.searchParams.get('sort'); + sort_direction = url.searchParams.get('sort_direction'); } else if (request.method.toUpperCase() === 'POST') { const body = await request.json(); q = body.q; lang = body.lang ?? 'en'; + uid = body.uid; + tags = body.tags?.split(',') ?? []; + sort = body.sort; + sort_direction = body.sort_direction; } else { return new Response(null, { status: 405, @@ -35,7 +43,7 @@ export async function getLiteApiResponse(request, url, env, serverContext) { let key; if (env.SKIP_CACHE !== 'true' && !request.headers.has('cache-check-complete')) { const requestStart = new Date(); - key = await cacheMachine.createKey(env, url.pathname, { q, lang, gameMode }); + key = await cacheMachine.createKey(env, url.pathname, { q, lang, gameMode, uid, tags, sort, sort_direction }); const cachedResponse = await cacheMachine.get(env, {key}); if (cachedResponse) { // Construct a new response with the cached data @@ -105,25 +113,46 @@ export async function getLiteApiResponse(request, url, env, serverContext) { if (endpoint.endsWith('/download')) { responseOptions.headers['Content-Disposition'] = 'attachment; filename="items.json"'; } + if (tags) { + items = await data.worker.item.getItemsByTypes(context, info, tags, items); + } } if (!items && endpoint.startsWith('item')) { - if (!q) { - throw new Error('No q parameter provided'); + if (!q && !uid) { + throw new Error('The item request requires either a q or uid parameter'); + } + if (q) { + items = await data.worker.item.getItemsByName(context, info, q); + } else if (uid) { + items = [await data.worker.item.getItem(context, info, uid)]; } - items = await data.worker.item.getItemsByName(context, info, q); } items = items.map(toLiteApiItem); ttl = data.getRequestTtl(context.requestId); } catch (error) { - throw (error); + return new Response(error.message, {status: 400}); } finally { data.clearRequestData(context.requestId); } + if (sort && items?.length) { + items.sort((a, b) => { + let aValue = sort_direction === 'desc' ? b[sort] : a[sort]; + let bValue = sort_direction === 'desc' ? a[sort] : b[sort]; + if (sort === 'updated') { + aValue = new Date(aValue); + bValue = new Date(bValue); + } + if (typeof aValue === 'string') { + return aValue.localeCompare(bValue, lang); + } + return aValue - bValue; + }); + } const responseBody = JSON.stringify(items ?? [], null, 4); // Update the cache with the results of the query if (env.SKIP_CACHE !== 'true' && ttl > 0) { - const putCachePromise = cacheMachine.put(env, responseBody, { key, query: url.pathname, variables: { q, lang, gameMode }, ttl: String(ttl)}); + const putCachePromise = cacheMachine.put(env, responseBody, { key, query: url.pathname, variables: { q, lang, gameMode, uid, tags, sort, sort_direction }, ttl: String(ttl)}); // using waitUntil doens't hold up returning a response but keeps the worker alive as long as needed if (request.ctx?.waitUntil) { request.ctx.waitUntil(putCachePromise); From 756d19d676531b0899d68ffe687426b545d1891d Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Wed, 18 Sep 2024 10:54:20 -0500 Subject: [PATCH 05/10] bug fix, don't forward invalid requests to origin --- index.mjs | 36 +++++++++++++++++++----------------- plugins/plugin-lite-api.mjs | 4 ++-- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/index.mjs b/index.mjs index 84afed77..b8ba0681 100644 --- a/index.mjs +++ b/index.mjs @@ -181,25 +181,28 @@ export default { const requestStart = new Date(); const url = new URL(request.url); - let response; - try { if (url.pathname === '/twitch') { - response = await getTwitchResponse(env); + const response = await getTwitchResponse(env); if (graphQLOptions.cors) { setCors(response, graphQLOptions.cors); } + return response; } if (url.pathname === graphQLOptions.playgroundEndpoint) { //response = playground(request, graphQLOptions); - response = graphiql(graphQLOptions); + return graphiql(graphQLOptions); + } + + if (!nightbotPaths.includes(url.pathname) && !url.pathname.match(liteApiPathRegex) && url.pathname !== graphQLOptions.baseEndpoint) { + return new Response('Not found', { status: 404 }); } // if an origin server is configured, pass the request if (env.USE_ORIGIN === 'true') { try { - response = await fetchWithTimeout(request.clone(), { + const response = await fetchWithTimeout(request.clone(), { headers: { 'cache-check-complete': 'true', }, @@ -209,35 +212,34 @@ export default { throw new Error(`${response.status} ${await response.text()}`); } console.log('Request served from origin server'); + return response; } catch (error) { console.error(`Error getting response from origin server: ${error}`); - response = undefined; } } - if (!response && nightbotPaths.includes(url.pathname)) { - response = await getNightbotResponse(request, url, env, ctx); + if (nightbotPaths.includes(url.pathname)) { + return await getNightbotResponse(request, url, env, ctx); } - if (!response && url.pathname.match(liteApiPathRegex)) { - response = await getLiteApiResponse(request, url, env, ctx); + if (url.pathname.match(liteApiPathRegex)) { + return await getLiteApiResponse(request, url, env, ctx); } - if (!response && url.pathname === graphQLOptions.baseEndpoint) { - response = await graphqlHandler(request, env, ctx); + if (url.pathname === graphQLOptions.baseEndpoint) { + const response = await graphqlHandler(request, env, ctx); if (graphQLOptions.cors) { setCors(response, graphQLOptions.cors); } + return response; } - if (!response) { - response = new Response('Not found', { status: 404 }); - } - console.log(`Response time: ${new Date() - requestStart} ms`); - return response; + return new Response('Not found', { status: 404 }); } catch (err) { console.log(err); return new Response(graphQLOptions.debug ? err : 'Something went wrong', { status: 500 }); + } finally { + console.log(`Response time: ${new Date() - requestStart} ms`); } }, }; diff --git a/plugins/plugin-lite-api.mjs b/plugins/plugin-lite-api.mjs index cd6dc8e4..1db1b9eb 100644 --- a/plugins/plugin-lite-api.mjs +++ b/plugins/plugin-lite-api.mjs @@ -16,7 +16,7 @@ export async function getLiteApiResponse(request, url, env, serverContext) { q = url.searchParams.get('q'); lang = url.searchParams.get('lang') ?? 'en'; uid = url.searchParams.get('uid'); - tags = url.searchParams.get('tags')?.split(',') ?? []; + tags = url.searchParams.get('tags')?.split(','); sort = url.searchParams.get('sort'); sort_direction = url.searchParams.get('sort_direction'); } else if (request.method.toUpperCase() === 'POST') { @@ -24,7 +24,7 @@ export async function getLiteApiResponse(request, url, env, serverContext) { q = body.q; lang = body.lang ?? 'en'; uid = body.uid; - tags = body.tags?.split(',') ?? []; + tags = body.tags?.split(','); sort = body.sort; sort_direction = body.sort_direction; } else { From 59b3e0e0700681c37c127eb5cdb66a207f2be1f7 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Wed, 18 Sep 2024 15:02:04 -0500 Subject: [PATCH 06/10] bump deps --- http/package-lock.json | 6 +++--- package-lock.json | 24 ++++++++++++------------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/http/package-lock.json b/http/package-lock.json index 8e2540c6..ea547a9a 100644 --- a/http/package-lock.json +++ b/http/package-lock.json @@ -333,9 +333,9 @@ } }, "node_modules/dset": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.3.tgz", - "integrity": "sha512-20TuZZHCEZ2O71q9/+8BwKwZ0QtD9D8ObhrihJPr+vLLYlSuAU3/zL4cSlgbfeoGHTjCSJBa7NGcrF9/Bx/WJQ==", + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.4.tgz", + "integrity": "sha512-2QF/g9/zTaPDc3BjNcVTGoBbXBgYfMTTceLaYcFJ/W9kggFUkhxD/hMEeuLKbugyef9SqAx8cpgwlIP/jinUTA==", "engines": { "node": ">=4" } diff --git a/package-lock.json b/package-lock.json index 64c1daab..aba4baf5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1211,9 +1211,9 @@ } }, "node_modules/dset": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.3.tgz", - "integrity": "sha512-20TuZZHCEZ2O71q9/+8BwKwZ0QtD9D8ObhrihJPr+vLLYlSuAU3/zL4cSlgbfeoGHTjCSJBa7NGcrF9/Bx/WJQ==", + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.4.tgz", + "integrity": "sha512-2QF/g9/zTaPDc3BjNcVTGoBbXBgYfMTTceLaYcFJ/W9kggFUkhxD/hMEeuLKbugyef9SqAx8cpgwlIP/jinUTA==", "engines": { "node": ">=4" } @@ -2031,9 +2031,9 @@ "dev": true }, "node_modules/path-to-regexp": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.1.tgz", - "integrity": "sha512-JLyh7xT1kizaEvcaXOQwOc2/Yhw6KZOvPf1S8401UyLk86CU79LN3vl7ztXGm/pZ+YjoyAJ4rxmHwbkBXJX+yw==", + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", + "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", "dev": true }, "node_modules/pathe": { @@ -3642,9 +3642,9 @@ } }, "dset": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.3.tgz", - "integrity": "sha512-20TuZZHCEZ2O71q9/+8BwKwZ0QtD9D8ObhrihJPr+vLLYlSuAU3/zL4cSlgbfeoGHTjCSJBa7NGcrF9/Bx/WJQ==" + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.4.tgz", + "integrity": "sha512-2QF/g9/zTaPDc3BjNcVTGoBbXBgYfMTTceLaYcFJ/W9kggFUkhxD/hMEeuLKbugyef9SqAx8cpgwlIP/jinUTA==" }, "ecc-jsbn": { "version": "0.1.2", @@ -4265,9 +4265,9 @@ "dev": true }, "path-to-regexp": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.1.tgz", - "integrity": "sha512-JLyh7xT1kizaEvcaXOQwOc2/Yhw6KZOvPf1S8401UyLk86CU79LN3vl7ztXGm/pZ+YjoyAJ4rxmHwbkBXJX+yw==", + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", + "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", "dev": true }, "pathe": { From 8a78ee72144ee1c367a47c7726244513eb46b89f Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Thu, 19 Sep 2024 10:16:43 -0500 Subject: [PATCH 07/10] fix cache check --- graphql-yoga.mjs | 6 ++-- index.mjs | 55 +++++++++++++++++------------- plugins/plugin-http-server.mjs | 30 ----------------- plugins/plugin-lite-api.mjs | 6 +++- plugins/plugin-nightbot.mjs | 6 +++- plugins/plugin-origin-server.mjs | 58 ++++++++++++++++++++++++++++++++ 6 files changed, 102 insertions(+), 59 deletions(-) delete mode 100644 plugins/plugin-http-server.mjs create mode 100644 plugins/plugin-origin-server.mjs diff --git a/graphql-yoga.mjs b/graphql-yoga.mjs index b6f4e069..82b22bc4 100644 --- a/graphql-yoga.mjs +++ b/graphql-yoga.mjs @@ -7,7 +7,7 @@ import graphqlUtil from './utils/graphql-util.mjs'; import graphQLOptions from './utils/graphql-options.mjs'; import useRequestTimer from './plugins/plugin-request-timer.mjs'; -import useHttpServer from './plugins/plugin-http-server.mjs'; +import useOriginServer from './plugins/plugin-origin-server.mjs'; import useCacheMachine from './plugins/plugin-use-cache-machine.mjs'; import useTwitch from './plugins/plugin-twitch.mjs'; import useNightbot from './plugins/plugin-nightbot.mjs'; @@ -46,10 +46,10 @@ export default async function getYoga(env) { useOptionMethod(), useTwitch(env), usePlayground(), + useCacheMachine(env), + useOriginServer(env), useNightbot(env), useLiteApi(env), - useHttpServer(env), - useCacheMachine(env), ], cors: { origin: graphQLOptions.cors.allowOrigin, diff --git a/index.mjs b/index.mjs index b8ba0681..8b00c018 100644 --- a/index.mjs +++ b/index.mjs @@ -26,9 +26,9 @@ import graphQLOptions from './utils/graphql-options.mjs'; import cacheMachine from './utils/cache-machine.mjs'; import fetchWithTimeout from './utils/fetch-with-timeout.mjs'; -import { getNightbotResponse, nightbotPaths } from './plugins/plugin-nightbot.mjs'; +import { getNightbotResponse, useNightbotOnUrl } from './plugins/plugin-nightbot.mjs'; import { getTwitchResponse } from './plugins/plugin-twitch.mjs'; -import { getLiteApiResponse, liteApiPathRegex } from './plugins/plugin-lite-api.mjs'; +import { getLiteApiResponse, useLiteApiOnUrl } from './plugins/plugin-lite-api.mjs'; let dataAPI; @@ -109,6 +109,32 @@ async function graphqlHandler(request, env, ctx) { //console.log(`Skipping cache in ${ENVIRONMENT} environment`); } + // if an origin server is configured, pass the request + if (env.USE_ORIGIN === 'true') { + try { + const serverUrl = `https://api.tarkov.dev${graphQLOptions.baseEndpoint}`; + const queryResult = await fetchWithTimeout(serverUrl, { + method: request.method, + body: JSON.stringify({ + query, + variables, + }), + headers: { + 'Content-Type': 'application/json', + 'cache-check-complete': 'true', + }, + timeout: 20000 + }); + if (queryResult.status !== 200) { + throw new Error(`${queryResult.status} ${await queryResult.text()}`); + } + console.log('Request served from origin server'); + return new Response(await queryResult.text(), responseOptions); + } catch (error) { + console.error(`Error getting response from origin server: ${error}`); + } + } + const context = graphqlUtil.getDefaultContext(dataAPI, requestId); let result, ttl; try { @@ -195,34 +221,15 @@ export default { return graphiql(graphQLOptions); } - if (!nightbotPaths.includes(url.pathname) && !url.pathname.match(liteApiPathRegex) && url.pathname !== graphQLOptions.baseEndpoint) { + if (!useNightbotOnUrl(url) && !useLiteApiOnUrl(url) && url.pathname !== graphQLOptions.baseEndpoint) { return new Response('Not found', { status: 404 }); } - - // if an origin server is configured, pass the request - if (env.USE_ORIGIN === 'true') { - try { - const response = await fetchWithTimeout(request.clone(), { - headers: { - 'cache-check-complete': 'true', - }, - timeout: 20000 - }); - if (response.status !== 200) { - throw new Error(`${response.status} ${await response.text()}`); - } - console.log('Request served from origin server'); - return response; - } catch (error) { - console.error(`Error getting response from origin server: ${error}`); - } - } - if (nightbotPaths.includes(url.pathname)) { + if (useNightbotOnUrl(url)) { return await getNightbotResponse(request, url, env, ctx); } - if (url.pathname.match(liteApiPathRegex)) { + if (useLiteApiOnUrl(url)) { return await getLiteApiResponse(request, url, env, ctx); } diff --git a/plugins/plugin-http-server.mjs b/plugins/plugin-http-server.mjs deleted file mode 100644 index a2c59ae6..00000000 --- a/plugins/plugin-http-server.mjs +++ /dev/null @@ -1,30 +0,0 @@ -import graphQLOptions from '../utils/graphql-options.mjs'; - -export default function useHttpServer(env) { - return { - async onParams({params, request, setParams, setResult, fetchAPI}) { - // if an HTTP GraphQL server is configured, pass the request to that - if (env.USE_ORIGIN !== 'true') { - return; - } - try { - const serverUrl = `https://api.tarkov.dev${graphQLOptions.baseEndpoint}`; - const queryResult = await fetch(serverUrl, { - method: request.method, - body: JSON.stringify(params), - headers: { - 'Content-Type': 'application/json', - }, - }); - if (queryResult.status !== 200) { - throw new Error(`${queryResult.status} ${queryResult.statusText}: ${await queryResult.text()}`); - } - console.log('Request served from graphql server'); - setResult(await queryResult.json()); - request.cached = true; - } catch (error) { - console.error(`Error getting response from GraphQL server: ${error}`); - } - }, - } -} \ No newline at end of file diff --git a/plugins/plugin-lite-api.mjs b/plugins/plugin-lite-api.mjs index 1db1b9eb..f276aeef 100644 --- a/plugins/plugin-lite-api.mjs +++ b/plugins/plugin-lite-api.mjs @@ -4,6 +4,10 @@ import graphqlUtil from '../utils/graphql-util.mjs'; export const liteApiPathRegex = /\/api\/v1(?\/\w+)?\/(?item[\w\/]*)/; +export function useLiteApiOnUrl(url) { + return !!url.pathname.match(liteApiPathRegex); +}; + const currencyMap = { RUB: '₽', USD: '$', @@ -167,7 +171,7 @@ export async function getLiteApiResponse(request, url, env, serverContext) { export default function useLiteApi(env) { return { async onRequest({ request, url, endResponse, serverContext, fetchAPI }) { - if (!url.pathname.match(liteApiPathRegex)) { + if (!useLiteApiOnUrl(url)) { return; } const response = await getLiteApiResponse(request, url, env, serverContext); diff --git a/plugins/plugin-nightbot.mjs b/plugins/plugin-nightbot.mjs index 5f2d3da2..748db9c5 100644 --- a/plugins/plugin-nightbot.mjs +++ b/plugins/plugin-nightbot.mjs @@ -12,6 +12,10 @@ export const nightbotPaths = [ '/webhook/moobot', ]; +export function useNightbotOnUrl(url) { + return nightbotPaths.includes(url.pathname); +}; + export async function getNightbotResponse(request, url, env, serverContext) { if (request.method.toUpperCase() !== 'GET') { return new Response(null, { @@ -88,7 +92,7 @@ export async function getNightbotResponse(request, url, env, serverContext) { export default function useNightbot(env) { return { async onRequest({ request, url, endResponse, serverContext, fetchAPI }) { - if (!nightbotPaths.includes(url.pathname)) { + if (!useNightbotOnUrl(url)) { return; } const response = await getNightbotResponse(request, url, env, serverContext); diff --git a/plugins/plugin-origin-server.mjs b/plugins/plugin-origin-server.mjs new file mode 100644 index 00000000..2c4d759c --- /dev/null +++ b/plugins/plugin-origin-server.mjs @@ -0,0 +1,58 @@ +// Pass the request to an origin server if USE_ORIGIN is set to 'true' +import graphQLOptions from '../utils/graphql-options.mjs'; +import fetchWithTimeout from '../utils/fetch-with-timeout.mjs'; +import { useNightbotOnUrl } from './plugin-nightbot.mjs'; +import { useLiteApiOnUrl } from './plugin-lite-api.mjs'; + +export default function useOriginServer(env) { + return { + async onRequest({ request, url, endResponse, serverContext, fetchAPI }) { + if (env.USE_ORIGIN !== 'true') { + return; + } + if (!useNightbotOnUrl(url) && !useLiteApiOnUrl(url)) { + return; + } + try { + const response = await fetchWithTimeout(request.clone(), { + headers: { + 'cache-check-complete': 'true', + }, + timeout: 20000 + }); + if (response.status !== 200) { + throw new Error(`${response.status} ${await response.text()}`); + } + console.log('Request served from origin server'); + endResponse(response); + } catch (error) { + console.error(`Error getting response from origin server: ${error}`); + } + }, + async onParams({params, request, setParams, setResult, fetchAPI}) { + if (env.USE_ORIGIN !== 'true') { + return; + } + try { + const serverUrl = `https://api.tarkov.dev${graphQLOptions.baseEndpoint}`; + //const serverUrl = `http://localhost:8788${graphQLOptions.baseEndpoint}`; + const queryResult = await fetchWithTimeout(serverUrl, { + method: request.method, + body: JSON.stringify(params), + headers: { + 'Content-Type': 'application/json', + }, + timeout: 20000 + }); + if (queryResult.status !== 200) { + throw new Error(`${queryResult.status} ${queryResult.statusText}: ${await queryResult.text()}`); + } + console.log('Request served from origin server'); + setResult(await queryResult.json()); + request.cached = true; + } catch (error) { + console.error(`Error getting response from origin server: ${error}`); + } + }, + } +} \ No newline at end of file From 178ce8d95bb45159d803b3514141d2ac1ca346ab Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Thu, 19 Sep 2024 14:33:10 -0500 Subject: [PATCH 08/10] add origin passthrough to other endpoints --- graphql-yoga.mjs | 4 +-- index.mjs | 9 +++-- plugins/plugin-graphql-origin.mjs | 34 ++++++++++++++++++ plugins/plugin-lite-api.mjs | 37 ++++++++++++++++++-- plugins/plugin-nightbot.mjs | 29 ++++++++++++++-- plugins/plugin-origin-server.mjs | 58 ------------------------------- 6 files changed, 104 insertions(+), 67 deletions(-) create mode 100644 plugins/plugin-graphql-origin.mjs delete mode 100644 plugins/plugin-origin-server.mjs diff --git a/graphql-yoga.mjs b/graphql-yoga.mjs index 82b22bc4..4742f17f 100644 --- a/graphql-yoga.mjs +++ b/graphql-yoga.mjs @@ -7,7 +7,7 @@ import graphqlUtil from './utils/graphql-util.mjs'; import graphQLOptions from './utils/graphql-options.mjs'; import useRequestTimer from './plugins/plugin-request-timer.mjs'; -import useOriginServer from './plugins/plugin-origin-server.mjs'; +import useGraphQLOrigin from './plugins/plugin-graphql-origin.mjs'; import useCacheMachine from './plugins/plugin-use-cache-machine.mjs'; import useTwitch from './plugins/plugin-twitch.mjs'; import useNightbot from './plugins/plugin-nightbot.mjs'; @@ -47,7 +47,7 @@ export default async function getYoga(env) { useTwitch(env), usePlayground(), useCacheMachine(env), - useOriginServer(env), + useGraphQLOrigin(env), useNightbot(env), useLiteApi(env), ], diff --git a/index.mjs b/index.mjs index 8b00c018..2595da95 100644 --- a/index.mjs +++ b/index.mjs @@ -112,9 +112,12 @@ async function graphqlHandler(request, env, ctx) { // if an origin server is configured, pass the request if (env.USE_ORIGIN === 'true') { try { - const serverUrl = `https://api.tarkov.dev${graphQLOptions.baseEndpoint}`; - const queryResult = await fetchWithTimeout(serverUrl, { - method: request.method, + const originUrl = new URL(request.url); + if (env.ORIGIN_OVERRIDE) { + originUrl.host = env.ORIGIN_OVERRIDE; + } + const queryResult = await fetchWithTimeout(originUrl, { + method: 'POST', body: JSON.stringify({ query, variables, diff --git a/plugins/plugin-graphql-origin.mjs b/plugins/plugin-graphql-origin.mjs new file mode 100644 index 00000000..e1b7a2f7 --- /dev/null +++ b/plugins/plugin-graphql-origin.mjs @@ -0,0 +1,34 @@ +// Pass the request to an origin server if USE_ORIGIN is set to 'true' +import fetchWithTimeout from '../utils/fetch-with-timeout.mjs'; + +export default function useGraphQLOrigin(env) { + return { + async onParams({params, request, setParams, setResult, fetchAPI}) { + if (env.USE_ORIGIN !== 'true') { + return; + } + try { + const originUrl = new URL(request.url); + if (env.ORIGIN_OVERRIDE) { + originUrl.host = env.ORIGIN_OVERRIDE; + } + const queryResult = await fetchWithTimeout(originUrl, { + method: request.method, + body: JSON.stringify(params), + headers: { + 'Content-Type': 'application/json', + }, + timeout: 20000 + }); + if (queryResult.status !== 200) { + throw new Error(`${queryResult.status} ${queryResult.statusText}: ${await queryResult.text()}`); + } + console.log('Request served from origin server'); + setResult(await queryResult.json()); + request.cached = true; + } catch (error) { + console.error(`Error getting response from origin server: ${error}`); + } + }, + } +} \ No newline at end of file diff --git a/plugins/plugin-lite-api.mjs b/plugins/plugin-lite-api.mjs index f276aeef..610a9a75 100644 --- a/plugins/plugin-lite-api.mjs +++ b/plugins/plugin-lite-api.mjs @@ -1,6 +1,7 @@ -import cacheMachine from '../utils/cache-machine.mjs'; import DataSource from '../datasources/index.mjs'; import graphqlUtil from '../utils/graphql-util.mjs'; +import cacheMachine from '../utils/cache-machine.mjs'; +import fetchWithTimeout from '../utils/fetch-with-timeout.mjs'; export const liteApiPathRegex = /\/api\/v1(?\/\w+)?\/(?item[\w\/]*)/; @@ -45,7 +46,7 @@ export async function getLiteApiResponse(request, url, env, serverContext) { const endpoint = pathInfo.groups.endpoint; let key; - if (env.SKIP_CACHE !== 'true' && !request.headers.has('cache-check-complete')) { + if (env.SKIP_CACHE !== 'true' && env.SKIP_CACHE_CHECK !== 'true' && !request.headers.has('cache-check-complete')) { const requestStart = new Date(); key = await cacheMachine.createKey(env, url.pathname, { q, lang, gameMode, uid, tags, sort, sort_direction }); const cachedResponse = await cacheMachine.get(env, {key}); @@ -64,6 +65,38 @@ export async function getLiteApiResponse(request, url, env, serverContext) { } else { //console.log(`Skipping cache in ${ENVIRONMENT} environment`); } + + if (env.USE_ORIGIN === 'true') { + try { + const originUrl = new URL(request.url); + if (env.ORIGIN_OVERRIDE) { + originUrl.host = env.ORIGIN_OVERRIDE; + } + const response = await fetchWithTimeout(originUrl, { + method: 'POST', + body: JSON.stringify({ + q, + lang, + uid, + tags: tags?.join(','), + sort, + sort_direction, + }), + headers: { + 'cache-check-complete': 'true', + }, + timeout: 20000 + }); + if (response.status !== 200) { + throw new Error(`${response.status} ${await response.text()}`); + } + console.log('Request served from origin server'); + return response; + } catch (error) { + console.error(`Error getting response from origin server: ${error}`); + } + } + const data = new DataSource(env); const context = graphqlUtil.getDefaultContext(data); diff --git a/plugins/plugin-nightbot.mjs b/plugins/plugin-nightbot.mjs index 748db9c5..908e0aeb 100644 --- a/plugins/plugin-nightbot.mjs +++ b/plugins/plugin-nightbot.mjs @@ -1,6 +1,7 @@ -import cacheMachine from '../utils/cache-machine.mjs'; import DataSource from '../datasources/index.mjs'; +import cacheMachine from '../utils/cache-machine.mjs'; import graphqlUtil from '../utils/graphql-util.mjs'; +import fetchWithTimeout from '../utils/fetch-with-timeout.mjs'; function capitalize(s) { return s && s[0].toUpperCase() + s.slice(1); @@ -36,7 +37,7 @@ export async function getNightbotResponse(request, url, env, serverContext) { const query = url.searchParams.get('q'); let key; - if (env.SKIP_CACHE !== 'true' && !request.headers.has('cache-check-complete')) { + if (env.SKIP_CACHE !== 'true' && env.SKIP_CACHE_CHECK !== 'true' && !request.headers.has('cache-check-complete')) { const requestStart = new Date(); key = await cacheMachine.createKey(env, 'nightbot', { q: query, l: lang, m: gameMode }); const cachedResponse = await cacheMachine.get(env, {key}); @@ -55,6 +56,30 @@ export async function getNightbotResponse(request, url, env, serverContext) { } else { //console.log(`Skipping cache in ${ENVIRONMENT} environment`); } + + if (env.USE_ORIGIN === 'true') { + try { + const originUrl = new URL(request.url); + if (env.ORIGIN_OVERRIDE) { + originUrl.host = env.ORIGIN_OVERRIDE; + } + const response = await fetchWithTimeout(originUrl, { + method: 'GET', + headers: { + 'cache-check-complete': 'true', + }, + timeout: 20000 + }); + if (response.status !== 200) { + throw new Error(`${response.status} ${await response.text()}`); + } + console.log('Request served from origin server'); + return response; + } catch (error) { + console.error(`Error getting response from origin server: ${error}`); + } + } + const data = new DataSource(env); const context = graphqlUtil.getDefaultContext(data); diff --git a/plugins/plugin-origin-server.mjs b/plugins/plugin-origin-server.mjs deleted file mode 100644 index 2c4d759c..00000000 --- a/plugins/plugin-origin-server.mjs +++ /dev/null @@ -1,58 +0,0 @@ -// Pass the request to an origin server if USE_ORIGIN is set to 'true' -import graphQLOptions from '../utils/graphql-options.mjs'; -import fetchWithTimeout from '../utils/fetch-with-timeout.mjs'; -import { useNightbotOnUrl } from './plugin-nightbot.mjs'; -import { useLiteApiOnUrl } from './plugin-lite-api.mjs'; - -export default function useOriginServer(env) { - return { - async onRequest({ request, url, endResponse, serverContext, fetchAPI }) { - if (env.USE_ORIGIN !== 'true') { - return; - } - if (!useNightbotOnUrl(url) && !useLiteApiOnUrl(url)) { - return; - } - try { - const response = await fetchWithTimeout(request.clone(), { - headers: { - 'cache-check-complete': 'true', - }, - timeout: 20000 - }); - if (response.status !== 200) { - throw new Error(`${response.status} ${await response.text()}`); - } - console.log('Request served from origin server'); - endResponse(response); - } catch (error) { - console.error(`Error getting response from origin server: ${error}`); - } - }, - async onParams({params, request, setParams, setResult, fetchAPI}) { - if (env.USE_ORIGIN !== 'true') { - return; - } - try { - const serverUrl = `https://api.tarkov.dev${graphQLOptions.baseEndpoint}`; - //const serverUrl = `http://localhost:8788${graphQLOptions.baseEndpoint}`; - const queryResult = await fetchWithTimeout(serverUrl, { - method: request.method, - body: JSON.stringify(params), - headers: { - 'Content-Type': 'application/json', - }, - timeout: 20000 - }); - if (queryResult.status !== 200) { - throw new Error(`${queryResult.status} ${queryResult.statusText}: ${await queryResult.text()}`); - } - console.log('Request served from origin server'); - setResult(await queryResult.json()); - request.cached = true; - } catch (error) { - console.error(`Error getting response from origin server: ${error}`); - } - }, - } -} \ No newline at end of file From 699882e8a89057a1a9388c2fae2ed90ee22ec460 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Thu, 19 Sep 2024 15:16:59 -0500 Subject: [PATCH 09/10] remove duplicate check --- index.mjs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/index.mjs b/index.mjs index 2595da95..3a4ad65b 100644 --- a/index.mjs +++ b/index.mjs @@ -218,15 +218,6 @@ export default { } return response; } - - if (url.pathname === graphQLOptions.playgroundEndpoint) { - //response = playground(request, graphQLOptions); - return graphiql(graphQLOptions); - } - - if (!useNightbotOnUrl(url) && !useLiteApiOnUrl(url) && url.pathname !== graphQLOptions.baseEndpoint) { - return new Response('Not found', { status: 404 }); - } if (useNightbotOnUrl(url)) { return await getNightbotResponse(request, url, env, ctx); From f0fffa866ad4a9ad107ce042eb4252acb487f428 Mon Sep 17 00:00:00 2001 From: Razzmatazz Date: Thu, 19 Sep 2024 15:33:41 -0500 Subject: [PATCH 10/10] add playground back --- index.mjs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/index.mjs b/index.mjs index 3a4ad65b..ca1a2065 100644 --- a/index.mjs +++ b/index.mjs @@ -218,6 +218,11 @@ export default { } return response; } + + if (url.pathname === graphQLOptions.playgroundEndpoint) { + //response = playground(request, graphQLOptions); + return graphiql(graphQLOptions); + } if (useNightbotOnUrl(url)) { return await getNightbotResponse(request, url, env, ctx);