fetch logic tweaks, fix download label (#1362)

components/Library/MetaData.tsx

@@ -55,7 +55,7 @@ function generateData({ github, score, npm, npmPkg }: LibraryType, isDark: boole
           icon: <Download fill={iconColor} width={16} height={18} />,
           content: (
             <A href={`https://www.npmjs.com/package/${npmPkg}`} style={styles.link}>
-              {`${npm.downloads.toLocaleString()}`} {npm.period}ly downloads
+              {`${npm.downloads.toLocaleString()}`} monthly downloads
             </A>
           ),
         }

scripts/build-and-score-data.js

@@ -71,18 +71,15 @@ const buildAndScoreData = async () => {
 
   if (SCRAPE_GH_IMAGES) {
     console.log('\n📝 Scraping images from README');
-    await sleep(1000);
     data = await Promise.all(data.map(project => fetchReadmeImages(project)));
   }
 
   console.log('\n🔖 Determining npm package names');
-  await sleep(1000);
   data = data.map(fillNpmName);
 
   console.log('\n⬇️ Fetching download stats from npm');
-  await sleep(1000);
 
-  // https://github.com/npm/registry/blob/master/docs/download-counts.md#bulk-queries
+  // https://github.com/npm/registry/blob/main/docs/download-counts.md#bulk-queries
   let bulkList = [];
 
   // Fetch scoped packages data
@@ -100,16 +97,18 @@ const buildAndScoreData = async () => {
     })
   );
 
+  // https://github.com/npm/registry/blob/main/docs/download-counts.md#limits
+  const CHUNK_SIZE = 128;
+
   // Assemble and fetch regular packages data in bulk queries
-  const CHUNK_SIZE = 32;
   bulkList = [...Array(Math.ceil(bulkList.length / CHUNK_SIZE))].map(_ =>
     bulkList.splice(0, CHUNK_SIZE)
   );
 
   const downloadsList = (
     await Promise.all(
       bulkList.map(async (chunk, index) => {
-        await sleep(250 * index);
+        await sleep(500 * index);
         return await fetchNpmDataBulk(chunk);
       })
     )
@@ -118,7 +117,7 @@ const buildAndScoreData = async () => {
   const downloadsListWeek = (
     await Promise.all(
       bulkList.map(async (chunk, index) => {
-        await sleep(250 * index);
+        await sleep(500 * index);
         return await fetchNpmDataBulk(chunk, 'week');
       })
     )

scripts/fetch-github-data.js

@@ -4,6 +4,7 @@ import { config } from 'dotenv';
 import { processTopics, sleep } from './helpers.js';
 import GitHubLicensesQuery from './queries/GitHubLicensesQuery.js';
 import GitHubRepositoryQuery from './queries/GitHubRepositoryQuery.js';
+import { REQUEST_SLEEP } from '../util/Constants.js';
 
 config();
 
@@ -116,15 +117,15 @@ export const fetchGithubData = async (data, retries = 2) => {
       }
 
       console.log(`[GH] Retrying fetch for ${data.githubUrl} due to error result`);
-      await sleep(2500);
+      await sleep(REQUEST_SLEEP, REQUEST_SLEEP * 2);
       return await fetchGithubData(data, retries - 1);
     }
 
     if (!result?.data?.repository) {
       console.log(
         `[GH] Retrying fetch for ${data.githubUrl} due to ${result?.message?.toLowerCase() ?? 'missing data'} (status: ${result?.status ?? 'Unknown'})`
       );
-      await sleep(2500);
+      await sleep(REQUEST_SLEEP, REQUEST_SLEEP * 2);
       return await fetchGithubData(data, retries - 1);
     }
 
@@ -135,7 +136,7 @@ export const fetchGithubData = async (data, retries = 2) => {
     };
   } catch (error) {
     console.log(`[GH] Retrying fetch for ${data.githubUrl} due to an error`, error);
-    await sleep(2500);
+    await sleep(REQUEST_SLEEP, REQUEST_SLEEP * 2);
     return await fetchGithubData(data, retries - 1);
   }
 };

scripts/fetch-npm-data.js

@@ -1,6 +1,7 @@
 import fetch from 'cross-fetch';
 
 import { sleep } from './helpers.js';
+import { REQUEST_SLEEP } from '../util/Constants.js';
 
 const ATTEMPTS_LIMIT = 5;
 
@@ -45,7 +46,7 @@ export const fetchNpmDataBulk = async (namesArray, period = 'month', attemptsCou
       console.error('[NPM] Looks like we have reach the NPM API rate limit!', error);
       return namesArray.map(name => ({ name, npm: null }));
     }
-    await sleep(2500 + 500 * attemptsCount, 5000 + 500 * attemptsCount);
+    await sleep(REQUEST_SLEEP, REQUEST_SLEEP * 2);
     console.log(`[NPM] Retrying fetch for ${namesArray} (${attemptsCount + 1})`);
     return await fetchNpmDataBulk(namesArray, period, attemptsCount + 1);
   }
@@ -85,7 +86,7 @@ export const fetchNpmData = async (pkgData, attemptsCount = 0) => {
       console.error('[NPM] Looks like we have reach the NPM API rate limit!', error);
       return { ...pkgData, npm: null };
     }
-    await sleep(2500 + 500 * attemptsCount, 5000 + 500 * attemptsCount);
+    await sleep(REQUEST_SLEEP, REQUEST_SLEEP * 2);
     console.log(`[NPM] Retrying fetch for ${npmPkg} (${attemptsCount + 1})`);
     return await fetchNpmData(pkgData, attemptsCount + 1);
   }

util/Constants.ts

@@ -1 +1,2 @@
 export const NUM_PER_PAGE = 30;
+export const REQUEST_SLEEP = 5000;