Adds background job scheduling and execution

.changesets/11238.md

@@ -0,0 +1,14 @@
+- Adds background job scheduling and execution (#10906) by @cannikin
+
+This new package provides scheduling and processing of background jobs. We want everything needed to run a modern web application to be included in Redwood itself—you shouldn't need any third party integrations if you don't want.
+
+Background jobs have been sorely missed, but the time has come! (If you do want to use a third party service we have had an [integration with Inngest](https://community.redwoodjs.com/t/ship-background-jobs-crons-webhooks-and-reliable-workflows-in-record-time-with-inngest-and-redwoodjs/4866) since May of 2023!)
+
+## Features
+
+- Named queues: you can schedule jobs in separate named queues and have a different number of workers monitoring each one—makes it much easier to scale your background processing
+- Priority: give your jobs a priority from 1 (highest) to 100 (lowest). Workers will sort available jobs by priority, working the most important ones first.
+- Configurable delay: run your job as soon as possible (default), wait a number of seconds before running, or run at a specific time in the future
+- Auto-retries with backoff: if your job fails it will back off at the rate of attempts \*\* 4 for a default of 24 tries, the time between the last two attempts is a little over three days.
+- Run inline: instead of scheduling to run in the background, run immediately
+- Integrates with Redwood's [logger](https://docs.redwoodjs.com/docs/logger): use your existing one in api/src/lib/logger or create a new one just for job logging

.vscode/settings.json

@@ -22,6 +22,7 @@
     "baremetal",
     "bazinga",
     "corepack",
+    "daemonized",
     "envinfo",
     "execa",
     "Fastify",

docs/sidebars.js

@@ -97,6 +97,7 @@ module.exports = {
             { type: 'doc', id: 'auth/supertokens' },
           ],
         },
+        'background-jobs',
         'builds',
         'cells',
         'cli-commands',

packages/api-server/src/logFormatter/bin.ts

@@ -6,3 +6,9 @@ const input = process.stdin
 const output = process.stdout
 
 input.pipe(split(LogFormatter())).pipe(output)
+
+// assume that receiving a SIGINT (Ctrl-C) is a normal event, so don't exit with
+// a 129 error code, which makes execa blow up. Just return a nice quiet 0.
+process.on('SIGINT', () => {
+  process.exit(0)
+})

packages/babel-config/src/api.ts

@@ -20,6 +20,7 @@ import pluginRedwoodContextWrapping from './plugins/babel-plugin-redwood-context
 import pluginRedwoodDirectoryNamedImport from './plugins/babel-plugin-redwood-directory-named-import'
 import pluginRedwoodGraphqlOptionsExtract from './plugins/babel-plugin-redwood-graphql-options-extract'
 import pluginRedwoodImportDir from './plugins/babel-plugin-redwood-import-dir'
+import pluginRedwoodJobPathInjector from './plugins/babel-plugin-redwood-job-path-injector'
 import pluginRedwoodOTelWrapping from './plugins/babel-plugin-redwood-otel-wrapping'
 
 export const TARGETS_NODE = '20.10'
@@ -178,6 +179,12 @@ export const getApiSideBabelOverrides = ({ projectIsEsm = false } = {}) => {
         ],
       ],
     },
+    // Add import names and paths to job definitions
+    {
+      // match */api/src/jobs/*.js|ts
+      test: /.+api(?:[\\|/])src(?:[\\|/])jobs(?:[\\|/]).+.(?:js|ts)$/,
+      plugins: [[pluginRedwoodJobPathInjector]],
+    },
   ].filter(Boolean)
   return overrides as TransformOptions[]
 }

packages/babel-config/src/plugins/babel-plugin-redwood-job-path-injector.ts

@@ -0,0 +1,110 @@
+import fsPath from 'node:path'
+
+import type { PluginObj, types } from '@babel/core'
+
+import { getPaths } from '@redwoodjs/project-config'
+
+// This plugin is responsible for injecting the import path and name of a job
+// into the object that is passed to createJob. This is later used by adapters
+// and workers to import the job.
+
+export default function ({ types: _t }: { types: typeof types }): PluginObj {
+  const paths = getPaths()
+  return {
+    name: 'babel-plugin-redwood-job-path-injector',
+    visitor: {
+      ExportNamedDeclaration(path, state) {
+        // Extract the variable declaration from the export
+        const declaration = path.node.declaration
+        if (!declaration) {
+          return
+        }
+        if (declaration.type !== 'VariableDeclaration') {
+          return
+        }
+        // Extract the variable declarator from the declaration
+        const declarator = declaration.declarations[0]
+        if (!declarator) {
+          return
+        }
+        if (declarator.type !== 'VariableDeclarator') {
+          return
+        }
+
+        // Confirm that the init it a call expression
+        const init = declarator.init
+        if (!init) {
+          return
+        }
+        if (init.type !== 'CallExpression') {
+          return
+        }
+        // Confirm that the callee is a member expression
+        const callee = init.callee
+        if (!callee) {
+          return
+        }
+        if (callee.type !== 'MemberExpression') {
+          return
+        }
+        // The object is imported and so could be aliased so lets check the property
+        const property = callee.property
+        if (!property) {
+          return
+        }
+        if (property.type !== 'Identifier') {
+          return
+        }
+        if (property.name !== 'createJob') {
+          return
+        }
+
+        // From this point on we're confident that we're looking at a createJob call
+        // so let's start throwing errors if we don't find what we expect
+
+        // Extract the variable name from the declarator
+        const id = declarator.id
+        if (!id) {
+          return
+        }
+        if (id.type !== 'Identifier') {
+          return
+        }
+
+        const filepath = state.file.opts.filename
+        if (!filepath) {
+          throw new Error('No file path was found in the state')
+        }
+
+        const importName = id.name
+        const importPath = fsPath.relative(paths.api.jobs, filepath)
+        const importPathWithoutExtension = importPath.replace(/\.[^/.]+$/, '')
+
+        // Get the first argument of the call expression
+        const firstArg = init.arguments[0]
+        if (!firstArg) {
+          throw new Error('No first argument found in the createJob call')
+        }
+        // confirm it's an object expression
+        if (firstArg.type !== 'ObjectExpression') {
+          throw new Error(
+            'The first argument of the createJob call is not an object expression',
+          )
+        }
+        // Add a property to the object expression
+        firstArg.properties.push(
+          _t.objectProperty(
+            _t.identifier('path'),
+            _t.stringLiteral(importPathWithoutExtension),
+          ),
+        )
+        firstArg.properties.push(
+          _t.objectProperty(
+            _t.identifier('name'),
+            _t.stringLiteral(importName),
+          ),
+        )
+      },
+    },
+  }
+}

packages/cli-helpers/src/lib/colors.ts

@@ -18,5 +18,5 @@ export const colors = {
   tip: chalk.green,
   important: chalk.magenta,
   caution: chalk.red,
-  link: chalk.hex('#e8e8e8'),
+  link: chalk.underline,
 }

packages/cli/src/commands/generate/job/__tests__/__snapshots__/job.test.ts.snap

@@ -0,0 +1,38 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`Single word default files > creates a single word function file > Scenario snapshot 1`] = `
+"import type { ScenarioData } from '@redwoodjs/testing/api'
+
+export const standard = defineScenario({
+  // Define the "fixture" to write into your test database here
+  // See guide: https://redwoodjs.com/docs/testing#scenarios
+})
+
+export type StandardScenario = ScenarioData<unknown>
+"
+`;
+
+exports[`Single word default files > creates a single word function file > Test snapshot 1`] = `
+"import { SampleJob } from './SampleJob'
+
+describe('SampleJob', () => {
+  it('should not throw any errors', async () => {
+    await expect(SampleJob.perform()).resolves.not.toThrow()
+  })
+})
+"
+`;
+
+exports[`Single word default files > creates a single word function file 1`] = `
+"import { jobs } from 'src/lib/jobs'
+
+export const SampleJob = jobs.createJob({
+  queue: 'default',
+  perform: async () => {
+    jobs.logger.info('SampleJob is performing...')
+  },
+})
+"
+`;
+
+exports[`multi-word files > creates a multi word function file 1`] = `undefined`;

packages/cli/src/commands/generate/job/__tests__/job.test.ts

@@ -0,0 +1,86 @@
+globalThis.__dirname = __dirname
+// Load shared mocks
+import '../../../../lib/test'
+
+import path from 'path'
+
+import { describe, it, expect } from 'vitest'
+
+import * as jobGenerator from '../job'
+
+// Should be refactored as it's repeated
+type WordFilesType = { [key: string]: string }
+
+describe('Single word default files', async () => {
+  const files: WordFilesType = await jobGenerator.files({
+    name: 'Sample',
+    queueName: 'default',
+    tests: true,
+    typescript: true,
+  })
+
+  it('creates a single word function file', () => {
+    expect(
+      files[
+        path.normalize('/path/to/project/api/src/jobs/SampleJob/SampleJob.ts')
+      ],
+    ).toMatchSnapshot()
+
+    expect(
+      files[
+        path.normalize(
+          '/path/to/project/api/src/jobs/SampleJob/SampleJob.test.ts',
+        )
+      ],
+    ).toMatchSnapshot('Test snapshot')
+
+    expect(
+      files[
+        path.normalize(
+          '/path/to/project/api/src/jobs/SampleJob/SampleJob.scenarios.ts',
+        )
+      ],
+    ).toMatchSnapshot('Scenario snapshot')
+  })
+})
+
+describe('multi-word files', () => {
+  it('creates a multi word function file', async () => {
+    const multiWordDefaultFiles = await jobGenerator.files({
+      name: 'send-mail',
+      queueName: 'default',
+      tests: false,
+      typescript: true,
+    })
+
+    expect(
+      multiWordDefaultFiles[
+        path.normalize(
+          '/path/to/project/api/src/functions/SendMailJob/SendMailJob.js',
+        )
+      ],
+    ).toMatchSnapshot()
+  })
+})
+
+describe('generation of js files', async () => {
+  const jsFiles: WordFilesType = await jobGenerator.files({
+    name: 'Sample',
+    queueName: 'default',
+    tests: true,
+    typescript: false,
+  })
+
+  it('returns tests, scenario and job file for JS', () => {
+    const fileNames = Object.keys(jsFiles)
+    expect(fileNames.length).toEqual(3)
+
+    expect(fileNames).toEqual(
+      expect.arrayContaining([
+        expect.stringContaining('SampleJob.js'),
+        expect.stringContaining('SampleJob.test.js'),
+        expect.stringContaining('SampleJob.scenarios.js'),
+      ]),
+    )
+  })
+})