Initial Emails service (#261)

* Initial Emails service

* Fix yarn.lock

* Metrics

* Fixes

.github/workflows/ci.yaml

@@ -114,6 +114,7 @@ jobs:
           AUTH0_CLIENT_ID: ${{ secrets.TEST_AUTH0_CLIENT_ID }}
           AUTH0_CLIENT_SECRET: ${{ secrets.TEST_AUTH0_CLIENT_SECRET }}
           AUTH0_USER_PASSWORD: ${{ secrets.AUTH0_TESTING_USER_PASSWORD }}
+          AUTH0_USER_ADMIN_EMAIL: kamil@graphql-hive.com
           AUTH0_USER_MAIN_EMAIL: contact@the-guild.dev
           AUTH0_USER_EXTRA_EMAIL: contact+extra@the-guild.dev
           AUTH0_SECRET: ${{ secrets.TEST_AUTH0_SECRET }}

.vscode/terminals.json

@@ -58,13 +58,6 @@
       "cwd": "packages/services/cdn-worker",
       "command": "yarn dev"
     },
-    {
-      "name": "billing:dev",
-      "description": "Run Billing Service",
-      "open": true,
-      "cwd": "packages/services/stripe-billing",
-      "command": "yarn dev"
-    },
     {
       "name": "usage:dev",
       "description": "Run Usage Service",

deployment/index.ts

@@ -3,6 +3,7 @@ import { DeploymentEnvironment } from './types';
 import { deployDbMigrations } from './services/db-migrations';
 import { deployTokens } from './services/tokens';
 import { deployWebhooks } from './services/webhooks';
+import { deployEmails } from './services/emails';
 import { deploySchema } from './services/schema';
 import { deployUsage } from './services/usage';
 import { deployUsageIngestor } from './services/usage-ingestor';
@@ -38,6 +39,7 @@ const appHostname = `${appDns}.${rootDns}`;
 const docsHostname = `${docsDns}.${rootDns}`;
 
 const heartbeatsConfig = new pulumi.Config('heartbeats');
+const emailConfig = new pulumi.Config('email');
 
 const resourceGroup = new azure.core.ResourceGroup(`hive-${envName}-rg`, {
   location: azure.Locations.EastUS,
@@ -102,6 +104,19 @@ const webhooksApi = deployWebhooks({
   heartbeat: heartbeatsConfig.get('webhooks'),
 });
 
+const emailsApi = deployEmails({
+  packageHelper,
+  storageContainer,
+  deploymentEnv,
+  redis: redisApi,
+  email: {
+    token: emailConfig.requireSecret('token'),
+    from: emailConfig.require('from'),
+    messageStream: emailConfig.require('messageStream'),
+  },
+  // heartbeat: heartbeatsConfig.get('emails'),
+});
+
 const usageEstimationApi = deployUsageEstimation({
   packageHelper,
   storageContainer,
@@ -124,6 +139,7 @@ const rateLimitApi = deployRateLimit({
   deploymentEnv,
   dbMigrations,
   usageEstimator: usageEstimationApi,
+  emails: emailsApi,
 });
 
 const usageApi = deployUsage({

deployment/services/emails.ts

@@ -0,0 +1,59 @@
+import * as pulumi from '@pulumi/pulumi';
+import * as azure from '@pulumi/azure';
+import { RemoteArtifactAsServiceDeployment } from '../utils/remote-artifact-as-service';
+import { DeploymentEnvironment } from '../types';
+import { Redis } from './redis';
+import { PackageHelper } from '../utils/pack';
+
+const commonConfig = new pulumi.Config('common');
+const commonEnv = commonConfig.requireObject<Record<string, string>>('env');
+
+export type Emails = ReturnType<typeof deployEmails>;
+
+export function deployEmails({
+  storageContainer,
+  packageHelper,
+  deploymentEnv,
+  redis,
+  heartbeat,
+  email,
+}: {
+  storageContainer: azure.storage.Container;
+  packageHelper: PackageHelper;
+  deploymentEnv: DeploymentEnvironment;
+  redis: Redis;
+  heartbeat?: string;
+  email: {
+    token: pulumi.Output<string>;
+    from: string;
+    messageStream: string;
+  };
+}) {
+  return new RemoteArtifactAsServiceDeployment(
+    'emails-service',
+    {
+      storageContainer,
+      env: {
+        ...deploymentEnv,
+        ...commonEnv,
+        HEARTBEAT_ENDPOINT: heartbeat ?? '',
+        METRICS_ENABLED: 'true',
+        RELEASE: packageHelper.currentReleaseId(),
+        REDIS_HOST: redis.config.host,
+        REDIS_PORT: String(redis.config.port),
+        REDIS_PASSWORD: redis.config.password,
+        BULLMQ_COMMANDS_FROM_ROOT: 'true',
+        EMAIL_PROVIDER: 'postmark',
+        EMAIL_FROM: email.from,
+        POSTMARK_TOKEN: email.token,
+        POSTMARK_MESSAGE_STREAM: email.messageStream,
+      },
+      readinessProbe: '/_readiness',
+      livenessProbe: '/_health',
+      exposesMetrics: true,
+      packageInfo: packageHelper.npmPack('@hive/emails'),
+      replicas: 1,
+    },
+    [redis.deployment, redis.service]
+  ).deploy();
+}

deployment/services/rate-limit.ts

@@ -5,6 +5,7 @@ import { PackageHelper } from '../utils/pack';
 import { DeploymentEnvironment } from '../types';
 import { DbMigrations } from './db-migrations';
 import { UsageEstimator } from './usage-estimation';
+import { Emails } from './emails';
 import { serviceLocalEndpoint } from '../utils/local-endpoint';
 
 const rateLimitConfig = new pulumi.Config('rateLimit');
@@ -20,12 +21,14 @@ export function deployRateLimit({
   deploymentEnv,
   dbMigrations,
   usageEstimator,
+  emails,
 }: {
   usageEstimator: UsageEstimator;
   storageContainer: azure.storage.Container;
   packageHelper: PackageHelper;
   deploymentEnv: DeploymentEnvironment;
   dbMigrations: DbMigrations;
+  emails: Emails;
 }) {
   return new RemoteArtifactAsServiceDeployment(
     'rate-limiter',
@@ -40,6 +43,7 @@ export function deployRateLimit({
         LIMIT_CACHE_UPDATE_INTERVAL_MS: rateLimitConfig.require('updateIntervalMs'),
         RELEASE: packageHelper.currentReleaseId(),
         USAGE_ESTIMATOR_ENDPOINT: serviceLocalEndpoint(usageEstimator.service),
+        EMAILS_ENDPOINT: serviceLocalEndpoint(emails.service),
         POSTGRES_CONNECTION_STRING: apiConfig.requireSecret('postgresConnectionString'),
       },
       exposesMetrics: true,

integration-tests/.env.template

@@ -2,6 +2,7 @@ AUTH0_DOMAIN="<sync>"
 AUTH0_CLIENT_ID="<sync>"
 AUTH0_CLIENT_SECRET="<sync>"
 AUTH0_USER_PASSWORD="<sync>"
+AUTH0_USER_ADMIN_EMAIL="<sync>"
 AUTH0_USER_MAIN_EMAIL="<sync>"
 AUTH0_USER_EXTRA_EMAIL="<sync>"
 AUTH0_SECRET="<sync>"

integration-tests/docker-compose.yml

@@ -144,6 +144,8 @@ services:
         condition: service_healthy
       webhooks:
         condition: service_healthy
+      emails:
+        condition: service_healthy
       schema:
         condition: service_healthy
       usage_estimator:
@@ -319,6 +321,37 @@ services:
       REDIS_PORT: 6379
       REDIS_PASSWORD: test
 
+  emails:
+    image: node:16.13.2-alpine3.14
+    entrypoint:
+      - '/bin/sh'
+      - '/run-emails.sh'
+    networks:
+      - 'stack'
+    depends_on:
+      redis:
+        condition: service_healthy
+    healthcheck:
+      test: ['CMD', 'wget', '--spider', '-q', 'localhost:3011/_readiness']
+      interval: 5s
+      timeout: 5s
+      retries: 6
+      start_period: 5s
+    ports:
+      - 3011:3011
+    volumes:
+      - './tarballs/emails.tgz:/emails.tgz'
+      - './run-emails.sh:/run-emails.sh'
+    environment:
+      NODE_ENV: production
+      BULLMQ_COMMANDS_FROM_ROOT: 'true'
+      PORT: 3011
+      REDIS_HOST: redis
+      REDIS_PORT: 6379
+      REDIS_PASSWORD: test
+      EMAIL_PROVIDER: mock
+      EMAIL_FROM: mock@graphql-hive.com
+
   usage:
     image: node:16.13.2-alpine3.14
     entrypoint:
@@ -449,10 +482,14 @@ services:
         condition: service_completed_successfully
       usage_estimator:
         condition: service_healthy
+      emails:
+        condition: service_healthy
     environment:
       NODE_ENV: production
+      LIMIT_CACHE_UPDATE_INTERVAL_MS: 2000
       POSTGRES_CONNECTION_STRING: 'postgresql://postgres:postgres@db:5432/registry'
       USAGE_ESTIMATOR_ENDPOINT: http://usage_estimator:3008
+      EMAILS_ENDPOINT: http://emails:3011
       ROARR_LOG: 'true'
       PORT: 3009
 

integration-tests/dockest.ts

@@ -24,7 +24,7 @@ async function main() {
         transform: {
           '^.+\\.ts$': 'ts-jest',
         },
-        testTimeout: 60_000,
+        testTimeout: 30_000,
         maxConcurrency: 1,
         setupFiles: ['dotenv/config'],
         setupFilesAfterEnv: ['./jest-setup.ts'],

integration-tests/run-emails.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+set -e
+
+npm install -g file:emails.tgz
+emails

integration-tests/testkit/auth.ts

@@ -7,17 +7,19 @@ const authenticationApi = new AuthenticationClient({
   clientSecret: ensureEnv('AUTH0_CLIENT_SECRET'),
 });
 
-type UserID = 'main' | 'extra';
+type UserID = 'main' | 'extra' | 'admin';
 const password = ensureEnv('AUTH0_USER_PASSWORD');
 
 const userEmails: Record<UserID, string> = {
   main: ensureEnv('AUTH0_USER_MAIN_EMAIL'),
   extra: ensureEnv('AUTH0_USER_EXTRA_EMAIL'),
+  admin: ensureEnv('AUTH0_USER_ADMIN_EMAIL'),
 };
 
 const tokenResponsePromise: Record<UserID, Promise<TokenResponse> | null> = {
   main: null,
   extra: null,
+  admin: null,
 };
 
 export function authenticate(userId: UserID) {

integration-tests/testkit/emails.ts

@@ -0,0 +1,18 @@
+import * as utils from 'dockest/test-helper';
+import axios from 'axios';
+
+const emailsAddress = utils.getServiceAddress('emails', 3011);
+
+export interface Email {
+  to: string;
+  subject: string;
+  body: string;
+}
+
+export async function history() {
+  const res = await axios.get<Email[]>(`http://${emailsAddress}/_history`, {
+    responseType: 'json',
+  });
+
+  return res.data;
+}

integration-tests/testkit/flow.ts

@@ -20,7 +20,9 @@ import type {
   UpdateTargetNameInput,
   SchemaVersionUpdateInput,
   TargetSelectorInput,
+  OrganizationSelectorInput,
   SchemaSyncCdnInput,
+  RateLimitInput,
 } from './gql/graphql';
 import { execute } from './graphql';
 
@@ -748,3 +750,24 @@ export async function fetchMetadataFromCDN(selector: TargetSelectorInput, token:
     status: res.status,
   };
 }
+
+export async function updateOrgRateLimit(
+  selector: OrganizationSelectorInput,
+  monthlyLimits: RateLimitInput,
+  authToken: string
+) {
+  return execute({
+    document: gql(/* GraphQL */ `
+      mutation updateOrgRateLimit($selector: OrganizationSelectorInput!, $monthlyLimits: RateLimitInput!) {
+        updateOrgRateLimit(selector: $selector, monthlyLimits: $monthlyLimits) {
+          id
+        }
+      }
+    `),
+    variables: {
+      selector,
+      monthlyLimits,
+    },
+    authToken,
+  });
+}

integration-tests/tests/api/organization/get-started.spec.ts

@@ -187,7 +187,7 @@ test('completing each step should result in updated Get Started progress', async
     token,
     authorizationHeader: 'authorization',
   });
-  await waitFor(10_000);
+  await waitFor(5_000);
 
   steps = await getSteps({
     organization: org.cleanId,