From cca09fee3ec6357b32938d9729c3619f420ac8d8 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Fri, 6 Dec 2024 11:01:49 +0100 Subject: [PATCH 01/67] Save user's registration token to users table when signing up --- .../app/components/matrix/register-user.gts | 18 ++++++- packages/host/app/services/matrix-service.ts | 15 +++++- packages/host/app/services/realm-server.ts | 39 ++++++++++++--- .../config/schema/1733393646040_schema.sql | 50 +++++++++++++++++++ ..._add-matrix-registration-token-to-users.js | 15 ++++++ .../handlers/handle-create-user.ts | 45 +++++++++++++++++ packages/realm-server/routes.ts | 6 +++ packages/realm-server/server.ts | 5 -- packages/runtime-common/user-queries.ts | 7 ++- 9 files changed, 184 insertions(+), 16 deletions(-) create mode 100644 packages/host/config/schema/1733393646040_schema.sql create mode 100644 packages/postgres/migrations/1733393646040_add-matrix-registration-token-to-users.js create mode 100644 packages/realm-server/handlers/handle-create-user.ts diff --git a/packages/host/app/components/matrix/register-user.gts b/packages/host/app/components/matrix/register-user.gts index 1096023af6..ceb3ec91ee 100644 --- a/packages/host/app/components/matrix/register-user.gts +++ b/packages/host/app/components/matrix/register-user.gts @@ -345,7 +345,7 @@ export default class RegisterUser extends Component { type: 'waitForEmailValidation'; username: string; password: string; - token?: string; + token: string; session: string; email: string; name: string; @@ -673,6 +673,7 @@ export default class RegisterUser extends Component { ); } let auth: RegisterResponse; + try { auth = await this.matrixService.client.registerRequest({ username: this.state.username, @@ -713,12 +714,18 @@ export default class RegisterUser extends Component { throw e; } + // If access_token and device_id are present, RegisterResponse matches LoginResponse // except for the optional well_known field - if (auth.access_token && auth.device_id) { + if ( + this.state.type === 'waitForEmailValidation' && + auth.access_token && + auth.device_id + ) { await this.matrixService.initializeNewUser( auth as LoginResponse, this.state.name, + this.state.token, ); } }); @@ -743,7 +750,14 @@ export default class RegisterUser extends Component { // it means we are polling the validation. if (this.state.type === 'waitForEmailValidation') { await timeout(1000); + } else { + if (this.state.type !== 'sendToken') { + throw new Error( + `invalid state: cannot go to 'm.login.email.identity' from state ${this.state.type}`, + ); + } } + this.state = { ...this.state, type: 'waitForEmailValidation', diff --git a/packages/host/app/services/matrix-service.ts b/packages/host/app/services/matrix-service.ts index 01e5ef1552..3b513f9cb0 100644 --- a/packages/host/app/services/matrix-service.ts +++ b/packages/host/app/services/matrix-service.ts @@ -261,11 +261,24 @@ export default class MatrixService extends Service { return this._isNewUser; } - async initializeNewUser(auth: LoginResponse, displayName: string) { + async initializeNewUser( + auth: LoginResponse, + displayName: string, + registrationToken: string, + ) { displayName = displayName.trim(); this._isInitializingNewUser = true; this.start({ auth }); this.setDisplayName(displayName); + let userId = this.client.getUserId(); + if (!userId) { + throw new Error( + `bug: there is no userId associated with the matrix client`, + ); + } + + await this.realmServer.createUser(userId, registrationToken); + await Promise.all([ this.createPersonalRealmForUser({ endpoint: 'personal', diff --git a/packages/host/app/services/realm-server.ts b/packages/host/app/services/realm-server.ts index 274459780d..347f3a9381 100644 --- a/packages/host/app/services/realm-server.ts +++ b/packages/host/app/services/realm-server.ts @@ -82,19 +82,44 @@ export default class RealmServerService extends Service { window.localStorage.getItem(sessionLocalStorageKey) ?? undefined; } + async ensureLoggedIn() { + if (this.auth.type !== 'logged-in') { + await this.login(); + } + } + + async createUser(matrixUserId: string, registrationToken: string) { + await this.ensureLoggedIn(); + let response = await this.network.authedFetch(`${this.url.href}_user`, { + method: 'POST', + headers: { + Accept: SupportedMimeType.JSONAPI, + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.token}`, + }, + body: JSON.stringify({ + data: { + type: 'user', + attributes: { matrixUserId, registrationToken }, + }, + }), + }); + if (!response.ok) { + let err = `Could not create user with parameters '${matrixUserId}' and '${registrationToken}': ${ + response.status + } - ${await response.text()}`; + console.error(err); + throw new Error(err); + } + } + async createRealm(args: { endpoint: string; name: string; iconURL?: string; backgroundURL?: string; }) { - if (!this.client) { - throw new Error(`Cannot create realm without matrix client`); - } - await this.login(); - if (this.auth.type !== 'logged-in') { - throw new Error('Could not login to realm server'); - } + await this.ensureLoggedIn(); let response = await this.network.authedFetch( `${this.url.href}_create-realm`, diff --git a/packages/host/config/schema/1733393646040_schema.sql b/packages/host/config/schema/1733393646040_schema.sql new file mode 100644 index 0000000000..04238f9494 --- /dev/null +++ b/packages/host/config/schema/1733393646040_schema.sql @@ -0,0 +1,50 @@ +-- This is auto-generated by packages/realm-server/scripts/convert-to-sqlite.ts +-- Please don't directly modify this file + + CREATE TABLE IF NOT EXISTS boxel_index ( + url TEXT NOT NULL, + file_alias TEXT NOT NULL, + type TEXT NOT NULL, + realm_version INTEGER NOT NULL, + realm_url TEXT NOT NULL, + pristine_doc BLOB, + search_doc BLOB, + error_doc BLOB, + deps BLOB, + types BLOB, + isolated_html TEXT, + indexed_at, + is_deleted BOOLEAN, + source TEXT, + transpiled_code TEXT, + last_modified, + embedded_html BLOB, + atom_html TEXT, + fitted_html BLOB, + display_names BLOB, + resource_created_at, + PRIMARY KEY ( url, realm_version, realm_url, type ) +); + + CREATE TABLE IF NOT EXISTS realm_meta ( + realm_url TEXT NOT NULL, + realm_version INTEGER NOT NULL, + value BLOB NOT NULL, + indexed_at, + PRIMARY KEY ( realm_url, realm_version ) +); + + CREATE TABLE IF NOT EXISTS realm_user_permissions ( + realm_url TEXT NOT NULL, + username TEXT NOT NULL, + read BOOLEAN NOT NULL, + write BOOLEAN NOT NULL, + realm_owner BOOLEAN DEFAULT false NOT NULL, + PRIMARY KEY ( realm_url, username ) +); + + CREATE TABLE IF NOT EXISTS realm_versions ( + realm_url TEXT NOT NULL, + current_version INTEGER NOT NULL, + PRIMARY KEY ( realm_url ) +); \ No newline at end of file diff --git a/packages/postgres/migrations/1733393646040_add-matrix-registration-token-to-users.js b/packages/postgres/migrations/1733393646040_add-matrix-registration-token-to-users.js new file mode 100644 index 0000000000..7746521537 --- /dev/null +++ b/packages/postgres/migrations/1733393646040_add-matrix-registration-token-to-users.js @@ -0,0 +1,15 @@ +/* eslint-disable camelcase */ + +exports.shorthands = undefined; + +exports.up = (pgm) => { + pgm.addColumns('users', { + matrix_registration_token: { + type: 'varchar', + }, + }); +}; + +exports.down = (pgm) => { + pgm.dropColumns('users', ['matrix_registration_token']); +}; diff --git a/packages/realm-server/handlers/handle-create-user.ts b/packages/realm-server/handlers/handle-create-user.ts new file mode 100644 index 0000000000..547e3d90f5 --- /dev/null +++ b/packages/realm-server/handlers/handle-create-user.ts @@ -0,0 +1,45 @@ +import { upsertUser } from '@cardstack/runtime-common'; +import Koa from 'koa'; +import { + fetchRequestFromContext, + sendResponseForBadRequest, + sendResponseForSystemError, + setContextResponse, +} from '../middleware'; +import { RealmServerTokenClaim } from '../utils/jwt'; +import { CreateRoutesArgs } from '../routes'; + +export default function handleCreateUserRequest({ + dbAdapter, +}: CreateRoutesArgs): (ctxt: Koa.Context, next: Koa.Next) => Promise { + return async function (ctxt: Koa.Context, _next: Koa.Next) { + let token = ctxt.state.token as RealmServerTokenClaim; + if (!token) { + await sendResponseForSystemError( + ctxt, + 'token is required to create user', + ); + return; + } + + let { user: matrixUserId } = token; + + let request = await fetchRequestFromContext(ctxt); + let body = await request.text(); + let json: Record; + try { + json = JSON.parse(body); + } catch (e) { + await sendResponseForBadRequest( + ctxt, + 'Request body is not valid JSON-API - invalid JSON', + ); + return; + } + + let registrationToken = json.data.attributes.registrationToken; + + await upsertUser(dbAdapter, matrixUserId, registrationToken); + await setContextResponse(ctxt, new Response('ok')); + }; +} diff --git a/packages/realm-server/routes.ts b/packages/realm-server/routes.ts index f62c2f8699..137662cd6e 100644 --- a/packages/realm-server/routes.ts +++ b/packages/realm-server/routes.ts @@ -9,6 +9,7 @@ import handleStripeWebhookRequest from './handlers/handle-stripe-webhook'; import { healthCheck, jwtMiddleware, livenessCheck } from './middleware'; import Koa from 'koa'; import handleStripeLinksRequest from './handlers/handle-stripe-links'; +import handleCreateUserRequest from './handlers/handle-create-user'; export type CreateRoutesArgs = { dbAdapter: DBAdapter; @@ -51,6 +52,11 @@ export function createRoutes(args: CreateRoutesArgs) { jwtMiddleware(args.secretSeed), handleFetchUserRequest(args), ); + router.post( + '/_user', + jwtMiddleware(args.secretSeed), + handleCreateUserRequest(args), + ); router.get('/_stripe-links', handleStripeLinksRequest()); return router.routes(); diff --git a/packages/realm-server/server.ts b/packages/realm-server/server.ts index 97e292d355..ae9ee92ecd 100644 --- a/packages/realm-server/server.ts +++ b/packages/realm-server/server.ts @@ -330,11 +330,6 @@ export class RealmServer { [ownerUserId]: DEFAULT_PERMISSIONS, }); - // It's not desirable to have user insertion entangled with realm creation– - // In the future we could refactor this to handle user creation in a separate - // endpoint - await upsertUser(this.dbAdapter, ownerUserId); - writeJSONSync(join(realmPath, '.realm.json'), { name, ...(iconURL ? { iconURL } : {}), diff --git a/packages/runtime-common/user-queries.ts b/packages/runtime-common/user-queries.ts index 192c05fda0..eaef549f76 100644 --- a/packages/runtime-common/user-queries.ts +++ b/packages/runtime-common/user-queries.ts @@ -2,9 +2,14 @@ import { DBAdapter } from './db'; import { query, asExpressions, upsert } from './expression'; -export async function upsertUser(dbAdapter: DBAdapter, matrixUserId: string) { +export async function upsertUser( + dbAdapter: DBAdapter, + matrixUserId: string, + matrixRegistrationToken: string, +) { let { valueExpressions, nameExpressions } = asExpressions({ matrix_user_id: matrixUserId, + matrix_registration_token: matrixRegistrationToken, }); await query( From 7c8424e9dda83468c67d5e50ff96268df13b79a0 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Fri, 6 Dec 2024 11:36:59 +0100 Subject: [PATCH 02/67] Oops - there can be a flow without registration token --- .../host/app/components/matrix/register-user.gts | 14 ++------------ packages/host/app/services/matrix-service.ts | 2 +- packages/host/app/services/realm-server.ts | 7 +++++-- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/packages/host/app/components/matrix/register-user.gts b/packages/host/app/components/matrix/register-user.gts index ceb3ec91ee..a2e30f076e 100644 --- a/packages/host/app/components/matrix/register-user.gts +++ b/packages/host/app/components/matrix/register-user.gts @@ -345,7 +345,7 @@ export default class RegisterUser extends Component { type: 'waitForEmailValidation'; username: string; password: string; - token: string; + token?: string; session: string; email: string; name: string; @@ -717,11 +717,7 @@ export default class RegisterUser extends Component { // If access_token and device_id are present, RegisterResponse matches LoginResponse // except for the optional well_known field - if ( - this.state.type === 'waitForEmailValidation' && - auth.access_token && - auth.device_id - ) { + if (auth.access_token && auth.device_id) { await this.matrixService.initializeNewUser( auth as LoginResponse, this.state.name, @@ -750,12 +746,6 @@ export default class RegisterUser extends Component { // it means we are polling the validation. if (this.state.type === 'waitForEmailValidation') { await timeout(1000); - } else { - if (this.state.type !== 'sendToken') { - throw new Error( - `invalid state: cannot go to 'm.login.email.identity' from state ${this.state.type}`, - ); - } } this.state = { diff --git a/packages/host/app/services/matrix-service.ts b/packages/host/app/services/matrix-service.ts index 3b513f9cb0..76c0e2f370 100644 --- a/packages/host/app/services/matrix-service.ts +++ b/packages/host/app/services/matrix-service.ts @@ -264,7 +264,7 @@ export default class MatrixService extends Service { async initializeNewUser( auth: LoginResponse, displayName: string, - registrationToken: string, + registrationToken?: string, ) { displayName = displayName.trim(); this._isInitializingNewUser = true; diff --git a/packages/host/app/services/realm-server.ts b/packages/host/app/services/realm-server.ts index 347f3a9381..05954063df 100644 --- a/packages/host/app/services/realm-server.ts +++ b/packages/host/app/services/realm-server.ts @@ -88,7 +88,7 @@ export default class RealmServerService extends Service { } } - async createUser(matrixUserId: string, registrationToken: string) { + async createUser(matrixUserId: string, registrationToken?: string) { await this.ensureLoggedIn(); let response = await this.network.authedFetch(`${this.url.href}_user`, { method: 'POST', @@ -100,7 +100,10 @@ export default class RealmServerService extends Service { body: JSON.stringify({ data: { type: 'user', - attributes: { matrixUserId, registrationToken }, + attributes: { + matrixUserId, + registrationToken: registrationToken ?? null, + }, }, }), }); From 2386091ed6d13e95477dcbda00cccd1050c7d09f Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Fri, 6 Dec 2024 11:37:06 +0100 Subject: [PATCH 03/67] Not needed --- packages/realm-server/server.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/realm-server/server.ts b/packages/realm-server/server.ts index ae9ee92ecd..69c90919fc 100644 --- a/packages/realm-server/server.ts +++ b/packages/realm-server/server.ts @@ -11,7 +11,6 @@ import { type DBAdapter, type QueuePublisher, type RealmPermissions, - upsertUser, } from '@cardstack/runtime-common'; import { ensureDirSync, writeJSONSync, readdirSync, copySync } from 'fs-extra'; import { setupCloseHandler } from './node-realm'; From e8f112389a25fa0dca145657345a72f372b7ec93 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Fri, 6 Dec 2024 11:57:11 +0100 Subject: [PATCH 04/67] Type fix --- packages/host/app/components/matrix/register-user.gts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/host/app/components/matrix/register-user.gts b/packages/host/app/components/matrix/register-user.gts index a2e30f076e..0a0c1706fc 100644 --- a/packages/host/app/components/matrix/register-user.gts +++ b/packages/host/app/components/matrix/register-user.gts @@ -717,7 +717,11 @@ export default class RegisterUser extends Component { // If access_token and device_id are present, RegisterResponse matches LoginResponse // except for the optional well_known field - if (auth.access_token && auth.device_id) { + if ( + auth.access_token && + auth.device_id && + this.state.type === 'waitForEmailValidation' + ) { await this.matrixService.initializeNewUser( auth as LoginResponse, this.state.name, From a4c627644ecf235dbe11fb5cd94e651434f81872 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Fri, 6 Dec 2024 12:35:57 +0100 Subject: [PATCH 05/67] Add tests --- packages/billing/billing-queries.ts | 3 ++ .../realm-server/tests/realm-server-test.ts | 48 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/packages/billing/billing-queries.ts b/packages/billing/billing-queries.ts index 9438c5cc10..0f4b5d19e9 100644 --- a/packages/billing/billing-queries.ts +++ b/packages/billing/billing-queries.ts @@ -17,6 +17,7 @@ export interface User { matrixUserId: string; stripeCustomerId: string; stripeCustomerEmail: string | null; + matrixRegistrationToken: string | null; } export interface Plan { @@ -167,6 +168,7 @@ export async function getUserByStripeId( id: results[0].id, matrixUserId: results[0].matrix_user_id, stripeCustomerId: results[0].stripe_customer_id, + matrixRegistrationToken: results[0].matrix_registration_token, } as User; } @@ -188,6 +190,7 @@ export async function getUserByMatrixUserId( matrixUserId: results[0].matrix_user_id, stripeCustomerId: results[0].stripe_customer_id, stripeCustomerEmail: results[0].stripe_customer_email, + matrixRegistrationToken: results[0].matrix_registration_token, } as User; } diff --git a/packages/realm-server/tests/realm-server-test.ts b/packages/realm-server/tests/realm-server-test.ts index 09be21e2a4..dc90ef7e8c 100644 --- a/packages/realm-server/tests/realm-server-test.ts +++ b/packages/realm-server/tests/realm-server-test.ts @@ -71,6 +71,7 @@ import { addToCreditsLedger, insertSubscriptionCycle, insertSubscription, + getUserByMatrixUserId, } from '@cardstack/billing/billing-queries'; import { createJWT as createRealmServerJWT, @@ -80,6 +81,7 @@ import { resetCatalogRealms } from '../handlers/handle-fetch-catalog-realms'; import Stripe from 'stripe'; import sinon from 'sinon'; import { getStripe } from '@cardstack/billing/stripe-webhook-handlers/stripe'; +import { fetchUserByMatrixUserId } from '@cardstack/billing/fetch-user-by-matrix-user-id'; setGracefulCleanup(); const testRealmURL = new URL('http://127.0.0.1:4444/'); @@ -3677,6 +3679,52 @@ module('Realm Server', function (hooks) { assert.strictEqual(response.status, 404, 'HTTP 404 status'); }); + test('can create a user', async function (assert) { + let ownerUserId = '@mango:boxel.ai'; + let response = await request2 + .post('/_user') + .set('Accept', 'application/json') + .set('Content-Type', 'application/json') + .set( + 'Authorization', + `Bearer ${createRealmServerJWT( + { user: ownerUserId, sessionRoom: 'session-room-test' }, + secretSeed, + )}`, + ) + .send({ + data: { + type: 'user', + attributes: { + registrationToken: 'reg_token_123', + }, + }, + }); + + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + assert.strictEqual(response.text, 'ok', 'response body is correct'); + + let user = await getUserByMatrixUserId(dbAdapter, ownerUserId); + if (!user) { + throw new Error('user does not exist in db'); + } + assert.strictEqual( + user.matrixUserId, + ownerUserId, + 'matrix user ID is correct', + ); + assert.strictEqual( + user.matrixRegistrationToken, + 'reg_token_123', + 'registration token is correct', + ); + }); + + test('can not create a user without a jwt', async function (assert) { + let response = await request2.post('/_user').send({}); + assert.strictEqual(response.status, 401, 'HTTP 401 status'); + }); + test('can dynamically load a card definition from own realm', async function (assert) { let ref = { module: `${testRealmHref}person`, From 9c9f855170807d7f3534815725827f83de1d1a4e Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Fri, 6 Dec 2024 12:47:22 +0100 Subject: [PATCH 06/67] Not needed --- packages/realm-server/tests/realm-server-test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/realm-server/tests/realm-server-test.ts b/packages/realm-server/tests/realm-server-test.ts index dc90ef7e8c..5dc14602e1 100644 --- a/packages/realm-server/tests/realm-server-test.ts +++ b/packages/realm-server/tests/realm-server-test.ts @@ -81,7 +81,6 @@ import { resetCatalogRealms } from '../handlers/handle-fetch-catalog-realms'; import Stripe from 'stripe'; import sinon from 'sinon'; import { getStripe } from '@cardstack/billing/stripe-webhook-handlers/stripe'; -import { fetchUserByMatrixUserId } from '@cardstack/billing/fetch-user-by-matrix-user-id'; setGracefulCleanup(); const testRealmURL = new URL('http://127.0.0.1:4444/'); From 4b92cb17b6b3d8b130014a0ae2b2b97df44dea57 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Fri, 6 Dec 2024 14:28:24 +0100 Subject: [PATCH 07/67] login() checks if already logged in --- packages/host/app/services/realm-server.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/packages/host/app/services/realm-server.ts b/packages/host/app/services/realm-server.ts index 05954063df..3e8643b9f5 100644 --- a/packages/host/app/services/realm-server.ts +++ b/packages/host/app/services/realm-server.ts @@ -82,14 +82,8 @@ export default class RealmServerService extends Service { window.localStorage.getItem(sessionLocalStorageKey) ?? undefined; } - async ensureLoggedIn() { - if (this.auth.type !== 'logged-in') { - await this.login(); - } - } - async createUser(matrixUserId: string, registrationToken?: string) { - await this.ensureLoggedIn(); + await this.login(); let response = await this.network.authedFetch(`${this.url.href}_user`, { method: 'POST', headers: { @@ -122,7 +116,7 @@ export default class RealmServerService extends Service { iconURL?: string; backgroundURL?: string; }) { - await this.ensureLoggedIn(); + await this.login(); let response = await this.network.authedFetch( `${this.url.href}_create-realm`, From b6ba37839dea4c75856d692388ad5910e3667e17 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Mon, 9 Dec 2024 10:01:57 +0100 Subject: [PATCH 08/67] `authedFetch` supports realm queries only, so don't use it for realm server queries `authedFetch` has a store and cache of tokens and is able to re(authenticate), but only when the response has the realm url header. but here, the realm server service has its own auth management system so we should use normal fetch and provide bearer token manually --- packages/host/app/services/realm-server.ts | 27 ++++++++++------------ 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/packages/host/app/services/realm-server.ts b/packages/host/app/services/realm-server.ts index 3e8643b9f5..222991c019 100644 --- a/packages/host/app/services/realm-server.ts +++ b/packages/host/app/services/realm-server.ts @@ -84,7 +84,7 @@ export default class RealmServerService extends Service { async createUser(matrixUserId: string, registrationToken?: string) { await this.login(); - let response = await this.network.authedFetch(`${this.url.href}_user`, { + let response = await this.network.fetch(`${this.url.href}_user`, { method: 'POST', headers: { Accept: SupportedMimeType.JSONAPI, @@ -118,20 +118,17 @@ export default class RealmServerService extends Service { }) { await this.login(); - let response = await this.network.authedFetch( - `${this.url.href}_create-realm`, - { - method: 'POST', - headers: { - Accept: SupportedMimeType.JSONAPI, - 'Content-Type': 'application/json', - Authorization: `Bearer ${this.token}`, - }, - body: JSON.stringify({ - data: { type: 'realm', attributes: args }, - }), + let response = await this.network.fetch(`${this.url.href}_create-realm`, { + method: 'POST', + headers: { + Accept: SupportedMimeType.JSONAPI, + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.token}`, }, - ); + body: JSON.stringify({ + data: { type: 'realm', attributes: args }, + }), + }); if (!response.ok) { let err = `Could not create realm with endpoint '${args.endpoint}': ${ response.status @@ -203,7 +200,7 @@ export default class RealmServerService extends Service { if (this.catalogRealmURLs.length > 0) { return; } - let response = await this.network.authedFetch( + let response = await this.network.fetch( `${this.url.origin}/_catalog-realms`, ); if (response.status !== 200) { From 33b666d4a1d333c581c0f6b6bdb4cebef7bf18da Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Mon, 9 Dec 2024 10:20:44 +0100 Subject: [PATCH 09/67] Cleanup old schemas --- .../config/schema/1733196359718_schema.sql | 50 ------------------- .../config/schema/1733253128046_schema.sql | 50 ------------------- .../config/schema/1733393646040_schema.sql | 2 +- 3 files changed, 1 insertion(+), 101 deletions(-) delete mode 100644 packages/host/config/schema/1733196359718_schema.sql delete mode 100644 packages/host/config/schema/1733253128046_schema.sql diff --git a/packages/host/config/schema/1733196359718_schema.sql b/packages/host/config/schema/1733196359718_schema.sql deleted file mode 100644 index 04238f9494..0000000000 --- a/packages/host/config/schema/1733196359718_schema.sql +++ /dev/null @@ -1,50 +0,0 @@ --- This is auto-generated by packages/realm-server/scripts/convert-to-sqlite.ts --- Please don't directly modify this file - - CREATE TABLE IF NOT EXISTS boxel_index ( - url TEXT NOT NULL, - file_alias TEXT NOT NULL, - type TEXT NOT NULL, - realm_version INTEGER NOT NULL, - realm_url TEXT NOT NULL, - pristine_doc BLOB, - search_doc BLOB, - error_doc BLOB, - deps BLOB, - types BLOB, - isolated_html TEXT, - indexed_at, - is_deleted BOOLEAN, - source TEXT, - transpiled_code TEXT, - last_modified, - embedded_html BLOB, - atom_html TEXT, - fitted_html BLOB, - display_names BLOB, - resource_created_at, - PRIMARY KEY ( url, realm_version, realm_url, type ) -); - - CREATE TABLE IF NOT EXISTS realm_meta ( - realm_url TEXT NOT NULL, - realm_version INTEGER NOT NULL, - value BLOB NOT NULL, - indexed_at, - PRIMARY KEY ( realm_url, realm_version ) -); - - CREATE TABLE IF NOT EXISTS realm_user_permissions ( - realm_url TEXT NOT NULL, - username TEXT NOT NULL, - read BOOLEAN NOT NULL, - write BOOLEAN NOT NULL, - realm_owner BOOLEAN DEFAULT false NOT NULL, - PRIMARY KEY ( realm_url, username ) -); - - CREATE TABLE IF NOT EXISTS realm_versions ( - realm_url TEXT NOT NULL, - current_version INTEGER NOT NULL, - PRIMARY KEY ( realm_url ) -); \ No newline at end of file diff --git a/packages/host/config/schema/1733253128046_schema.sql b/packages/host/config/schema/1733253128046_schema.sql deleted file mode 100644 index 5577b1a87b..0000000000 --- a/packages/host/config/schema/1733253128046_schema.sql +++ /dev/null @@ -1,50 +0,0 @@ --- This is auto-generated by packages/realm-server/scripts/convert-to-sqlite.ts --- Please don't directly modify this file - - CREATE TABLE IF NOT EXISTS boxel_index ( - url TEXT NOT NULL, - file_alias TEXT NOT NULL, - type TEXT NOT NULL, - realm_version INTEGER NOT NULL, - realm_url TEXT NOT NULL, - pristine_doc BLOB, - search_doc BLOB, - error_doc BLOB, - deps BLOB, - types BLOB, - isolated_html TEXT, - indexed_at, - is_deleted BOOLEAN, - source TEXT, - transpiled_code TEXT, - last_modified, - embedded_html BLOB, - atom_html TEXT, - fitted_html BLOB, - display_names BLOB, - resource_created_at, - PRIMARY KEY ( url, realm_version, realm_url ) -); - - CREATE TABLE IF NOT EXISTS realm_meta ( - realm_url TEXT NOT NULL, - realm_version INTEGER NOT NULL, - value BLOB NOT NULL, - indexed_at, - PRIMARY KEY ( realm_url, realm_version ) -); - - CREATE TABLE IF NOT EXISTS realm_user_permissions ( - realm_url TEXT NOT NULL, - username TEXT NOT NULL, - read BOOLEAN NOT NULL, - write BOOLEAN NOT NULL, - realm_owner BOOLEAN DEFAULT false NOT NULL, - PRIMARY KEY ( realm_url, username ) -); - - CREATE TABLE IF NOT EXISTS realm_versions ( - realm_url TEXT NOT NULL, - current_version INTEGER NOT NULL, - PRIMARY KEY ( realm_url ) -); \ No newline at end of file diff --git a/packages/host/config/schema/1733393646040_schema.sql b/packages/host/config/schema/1733393646040_schema.sql index 04238f9494..5577b1a87b 100644 --- a/packages/host/config/schema/1733393646040_schema.sql +++ b/packages/host/config/schema/1733393646040_schema.sql @@ -23,7 +23,7 @@ fitted_html BLOB, display_names BLOB, resource_created_at, - PRIMARY KEY ( url, realm_version, realm_url, type ) + PRIMARY KEY ( url, realm_version, realm_url ) ); CREATE TABLE IF NOT EXISTS realm_meta ( From af9d238588c00e4647bed1ac258362bdfae92de9 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Mon, 9 Dec 2024 13:49:34 +0100 Subject: [PATCH 10/67] Monthly price should be number, not a string --- packages/billing/billing-queries.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/billing/billing-queries.ts b/packages/billing/billing-queries.ts index 9438c5cc10..8eebb33e6b 100644 --- a/packages/billing/billing-queries.ts +++ b/packages/billing/billing-queries.ts @@ -101,7 +101,7 @@ export async function getPlanByStripeId( return { id: results[0].id, name: results[0].name, - monthlyPrice: results[0].monthly_price, + monthlyPrice: parseFloat(results[0].monthly_price as string), creditsIncluded: results[0].credits_included, stripePlanId: results[0].stripe_plan_id, } as Plan; @@ -456,7 +456,7 @@ export async function getPlanById( return { id: results[0].id, name: results[0].name, - monthlyPrice: results[0].monthly_price, + monthlyPrice: parseFloat(results[0].monthly_price as string), creditsIncluded: results[0].credits_included, stripePlanId: results[0].stripe_plan_id, } as Plan; @@ -478,7 +478,7 @@ export async function getPlanByMonthlyPrice( return { id: results[0].id, name: results[0].name, - monthlyPrice: results[0].monthly_price, + monthlyPrice: parseFloat(results[0].monthly_price as string), creditsIncluded: results[0].credits_included, stripePlanId: results[0].stripe_plan_id, } as Plan; From 086ef43c4e986723fa5c53257c94b97b4061e448 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Mon, 9 Dec 2024 13:51:07 +0100 Subject: [PATCH 11/67] Free plan credits should not carry over when upgrading to a paid plan --- packages/billing/proration-calculator.ts | 26 +++++++++++++-------- packages/realm-server/tests/billing-test.ts | 8 +++---- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/packages/billing/proration-calculator.ts b/packages/billing/proration-calculator.ts index 49485c412d..b5e53069a7 100644 --- a/packages/billing/proration-calculator.ts +++ b/packages/billing/proration-calculator.ts @@ -20,17 +20,23 @@ export class ProrationCalculator { }) { let { currentPlan, newPlan, invoiceLines, currentAllowance } = params; - // Sum up monetary credit (refunds) given to the user by Stripe for unused time on previous plans - // (there can be multiple such lines if user switches to larger plans multiple times in the same billing period) - // and convert it to credits. In other words, take away the credits calculated from the money that Stripe - // returned to the user for unused time. let creditsToExpireForUnusedTime = 0; - for (let line of invoiceLines) { - if (line.amount > 0) continue; - creditsToExpireForUnusedTime += this.centsToCredits( - -line.amount, - currentPlan, - ); + + if (currentPlan.monthlyPrice === 0) { + // If the user is upgrading from a free plan, we do not want to carry over any credits from the free plan into the paid plan + creditsToExpireForUnusedTime = currentAllowance; + } else { + // Sum up monetary credit (refunds) given to the user by Stripe for unused time on previous plans + // (there can be multiple such lines if user switches to larger plans multiple times in the same billing period) + // and convert it to credits. In other words, take away the credits calculated from the money that Stripe + // returned to the user for unused time. + for (let line of invoiceLines) { + if (line.amount > 0) continue; + creditsToExpireForUnusedTime += this.centsToCredits( + -line.amount, + currentPlan, + ); + } } // Find invoice line for the new plan the user is subscribing to diff --git a/packages/realm-server/tests/billing-test.ts b/packages/realm-server/tests/billing-test.ts index daa19fbc99..d5cb1da83a 100644 --- a/packages/realm-server/tests/billing-test.ts +++ b/packages/realm-server/tests/billing-test.ts @@ -350,8 +350,8 @@ module('billing', function (hooks) { subscriptionCycle = subscriptionCycles[0]; - // User received 5000 credits from the creator plan, plus 500 from the plan allowance they had left from the free plan - assert.strictEqual(creditsBalance, 5500); + // User received 5000 credits from the creator plan, but the 500 credits from the plan allowance they had left from the free plan were expired + assert.strictEqual(creditsBalance, 5000); // User spent 2000 credits from the plan allowance await addToCreditsLedger(dbAdapter, { @@ -361,11 +361,11 @@ module('billing', function (hooks) { subscriptionCycleId: subscriptionCycle.id, }); - // Assert that the user now has 3500 credits left + // Assert that the user now has 3000 credits left creditsBalance = await sumUpCreditsLedger(dbAdapter, { userId: user.id, }); - assert.strictEqual(creditsBalance, 3500); + assert.strictEqual(creditsBalance, 3000); // Now, user upgrades to power user plan ($49 monthly) in the middle of the month: From 7d2658f52ba0bdbd988ea4711fb52685bc7a83c1 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Mon, 9 Dec 2024 13:51:20 +0100 Subject: [PATCH 12/67] Cleanup --- packages/billing/stripe-webhook-handlers/payment-succeeded.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/billing/stripe-webhook-handlers/payment-succeeded.ts b/packages/billing/stripe-webhook-handlers/payment-succeeded.ts index f109ee73e0..64ae4f88b4 100644 --- a/packages/billing/stripe-webhook-handlers/payment-succeeded.ts +++ b/packages/billing/stripe-webhook-handlers/payment-succeeded.ts @@ -22,10 +22,6 @@ import { StripeInvoicePaymentSucceededWebhookEvent } from '.'; import { PgAdapter, TransactionManager } from '@cardstack/postgres'; import { ProrationCalculator } from '../proration-calculator'; -// TODOs that will be handled in a separated PRs: -// - signal to frontend that subscription has been created and credits have been added -// - put this in a background job - export async function handlePaymentSucceeded( dbAdapter: DBAdapter, event: StripeInvoicePaymentSucceededWebhookEvent, From 3a2bae3b37c86b9ba4edfd66639367d7dbe32902 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Mon, 9 Dec 2024 14:38:41 +0100 Subject: [PATCH 13/67] price should be a number not a string --- packages/realm-server/tests/helpers/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/realm-server/tests/helpers/index.ts b/packages/realm-server/tests/helpers/index.ts index 5dfc8b7749..084dd8411d 100644 --- a/packages/realm-server/tests/helpers/index.ts +++ b/packages/realm-server/tests/helpers/index.ts @@ -461,7 +461,7 @@ export async function insertPlan( return { id: result[0].id, name: result[0].name, - monthlyPrice: result[0].monthly_price, + monthlyPrice: parseFloat(result[0].monthly_price as string), creditsIncluded: result[0].credits_included, stripePlanId: result[0].stripe_plan_id, } as Plan; From 8e03d437861d1bfb1cc341298d92d3677eb14ab4 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Mon, 9 Dec 2024 14:38:53 +0100 Subject: [PATCH 14/67] Better assertion texts --- packages/realm-server/tests/realm-server-test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/realm-server/tests/realm-server-test.ts b/packages/realm-server/tests/realm-server-test.ts index 09be21e2a4..ff8a4e753b 100644 --- a/packages/realm-server/tests/realm-server-test.ts +++ b/packages/realm-server/tests/realm-server-test.ts @@ -2761,7 +2761,7 @@ module('Realm Server', function (hooks) { john: ['read', 'write'], }); - test('user is not found', async function (assert) { + test('responds with 404 if user is not found', async function (assert) { let response = await request .get(`/_user`) .set('Accept', 'application/vnd.api+json') @@ -2772,7 +2772,7 @@ module('Realm Server', function (hooks) { assert.strictEqual(response.status, 404, 'HTTP 404 status'); }); - test('subscription is not found', async function (assert) { + test('responds with 200 and null subscription values if user is not subscribed', async function (assert) { let user = await insertUser( dbAdapter, 'user@test', @@ -2812,7 +2812,7 @@ module('Realm Server', function (hooks) { ); }); - test('user subscibes to a plan and has extra credit', async function (assert) { + test('response has correct values for subscribed user who has some extra credits', async function (assert) { let user = await insertUser( dbAdapter, 'user@test', From f5cc5e81bd773380a77341278dd363b06c01d078 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Mon, 9 Dec 2024 14:12:11 -0500 Subject: [PATCH 15/67] Add live card support for card errors --- .../operator-mode/card-error-detail.gts | 2 +- .../operator-mode/submode-layout.gts | 14 +- packages/host/app/resources/card-resource.ts | 261 +++++++++--------- packages/host/app/services/realm.ts | 11 + .../tests/acceptance/code-submode-test.ts | 89 ++++++ .../acceptance/interact-submode-test.gts | 105 +++++++ 6 files changed, 352 insertions(+), 130 deletions(-) diff --git a/packages/host/app/components/operator-mode/card-error-detail.gts b/packages/host/app/components/operator-mode/card-error-detail.gts index 6106e068b7..2abf8a51d1 100644 --- a/packages/host/app/components/operator-mode/card-error-detail.gts +++ b/packages/host/app/components/operator-mode/card-error-detail.gts @@ -19,7 +19,7 @@ import type CommandService from '../../services/command-service'; interface Signature { Args: { - error: CardError['errors'][0]; + error: CardError; viewInCodeMode?: true; title?: string; }; diff --git a/packages/host/app/components/operator-mode/submode-layout.gts b/packages/host/app/components/operator-mode/submode-layout.gts index 2291be556f..96c908250a 100644 --- a/packages/host/app/components/operator-mode/submode-layout.gts +++ b/packages/host/app/components/operator-mode/submode-layout.gts @@ -30,8 +30,6 @@ import type IndexController from '@cardstack/host/controllers'; import { assertNever } from '@cardstack/host/utils/assert-never'; -import type { CardDef } from 'https://cardstack.com/base/card-api'; - import SearchSheet, { SearchSheetMode, SearchSheetModes, @@ -99,12 +97,16 @@ export default class SubmodeLayout extends Component { return this.operatorModeStateService.state?.stacks.flat() ?? []; } - private get lastCardInRightMostStack(): CardDef | null { + private get lastCardIdInRightMostStack() { if (this.allStackItems.length <= 0) { return null; } - return this.allStackItems[this.allStackItems.length - 1].card; + let stackItem = this.allStackItems[this.allStackItems.length - 1]; + if (stackItem.cardError) { + return stackItem.cardError.id; + } + return stackItem.card.id; } private get isToggleWorkspaceChooserDisabled() { @@ -118,8 +120,8 @@ export default class SubmodeLayout extends Component { break; case Submodes.Code: this.operatorModeStateService.updateCodePath( - this.lastCardInRightMostStack - ? new URL(this.lastCardInRightMostStack.id + '.json') + this.lastCardIdInRightMostStack + ? new URL(this.lastCardIdInRightMostStack + '.json') : null, ); break; diff --git a/packages/host/app/resources/card-resource.ts b/packages/host/app/resources/card-resource.ts index 61396f04cc..b538fe72e2 100644 --- a/packages/host/app/resources/card-resource.ts +++ b/packages/host/app/resources/card-resource.ts @@ -4,6 +4,7 @@ import { registerDestructor } from '@ember/destroyable'; import { getOwner } from '@ember/owner'; +import { service } from '@ember/service'; import { tracked } from '@glimmer/tracking'; import { restartableTask } from 'ember-concurrency'; @@ -13,11 +14,10 @@ import { Resource } from 'ember-resources'; import status from 'statuses'; import { - Loader, isSingleCardDocument, apiFor, - loaderFor, hasExecutableExtension, + isCardInstance, } from '@cardstack/runtime-common'; import type MessageService from '@cardstack/host/services/message-service'; @@ -31,8 +31,9 @@ import type * as CardAPI from 'https://cardstack.com/base/card-api'; import type CardService from '../services/card-service'; import type LoaderService from '../services/loader-service'; +import type RealmService from '../services/realm'; -export interface CardError { +interface CardErrors { errors: { id: string; status: number; @@ -47,12 +48,13 @@ export interface CardError { }[]; } +export type CardError = CardErrors['errors'][0]; + interface Args { named: { // using string type here so that URL's that have the same href but are // different instances don't result in re-running the resource url: string | undefined; - loader: Loader; isLive: boolean; // this is not always constructed within a container so we pass in our services cardService: CardService; @@ -69,7 +71,7 @@ class LiveCardIdentityContext implements IdentityContext { #cards = new Map< string, { - card: CardDef; + card: CardDef | undefined; // undefined means that the card is in an error state subscribers: Set; } >(); @@ -83,13 +85,23 @@ class LiveCardIdentityContext implements IdentityContext { delete(url: string): void { this.#cards.delete(url); } - + update(url: string, instance: CardDef | undefined) { + let entry = this.#cards.get(url); + if (!entry) { + this.#cards.set(url, { card: instance, subscribers: new Set() }); + } else { + entry.card = instance; + } + } + hasError(url: string) { + return this.#cards.has(url) && !this.#cards.get(url)?.card; + } subscribers(url: string): Set | undefined { return this.#cards.get(url)?.subscribers; } } -const liveCards: WeakMap = new WeakMap(); +const liveCardIdentityContext = new LiveCardIdentityContext(); const realmSubscriptions: Map< string, WeakMap void }> @@ -98,7 +110,8 @@ const realmSubscriptions: Map< export class CardResource extends Resource { url: string | undefined; @tracked loaded: Promise | undefined; - @tracked cardError: CardError['errors'][0] | undefined; + @tracked cardError: CardError | undefined; + @service private declare realm: RealmService; @tracked private _card: CardDef | undefined; @tracked private _api: typeof CardAPI | undefined; @tracked private staleCard: CardDef | undefined; @@ -106,7 +119,6 @@ export class CardResource extends Resource { private declare messageService: MessageService; private declare loaderService: LoaderService; private declare resetLoader: () => void; - private _loader: Loader | undefined; private onCardInstanceChange?: ( oldCard: CardDef | undefined, newCard: CardDef | undefined, @@ -120,7 +132,6 @@ export class CardResource extends Resource { let { url, - loader, isLive, onCardInstanceChange, messageService, @@ -129,8 +140,7 @@ export class CardResource extends Resource { } = named; this.messageService = messageService; this.cardService = cardService; - this.url = url; - this._loader = loader; + this.url = url?.replace(/\.json$/, ''); this.onCardInstanceChange = onCardInstanceChange; this.cardError = undefined; this.resetLoader = resetLoader; @@ -141,8 +151,8 @@ export class CardResource extends Resource { } registerDestructor(this, () => { - if (this.card) { - this.removeLiveCardEntry(this.card); + if (this.url) { + this.removeLiveCardEntry(this.url); } this.unsubscribeFromRealm(); }); @@ -164,43 +174,37 @@ export class CardResource extends Resource { return this._api; } - private get loader() { - if (!this._loader) { - throw new Error( - `bug: should never get here, loader is obtained via owner`, - ); - } - return this._loader; - } - private loadStaticModel = restartableTask(async (url: URL) => { - let card = await this.getCard(url); - await this.updateCardInstance(card); + let cardOrError = await this.getCard(url); + await this.updateCardInstance(cardOrError); }); private loadLiveModel = restartableTask(async (url: URL) => { - let identityContext = liveCards.get(this.loader); - if (!identityContext) { - identityContext = new LiveCardIdentityContext(); - liveCards.set(this.loader, identityContext); - } - let card = await this.getCard(url, identityContext); - if (!card) { - if (this.cardError) { - console.warn(`cannot load card ${this.cardError.id}`, this.cardError); - } - this.clearCardInstance(); - return; + let cardOrError = await this.getCard(url, liveCardIdentityContext); + if (isCardInstance(cardOrError)) { + let subscribers = liveCardIdentityContext.subscribers(cardOrError.id)!; + subscribers.add(this); + } else { + console.warn(`cannot load card ${cardOrError.id}`, cardOrError); + this.subscribeToRealm(url.href); } - let subscribers = identityContext.subscribers(card.id)!; - subscribers.add(this); - await this.updateCardInstance(card); + await this.updateCardInstance(cardOrError); }); - private subscribeToRealm(card: CardDef) { - let realmURL = card[this.api.realmURL]; + private subscribeToRealm(cardOrId: CardDef | string) { + let card: CardDef | undefined; + let id: string; + let realmURL: URL | undefined; + if (typeof cardOrId === 'string') { + id = cardOrId; + realmURL = this.realm.realmOfURL(new URL(id)); + } else { + card = cardOrId; + id = card.id; + realmURL = card[this.api.realmURL]; + } if (!realmURL) { - throw new Error(`could not determine realm for card ${card.id}`); + throw new Error(`could not determine realm for card ${id}`); } let realmSubscribers = realmSubscriptions.get(realmURL.href); if (!realmSubscribers) { @@ -211,7 +215,6 @@ export class CardResource extends Resource { return; } realmSubscribers.set(this, { - // TODO figure out how to go in an out of errors via SSE unsubscribe: this.messageService.subscribe( realmURL.href, ({ type, data: dataStr }) => { @@ -224,13 +227,21 @@ export class CardResource extends Resource { } let invalidations = data.invalidations as string[]; let card = this.url - ? liveCards.get(this.loader)?.get(this.url) + ? liveCardIdentityContext.get(this.url) : undefined; if (!card) { - // the initial card static load has not actually completed yet - // (perhaps the loader just changed). in this case we ignore this - // message. + if (this.url && liveCardIdentityContext.hasError(this.url)) { + if (invalidations.find((i) => hasExecutableExtension(i))) { + // the invalidation included code changes too. in this case we + // need to flush the loader so that we can pick up any updated + // code before re-running the card + this.resetLoader(); + } + // we've already established a subscription--we're in it, just + // load the updated instance + this.loadStaticModel.perform(new URL(this.url)); + } return; } @@ -253,10 +264,7 @@ export class CardResource extends Resource { }); } - private async getCard( - url: URL, - identityContext?: IdentityContext, - ): Promise { + private async getCard(url: URL, identityContext?: LiveCardIdentityContext) { if (typeof url === 'string') { url = new URL(url); } @@ -266,7 +274,6 @@ export class CardResource extends Resource { if (existingCard) { return existingCard; } - this.cardError = undefined; try { let json = await this.cardService.fetchJSON(url); if (!isSingleCardDocument(json)) { @@ -283,65 +290,28 @@ export class CardResource extends Resource { identityContext, }, ); + if (identityContext && identityContext.hasError(url.href)) { + liveCardIdentityContext.update(url.href, card); + } return card; } catch (error: any) { - let errorResponse: CardError; - try { - errorResponse = JSON.parse(error.responseText) as CardError; - } catch (parseError) { - switch (error.status) { - // tailor HTTP responses as necessary for better user feedback - case 404: - errorResponse = { - errors: [ - { - id: url.href, - status: 404, - title: 'Card Not Found', - message: `The card ${url.href} does not exist`, - realm: error.responseHeaders?.get('X-Boxel-Realm-Url'), - meta: { - lastKnownGoodHtml: null, - scopedCssUrls: [], - stack: null, - }, - }, - ], - }; - break; - default: - errorResponse = { - errors: [ - { - id: url.href, - status: error.status, - title: status.message[error.status] ?? `HTTP ${error.status}`, - message: `Received HTTP ${error.status} from server ${ - error.responseText ?? '' - }`.trim(), - realm: error.responseHeaders?.get('X-Boxel-Realm-Url'), - meta: { - lastKnownGoodHtml: null, - scopedCssUrls: [], - stack: null, - }, - }, - ], - }; - } + if (identityContext) { + liveCardIdentityContext.update(url.href, undefined); } - this.cardError = errorResponse.errors[0]; - return; + let errorResponse = processCardError(url, error); + return errorResponse.errors[0]; } } - // TODO deal with live update of card that goes into and out of an error state private reload = task(async (card: CardDef) => { try { await this.cardService.reloadCard(card); } catch (err: any) { if (err.status !== 404) { - throw err; + liveCardIdentityContext.update(card.id, undefined); + let errorResponse = processCardError(new URL(card.id), err); + this.setCardOrError(errorResponse.errors[0]); + return; } // in this case the document was invalidated in the index because the // file was deleted @@ -361,39 +331,38 @@ export class CardResource extends Resource { } }; - private async updateCardInstance(maybeCard: CardDef | undefined) { - if (maybeCard) { + private async updateCardInstance(maybeCard: CardDef | CardError) { + let instance: CardDef | undefined; + if (isCardInstance(maybeCard)) { + instance = maybeCard; this._api = await apiFor(maybeCard); - } else { - this._api = undefined; } if (this.onCardInstanceChange) { - this.onCardInstanceChange(this._card, maybeCard); - } - if (maybeCard) { - this.subscribeToRealm(maybeCard); + this.onCardInstanceChange(this._card, instance); } + this.subscribeToRealm(maybeCard.id); + this.setCardOrError(maybeCard); + } - // clean up the live card entry if the new card is undefined or if it's - // using a different loader - if ( - this._card && - (!maybeCard || loaderFor(maybeCard) !== loaderFor(this._card)) - ) { - this.removeLiveCardEntry(this._card); + private setCardOrError(cardOrError: CardDef | CardError) { + if (isCardInstance(cardOrError)) { + this._card = cardOrError; + this.staleCard = cardOrError; + this.cardError = undefined; + } else { + this.cardError = cardOrError; + this._card = undefined; + this.staleCard = undefined; } - this._card = maybeCard; - this.staleCard = maybeCard; } - private removeLiveCardEntry(card: CardDef) { - let loader = loaderFor(card); - let subscribers = liveCards.get(loader)?.subscribers(card.id); + private removeLiveCardEntry(id: string) { + let subscribers = liveCardIdentityContext.subscribers(id); if (subscribers && subscribers.has(this)) { subscribers.delete(this); } if (subscribers && subscribers.size === 0) { - liveCards.get(loader)!.delete(card.id); + liveCardIdentityContext.delete(id); } } @@ -428,7 +397,6 @@ export function getCard( onCardInstanceChange: opts?.onCardInstanceChange ? opts.onCardInstanceChange() : undefined, - loader: loaderService.loader, resetLoader: loaderService.reset.bind(loaderService), messageService: (getOwner(parent) as any).lookup( 'service:message-service', @@ -439,3 +407,50 @@ export function getCard( }, })); } + +function processCardError(url: URL, error: any): CardErrors { + try { + let errorResponse = JSON.parse(error.responseText) as CardErrors; + return errorResponse; + } catch (parseError) { + switch (error.status) { + // tailor HTTP responses as necessary for better user feedback + case 404: + return { + errors: [ + { + id: url.href, + status: 404, + title: 'Card Not Found', + message: `The card ${url.href} does not exist`, + realm: error.responseHeaders?.get('X-Boxel-Realm-Url'), + meta: { + lastKnownGoodHtml: null, + scopedCssUrls: [], + stack: null, + }, + }, + ], + }; + default: + return { + errors: [ + { + id: url.href, + status: error.status, + title: status.message[error.status] ?? `HTTP ${error.status}`, + message: `Received HTTP ${error.status} from server ${ + error.responseText ?? '' + }`.trim(), + realm: error.responseHeaders?.get('X-Boxel-Realm-Url'), + meta: { + lastKnownGoodHtml: null, + scopedCssUrls: [], + stack: null, + }, + }, + ], + }; + } + } +} diff --git a/packages/host/app/services/realm.ts b/packages/host/app/services/realm.ts index 217b5075fc..4dd57e8014 100644 --- a/packages/host/app/services/realm.ts +++ b/packages/host/app/services/realm.ts @@ -27,6 +27,7 @@ import { type RealmInfo, type IndexEventData, RealmPermissions, + RealmPaths, } from '@cardstack/runtime-common'; import ENV from '@cardstack/host/config/environment'; @@ -482,6 +483,16 @@ export default class RealmService extends Service { return realmsMeta; } + realmOfURL(url: URL) { + for (const realm of this.realms.keys()) { + let realmURL = new URL(realm); + if (new RealmPaths(realmURL).inRealm(url)) { + return new URL(realmURL); + } + } + return undefined; + } + @cached get defaultWritableRealm(): { path: string; info: RealmInfo } | null { let maybePersonalRealm = `${this.realmServer.url.href}${this.matrixService.userName}/personal/`; diff --git a/packages/host/tests/acceptance/code-submode-test.ts b/packages/host/tests/acceptance/code-submode-test.ts index 83d78a0114..6ff81b2c4e 100644 --- a/packages/host/tests/acceptance/code-submode-test.ts +++ b/packages/host/tests/acceptance/code-submode-test.ts @@ -1527,6 +1527,95 @@ module('Acceptance | code submode tests', function (_hooks) { .includesText('FadhlanXXX'); }); + test('card preview live updates with error', async function (assert) { + let expectedEvents = [ + { + type: 'index', + data: { + type: 'incremental-index-initiation', + realmURL: testRealmURL, + updatedFile: `${testRealmURL}Person/fadhlan`, + }, + }, + { + type: 'index', + data: { + type: 'incremental', + invalidations: [`${testRealmURL}Person/fadhlan`], + }, + }, + ]; + await visitOperatorMode({ + submode: 'code', + codePath: `${testRealmURL}Person/fadhlan.json`, + }); + await waitFor('[data-test-card-resource-loaded]'); + assert + .dom('[data-test-card-error]') + .doesNotExist('card error state is not displayed'); + await this.expectEvents({ + assert, + realm, + expectedEvents, + callback: async () => { + await realm.write( + 'Person/fadhlan.json', + JSON.stringify({ + data: { + type: 'card', + relationships: { + 'friends.0': { + links: { self: './missing' }, + }, + }, + meta: { + adoptsFrom: { + module: '../person', + name: 'Person', + }, + }, + }, + } as LooseSingleCardDocument), + ); + }, + }); + await waitFor('[data-test-card-error]'); + assert + .dom('[data-test-card-error]') + .exists('card error state is displayed'); + + await this.expectEvents({ + assert, + realm, + expectedEvents, + callback: async () => { + await realm.write( + 'Person/fadhlan.json', + JSON.stringify({ + data: { + type: 'card', + relationships: { + 'friends.0': { + links: { self: null }, + }, + }, + meta: { + adoptsFrom: { + module: '../person', + name: 'Person', + }, + }, + }, + } as LooseSingleCardDocument), + ); + }, + }); + await waitFor('[data-test-card-error]', { count: 0 }); + assert + .dom('[data-test-card-error]') + .doesNotExist('card error state is not displayed'); + }); + test('card-catalog does not offer to "create new card" when editing linked fields in code mode', async function (assert) { await visitOperatorMode({ submode: 'code', diff --git a/packages/host/tests/acceptance/interact-submode-test.gts b/packages/host/tests/acceptance/interact-submode-test.gts index 8a6f599b2b..a8aaf89002 100644 --- a/packages/host/tests/acceptance/interact-submode-test.gts +++ b/packages/host/tests/acceptance/interact-submode-test.gts @@ -1692,6 +1692,111 @@ module('Acceptance | interact submode tests', function (hooks) { .dom('[data-test-operator-mode-stack="0"] [data-test-person]') .hasText('FadhlanXXX'); }); + + test('stack item live updates with error', async function (assert) { + assert.expect(7); + let expectedEvents = [ + { + type: 'index', + data: { + type: 'incremental-index-initiation', + realmURL: testRealmURL, + updatedFile: `${testRealmURL}Person/fadhlan`, + }, + }, + { + type: 'index', + data: { + type: 'incremental', + invalidations: [`${testRealmURL}Person/fadhlan`], + }, + }, + ]; + await visitOperatorMode({ + stacks: [ + [ + { + id: `${testRealmURL}Person/fadhlan`, + format: 'isolated', + }, + ], + ], + }); + assert + .dom(`[data-test-stack-card="${testRealmURL}Person/fadhlan"]`) + .exists('card is displayed'); + assert + .dom( + `[data-test-stack-card="${testRealmURL}Person/fadhlan"] [data-test-card-error]`, + ) + .doesNotExist('card error state is NOT displayed'); + + await this.expectEvents({ + assert, + realm, + expectedEvents, + callback: async () => { + await realm.write( + 'Person/fadhlan.json', + JSON.stringify({ + data: { + type: 'card', + relationships: { + pet: { + links: { self: './missing' }, + }, + }, + meta: { + adoptsFrom: { + module: '../person', + name: 'Person', + }, + }, + }, + } as LooseSingleCardDocument), + ); + }, + }); + + assert + .dom( + `[data-test-stack-card="${testRealmURL}Person/fadhlan"] [data-test-card-error]`, + ) + .exists('card error state is displayed'); + + await this.expectEvents({ + assert, + realm, + expectedEvents, + callback: async () => { + await realm.write( + 'Person/fadhlan.json', + JSON.stringify({ + data: { + type: 'card', + relationships: { + pet: { links: { self: null } }, + }, + meta: { + adoptsFrom: { + module: '../person', + name: 'Person', + }, + }, + }, + } as LooseSingleCardDocument), + ); + }, + }); + assert + .dom(`[data-test-stack-card="${testRealmURL}Person/fadhlan"]`) + .exists('card is displayed'); + assert + .dom( + `[data-test-stack-card="${testRealmURL}Person/fadhlan"] [data-test-card-error]`, + ) + .doesNotExist('card error state is NOT displayed'); + }); }); module('workspace index card', function () { From 2b74586bd0133c2f2d263c2d690eed4827427ce0 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Mon, 9 Dec 2024 15:02:41 -0500 Subject: [PATCH 16/67] fix issue where 404 errors don't have an "id" --- packages/host/app/resources/card-resource.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/host/app/resources/card-resource.ts b/packages/host/app/resources/card-resource.ts index b538fe72e2..fe0167db6f 100644 --- a/packages/host/app/resources/card-resource.ts +++ b/packages/host/app/resources/card-resource.ts @@ -35,7 +35,7 @@ import type RealmService from '../services/realm'; interface CardErrors { errors: { - id: string; + id?: string; // 404 errors won't necessarily have an id status: number; title: string; message: string; @@ -340,7 +340,9 @@ export class CardResource extends Resource { if (this.onCardInstanceChange) { this.onCardInstanceChange(this._card, instance); } - this.subscribeToRealm(maybeCard.id); + if (maybeCard.id) { + this.subscribeToRealm(maybeCard.id); + } this.setCardOrError(maybeCard); } From 2f60609bd7fbd82438e3fd96f02ba5031ba73cf4 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Mon, 9 Dec 2024 15:46:46 -0500 Subject: [PATCH 17/67] reset module scoped identity context state for SSE tests --- packages/host/app/resources/card-resource.ts | 9 +++++++-- packages/host/tests/helpers/index.gts | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/host/app/resources/card-resource.ts b/packages/host/app/resources/card-resource.ts index fe0167db6f..5798816bdc 100644 --- a/packages/host/app/resources/card-resource.ts +++ b/packages/host/app/resources/card-resource.ts @@ -101,12 +101,16 @@ class LiveCardIdentityContext implements IdentityContext { } } -const liveCardIdentityContext = new LiveCardIdentityContext(); +let liveCardIdentityContext = new LiveCardIdentityContext(); const realmSubscriptions: Map< string, WeakMap void }> > = new Map(); +export function testOnlyResetLiveCardIdentityContext() { + liveCardIdentityContext = new LiveCardIdentityContext(); +} + export class CardResource extends Resource { url: string | undefined; @tracked loaded: Promise | undefined; @@ -181,6 +185,7 @@ export class CardResource extends Resource { private loadLiveModel = restartableTask(async (url: URL) => { let cardOrError = await this.getCard(url, liveCardIdentityContext); + await this.updateCardInstance(cardOrError); if (isCardInstance(cardOrError)) { let subscribers = liveCardIdentityContext.subscribers(cardOrError.id)!; subscribers.add(this); @@ -188,7 +193,6 @@ export class CardResource extends Resource { console.warn(`cannot load card ${cardOrError.id}`, cardOrError); this.subscribeToRealm(url.href); } - await this.updateCardInstance(cardOrError); }); private subscribeToRealm(cardOrId: CardDef | string) { @@ -306,6 +310,7 @@ export class CardResource extends Resource { private reload = task(async (card: CardDef) => { try { await this.cardService.reloadCard(card); + this.setCardOrError(card); } catch (err: any) { if (err.status !== 404) { liveCardIdentityContext.update(card.id, undefined); diff --git a/packages/host/tests/helpers/index.gts b/packages/host/tests/helpers/index.gts index 3ef7de4e01..d7f00d0fcc 100644 --- a/packages/host/tests/helpers/index.gts +++ b/packages/host/tests/helpers/index.gts @@ -40,6 +40,8 @@ import CardPrerender from '@cardstack/host/components/card-prerender'; import ENV from '@cardstack/host/config/environment'; import SQLiteAdapter from '@cardstack/host/lib/sqlite-adapter'; +import { testOnlyResetLiveCardIdentityContext } from '@cardstack/host/resources/card-resource'; + import type CardService from '@cardstack/host/services/card-service'; import type { CardSaveSubscriber } from '@cardstack/host/services/card-service'; @@ -293,6 +295,7 @@ export function setupServerSentEvents(hooks: NestedHooks) { hooks.beforeEach(function () { this.subscribers = []; let self = this; + testOnlyResetLiveCardIdentityContext(); class MockMessageService extends Service { register() { From 05ef56c406e6e17c4c013fa658a074631006f68d Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Mon, 9 Dec 2024 15:49:10 -0500 Subject: [PATCH 18/67] more test state cleanup --- packages/host/app/resources/card-resource.ts | 5 +++-- packages/host/tests/helpers/index.gts | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/host/app/resources/card-resource.ts b/packages/host/app/resources/card-resource.ts index 5798816bdc..3b9d59b0d4 100644 --- a/packages/host/app/resources/card-resource.ts +++ b/packages/host/app/resources/card-resource.ts @@ -102,13 +102,14 @@ class LiveCardIdentityContext implements IdentityContext { } let liveCardIdentityContext = new LiveCardIdentityContext(); -const realmSubscriptions: Map< +let realmSubscriptions: Map< string, WeakMap void }> > = new Map(); -export function testOnlyResetLiveCardIdentityContext() { +export function testOnlyResetCardResourceModuleState() { liveCardIdentityContext = new LiveCardIdentityContext(); + realmSubscriptions = new Map(); } export class CardResource extends Resource { diff --git a/packages/host/tests/helpers/index.gts b/packages/host/tests/helpers/index.gts index d7f00d0fcc..4dae4763ae 100644 --- a/packages/host/tests/helpers/index.gts +++ b/packages/host/tests/helpers/index.gts @@ -40,7 +40,7 @@ import CardPrerender from '@cardstack/host/components/card-prerender'; import ENV from '@cardstack/host/config/environment'; import SQLiteAdapter from '@cardstack/host/lib/sqlite-adapter'; -import { testOnlyResetLiveCardIdentityContext } from '@cardstack/host/resources/card-resource'; +import { testOnlyResetCardResourceModuleState } from '@cardstack/host/resources/card-resource'; import type CardService from '@cardstack/host/services/card-service'; import type { CardSaveSubscriber } from '@cardstack/host/services/card-service'; @@ -295,7 +295,7 @@ export function setupServerSentEvents(hooks: NestedHooks) { hooks.beforeEach(function () { this.subscribers = []; let self = this; - testOnlyResetLiveCardIdentityContext(); + testOnlyResetCardResourceModuleState(); class MockMessageService extends Service { register() { From 5f05d28c71cec762f39cdfe2ae5d29d016e0b28d Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Mon, 9 Dec 2024 15:53:16 -0500 Subject: [PATCH 19/67] better naming --- packages/host/app/resources/card-resource.ts | 2 +- packages/host/tests/helpers/index.gts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/host/app/resources/card-resource.ts b/packages/host/app/resources/card-resource.ts index 3b9d59b0d4..934b3e4eed 100644 --- a/packages/host/app/resources/card-resource.ts +++ b/packages/host/app/resources/card-resource.ts @@ -107,7 +107,7 @@ let realmSubscriptions: Map< WeakMap void }> > = new Map(); -export function testOnlyResetCardResourceModuleState() { +export function testOnlyResetLiveCardState() { liveCardIdentityContext = new LiveCardIdentityContext(); realmSubscriptions = new Map(); } diff --git a/packages/host/tests/helpers/index.gts b/packages/host/tests/helpers/index.gts index 4dae4763ae..c4c41fa264 100644 --- a/packages/host/tests/helpers/index.gts +++ b/packages/host/tests/helpers/index.gts @@ -40,7 +40,7 @@ import CardPrerender from '@cardstack/host/components/card-prerender'; import ENV from '@cardstack/host/config/environment'; import SQLiteAdapter from '@cardstack/host/lib/sqlite-adapter'; -import { testOnlyResetCardResourceModuleState } from '@cardstack/host/resources/card-resource'; +import { testOnlyResetLiveCardState } from '@cardstack/host/resources/card-resource'; import type CardService from '@cardstack/host/services/card-service'; import type { CardSaveSubscriber } from '@cardstack/host/services/card-service'; @@ -295,7 +295,7 @@ export function setupServerSentEvents(hooks: NestedHooks) { hooks.beforeEach(function () { this.subscribers = []; let self = this; - testOnlyResetCardResourceModuleState(); + testOnlyResetLiveCardState(); class MockMessageService extends Service { register() { From 05333831c3a686ce41f86a09652695e73faf0404 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 9 Dec 2024 15:41:26 -0500 Subject: [PATCH 20/67] Serialize matrix room mutations to avoid network race conditions with swift actions like enabling/disabling skills. - Add ember test waiter for mutex --- packages/host/app/components/matrix/room.gts | 20 ++- packages/host/app/lib/matrix-classes/room.ts | 4 + packages/host/app/lib/mutex.ts | 43 +++++++ packages/host/app/services/matrix-service.ts | 116 ++++++++++-------- .../components/ai-assistant-panel-test.gts | 5 +- 5 files changed, 126 insertions(+), 62 deletions(-) create mode 100644 packages/host/app/lib/mutex.ts diff --git a/packages/host/app/components/matrix/room.gts b/packages/host/app/components/matrix/room.gts index dffea9b10e..ef3a055832 100644 --- a/packages/host/app/components/matrix/room.gts +++ b/packages/host/app/components/matrix/room.gts @@ -7,7 +7,6 @@ import Component from '@glimmer/component'; import { tracked, cached } from '@glimmer/tracking'; import { enqueueTask, restartableTask, timeout, all } from 'ember-concurrency'; -import perform from 'ember-concurrency/helpers/perform'; import max from 'lodash/max'; @@ -102,7 +101,7 @@ export default class Room extends Component { class='skills' @skills={{this.sortedSkills}} @onChooseCard={{this.attachSkill}} - @onUpdateSkillIsActive={{perform this.updateSkillIsActiveTask}} + @onUpdateSkillIsActive={{this.updateSkillIsActiveTask}} data-test-skill-menu /> {{/if}} @@ -586,16 +585,13 @@ export default class Room extends Component { return this.autoAttachmentResource.cards; } - // using enqueue task on this ensures that updates don't step on each other - private updateSkillIsActiveTask = enqueueTask( - async (skillEventId: string, isActive: boolean) => { - await this.matrixService.updateSkillIsActive( - this.args.roomId, - skillEventId, - isActive, - ); - }, - ); + updateSkillIsActiveTask = async (skillEventId: string, isActive: boolean) => { + await this.matrixService.updateSkillIsActive( + this.args.roomId, + skillEventId, + isActive, + ); + }; private get canSend() { return ( diff --git a/packages/host/app/lib/matrix-classes/room.ts b/packages/host/app/lib/matrix-classes/room.ts index 01aad561bc..1a6aaf9eb8 100644 --- a/packages/host/app/lib/matrix-classes/room.ts +++ b/packages/host/app/lib/matrix-classes/room.ts @@ -5,6 +5,8 @@ import { type IEvent } from 'matrix-js-sdk'; import type { MatrixEvent as DiscreteMatrixEvent } from 'https://cardstack.com/base/matrix-event'; +import Mutex from '../mutex'; + import type * as MatrixSDK from 'matrix-js-sdk'; export type TempEvent = Partial & { @@ -25,6 +27,8 @@ export default class Room { disabledEventIds: [], }; + readonly mutex = new Mutex(); + get events() { return this._events; } diff --git a/packages/host/app/lib/mutex.ts b/packages/host/app/lib/mutex.ts new file mode 100644 index 0000000000..5b2cf82505 --- /dev/null +++ b/packages/host/app/lib/mutex.ts @@ -0,0 +1,43 @@ +import { buildWaiter } from '@ember/test-waiters'; + +/* Usage example: + +// Usage example +const mutex = new Mutex(); + +async function criticalSection() { + await mutex.dispatch(async () => { + // Your critical section code here + console.log('Critical section'); + }); +} +*/ + +const waiter = buildWaiter('mutex:waiter'); + +export default class Mutex { + private mutex = Promise.resolve(); + + private async lock(): Promise<() => void> { + let begin: (unlock: () => void) => void = (_unlock) => {}; + + this.mutex = this.mutex.then(() => { + return new Promise(begin); + }); + + return new Promise((res) => { + begin = res; + }); + } + + async dispatch(fn: (() => T) | (() => Promise)): Promise { + const token = waiter.beginAsync(); + const unlock = await this.lock(); + try { + return await fn(); + } finally { + unlock(); + waiter.endAsync(token); + } + } +} diff --git a/packages/host/app/services/matrix-service.ts b/packages/host/app/services/matrix-service.ts index ad99cdc7be..47b4e6ac19 100644 --- a/packages/host/app/services/matrix-service.ts +++ b/packages/host/app/services/matrix-service.ts @@ -450,10 +450,18 @@ export default class MatrixService extends Service { }`, ), }); + let roomData = this.ensureRoomData(roomId); invites.map((i) => { let fullId = i.startsWith('@') ? i : `@${i}:${userId!.split(':')[1]}`; if (i === aiBotUsername) { - this.client.setPowerLevel(roomId, fullId, AI_BOT_POWER_LEVEL, null); + roomData.mutex.dispatch(async () => { + return this.client.setPowerLevel( + roomId, + fullId, + AI_BOT_POWER_LEVEL, + null, + ); + }); } }); this.addSkillCardsToRoom(roomId, await this.loadDefaultSkills()); @@ -469,15 +477,18 @@ export default class MatrixService extends Service { | ReactionEventContent | CommandResultContent, ) { - if ('data' in content) { - const encodedContent = { - ...content, - data: JSON.stringify(content.data), - }; - return await this.client.sendEvent(roomId, eventType, encodedContent); - } else { - return await this.client.sendEvent(roomId, eventType, content); - } + let roomData = this.ensureRoomData(roomId); + return roomData.mutex.dispatch(async () => { + if ('data' in content) { + const encodedContent = { + ...content, + data: JSON.stringify(content.data), + }; + return await this.client.sendEvent(roomId, eventType, encodedContent); + } else { + return await this.client.sendEvent(roomId, eventType, content); + } + }); } async sendReactionEvent(roomId: string, eventId: string, status: string) { @@ -489,7 +500,7 @@ export default class MatrixService extends Service { }, }; try { - return await this.client.sendEvent(roomId, 'm.reaction', content); + return await this.sendEvent(roomId, 'm.reaction', content); } catch (e) { throw new Error( `Error sending reaction event: ${ @@ -646,14 +657,19 @@ export default class MatrixService extends Service { throw e; } } - this.client.sendStateEvent(roomId, SKILLS_STATE_EVENT_TYPE, { - enabledEventIds: [ - ...new Set([ - ...(skillEventIdsStateEvent?.enabledEventIds || []), - ...attachedSkillEventIds, - ]), - ], - disabledEventIds: [...(skillEventIdsStateEvent?.disabledEventIds || [])], + let roomData = this.ensureRoomData(roomId); + await roomData.mutex.dispatch(async () => { + await this.client.sendStateEvent(roomId, SKILLS_STATE_EVENT_TYPE, { + enabledEventIds: [ + ...new Set([ + ...(skillEventIdsStateEvent?.enabledEventIds || []), + ...attachedSkillEventIds, + ]), + ], + disabledEventIds: [ + ...(skillEventIdsStateEvent?.disabledEventIds || []), + ], + }); }); } @@ -662,30 +678,32 @@ export default class MatrixService extends Service { skillEventId: string, isActive: boolean, ) => { - let currentSkillsConfig = await this.client.getStateEvent( - roomId, - SKILLS_STATE_EVENT_TYPE, - '', - ); - let newSkillsConfig = { - enabledEventIds: [...(currentSkillsConfig.enabledEventIds || [])], - disabledEventIds: [...(currentSkillsConfig.disabledEventIds || [])], - }; - if (isActive) { - newSkillsConfig.enabledEventIds.push(skillEventId); - newSkillsConfig.disabledEventIds = - newSkillsConfig.disabledEventIds.filter((id) => id !== skillEventId); - } else { - newSkillsConfig.disabledEventIds.push(skillEventId); - newSkillsConfig.enabledEventIds = newSkillsConfig.enabledEventIds.filter( - (id) => id !== skillEventId, + let roomData = this.ensureRoomData(roomId); + await roomData.mutex.dispatch(async () => { + let currentSkillsConfig = await this.client.getStateEvent( + roomId, + SKILLS_STATE_EVENT_TYPE, + '', ); - } - await this.client.sendStateEvent( - roomId, - SKILLS_STATE_EVENT_TYPE, - newSkillsConfig, - ); + let newSkillsConfig = { + enabledEventIds: [...(currentSkillsConfig.enabledEventIds || [])], + disabledEventIds: [...(currentSkillsConfig.disabledEventIds || [])], + }; + if (isActive) { + newSkillsConfig.enabledEventIds.push(skillEventId); + newSkillsConfig.disabledEventIds = + newSkillsConfig.disabledEventIds.filter((id) => id !== skillEventId); + } else { + newSkillsConfig.disabledEventIds.push(skillEventId); + newSkillsConfig.enabledEventIds = + newSkillsConfig.enabledEventIds.filter((id) => id !== skillEventId); + } + await this.client.sendStateEvent( + roomId, + SKILLS_STATE_EVENT_TYPE, + newSkillsConfig, + ); + }); }; public async sendAiAssistantMessage(params: { @@ -949,13 +967,17 @@ export default class MatrixService extends Service { `bug: roomId is undefined for event ${JSON.stringify(event, null, 2)}`, ); } + let roomData = this.ensureRoomData(roomId); + roomData.addEvent(event, oldEventId); + } + + private ensureRoomData(roomId: string) { let roomData = this.getRoomData(roomId); if (!roomData) { roomData = new Room(); this.setRoomData(roomId, roomData); } - - roomData.addEvent(event, oldEventId); + return roomData; } private onMembership = (event: MatrixEvent, member: RoomMember) => { @@ -1071,11 +1093,7 @@ export default class MatrixService extends Service { } roomStates = Array.from(roomStateMap.values()); for (let rs of roomStates) { - let roomData = this.getRoomData(rs.roomId); - if (!roomData) { - roomData = new Room(); - this.setRoomData(rs.roomId, roomData); - } + let roomData = this.ensureRoomData(rs.roomId); let name = rs.events.get('m.room.name')?.get('')?.event.content?.name; if (name) { roomData.updateName(name); diff --git a/packages/host/tests/integration/components/ai-assistant-panel-test.gts b/packages/host/tests/integration/components/ai-assistant-panel-test.gts index b5d1572d0a..690e0005b5 100644 --- a/packages/host/tests/integration/components/ai-assistant-panel-test.gts +++ b/packages/host/tests/integration/components/ai-assistant-panel-test.gts @@ -1184,9 +1184,12 @@ module('Integration | ai-assistant-panel', function (hooks) { assert.dom('[data-test-ai-bot-retry-button]').exists(); await percySnapshot(assert); - await click('[data-test-ai-bot-retry-button]'); + click('[data-test-ai-bot-retry-button]'); + await waitFor('[data-test-ai-assistant-message].is-pending'); assert.dom('[data-test-ai-assistant-message]').exists({ count: 1 }); assert.dom('[data-test-ai-assistant-message]').hasNoClass('is-error'); + await settled(); + assert.dom('[data-test-ai-assistant-message]').hasClass('is-error'); }); test('it does not display the streaming indicator when ai bot sends an option', async function (assert) { From 57502ed0ba9c4718c9d5196888ec826da0cb27ac Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Mon, 9 Dec 2024 16:15:57 -0500 Subject: [PATCH 21/67] fix tests --- packages/host/app/resources/card-resource.ts | 5 ++++- .../integration/components/ai-assistant-panel-test.gts | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/host/app/resources/card-resource.ts b/packages/host/app/resources/card-resource.ts index 934b3e4eed..87de89bae3 100644 --- a/packages/host/app/resources/card-resource.ts +++ b/packages/host/app/resources/card-resource.ts @@ -209,7 +209,10 @@ export class CardResource extends Resource { realmURL = card[this.api.realmURL]; } if (!realmURL) { - throw new Error(`could not determine realm for card ${id}`); + console.warn( + `could not determine realm for card ${id} when trying to subscribe to realm`, + ); + return; } let realmSubscribers = realmSubscriptions.get(realmURL.href); if (!realmSubscribers) { diff --git a/packages/host/tests/integration/components/ai-assistant-panel-test.gts b/packages/host/tests/integration/components/ai-assistant-panel-test.gts index b5d1572d0a..5ecd023d74 100644 --- a/packages/host/tests/integration/components/ai-assistant-panel-test.gts +++ b/packages/host/tests/integration/components/ai-assistant-panel-test.gts @@ -2172,8 +2172,8 @@ module('Integration | ai-assistant-panel', function (hooks) { }); test('it can copy search results card to workspace', async function (assert) { - const id = `${testRealmURL}Person/fadhlan.json`; - const roomId = await renderAiAssistantPanel(id); + const id = `${testRealmURL}Person/fadhlan`; + const roomId = await renderAiAssistantPanel(`${id}.json`); const toolArgs = { description: 'Search for Person cards', attributes: { @@ -2247,8 +2247,8 @@ module('Integration | ai-assistant-panel', function (hooks) { }); test('it can copy search results card to workspace (no cards in stack)', async function (assert) { - const id = `${testRealmURL}Person/fadhlan.json`; - const roomId = await renderAiAssistantPanel(id); + const id = `${testRealmURL}Person/fadhlan`; + const roomId = await renderAiAssistantPanel(`${id}.json`); const toolArgs = { description: 'Search for Person cards', attributes: { From c5ba3466b4c3810941bbc4dcf86df3b1d1d61beb Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Mon, 9 Dec 2024 17:17:54 -0500 Subject: [PATCH 22/67] fix issue where card implementation changes as part of live load --- packages/host/app/resources/card-resource.ts | 28 +++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/packages/host/app/resources/card-resource.ts b/packages/host/app/resources/card-resource.ts index 87de89bae3..f50ad43e3b 100644 --- a/packages/host/app/resources/card-resource.ts +++ b/packages/host/app/resources/card-resource.ts @@ -85,13 +85,23 @@ class LiveCardIdentityContext implements IdentityContext { delete(url: string): void { this.#cards.delete(url); } - update(url: string, instance: CardDef | undefined) { + update( + url: string, + instance: CardDef | undefined, + subscribers?: Set, + ) { let entry = this.#cards.get(url); if (!entry) { - this.#cards.set(url, { card: instance, subscribers: new Set() }); + entry = { card: instance, subscribers: new Set() }; + this.#cards.set(url, entry); } else { entry.card = instance; } + if (subscribers) { + for (let subscriber of subscribers) { + entry.subscribers.add(subscriber); + } + } } hasError(url: string) { return this.#cards.has(url) && !this.#cards.get(url)?.card; @@ -261,10 +271,20 @@ export class CardResource extends Resource { if (invalidations.find((i) => hasExecutableExtension(i))) { // the invalidation included code changes too. in this case we // need to flush the loader so that we can pick up any updated - // code before re-running the card + // code before re-running the card as well as clear out the + // identity context as the card has a new implementation this.resetLoader(); + let subscribers = liveCardIdentityContext.subscribers(card.id); + liveCardIdentityContext.delete(card.id); + this.loadStaticModel.perform(new URL(card.id)); + liveCardIdentityContext.update( + card.id, + this._card, + subscribers, + ); + } else { + this.reload.perform(card); } - this.reload.perform(card); } } }, From 1061d159b15b6d8c25e2276b77b48156197629bc Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Mon, 9 Dec 2024 16:22:04 -0600 Subject: [PATCH 23/67] host: Remove stack trace from Percy snapshots (#1900) --- packages/host/app/components/operator-mode/card-error-detail.gts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/host/app/components/operator-mode/card-error-detail.gts b/packages/host/app/components/operator-mode/card-error-detail.gts index 6106e068b7..d5d9f35b3c 100644 --- a/packages/host/app/components/operator-mode/card-error-detail.gts +++ b/packages/host/app/components/operator-mode/card-error-detail.gts @@ -78,6 +78,7 @@ export default class CardErrorDetail extends Component {
Stack trace:
 {{@error.meta.stack}}
From 4fd99ea013226277f26fe61391f9d924029676b8 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Mon, 9 Dec 2024 16:54:53 -0600 Subject: [PATCH 24/67] tools: Add publishing to Open VSX (#1904) --- .github/workflows/manual-vscode-boxel-tools.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual-vscode-boxel-tools.yml b/.github/workflows/manual-vscode-boxel-tools.yml index 61df8cb269..cdfb00fb54 100644 --- a/.github/workflows/manual-vscode-boxel-tools.yml +++ b/.github/workflows/manual-vscode-boxel-tools.yml @@ -94,7 +94,7 @@ jobs: - name: Package run: pnpm vscode:package working-directory: packages/vscode-boxel-tools - - name: Publish + - name: Publish to Visual Studio Marketplace run: | if [ "${{ inputs.environment }}" = "production" ]; then pnpm vscode:publish @@ -104,3 +104,13 @@ jobs: working-directory: packages/vscode-boxel-tools env: VSCE_PAT: ${{ secrets.VSCE_PAT }} + - name: Publish to Open VSX + run: | + if [ "${{ inputs.environment }}" = "production" ]; then + npx ovsx publish --no-dependencies --pat $OVSX_TOKEN + else + npx ovsx publish --no-dependencies --pre-release --pat $OVSX_TOKEN + fi + working-directory: packages/vscode-boxel-tools + env: + OVSX_TOKEN: ${{ secrets.OVSX_TOKEN }} From 72d4bf38665ae768cef229b89d96ebc37f0960fa Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Mon, 9 Dec 2024 17:08:45 -0600 Subject: [PATCH 25/67] tools: Update to version 0.1.0 (#1908) --- packages/vscode-boxel-tools/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vscode-boxel-tools/package.json b/packages/vscode-boxel-tools/package.json index 8f25f3cc10..97b457b98b 100644 --- a/packages/vscode-boxel-tools/package.json +++ b/packages/vscode-boxel-tools/package.json @@ -2,7 +2,7 @@ "name": "boxel-tools", "displayName": "Boxel Tools", "description": "Access boxel realm data and code from your vscode workspace.", - "version": "0.0.23", + "version": "0.1.0", "publisher": "cardstack", "private": true, "license": "MIT", From 71e682aad1c77bd1b7afc031708b658b53ea35c1 Mon Sep 17 00:00:00 2001 From: tintinthong Date: Tue, 10 Dec 2024 09:39:57 +0800 Subject: [PATCH 26/67] Create account card (#1888) * setup account card * cleanup comments in contact * wip * account wip * remove check for model.id * apply entitydisplay to all the possible field & apply atomformat to contact card * fix lint * fix social links * fix action not in decorator link for phone --------- Co-authored-by: lucaslyl --- .../addon/src/components/avatar/index.gts | 4 +- .../addon/src/components/pill/index.gts | 11 +- .../deterministic-color-from-string.ts | 2 +- .../77976251-6bc8-4a8c-972f-f9d588e8434d.json | 16 +- .../bf437dc3-1f96-45e5-a057-3487c6a5c2f7.json | 41 +- .../a01fc5c9-d70d-4b9c-aae4-384cf2b79b25.json | 19 +- .../0e5aec99-798b-4417-9426-a338432e0ee5.json | 8 +- .../7db76dab-19af-4069-9850-878fd54cfc2a.json | 43 -- .../4fdd3053-14f1-4800-827f-d3e2a5c44ae8.json | 2 +- .../de720f57-964b-4a09-8d52-80cd8bb4b739.json | 13 + .../1dbcc3e8-fe3c-4c4a-ba66-ff7d637a7358.json | 8 +- .../9d7b2f20-c7da-4ddc-8e77-b75d88c97b48.json | 8 +- .../components/avatar-group.gts | 6 +- .../components/entity-display.gts | 43 ++ .../components/status-pill.gts | 16 +- packages/experiments-realm/crm-app.gts | 4 +- packages/experiments-realm/crm/account.gts | 167 +++++++- packages/experiments-realm/crm/contact.gts | 366 ++++++------------ packages/experiments-realm/email.gts | 16 +- .../experiments_fields_preview.gts | 6 + packages/experiments-realm/phone.gts | 120 ++++++ packages/experiments-realm/website.gts | 21 + 22 files changed, 584 insertions(+), 356 deletions(-) delete mode 100644 packages/experiments-realm/Customer/7db76dab-19af-4069-9850-878fd54cfc2a.json create mode 100644 packages/experiments-realm/components/entity-display.gts create mode 100644 packages/experiments-realm/phone.gts create mode 100644 packages/experiments-realm/website.gts diff --git a/packages/boxel-ui/addon/src/components/avatar/index.gts b/packages/boxel-ui/addon/src/components/avatar/index.gts index ec1fc3d0c7..098da568d6 100644 --- a/packages/boxel-ui/addon/src/components/avatar/index.gts +++ b/packages/boxel-ui/addon/src/components/avatar/index.gts @@ -10,7 +10,7 @@ interface Signature { displayName?: string; isReady: boolean; thumbnailURL?: string; - userId: string; + userId?: string | null; }; Element: HTMLDivElement; } @@ -73,7 +73,7 @@ export default class Avatar extends Component { } let name = this.args.displayName?.length ? this.args.displayName - : this.args.userId.replace(/^@/, ''); + : this.args.userId?.replace(/^@/, '') ?? ''; return name.slice(0, 1).toUpperCase(); } } diff --git a/packages/boxel-ui/addon/src/components/pill/index.gts b/packages/boxel-ui/addon/src/components/pill/index.gts index 462bac0d8e..9139def271 100644 --- a/packages/boxel-ui/addon/src/components/pill/index.gts +++ b/packages/boxel-ui/addon/src/components/pill/index.gts @@ -49,15 +49,18 @@ const Pill: TemplateOnlyComponent = +} diff --git a/packages/experiments-realm/components/status-pill.gts b/packages/experiments-realm/components/status-pill.gts index 7c22ce035d..00f11c1e8c 100644 --- a/packages/experiments-realm/components/status-pill.gts +++ b/packages/experiments-realm/components/status-pill.gts @@ -26,14 +26,15 @@ export class StatusPill extends GlimmerComponent { <:iconLeft> - + > + + <:default> @@ -44,6 +45,15 @@ export class StatusPill extends GlimmerComponent { + + }; +} + +interface ContactRowArgs { + Args: { + userID: string; + name: string; + thumbnailURL: string; + isPrimary: boolean; + }; + Blocks: {}; + Element: HTMLElement; +} + +class ContactRow extends GlimmerComponent { + +} class IsolatedTemplate extends Component { //Mock Data: @@ -20,6 +106,17 @@ class IsolatedTemplate extends Component { return 'TechNova Solutions'; } + get hasCompanyInfo() { + return this.args.model.website || this.args.model.address?.country?.name; + } + + get hasContacts() { + return ( + this.args.model.primaryContact?.name || + (this.args.model.contacts?.length ?? 0) > 0 //contacts is a proxy array + ); + } + } export class Account extends CardDef { - static displayName = 'Account'; + static displayName = 'CRM Account'; @field company = linksTo(Company); @field primaryContact = linksTo(Contact); @field contacts = linksToMany(Contact); - @field deals = linksToMany(Deal); + @field address = contains(LocationField); + @field shippingAddress = contains(AddressField); + @field billingAddress = contains(AddressField); + @field website = contains(WebsiteField); + + @field title = contains(StringField, { + computeVia: function (this: Account) { + return this.company?.name; + }, + }); static isolated = IsolatedTemplate; } diff --git a/packages/experiments-realm/crm/contact.gts b/packages/experiments-realm/crm/contact.gts index 3791862f11..0033869716 100644 --- a/packages/experiments-realm/crm/contact.gts +++ b/packages/experiments-realm/crm/contact.gts @@ -1,6 +1,8 @@ import StringField from 'https://cardstack.com/base/string'; import NumberField from 'https://cardstack.com/base/number'; - +import { PhoneField } from '../phone'; +import { EmailField } from '../email'; +import { ContactLinkField } from '../fields/contact-link'; import { Component, CardDef, @@ -8,13 +10,12 @@ import { field, contains, linksTo, + containsMany, } from 'https://cardstack.com/base/card-api'; import { tracked } from '@glimmer/tracking'; import { action } from '@ember/object'; import { fn } from '@ember/helper'; -import { RadioInput, Pill } from '@cardstack/boxel-ui/components'; -import MailIcon from '@cardstack/boxel-icons/mail'; -import PhoneIcon from '@cardstack/boxel-icons/phone'; +import { RadioInput } from '@cardstack/boxel-ui/components'; import HeartHandshakeIcon from '@cardstack/boxel-icons/heart-handshake'; import TargetArrowIcon from '@cardstack/boxel-icons/target-arrow'; import AvatarGroup from '../components/avatar-group'; @@ -22,6 +23,35 @@ import { Company } from './company'; import { Avatar } from '@cardstack/boxel-ui/components'; import { StatusPill } from '../components/status-pill'; import type IconComponent from '@cardstack/boxel-icons/captions'; +import ContactIcon from '@cardstack/boxel-icons/contact'; +import Email from '@cardstack/boxel-icons/mail'; +import Linkedin from '@cardstack/boxel-icons/linkedin'; +import XIcon from '@cardstack/boxel-icons/brand-x'; + +export class SocialLinkField extends ContactLinkField { + static displayName = 'social-link'; + + static values = [ + { + type: 'social', + label: 'X', + icon: XIcon, + cta: 'Follow', + }, + { + type: 'social', + label: 'LinkedIn', + icon: Linkedin, + cta: 'Connect', + }, + { + type: 'email', + label: 'Email', + icon: Email, + cta: 'Contact', + }, + ]; +} const getStatusData = ( label: string | undefined, @@ -29,16 +59,6 @@ const getStatusData = ( return StatusField.values.find((status) => status.label === label); }; -const formatPhone = (phone: any) => { - if (!phone) return undefined; - return `+${phone.country} (${phone.area}) ${phone.phoneNumber}`; -}; - -const formatEmail = (email: string) => { - if (!email) return undefined; - return email; -}; - export interface LooseyGooseyData { index: number; label: string; @@ -118,72 +138,28 @@ export class StatusField extends LooseGooseyField { }; } -export class PhoneField extends FieldDef { - static displayName = 'phoneMobile'; - @field country = contains(NumberField); - @field area = contains(NumberField); - @field phoneNumber = contains(NumberField); - - static embedded = class Embedded extends Component { - - }; -} - class EmbeddedTemplate extends Component { } class FittedTemplate extends Component { + get hasSocialLinks() { + return ( + this.args.model.socialLinks && this.args.model.socialLinks.length > 0 + ); + }