refactor setup script (#13323)

* refactor setup script

* debugg

* move dependency step after setup script

* temporarly use profile services

* remove profile services

* remove , from gitea api

* add debugg

* more debugg logs

* more logs

* try fix

* degubb

* extended gitea-api, made it possible to configure hostname

* removed debugg console.log

* wait for healthy container

* This should be the final commit

* added health

* fix healthchecks

---------

Co-authored-by: davidovrelid.com <david@framit.no>

.github/workflows/run-playwright-on-pr.yaml

@@ -31,9 +31,6 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: 'Installing Dependencies'
-        uses: ./.github/actions/yarn-install
-
       - name: Generate .env file
         run: |
           echo PLAYWRIGHT_TEST_APP=autodeploy-v3 >> .env
@@ -55,6 +52,9 @@ jobs:
         run: |
           node ./development/setup.js
 
+      - name: 'Installing Dependencies'
+        uses: ./.github/actions/yarn-install
+
       - name: Playwright run
         working-directory: frontend/testing/playwright
         env:

Dockerfile

@@ -56,7 +56,7 @@ EXPOSE 80
 WORKDIR /app
 ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=false \
   DOTNET_RUNNING_IN_CONTAINER=true
-RUN apk add --no-cache icu-libs krb5-libs libgcc libintl openssl libstdc++ zlib
+RUN apk add --no-cache icu-libs krb5-libs libgcc libintl openssl libstdc++ zlib curl
 
 COPY --from=generate-studio-backend /app_output .
 COPY --from=generate-studio-frontend /build/frontend/dist/app-development ./wwwroot/designer/frontend/app-development

compose.yaml

@@ -40,6 +40,8 @@ services:
     depends_on:
       studio_repositories:
         condition: service_healthy
+      studio_designer:
+        condition: service_healthy
     extra_hosts:
       - 'host.docker.internal:host-gateway'
     volumes:
@@ -83,21 +85,22 @@ services:
       - OidcLoginSettings:Authority=http://studio.localhost/repos/
       - OidcLoginSettings:RequireHttpsMetadata=false
       - OidcLoginSettings:CookieExpiryTimeInMinutes=59
+      - OidcLoginSettings:FetchClientIdAndSecretFromRootEnvFile=false
     ports:
       - '6000:6000'
     depends_on:
-        studio_repositories:
-          condition: service_healthy
+      studio_db:
+        condition: service_healthy
     build:
       context: .
     extra_hosts:
       - 'host.docker.internal:host-gateway'
       - 'studio.localhost:host-gateway'
     healthcheck:
-      test: [ "CMD", "curl", "--fail", "http://localhost:6000/" ]
-      interval: 30s
+      test: [ "CMD", "curl", "--fail", "http://localhost:6000/health" ]
+      interval: 5s
       timeout: 10s
-      retries: 3
+      retries: 5
 
   studio_repositories:
     container_name: studio-repositories
@@ -136,11 +139,14 @@ services:
       context: ./gitea/
     extra_hosts:
       - 'host.docker.internal:host-gateway'
+    depends_on:
+      studio_db:
+        condition: service_healthy
     healthcheck:
-          test: [ "CMD", "curl", "--fail", "http://localhost:3000" ]
-          interval: 30s
-          timeout: 10s
-          retries: 3
+      test: [ "CMD", "curl", "--fail", "http://localhost:3000/api/healthz" ]
+      interval: 5s
+      timeout: 10s
+      retries: 5
 
 
   studio_db:
@@ -158,7 +164,7 @@ services:
       - ./development/db/data.sql:/data.sql
       - pgdata:/var/lib/postgresql/data
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U designer_admin -d designerdb"]
+      test: [ "CMD-SHELL", "pg_isready -U designer_admin -d designerdb" ]
       interval: 10s
       timeout: 5s
       retries: 5
@@ -167,8 +173,7 @@ services:
     container_name: db-database_migrations
     depends_on:
       studio_db:
-          condition: service_healthy
-
+        condition: service_healthy
     build:
       context: backend
       dockerfile: Migrations.Dockerfile
@@ -178,4 +183,3 @@ services:
       - PGUSER=designer_admin
       - PGPASSWORD=${POSTGRES_PASSWORD}
       - PGDATABASE=designerdb
-

development/setup.js

@@ -6,10 +6,12 @@ const dnsIsOk = require('./utils/check-if-dns-is-correct.js');
 const createCypressEnvFile = require('./utils/create-cypress-env-file.js');
 const path = require('path');
 const writeEnvFile = require('./utils/write-env-file.js');
+const waitForHealthy = require('./utils/wait-for-healthy.js');
 
 const startingDockerCompose = () => runCommand('docker compose up -d --remove-orphans --build');
-const restartComposeServices = () =>
-  runCommand('docker compose down && docker compose up -d --remove-orphans');
+const buildAndStartComposeService = (service) =>
+  runCommand(`docker compose up -d ${service} --build`);
+const stopComposeService = (service) => runCommand(`docker compose down ${service}`);
 
 const createUser = (username, password, admin) =>
   runCommand(
@@ -23,19 +25,9 @@ const createUser = (username, password, admin) =>
     ].join(' '),
   );
 
-const ensureUserPassword = (username, password) =>
-  runCommand(
-    [
-      `docker exec studio-repositories gitea admin user change-password`,
-      `--username ${username}`,
-      `--password ${password}`,
-      `--must-change-password=false`,
-    ].join(' '),
-  );
-
 const createTestDepOrg = (env) =>
   giteaApi({
-    path: '/repos/api/v1/orgs',
+    path: '/api/v1/orgs',
     method: 'POST',
     user: env.GITEA_ADMIN_USER,
     pass: env.GITEA_ADMIN_PASS,
@@ -49,7 +41,7 @@ const createTestDepTeams = async (env) => {
   const allTeams = require(path.resolve(__dirname, 'data', 'gitea-teams.json'));
 
   const existingTeams = await giteaApi({
-    path: `/repos/api/v1/orgs/${env.GITEA_ORG_USER}/teams`,
+    path: `/api/v1/orgs/${env.GITEA_ORG_USER}/teams`,
     method: 'GET',
     user: env.GITEA_ADMIN_USER,
     pass: env.GITEA_ADMIN_PASS,
@@ -59,7 +51,7 @@ const createTestDepTeams = async (env) => {
     const existing = existingTeams.find((t) => t.name === team.name);
     if (!existing) {
       await giteaApi({
-        path: `/repos/api/v1/orgs/${env.GITEA_ORG_USER}/teams`,
+        path: `/api/v1/orgs/${env.GITEA_ORG_USER}/teams`,
         method: 'POST',
         user: env.GITEA_ADMIN_USER,
         pass: env.GITEA_ADMIN_PASS,
@@ -76,19 +68,19 @@ const createTestDepTeams = async (env) => {
 
 const createOidcClientIfNotExists = async (env) => {
   const clients = await giteaApi({
-    path: `/repos/api/v1/user/applications/oauth2`,
+    path: `/api/v1/user/applications/oauth2`,
     method: 'GET',
     user: env.GITEA_ADMIN_USER,
     pass: env.GITEA_ADMIN_PASS,
   });
 
   const shouldCreateClient = !clients.some((app) => app.name === 'LocalTestOidcClient');
   if (!shouldCreateClient) {
-    return;
+    return null;
   }
 
   var createdClient = await giteaApi({
-    path: `/repos/api/v1/user/applications/oauth2`,
+    path: `/api/v1/user/applications/oauth2`,
     method: 'POST',
     user: env.GITEA_ADMIN_USER,
     pass: env.GITEA_ADMIN_PASS,
@@ -102,19 +94,17 @@ const createOidcClientIfNotExists = async (env) => {
   env.CLIENT_ID = createdClient.client_id;
   env.CLIENT_SECRET = createdClient.client_secret;
 
-  writeEnvFile(env);
-  // reload designer with new clientid and secret
-  restartComposeServices();
-  await waitFor('http://studio.localhost/repos/');
+  return env;
 };
 
 const addUserToSomeTestDepTeams = async (env) => {
   const teams = await giteaApi({
-    path: `/repos/api/v1/orgs/${env.GITEA_ORG_USER}/teams`,
+    path: `/api/v1/orgs/${env.GITEA_ORG_USER}/teams`,
     method: 'GET',
     user: env.GITEA_ADMIN_USER,
     pass: env.GITEA_ADMIN_PASS,
   });
+
   for (const teamName of [
     'Owners',
     'Deploy-TT02',
@@ -134,8 +124,9 @@ const addUserToSomeTestDepTeams = async (env) => {
     'AccessLists-TT02',
   ]) {
     const existing = teams.find((t) => t.name === teamName);
+
     await giteaApi({
-      path: `/repos/api/v1/teams/${existing.id}/members/${env.GITEA_ADMIN_USER}`,
+      path: `/api/v1/teams/${existing.id}/members/${env.GITEA_ADMIN_USER}`,
       method: 'PUT',
       user: env.GITEA_ADMIN_USER,
       pass: env.GITEA_ADMIN_PASS,
@@ -160,8 +151,9 @@ const addUserToSomeTestDepTeams = async (env) => {
     'AccessLists-TT02',
   ]) {
     const existing = teams.find((t) => t.name === teamName);
+
     await giteaApi({
-      path: `/repos/api/v1/teams/${existing.id}/members/${env.GITEA_CYPRESS_USER}`,
+      path: `/api/v1/teams/${existing.id}/members/${env.GITEA_CYPRESS_USER}`,
       method: 'PUT',
       user: env.GITEA_ADMIN_USER,
       pass: env.GITEA_ADMIN_PASS,
@@ -178,25 +170,47 @@ const addReleaseAndDeployTestDataToDb = async () =>
     ].join(' '),
   );
 
+const setupEnvironment = async (env) => {
+  buildAndStartComposeService('studio_db');
+  buildAndStartComposeService('studio_repositories');
+  await waitForHealthy('studio-repositories');
+
+  createUser(env.GITEA_ADMIN_USER, env.GITEA_ADMIN_PASS, true);
+  createUser(env.GITEA_CYPRESS_USER, env.GITEA_CYPRESS_PASS, false);
+  await createTestDepOrg(env);
+  await createTestDepTeams(env);
+  await addUserToSomeTestDepTeams(env);
+  const result = await createOidcClientIfNotExists(env);
+
+  await createCypressEnvFile(env);
+
+  stopComposeService('studio_db');
+  stopComposeService('studio_repositories');
+  return result;
+};
+
 const script = async () => {
   const env = ensureDotEnv();
   await dnsIsOk('studio.localhost');
   if (!(env.IGNORE_DOCKER_DNS_LOOKUP === 'true')) {
     await dnsIsOk('host.docker.internal');
   }
-  await startingDockerCompose();
-  await waitFor('http://studio.localhost/repos/');
-  await createUser(env.GITEA_ADMIN_USER, env.GITEA_ADMIN_PASS, true);
-  await ensureUserPassword(env.GITEA_ADMIN_USER, env.GITEA_ADMIN_PASS);
-  await createUser(env.GITEA_CYPRESS_USER, env.GITEA_CYPRESS_PASS, false);
-  await ensureUserPassword(env.GITEA_CYPRESS_USER, env.GITEA_CYPRESS_PASS);
-  await createTestDepOrg(env);
-  await createTestDepTeams(env);
-  await addUserToSomeTestDepTeams(env);
-  await createOidcClientIfNotExists(env);
-  await createCypressEnvFile(env);
+
+  const result = await setupEnvironment(env);
+  if (result) {
+    writeEnvFile(result);
+  }
+
+  startingDockerCompose();
+  await waitFor('http://studio.localhost', 120);
+
   await addReleaseAndDeployTestDataToDb();
   process.exit(0);
 };
 
-script().then().catch(console.error);
+script()
+  .then()
+  .catch((error) => {
+    console.error(error);
+    process.exit(1);
+  });

development/utils/create-cypress-env-file.js

@@ -7,7 +7,7 @@ module.exports = async (env) => {
   const tokenPrefix = 'setup.js';
 
   const allTokens = await giteaApi({
-    path: `/repos/api/v1/users/${env.GITEA_ADMIN_USER}/tokens`,
+    path: `/api/v1/users/${env.GITEA_ADMIN_USER}/tokens`,
     method: 'GET',
     user: env.GITEA_ADMIN_USER,
     pass: env.GITEA_ADMIN_PASS,
@@ -17,22 +17,32 @@ module.exports = async (env) => {
     if (token.name.startsWith(tokenPrefix)) {
       deleteTokenOperations.push(
         giteaApi({
-          path: `/repos/api/v1/users/${env.GITEA_ADMIN_USER}/tokens/${token.id}`,
+          path: `/api/v1/users/${env.GITEA_ADMIN_USER}/tokens/${token.id}`,
           method: 'DELETE',
           user: env.GITEA_ADMIN_USER,
           pass: env.GITEA_ADMIN_PASS,
-        })
+        }),
       );
     }
   });
   const result = await giteaApi({
-    path: `/repos/api/v1/users/${env.GITEA_ADMIN_USER}/tokens`,
+    path: `/api/v1/users/${env.GITEA_ADMIN_USER}/tokens`,
     method: 'POST',
     user: env.GITEA_ADMIN_USER,
     pass: env.GITEA_ADMIN_PASS,
     body: {
       name: tokenPrefix + ' ' + Date.now(),
-      scopes: ['write:activitypub', 'write:admin', 'write:issue', 'write:misc', 'write:notification', 'write:organization', 'write:package', 'write:repository', 'write:user'],
+      scopes: [
+        'write:activitypub',
+        'write:admin',
+        'write:issue',
+        'write:misc',
+        'write:notification',
+        'write:organization',
+        'write:package',
+        'write:repository',
+        'write:user',
+      ],
     },
   });
   const envFile = {
@@ -53,7 +63,7 @@ module.exports = async (env) => {
     'frontend',
     'testing',
     'cypress',
-    'cypress.env.json'
+    'cypress.env.json',
   );
   fs.writeFileSync(cypressEnvFilePath, JSON.stringify(envFile, null, 2) + os.EOL, 'utf-8');
   console.log('Wrote a new:', cypressEnvFilePath);

development/utils/gitea-api.js

@@ -6,33 +6,24 @@ const { request } = require('http');
  * @param options
  * @returns {Promise<unknown>}
  */
-module.exports = (options) =>
-  new Promise(function (resolve, reject) {
-    const req = request(
-      {
-        host: 'studio.localhost',
-        path: options.path,
-        auth: [options.user, options.pass].join(':'),
-        method: options.method,
-        headers: {
-          'Content-Type': 'application/json',
-        },
+module.exports = async (options) => {
+  try {
+    const response = await fetch(`${options.hostname || 'http://localhost:3000'}${options.path}`, {
+      method: options.method,
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Basic ${Buffer.from(`${options.user}:${options.pass}`).toString('base64')}`,
       },
-      (response) => {
-        const data = [];
-        response.on('data', (chunk) => data.push(chunk));
-        response.on('end', () => {
-          console.log(options.method, options.path, response.statusCode, response.statusMessage);
-          if (data.length) {
-            resolve(JSON.parse(data.join()));
-          } else {
-            resolve();
-          }
-        });
-      }
-    );
-    if (options.body) {
-      req.write(JSON.stringify(options.body));
-    }
-    req.end(() => {});
-  });
+      body: options.body ? JSON.stringify(options.body) : undefined,
+    });
+
+    const data = await response.json().catch((err) => {
+      console.error('Error:', err);
+    });
+    console.log(options.method, options.path, response.status, response.statusText);
+    return data;
+  } catch (error) {
+    console.error('Error:', error);
+    throw error;
+  }
+};