Retry user creation when role is not ready (#7096)

Co-authored-by: Harold Shen <hlshen@google.com>
firebase · May 9, 2024 · 2361e0d · 2361e0d
1 parent d30d96c
commit 2361e0d
Showing 1 changed file with 44 additions and 22 deletions.
diff --git a/src/gcp/cloudsql/cloudsqladmin.ts b/src/gcp/cloudsql/cloudsqladmin.ts
@@ -162,28 +162,50 @@ export async function createUser(
   username: string,
   password?: string,
 ): Promise<User> {
-  const op = await client.post<User, Operation>(
-    `projects/${projectId}/instances/${instanceId}/users`,
-    {
-      name: username,
-      instance: instanceId,
-      project: projectId,
-      password: password,
-      sqlserverUserDetails: {
-        disabled: false,
-        serverRoles: ["cloudsqlsuperuser"],
-      },
-      type,
-    },
-  );
-  const opName = `projects/${projectId}/operations/${op.body.name}`;
-  const pollRes = await operationPoller.pollOperation<User>({
-    apiOrigin: cloudSQLAdminOrigin(),
-    apiVersion: API_VERSION,
-    operationResourceName: opName,
-    doneFn: (op: Operation) => op.status === "DONE",
-  });
-  return pollRes;
+  const maxRetries = 3;
+  let retries = 0;
+  while (true) {
+    try {
+      const op = await client.post<User, Operation>(
+        `projects/${projectId}/instances/${instanceId}/users`,
+        {
+          name: username,
+          instance: instanceId,
+          project: projectId,
+          password: password,
+          sqlserverUserDetails: {
+            disabled: false,
+            serverRoles: ["cloudsqlsuperuser"],
+          },
+          type,
+        },
+      );
+      const opName = `projects/${projectId}/operations/${op.body.name}`;
+      const pollRes = await operationPoller.pollOperation<User>({
+        apiOrigin: cloudSQLAdminOrigin(),
+        apiVersion: API_VERSION,
+        operationResourceName: opName,
+        doneFn: (op: Operation) => op.status === "DONE",
+      });
+      return pollRes;
+    } catch (err: any) {
+      if (builtinRoleNotReady(err.message) && retries < maxRetries) {
+        retries++;
+        await new Promise((resolve) => {
+          setTimeout(resolve, 1000 * retries);
+        });
+      } else {
+        throw err;
+      }
+    }
+  }
+}
+
+// CloudSQL built in roles get created _after_ the operation is complete.
+// This means that we occasionally bump into cases where we try to create the user
+// before the role required for IAM users exists.
+function builtinRoleNotReady(message: string): boolean {
+  return message.includes("cloudsqliamuser");
 }
 
 export async function getUser(