maintenance/update cli tooling

.envrc

@@ -0,0 +1 @@
+dotenv

clients/dotnet-core-rest/src/Tika.RestClient/Features/Acls/Models/Acl.cs

@@ -2,7 +2,7 @@ namespace Tika.RestClient.Features.Acls.Models
 {
     public class Acl
     {
-        public long UserId { get; set; }
+        public string UserId { get; set; }
         public string ServiceAccountId { get; set; }
         public string Permission { get; set; }
         public string Resource { get; set; }

clients/dotnet-core-rest/src/Tika.RestClient/Features/Acls/Models/AclCreateDelete.cs

@@ -2,15 +2,15 @@ namespace Tika.RestClient.Features.Acls.Models
 {
     public class AclCreateDelete
     {
-        public long ServiceAccountId { get; set; }
+        public string ServiceAccountId { get; set; }
         public bool Allow { get; set; }
         public string Operation { get; set; }
         public string TopicPrefix { get; set; }
         public string ConsumerGroupPrefix { get; set; }
 
         public AclCreateDelete() {}
 
-        public AclCreateDelete(long serviceAccountId, bool allow, string operation, string topicPrefix = "", string consumerGroupPrefix = "")
+        public AclCreateDelete(string serviceAccountId, bool allow, string operation, string topicPrefix = "", string consumerGroupPrefix = "")
         {
             ServiceAccountId = serviceAccountId;
             Allow = allow;

server/Dockerfile

@@ -22,14 +22,13 @@ RUN apk add --update \
     curl expect \
     && rm -rf /var/cache/apk/*
 
-# Install ccloud cli tool
-RUN curl -L https://s3-us-west-2.amazonaws.com/confluent.cloud/ccloud-cli/install.sh | sh -s -- -b /ccloud/bin
+# Install confluent cli tool
+RUN apk --no-cache add ca-certificates bash curl
 
-ENV TIKA_CCLOUD_BIN_PATH="/ccloud/bin/ccloud"
-ENV PATH "$PATH:/ccloud/bin"
-
-RUN ccloud version
+ENV CONFLUENT_CLI_VERSION="v2.12.0"
+RUN curl -sL --http1.1 https://cnfl.io/cli | sh -s -- -b /usr/local/bin $CONFLUENT_CLI_VERSION
 
+RUN confluent version
 
 # Copy app & supporting scripts
 COPY --from=Builder /app/dist/main.js /app/main.js

server/login.sh

@@ -1,15 +1,4 @@
-#!/usr/bin/expect
-spawn ccloud login
-
-expect "Email: "
-
-send -- "$env(TIKA_CC_USER)\r"
-
-expect "Password: "
-
-send -- "$env(TIKA_CC_PASS)\r"
-
-set timeout 600
-expect eof
-
-send_user "Confluent cloud login successful\n"
+#!/bin/ash
+export CONFLUENT_CLOUD_EMAIL="$TIKA_CC_USER"
+export CONFLUENT_CLOUD_PASSWORD="$TIKA_CC_PASS"
+confluent login --save

server/makefile

@@ -5,7 +5,7 @@ build:
 	docker build -t $(IMAGE_NAME) .
 
 run:
-	docker run -it -p 3000:3000 --rm $(IMAGE_NAME)
+	docker run -it -p 3000:3000 --rm --env-file ../.env $(IMAGE_NAME)
 
 release: build
 	chmod +x ../scripts/push_container_image.sh && ../scripts/push_container_image.sh $(IMAGE_NAME) $(BUILD_NUMBER)

server/src/server/api/api-keys.ts

@@ -7,7 +7,7 @@ export class ApiKeysInterface {
 
             try {
                 let apiKey = await apiKeys.createApiKey(
-                    parseInt(req.body.serviceAccountId),
+                    req.body.serviceAccountId,
                     req.body.description
                 );
                 res.json(apiKey);

server/src/server/wrapper/connected/CCloudTopics.ts

@@ -2,26 +2,47 @@ import { parse, parseTopicDescription } from "./../parser";
 import { executeCli } from "./executeCli";
 import { TopicAlreadyExistsException } from "../model/error";
 import { GetConfig } from "../../config";
+import { Deserializer, ConcatOutput } from "../utils";
+import { ListTopics, DescribeTopic } from "../model/topics";
+
 
 export class CcloudTopics implements Topics {
 
     async getTopics(): Promise<string[]> {
         let config = GetConfig();
 
-        let result = await executeCli(["kafka", "topic", "list", "--cluster", config.clusterId, "--environment", config.environmentId]);
-        result =
-            parse(result)
-                .filter(t => t.Name.startsWith("_confluent") === false)
-                .map(t => t.Name);
-
-        return result;
+        let result = await executeCli(["kafka", "topic", "list", "--cluster", config.clusterId, "--environment", config.environmentId, "--output", "json"]);
+
+        let combinedResult = ConcatOutput(result);
+        let deserializedResult : ListTopics;
+        try {
+            deserializedResult = Deserializer<ListTopics>(combinedResult);
+        } catch (error) {
+            return error;
+        }
+
+        return deserializedResult
+            .filter(t => t.name.startsWith("_confluent") === false)
+            .map(t => t.name);
     }
 
     async describeTopic(name: string): Promise<Topic> {
         let config = GetConfig();
-        let consoleLines = await executeCli(["kafka", "topic", "describe", name, "--cluster", config.clusterId, "--environment", config.environmentId]);
-
-        var topic = parseTopicDescription(consoleLines);
+        let result = await executeCli(["kafka", "topic", "describe", name, "--cluster", config.clusterId, "--environment", config.environmentId, "--output", "json"]);
+
+        let combinedResult = ConcatOutput(result);
+        let deserializedResult : DescribeTopic;
+        try {
+            deserializedResult = Deserializer<DescribeTopic>(combinedResult);
+        } catch (error) {
+            return error;
+        }
+
+        let topic = {
+            Name: deserializedResult.topic_name,
+            PartitionCount: deserializedResult.config["num.partitions"], // might be an issue that partitionCount is present twice
+            Configurations: deserializedResult.config // might be an issue that partitionCount is present twice
+        };
 
         return topic;
     }

server/src/server/wrapper/connected/CcloudAccessControlLists.ts

@@ -1,19 +1,35 @@
 import { parse } from "./../parser";
 import { executeCli } from "./executeCli";
 import { GetConfig } from "../../config";
+import { Deserializer, ConcatOutput } from "../utils";
+import { ListAcls } from "../model/acls";
 
 export class CcloudAccessControlLists implements AccessControlLists {
     async getAccessControlLists(): Promise<AccessControlList[]> {
         let config = GetConfig();
-        let result = await executeCli(["kafka", "acl", "list", "--cluster", config.clusterId, "--environment", config.environmentId]);
-        let resultObjects = parse(result) as AccessControlList[];
+        let result = await executeCli(["kafka", "acl", "list", "--cluster", config.clusterId, "--environment", config.environmentId, "--output", "json"]);
 
-        resultObjects.forEach(elem => {
-            elem.UserId = elem.UserId.split(':')[1];
-        });
-
-        return resultObjects;
-    }
+        let combinedResult = ConcatOutput(result);
+        let deserializedResult : ListAcls;
+        try {
+            deserializedResult = Deserializer<ListAcls>(combinedResult);
+        } catch (error) {
+            return error;
+        }
+
+        return deserializedResult.map(t => {
+          let obj = {
+            UserId: "", // removed property by Confluent
+            ServiceAccountId: t.principal.split(":")[1],
+            Permission: t.permission,
+            Operation: t.operation,
+            Resource: t.resource_type,
+            Name: t.resource_name,
+            Type: t.pattern_type
+          };
+          return obj;
+        })      
+      }
 
     async createAccessControlList(
         serviceAccountId: number,

server/src/server/wrapper/connected/CcloudApiKeys.ts

@@ -1,40 +1,65 @@
 import { parse, parseSideColumns } from "./../parser";
 import {executeCli } from "./executeCli";
 import { GetConfig } from "../../config";
+import { Deserializer, ConcatOutput } from "../utils";
+import { CreateApiKey, ListApiKeys } from "../model/api-keys";
 
 export class CcloudApiKeys implements ApiKeys {
     ccloud: CCloudCliWrapper;
 
-    async createApiKey(serviceAccountId: number, description: string): Promise<ApiKeySet> {
+    async createApiKey(serviceAccountId: string, description: string): Promise<ApiKeySet> {
       let config = GetConfig();
 
       let cliOutput = await executeCli([
         "api-key",
         "create",
         "--resource", config.clusterId,
         "--environment", config.environmentId,
-        "--service-account", serviceAccountId + "",
-        "--description", description]
+        "--service-account", serviceAccountId,
+        "--description", description, "--output", "json"]
       );
 
       let cliObjects: any = parseSideColumns(cliOutput);
       let apiKeySet: ApiKeySet = { Key: cliObjects.APIKey, Secret: cliObjects.Secret }
+
+      let combinedResult = ConcatOutput(cliOutput);
+      let deserializedResult : CreateApiKey;
+      try {
+          deserializedResult = Deserializer<CreateApiKey>(combinedResult);
+      } catch (error) {
+          return error;
+      }
 
-      return apiKeySet;
+      return {
+        Key: deserializedResult.key,
+        Secret: deserializedResult.secret
+      };
     }
 
     async deleteApiKey(key: string): Promise<void> {
       await executeCli(["api-key", "delete", key]);
     }
+
     async getApiKeys(): Promise<ApiKey[]> {
-      let cliOutput = await executeCli(["api-key", "list"]);
-      let cliObjects = parse(cliOutput);
+      let cliOutput = await executeCli(["api-key", "list", "--output", "json"]);
 
-      let apiKeys = cliObjects.map(function (obj) {
-        return { Key: obj.Key, Description: obj.Description, Owner: obj.Owner, Resource: obj.ResourceID } as ApiKey
-      });
-
-      return apiKeys;
+      let combinedResult = ConcatOutput(cliOutput);
+      let deserializedResult : ListApiKeys;
+      try {
+          deserializedResult = Deserializer<ListApiKeys>(combinedResult);
+      } catch (error) {
+          return error;
+      }
+
+      return deserializedResult.map(t => {
+        let obj = {
+          Key: t.key,
+          Description: t.description,
+          Owner: t.owner_resource_id,
+          Resource: t.resource_id
+        };
+        return obj;
+      })      
     }
 
     constructor(ccloud: CCloudCliWrapper) {

server/src/server/wrapper/connected/CcloudCluster.ts

@@ -1,19 +1,26 @@
 import { parse, parseSideColumns } from "./../parser";
 import {executeCli } from "./executeCli";
 import { GetConfig } from "../../config";
+import { Deserializer, ConcatOutput } from "../utils";
+import Clusters from "../model/clusters";
 
 export class CcloudCluster {
     ccloud: CCloudCliWrapper;
 
     async list(): Promise<any[]> {
       let config = GetConfig();
 
-      let result = await executeCli(["kafka", "cluster", "list", "--environment", config.environmentId]);
-      parse(result);
-      console.log("\n::SEP::\n");
-      console.log(result);
+      let result = await executeCli(["kafka", "cluster", "list", "--environment", config.environmentId, "--output", "json"]);
+
+      let combinedResult = ConcatOutput(result);
+      let deserializedResult : Clusters;
+      try {
+        deserializedResult = Deserializer<Clusters>(combinedResult);
+      } catch (error) {
+        return error;
+      }
 
-      return result;
+      return deserializedResult;
     }
 
     constructor(ccloud: CCloudCliWrapper) {

server/src/server/wrapper/connected/CcloudServiceAccount.ts

@@ -1,28 +1,43 @@
 import { parse, parseSideColumns } from "./../parser";
 import { executeCli } from "./executeCli";
 import { CliException, ServiceAccountAlreadyExistsException } from "./../model/error";
+import { Deserializer, ConcatOutput } from "../utils";
+import { ListServiceAccounts, ListServiceAccount } from "../model/service-account";
 
 export class CcloudServiceAccount implements ServiceAccounts {
   ccloud: CCloudCliWrapper;
 
   async getServiceAccounts(): Promise<ServiceAccount[]> {
-    let result = await executeCli(["service-account", "list"]);
-    result = parse(result);
+    let result = await executeCli(["iam", "service-account", "list", "--output", "json"]);
+    let combinedResult = ConcatOutput(result);
+    let deserializedResult : ListServiceAccounts;
+    try {
+        deserializedResult = Deserializer<ListServiceAccounts>(combinedResult);
+    } catch (error) {
+        return error;
+    }
 
-    return (result as any) as ServiceAccount[];
+    return deserializedResult.map(t => {
+      let obj = {
+        Name: t.name,
+        Id: t.id,
+        Description: t.description
+      };
+      return obj;
+    })
   }
 
   async createServiceAccount(accountName: string, description: string = ""): Promise<ServiceAccount> {
     let cliResult;
     try {
-      cliResult = await executeCli(["service-account", "create", accountName, "--description", description]);
+      cliResult = await executeCli(["iam", "service-account", "create", accountName, "--description", description, "--output", "json"]);
     }
     catch (error) {
       if (error.name.valueOf() !== "CliException") {
         throw (error);
       }
 
-      if (error.consoleLines.some((l: string): boolean => l.includes("Service name is already in use"))) {
+      if (error.consoleLines.some((l: string): boolean => l.includes("is already in use"))) {
         let existingServicesAccounts = await this.getServiceAccounts();
 
         let existingServicesAccount = existingServicesAccounts.find(s => s.Name === accountName);
@@ -39,15 +54,23 @@ export class CcloudServiceAccount implements ServiceAccounts {
       throw (error);
     }
 
+    let combinedResult = ConcatOutput(cliResult);
+    let deserializedResult : ListServiceAccount;
+    try {
+        deserializedResult = Deserializer<ListServiceAccount>(combinedResult);
+    } catch (error) {
+        return error;
+    }
 
-
-    let result = parseSideColumns(cliResult);
-
-    return (result as any) as ServiceAccount;
+    return {
+      Name: deserializedResult.name,
+      Id: deserializedResult.id,
+      Description: deserializedResult.description
+    };
   }
 
   async deleteServiceAccount(accountId: number): Promise<boolean> {
-    await executeCli(["service-account", "delete", accountId.toString()]);
+    await executeCli(["iam", "service-account", "delete", accountId.toString()]);
     return true;
   }
 

server/src/server/wrapper/connected/executeCli.ts

@@ -3,7 +3,7 @@ import * as readline from "readline";
 import { CcloudSessionExpiredException, CliException } from "./../model/error";
 
 export function executeCli(args: string[]): Promise<string[]> {
-  const cli = process.env.TIKA_CCLOUD_BIN_PATH ?? "ccloud";
+  const cli = process.env.TIKA_CCLOUD_BIN_PATH ?? "confluent";
 
   return new Promise((resolve, reject) => {
     const lines: Array<string> = [];
@@ -21,7 +21,7 @@ export function executeCli(args: string[]): Promise<string[]> {
         return resolve(lines);
       }
 
-      if (errLines.some((l: string): boolean => l.includes("You must log in to run that command."))) {
+      if (errLines.some((l: string): boolean => l.includes("You must be logged in to run this command"))) {
         return reject(new CcloudSessionExpiredException());
       }