diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..40448e6 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +dotenv \ No newline at end of file diff --git a/clients/dotnet-core-rest/src/Tika.RestClient/Features/Acls/Models/Acl.cs b/clients/dotnet-core-rest/src/Tika.RestClient/Features/Acls/Models/Acl.cs index a286372..e0db727 100644 --- a/clients/dotnet-core-rest/src/Tika.RestClient/Features/Acls/Models/Acl.cs +++ b/clients/dotnet-core-rest/src/Tika.RestClient/Features/Acls/Models/Acl.cs @@ -2,7 +2,7 @@ namespace Tika.RestClient.Features.Acls.Models { public class Acl { - public long UserId { get; set; } + public string UserId { get; set; } public string ServiceAccountId { get; set; } public string Permission { get; set; } public string Resource { get; set; } diff --git a/clients/dotnet-core-rest/src/Tika.RestClient/Features/Acls/Models/AclCreateDelete.cs b/clients/dotnet-core-rest/src/Tika.RestClient/Features/Acls/Models/AclCreateDelete.cs index 3f6ec6d..f506880 100644 --- a/clients/dotnet-core-rest/src/Tika.RestClient/Features/Acls/Models/AclCreateDelete.cs +++ b/clients/dotnet-core-rest/src/Tika.RestClient/Features/Acls/Models/AclCreateDelete.cs @@ -2,7 +2,7 @@ namespace Tika.RestClient.Features.Acls.Models { public class AclCreateDelete { - public long ServiceAccountId { get; set; } + public string ServiceAccountId { get; set; } public bool Allow { get; set; } public string Operation { get; set; } public string TopicPrefix { get; set; } @@ -10,7 +10,7 @@ public class AclCreateDelete public AclCreateDelete() {} - public AclCreateDelete(long serviceAccountId, bool allow, string operation, string topicPrefix = "", string consumerGroupPrefix = "") + public AclCreateDelete(string serviceAccountId, bool allow, string operation, string topicPrefix = "", string consumerGroupPrefix = "") { ServiceAccountId = serviceAccountId; Allow = allow; diff --git a/server/Dockerfile b/server/Dockerfile index 3e50d3e..13e61a1 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -22,14 +22,13 @@ RUN apk add --update \ curl expect \ && rm -rf /var/cache/apk/* -# Install ccloud cli tool -RUN curl -L https://s3-us-west-2.amazonaws.com/confluent.cloud/ccloud-cli/install.sh | sh -s -- -b /ccloud/bin +# Install confluent cli tool +RUN apk --no-cache add ca-certificates bash curl -ENV TIKA_CCLOUD_BIN_PATH="/ccloud/bin/ccloud" -ENV PATH "$PATH:/ccloud/bin" - -RUN ccloud version +ENV CONFLUENT_CLI_VERSION="v2.12.0" +RUN curl -sL --http1.1 https://cnfl.io/cli | sh -s -- -b /usr/local/bin $CONFLUENT_CLI_VERSION +RUN confluent version # Copy app & supporting scripts COPY --from=Builder /app/dist/main.js /app/main.js diff --git a/server/login.sh b/server/login.sh index 3cabf89..4cecb5a 100755 --- a/server/login.sh +++ b/server/login.sh @@ -1,15 +1,4 @@ -#!/usr/bin/expect -spawn ccloud login - -expect "Email: " - -send -- "$env(TIKA_CC_USER)\r" - -expect "Password: " - -send -- "$env(TIKA_CC_PASS)\r" - -set timeout 600 -expect eof - -send_user "Confluent cloud login successful\n" \ No newline at end of file +#!/bin/ash +export CONFLUENT_CLOUD_EMAIL="$TIKA_CC_USER" +export CONFLUENT_CLOUD_PASSWORD="$TIKA_CC_PASS" +confluent login --save diff --git a/server/makefile b/server/makefile index 387b9c6..f988745 100644 --- a/server/makefile +++ b/server/makefile @@ -5,7 +5,7 @@ build: docker build -t $(IMAGE_NAME) . run: - docker run -it -p 3000:3000 --rm $(IMAGE_NAME) + docker run -it -p 3000:3000 --rm --env-file ../.env $(IMAGE_NAME) release: build chmod +x ../scripts/push_container_image.sh && ../scripts/push_container_image.sh $(IMAGE_NAME) $(BUILD_NUMBER) diff --git a/server/src/server/api/api-keys.ts b/server/src/server/api/api-keys.ts index ced0de9..fb82be7 100644 --- a/server/src/server/api/api-keys.ts +++ b/server/src/server/api/api-keys.ts @@ -7,7 +7,7 @@ export class ApiKeysInterface { try { let apiKey = await apiKeys.createApiKey( - parseInt(req.body.serviceAccountId), + req.body.serviceAccountId, req.body.description ); res.json(apiKey); diff --git a/server/src/server/wrapper/connected/CCloudTopics.ts b/server/src/server/wrapper/connected/CCloudTopics.ts index e5c2c35..fa7a13b 100644 --- a/server/src/server/wrapper/connected/CCloudTopics.ts +++ b/server/src/server/wrapper/connected/CCloudTopics.ts @@ -2,26 +2,47 @@ import { parse, parseTopicDescription } from "./../parser"; import { executeCli } from "./executeCli"; import { TopicAlreadyExistsException } from "../model/error"; import { GetConfig } from "../../config"; +import { Deserializer, ConcatOutput } from "../utils"; +import { ListTopics, DescribeTopic } from "../model/topics"; + export class CcloudTopics implements Topics { async getTopics(): Promise { let config = GetConfig(); - let result = await executeCli(["kafka", "topic", "list", "--cluster", config.clusterId, "--environment", config.environmentId]); - result = - parse(result) - .filter(t => t.Name.startsWith("_confluent") === false) - .map(t => t.Name); - - return result; + let result = await executeCli(["kafka", "topic", "list", "--cluster", config.clusterId, "--environment", config.environmentId, "--output", "json"]); + + let combinedResult = ConcatOutput(result); + let deserializedResult : ListTopics; + try { + deserializedResult = Deserializer(combinedResult); + } catch (error) { + return error; + } + + return deserializedResult + .filter(t => t.name.startsWith("_confluent") === false) + .map(t => t.name); } async describeTopic(name: string): Promise { let config = GetConfig(); - let consoleLines = await executeCli(["kafka", "topic", "describe", name, "--cluster", config.clusterId, "--environment", config.environmentId]); - - var topic = parseTopicDescription(consoleLines); + let result = await executeCli(["kafka", "topic", "describe", name, "--cluster", config.clusterId, "--environment", config.environmentId, "--output", "json"]); + + let combinedResult = ConcatOutput(result); + let deserializedResult : DescribeTopic; + try { + deserializedResult = Deserializer(combinedResult); + } catch (error) { + return error; + } + + let topic = { + Name: deserializedResult.topic_name, + PartitionCount: deserializedResult.config["num.partitions"], // might be an issue that partitionCount is present twice + Configurations: deserializedResult.config // might be an issue that partitionCount is present twice + }; return topic; } diff --git a/server/src/server/wrapper/connected/CcloudAccessControlLists.ts b/server/src/server/wrapper/connected/CcloudAccessControlLists.ts index 8610108..72efddb 100644 --- a/server/src/server/wrapper/connected/CcloudAccessControlLists.ts +++ b/server/src/server/wrapper/connected/CcloudAccessControlLists.ts @@ -1,19 +1,35 @@ import { parse } from "./../parser"; import { executeCli } from "./executeCli"; import { GetConfig } from "../../config"; +import { Deserializer, ConcatOutput } from "../utils"; +import { ListAcls } from "../model/acls"; export class CcloudAccessControlLists implements AccessControlLists { async getAccessControlLists(): Promise { let config = GetConfig(); - let result = await executeCli(["kafka", "acl", "list", "--cluster", config.clusterId, "--environment", config.environmentId]); - let resultObjects = parse(result) as AccessControlList[]; + let result = await executeCli(["kafka", "acl", "list", "--cluster", config.clusterId, "--environment", config.environmentId, "--output", "json"]); - resultObjects.forEach(elem => { - elem.UserId = elem.UserId.split(':')[1]; - }); - - return resultObjects; - } + let combinedResult = ConcatOutput(result); + let deserializedResult : ListAcls; + try { + deserializedResult = Deserializer(combinedResult); + } catch (error) { + return error; + } + + return deserializedResult.map(t => { + let obj = { + UserId: "", // removed property by Confluent + ServiceAccountId: t.principal.split(":")[1], + Permission: t.permission, + Operation: t.operation, + Resource: t.resource_type, + Name: t.resource_name, + Type: t.pattern_type + }; + return obj; + }) + } async createAccessControlList( serviceAccountId: number, diff --git a/server/src/server/wrapper/connected/CcloudApiKeys.ts b/server/src/server/wrapper/connected/CcloudApiKeys.ts index e9916a8..28ae9bb 100644 --- a/server/src/server/wrapper/connected/CcloudApiKeys.ts +++ b/server/src/server/wrapper/connected/CcloudApiKeys.ts @@ -1,11 +1,13 @@ import { parse, parseSideColumns } from "./../parser"; import {executeCli } from "./executeCli"; import { GetConfig } from "../../config"; +import { Deserializer, ConcatOutput } from "../utils"; +import { CreateApiKey, ListApiKeys } from "../model/api-keys"; export class CcloudApiKeys implements ApiKeys { ccloud: CCloudCliWrapper; - async createApiKey(serviceAccountId: number, description: string): Promise { + async createApiKey(serviceAccountId: string, description: string): Promise { let config = GetConfig(); let cliOutput = await executeCli([ @@ -13,28 +15,51 @@ export class CcloudApiKeys implements ApiKeys { "create", "--resource", config.clusterId, "--environment", config.environmentId, - "--service-account", serviceAccountId + "", - "--description", description] + "--service-account", serviceAccountId, + "--description", description, "--output", "json"] ); let cliObjects: any = parseSideColumns(cliOutput); let apiKeySet: ApiKeySet = { Key: cliObjects.APIKey, Secret: cliObjects.Secret } + + let combinedResult = ConcatOutput(cliOutput); + let deserializedResult : CreateApiKey; + try { + deserializedResult = Deserializer(combinedResult); + } catch (error) { + return error; + } - return apiKeySet; + return { + Key: deserializedResult.key, + Secret: deserializedResult.secret + }; } async deleteApiKey(key: string): Promise { await executeCli(["api-key", "delete", key]); } + async getApiKeys(): Promise { - let cliOutput = await executeCli(["api-key", "list"]); - let cliObjects = parse(cliOutput); + let cliOutput = await executeCli(["api-key", "list", "--output", "json"]); - let apiKeys = cliObjects.map(function (obj) { - return { Key: obj.Key, Description: obj.Description, Owner: obj.Owner, Resource: obj.ResourceID } as ApiKey - }); - - return apiKeys; + let combinedResult = ConcatOutput(cliOutput); + let deserializedResult : ListApiKeys; + try { + deserializedResult = Deserializer(combinedResult); + } catch (error) { + return error; + } + + return deserializedResult.map(t => { + let obj = { + Key: t.key, + Description: t.description, + Owner: t.owner_resource_id, + Resource: t.resource_id + }; + return obj; + }) } constructor(ccloud: CCloudCliWrapper) { diff --git a/server/src/server/wrapper/connected/CcloudCluster.ts b/server/src/server/wrapper/connected/CcloudCluster.ts index 316aabe..d17a0ed 100644 --- a/server/src/server/wrapper/connected/CcloudCluster.ts +++ b/server/src/server/wrapper/connected/CcloudCluster.ts @@ -1,6 +1,8 @@ import { parse, parseSideColumns } from "./../parser"; import {executeCli } from "./executeCli"; import { GetConfig } from "../../config"; +import { Deserializer, ConcatOutput } from "../utils"; +import Clusters from "../model/clusters"; export class CcloudCluster { ccloud: CCloudCliWrapper; @@ -8,12 +10,17 @@ export class CcloudCluster { async list(): Promise { let config = GetConfig(); - let result = await executeCli(["kafka", "cluster", "list", "--environment", config.environmentId]); - parse(result); - console.log("\n::SEP::\n"); - console.log(result); + let result = await executeCli(["kafka", "cluster", "list", "--environment", config.environmentId, "--output", "json"]); + + let combinedResult = ConcatOutput(result); + let deserializedResult : Clusters; + try { + deserializedResult = Deserializer(combinedResult); + } catch (error) { + return error; + } - return result; + return deserializedResult; } constructor(ccloud: CCloudCliWrapper) { diff --git a/server/src/server/wrapper/connected/CcloudServiceAccount.ts b/server/src/server/wrapper/connected/CcloudServiceAccount.ts index a95e1f2..5fd9fda 100644 --- a/server/src/server/wrapper/connected/CcloudServiceAccount.ts +++ b/server/src/server/wrapper/connected/CcloudServiceAccount.ts @@ -1,28 +1,43 @@ import { parse, parseSideColumns } from "./../parser"; import { executeCli } from "./executeCli"; import { CliException, ServiceAccountAlreadyExistsException } from "./../model/error"; +import { Deserializer, ConcatOutput } from "../utils"; +import { ListServiceAccounts, ListServiceAccount } from "../model/service-account"; export class CcloudServiceAccount implements ServiceAccounts { ccloud: CCloudCliWrapper; async getServiceAccounts(): Promise { - let result = await executeCli(["service-account", "list"]); - result = parse(result); + let result = await executeCli(["iam", "service-account", "list", "--output", "json"]); + let combinedResult = ConcatOutput(result); + let deserializedResult : ListServiceAccounts; + try { + deserializedResult = Deserializer(combinedResult); + } catch (error) { + return error; + } - return (result as any) as ServiceAccount[]; + return deserializedResult.map(t => { + let obj = { + Name: t.name, + Id: t.id, + Description: t.description + }; + return obj; + }) } async createServiceAccount(accountName: string, description: string = ""): Promise { let cliResult; try { - cliResult = await executeCli(["service-account", "create", accountName, "--description", description]); + cliResult = await executeCli(["iam", "service-account", "create", accountName, "--description", description, "--output", "json"]); } catch (error) { if (error.name.valueOf() !== "CliException") { throw (error); } - if (error.consoleLines.some((l: string): boolean => l.includes("Service name is already in use"))) { + if (error.consoleLines.some((l: string): boolean => l.includes("is already in use"))) { let existingServicesAccounts = await this.getServiceAccounts(); let existingServicesAccount = existingServicesAccounts.find(s => s.Name === accountName); @@ -39,15 +54,23 @@ export class CcloudServiceAccount implements ServiceAccounts { throw (error); } + let combinedResult = ConcatOutput(cliResult); + let deserializedResult : ListServiceAccount; + try { + deserializedResult = Deserializer(combinedResult); + } catch (error) { + return error; + } - - let result = parseSideColumns(cliResult); - - return (result as any) as ServiceAccount; + return { + Name: deserializedResult.name, + Id: deserializedResult.id, + Description: deserializedResult.description + }; } async deleteServiceAccount(accountId: number): Promise { - await executeCli(["service-account", "delete", accountId.toString()]); + await executeCli(["iam", "service-account", "delete", accountId.toString()]); return true; } diff --git a/server/src/server/wrapper/connected/executeCli.ts b/server/src/server/wrapper/connected/executeCli.ts index ac035bc..4cdfbc7 100644 --- a/server/src/server/wrapper/connected/executeCli.ts +++ b/server/src/server/wrapper/connected/executeCli.ts @@ -3,7 +3,7 @@ import * as readline from "readline"; import { CcloudSessionExpiredException, CliException } from "./../model/error"; export function executeCli(args: string[]): Promise { - const cli = process.env.TIKA_CCLOUD_BIN_PATH ?? "ccloud"; + const cli = process.env.TIKA_CCLOUD_BIN_PATH ?? "confluent"; return new Promise((resolve, reject) => { const lines: Array = []; @@ -21,7 +21,7 @@ export function executeCli(args: string[]): Promise { return resolve(lines); } - if (errLines.some((l: string): boolean => l.includes("You must log in to run that command."))) { + if (errLines.some((l: string): boolean => l.includes("You must be logged in to run this command"))) { return reject(new CcloudSessionExpiredException()); } diff --git a/server/src/server/wrapper/definitions.d.ts b/server/src/server/wrapper/definitions.d.ts index f69e8f4..b2661d7 100644 --- a/server/src/server/wrapper/definitions.d.ts +++ b/server/src/server/wrapper/definitions.d.ts @@ -48,7 +48,7 @@ interface ServiceAccounts { interface ApiKeys { createApiKey( - serviceAccountId: number, + serviceAccountId: string, description: string ): Promise; diff --git a/server/src/server/wrapper/model/acls.ts b/server/src/server/wrapper/model/acls.ts new file mode 100644 index 0000000..c629463 --- /dev/null +++ b/server/src/server/wrapper/model/acls.ts @@ -0,0 +1,10 @@ +export class ListAcl { + operation: string + pattern_type: string + permission: string + principal: string + resource_name: string + resource_type: string +} + +export type ListAcls = Array diff --git a/server/src/server/wrapper/model/api-keys.ts b/server/src/server/wrapper/model/api-keys.ts new file mode 100644 index 0000000..794fc02 --- /dev/null +++ b/server/src/server/wrapper/model/api-keys.ts @@ -0,0 +1,16 @@ +export class CreateApiKey { + key: string + secret: string +} + +export class ListApiKey{ + created: string + description: string + key: string + owner_email: string + owner_resource_id: string + resource_id: string + resource_type: string +} + +export type ListApiKeys = Array \ No newline at end of file diff --git a/server/src/server/wrapper/model/clusters.ts b/server/src/server/wrapper/model/clusters.ts new file mode 100644 index 0000000..955b823 --- /dev/null +++ b/server/src/server/wrapper/model/clusters.ts @@ -0,0 +1,13 @@ +export class Cluster { + availability: string + id: string + name: string + provider: string + region: string + status: string + type: string +} + +type Clusters = Array; + +export default Clusters; \ No newline at end of file diff --git a/server/src/server/wrapper/model/service-account.ts b/server/src/server/wrapper/model/service-account.ts index 847f6f9..cbab58d 100644 --- a/server/src/server/wrapper/model/service-account.ts +++ b/server/src/server/wrapper/model/service-account.ts @@ -1,5 +1,7 @@ -export default class ServiceAccount { - Id: number; - Name: string; - Description: string; // Currently isn't parsed properly (whitespace and space in general gets trimemd away) +export class ListServiceAccount { + id: number + name: string + description: string // Currently isn't parsed properly (whitespace and space in general gets trimemd away) } + +export type ListServiceAccounts = Array; \ No newline at end of file diff --git a/server/src/server/wrapper/model/topics.ts b/server/src/server/wrapper/model/topics.ts new file mode 100644 index 0000000..09cd930 --- /dev/null +++ b/server/src/server/wrapper/model/topics.ts @@ -0,0 +1,10 @@ +export class ListTopic { + name: string +} + +export type ListTopics = Array; + +export class DescribeTopic { + topic_name: string + config: {[key: string]: any} +} \ No newline at end of file diff --git a/server/src/server/wrapper/notConnected/NotConnectedApiKeys.ts b/server/src/server/wrapper/notConnected/NotConnectedApiKeys.ts index bc3dda6..aec323d 100644 --- a/server/src/server/wrapper/notConnected/NotConnectedApiKeys.ts +++ b/server/src/server/wrapper/notConnected/NotConnectedApiKeys.ts @@ -15,7 +15,7 @@ export class NotConnectedApiKeys implements ApiKeys { constructor() { this.apiKeys = []; } - async createApiKey(serviceAccountId: number, description: string): Promise { + async createApiKey(serviceAccountId: string, description: string): Promise { let key = NotConnectedApiKeys.createRandomString(16, true); let apiKey: ApiKey = { diff --git a/server/src/server/wrapper/utils.ts b/server/src/server/wrapper/utils.ts new file mode 100644 index 0000000..44968ab --- /dev/null +++ b/server/src/server/wrapper/utils.ts @@ -0,0 +1,10 @@ +export function Deserializer(json:string) : T { + let payload = JSON.parse(json); + return payload; +} + +export function ConcatOutput(output:Array) : string { + return output.join("\n"); +} + +export default Deserializer; \ No newline at end of file