Skip to content
This repository has been archived by the owner on Jul 25, 2023. It is now read-only.

Commit

Permalink
Merge pull request #46 from dfds/maintenance/update-cli-tooling
Browse files Browse the repository at this point in the history
maintenance/update cli tooling
  • Loading branch information
wcarlsen authored May 3, 2022
2 parents 575c2e2 + 9e22749 commit a77b92e
Show file tree
Hide file tree
Showing 21 changed files with 220 additions and 78 deletions.
1 change: 1 addition & 0 deletions .envrc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
dotenv
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ namespace Tika.RestClient.Features.Acls.Models
{
public class Acl
{
public long UserId { get; set; }
public string UserId { get; set; }
public string ServiceAccountId { get; set; }
public string Permission { get; set; }
public string Resource { get; set; }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@ namespace Tika.RestClient.Features.Acls.Models
{
public class AclCreateDelete
{
public long ServiceAccountId { get; set; }
public string ServiceAccountId { get; set; }
public bool Allow { get; set; }
public string Operation { get; set; }
public string TopicPrefix { get; set; }
public string ConsumerGroupPrefix { get; set; }

public AclCreateDelete() {}

public AclCreateDelete(long serviceAccountId, bool allow, string operation, string topicPrefix = "", string consumerGroupPrefix = "")
public AclCreateDelete(string serviceAccountId, bool allow, string operation, string topicPrefix = "", string consumerGroupPrefix = "")
{
ServiceAccountId = serviceAccountId;
Allow = allow;
Expand Down
11 changes: 5 additions & 6 deletions server/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,13 @@ RUN apk add --update \
curl expect \
&& rm -rf /var/cache/apk/*

# Install ccloud cli tool
RUN curl -L https://s3-us-west-2.amazonaws.com/confluent.cloud/ccloud-cli/install.sh | sh -s -- -b /ccloud/bin
# Install confluent cli tool
RUN apk --no-cache add ca-certificates bash curl

ENV TIKA_CCLOUD_BIN_PATH="/ccloud/bin/ccloud"
ENV PATH "$PATH:/ccloud/bin"

RUN ccloud version
ENV CONFLUENT_CLI_VERSION="v2.12.0"
RUN curl -sL --http1.1 https://cnfl.io/cli | sh -s -- -b /usr/local/bin $CONFLUENT_CLI_VERSION

RUN confluent version

# Copy app & supporting scripts
COPY --from=Builder /app/dist/main.js /app/main.js
Expand Down
19 changes: 4 additions & 15 deletions server/login.sh
Original file line number Diff line number Diff line change
@@ -1,15 +1,4 @@
#!/usr/bin/expect
spawn ccloud login

expect "Email: "

send -- "$env(TIKA_CC_USER)\r"

expect "Password: "

send -- "$env(TIKA_CC_PASS)\r"

set timeout 600
expect eof

send_user "Confluent cloud login successful\n"
#!/bin/ash
export CONFLUENT_CLOUD_EMAIL="$TIKA_CC_USER"
export CONFLUENT_CLOUD_PASSWORD="$TIKA_CC_PASS"
confluent login --save
2 changes: 1 addition & 1 deletion server/makefile
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ build:
docker build -t $(IMAGE_NAME) .

run:
docker run -it -p 3000:3000 --rm $(IMAGE_NAME)
docker run -it -p 3000:3000 --rm --env-file ../.env $(IMAGE_NAME)

release: build
chmod +x ../scripts/push_container_image.sh && ../scripts/push_container_image.sh $(IMAGE_NAME) $(BUILD_NUMBER)
Expand Down
2 changes: 1 addition & 1 deletion server/src/server/api/api-keys.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ export class ApiKeysInterface {

try {
let apiKey = await apiKeys.createApiKey(
parseInt(req.body.serviceAccountId),
req.body.serviceAccountId,
req.body.description
);
res.json(apiKey);
Expand Down
41 changes: 31 additions & 10 deletions server/src/server/wrapper/connected/CCloudTopics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,26 +2,47 @@ import { parse, parseTopicDescription } from "./../parser";
import { executeCli } from "./executeCli";
import { TopicAlreadyExistsException } from "../model/error";
import { GetConfig } from "../../config";
import { Deserializer, ConcatOutput } from "../utils";
import { ListTopics, DescribeTopic } from "../model/topics";


export class CcloudTopics implements Topics {

async getTopics(): Promise<string[]> {
let config = GetConfig();

let result = await executeCli(["kafka", "topic", "list", "--cluster", config.clusterId, "--environment", config.environmentId]);
result =
parse(result)
.filter(t => t.Name.startsWith("_confluent") === false)
.map(t => t.Name);

return result;
let result = await executeCli(["kafka", "topic", "list", "--cluster", config.clusterId, "--environment", config.environmentId, "--output", "json"]);

let combinedResult = ConcatOutput(result);
let deserializedResult : ListTopics;
try {
deserializedResult = Deserializer<ListTopics>(combinedResult);
} catch (error) {
return error;
}

return deserializedResult
.filter(t => t.name.startsWith("_confluent") === false)
.map(t => t.name);
}

async describeTopic(name: string): Promise<Topic> {
let config = GetConfig();
let consoleLines = await executeCli(["kafka", "topic", "describe", name, "--cluster", config.clusterId, "--environment", config.environmentId]);

var topic = parseTopicDescription(consoleLines);
let result = await executeCli(["kafka", "topic", "describe", name, "--cluster", config.clusterId, "--environment", config.environmentId, "--output", "json"]);

let combinedResult = ConcatOutput(result);
let deserializedResult : DescribeTopic;
try {
deserializedResult = Deserializer<DescribeTopic>(combinedResult);
} catch (error) {
return error;
}

let topic = {
Name: deserializedResult.topic_name,
PartitionCount: deserializedResult.config["num.partitions"], // might be an issue that partitionCount is present twice
Configurations: deserializedResult.config // might be an issue that partitionCount is present twice
};

return topic;
}
Expand Down
32 changes: 24 additions & 8 deletions server/src/server/wrapper/connected/CcloudAccessControlLists.ts
Original file line number Diff line number Diff line change
@@ -1,19 +1,35 @@
import { parse } from "./../parser";
import { executeCli } from "./executeCli";
import { GetConfig } from "../../config";
import { Deserializer, ConcatOutput } from "../utils";
import { ListAcls } from "../model/acls";

export class CcloudAccessControlLists implements AccessControlLists {
async getAccessControlLists(): Promise<AccessControlList[]> {
let config = GetConfig();
let result = await executeCli(["kafka", "acl", "list", "--cluster", config.clusterId, "--environment", config.environmentId]);
let resultObjects = parse(result) as AccessControlList[];
let result = await executeCli(["kafka", "acl", "list", "--cluster", config.clusterId, "--environment", config.environmentId, "--output", "json"]);

resultObjects.forEach(elem => {
elem.UserId = elem.UserId.split(':')[1];
});

return resultObjects;
}
let combinedResult = ConcatOutput(result);
let deserializedResult : ListAcls;
try {
deserializedResult = Deserializer<ListAcls>(combinedResult);
} catch (error) {
return error;
}

return deserializedResult.map(t => {
let obj = {
UserId: "", // removed property by Confluent
ServiceAccountId: t.principal.split(":")[1],
Permission: t.permission,
Operation: t.operation,
Resource: t.resource_type,
Name: t.resource_name,
Type: t.pattern_type
};
return obj;
})
}

async createAccessControlList(
serviceAccountId: number,
Expand Down
47 changes: 36 additions & 11 deletions server/src/server/wrapper/connected/CcloudApiKeys.ts
Original file line number Diff line number Diff line change
@@ -1,40 +1,65 @@
import { parse, parseSideColumns } from "./../parser";
import {executeCli } from "./executeCli";
import { GetConfig } from "../../config";
import { Deserializer, ConcatOutput } from "../utils";
import { CreateApiKey, ListApiKeys } from "../model/api-keys";

export class CcloudApiKeys implements ApiKeys {
ccloud: CCloudCliWrapper;

async createApiKey(serviceAccountId: number, description: string): Promise<ApiKeySet> {
async createApiKey(serviceAccountId: string, description: string): Promise<ApiKeySet> {
let config = GetConfig();

let cliOutput = await executeCli([
"api-key",
"create",
"--resource", config.clusterId,
"--environment", config.environmentId,
"--service-account", serviceAccountId + "",
"--description", description]
"--service-account", serviceAccountId,
"--description", description, "--output", "json"]
);

let cliObjects: any = parseSideColumns(cliOutput);
let apiKeySet: ApiKeySet = { Key: cliObjects.APIKey, Secret: cliObjects.Secret }

let combinedResult = ConcatOutput(cliOutput);
let deserializedResult : CreateApiKey;
try {
deserializedResult = Deserializer<CreateApiKey>(combinedResult);
} catch (error) {
return error;
}

return apiKeySet;
return {
Key: deserializedResult.key,
Secret: deserializedResult.secret
};
}

async deleteApiKey(key: string): Promise<void> {
await executeCli(["api-key", "delete", key]);
}

async getApiKeys(): Promise<ApiKey[]> {
let cliOutput = await executeCli(["api-key", "list"]);
let cliObjects = parse(cliOutput);
let cliOutput = await executeCli(["api-key", "list", "--output", "json"]);

let apiKeys = cliObjects.map(function (obj) {
return { Key: obj.Key, Description: obj.Description, Owner: obj.Owner, Resource: obj.ResourceID } as ApiKey
});

return apiKeys;
let combinedResult = ConcatOutput(cliOutput);
let deserializedResult : ListApiKeys;
try {
deserializedResult = Deserializer<ListApiKeys>(combinedResult);
} catch (error) {
return error;
}

return deserializedResult.map(t => {
let obj = {
Key: t.key,
Description: t.description,
Owner: t.owner_resource_id,
Resource: t.resource_id
};
return obj;
})
}

constructor(ccloud: CCloudCliWrapper) {
Expand Down
17 changes: 12 additions & 5 deletions server/src/server/wrapper/connected/CcloudCluster.ts
Original file line number Diff line number Diff line change
@@ -1,19 +1,26 @@
import { parse, parseSideColumns } from "./../parser";
import {executeCli } from "./executeCli";
import { GetConfig } from "../../config";
import { Deserializer, ConcatOutput } from "../utils";
import Clusters from "../model/clusters";

export class CcloudCluster {
ccloud: CCloudCliWrapper;

async list(): Promise<any[]> {
let config = GetConfig();

let result = await executeCli(["kafka", "cluster", "list", "--environment", config.environmentId]);
parse(result);
console.log("\n::SEP::\n");
console.log(result);
let result = await executeCli(["kafka", "cluster", "list", "--environment", config.environmentId, "--output", "json"]);

let combinedResult = ConcatOutput(result);
let deserializedResult : Clusters;
try {
deserializedResult = Deserializer<Clusters>(combinedResult);
} catch (error) {
return error;
}

return result;
return deserializedResult;
}

constructor(ccloud: CCloudCliWrapper) {
Expand Down
43 changes: 33 additions & 10 deletions server/src/server/wrapper/connected/CcloudServiceAccount.ts
Original file line number Diff line number Diff line change
@@ -1,28 +1,43 @@
import { parse, parseSideColumns } from "./../parser";
import { executeCli } from "./executeCli";
import { CliException, ServiceAccountAlreadyExistsException } from "./../model/error";
import { Deserializer, ConcatOutput } from "../utils";
import { ListServiceAccounts, ListServiceAccount } from "../model/service-account";

export class CcloudServiceAccount implements ServiceAccounts {
ccloud: CCloudCliWrapper;

async getServiceAccounts(): Promise<ServiceAccount[]> {
let result = await executeCli(["service-account", "list"]);
result = parse(result);
let result = await executeCli(["iam", "service-account", "list", "--output", "json"]);
let combinedResult = ConcatOutput(result);
let deserializedResult : ListServiceAccounts;
try {
deserializedResult = Deserializer<ListServiceAccounts>(combinedResult);
} catch (error) {
return error;
}

return (result as any) as ServiceAccount[];
return deserializedResult.map(t => {
let obj = {
Name: t.name,
Id: t.id,
Description: t.description
};
return obj;
})
}

async createServiceAccount(accountName: string, description: string = ""): Promise<ServiceAccount> {
let cliResult;
try {
cliResult = await executeCli(["service-account", "create", accountName, "--description", description]);
cliResult = await executeCli(["iam", "service-account", "create", accountName, "--description", description, "--output", "json"]);
}
catch (error) {
if (error.name.valueOf() !== "CliException") {
throw (error);
}

if (error.consoleLines.some((l: string): boolean => l.includes("Service name is already in use"))) {
if (error.consoleLines.some((l: string): boolean => l.includes("is already in use"))) {
let existingServicesAccounts = await this.getServiceAccounts();

let existingServicesAccount = existingServicesAccounts.find(s => s.Name === accountName);
Expand All @@ -39,15 +54,23 @@ export class CcloudServiceAccount implements ServiceAccounts {
throw (error);
}

let combinedResult = ConcatOutput(cliResult);
let deserializedResult : ListServiceAccount;
try {
deserializedResult = Deserializer<ListServiceAccount>(combinedResult);
} catch (error) {
return error;
}


let result = parseSideColumns(cliResult);

return (result as any) as ServiceAccount;
return {
Name: deserializedResult.name,
Id: deserializedResult.id,
Description: deserializedResult.description
};
}

async deleteServiceAccount(accountId: number): Promise<boolean> {
await executeCli(["service-account", "delete", accountId.toString()]);
await executeCli(["iam", "service-account", "delete", accountId.toString()]);
return true;
}

Expand Down
4 changes: 2 additions & 2 deletions server/src/server/wrapper/connected/executeCli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import * as readline from "readline";
import { CcloudSessionExpiredException, CliException } from "./../model/error";

export function executeCli(args: string[]): Promise<string[]> {
const cli = process.env.TIKA_CCLOUD_BIN_PATH ?? "ccloud";
const cli = process.env.TIKA_CCLOUD_BIN_PATH ?? "confluent";

return new Promise((resolve, reject) => {
const lines: Array<string> = [];
Expand All @@ -21,7 +21,7 @@ export function executeCli(args: string[]): Promise<string[]> {
return resolve(lines);
}

if (errLines.some((l: string): boolean => l.includes("You must log in to run that command."))) {
if (errLines.some((l: string): boolean => l.includes("You must be logged in to run this command"))) {
return reject(new CcloudSessionExpiredException());
}

Expand Down
Loading

0 comments on commit a77b92e

Please sign in to comment.