diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 257eb56d..3c68f491 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -10,8 +10,12 @@ on: jobs: apisix: runs-on: ubuntu-latest + strategy: + matrix: + version: [3.8.1, 3.9.1, 3.10.0, 3.11.0] env: - BACKEND_APISIX_VERSION: 3.9.1-debian + BACKEND_APISIX_VERSION: ${{ matrix.version }} + BACKEND_APISIX_IMAGE: ${{ matrix.version }}-debian steps: - uses: actions/checkout@v4 diff --git a/apps/cli/src/command/utils.ts b/apps/cli/src/command/utils.ts index ec81ed02..c1248dfb 100644 --- a/apps/cli/src/command/utils.ts +++ b/apps/cli/src/command/utils.ts @@ -253,7 +253,7 @@ export const recursiveRemoveMetadataField = (c: ADCSDK.Configuration) => { if ('metadata' in obj) delete obj.metadata; }; Object.entries(c).forEach(([key, value]) => { - if (['global_rules', 'plugin_metadata', 'consumers'].includes(key)) return; + if (['global_rules', 'plugin_metadata'].includes(key)) return; if (Array.isArray(value)) value.forEach((item) => { removeMetadata(item); @@ -265,6 +265,9 @@ export const recursiveRemoveMetadataField = (c: ADCSDK.Configuration) => { } else if (key === 'consumer_groups') { if ('consumers' in item && Array.isArray(item.consumers)) item.consumers.forEach((c) => removeMetadata(c)); + } else if (key === 'consumers') { + if ('credentials' in item && Array.isArray(item.credentials)) + item.credentials.forEach((c) => removeMetadata(c)); } }); }); diff --git a/libs/backend-apisix/e2e/assets/docker-compose.yaml b/libs/backend-apisix/e2e/assets/docker-compose.yaml index 457520a6..d6c80351 100644 --- a/libs/backend-apisix/e2e/assets/docker-compose.yaml +++ b/libs/backend-apisix/e2e/assets/docker-compose.yaml @@ -1,6 +1,6 @@ services: apisix_http: - image: apache/apisix:${BACKEND_APISIX_VERSION:-3.9.0-debian} + image: apache/apisix:${BACKEND_APISIX_IMAGE:-3.9.0-debian} restart: always volumes: - ./apisix_conf/http.yaml:/usr/local/apisix/conf/config.yaml:ro @@ -13,7 +13,7 @@ services: apisix: apisix_mtls: - image: apache/apisix:${BACKEND_APISIX_VERSION:-3.9.0-debian} + image: apache/apisix:${BACKEND_APISIX_IMAGE:-3.9.0-debian} restart: always volumes: - ./apisix_conf/mtls.yaml:/usr/local/apisix/conf/config.yaml:ro diff --git a/libs/backend-apisix/e2e/resources/consumer.e2e-spec.ts b/libs/backend-apisix/e2e/resources/consumer.e2e-spec.ts new file mode 100644 index 00000000..39dad100 --- /dev/null +++ b/libs/backend-apisix/e2e/resources/consumer.e2e-spec.ts @@ -0,0 +1,116 @@ +import * as ADCSDK from '@api7/adc-sdk'; +import { gte, lt } from 'semver'; + +import { BackendAPISIX } from '../../src'; +import { server, token } from '../support/constants'; +import { conditionalDescribe, semverCondition } from '../support/utils'; +import { + createEvent, + deleteEvent, + dumpConfiguration, + syncEvents, + updateEvent, +} from '../support/utils'; + +describe('Consumer E2E', () => { + let backend: BackendAPISIX; + + beforeAll(() => { + backend = new BackendAPISIX({ + server, + token, + tlsSkipVerify: true, + }); + }); + + conditionalDescribe(semverCondition(gte, '3.11.0'))( + 'Sync and dump consumers (with credential support)', + () => { + const consumer1Name = 'consumer1'; + const consumer1Key = 'consumer1-key'; + const consumer1Cred = { + name: consumer1Key, + type: 'key-auth', + config: { key: consumer1Key }, + }; + const consumer1 = { + username: consumer1Name, + credentials: [consumer1Cred], + } as ADCSDK.Consumer; + + it('Create consumers', async () => + syncEvents(backend, [ + createEvent(ADCSDK.ResourceType.CONSUMER, consumer1Name, consumer1), + createEvent( + ADCSDK.ResourceType.CONSUMER_CREDENTIAL, + consumer1Key, + consumer1Cred, + consumer1Name, + ), + ])); + + it('Dump', async () => { + const result = (await dumpConfiguration( + backend, + )) as ADCSDK.Configuration; + expect(result.consumers).toHaveLength(1); + expect(result.consumers[0]).toMatchObject(consumer1); + expect(result.consumers[0].credentials).toMatchObject( + consumer1.credentials, + ); + }); + + it('Update consumer credential', async () => { + consumer1.credentials[0].config.key = 'new-key'; + await syncEvents(backend, [ + updateEvent( + ADCSDK.ResourceType.CONSUMER_CREDENTIAL, + consumer1Key, + consumer1Cred, + consumer1Name, + ), + ]); + }); + + it('Dump again (consumer credential updated)', async () => { + const result = (await dumpConfiguration( + backend, + )) as ADCSDK.Configuration; + expect(result.consumers[0]).toMatchObject(consumer1); + expect(result.consumers[0].credentials[0].config.key).toEqual( + 'new-key', + ); + }); + + it('Delete consumer credential', async () => + syncEvents(backend, [ + deleteEvent( + ADCSDK.ResourceType.CONSUMER_CREDENTIAL, + consumer1Key, + consumer1Name, + ), + ])); + + it('Dump again (consumer credential should not exist)', async () => { + const result = (await dumpConfiguration( + backend, + )) as ADCSDK.Configuration; + expect(result.consumers).toHaveLength(1); + console.log(result.consumers[0]); + expect(result.consumers[0].credentials).toBeUndefined(); + }); + + it('Delete consumer', async () => + syncEvents(backend, [ + deleteEvent(ADCSDK.ResourceType.CONSUMER, consumer1Name), + ])); + + it('Dump again (consumer should not exist)', async () => { + const result = (await dumpConfiguration( + backend, + )) as ADCSDK.Configuration; + expect(result.consumers).toHaveLength(0); + }); + }, + ); +}); diff --git a/libs/backend-apisix/e2e/support/utils.ts b/libs/backend-apisix/e2e/support/utils.ts index 83100439..24dc14ba 100644 --- a/libs/backend-apisix/e2e/support/utils.ts +++ b/libs/backend-apisix/e2e/support/utils.ts @@ -1,5 +1,6 @@ import * as ADCSDK from '@api7/adc-sdk'; import { Listr, SilentRenderer } from 'listr2'; +import semver from 'semver'; import { BackendAPISIX } from '../../src'; @@ -49,7 +50,11 @@ export const createEvent = ( parentName ? `${parentName}.${resourceName}` : resourceName, ), newValue: resource, - parentId: parentName ? ADCSDK.utils.generateId(parentName) : undefined, + parentId: parentName + ? resourceType === ADCSDK.ResourceType.CONSUMER_CREDENTIAL + ? parentName + : ADCSDK.utils.generateId(parentName) + : undefined, }); export const updateEvent = ( @@ -79,5 +84,22 @@ export const deleteEvent = ( : ADCSDK.utils.generateId( parentName ? `${parentName}.${resourceName}` : resourceName, ), - parentId: parentName ? ADCSDK.utils.generateId(parentName) : undefined, + parentId: parentName + ? resourceType === ADCSDK.ResourceType.CONSUMER_CREDENTIAL + ? parentName + : ADCSDK.utils.generateId(parentName) + : undefined, }); + +type cond = boolean | (() => boolean); + +export const conditionalDescribe = (cond: cond) => + cond ? describe : describe.skip; + +export const conditionalIt = (cond: cond) => (cond ? it : it.skip); + +export const semverCondition = ( + op: (v1: string | semver.SemVer, v2: string | semver.SemVer) => boolean, + base: string, + target = semver.coerce(process.env.BACKEND_APISIX_VERSION) ?? '0.0.0', +) => op(target, base); diff --git a/libs/backend-apisix/e2e/sync-and-dump-1.e2e-spec.ts b/libs/backend-apisix/e2e/sync-and-dump-1.e2e-spec.ts index 9f340a2c..8d854a4e 100644 --- a/libs/backend-apisix/e2e/sync-and-dump-1.e2e-spec.ts +++ b/libs/backend-apisix/e2e/sync-and-dump-1.e2e-spec.ts @@ -21,7 +21,6 @@ describe('Sync and Dump - 1', () => { server, token, tlsSkipVerify: true, - gatewayGroup: 'default', }); }); diff --git a/libs/backend-apisix/src/fetcher.ts b/libs/backend-apisix/src/fetcher.ts index 5157543e..f9f462c4 100644 --- a/libs/backend-apisix/src/fetcher.ts +++ b/libs/backend-apisix/src/fetcher.ts @@ -1,6 +1,7 @@ import * as ADCSDK from '@api7/adc-sdk'; import { Axios } from 'axios'; import { ListrTask } from 'listr2'; +import { SemVer, gte as semVerGTE } from 'semver'; import { ToADC } from './transformer'; import * as typing from './typing'; @@ -8,6 +9,8 @@ import { buildReqAndRespDebugOutput, resourceTypeToAPIName } from './utils'; type FetchTask = ListrTask<{ remote: ADCSDK.Configuration; + + apisixVersion: SemVer; apisixResources?: typing.Resources; }>; @@ -65,7 +68,6 @@ export class Fetcher { ) return; - // resourceType === ADCSDK.ResourceType.GLOBAL_RULE || if (resourceType === ADCSDK.ResourceType.PLUGIN_METADATA) { ctx.apisixResources[ADCSDK.ResourceType.PLUGIN_METADATA] = Object.fromEntries( @@ -92,6 +94,35 @@ export class Fetcher { (item) => item.value, ); } + + if ( + resourceType === ADCSDK.ResourceType.CONSUMER && + semVerGTE(ctx.apisixVersion, '3.11.0') + ) { + await Promise.all( + ctx.apisixResources[resourceType].map(async (item) => { + const resp = await this.client.get<{ + list: Array<{ + key: string; + value: typing.ConsumerCredential; + createdIndex: number; + modifiedIndex: number; + }>; + total: number; + }>(`/apisix/admin/consumers/${item.username}/credentials`, { + validateStatus: () => true, + }); + task.output = buildReqAndRespDebugOutput( + resp, + `Get credentials of consumer "${item.username}"`, + ); + if (resp.status === 200) + item.credentials = resp.data.list.map( + (credential) => credential.value, + ); + }), + ); + } }, }), ); diff --git a/libs/backend-apisix/src/index.ts b/libs/backend-apisix/src/index.ts index ba8f124a..d8a8a562 100644 --- a/libs/backend-apisix/src/index.ts +++ b/libs/backend-apisix/src/index.ts @@ -3,9 +3,11 @@ import axios, { Axios, CreateAxiosDefaults } from 'axios'; import { Listr, ListrTask } from 'listr2'; import { readFileSync } from 'node:fs'; import { AgentOptions, Agent as httpsAgent } from 'node:https'; +import semver from 'semver'; import { Fetcher } from './fetcher'; import { Operator } from './operator'; +import { buildReqAndRespDebugOutput } from './utils'; export class BackendAPISIX implements ADCSDK.Backend { private readonly client: Axios; @@ -45,6 +47,24 @@ export class BackendAPISIX implements ADCSDK.Backend { await this.client.get(`/apisix/admin/routes`); } + private getAPISIXVersionTask(): ListrTask { + return { + enabled: (ctx) => !ctx.apisixVersion, + task: async (ctx, task) => { + const resp = await this.client.get<{ value: string }>( + '/apisix/admin/routes', + ); + task.output = buildReqAndRespDebugOutput(resp, `Get APISIX version`); + + ctx.apisixVersion = semver.coerce('0.0.0'); + if (resp.headers.server) { + const version = (resp.headers.server as string).match(/APISIX\/(.*)/); + if (version) ctx.apisixVersion = semver.coerce(version[1]); + } + }, + }; + } + public getResourceDefaultValueTask(): Array { return []; } @@ -52,7 +72,11 @@ export class BackendAPISIX implements ADCSDK.Backend { public async dump(): Promise> { const fetcher = new Fetcher(this.client); return new Listr( - [...this.getResourceDefaultValueTask(), ...fetcher.fetch()], + [ + this.getAPISIXVersionTask(), + ...this.getResourceDefaultValueTask(), + ...fetcher.fetch(), + ], { rendererOptions: { scope: BackendAPISIX.logScope }, }, @@ -63,6 +87,7 @@ export class BackendAPISIX implements ADCSDK.Backend { const operator = new Operator(this.client); return new Listr( [ + this.getAPISIXVersionTask(), ...this.getResourceDefaultValueTask(), { task: (ctx, task) => diff --git a/libs/backend-apisix/src/operator.ts b/libs/backend-apisix/src/operator.ts index 6025f480..da1d3f1d 100644 --- a/libs/backend-apisix/src/operator.ts +++ b/libs/backend-apisix/src/operator.ts @@ -1,6 +1,7 @@ import * as ADCSDK from '@api7/adc-sdk'; import { Axios } from 'axios'; import { ListrTask } from 'listr2'; +import { SemVer, lt as semVerLT } from 'semver'; import { FromADC } from './transformer'; import * as typing from './typing'; @@ -14,6 +15,8 @@ export interface OperateContext { diff: Array; gatewayGroupId: string; needPublishServices: Record; + + apisixVersion: SemVer; } type OperateTask = ListrTask; @@ -24,17 +27,31 @@ export class Operator { return { title: this.generateTaskName(event), task: async (ctx, task) => { - const resp = await this.client.put( - `/apisix/admin/${resourceTypeToAPIName(event.resourceType)}/${event.resourceId}`, - this.fromADC(event), - { - validateStatus: () => true, - }, - ); - task.output = buildReqAndRespDebugOutput(resp); + if (event.resourceType === ADCSDK.ResourceType.CONSUMER_CREDENTIAL) { + if (semVerLT(ctx.apisixVersion, '3.11.0')) return; + + const resp = await this.client.put( + `/apisix/admin/consumers/${event.parentId}/credentials/${event.resourceId}`, + this.fromADC(event), + { + validateStatus: () => true, + }, + ); + task.output = buildReqAndRespDebugOutput(resp); + + if (resp.data?.error_msg) throw new Error(resp.data.error_msg); + } else { + const resp = await this.client.put( + `/apisix/admin/${resourceTypeToAPIName(event.resourceType)}/${event.resourceId}`, + this.fromADC(event), + { + validateStatus: () => true, + }, + ); + task.output = buildReqAndRespDebugOutput(resp); - if (resp.data?.error_msg) throw new Error(resp.data.error_msg); - // [200, 201].includes(resp.status); + if (resp.data?.error_msg) throw new Error(resp.data.error_msg); + } }, }; } @@ -43,22 +60,38 @@ export class Operator { return { title: this.generateTaskName(event), task: async (ctx, task) => { - const resp = await this.client.delete( - `/apisix/admin/${resourceTypeToAPIName(event.resourceType)}/${event.resourceId}`, - { - validateStatus: () => true, - }, - ); - task.output = buildReqAndRespDebugOutput(resp); + if (event.resourceType === ADCSDK.ResourceType.CONSUMER_CREDENTIAL) { + const resp = await this.client.delete( + `/apisix/admin/consumers/${event.parentId}/credentials/${event.resourceId}`, + { + validateStatus: () => true, + }, + ); + task.output = buildReqAndRespDebugOutput(resp); - // If the resource does not exist, it is not an error for the delete operation - if (resp.status === 404) return; - if (resp.data?.error_msg) throw new Error(resp.data.error_msg); - if (resp.data?.deleted <= 0) - throw new Error( - `Unexpected number of deletions of resources: ${resp.data?.deleted}`, + if (resp.status === 404) return; + if (resp.data?.error_msg) throw new Error(resp.data.error_msg); + if (resp.data?.deleted <= 0) + throw new Error( + `Unexpected number of deletions of resources: ${resp.data?.deleted}`, + ); + } else { + const resp = await this.client.delete( + `/apisix/admin/${resourceTypeToAPIName(event.resourceType)}/${event.resourceId}`, + { + validateStatus: () => true, + }, ); - // [200, 404].includes(resp.status); + task.output = buildReqAndRespDebugOutput(resp); + + // If the resource does not exist, it is not an error for the delete operation + if (resp.status === 404) return; + if (resp.data?.error_msg) throw new Error(resp.data.error_msg); + if (resp.data?.deleted <= 0) + throw new Error( + `Unexpected number of deletions of resources: ${resp.data?.deleted}`, + ); + } }, }; } @@ -78,6 +111,10 @@ export class Operator { return fromADC.transformConsumerGroup( event.newValue as ADCSDK.ConsumerGroup, )[0]; + case ADCSDK.ResourceType.CONSUMER_CREDENTIAL: + return fromADC.transformConsumerCredential( + event.newValue as ADCSDK.ConsumerCredential, + ); case ADCSDK.ResourceType.GLOBAL_RULE: return { plugins: { diff --git a/libs/backend-apisix/src/transformer.ts b/libs/backend-apisix/src/transformer.ts index 86ab68ad..d49f24c2 100644 --- a/libs/backend-apisix/src/transformer.ts +++ b/libs/backend-apisix/src/transformer.ts @@ -63,6 +63,31 @@ export class ToADC { plugins: consumer.plugins, group_id: !removeGroupId ? consumer.group_id : undefined, + + credentials: consumer.credentials + ?.map((item) => this.transformConsumerCredential(item)) + .filter((item) => !!item), + } as ADCSDK.Consumer); + } + + public transformConsumerCredential( + credential: typing.ConsumerCredential, + ): ADCSDK.ConsumerCredential { + if (!credential.plugins || Object.keys(credential.plugins).length <= 0) + return; + + const [pluginName, config] = Object.entries(credential.plugins)[0]; + if ( + !['key-auth', 'basic-auth', 'jwt-auth', 'hmac-auth'].includes(pluginName) + ) + return; + return ADCSDK.utils.recursiveOmitUndefined({ + name: credential.name, + description: credential.desc, + labels: credential.labels, + type: pluginName as ADCSDK.ConsumerCredential['type'], + config, + metadata: { id: credential.id }, }); } @@ -258,12 +283,24 @@ export class FromADC { public transformConsumer(consumer: ADCSDK.Consumer): typing.Consumer { return ADCSDK.utils.recursiveOmitUndefined({ - ...consumer, - id: undefined, + username: consumer.username, + desc: consumer.description, labels: FromADC.transformLabels(consumer.labels), - desc: consumer.description, - description: undefined, + plugins: consumer.plugins, + } as typing.Consumer); + } + + public transformConsumerCredential( + credential: ADCSDK.ConsumerCredential, + ): typing.ConsumerCredential { + return ADCSDK.utils.recursiveOmitUndefined({ + name: credential.name, + desc: credential.description, + labels: FromADC.transformLabels(credential.labels), + plugins: { + [credential.type]: credential.config, + }, }); } diff --git a/libs/backend-apisix/src/typing.ts b/libs/backend-apisix/src/typing.ts index d75122f4..69a60675 100644 --- a/libs/backend-apisix/src/typing.ts +++ b/libs/backend-apisix/src/typing.ts @@ -59,6 +59,14 @@ export interface Service { script?: string; enable_websocket?: boolean; } +export interface ConsumerCredential { + id?: string; + name: string; + desc?: string; + labels?: Labels; + + plugins?: Plugins; +} export interface Consumer { username: string; desc?: string; @@ -66,6 +74,7 @@ export interface Consumer { group_id?: string; plugins?: Plugins; + credentials?: Array; } export interface SSL { id: string; diff --git a/libs/backend-apisix/src/utils.ts b/libs/backend-apisix/src/utils.ts index 21410072..5a221f3c 100644 --- a/libs/backend-apisix/src/utils.ts +++ b/libs/backend-apisix/src/utils.ts @@ -1,10 +1,16 @@ import * as ADCSDK from '@api7/adc-sdk'; import axios, { AxiosResponse } from 'axios'; -export const resourceTypeToAPIName = (resourceType: ADCSDK.ResourceType) => - resourceType !== ADCSDK.ResourceType.PLUGIN_METADATA - ? `${resourceType}s` - : resourceType; +export const resourceTypeToAPIName = (resourceType: ADCSDK.ResourceType) => { + switch (resourceType) { + case ADCSDK.ResourceType.PLUGIN_METADATA: + return resourceType; + case ADCSDK.ResourceType.CONSUMER_CREDENTIAL: + return `consumers/%s/credentials`; + default: + return `${resourceType}s`; + } +}; export const capitalizeFirstLetter = (str: string) => str.charAt(0).toUpperCase() + str.slice(1);