feat(apisix): support consumer credentials (#200)

.github/workflows/e2e.yaml

@@ -10,8 +10,12 @@ on:
 jobs:
   apisix:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        version: [3.8.1, 3.9.1, 3.10.0, 3.11.0]
     env:
-      BACKEND_APISIX_VERSION: 3.9.1-debian
+      BACKEND_APISIX_VERSION: ${{ matrix.version }}
+      BACKEND_APISIX_IMAGE: ${{ matrix.version }}-debian
     steps:
       - uses: actions/checkout@v4
 

apps/cli/src/command/utils.ts

@@ -253,7 +253,7 @@ export const recursiveRemoveMetadataField = (c: ADCSDK.Configuration) => {
     if ('metadata' in obj) delete obj.metadata;
   };
   Object.entries(c).forEach(([key, value]) => {
-    if (['global_rules', 'plugin_metadata', 'consumers'].includes(key)) return;
+    if (['global_rules', 'plugin_metadata'].includes(key)) return;
     if (Array.isArray(value))
       value.forEach((item) => {
         removeMetadata(item);
@@ -265,6 +265,9 @@ export const recursiveRemoveMetadataField = (c: ADCSDK.Configuration) => {
         } else if (key === 'consumer_groups') {
           if ('consumers' in item && Array.isArray(item.consumers))
             item.consumers.forEach((c) => removeMetadata(c));
+        } else if (key === 'consumers') {
+          if ('credentials' in item && Array.isArray(item.credentials))
+            item.credentials.forEach((c) => removeMetadata(c));
         }
       });
   });

libs/backend-apisix/e2e/assets/docker-compose.yaml

@@ -1,6 +1,6 @@
 services:
   apisix_http:
-    image: apache/apisix:${BACKEND_APISIX_VERSION:-3.9.0-debian}
+    image: apache/apisix:${BACKEND_APISIX_IMAGE:-3.9.0-debian}
     restart: always
     volumes:
       - ./apisix_conf/http.yaml:/usr/local/apisix/conf/config.yaml:ro
@@ -13,7 +13,7 @@ services:
       apisix:
 
   apisix_mtls:
-    image: apache/apisix:${BACKEND_APISIX_VERSION:-3.9.0-debian}
+    image: apache/apisix:${BACKEND_APISIX_IMAGE:-3.9.0-debian}
     restart: always
     volumes:
       - ./apisix_conf/mtls.yaml:/usr/local/apisix/conf/config.yaml:ro

libs/backend-apisix/e2e/resources/consumer.e2e-spec.ts

@@ -0,0 +1,116 @@
+import * as ADCSDK from '@api7/adc-sdk';
+import { gte, lt } from 'semver';
+
+import { BackendAPISIX } from '../../src';
+import { server, token } from '../support/constants';
+import { conditionalDescribe, semverCondition } from '../support/utils';
+import {
+  createEvent,
+  deleteEvent,
+  dumpConfiguration,
+  syncEvents,
+  updateEvent,
+} from '../support/utils';
+
+describe('Consumer E2E', () => {
+  let backend: BackendAPISIX;
+
+  beforeAll(() => {
+    backend = new BackendAPISIX({
+      server,
+      token,
+      tlsSkipVerify: true,
+    });
+  });
+
+  conditionalDescribe(semverCondition(gte, '3.11.0'))(
+    'Sync and dump consumers (with credential support)',
+    () => {
+      const consumer1Name = 'consumer1';
+      const consumer1Key = 'consumer1-key';
+      const consumer1Cred = {
+        name: consumer1Key,
+        type: 'key-auth',
+        config: { key: consumer1Key },
+      };
+      const consumer1 = {
+        username: consumer1Name,
+        credentials: [consumer1Cred],
+      } as ADCSDK.Consumer;
+
+      it('Create consumers', async () =>
+        syncEvents(backend, [
+          createEvent(ADCSDK.ResourceType.CONSUMER, consumer1Name, consumer1),
+          createEvent(
+            ADCSDK.ResourceType.CONSUMER_CREDENTIAL,
+            consumer1Key,
+            consumer1Cred,
+            consumer1Name,
+          ),
+        ]));
+
+      it('Dump', async () => {
+        const result = (await dumpConfiguration(
+          backend,
+        )) as ADCSDK.Configuration;
+        expect(result.consumers).toHaveLength(1);
+        expect(result.consumers[0]).toMatchObject(consumer1);
+        expect(result.consumers[0].credentials).toMatchObject(
+          consumer1.credentials,
+        );
+      });
+
+      it('Update consumer credential', async () => {
+        consumer1.credentials[0].config.key = 'new-key';
+        await syncEvents(backend, [
+          updateEvent(
+            ADCSDK.ResourceType.CONSUMER_CREDENTIAL,
+            consumer1Key,
+            consumer1Cred,
+            consumer1Name,
+          ),
+        ]);
+      });
+
+      it('Dump again (consumer credential updated)', async () => {
+        const result = (await dumpConfiguration(
+          backend,
+        )) as ADCSDK.Configuration;
+        expect(result.consumers[0]).toMatchObject(consumer1);
+        expect(result.consumers[0].credentials[0].config.key).toEqual(
+          'new-key',
+        );
+      });
+
+      it('Delete consumer credential', async () =>
+        syncEvents(backend, [
+          deleteEvent(
+            ADCSDK.ResourceType.CONSUMER_CREDENTIAL,
+            consumer1Key,
+            consumer1Name,
+          ),
+        ]));
+
+      it('Dump again (consumer credential should not exist)', async () => {
+        const result = (await dumpConfiguration(
+          backend,
+        )) as ADCSDK.Configuration;
+        expect(result.consumers).toHaveLength(1);
+        console.log(result.consumers[0]);
+        expect(result.consumers[0].credentials).toBeUndefined();
+      });
+
+      it('Delete consumer', async () =>
+        syncEvents(backend, [
+          deleteEvent(ADCSDK.ResourceType.CONSUMER, consumer1Name),
+        ]));
+
+      it('Dump again (consumer should not exist)', async () => {
+        const result = (await dumpConfiguration(
+          backend,
+        )) as ADCSDK.Configuration;
+        expect(result.consumers).toHaveLength(0);
+      });
+    },
+  );
+});

libs/backend-apisix/e2e/support/utils.ts

@@ -1,5 +1,6 @@
 import * as ADCSDK from '@api7/adc-sdk';
 import { Listr, SilentRenderer } from 'listr2';
+import semver from 'semver';
 
 import { BackendAPISIX } from '../../src';
 
@@ -49,7 +50,11 @@ export const createEvent = (
             parentName ? `${parentName}.${resourceName}` : resourceName,
           ),
   newValue: resource,
-  parentId: parentName ? ADCSDK.utils.generateId(parentName) : undefined,
+  parentId: parentName
+    ? resourceType === ADCSDK.ResourceType.CONSUMER_CREDENTIAL
+      ? parentName
+      : ADCSDK.utils.generateId(parentName)
+    : undefined,
 });
 
 export const updateEvent = (
@@ -79,5 +84,22 @@ export const deleteEvent = (
       : ADCSDK.utils.generateId(
           parentName ? `${parentName}.${resourceName}` : resourceName,
         ),
-  parentId: parentName ? ADCSDK.utils.generateId(parentName) : undefined,
+  parentId: parentName
+    ? resourceType === ADCSDK.ResourceType.CONSUMER_CREDENTIAL
+      ? parentName
+      : ADCSDK.utils.generateId(parentName)
+    : undefined,
 });
+
+type cond = boolean | (() => boolean);
+
+export const conditionalDescribe = (cond: cond) =>
+  cond ? describe : describe.skip;
+
+export const conditionalIt = (cond: cond) => (cond ? it : it.skip);
+
+export const semverCondition = (
+  op: (v1: string | semver.SemVer, v2: string | semver.SemVer) => boolean,
+  base: string,
+  target = semver.coerce(process.env.BACKEND_APISIX_VERSION) ?? '0.0.0',
+) => op(target, base);

libs/backend-apisix/e2e/sync-and-dump-1.e2e-spec.ts

@@ -21,7 +21,6 @@ describe('Sync and Dump - 1', () => {
       server,
       token,
       tlsSkipVerify: true,
-      gatewayGroup: 'default',
     });
   });
 

libs/backend-apisix/src/fetcher.ts

@@ -1,13 +1,16 @@
 import * as ADCSDK from '@api7/adc-sdk';
 import { Axios } from 'axios';
 import { ListrTask } from 'listr2';
+import { SemVer, gte as semVerGTE } from 'semver';
 
 import { ToADC } from './transformer';
 import * as typing from './typing';
 import { buildReqAndRespDebugOutput, resourceTypeToAPIName } from './utils';
 
 type FetchTask = ListrTask<{
   remote: ADCSDK.Configuration;
+
+  apisixVersion: SemVer;
   apisixResources?: typing.Resources;
 }>;
 
@@ -65,7 +68,6 @@ export class Fetcher {
             )
               return;
 
-            // resourceType === ADCSDK.ResourceType.GLOBAL_RULE ||
             if (resourceType === ADCSDK.ResourceType.PLUGIN_METADATA) {
               ctx.apisixResources[ADCSDK.ResourceType.PLUGIN_METADATA] =
                 Object.fromEntries(
@@ -92,6 +94,35 @@ export class Fetcher {
                 (item) => item.value,
               );
             }
+
+            if (
+              resourceType === ADCSDK.ResourceType.CONSUMER &&
+              semVerGTE(ctx.apisixVersion, '3.11.0')
+            ) {
+              await Promise.all(
+                ctx.apisixResources[resourceType].map(async (item) => {
+                  const resp = await this.client.get<{
+                    list: Array<{
+                      key: string;
+                      value: typing.ConsumerCredential;
+                      createdIndex: number;
+                      modifiedIndex: number;
+                    }>;
+                    total: number;
+                  }>(`/apisix/admin/consumers/${item.username}/credentials`, {
+                    validateStatus: () => true,
+                  });
+                  task.output = buildReqAndRespDebugOutput(
+                    resp,
+                    `Get credentials of consumer "${item.username}"`,
+                  );
+                  if (resp.status === 200)
+                    item.credentials = resp.data.list.map(
+                      (credential) => credential.value,
+                    );
+                }),
+              );
+            }
           },
         }),
       );

libs/backend-apisix/src/index.ts

@@ -3,9 +3,11 @@ import axios, { Axios, CreateAxiosDefaults } from 'axios';
 import { Listr, ListrTask } from 'listr2';
 import { readFileSync } from 'node:fs';
 import { AgentOptions, Agent as httpsAgent } from 'node:https';
+import semver from 'semver';
 
 import { Fetcher } from './fetcher';
 import { Operator } from './operator';
+import { buildReqAndRespDebugOutput } from './utils';
 
 export class BackendAPISIX implements ADCSDK.Backend {
   private readonly client: Axios;
@@ -45,14 +47,36 @@ export class BackendAPISIX implements ADCSDK.Backend {
     await this.client.get(`/apisix/admin/routes`);
   }
 
+  private getAPISIXVersionTask(): ListrTask {
+    return {
+      enabled: (ctx) => !ctx.apisixVersion,
+      task: async (ctx, task) => {
+        const resp = await this.client.get<{ value: string }>(
+          '/apisix/admin/routes',
+        );
+        task.output = buildReqAndRespDebugOutput(resp, `Get APISIX version`);
+
+        ctx.apisixVersion = semver.coerce('0.0.0');
+        if (resp.headers.server) {
+          const version = (resp.headers.server as string).match(/APISIX\/(.*)/);
+          if (version) ctx.apisixVersion = semver.coerce(version[1]);
+        }
+      },
+    };
+  }
+
   public getResourceDefaultValueTask(): Array<ListrTask> {
     return [];
   }
 
   public async dump(): Promise<Listr<{ remote: ADCSDK.Configuration }>> {
     const fetcher = new Fetcher(this.client);
     return new Listr(
-      [...this.getResourceDefaultValueTask(), ...fetcher.fetch()],
+      [
+        this.getAPISIXVersionTask(),
+        ...this.getResourceDefaultValueTask(),
+        ...fetcher.fetch(),
+      ],
       {
         rendererOptions: { scope: BackendAPISIX.logScope },
       },
@@ -63,6 +87,7 @@ export class BackendAPISIX implements ADCSDK.Backend {
     const operator = new Operator(this.client);
     return new Listr(
       [
+        this.getAPISIXVersionTask(),
         ...this.getResourceDefaultValueTask(),
         {
           task: (ctx, task) =>