Initial decomposition pass on matter.js

This is a partial pass deconstructing the matter.js package.  I tried to stick to decisions we'd previously agreed to.

The new @project-chip/matter.js-general package contains those portions of matter.js that are not Matter specific.

The new @project-chip/matter.js-model package contains the Matter object model.

Since I was moving stuff around I got rid of the Uint8Array monkey patches we did in ByteArray.  For the most part this just consisted of replacing the "ByteArray" type with "Uint8Array".  The methods we injected I moved to a "Bytes" namespace containing utility functions.  This is a breaking change.

Following new conventions, the new packages don't have "matter.js" in their names, so they're just in "packages/general" and "packages/model".  I also renamed packages/matter.js-tools to packages/tools.  I did not rename the other directories yet.

I updated codegen and the intermediate model package dependencies, so codegen errors do not prevent codegen from running except for the generated model files in @project-chip/matter.js-model.

The new packages are smaller so I moved to a flat namespace for exports.  So there are no subpaths exported from the "general" package, even though most of the exports were previously exported as subpaths of matter.js.  I think this improves readability and makes consumption simpler.

"model" also has a flat namespace but this was true of the previous "@project-chip/matter.js/model" module so this isn't as big of a change.

I did my best to add exports to matter.js to support backward compatibility.  However I used a little editorial control to select which exports to reproduce at the old location...  I made sure to cover all of those we used previously in the examples.  Many of the other exports were probably not ever used externally.  Shouldn't be a big deal if I got it wrong; we can add additional exports or just inform users of the new locations.

Fixes project-chip#156

.gitignore

@@ -1,4 +1,6 @@
-/node_modules/
+node_modules/
+build/
+dist/
 .DS_Store
 .idea/
 /bin/

.prettierignore

@@ -2,7 +2,7 @@
 **/build/**
 models/src/chip.ts
 models/src/spec.ts
-packages/matter.js/src/model/standard/elements/*
+packages/model/src/standard/elements/*
 packages/matter.js/src/cluster/definitions/*
 packages/matter.js/src/cluster/globals/*
 packages/matter.js/src/endpoint/definitions/*

CHANGELOG.md

@@ -9,6 +9,15 @@ The main work (all changes without a GitHub username in brackets in the below li
 	## __WORK IN PROGRESS__
 -->
 
+## __WORK IN PROGRESS__
+
+-   IMPORTANT: As of 0.10.0 the @project-chip/matter.js module has grown quite large.  This release includes major refactoring that moves functional areas into independent NPM packages.  We have added exports to maintain backwards compatibility but these are not exhaustive.  In some cases you may need to update imports to reference new code locations.
+
+-   Matter-Core functionality:
+    -   BREAKING: The "ByteArray" type is removed, replaced with native-JS Uint8Array and a small collection of utility functions in the "Bytes" namespace
+    -   General functionality that is not Matter specific previously resided in @project-chip/matter.js.  It now lives in @project-chip/matter.js-general
+    -   The Matter object model previously exported as @project-chip/matter.js/model now resides in @project-chip/matter.js-model
+
 ## 0.10.1 (2024-09-08)
 
 -   Matter-Core functionality:

chip-testing/package.json

@@ -20,6 +20,7 @@
         "build-clean": "matter-build --clean"
     },
     "dependencies": {
+        "@project-chip/matter.js-general": "*",
         "@project-chip/matter-node.js": "*",
         "@project-chip/matter.js": "*",
         "@project-chip/matter.js-tools": "*"

chip-testing/src/AllClustersTestInstance.ts

@@ -4,6 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import { Bytes, Storage } from "@project-chip/matter.js-general";
 import { AdministratorCommissioningServer } from "@project-chip/matter.js/behavior/definitions/administrator-commissioning";
 import { BooleanStateServer } from "@project-chip/matter.js/behavior/definitions/boolean-state";
 import { CarbonDioxideConcentrationMeasurementServer } from "@project-chip/matter.js/behavior/definitions/carbon-dioxide-concentration-measurement";
@@ -62,9 +63,7 @@ import { OnOffLightDevice } from "@project-chip/matter.js/devices/OnOffLightDevi
 import { Endpoint } from "@project-chip/matter.js/endpoint";
 import { Environment, StorageService } from "@project-chip/matter.js/environment";
 import { ServerNode } from "@project-chip/matter.js/node";
-import { Storage } from "@project-chip/matter.js/storage";
 import { NumberTag } from "@project-chip/matter.js/tags/NumberTag";
-import { ByteArray } from "@project-chip/matter.js/util";
 import { TestActivatedCarbonFilterMonitoringServer } from "./cluster/TestActivatedCarbonFilterMonitoringServer.js";
 import { TestGeneralDiagnosticsServer } from "./cluster/TestGeneralDiagnosticsServer.js";
 import { TestHepaFilterMonitoringServer } from "./cluster/TestHEPAFilterMonitoringServer.js";
@@ -164,12 +163,12 @@ export class AllClustersTestInstance implements TestInstance {
     async setupServer(): Promise<ServerNode> {
         Environment.default.get(StorageService).factory = (_namespace: string) => this.storage;
 
-        const networkId = new ByteArray(32);
+        const networkId = new Uint8Array(32);
 
-        let deviceTestEnableKey = ByteArray.fromHex("00112233445566778899aabbccddeeff");
+        let deviceTestEnableKey = Bytes.fromHex("00112233445566778899aabbccddeeff");
         const argsEnableKeyIndex = process.argv.indexOf("--enable-key");
         if (argsEnableKeyIndex !== -1) {
-            deviceTestEnableKey = ByteArray.fromHex(process.argv[argsEnableKeyIndex + 1]);
+            deviceTestEnableKey = Bytes.fromHex(process.argv[argsEnableKeyIndex + 1]);
         }
 
         const serverNode = await ServerNode.create(

chip-testing/src/AllClustersTestInstanceLegacy.ts

@@ -6,7 +6,7 @@
 
 import { CommissioningServer, MatterServer } from "@project-chip/matter-node.js";
 
-import { Storage, StorageManager } from "@project-chip/matter.js/storage";
+import { Storage, StorageManager } from "@project-chip/matter.js-general";
 
 import {
     AdministratorCommissioning,
@@ -18,7 +18,6 @@ import {
 } from "@project-chip/matter.js/cluster";
 import { DeviceTypeId, EndpointNumber, VendorId } from "@project-chip/matter.js/datatype";
 import { OnOffLightDevice } from "@project-chip/matter.js/device";
-import { ByteArray } from "@project-chip/matter.js/util";
 import { TestInstance } from "./GenericTestApp.js";
 
 export class AllClustersTestInstanceLegacy implements TestInstance {
@@ -132,7 +131,7 @@ export class AllClustersTestInstanceLegacy implements TestInstance {
             ),
         );
 
-        const networkId = new ByteArray(32);
+        const networkId = new Uint8Array(32);
         commissioningServer.addRootClusterServer(
             ClusterServer(
                 NetworkCommissioning.Cluster.with("EthernetNetworkInterface"),

chip-testing/src/BridgeTestInstance.ts

@@ -4,6 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import { Storage } from "@project-chip/matter.js-general";
 import { AdministratorCommissioningServer } from "@project-chip/matter.js/behavior/definitions/administrator-commissioning";
 import { BridgedDeviceBasicInformationServer } from "@project-chip/matter.js/behavior/definitions/bridged-device-basic-information";
 import { NetworkCommissioningServer } from "@project-chip/matter.js/behavior/definitions/network-commissioning";
@@ -13,8 +14,6 @@ import { Endpoint } from "@project-chip/matter.js/endpoint";
 import { AggregatorEndpoint, DimmableLightDevice } from "@project-chip/matter.js/endpoint/definitions";
 import { Environment, StorageService } from "@project-chip/matter.js/environment";
 import { ServerNode } from "@project-chip/matter.js/node";
-import { Storage } from "@project-chip/matter.js/storage";
-import { ByteArray } from "@project-chip/matter.js/util";
 import { TestInstance } from "./GenericTestApp.js";
 
 export class BridgeTestInstance implements TestInstance {
@@ -89,7 +88,7 @@ export class BridgeTestInstance implements TestInstance {
     async setupServer(): Promise<ServerNode> {
         Environment.default.get(StorageService).factory = (_namespace: string) => this.storage;
 
-        const networkId = new ByteArray(32);
+        const networkId = new Uint8Array(32);
 
         const serverNode = await ServerNode.create(
             ServerNode.RootEndpoint.with(

chip-testing/src/BridgeTestInstanceLegacy.ts

@@ -15,7 +15,6 @@ import {
 } from "@project-chip/matter.js/cluster";
 import { DeviceTypeId, EndpointNumber, VendorId } from "@project-chip/matter.js/datatype";
 import { Aggregator, DimmableLightDevice } from "@project-chip/matter.js/device";
-import { ByteArray } from "@project-chip/matter.js/util";
 import { AllClustersTestInstanceLegacy } from "./AllClustersTestInstanceLegacy.js";
 
 export class BridgeTestInstanceLegacy extends AllClustersTestInstanceLegacy {
@@ -65,7 +64,7 @@ export class BridgeTestInstanceLegacy extends AllClustersTestInstanceLegacy {
             ),
         );
 
-        const networkId = new ByteArray(32);
+        const networkId = new Uint8Array(32);
         commissioningServer.addRootClusterServer(
             ClusterServer(
                 NetworkCommissioning.Cluster.with("EthernetNetworkInterface"),

chip-testing/src/GenericTestApp.ts

@@ -6,8 +6,8 @@
  */
 
 import { getIntParameter, getParameter, hasParameter } from "@project-chip/matter-node.js/util";
+import { ClassExtends } from "@project-chip/matter.js-general";
 import { Environment } from "@project-chip/matter.js/environment";
-import { ClassExtends } from "@project-chip/matter.js/util";
 import { StorageBackendAsyncJsonFile } from "./storage/StorageBackendAsyncJsonFile.js";
 import { StorageBackendSyncJsonFile } from "./storage/StorageBackendSyncJsonFile.js";
 

chip-testing/src/NamedPipeCommandHandler.ts

@@ -4,10 +4,10 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import { isObject } from "@project-chip/matter.js-general";
 import { SwitchServer } from "@project-chip/matter.js/behaviors/switch";
 import { Endpoint } from "@project-chip/matter.js/endpoint";
 import { ServerNode } from "@project-chip/matter.js/node";
-import { isObject } from "@project-chip/matter.js/util";
 import { execSync } from "child_process";
 import { constants } from "node:fs";
 import { FileHandle, open, unlink } from "node:fs/promises";

chip-testing/src/TvTestInstance.ts

@@ -4,6 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import { Bytes, Storage } from "@project-chip/matter.js-general";
 import { AdministratorCommissioningServer } from "@project-chip/matter.js/behavior/definitions/administrator-commissioning";
 import { ApplicationBasicServer } from "@project-chip/matter.js/behavior/definitions/application-basic";
 import { WakeOnLanServer } from "@project-chip/matter.js/behavior/definitions/wake-on-lan";
@@ -13,8 +14,6 @@ import { DimmableLightDevice } from "@project-chip/matter.js/devices/DimmableLig
 import { Endpoint } from "@project-chip/matter.js/endpoint";
 import { Environment, StorageService } from "@project-chip/matter.js/environment";
 import { ServerNode } from "@project-chip/matter.js/node";
-import { Storage } from "@project-chip/matter.js/storage";
-import { ByteArray } from "@project-chip/matter.js/util";
 import { TestInstance } from "./GenericTestApp.js";
 import { TestLowPowerServer } from "./cluster/TestLowPowerServer.js";
 
@@ -153,7 +152,7 @@ export class TvTestInstance implements TestInstance {
                 },
                 wakeOnLan: {
                     macAddress: "000000000000",
-                    linkLocalAddress: ByteArray.fromHex("00000000000000000000000000000000"),
+                    linkLocalAddress: Bytes.fromHex("00000000000000000000000000000000"),
                 },
             },
         );

chip-testing/src/cluster/TestGeneralDiagnosticsServer.ts

@@ -3,10 +3,10 @@
  * Copyright 2022-2024 Matter.js Authors
  * SPDX-License-Identifier: Apache-2.0
  */
+import { Logger } from "@project-chip/matter.js-general";
 import { GeneralDiagnosticsServer } from "@project-chip/matter.js/behavior/definitions/general-diagnostics";
 import { Status } from "@project-chip/matter.js/cluster";
 import { StatusResponseError } from "@project-chip/matter.js/interaction";
-import { Logger } from "@project-chip/matter.js/log";
 
 const logger = Logger.get("TestGeneralDiagnosticServer");
 

chip-testing/src/cluster/TestIdentifyServer.ts

@@ -3,9 +3,9 @@
  * Copyright 2022-2024 Matter.js Authors
  * SPDX-License-Identifier: Apache-2.0
  */
+import { Logger } from "@project-chip/matter.js-general";
 import { IdentifyServer } from "@project-chip/matter.js/behavior/definitions/identify";
 import { Identify } from "@project-chip/matter.js/cluster";
-import { Logger } from "@project-chip/matter.js/log";
 
 const logger = Logger.get("TestIdentifyServer");
 

chip-testing/src/cluster/TestLevelControlServer.ts

@@ -3,9 +3,9 @@
  * Copyright 2022-2024 Matter.js Authors
  * SPDX-License-Identifier: Apache-2.0
  */
+import { Logger } from "@project-chip/matter.js-general";
 import { LevelControlServer } from "@project-chip/matter.js/behavior/definitions/level-control";
 import { LevelControl } from "@project-chip/matter.js/cluster";
-import { Logger } from "@project-chip/matter.js/log";
 
 const logger = Logger.get("TestLevelControlServer");
 

chip-testing/src/cluster/TestLowPowerServer.ts

@@ -3,8 +3,8 @@
  * Copyright 2022-2024 Matter.js Authors
  * SPDX-License-Identifier: Apache-2.0
  */
+import { Logger } from "@project-chip/matter.js-general";
 import { LowPowerServer } from "@project-chip/matter.js/behavior/definitions/low-power";
-import { Logger } from "@project-chip/matter.js/log";
 
 const logger = Logger.get("TestLowPowerServer");
 

chip-testing/src/cluster/TestWindowCoveringServer.ts

@@ -3,14 +3,13 @@
  * Copyright 2022-2024 Matter.js Authors
  * SPDX-License-Identifier: Apache-2.0
  */
+import { Logger, Time, Timer } from "@project-chip/matter.js-general";
 import {
     MovementDirection,
     MovementType,
     WindowCoveringServer,
 } from "@project-chip/matter.js/behavior/definitions/window-covering";
 import { WindowCovering } from "@project-chip/matter.js/cluster";
-import { Logger } from "@project-chip/matter.js/log";
-import { Time, Timer } from "@project-chip/matter.js/time";
 
 const logger = Logger.get("TestWindowCoveringServer");
 

chip-testing/src/simulators/SwitchSimulator.ts

@@ -4,11 +4,11 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import { Time, Timer } from "@project-chip/matter.js-general";
 import { SwitchServer } from "@project-chip/matter.js/behaviors/switch";
 import { Switch } from "@project-chip/matter.js/cluster";
 import { Endpoint } from "@project-chip/matter.js/endpoint";
 import { BitFlag, BitmapSchema } from "@project-chip/matter.js/schema";
-import { Time, Timer } from "@project-chip/matter.js/time";
 import { SimulateLongPressCommand, SimulateMultiPressCommand } from "../NamedPipeCommands.js";
 
 const NEUTRAL_SWITCH_POSITION = 0;

chip-testing/src/storage/StorageBackendAsyncJsonFile.ts

@@ -4,14 +4,14 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { InternalError } from "@project-chip/matter.js/common";
 import {
+    InternalError,
     MaybeAsyncStorage,
     StorageBackendMemory,
     SupportedStorageTypes,
     fromJson,
     toJson,
-} from "@project-chip/matter.js/storage";
+} from "@project-chip/matter.js-general";
 import { readFile, writeFile } from "fs/promises";
 
 export class StorageBackendAsyncJsonFile extends MaybeAsyncStorage {

chip-testing/src/storage/StorageBackendSyncJsonFile.ts

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { fromJson, StorageBackendMemory, SupportedStorageTypes, toJson } from "@project-chip/matter.js/storage";
+import { fromJson, StorageBackendMemory, SupportedStorageTypes, toJson } from "@project-chip/matter.js-general";
 import { readFileSync, writeFileSync } from "fs";
 
 export class StorageBackendSyncJsonFile extends StorageBackendMemory {

chip-testing/src/tsconfig.json

@@ -1,5 +1,5 @@
 {
-    "extends": "../../packages/matter.js-tools/tsconfig.base.json",
+    "extends": "../../packages/tools/tsconfig.base.json",
     "compilerOptions": {
         "outDir": "../dist/cjs",
         "types": ["node"]

codegen/package.json

@@ -28,7 +28,8 @@
     },
     "homepage": "https://github.com/project-chip/matter.js#readme",
     "dependencies": {
-        "@project-chip/matter.js": "*",
+        "@project-chip/matter.js-general": "*",
+        "@project-chip/matter.js-model": "*",
         "@project-chip/matter.js-intermediate-models": "*"
     },
     "engines": {

codegen/src/clusters/ClusterComponentGenerator.ts

@@ -16,7 +16,7 @@ import {
     Model,
     NamedComponent,
     ValueModel,
-} from "@project-chip/matter.js/model";
+} from "@project-chip/matter.js-model";
 import { Block } from "../util/TsFile.js";
 import { camelize } from "../util/string.js";
 import { ClusterFile } from "./ClusterFile.js";
@@ -199,7 +199,7 @@ export class ClusterComponentGenerator {
     }
 
     private mapPrivilege(privilege: Access.Privilege) {
-        this.file.addImport("#/cluster/Cluster.js", "AccessLevel");
+        this.file.addImport("@project-chip/matter.js-model", "AccessLevel");
         return `AccessLevel.${Access.PrivilegeName[privilege]}`;
     }
 

codegen/src/clusters/ClusterFile.ts

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { ClusterModel } from "@project-chip/matter.js/model";
+import { ClusterModel } from "@project-chip/matter.js-model";
 import { ScopeFile } from "../util/ScopeFile.js";
 import { Block } from "../util/TsFile.js";
 

codegen/src/clusters/ConditionalElements.ts

@@ -11,7 +11,7 @@ import {
     Model,
     translateBitmap,
     VarianceCondition,
-} from "@project-chip/matter.js/model";
+} from "@project-chip/matter.js-model";
 import { serialize } from "../util/string.js";
 
 export type ConditionBitmap = {

codegen/src/clusters/DefaultValueGenerator.ts

@@ -4,9 +4,8 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { InternalError } from "@project-chip/matter.js/common";
-import { DefaultValue, Metatype, ValueModel } from "@project-chip/matter.js/model";
-import { Properties } from "@project-chip/matter.js/util";
+import { InternalError, Properties } from "@project-chip/matter.js-general";
+import { DefaultValue, Metatype, ValueModel } from "@project-chip/matter.js-model";
 import { camelize, serialize } from "../util/string.js";
 import { SpecializedNumbers, specializedNumberTypeFor } from "./NumberConstants.js";
 import { TlvGenerator } from "./TlvGenerator.js";

codegen/src/clusters/NumberConstants.ts

@@ -4,8 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import * as Elements from "@project-chip/matter.js/elements";
-import { Model } from "@project-chip/matter.js/model";
 import {
     FLOAT32_MAX,
     FLOAT32_MIN,
@@ -22,7 +20,9 @@ import {
     UINT32_MAX,
     UINT64_MAX,
     UINT8_MAX,
-} from "@project-chip/matter.js/util";
+} from "@project-chip/matter.js-general";
+import * as Elements from "@project-chip/matter.js-model";
+import { Model } from "@project-chip/matter.js-model";
 
 function special(type: string, category: "datatype" | "number" = "datatype") {
     return { type, category };

codegen/src/clusters/Scope.ts

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { InternalError } from "@project-chip/matter.js/common";
+import { InternalError } from "@project-chip/matter.js-general";
 import {
     ClusterModel,
     CommandModel,
@@ -16,7 +16,7 @@ import {
     Metatype,
     Model,
     ValueModel,
-} from "@project-chip/matter.js/model";
+} from "@project-chip/matter.js-model";
 import { camelize } from "../util/string.js";
 
 /**