From a7f16e6b0d6c9cf3b94f8c14f3989d525ca2cd7a Mon Sep 17 00:00:00 2001 From: Michael L Perry Date: Thu, 19 Dec 2024 07:01:12 -0600 Subject: [PATCH 1/8] Load rule sets --- index.ts | 33 ++++++++++++++++++-------- loadRuleSets.ts | 60 +++++++++++++++++++++++++++++++++++++++++++++++ package-lock.json | 8 +++---- package.json | 2 +- 4 files changed, 88 insertions(+), 15 deletions(-) create mode 100644 loadRuleSets.ts diff --git a/index.ts b/index.ts index 7ae8477..6dc5847 100644 --- a/index.ts +++ b/index.ts @@ -2,6 +2,7 @@ import express = require("express"); import * as http from "http"; import { JinagaServer } from "jinaga-server"; import process = require("process"); +import { loadRuleSets } from "./loadRuleSets"; process.on('SIGINT', () => { console.log("\n\nStopping replicator\n"); @@ -15,15 +16,27 @@ app.set('port', process.env.PORT || 8080); app.use(express.json()); app.use(express.text()); -const pgConnection = process.env.JINAGA_POSTGRESQL || - 'postgresql://repl:replpw@localhost:5432/replicator'; -const { handler } = JinagaServer.create({ - pgStore: pgConnection -}); +async function initializeReplicator() { + const pgConnection = process.env.JINAGA_POSTGRESQL || + 'postgresql://repl:replpw@localhost:5432/replicator'; + const ruleSetsPath = process.env.JINAGA_RULES || 'rules'; + const ruleSet = await loadRuleSets(ruleSetsPath); + const { handler } = JinagaServer.create({ + pgStore: pgConnection, + authorization: ruleSet ? a => ruleSet.authorizationRules : undefined, + distribution: ruleSet ? d => ruleSet.distributionRules : undefined, + purgeConditions: ruleSet ? p => ruleSet.purgeConditions : undefined + }); -app.use('/jinaga', handler); + app.use('/jinaga', handler); -server.listen(app.get('port'), () => { - console.log(` Replicator is running at http://localhost:${app.get('port')} in ${app.get('env')} mode`); - console.log(' Press CTRL-C to stop\n'); -}); + server.listen(app.get('port'), () => { + console.log(` Replicator is running at http://localhost:${app.get('port')} in ${app.get('env')} mode`); + console.log(' Press CTRL-C to stop\n'); + }); +} + +initializeReplicator() + .catch((error) => { + console.error("Error initializing replicator.", error); + }); \ No newline at end of file diff --git a/loadRuleSets.ts b/loadRuleSets.ts new file mode 100644 index 0000000..cc76fc5 --- /dev/null +++ b/loadRuleSets.ts @@ -0,0 +1,60 @@ +import { Dirent, readFile, readdir } from "fs"; +import { RuleSet } from "jinaga"; +import { join } from "path"; + +export async function loadRuleSets(path: string): Promise { + const fileNames = await findRuleFiles(path); + if (fileNames.length === 0) { + // No rules defined; apply no rules. + return undefined; + } + + let ruleSet = RuleSet.empty; + + for (const fileName of fileNames) { + const rules = await loadRuleSetFromFile(fileName); + ruleSet = ruleSet.merge(rules); + } + + return ruleSet; +} + +async function findRuleFiles(dir: string): Promise { + const fileNames: string[] = []; + + const entries = await new Promise((resolve, reject) => { + readdir(dir, { withFileTypes: true }, (err, files) => { + if (err) { + reject(err); + } else { + resolve(files); + } + }); + }); + + for (const entry of entries) { + const fullPath = join(dir, entry.name); + if (entry.isDirectory()) { + await findRuleFiles(fullPath); + } else if (entry.isFile() && entry.name.endsWith('.rule')) { + fileNames.push(fullPath); + } + } + + return fileNames; +} + +async function loadRuleSetFromFile(path: string): Promise { + const description = await new Promise((resolve, reject) => { + readFile(path, 'utf8', (err, data) => { + if (err) { + reject(err); + } else { + resolve(data); + } + }); + }); + + const ruleSet = RuleSet.loadFromDescription(description); + return ruleSet; +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 2a2d921..a2f5518 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "MIT", "dependencies": { "express": "^4.21.1", - "jinaga": "^6.3.0", + "jinaga": "^6.5.2", "jinaga-server": "^3.2.0", "pg": "^8.13.0" }, @@ -566,9 +566,9 @@ } }, "node_modules/jinaga": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/jinaga/-/jinaga-6.3.0.tgz", - "integrity": "sha512-3VFgi5X095YxABpFdL1shPC9SnVDWCcHwkR4xkevG0VY1vQj7U1Gn6cZ9gL4MbCHctIGh/EHKNsQkD2CO/s3lw==", + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/jinaga/-/jinaga-6.5.2.tgz", + "integrity": "sha512-GPv9dk6BYGbD4zFQNuvT6vAgpOqqJyTPmD5frhqjSIE6vCCNIsMCQnxRogmB1fnihDAjz83I+5aHusiW3RB65A==", "dependencies": { "@stablelib/base64": "^1.0.1", "@stablelib/sha512": "^1.0.1", diff --git a/package.json b/package.json index bc13daf..c1bd1ea 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "license": "MIT", "dependencies": { "express": "^4.21.1", - "jinaga": "^6.3.0", + "jinaga": "^6.5.2", "jinaga-server": "^3.2.0", "pg": "^8.13.0" }, From 49d0f222f5c49cd3a80c44491825582f4be14dd9 Mon Sep 17 00:00:00 2001 From: Michael L Perry Date: Thu, 19 Dec 2024 07:25:35 -0600 Subject: [PATCH 2/8] Mount rules folder as a volume --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index 2cded5d..fc1a2f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,4 +34,9 @@ COPY --from=build /replicator . COPY start.sh /usr/local/bin/start.sh +RUN mkdir -p /var/lib/replicator/rules +VOLUME /var/lib/replicator/rules + +ENV JINAGA_RULES /var/lib/replicator/rules + ENTRYPOINT [ "start.sh" ] \ No newline at end of file From 8da29217a3d9ad621a00f1bb09f12904f44f25f0 Mon Sep 17 00:00:00 2001 From: Michael L Perry Date: Thu, 19 Dec 2024 07:35:16 -0600 Subject: [PATCH 3/8] Update documentation to describe importing rule files --- README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/README.md b/README.md index 31173c1..394acae 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,36 @@ Run: docker run -it --rm -p8080:8080 jinaga/jinaga-replicator ``` +If you have rule files that you want to use with this image, you can mount a directory containing your rule files to the container's `/var/lib/replicator/rules` directory: + +```bash +docker run -it --rm -p8080:8080 -v /path/to/your/rules:/var/lib/replicator/rules jinaga/jinaga-replicator +``` + +Replace `/path/to/your/rules` with the path to the directory on your host machine that contains your rule files. + +### Using as a Base Image + +To use this image as a base image and copy your rule files into the `/var/lib/replicator/rules` directory, create a `Dockerfile` like this: + +```dockerfile +FROM jinaga/jinaga-replicator + +# Copy rule files into the /var/lib/replicator/rules directory +COPY *.rule /var/lib/replicator/rules/ + +# Ensure the rule files have the correct permissions +RUN chmod -R 755 /var/lib/replicator/rules +``` + +Build the new Docker image: + +```bash +docker build -t my-replicator-with-rules . +``` + +This will create a new Docker image named `my-replicator-with-rules` with the rule files included. + ## Release To release a new version of Jinaga replicator, bump the version number, push the tag, and let GitHub Actions do the rest. From a978647f2231cd4b5f211da195a86cd051288f3e Mon Sep 17 00:00:00 2001 From: Michael L Perry Date: Thu, 19 Dec 2024 08:23:55 -0600 Subject: [PATCH 4/8] Upgrade express to fix vulnerable regex dependency --- package-lock.json | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index a2f5518..f4501df 100644 --- a/package-lock.json +++ b/package-lock.json @@ -359,9 +359,9 @@ } }, "node_modules/express": { - "version": "4.21.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", - "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", + "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", "license": "MIT", "dependencies": { "accepts": "~1.3.8", @@ -383,7 +383,7 @@ "methods": "~1.1.2", "on-finished": "2.4.1", "parseurl": "~1.3.3", - "path-to-regexp": "0.1.10", + "path-to-regexp": "0.1.12", "proxy-addr": "~2.0.7", "qs": "6.13.0", "range-parser": "~1.2.1", @@ -398,6 +398,10 @@ }, "engines": { "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/finalhandler": { @@ -702,9 +706,9 @@ } }, "node_modules/path-to-regexp": { - "version": "0.1.10", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", - "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==", + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", + "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", "license": "MIT" }, "node_modules/pg": { From c4387c1a6ec4f55e1305ac3c014851d86d0d6793 Mon Sep 17 00:00:00 2001 From: Michael L Perry Date: Fri, 20 Dec 2024 07:14:15 -0600 Subject: [PATCH 5/8] Copy all TypeScript files --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fc1a2f2..6a212fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /replicator COPY package.json . COPY package-lock.json . COPY tsconfig.json . -COPY index.ts . +COPY *.ts . RUN npm ci RUN npm run build From 9074038bae9e5d2813589e724ae3e437c3f4710b Mon Sep 17 00:00:00 2001 From: Michael L Perry Date: Fri, 20 Dec 2024 20:35:48 -0600 Subject: [PATCH 6/8] Rename rules to policies --- Dockerfile | 6 +++--- README.md | 20 ++++++++++---------- index.ts | 6 +++--- loadRuleSets.ts => loadPolicies.ts | 12 ++++++------ 4 files changed, 22 insertions(+), 22 deletions(-) rename loadRuleSets.ts => loadPolicies.ts (79%) diff --git a/Dockerfile b/Dockerfile index 6a212fb..de477e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,9 +34,9 @@ COPY --from=build /replicator . COPY start.sh /usr/local/bin/start.sh -RUN mkdir -p /var/lib/replicator/rules -VOLUME /var/lib/replicator/rules +RUN mkdir -p /var/lib/replicator/policies +VOLUME /var/lib/replicator/policies -ENV JINAGA_RULES /var/lib/replicator/rules +ENV JINAGA_POLICIES /var/lib/replicator/policies ENTRYPOINT [ "start.sh" ] \ No newline at end of file diff --git a/README.md b/README.md index 394acae..7d9950c 100644 --- a/README.md +++ b/README.md @@ -38,35 +38,35 @@ Run: docker run -it --rm -p8080:8080 jinaga/jinaga-replicator ``` -If you have rule files that you want to use with this image, you can mount a directory containing your rule files to the container's `/var/lib/replicator/rules` directory: +If you have policy files that you want to use with this image, you can mount a directory containing your policy files to the container's `/var/lib/replicator/policies` directory: ```bash -docker run -it --rm -p8080:8080 -v /path/to/your/rules:/var/lib/replicator/rules jinaga/jinaga-replicator +docker run -it --rm -p8080:8080 -v /path/to/your/policies:/var/lib/replicator/policies jinaga/jinaga-replicator ``` -Replace `/path/to/your/rules` with the path to the directory on your host machine that contains your rule files. +Replace `/path/to/your/policies` with the path to the directory on your host machine that contains your policy files. ### Using as a Base Image -To use this image as a base image and copy your rule files into the `/var/lib/replicator/rules` directory, create a `Dockerfile` like this: +To use this image as a base image and copy your policy files into the `/var/lib/replicator/policies` directory, create a `Dockerfile` like this: ```dockerfile FROM jinaga/jinaga-replicator -# Copy rule files into the /var/lib/replicator/rules directory -COPY *.rule /var/lib/replicator/rules/ +# Copy policy files into the /var/lib/replicator/policies directory +COPY *.policy /var/lib/replicator/policies/ -# Ensure the rule files have the correct permissions -RUN chmod -R 755 /var/lib/replicator/rules +# Ensure the policy files have the correct permissions +RUN chmod -R 755 /var/lib/replicator/policies ``` Build the new Docker image: ```bash -docker build -t my-replicator-with-rules . +docker build -t my-replicator-with-policies . ``` -This will create a new Docker image named `my-replicator-with-rules` with the rule files included. +This will create a new Docker image named `my-replicator-with-policies` with the policy files included. ## Release diff --git a/index.ts b/index.ts index 6dc5847..87b5a72 100644 --- a/index.ts +++ b/index.ts @@ -2,7 +2,7 @@ import express = require("express"); import * as http from "http"; import { JinagaServer } from "jinaga-server"; import process = require("process"); -import { loadRuleSets } from "./loadRuleSets"; +import { loadPolicies } from "./loadPolicies"; process.on('SIGINT', () => { console.log("\n\nStopping replicator\n"); @@ -19,8 +19,8 @@ app.use(express.text()); async function initializeReplicator() { const pgConnection = process.env.JINAGA_POSTGRESQL || 'postgresql://repl:replpw@localhost:5432/replicator'; - const ruleSetsPath = process.env.JINAGA_RULES || 'rules'; - const ruleSet = await loadRuleSets(ruleSetsPath); + const policiesPath = process.env.JINAGA_POLICIES || 'policies'; + const ruleSet = await loadPolicies(policiesPath); const { handler } = JinagaServer.create({ pgStore: pgConnection, authorization: ruleSet ? a => ruleSet.authorizationRules : undefined, diff --git a/loadRuleSets.ts b/loadPolicies.ts similarity index 79% rename from loadRuleSets.ts rename to loadPolicies.ts index cc76fc5..c142cb4 100644 --- a/loadRuleSets.ts +++ b/loadPolicies.ts @@ -2,10 +2,10 @@ import { Dirent, readFile, readdir } from "fs"; import { RuleSet } from "jinaga"; import { join } from "path"; -export async function loadRuleSets(path: string): Promise { - const fileNames = await findRuleFiles(path); +export async function loadPolicies(path: string): Promise { + const fileNames = await findPolicyFiles(path); if (fileNames.length === 0) { - // No rules defined; apply no rules. + // Leave the replicator wide open return undefined; } @@ -19,7 +19,7 @@ export async function loadRuleSets(path: string): Promise { return ruleSet; } -async function findRuleFiles(dir: string): Promise { +async function findPolicyFiles(dir: string): Promise { const fileNames: string[] = []; const entries = await new Promise((resolve, reject) => { @@ -35,8 +35,8 @@ async function findRuleFiles(dir: string): Promise { for (const entry of entries) { const fullPath = join(dir, entry.name); if (entry.isDirectory()) { - await findRuleFiles(fullPath); - } else if (entry.isFile() && entry.name.endsWith('.rule')) { + await findPolicyFiles(fullPath); + } else if (entry.isFile() && entry.name.endsWith('.policy')) { fileNames.push(fullPath); } } From a1e21d4750689dedef0cfb3f5f159edd8dab6d43 Mon Sep 17 00:00:00 2001 From: Michael L Perry Date: Fri, 20 Dec 2024 20:48:50 -0600 Subject: [PATCH 7/8] Use a marker file to make no-security explicit --- index.ts | 2 +- loadPolicies.ts | 36 +++++++++++++++++++++++++++--------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/index.ts b/index.ts index 87b5a72..2ca7f0d 100644 --- a/index.ts +++ b/index.ts @@ -17,7 +17,7 @@ app.use(express.json()); app.use(express.text()); async function initializeReplicator() { - const pgConnection = process.env.JINAGA_POSTGRESQL || + const pgConnection = process.env.JINAGA_POSTGRESQL || 'postgresql://repl:replpw@localhost:5432/replicator'; const policiesPath = process.env.JINAGA_POLICIES || 'policies'; const ruleSet = await loadPolicies(policiesPath); diff --git a/loadPolicies.ts b/loadPolicies.ts index c142cb4..ab6d82f 100644 --- a/loadPolicies.ts +++ b/loadPolicies.ts @@ -2,16 +2,27 @@ import { Dirent, readFile, readdir } from "fs"; import { RuleSet } from "jinaga"; import { join } from "path"; +const MARKER_FILE_NAME = "no-security-policies"; + export async function loadPolicies(path: string): Promise { - const fileNames = await findPolicyFiles(path); - if (fileNames.length === 0) { + const { policyFiles, hasMarkerFile } = await findPolicyFiles(path); + + if (hasMarkerFile && policyFiles.length > 0) { + throw new Error(`Security policies are disabled, but there are policy files in ${path}.`); + } + + if (!hasMarkerFile && policyFiles.length === 0) { + throw new Error(`No security policies found in ${path}.`); + } + + if (policyFiles.length === 0) { // Leave the replicator wide open return undefined; } let ruleSet = RuleSet.empty; - for (const fileName of fileNames) { + for (const fileName of policyFiles) { const rules = await loadRuleSetFromFile(fileName); ruleSet = ruleSet.merge(rules); } @@ -19,8 +30,9 @@ export async function loadPolicies(path: string): Promise { return ruleSet; } -async function findPolicyFiles(dir: string): Promise { - const fileNames: string[] = []; +async function findPolicyFiles(dir: string): Promise<{ policyFiles: string[], hasMarkerFile: boolean }> { + const policyFiles: string[] = []; + let hasMarkerFile = false; const entries = await new Promise((resolve, reject) => { readdir(dir, { withFileTypes: true }, (err, files) => { @@ -35,13 +47,19 @@ async function findPolicyFiles(dir: string): Promise { for (const entry of entries) { const fullPath = join(dir, entry.name); if (entry.isDirectory()) { - await findPolicyFiles(fullPath); - } else if (entry.isFile() && entry.name.endsWith('.policy')) { - fileNames.push(fullPath); + const result = await findPolicyFiles(fullPath); + policyFiles.push(...result.policyFiles); + hasMarkerFile = hasMarkerFile || result.hasMarkerFile; + } else if (entry.isFile()) { + if (entry.name.endsWith('.policy')) { + policyFiles.push(fullPath); + } else if (entry.name === MARKER_FILE_NAME) { + hasMarkerFile = true; + } } } - return fileNames; + return { policyFiles, hasMarkerFile }; } async function loadRuleSetFromFile(path: string): Promise { From eccb3d407c04e52cc1240075328b9ffcd17bc50d Mon Sep 17 00:00:00 2001 From: Michael L Perry Date: Fri, 20 Dec 2024 21:06:54 -0600 Subject: [PATCH 8/8] Document configuring security policies --- README.md | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d9950c..f03a074 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,74 @@ docker build -t my-replicator-with-policies . This will create a new Docker image named `my-replicator-with-policies` with the policy files included. +## Security Policies + +Policies determine who is authorized to write facts, and to whom to distribute facts. +They also determine the conditions under which to purge facts. +Authorization, distribution, and purge rules are defined in policy files. + +### Policy Files + +Policy files have three sections: + +- **authorization**: Who is authorized to write facts. +- **distribution**: To whom to distribute facts. +- **purge**: Conditions under which to purge facts. + +Here is an example policy file: + +``` +authorization { + (post: Blog.Post) { + creator: Jinaga.User [ + creator = post->site: Blog.Site->creator: Jinaga.User + ] + } => creator + (deleted: Blog.Post.Deleted) { + creator: Jinaga.User [ + creator = deleted->post: Blog.Post->site: Blog.Site->creator: Jinaga.User + ] + } => creator +} +distribution { + share (user: Jinaga.User) { + name: Blog.User.Name [ + name->user: Jinaga.User = user + !E { + next: Blog.User.Name [ + next->prior: Blog.User.Name = name + ] + } + ] + } => name + with (user: Jinaga.User) { + self: Jinaga.User [ + self = user + ] + } => self +} +purge { + (post: Blog.Post) { + deleted: Blog.Post.Deleted [ + deleted->post: Blog.Post = post + ] + } => deleted +} +``` + +You can produce a policy file from .NET using the `dotnet jinaga` command line tool, or from JavaScript using the `jinaga` package. + +### No Security Policies + +To run a replicator with no security policies, create an empty `no-security-policies` file in the policy directory. + +```bash +touch /var/lib/replicator/policies/no-security-policies +``` + +This file opts-in to running the replicator with no security policies. +If the file is not present, the replicator will exit with an error message indicating that no security policies are found. + ## Release To release a new version of Jinaga replicator, bump the version number, push the tag, and let GitHub Actions do the rest. @@ -77,4 +145,4 @@ git checkout main git pull npm version patch git push --follow-tags -``` \ No newline at end of file +```