From b992be96a9abbc6e8a389e52fdc6a8dc6a1f3dda Mon Sep 17 00:00:00 2001 From: Ronnie Miller Date: Tue, 7 May 2024 21:53:50 -0700 Subject: [PATCH] Updates for unified build (#176) --------- Co-authored-by: Eric Schneider <37347760+eric-schneider@users.noreply.github.com> --- .github/workflows/dispatch-deploy-draft.yml | 45 ++++ docs/.gitignore | 15 ++ docs/README.adoc | 161 +++++++++++--- docs/docs-src/core/antora-cdc-cassandra.yml | 17 -- docs/docs-src/core/antora.yml | 18 +- docs/docs-src/core/modules/ROOT/nav.adoc | 1 + .../core/modules/ROOT/pages/index.adoc | 8 +- .../core/modules/ROOT/pages/install.adoc | 26 +-- .../core/modules/ROOT/pages/monitor.adoc | 4 +- .../ROOT/{pages => partials}/agentParams.adoc | 0 .../{pages => partials}/cfgCassandraAuth.adoc | 0 .../cfgCassandraJavaDriverSettings.adoc | 0 .../{pages => partials}/cfgCassandraSSL.adoc | 0 .../cfgCassandraSource.adoc | 0 docs/lib/remote-include-processor.js | 199 ++++++++++++++++++ docs/lib/svg-macro.js | 110 ++++++++++ docs/lib/tailwind-processor.js | 12 ++ docs/lib/unlisted-pages-extension.js | 41 ++++ docs/local-preview-playbook.yml | 73 +++++++ docs/package.json | 29 ++- docs/playbooks/site-local-cdc-cassandra.yaml | 51 ----- .../playbooks/site-publish-cdc-cassandra.yaml | 61 ------ 22 files changed, 685 insertions(+), 186 deletions(-) create mode 100644 .github/workflows/dispatch-deploy-draft.yml create mode 100644 docs/.gitignore delete mode 100644 docs/docs-src/core/antora-cdc-cassandra.yml mode change 120000 => 100644 docs/docs-src/core/antora.yml rename docs/docs-src/core/modules/ROOT/{pages => partials}/agentParams.adoc (100%) rename docs/docs-src/core/modules/ROOT/{pages => partials}/cfgCassandraAuth.adoc (100%) rename docs/docs-src/core/modules/ROOT/{pages => partials}/cfgCassandraJavaDriverSettings.adoc (100%) rename docs/docs-src/core/modules/ROOT/{pages => partials}/cfgCassandraSSL.adoc (100%) rename docs/docs-src/core/modules/ROOT/{pages => partials}/cfgCassandraSource.adoc (100%) create mode 100644 docs/lib/remote-include-processor.js create mode 100644 docs/lib/svg-macro.js create mode 100644 docs/lib/tailwind-processor.js create mode 100644 docs/lib/unlisted-pages-extension.js create mode 100644 docs/local-preview-playbook.yml delete mode 100644 docs/playbooks/site-local-cdc-cassandra.yaml delete mode 100644 docs/playbooks/site-publish-cdc-cassandra.yaml diff --git a/.github/workflows/dispatch-deploy-draft.yml b/.github/workflows/dispatch-deploy-draft.yml new file mode 100644 index 00000000..dbbbf7b8 --- /dev/null +++ b/.github/workflows/dispatch-deploy-draft.yml @@ -0,0 +1,45 @@ +name: Deploy Draft + +on: + pull_request: + branches: + - '*' + +jobs: + dispatch-deploy: + runs-on: ubuntu-latest + + steps: + # Determine the build branch and draft branch for dispatch. + - name: Determine Dispatch Parameters + run: | + if [ "${{ github.event_name }}" == "pull_request" ]; then + # If this workflow is kicked off by a pull request, build + # a draft using the pull request base branch and PR branch. + build_branch="${{ github.base_ref }}" + draft_branch="${{ github.event.pull_request.head.ref }}" + else + if [ "$(basename ${{ github.event.ref }})" == "stage" ]; then + # This was a merge to stage so kick off a build to update stage draft. + build_branch=stage + draft_branch=stage + else + # Otherwise this is a push to one of the source branches so + # dispatch a build for the main draft to pick up the changes. + build_branch=main + draft_branch=main + fi + fi + echo "build_branch=$build_branch" >> $GITHUB_OUTPUT + echo "draft_branch=$draft_branch" >> $GITHUB_OUTPUT + id: branches + + - name: Deploy Draft + uses: convictional/trigger-workflow-and-wait@v1.6.1 + with: + owner: riptano + repo: datastax-docs-site + github_token: ${{ secrets.DISPATCH_GITHUB_TOKEN }} + github_user: ${{ secrets.DISPATCH_GITHUB_USER }} + workflow_file_name: deploy-draft.yml + client_payload: '{ "build_repository": "${{ github.event.repository.full_name }}", "build_branch": "${{ steps.branches.outputs.build_branch }}", "draft_branch": "${{ steps.branches.outputs.draft_branch }}", "pull_request_number": "${{ github.event.pull_request.number }}" }' diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..5f28c239 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,15 @@ +#OS +**/.DS_Store +**/.git-credentials + +#IDE +**/.vscode/ +**/.idea +**/*.iml +*.http +**/.java-version + +#build +**/package-lock.json +**/node_modules/ +/build/ diff --git a/docs/README.adoc b/docs/README.adoc index 423138bf..9ae4e57f 100644 --- a/docs/README.adoc +++ b/docs/README.adoc @@ -1,50 +1,159 @@ -= DataStax CDC for Apache Cassandra documentation += {company} {product} Docs +// Variables: +:company: DataStax +:product: CDC for Apache Cassandra +:repo-name: cdc-apache-cassandra +:github-org: datastax +// Settings: +:toc: macro +:!example-caption: +:experimental: +:hide-uri-scheme: +ifdef::env-github[] +:icons: font +:toclevels: 1 +:toc-title: Contents +:tip-caption: :bulb: +:note-caption: :information_source: +:important-caption: :heavy_exclamation_mark: +:caution-caption: :fire: +:warning-caption: :warning: +:badges: +endif::[] +// Project URLs: +:url-github-org: https://github.com/{github-org} +:url-project-repo: {url-github-org}/{repo-name} +:url-ui-repo: https://github.com/riptano/docs-ui +:url-playbook-repo: https://github.com/riptano/datastax-docs-site +:url-contribute: +:url-datastax: https://datastax.com +:url-datastax-docs: https://docs.datastax.com +:url-docs-preview: http://docs-preview.datastax.com +// External URLs: +:asciidoc-language: https://docs.asciidoctor.org/asciidoc/latest/ -This repo contains the source files for the {csc_pulsar_first} documentation. +This repository contains the source files for the {company} {product} documentation. -The docs are written in asciidoc and use Antora to generate the output. +toc::[] -== Dependencies +== Get started -Antora requires NodeJS. Install NodeJS, then update project dependencies: +The documentation is written in {asciidoc-language}[AsciiDoc]-formatted source files located in the `modules` directory. -[source,bash] ----- -brew install node ----- +=== Make a simple update -Install Antora: +For simple updates like fixing typos or modifying existing prose, it's easiest to edit the source files directly on GitHub. -[source,bash] ----- -npm i -g @antora/cli@2.3 @antora/site-generator-default@2.3 ----- +NOTE: You'll need Write privileges on the repository to edit files directly on GitHub. -== Create the antora config file +. Find the file you want to edit in the `modules` directory. -Generate the antory.yaml file using the project version: +. Click the *Edit* icon in the upper-right corner of the file view. -[source,bash] +. Make your changes in the editor. + +. Click *Commit changes...* + +. Enter a description for your commit and click *Propose changes*. + +. On the *Open a pull request* screen, enter a title and description for your change, assign reviewers, then click *Create pull request*. + +. Once the pull request is open, an automatic draft preview build is triggered. +Once complete, the build system posts a comment on the pull request with a link to the draft site for you to preview your changes. + +=== Edit docs locally + +If you need to make substantial updates to the documentation, you'll want to clone the repository so you can work with the source files locally. + +. Clone this repository ++ +[source,bash,subs="attributes"] ---- -./gradlew docs:antoraConfig +git clone {url-project-repo}.git ---- -== Generating and viewing the HTML output locally +. https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-personal-access-token-classic[Create a _classic_ personal access token] for your GitHub account. +When configuring the token, set the *Expiration* to at least 90 days and select everything under the *Repo* https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes[scope]. ++ +[IMPORTANT] +==== +Copy your personal access token to a temporary location -- you'll need it later. +==== + +. https://docs.github.com/en/enterprise-cloud@latest/authentication/authenticating-with-saml-single-sign-on/authorizing-a-personal-access-token-for-use-with-saml-single-sign-on[Authorize your personal access token] so that it can access repositories in the Riptano and DataStax organizations in GitHub. + +. https://docs.antora.org/antora/latest/playbook/private-repository-auth/#populate-credentials-directly[Populate the credential store] with your personal access token. +For most people this means doing the following: ++ +.. Create the file `$HOME/.git-credentials` and open it in your editor. +.. Add the following line: ++ +[source,subs="verbatim,quotes"] +---- +https://**TOKEN**:@github.com +---- ++ +Replace *`TOKEN`* with the personal access token you copied from GitHub. +.. Save and close the file. -The docs can be generated locally during development, to check work. +. If you don't already have Node.js installed, do the following: +.. Install https://github.com/nvm-sh/nvm[nvm]. ++ +If you're on macOS, you can install nvm using https://brew.sh/[Homebrew]: ++ [source,bash] ---- -npm run build:local +brew install nvm ---- -Output files are located in the build/site directory. - -== Publishing the HTML output +.. Use nvm to install Node.js. ++ +[source,bash] +---- +nvm install --lts +---- ++ +[source,bash] +---- +nvm use --lts +---- ++ +[source,bash] +---- +nvm alias default node +---- -To generate files for publishing: +. Install the project dependencies. ++ +[source,bash,subs="attributes"] +---- +cd {repo-name}/docs +---- ++ +[source,bash] +---- +npm install +---- +. Build the site. ++ [source,bash] ---- -npm run build:publish +npm run build:local ---- ++ +If the build was successful, you'll see the following output in your terminal: ++ +[source,console,subs="attributes"] +---- +Site generation complete! +Open file:///Users/USERNAME/repos/{repo-name}/build/site/index.html in a browser to view your site. +---- ++ +To view the site, paste the entire `\file:///` path into your browser's address bar and press kbd:[Return]. + +[#publish-docs] +== Publish docs + +To learn how to publish documentation to {url-datastax-docs}, see the {url-playbook-repo}#deploy-production[datastax-docs-site README]. diff --git a/docs/docs-src/core/antora-cdc-cassandra.yml b/docs/docs-src/core/antora-cdc-cassandra.yml deleted file mode 100644 index 2c5c4db8..00000000 --- a/docs/docs-src/core/antora-cdc-cassandra.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: docs -title: DataStax CDC for Apache Cassandra -version: '2.2.9' -display_version: '2.2.9' -start_page: index.adoc -asciidoc: - attributes: - cdc_cass_first: 'DataStax CDC for Apache Cassandra®' - cdc_cass: 'CDC for Cassandra' - csc_pulsar_first: 'DataStax Cassandra Source Connector for Apache Pulsar™' - csc_pulsar: 'CSC for Pulsar' - cdc_pulsar: 'CDC for Cassandra' - luna_version: '2.10' - pulsar_version: '2.10' - version: '2.2.9' # cdc-apache-cassandra latest {version} -nav: -- modules/ROOT/nav.adoc diff --git a/docs/docs-src/core/antora.yml b/docs/docs-src/core/antora.yml deleted file mode 120000 index cd383fcf..00000000 --- a/docs/docs-src/core/antora.yml +++ /dev/null @@ -1 +0,0 @@ -antora-cdc-cassandra.yml \ No newline at end of file diff --git a/docs/docs-src/core/antora.yml b/docs/docs-src/core/antora.yml new file mode 100644 index 00000000..511924ed --- /dev/null +++ b/docs/docs-src/core/antora.yml @@ -0,0 +1,17 @@ +name: cdc-for-cassandra +title: DataStax CDC for Apache Cassandra +version: '2.2.9' +display_version: '2.2.9' +start_page: index.adoc +asciidoc: + attributes: + cdc_cass_first: 'DataStax CDC for Apache Cassandra(R)' + cdc_cass: 'CDC for Cassandra' + csc_pulsar_first: 'DataStax Cassandra Source Connector for Apache Pulsar(TM)' + csc_pulsar: 'CSC for Pulsar' + cdc_pulsar: 'CDC for Cassandra' + luna_version: '2.10' + pulsar_version: '2.10' + version: '2.2.9' # cdc-apache-cassandra latest {version} +nav: +- modules/ROOT/nav.adoc diff --git a/docs/docs-src/core/modules/ROOT/nav.adoc b/docs/docs-src/core/modules/ROOT/nav.adoc index 7308ad5e..8e32bbca 100644 --- a/docs/docs-src/core/modules/ROOT/nav.adoc +++ b/docs/docs-src/core/modules/ROOT/nav.adoc @@ -4,4 +4,5 @@ * xref:backfill-cli.adoc[] * xref:faqs.adoc[] * xref:cdc-cassandra-events.adoc[] +* xref:stringMappings.adoc[] * xref:cdcExample.adoc[] diff --git a/docs/docs-src/core/modules/ROOT/pages/index.adoc b/docs/docs-src/core/modules/ROOT/pages/index.adoc index 3747e03f..aeae47bb 100644 --- a/docs/docs-src/core/modules/ROOT/pages/index.adoc +++ b/docs/docs-src/core/modules/ROOT/pages/index.adoc @@ -1,7 +1,7 @@ = About {cdc_cass} {cdc_cass_first} is open-source software (OSS) that sends Cassandra mutations -for tables having Change Data Capture (CDC) enabled to https://www.datastax.com/products/luna-streaming[Luna Streaming] or https://pulsar.apache.org/[Apache Pulsar™], which in turn can write the data to platforms such as Elasticsearch® or Snowflake®. +for tables having Change Data Capture (CDC) enabled to https://www.datastax.com/products/luna-streaming[Luna Streaming] or https://pulsar.apache.org/[Apache Pulsar(TM)], which in turn can write the data to platforms such as Elasticsearch(R) or Snowflake(R). == Key Features @@ -85,12 +85,12 @@ For each update to the table, an MD5 digest is calculated to de-duplicate the up [#supported-databases] == Supported databases -* Apache Cassandra® 3.11.x and 4.x databases +* Apache Cassandra(R) 3.11.x and 4.x databases * Datastax Server Enterprise 6.8.16+ == Supported Cassandra data structures -The following CQL data types are encoded as AVRO logical types: +The following CQL data types are encoded as AVRO logical types: * ascii (string) * bigint (long) @@ -134,7 +134,7 @@ If a row update contains both supported and unsupported data types, the event wi {cdc_cass} has the following limitations: -* Does not manage table truncates. The `TRUNCATE [TABLE]` command should not be used. +* Does not manage table truncates. The `TRUNCATE [TABLE]` command should not be used. * Does not sync data available before starting the CDC agent. * Does not replay logged batches. * Does not manage time-to-live. diff --git a/docs/docs-src/core/modules/ROOT/pages/install.adoc b/docs/docs-src/core/modules/ROOT/pages/install.adoc index 8f112af3..5a19a138 100644 --- a/docs/docs-src/core/modules/ROOT/pages/install.adoc +++ b/docs/docs-src/core/modules/ROOT/pages/install.adoc @@ -1,6 +1,6 @@ = Installing {cdc_cass} for VM deployment -== Download the DataStax Change Data Capture (CDC) Agent for Apache Cassandra® +== Download the DataStax Change Data Capture (CDC) Agent for Apache Cassandra(R) [IMPORTANT] ==== @@ -58,7 +58,7 @@ JVM_OPTS="$JVM_OPTS -javaagent:/home/automaton/cdc104/agent-dse4-pulsar-1.0.5-al ---- The CDC parameter mappings between JVM and C* environment variables are provided in xref:stringMappings.adoc[CDC Environment Parameter Strings]. + -For the full set of JVM configuration options, see xref:agentParams.adoc[Table Change Agent Parameters]. +For the full set of JVM configuration options, see xref:install.adoc#agentParams[]. == `Cassandra.yaml` In addition to configuring the change agent, enable CDC on the Cassandra node by setting `cdc_enabled` to `true` in the `cassandra.yaml` file. @@ -85,7 +85,7 @@ cdc_total_space_in_mb: 50000 [#agentParams] == Change Agent Parameters -include::agentParams.adoc[] +include::partial$agentParams.adoc[] == Download {cdc_pulsar} IMPORTANT @@ -99,7 +99,7 @@ The following files are available: [cols="1"] |=== -| Streaming platform | NAR file +| Streaming platform | NAR file | Apache Pulsar 2.8 and Luna Streaming 2.8 | pulsar-cassandra-source-.nar @@ -166,10 +166,10 @@ pulsar-admin source status --name cassandra-source-1 Once the connector is running, it will process events from the `events` topic and publish the result to the `data` topic. -For the full set of source configuration options, see xref:cfgCassandraSource.adoc[{cdc_pulsar} settings]. -For the full set of Cassandra authentication options, see xref:cfgCassandraAuth.adoc[Cassandra Authentication settings]. -For the full set of Cassandra SSL settings, see xref:cfgCassandraSSL.adoc[Cassandra SSL/TLS settings]. -For advanced configuration of the Cassandra driver in the {cdc_pulsar}, see xref:cfgCassandraJavaDriverSettings.adoc[Pass {cdc_pulsar} settings directly to the DataStax Java driver]. +For the full set of source configuration options, see xref:install.adoc#datastax-cassandra-source-connector-for-apache-pulsar-settings[{cdc_pulsar} settings]. +For the full set of Cassandra authentication options, see xref:install.adoc#cassandra-authentication-settings[Cassandra Authentication settings]. +For the full set of Cassandra SSL settings, see xref:install.adoc#cassandra-ssltls-settings[Cassandra SSL/TLS settings]. +For advanced configuration of the Cassandra driver in the {cdc_pulsar}, see xref:install.adoc#pass-cdc-for-cassandra-settings-directly-to-the-datastax-java-driver[Pass {cdc_pulsar} settings directly to the DataStax Java driver]. == Enabling and disabling CDC on a table @@ -184,13 +184,13 @@ ALTER TABLE foo WITH cdc=false; When CDC is enabled on a table, updates to that table are sent by the change agent to the {cdc_pulsar} which further processes the event and then sends it to the data topic when it can be processed by other connectors (for example, Elasticsearch). -include::cfgCassandraSource.adoc[] +include::partial$cfgCassandraSource.adoc[] -include::cfgCassandraAuth.adoc[] +include::partial$cfgCassandraAuth.adoc[] -include::cfgCassandraSSL.adoc[] +include::partial$cfgCassandraSSL.adoc[] -include::cfgCassandraJavaDriverSettings.adoc[] +include::partial$cfgCassandraJavaDriverSettings.adoc[] == Scaling up your configuration @@ -206,4 +206,4 @@ To further improve the throughput, you can adjust the `pulsarBatchDelayInMs` in To improve performance on individual connector instances as they read data from Cassandra, you can adjust the `batch.size` and the `query.executors`. Increasing these values from their defaults will increase parallelism within the connector instances. -The de-duplication cache is configurable, including the cache size with `cache.max.capacity`, the entry retention duration `cache.expire.after.ms` and the number of MD5 digest per primary key entry with `cache.max.digest`. \ No newline at end of file +The de-duplication cache is configurable, including the cache size with `cache.max.capacity`, the entry retention duration `cache.expire.after.ms` and the number of MD5 digest per primary key entry with `cache.max.digest`. diff --git a/docs/docs-src/core/modules/ROOT/pages/monitor.adoc b/docs/docs-src/core/modules/ROOT/pages/monitor.adoc index dc39a5d6..4d2b0844 100644 --- a/docs/docs-src/core/modules/ROOT/pages/monitor.adoc +++ b/docs/docs-src/core/modules/ROOT/pages/monitor.adoc @@ -2,7 +2,7 @@ == Change Agent Metrics -The change agent is a JVM agent running in Apache Cassandra® nodes and provides a dedicated MBean type=`CdcAgent` with the following metrics: +The change agent is a JVM agent running in Apache Cassandra(R) nodes and provides a dedicated MBean type=`CdcAgent` with the following metrics: [cols="2,1,3"] |=== @@ -116,7 +116,7 @@ A description of the last seen error is displayed in the `error` field. |=== -Here an example of those user-defined metrics aggregated by Apache Pulsar™ when processing 2000 mutations: +Here an example of those user-defined metrics aggregated by Apache Pulsar(TM) when processing 2000 mutations: [source,bash] ---- diff --git a/docs/docs-src/core/modules/ROOT/pages/agentParams.adoc b/docs/docs-src/core/modules/ROOT/partials/agentParams.adoc similarity index 100% rename from docs/docs-src/core/modules/ROOT/pages/agentParams.adoc rename to docs/docs-src/core/modules/ROOT/partials/agentParams.adoc diff --git a/docs/docs-src/core/modules/ROOT/pages/cfgCassandraAuth.adoc b/docs/docs-src/core/modules/ROOT/partials/cfgCassandraAuth.adoc similarity index 100% rename from docs/docs-src/core/modules/ROOT/pages/cfgCassandraAuth.adoc rename to docs/docs-src/core/modules/ROOT/partials/cfgCassandraAuth.adoc diff --git a/docs/docs-src/core/modules/ROOT/pages/cfgCassandraJavaDriverSettings.adoc b/docs/docs-src/core/modules/ROOT/partials/cfgCassandraJavaDriverSettings.adoc similarity index 100% rename from docs/docs-src/core/modules/ROOT/pages/cfgCassandraJavaDriverSettings.adoc rename to docs/docs-src/core/modules/ROOT/partials/cfgCassandraJavaDriverSettings.adoc diff --git a/docs/docs-src/core/modules/ROOT/pages/cfgCassandraSSL.adoc b/docs/docs-src/core/modules/ROOT/partials/cfgCassandraSSL.adoc similarity index 100% rename from docs/docs-src/core/modules/ROOT/pages/cfgCassandraSSL.adoc rename to docs/docs-src/core/modules/ROOT/partials/cfgCassandraSSL.adoc diff --git a/docs/docs-src/core/modules/ROOT/pages/cfgCassandraSource.adoc b/docs/docs-src/core/modules/ROOT/partials/cfgCassandraSource.adoc similarity index 100% rename from docs/docs-src/core/modules/ROOT/pages/cfgCassandraSource.adoc rename to docs/docs-src/core/modules/ROOT/partials/cfgCassandraSource.adoc diff --git a/docs/lib/remote-include-processor.js b/docs/lib/remote-include-processor.js new file mode 100644 index 00000000..731a0a08 --- /dev/null +++ b/docs/lib/remote-include-processor.js @@ -0,0 +1,199 @@ +const CIRCUMFIX_COMMENT_SUFFIX_RX = / (?:\*[/)]|--%?>)$/ +const NEWLINE_RX = /\r\n?|\n/ +const TAG_DELIMITER_RX = /[,;]/ +const TAG_DIRECTIVE_RX = /\b(?:tag|(end))::(\S+)\[\]$/ +const LINES_DOTDOT_RX = /\.\./ + + +function getTags (attrs) { + if ( 'tag' in attrs ) { + const tag = attrs['tag'] + if (tag && tag !== '!') { + return tag.charAt() === '!' ? new Map().set(tag.substr(1), false) : new Map().set(tag, true) + } + } else if ( 'tags' in attrs ) { + const tags = attrs['tags'] + if (tags) { + let result = new Map() + let any = false + tags.split(TAG_DELIMITER_RX).forEach((tag) => { + if (tag && tag !== '!') { + any = true + tag.charAt() === '!' ? result.set(tag.substr(1), false) : result.set(tag, true) + } + }) + if (any) return result + } + } +} + + +function applyTagFiltering (contents, tags) { + let selecting, selectingDefault, wildcard + if (tags.has('**')) { + if (tags.has('*')) { + selectingDefault = selecting = tags.get('**') + wildcard = tags.get('*') + tags.delete('*') + } else { + selectingDefault = selecting = wildcard = tags.get('**') + } + tags.delete('**') + } else { + selectingDefault = selecting = !Array.from(tags.values()).includes(true) + if (tags.has('*')) { + wildcard = tags.get('*') + tags.delete('*') + } + } + + const lines = [] + const tagStack = [] + const usedTags = [] + let activeTag + let lineNum = 0 + let startLineNum + contents.split(NEWLINE_RX).forEach((line) => { + lineNum++ + let m + let l = line + if ( + (l.endsWith('[]') || + (~l.indexOf('[] ') && + (m = l.match(CIRCUMFIX_COMMENT_SUFFIX_RX)) && + (l = l.substr(0, m.index)).endsWith('[]'))) && + (m = l.match(TAG_DIRECTIVE_RX)) + ) { + const thisTag = m[2] + if (m[1]) { + if (thisTag === activeTag) { + tagStack.shift() + ;[activeTag, selecting] = tagStack.length ? tagStack[0] : [undefined, selectingDefault] + } else if (tags.has(thisTag)) { + const idx = tagStack.findIndex(([name]) => name === thisTag) + if (~idx) { + tagStack.splice(idx, 1) + //console.warn(`line ${lineNum}: mismatched end tag in include: expected ${activeTag}, found ${thisTag}`) + } + //} else { + // //console.warn(`line ${lineNum}: unexpected end tag in include: ${thisTag}`) + //} + } + } else if (tags.has(thisTag)) { + usedTags.push(thisTag) + tagStack.unshift([(activeTag = thisTag), (selecting = tags.get(thisTag))]) + } else if (wildcard !== undefined) { + selecting = activeTag && !selecting ? false : wildcard + tagStack.unshift([(activeTag = thisTag), selecting]) + } + } else if (selecting) { + if (!startLineNum) startLineNum = lineNum + lines.push(line) + } + }) + // Q: use _.difference(Object.keys(tags), usedTags)? + //const missingTags = Object.keys(tags).filter((e) => !usedTags.includes(e)) + //if (missingTags.length) { + // console.warn(`tag${missingTags.length > 1 ? 's' : ''} '${missingTags.join(',')}' not found in include`) + //} + return [lines, startLineNum || 1] +} + +function getLines (attrs) { + if ( 'lines' in attrs ) { + const lines = attrs['lines'] + if (lines) { + // console.warn(`have lines` + lines) + let result = [] // new Map() + let any = false + lines.split(TAG_DELIMITER_RX).forEach((line) => { + if (line && line !== '!') { + let tryMultipleLines = line.split(LINES_DOTDOT_RX) + if ( tryMultipleLines.length === 1 ) { + any = true + result.push([tryMultipleLines[0], tryMultipleLines[0]]) + } + else if ( tryMultipleLines.length === 2 ) { + any = true + result.push([tryMultipleLines[0], tryMultipleLines[1]]) + } + } + }) + if (any) return result + } + } +} + +function filterChars(content){ + let myRegexp = new RegExp(/[^${\}]+(?=})/g); + let matches = myRegexp.exec(content); + + if(matches == null) + return content; + + let ret = content; + // matches.forEach(x => { + // let a = new RegExp("\\${"+x+"}",'gm'); + // ret = ret.replace(a,'asd'); + // }) + + return ret; +} + +function applyLineFiltering (contents, linesToInclude) { + const lines = [] + let lineNum = 0 + let startLineNum + let registerCurrentLine = false + + const nLinesPair = linesToInclude.length + let currentLinePair = 0 + let startLine = linesToInclude[currentLinePair][0] + let endLine = linesToInclude[currentLinePair][1] + // console.warn(`applyLineFiltering ` + startLine + ' and ' + endLine ) + + contents.split(NEWLINE_RX).forEach((line) => { + lineNum++ + + if ( !registerCurrentLine ) { + if ( lineNum == startLine ) + registerCurrentLine = true + } + + if ( registerCurrentLine ) + { + if (!startLineNum) startLineNum = lineNum + lines.push(line) + + if ( lineNum == endLine ) { + registerCurrentLine = false + currentLinePair++ + if ( currentLinePair >= nLinesPair ) + return [lines, startLineNum] + else { + startLine = linesToInclude[currentLinePair][0] + endLine = linesToInclude[currentLinePair][1] + } + } + } + }) + return [lines, startLineNum || 1] +} + +module.exports = function () { + this.includeProcessor(function () { + this.$option('position', '>>') + this.handles((target) => target.startsWith('https://')) + this.process((doc, reader, target, attrs) => { + const contents = require('child_process').execFileSync('curl', ['--silent', '-L', target], { encoding: 'utf8' }) + let includeContents = filterChars(contents) + let startLineNum = 1 + const tags = getTags(attrs) + const lines = getLines(attrs) + if (tags) [includeContents, startLineNum] = applyTagFiltering(includeContents, tags) + else if (lines) [includeContents, startLineNum] = applyLineFiltering(includeContents, lines) + reader.pushInclude(includeContents, target, target, startLineNum, attrs) + // reader.pushInclude(contents, target, target, 1, attrs) + }) + }) +} diff --git a/docs/lib/svg-macro.js b/docs/lib/svg-macro.js new file mode 100644 index 00000000..e7f59282 --- /dev/null +++ b/docs/lib/svg-macro.js @@ -0,0 +1,110 @@ +const logger = require("@antora/logger")("asciidoctor:svg-macro"); + +/** + * @example Inline Embedded SVG + * svg:ROOT:ui/icons/vector.svg[] + */ +function inlineSvgMacro({ contentCatalog, file }) { + return function () { + this.process((parent, target, attrs) => { + svgContent = getSvgContent(target, file, contentCatalog); + if (!svgContent) return; + return this.createInlinePass( + parent, + svgContent.replace(" { + svgContent = getSvgContent(target, file, contentCatalog); + if (!svgContent) return; + const svgHtmlAttrs = htmlAttrs({ ...attrs, role: undefined }); + const containerHtmlAttrs = attrs.role + ? `class="imageblock ${attrs.role}"` + : 'class="imageblock"'; + const html = + `
` + + svgContent.replace("
"; + return this.createBlock(parent, "pass", html); + }); + }; +} + +/** + * This macro relies on the material-icons font being loaded in UI bundle. + * + * @example Material Icon + * icon:material-icons:menu_open[] + * + * @example Embedded SVG + * icon:ROOT:ui/icons/vector.svg[] + */ +function inlineIconMacro({ contentCatalog, file }) { + return function () { + this.process((parent, target, attrs) => { + if (target.startsWith("material-icons")) { + iconTarget = target + .replace("material-icons:", "") + .trim() + .replace("-", "_"); + return this.createInlinePass( + parent, + `${iconTarget}` + ); + } else { + svgContent = getSvgContent(target, file, contentCatalog); + if (!svgContent) return; + return this.createInlinePass( + parent, + svgContent.replace(" { + context.once('sitePublished', () => { + const logger = context.getLogger('tailwind-processor-extension') + logger.info('Building Tailwind') + execSync('npm run tailwindcss', { stdio: 'inherit' }) + logger.info('Tailwind Build Successful') + }) +} diff --git a/docs/lib/unlisted-pages-extension.js b/docs/lib/unlisted-pages-extension.js new file mode 100644 index 00000000..ec1b337f --- /dev/null +++ b/docs/lib/unlisted-pages-extension.js @@ -0,0 +1,41 @@ +module.exports.register = function ({ config }) { + const { addToNavigation, unlistedPagesHeading = 'Unlisted Pages' } = config + const logger = this.getLogger('unlisted-pages-extension') + this + .on('navigationBuilt', ({ contentCatalog }) => { + contentCatalog.getComponents().forEach(({ versions }) => { + versions.forEach(({ name: component, version, navigation: nav, url: defaultUrl }) => { + const navEntriesByUrl = getNavEntriesByUrl(nav) + const unlistedPages = contentCatalog + .findBy({ component, version, family: 'page' }) + .filter((page) => page.out) + .reduce((collector, page) => { + // Check if the 'unlisted-page' attribute is set to true + if (page.asciidoc.attributes['unlisted-page'] === 'true') { + return collector; // Skip this page + } + if ((page.pub.url in navEntriesByUrl) || page.pub.url === defaultUrl) return collector + logger.warn({ file: page.src, source: page.src.origin }, 'detected unlisted page') + return collector.concat(page) + }, []) + if (unlistedPages.length && addToNavigation) { + nav.push({ + content: unlistedPagesHeading, + items: unlistedPages.map((page) => { + return { content: page.asciidoc.navtitle, url: page.pub.url, urlType: 'internal' } + }), + root: true, + }) + } + }) + }) + }) +} + +function getNavEntriesByUrl (items = [], accum = {}) { + items.forEach((item) => { + if (item.urlType === 'internal') accum[item.url.split('#')[0]] = item + getNavEntriesByUrl(item.items, accum) + }) + return accum +} diff --git a/docs/local-preview-playbook.yml b/docs/local-preview-playbook.yml new file mode 100644 index 00000000..7f06a60d --- /dev/null +++ b/docs/local-preview-playbook.yml @@ -0,0 +1,73 @@ +runtime: + log: + failure_level: warn +git: + # ensure_git_suffix: false # Enable if necessary -- some git services don’t recognize the URL if it contains the .git extension. + fetch_concurrency: 10 + +site: + title: DataStax Docs + start_page: cdc-cassandra::index.adoc + robots: disallow + +content: + branches: main # Sources default to this branch if none are specified. + sources: + - url: .. + start_path: docs/docs-src/core + branches: HEAD + +antora: + extensions: + - '@antora/collector-extension' + - lib/tailwind-processor.js + - id: unlisted-pages + enabled: true + require: lib/unlisted-pages-extension.js + add_to_navigation: false + unlisted_pages_heading: Orphans + +asciidoc: + extensions: + - '@asciidoctor/tabs' + - lib/remote-include-processor.js + - lib/svg-macro.js + - asciidoctor-kroki + - asciidoctor-external-callout + attributes: + # BUILT-IN ATTRIBUTES + allow-uri-read: '' # this has no effect in antora, but does help development in Intellij + experimental: '' + idprefix: '' + idseparator: '-' + # kroki-fetch-diagram: true + # kroki-server-url: + max-include-depth: 10 + page-toclevels: 2@ + sectlinks: '' + tabs-sync-option: '' + example-caption: false + figure-caption: false + table-caption: false + xrefstyle: short + # CUSTOM ATTRIBUTES + company: 'DataStax' + astra_db: 'Astra DB' + astra_stream: 'Astra Streaming' + astra_ui: 'Astra Portal' + support_url: 'https://support.datastax.com' + glossary-url: 'https://docs.datastax.com/en/glossary/docs/index.html#' + +urls: + latest_version_segment_strategy: redirect:from + latest_version_segment: 'latest' + +ui: + bundle: + url: https://github.com/riptano/docs-ui/releases/latest/download/ui-bundle.zip + # url: https://gitlab.com/antora/antora-ui-default/-/jobs/artifacts/HEAD/raw/build/ui-bundle.zip?job=bundle-stable + snapshot: true + # supplemental_files: supplemental-ui + +output: + dir: 'build/site' \ No newline at end of file diff --git a/docs/package.json b/docs/package.json index 35f7c7e7..463c4977 100644 --- a/docs/package.json +++ b/docs/package.json @@ -1,18 +1,25 @@ { - "name": "cdcrep-docs", - "scripts": { - "build:prod:cdc-cassandra": "npm run build:publish:cdc-cassandra", - "build:dev:cdc-cassandra": "npm run build:local:cdc-cassandra", - "build:publish:cdc-cassandra": "antora --stacktrace --fetch --clean playbooks/site-publish-cdc-cassandra.yaml", - "build:local:cdc-cassandra": "antora --stacktrace --fetch --clean playbooks/site-local-cdc-cassandra.yaml" - }, + "private": true, + "name": "cdc-apache-cassandra", + "description": "The content source for DataStax CDC for Apache Cassandra documentation.", + "author": "DataStax Inc.", + "homepage": "https://docs.datastax.com", "repository": { "type": "git", - "url": "git+https://github.com/riptano/cdc-apache-cassandra.git" + "url": "https://github.com/datastax/cdc-apache-cassandra" + }, + "scripts": { + "build:local": "env FORCE_SHOW_EDIT_PAGE_LINK=true antora --clean --stacktrace local-preview-playbook.yml", + "tailwindcss": "tailwindcss build -c ./build/site/_/js/tailwind.config.js -i ./build/site/_/css/site.css -o ./build/site/_/css/site.css --minify" }, "dependencies": { - "@antora/cli": "~3.1", - "@antora/site-generator": "~3.1", - "@asciidoctor/tabs": "^1.0.0-beta.5" + "@antora/collector-extension": "^1.0.0-alpha.3", + "@asciidoctor/tabs": "^1.0.0-beta.6", + "antora": "~3.1", + "asciidoctor-external-callout": "~1.2.1", + "asciidoctor-kroki": "~0.18.1", + "csv-parser": "^3.0.0", + "npm-run-all": "^4.1.5", + "tailwindcss": "^3.3.5" } } diff --git a/docs/playbooks/site-local-cdc-cassandra.yaml b/docs/playbooks/site-local-cdc-cassandra.yaml deleted file mode 100644 index 6472b98c..00000000 --- a/docs/playbooks/site-local-cdc-cassandra.yaml +++ /dev/null @@ -1,51 +0,0 @@ -site: - title: DataStax CDC for Apache Cassandra - url: https://docs.datastax.com/en/cdc-for-cassandra - start_page: docs::index.adoc - -output: - # The output.dir in the site-local playbook should take the form - # "./../build/". The reason for this is because DataStax docs are - # often stored on separate branches of the same repo. Using the docset name - # for the output directory ensures that different docset builds within the - # same repo don't overwrite each other - dir: ./../build/cdc-apache-cassandra - -content: - edit_url: false - sources: - - url: ./../.. - branches: HEAD - start_path: docs/docs-src/core - -ui: - bundle: - url: https://github.com/riptano/antora-ui-docs/releases/latest/download/ui-bundle.zip - snapshot: true - # Uncomment the "supplemental_files" line below ONLY IF you're trying to - # change the banner logo to something other than the default logo provided by - # the UI bundle. NOTE: The non-default logo must be named "logo.svg" and be - # located @ "playbooks/supplemental-ui/img/logo.svg". - #supplemental_files: ./supplemental-ui - output_dir: assets - -asciidoc: - attributes: - page-pagination: '' - sectanchors: '' - sectlinks: '' - idprefix: '' - idseparator: '-' - - support_url: 'https://support.datastax.com' - astra_docs_base_url: 'https://docs.datastax.com/en/astra/docs' - - # The "glossary-url" attribute below is used by writers when linking to a - # term in the glossary. Referencing this attribute in an adoc file will - # automatically insert the root URL where the glossary pages are located. - # For example, the following syntax will generate a link to the glossary - # page for "agent": {glossary-url}agent[agent,window="_blank"] - glossary-url: 'https://docs.datastax.com/en/glossary/docs/index.html#' - - extensions: - - '@asciidoctor/tabs' diff --git a/docs/playbooks/site-publish-cdc-cassandra.yaml b/docs/playbooks/site-publish-cdc-cassandra.yaml deleted file mode 100644 index b0620c57..00000000 --- a/docs/playbooks/site-publish-cdc-cassandra.yaml +++ /dev/null @@ -1,61 +0,0 @@ -site: - title: DataStax CDC for Apache Cassandra - url: https://docs.datastax.com/en/cdc-for-cassandra - start_page: docs::index.adoc - # The google_analytics and segment_io site keys must be in the release playbook - # of each docset. This ensures that the Google Tag Manager (GTM) and Segment - # scripts get loaded into the docsets that get published to docs.datastax.com. - keys: - google_analytics: 'GTM-5FSG7Q' - segment_io: 'd24gQtyKIUu5mLdkp11xjfiXLhRqx0HH' - -output: - # Bsys requires the output.dir in the site-publish playbook to be specifically - # set to "~/work//build/site". This is because bsys clones the - # content source repo into "/work", and then looks for the Antora-generated - # site in "build/site" within the repo directory when deploying/syncing. - dir: '~/work/cdc-apache-cassandra/docs/build/site' - -content: - edit_url: 'https://github.com/datastax/cdc-apache-cassandra/blob/master/docs/{path}' - sources: - - url: '~/work/cdc-apache-cassandra' - branches: HEAD - start_path: docs/docs-src/core - -ui: - bundle: - url: https://github.com/riptano/antora-ui-docs/releases/latest/download/ui-bundle.zip - snapshot: true - # Uncomment the "supplemental_files" line below ONLY IF you're trying to - # change the banner logo to something other than the default logo provided by - # the UI bundle. NOTE: The non-default logo must be named "logo.svg" and be - # located @ "playbooks/supplemental-ui/img/logo.svg". - #supplemental_files: '~/work/cdc-apache-cassandra/docs/playbooks/supplemental-ui' - output_dir: assets - -asciidoc: - attributes: - page-pagination: '' - sectanchors: '' - sectlinks: '' - idprefix: '' - idseparator: '-' - - support_url: 'https://support.datastax.com' - astra_docs_base_url: 'https://docs.datastax.com/en/astra/docs' - - # The "glossary-url" attribute below is used by writers when linking to a - # term in the glossary. Referencing this attribute in an adoc file will - # automatically insert the root URL where the glossary pages are located. - # For example, the following syntax will generate a link to the glossary - # page for "agent": {glossary-url}agent[agent,window="_blank"] - glossary-url: 'https://docs.datastax.com/en/glossary/docs/index.html#' - - # Bsys requires the URLs of the content source repos of the current docset - # to be listed as attributes in the site-publish playbook in the form - # ": 'ssh://github.com//.git'". - cdc-apache-cassandra: 'ssh://github.com/datastax/cdc-apache-cassandra' - - extensions: - - '@asciidoctor/tabs'